Skip to content
Snippets Groups Projects
Unverified Commit d36eceda authored by Martin Weise's avatar Martin Weise
Browse files

Fixed the tests

parent e2d816c5
No related branches found
No related tags found
4 merge requests!231CI: Remove build for log-service,!228Better error message handling in the frontend,!223Release of version 1.4.0,!218Replacing minIO with SeaweedFS
Show whitespace changes
Inline Side-by-side
.docs/system-services-storage.md
+ 11
− 3
View file @ d36eceda
......@@ -36,14 +36,19 @@ The default configuration creates two buckets `dbrepo-upload`, `dbrepo-download`
Upload a CSV-file into the `dbrepo-upload` bucket with the AWS CLI:
```console
$ aws --endpoint-url http://<hostname>:9000 s3 cp /path/to/file.csv s3://dbrepo-upload/
$ aws --endpoint-url http://<hostname>:9000 \
s3 \
cp /path/to/file.csv \
s3://dbrepo-upload/
upload: /path/to/file.csv to s3://dbrepo-upload/file.csv
```
You can list the buckets:
```console
$ aws --endpoint-url http://<hostname>:9000 s3 ls
$ aws --endpoint-url http://<hostname>:9000 \
s3 \
ls
2023-12-03 16:23:15 dbrepo-download
2023-12-03 16:28:05 dbrepo-upload
```
......@@ -51,7 +56,10 @@ $ aws --endpoint-url http://<hostname>:9000 s3 ls
And list the files in the bucket `dbrepo-upload` with:
```console
$ aws --endpoint-url http://<hostname>:9000 s3 ls dbrepo-upload
$ aws --endpoint-url http://<hostname>:9000 \
s3 \
ls \
dbrepo-upload
2023-12-03 16:28:05 535219 file.csv
```
......
dbrepo-analyse-service/test.sh
+ 1
− 1
View file @ d36eceda
#!/bin/bash
source ./dbrepo-analyse-service/venv/bin/activate
cd ./dbrepo-analyse-service/ && coverage run -m pytest test/test_determine_dt.py test/test_determine_pk.py test/test_minio_client.py --junitxml=report.xml && coverage html --omit="test/*" && coverage report --omit="test/*" > ./coverage.txt
\ No newline at end of file
cd ./dbrepo-analyse-service/ && coverage run -m pytest test/test_determine_dt.py test/test_determine_pk.py test/test_s3_client.py --junitxml=report.xml && coverage html --omit="test/*" && coverage report --omit="test/*" > ./coverage.txt
\ No newline at end of file
dbrepo-analyse-service/test/conftest.py
+ 3
− 5
View file @ d36eceda
......@@ -6,8 +6,6 @@ import logging
from minio.deleteobjects import DeleteObject
from testcontainers.minio import MinioContainer
from clients.minio_client import MinioClient
@pytest.fixture(scope="session")
def session(request):
......@@ -16,9 +14,9 @@ def session(request):
:param request: /
:return: The minIO container
"""
logging.debug("[fixture] creating minIO container")
container = MinioContainer(access_key="minioadmin", secret_key="minioadmin")
logging.debug("[fixture] starting minIO container")
logging.debug("[fixture] creating container")
container = MinioContainer(access_key="seaweedfsadmin", secret_key="seaweedfsadmin")
logging.debug("[fixture] starting container")
container.start()
# set the environment for the client
endpoint = 'http://' + container.get_container_host_ip() + ':' + container.get_exposed_port(9000)
......
dbrepo-analyse-service/test/s3_config.json 0 → 100644
+ 20
− 0
View file @ d36eceda
{
"identities": [
{
"name": "admin",
"credentials": [
{
"accessKey": "seaweedfsadmin",
"secretKey": "seaweedfsadmin"
}
],
"actions": [
"Read",
"Write",
"List",
"Tagging",
"Admin"
]
}
]
}
\ No newline at end of file
dbrepo-analyse-service/test/test_determine_dt.py
+ 9
− 9
View file @ d36eceda
......@@ -9,7 +9,7 @@ import unittest
import json
from clients.minio_client import MinioClient
from clients.s3_client import S3Client
from botocore.exceptions import ClientError
from determine_dt import determine_datatypes
......@@ -32,7 +32,7 @@ class DetermineDatatypesTest(unittest.TestCase):
}
# mock
MinioClient().upload_file("datetime.csv", './data/test_dt/', 'dbrepo-upload')
S3Client().upload_file("datetime.csv", './data/test_dt/', 'dbrepo-upload')
# test
response = determine_datatypes(filename="datetime.csv", separator=",")
......@@ -54,7 +54,7 @@ class DetermineDatatypesTest(unittest.TestCase):
}
# mock
MinioClient().upload_file("datetime_tz.csv", './data/test_dt/', 'dbrepo-upload')
S3Client().upload_file("datetime_tz.csv", './data/test_dt/', 'dbrepo-upload')
# test
response = determine_datatypes(filename="datetime_tz.csv", separator=",")
......@@ -76,7 +76,7 @@ class DetermineDatatypesTest(unittest.TestCase):
}
# mock
MinioClient().upload_file("datetime_t.csv", './data/test_dt/', 'dbrepo-upload')
S3Client().upload_file("datetime_t.csv", './data/test_dt/', 'dbrepo-upload')
# test
response = determine_datatypes(filename="datetime_t.csv", separator=",")
......@@ -98,7 +98,7 @@ class DetermineDatatypesTest(unittest.TestCase):
}
# mock
MinioClient().upload_file("datatypes.csv", './data/test_dt/', 'dbrepo-upload')
S3Client().upload_file("datatypes.csv", './data/test_dt/', 'dbrepo-upload')
# test
response = determine_datatypes(filename="datatypes.csv", separator=",")
......@@ -121,7 +121,7 @@ class DetermineDatatypesTest(unittest.TestCase):
def test_determine_datatypes_fileEmpty_succeeds(self):
# mock
MinioClient().upload_file("empty.csv", './data/test_dt/', 'dbrepo-upload')
S3Client().upload_file("empty.csv", './data/test_dt/', 'dbrepo-upload')
# test
response = determine_datatypes("empty.csv")
......@@ -133,7 +133,7 @@ class DetermineDatatypesTest(unittest.TestCase):
def test_determine_datatypes_separatorSemicolon_succeeds(self):
# mock
MinioClient().upload_file("separator.csv", './data/test_dt/', 'dbrepo-upload')
S3Client().upload_file("separator.csv", './data/test_dt/', 'dbrepo-upload')
# test
response = determine_datatypes(filename="separator.csv", separator=";")
......@@ -144,7 +144,7 @@ class DetermineDatatypesTest(unittest.TestCase):
def test_determine_datatypes_separatorGuess_succeeds(self):
# mock
MinioClient().upload_file("separator.csv", './data/test_dt/', 'dbrepo-upload')
S3Client().upload_file("separator.csv", './data/test_dt/', 'dbrepo-upload')
# test
response = determine_datatypes(filename="separator.csv")
......@@ -155,7 +155,7 @@ class DetermineDatatypesTest(unittest.TestCase):
def test_determine_datatypes_separatorGuessLargeDataset_succeeds(self):
# mock
MinioClient().upload_file("large.csv", './data/test_dt/', 'dbrepo-upload')
S3Client().upload_file("large.csv", './data/test_dt/', 'dbrepo-upload')
# test
response = determine_datatypes(filename="large.csv")
......
dbrepo-analyse-service/test/test_determine_pk.py
+ 49
− 77
View file @ d36eceda
......@@ -9,31 +9,17 @@ import unittest
import os
import json
from clients.minio_client import MinioClient
from testcontainers.minio import MinioContainer
from clients.s3_client import S3Client
from determine_pk import determine_pk
def before():
container = MinioContainer(access_key="minioadmin", secret_key="minioadmin").start()
endpoint = 'http://' + container.get_container_host_ip() + ':' + container.get_exposed_port(9000)
os.environ['S3_STORAGE_ENDPOINT'] = endpoint
client = container.get_client()
# create buckets
client.make_bucket('dbrepo-upload')
client.make_bucket('dbrepo-download')
return container
class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test
def test_determine_pk_largeFileIdFirst_succeeds(self):
with before() as minio:
# mock
MinioClient().upload_file("largefile_idfirst.csv", './data/test_pk/', 'dbrepo-upload')
S3Client().upload_file("largefile_idfirst.csv", './data/test_pk/', 'dbrepo-upload')
# test
response = determine_pk('largefile_idfirst.csv')
......@@ -43,10 +29,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test
def test_determine_pk_largeFileIdInBetween_succeeds(self):
with before() as minio:
# mock
MinioClient().upload_file("largefile_idinbtw.csv", './data/test_pk/', 'dbrepo-upload')
S3Client().upload_file("largefile_idinbtw.csv", './data/test_pk/', 'dbrepo-upload')
# test
response = determine_pk('largefile_idinbtw.csv')
......@@ -56,10 +40,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test
def test_determine_pk_largeFileNoPrimaryKey_fails(self):
with before() as minio:
# mock
MinioClient().upload_file("largefile_no_pk.csv", './data/test_pk/', 'dbrepo-upload')
S3Client().upload_file("largefile_no_pk.csv", './data/test_pk/', 'dbrepo-upload')
# test
response = determine_pk('largefile_no_pk.csv')
......@@ -69,10 +51,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test
def test_determine_pk_largeFileNullInUnique_fails(self):
with before() as minio:
# mock
MinioClient().upload_file("largefile_nullinunique.csv", './data/test_pk/', 'dbrepo-upload')
S3Client().upload_file("largefile_nullinunique.csv", './data/test_pk/', 'dbrepo-upload')
# test
response = determine_pk('largefile_nullinunique.csv')
......@@ -82,10 +62,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test
def test_determine_pk_smallFileIdFirst_fails(self):
with before() as minio:
# mock
MinioClient().upload_file("smallfile_idfirst.csv", './data/test_pk/', 'dbrepo-upload')
S3Client().upload_file("smallfile_idfirst.csv", './data/test_pk/', 'dbrepo-upload')
# test
response = determine_pk('smallfile_idfirst.csv')
......@@ -95,10 +73,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test
def test_determine_pk_smallFileIdIntBetween_fails(self):
with before() as minio:
# mock
MinioClient().upload_file("smallfile_idinbtw.csv", './data/test_pk/', 'dbrepo-upload')
S3Client().upload_file("smallfile_idinbtw.csv", './data/test_pk/', 'dbrepo-upload')
# test
response = determine_pk('smallfile_idinbtw.csv')
......@@ -108,10 +84,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test
def test_determine_pk_smallFileNoPrimaryKey_fails(self):
with before() as minio:
# mock
MinioClient().upload_file("smallfile_no_pk.csv", './data/test_pk/', 'dbrepo-upload')
S3Client().upload_file("smallfile_no_pk.csv", './data/test_pk/', 'dbrepo-upload')
# test
response = determine_pk('smallfile_no_pk.csv')
......@@ -121,10 +95,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test
def test_determine_pk_smallFileNullInUnique_fails(self):
with before() as minio:
# mock
MinioClient().upload_file("smallfile_nullinunique.csv", './data/test_pk/', 'dbrepo-upload')
S3Client().upload_file("smallfile_nullinunique.csv", './data/test_pk/', 'dbrepo-upload')
# test
response = determine_pk('smallfile_nullinunique.csv')
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment