Skip to content
Snippets Groups Projects
Verified Commit d36eceda authored by Martin Weise's avatar Martin Weise
Browse files

Fixed the tests

parent e2d816c5
No related branches found
No related tags found
4 merge requests!231CI: Remove build for log-service,!228Better error message handling in the frontend,!223Release of version 1.4.0,!218Replacing minIO with SeaweedFS
...@@ -36,14 +36,19 @@ The default configuration creates two buckets `dbrepo-upload`, `dbrepo-download` ...@@ -36,14 +36,19 @@ The default configuration creates two buckets `dbrepo-upload`, `dbrepo-download`
Upload a CSV-file into the `dbrepo-upload` bucket with the AWS CLI: Upload a CSV-file into the `dbrepo-upload` bucket with the AWS CLI:
```console ```console
$ aws --endpoint-url http://<hostname>:9000 s3 cp /path/to/file.csv s3://dbrepo-upload/ $ aws --endpoint-url http://<hostname>:9000 \
s3 \
cp /path/to/file.csv \
s3://dbrepo-upload/
upload: /path/to/file.csv to s3://dbrepo-upload/file.csv upload: /path/to/file.csv to s3://dbrepo-upload/file.csv
``` ```
You can list the buckets: You can list the buckets:
```console ```console
$ aws --endpoint-url http://<hostname>:9000 s3 ls $ aws --endpoint-url http://<hostname>:9000 \
s3 \
ls
2023-12-03 16:23:15 dbrepo-download 2023-12-03 16:23:15 dbrepo-download
2023-12-03 16:28:05 dbrepo-upload 2023-12-03 16:28:05 dbrepo-upload
``` ```
...@@ -51,7 +56,10 @@ $ aws --endpoint-url http://<hostname>:9000 s3 ls ...@@ -51,7 +56,10 @@ $ aws --endpoint-url http://<hostname>:9000 s3 ls
And list the files in the bucket `dbrepo-upload` with: And list the files in the bucket `dbrepo-upload` with:
```console ```console
$ aws --endpoint-url http://<hostname>:9000 s3 ls dbrepo-upload $ aws --endpoint-url http://<hostname>:9000 \
s3 \
ls \
dbrepo-upload
2023-12-03 16:28:05 535219 file.csv 2023-12-03 16:28:05 535219 file.csv
``` ```
......
#!/bin/bash #!/bin/bash
source ./dbrepo-analyse-service/venv/bin/activate source ./dbrepo-analyse-service/venv/bin/activate
cd ./dbrepo-analyse-service/ && coverage run -m pytest test/test_determine_dt.py test/test_determine_pk.py test/test_minio_client.py --junitxml=report.xml && coverage html --omit="test/*" && coverage report --omit="test/*" > ./coverage.txt cd ./dbrepo-analyse-service/ && coverage run -m pytest test/test_determine_dt.py test/test_determine_pk.py test/test_s3_client.py --junitxml=report.xml && coverage html --omit="test/*" && coverage report --omit="test/*" > ./coverage.txt
\ No newline at end of file \ No newline at end of file
...@@ -6,8 +6,6 @@ import logging ...@@ -6,8 +6,6 @@ import logging
from minio.deleteobjects import DeleteObject from minio.deleteobjects import DeleteObject
from testcontainers.minio import MinioContainer from testcontainers.minio import MinioContainer
from clients.minio_client import MinioClient
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
def session(request): def session(request):
...@@ -16,9 +14,9 @@ def session(request): ...@@ -16,9 +14,9 @@ def session(request):
:param request: / :param request: /
:return: The minIO container :return: The minIO container
""" """
logging.debug("[fixture] creating minIO container") logging.debug("[fixture] creating container")
container = MinioContainer(access_key="minioadmin", secret_key="minioadmin") container = MinioContainer(access_key="seaweedfsadmin", secret_key="seaweedfsadmin")
logging.debug("[fixture] starting minIO container") logging.debug("[fixture] starting container")
container.start() container.start()
# set the environment for the client # set the environment for the client
endpoint = 'http://' + container.get_container_host_ip() + ':' + container.get_exposed_port(9000) endpoint = 'http://' + container.get_container_host_ip() + ':' + container.get_exposed_port(9000)
......
{
"identities": [
{
"name": "admin",
"credentials": [
{
"accessKey": "seaweedfsadmin",
"secretKey": "seaweedfsadmin"
}
],
"actions": [
"Read",
"Write",
"List",
"Tagging",
"Admin"
]
}
]
}
\ No newline at end of file
...@@ -9,7 +9,7 @@ import unittest ...@@ -9,7 +9,7 @@ import unittest
import json import json
from clients.minio_client import MinioClient from clients.s3_client import S3Client
from botocore.exceptions import ClientError from botocore.exceptions import ClientError
from determine_dt import determine_datatypes from determine_dt import determine_datatypes
...@@ -32,7 +32,7 @@ class DetermineDatatypesTest(unittest.TestCase): ...@@ -32,7 +32,7 @@ class DetermineDatatypesTest(unittest.TestCase):
} }
# mock # mock
MinioClient().upload_file("datetime.csv", './data/test_dt/', 'dbrepo-upload') S3Client().upload_file("datetime.csv", './data/test_dt/', 'dbrepo-upload')
# test # test
response = determine_datatypes(filename="datetime.csv", separator=",") response = determine_datatypes(filename="datetime.csv", separator=",")
...@@ -54,7 +54,7 @@ class DetermineDatatypesTest(unittest.TestCase): ...@@ -54,7 +54,7 @@ class DetermineDatatypesTest(unittest.TestCase):
} }
# mock # mock
MinioClient().upload_file("datetime_tz.csv", './data/test_dt/', 'dbrepo-upload') S3Client().upload_file("datetime_tz.csv", './data/test_dt/', 'dbrepo-upload')
# test # test
response = determine_datatypes(filename="datetime_tz.csv", separator=",") response = determine_datatypes(filename="datetime_tz.csv", separator=",")
...@@ -76,7 +76,7 @@ class DetermineDatatypesTest(unittest.TestCase): ...@@ -76,7 +76,7 @@ class DetermineDatatypesTest(unittest.TestCase):
} }
# mock # mock
MinioClient().upload_file("datetime_t.csv", './data/test_dt/', 'dbrepo-upload') S3Client().upload_file("datetime_t.csv", './data/test_dt/', 'dbrepo-upload')
# test # test
response = determine_datatypes(filename="datetime_t.csv", separator=",") response = determine_datatypes(filename="datetime_t.csv", separator=",")
...@@ -98,7 +98,7 @@ class DetermineDatatypesTest(unittest.TestCase): ...@@ -98,7 +98,7 @@ class DetermineDatatypesTest(unittest.TestCase):
} }
# mock # mock
MinioClient().upload_file("datatypes.csv", './data/test_dt/', 'dbrepo-upload') S3Client().upload_file("datatypes.csv", './data/test_dt/', 'dbrepo-upload')
# test # test
response = determine_datatypes(filename="datatypes.csv", separator=",") response = determine_datatypes(filename="datatypes.csv", separator=",")
...@@ -121,7 +121,7 @@ class DetermineDatatypesTest(unittest.TestCase): ...@@ -121,7 +121,7 @@ class DetermineDatatypesTest(unittest.TestCase):
def test_determine_datatypes_fileEmpty_succeeds(self): def test_determine_datatypes_fileEmpty_succeeds(self):
# mock # mock
MinioClient().upload_file("empty.csv", './data/test_dt/', 'dbrepo-upload') S3Client().upload_file("empty.csv", './data/test_dt/', 'dbrepo-upload')
# test # test
response = determine_datatypes("empty.csv") response = determine_datatypes("empty.csv")
...@@ -133,7 +133,7 @@ class DetermineDatatypesTest(unittest.TestCase): ...@@ -133,7 +133,7 @@ class DetermineDatatypesTest(unittest.TestCase):
def test_determine_datatypes_separatorSemicolon_succeeds(self): def test_determine_datatypes_separatorSemicolon_succeeds(self):
# mock # mock
MinioClient().upload_file("separator.csv", './data/test_dt/', 'dbrepo-upload') S3Client().upload_file("separator.csv", './data/test_dt/', 'dbrepo-upload')
# test # test
response = determine_datatypes(filename="separator.csv", separator=";") response = determine_datatypes(filename="separator.csv", separator=";")
...@@ -144,7 +144,7 @@ class DetermineDatatypesTest(unittest.TestCase): ...@@ -144,7 +144,7 @@ class DetermineDatatypesTest(unittest.TestCase):
def test_determine_datatypes_separatorGuess_succeeds(self): def test_determine_datatypes_separatorGuess_succeeds(self):
# mock # mock
MinioClient().upload_file("separator.csv", './data/test_dt/', 'dbrepo-upload') S3Client().upload_file("separator.csv", './data/test_dt/', 'dbrepo-upload')
# test # test
response = determine_datatypes(filename="separator.csv") response = determine_datatypes(filename="separator.csv")
...@@ -155,7 +155,7 @@ class DetermineDatatypesTest(unittest.TestCase): ...@@ -155,7 +155,7 @@ class DetermineDatatypesTest(unittest.TestCase):
def test_determine_datatypes_separatorGuessLargeDataset_succeeds(self): def test_determine_datatypes_separatorGuessLargeDataset_succeeds(self):
# mock # mock
MinioClient().upload_file("large.csv", './data/test_dt/', 'dbrepo-upload') S3Client().upload_file("large.csv", './data/test_dt/', 'dbrepo-upload')
# test # test
response = determine_datatypes(filename="large.csv") response = determine_datatypes(filename="large.csv")
......
...@@ -9,31 +9,17 @@ import unittest ...@@ -9,31 +9,17 @@ import unittest
import os import os
import json import json
from clients.minio_client import MinioClient from clients.s3_client import S3Client
from testcontainers.minio import MinioContainer
from determine_pk import determine_pk from determine_pk import determine_pk
def before():
container = MinioContainer(access_key="minioadmin", secret_key="minioadmin").start()
endpoint = 'http://' + container.get_container_host_ip() + ':' + container.get_exposed_port(9000)
os.environ['S3_STORAGE_ENDPOINT'] = endpoint
client = container.get_client()
# create buckets
client.make_bucket('dbrepo-upload')
client.make_bucket('dbrepo-download')
return container
class DeterminePrimaryKeyTest(unittest.TestCase): class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test # @Test
def test_determine_pk_largeFileIdFirst_succeeds(self): def test_determine_pk_largeFileIdFirst_succeeds(self):
with before() as minio:
# mock # mock
MinioClient().upload_file("largefile_idfirst.csv", './data/test_pk/', 'dbrepo-upload') S3Client().upload_file("largefile_idfirst.csv", './data/test_pk/', 'dbrepo-upload')
# test # test
response = determine_pk('largefile_idfirst.csv') response = determine_pk('largefile_idfirst.csv')
...@@ -43,10 +29,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase): ...@@ -43,10 +29,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test # @Test
def test_determine_pk_largeFileIdInBetween_succeeds(self): def test_determine_pk_largeFileIdInBetween_succeeds(self):
with before() as minio:
# mock # mock
MinioClient().upload_file("largefile_idinbtw.csv", './data/test_pk/', 'dbrepo-upload') S3Client().upload_file("largefile_idinbtw.csv", './data/test_pk/', 'dbrepo-upload')
# test # test
response = determine_pk('largefile_idinbtw.csv') response = determine_pk('largefile_idinbtw.csv')
...@@ -56,10 +40,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase): ...@@ -56,10 +40,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test # @Test
def test_determine_pk_largeFileNoPrimaryKey_fails(self): def test_determine_pk_largeFileNoPrimaryKey_fails(self):
with before() as minio:
# mock # mock
MinioClient().upload_file("largefile_no_pk.csv", './data/test_pk/', 'dbrepo-upload') S3Client().upload_file("largefile_no_pk.csv", './data/test_pk/', 'dbrepo-upload')
# test # test
response = determine_pk('largefile_no_pk.csv') response = determine_pk('largefile_no_pk.csv')
...@@ -69,10 +51,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase): ...@@ -69,10 +51,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test # @Test
def test_determine_pk_largeFileNullInUnique_fails(self): def test_determine_pk_largeFileNullInUnique_fails(self):
with before() as minio:
# mock # mock
MinioClient().upload_file("largefile_nullinunique.csv", './data/test_pk/', 'dbrepo-upload') S3Client().upload_file("largefile_nullinunique.csv", './data/test_pk/', 'dbrepo-upload')
# test # test
response = determine_pk('largefile_nullinunique.csv') response = determine_pk('largefile_nullinunique.csv')
...@@ -82,10 +62,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase): ...@@ -82,10 +62,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test # @Test
def test_determine_pk_smallFileIdFirst_fails(self): def test_determine_pk_smallFileIdFirst_fails(self):
with before() as minio:
# mock # mock
MinioClient().upload_file("smallfile_idfirst.csv", './data/test_pk/', 'dbrepo-upload') S3Client().upload_file("smallfile_idfirst.csv", './data/test_pk/', 'dbrepo-upload')
# test # test
response = determine_pk('smallfile_idfirst.csv') response = determine_pk('smallfile_idfirst.csv')
...@@ -95,10 +73,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase): ...@@ -95,10 +73,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test # @Test
def test_determine_pk_smallFileIdIntBetween_fails(self): def test_determine_pk_smallFileIdIntBetween_fails(self):
with before() as minio:
# mock # mock
MinioClient().upload_file("smallfile_idinbtw.csv", './data/test_pk/', 'dbrepo-upload') S3Client().upload_file("smallfile_idinbtw.csv", './data/test_pk/', 'dbrepo-upload')
# test # test
response = determine_pk('smallfile_idinbtw.csv') response = determine_pk('smallfile_idinbtw.csv')
...@@ -108,10 +84,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase): ...@@ -108,10 +84,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test # @Test
def test_determine_pk_smallFileNoPrimaryKey_fails(self): def test_determine_pk_smallFileNoPrimaryKey_fails(self):
with before() as minio:
# mock # mock
MinioClient().upload_file("smallfile_no_pk.csv", './data/test_pk/', 'dbrepo-upload') S3Client().upload_file("smallfile_no_pk.csv", './data/test_pk/', 'dbrepo-upload')
# test # test
response = determine_pk('smallfile_no_pk.csv') response = determine_pk('smallfile_no_pk.csv')
...@@ -121,10 +95,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase): ...@@ -121,10 +95,8 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test # @Test
def test_determine_pk_smallFileNullInUnique_fails(self): def test_determine_pk_smallFileNullInUnique_fails(self):
with before() as minio:
# mock # mock
MinioClient().upload_file("smallfile_nullinunique.csv", './data/test_pk/', 'dbrepo-upload') S3Client().upload_file("smallfile_nullinunique.csv", './data/test_pk/', 'dbrepo-upload')
# test # test
response = determine_pk('smallfile_nullinunique.csv') response = determine_pk('smallfile_nullinunique.csv')
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment