diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cbad2b94b4c70d6e6ebfad55cf22d251e3c3ed78..94468b42ac18c66c75c8a64b3bd1dd7ab98fd91b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -119,7 +119,7 @@ test-analyse-service:
script:
- "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy"
- - "cd ./dbrepo-analyse-service/ && coverage run -m pytest test/test_determine_dt.py test/test_determine_pk.py --junitxml=report.xml && coverage html && coverage report > ./coverage.txt"
+ - "cd ./dbrepo-analyse-service/ && coverage run -m pytest test/test_determine_dt.py test/test_determine_pk.py test/test_determine_stats.py --junitxml=report.xml && coverage html && coverage report > ./coverage.txt"
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts:
when: always
diff --git a/dbrepo-analyse-service/Dockerfile b/dbrepo-analyse-service/Dockerfile
index 50d56f60e22eb354ac003b821148c2ab0313ef3c..714cfc9e8580741f3dc9e24f3cae048001db95ba 100644
--- a/dbrepo-analyse-service/Dockerfile
+++ b/dbrepo-analyse-service/Dockerfile
@@ -1,7 +1,7 @@
FROM python:3.9-slim
MAINTAINER Martin Weise <martin.weise@tuwien.ac.at>
-RUN apt update && apt install -y curl
+RUN apt update && apt install -y curl gcc libmariadb-dev
WORKDIR /app
diff --git a/dbrepo-analyse-service/Pipfile b/dbrepo-analyse-service/Pipfile
index 29d57acc66b8dab57afb21b1a9922439e753af5a..99ab9195d0612621ea6fa0bd535b880e51807896 100644
--- a/dbrepo-analyse-service/Pipfile
+++ b/dbrepo-analyse-service/Pipfile
@@ -16,11 +16,16 @@ numpy = "*"
pandas = "*"
messytables = "*"
minio = "*"
+flask-sqlalchemy = "*"
+opensearch-py = "*"
+pymysql = "*"
[dev-packages]
coverage = "*"
pytest = "*"
testcontainers-minio = "*"
+testcontainers-mysql = "*"
+testcontainers-opensearch = "*"
[requires]
python_version = "3.9"
diff --git a/dbrepo-analyse-service/Pipfile.lock b/dbrepo-analyse-service/Pipfile.lock
index 1000eddec96027aff2f5400389a7d6fa52842157..776c27c79292e25b1ef041eaf9e7cc4c136f17ed 100644
--- a/dbrepo-analyse-service/Pipfile.lock
+++ b/dbrepo-analyse-service/Pipfile.lock
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
- "sha256": "fe382056038fea58538a52ab9f8ac7bd80659537f8336a534cb79a71e515bceb"
+ "sha256": "593bb7bb2fab9041c4da489021bb7fe2f857624e18f484d53f1c4f3958b04837"
},
"pipfile-spec": 6,
"requires": {
@@ -69,19 +69,19 @@
},
"boto3": {
"hashes": [
- "sha256:1f94042f4efb5133b6b9b8b3243afc01143a81d21b3197a3afadf5780f97b05d",
- "sha256:5c1bb487c68120aae236354d81b8a1a55d0aa3395d30748a01825ef90891921e"
+ "sha256:1efc02be786884034d503d59c018cf7650d0cff9fcb37cd2eb49b802a6fe6111",
+ "sha256:8ca248cc84e7e859e4e276eb9c4309fa01a3e58473bf48d6c33448be870c2bb8"
],
"index": "pypi",
- "version": "==1.34.14"
+ "version": "==1.34.17"
},
"botocore": {
"hashes": [
- "sha256:041bed0852649cab7e4dcd4d87f9d1cc084467fb846e5b60015e014761d96414",
- "sha256:3b592f50f0406e236782a3a0a9ad1c3976060fdb2e04a23d18c3df5b7dfad3e0"
+ "sha256:7272c39032c6f1d62781e4c8445d9a1d9140c2bf52ba7ee66bf6db559c4b2427",
+ "sha256:e48a662f3a6919219276b55085e8f73c3347966675f55e9d448be30cf79678ee"
],
"markers": "python_version >= '3.8'",
- "version": "==1.34.14"
+ "version": "==1.34.17"
},
"certifi": {
"hashes": [
@@ -250,7 +250,7 @@
"sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519",
"sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"
],
- "markers": "python_full_version >= '3.7.0'",
+ "markers": "python_version >= '3.7'",
"version": "==3.3.2"
},
"click": {
@@ -284,6 +284,14 @@
"index": "pypi",
"version": "==3.0.0"
},
+ "flask-sqlalchemy": {
+ "hashes": [
+ "sha256:4ba4be7f419dc72f4efd8802d69974803c37259dd42f3913b0dcf75c9447e0a0",
+ "sha256:e4b68bb881802dda1a7d878b2fc84c06d1ee57fb40b874d3dc97dabfa36b8312"
+ ],
+ "index": "pypi",
+ "version": "==3.1.1"
+ },
"gevent": {
"hashes": [
"sha256:272cffdf535978d59c38ed837916dfd2b5d193be1e9e5dcc60a5f4d5025dd98a",
@@ -391,7 +399,7 @@
"sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da",
"sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33"
],
- "markers": "python_version >= '3.11' and platform_python_implementation == 'CPython'",
+ "index": "pypi",
"version": "==3.0.3"
},
"gunicorn": {
@@ -418,6 +426,14 @@
"markers": "python_version >= '3.5'",
"version": "==3.6"
},
+ "importlib-metadata": {
+ "hashes": [
+ "sha256:4805911c3a4ec7c3966410053e9ec6a1fecd629117df5adee56dfc9432a1081e",
+ "sha256:f238736bb06590ae52ac1fab06a3a9ef1d8dce2b7a35b5ab329371d6c8f5d2cc"
+ ],
+ "markers": "python_version < '3.10'",
+ "version": "==7.0.1"
+ },
"itsdangerous": {
"hashes": [
"sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44",
@@ -428,11 +444,11 @@
},
"jinja2": {
"hashes": [
- "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852",
- "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"
+ "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa",
+ "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"
],
"markers": "python_version >= '3.7'",
- "version": "==3.1.2"
+ "version": "==3.1.3"
},
"jmespath": {
"hashes": [
@@ -466,97 +482,87 @@
},
"lxml": {
"hashes": [
- "sha256:00bfccab28f710bb13f7f644c980ad50ce3e5b6a412b5bb9a6c30136b298fb2c",
- "sha256:03c977ffc9a4bf17b3e0f8db0451dc38e9f4ec92cfdb5df462d38fbe6e6e0825",
- "sha256:0499310df9afc0ce634ce14cacbb333d62f561038ea4db640494e4a22ac4f2e9",
- "sha256:070469a23f2ef3a9e72165af7e0b12eca9a6e47c3a8ec1cad60d14fb2f2c3aa8",
- "sha256:0963de4fe463caff48e6ce4d83d19a3c099126874185d10cf490c29057ca518d",
- "sha256:0a79eca2ef5e032c8ed9da07f84a07a29105f220b777613dfe7fc31445691ee3",
- "sha256:0dc36ec06514fe8848c4733d47f96a0636f82d9ca3eaa2132373426bc03f178f",
- "sha256:0ecc0f1e1d901b66f2f68edff85b8ff421fa9683d02eaea6742a42c145d741b6",
- "sha256:0f70e5de6b3e24ababeca597f776e5f37973f05d28a4d9f467aa5b45745af762",
- "sha256:0fb55d77e685def5218701d5d296fca62f595752c88402404da685864b06b67e",
- "sha256:1215c8e57a25ad68488abb83a36734f6c6b3f0ccd880f0c68da98682d463ef09",
- "sha256:1546fa25a6dde5698271e19787062446f433c98ca7eab35545f665dde2c1bd34",
- "sha256:18b456f1bfb338be94a916166ac5507e73b9eb9f6e1f0fbd1c8caff2f3fa5535",
- "sha256:18caa0d3925902949cb060fe5f2da70c953d60bd9ef78657bd389f6db30533cc",
- "sha256:19251f782ea51a4841e747158195312ef63e06b47889e159dc5f1b2e5d668465",
- "sha256:1d78e91cbffe733ff325e0d258bb64702c8d91f8f652db088bd2989c6a8f90cc",
- "sha256:2ec9fa65e0638551a5ad31cb9fa160b321f19632e5ec517fe68d7b4110133e69",
- "sha256:3013823b0069cb4bd9b558e87076a18142703c6d2ac3a5d5521dd35734d23a72",
- "sha256:31362b900b8fd2a026b9d24695ebe5449ea8f0c948af2445d7880b738d9fc368",
- "sha256:32a135d4ef8f966bc087d450d641df73fc6874f04cf6608111541b50090e6f13",
- "sha256:331237209fe76951450c1119af0879f04f32d1b07b21e83a34ba439520492feb",
- "sha256:3714f339449d2738b4fadd078a6022704a2af3cab06bec9627b19eaa4205090d",
- "sha256:387c5416b8bb4b8ad7306797cb2719a841f5f3836b3c39fcaa56b9af5448dd2a",
- "sha256:39a3cf581e9040e66aaa719b2f338b2b7eb43ce1db059089c82ae72e0fd48b47",
- "sha256:3ffc56d68b9782919d69ae9a6fac99efd7547f2666ccb7ecfd12871564d16133",
- "sha256:42069ce16dab9755b381b90defd846ca407b9ead05dc20732fd5502b5cc49b87",
- "sha256:430780608d16b3bb96ef99a67a1a0626c8a295193af53ce9c4d5ec3fef2fbc79",
- "sha256:4432a1d89a9b340bc6bd1201aef3ba03112f151d3f340d9218247dc0c85028ab",
- "sha256:45e747b4e82390ffd802528b9e7b39333c1ce71423057bf5961b30ec0d52f76f",
- "sha256:473f2d0511dd84697326ee9362b0c0c2e9f99a433dcb1fbb5aa8df3d1b2185db",
- "sha256:47f46a2ebade07f3afa47695882e7725440c49bf77cba39c3762a42597e5aad3",
- "sha256:49dc4dcf14a08f160bb3f5f571f63514d918b8933a25c221536571a5fc010271",
- "sha256:4ae66b6b0f82e7839b6b8d009182c652d48e7d2ea21a6709f3033ce5fbf199c4",
- "sha256:4b49a1569ed6d05808f4d163a316e7bf4a230e0c36855b59f56020ae27ae586a",
- "sha256:4d58af4ebd711dad40f1c024a779950d9918d04d74f49230edf5d271dcd33c28",
- "sha256:4df6be79be4d7574e9e4002aeb6aa03d3f3809582db07abb166df7fc6e7438af",
- "sha256:57f5c362cbd9d688fb1fa07c8955cec26d5c066fbcb4163aa341ff751eba7587",
- "sha256:5892cff0a6817743fe470b7402677310ffc8514a74de14d4e591cecdc457ff61",
- "sha256:60974986aa80a8bb883da5663f90bc632bd4ce0d0508e66a9132412facec65f6",
- "sha256:62a3c0fdf70f785cd29824666d1dcea88c207f0b73ddbc28fb7a6a1a5bbb1af7",
- "sha256:67ddff0272905a0b78a2c3ea01487e0551cc38094cd5994f73af370f355ecb47",
- "sha256:6af86081c888ce81ca7e361ed7fa2ba1678e2a86eb5a059c96d5a719364d319e",
- "sha256:6cdd0fb749c80ffcf408f659b209e82333f10b517786083a3dd3c3b5adc60111",
- "sha256:737a4dba9f7ee842c6597f719dda2eabeeaefe42443f7f24de20c262f88527cd",
- "sha256:73e71b95c5215310a92e560369ac1f0e2cd018d5a36be182da88958f3d6084f5",
- "sha256:745383c124f096cc03fb942c8a05ea1e8cb4f44c5b28887adce6224e4540808e",
- "sha256:7febf50135363e981097eeada84781eeae92bfc3c203495f63d6b542a7132ba7",
- "sha256:821fb791565536617b058c2cd5b8d28a1285b3375eecc5bd6b5c6d87add0d3dd",
- "sha256:83ff41e1bc0666f31acda52407d869ea257f232c2d9394806647a0e7454de73e",
- "sha256:8991837fdf8086486f1c300d936bacd757e2e5398be84cd54a1fba0e6b6d5591",
- "sha256:8a70c47c14f89b8bfb430f85b608aa460204fe7c005545d79afd31b925cc6669",
- "sha256:8ba56c3686fa60cc04191d22d1733aad484c9cbc153cdc3e8eb9bdfcad30f704",
- "sha256:8de180f748a17382dd695b3999be03a647d09af16ae589c4e9c37138ddb6d4c6",
- "sha256:96c2abcbcb24f850f00f279c4660ce6498cae16ff1659845838d752c26890748",
- "sha256:99b5ca5775435aa296d32ea711e194aaae871b21dbf0d57cb7d4431e5d3ad699",
- "sha256:9a4eff4d8ad0bbc9f470a9be19c5e718af4baf47111d7c2d9b036b9986107e7c",
- "sha256:9a99dae826c80cf0a21dd21eb66db16310d1f58ac4c27c804da980212fbcabe2",
- "sha256:9e8a4782ecaaacf8e9355f39179b1f00e7f27b774306eccbe80f6215724be4cd",
- "sha256:a28eab2d9ea79b830be50e3350d827ae8abf5b23e278e14929824d5ab2069008",
- "sha256:ae3a0ec0f1b6cf1e8bca41bc86cd64ba02e31c71716efbf149a0f7ebc168cf0b",
- "sha256:b005101a257c494e84d36ecb62b02ba195b02b7f8892f57b1f5aaa352ed44467",
- "sha256:b4eef43c5dc5c579d0804e55a32dd1bacbd008c8191ed4d65be278bbb11ddc61",
- "sha256:b6bb5a0a87ab1e01f086cbb418be9e409719cd216954aa38b1cceee36a561ce1",
- "sha256:b7bb0010c2969c23bf3d2b3892e638a7cb83e7daeb749e3db5f3c002fd191e11",
- "sha256:b889c0b9be774466308c3590e093ce9a6f9115c78fc8624aa6ba8dfeaf5188ab",
- "sha256:bcded868b05583d41ab0b024f39f90a04e486a2349a9b493d8d17024f1433aa6",
- "sha256:c21e60c017cab3a7e7786185cc8853b8614b01ccd69cc8b24608e5356784631b",
- "sha256:c32a4fbae218b336d547bc626897325202e4e9f1794ac5f4d0bb3caacf41df21",
- "sha256:c4eaa83b610595ef9f20aa69b96053d5b7f3f70c67c7a3af8f433136a9d315ab",
- "sha256:c5b23f63fcec652bf1e775eca5e03a713a4994d2a7ce2e70a91e964a26220e0d",
- "sha256:cfdac95815d3025e4c9edce2ef2ebe4e034edc35a2c44a606dd846554387ae38",
- "sha256:d0047c90e0ebd0d8f3c1e6636e10f597b8f25e4ef9e6416dd2e5c4c0960270cc",
- "sha256:d05cf827f160340f67c25ce6f271689a844605aa123849f1a80e21c9bd21f00b",
- "sha256:d64e543b07964ff73b4eb994bee894803a80e19fd3b29a5ffbe3c637fe43e788",
- "sha256:d6d3ce5638cd4ed3fa684507f164e7039e1b07475bc8f37ba6e12271c1a2e9e0",
- "sha256:d80d9f4d986bb6ad65bae86f07391152f7b6c65cfc63d118616b18b0be2e79da",
- "sha256:dbff288e1869db78f8731ca257553dd699edef07e173b35e71b1122b630d6008",
- "sha256:dcb25128c9e7f01c00ad116d2c762c3942724981a35c6e5c551ab55d4c2bfcfe",
- "sha256:dcc7dc4b9b65f185aa8533abc78f0a3b2ac38fe075bb23d3c1590ba0990f6c80",
- "sha256:e2388a792f9c239510d62a9e615662b8202e4ca275aafcc9c4af154654462a14",
- "sha256:e2bd8faf6a9682ca385e4bca1a38a057be716dc303f16ddec9e4c9cf01b7d722",
- "sha256:e97c74725e86d84a477df081eef69b70f048128afee841dbd8c690a9e3d2e8e0",
- "sha256:ea5e4b3eff9029a02fe7736540675ab8fca44277232f0027397b0d7111d04b1c",
- "sha256:f36c3103a6f2641777b64f1da860f37cbaa718ce3feea621583627269e68eb03",
- "sha256:f56e6a38c64a79e228d48344bb3bec265ac035fc1277ce8c049445bb18e4cd41",
- "sha256:f9464ff2fd1f2ba4d0d246a560d369ee7e7213c556a30db9ae9426850ce1baf9",
- "sha256:feb1102d9e5de08120d46a1011110c43b2547ecb3ae80030801e0e2dacd1ee18",
- "sha256:ff29353c12f0abc9cb3395899b7192a970d5a63f80ac1e7f0c3247ed83f5dcd4"
+ "sha256:13521a321a25c641b9ea127ef478b580b5ec82aa2e9fc076c86169d161798b01",
+ "sha256:14deca1460b4b0f6b01f1ddc9557704e8b365f55c63070463f6c18619ebf964f",
+ "sha256:16018f7099245157564d7148165132c70adb272fb5a17c048ba70d9cc542a1a1",
+ "sha256:16dd953fb719f0ffc5bc067428fc9e88f599e15723a85618c45847c96f11f431",
+ "sha256:19a1bc898ae9f06bccb7c3e1dfd73897ecbbd2c96afe9095a6026016e5ca97b8",
+ "sha256:1ad17c20e3666c035db502c78b86e58ff6b5991906e55bdbef94977700c72623",
+ "sha256:22b7ee4c35f374e2c20337a95502057964d7e35b996b1c667b5c65c567d2252a",
+ "sha256:24ef5a4631c0b6cceaf2dbca21687e29725b7c4e171f33a8f8ce23c12558ded1",
+ "sha256:25663d6e99659544ee8fe1b89b1a8c0aaa5e34b103fab124b17fa958c4a324a6",
+ "sha256:262bc5f512a66b527d026518507e78c2f9c2bd9eb5c8aeeb9f0eb43fcb69dc67",
+ "sha256:280f3edf15c2a967d923bcfb1f8f15337ad36f93525828b40a0f9d6c2ad24890",
+ "sha256:2ad3a8ce9e8a767131061a22cd28fdffa3cd2dc193f399ff7b81777f3520e372",
+ "sha256:2befa20a13f1a75c751f47e00929fb3433d67eb9923c2c0b364de449121f447c",
+ "sha256:2f37c6d7106a9d6f0708d4e164b707037b7380fcd0b04c5bd9cae1fb46a856fb",
+ "sha256:304128394c9c22b6569eba2a6d98392b56fbdfbad58f83ea702530be80d0f9df",
+ "sha256:342e95bddec3a698ac24378d61996b3ee5ba9acfeb253986002ac53c9a5f6f84",
+ "sha256:3aeca824b38ca78d9ee2ab82bd9883083d0492d9d17df065ba3b94e88e4d7ee6",
+ "sha256:3d184e0d5c918cff04cdde9dbdf9600e960161d773666958c9d7b565ccc60c45",
+ "sha256:3e3898ae2b58eeafedfe99e542a17859017d72d7f6a63de0f04f99c2cb125936",
+ "sha256:3eea6ed6e6c918e468e693c41ef07f3c3acc310b70ddd9cc72d9ef84bc9564ca",
+ "sha256:3f14a4fb1c1c402a22e6a341a24c1341b4a3def81b41cd354386dcb795f83897",
+ "sha256:436a943c2900bb98123b06437cdd30580a61340fbdb7b28aaf345a459c19046a",
+ "sha256:4946e7f59b7b6a9e27bef34422f645e9a368cb2be11bf1ef3cafc39a1f6ba68d",
+ "sha256:49a9b4af45e8b925e1cd6f3b15bbba2c81e7dba6dce170c677c9cda547411e14",
+ "sha256:4f8b0c78e7aac24979ef09b7f50da871c2de2def043d468c4b41f512d831e912",
+ "sha256:52427a7eadc98f9e62cb1368a5079ae826f94f05755d2d567d93ee1bc3ceb354",
+ "sha256:5e53d7e6a98b64fe54775d23a7c669763451340c3d44ad5e3a3b48a1efbdc96f",
+ "sha256:5fcfbebdb0c5d8d18b84118842f31965d59ee3e66996ac842e21f957eb76138c",
+ "sha256:601f4a75797d7a770daed8b42b97cd1bb1ba18bd51a9382077a6a247a12aa38d",
+ "sha256:61c5a7edbd7c695e54fca029ceb351fc45cd8860119a0f83e48be44e1c464862",
+ "sha256:6a2a2c724d97c1eb8cf966b16ca2915566a4904b9aad2ed9a09c748ffe14f969",
+ "sha256:6d48fc57e7c1e3df57be5ae8614bab6d4e7b60f65c5457915c26892c41afc59e",
+ "sha256:6f11b77ec0979f7e4dc5ae081325a2946f1fe424148d3945f943ceaede98adb8",
+ "sha256:704f5572ff473a5f897745abebc6df40f22d4133c1e0a1f124e4f2bd3330ff7e",
+ "sha256:725e171e0b99a66ec8605ac77fa12239dbe061482ac854d25720e2294652eeaa",
+ "sha256:7cfced4a069003d8913408e10ca8ed092c49a7f6cefee9bb74b6b3e860683b45",
+ "sha256:7ec465e6549ed97e9f1e5ed51c657c9ede767bc1c11552f7f4d022c4df4a977a",
+ "sha256:82bddf0e72cb2af3cbba7cec1d2fd11fda0de6be8f4492223d4a268713ef2147",
+ "sha256:82cd34f1081ae4ea2ede3d52f71b7be313756e99b4b5f829f89b12da552d3aa3",
+ "sha256:843b9c835580d52828d8f69ea4302537337a21e6b4f1ec711a52241ba4a824f3",
+ "sha256:877efb968c3d7eb2dad540b6cabf2f1d3c0fbf4b2d309a3c141f79c7e0061324",
+ "sha256:8b9f19df998761babaa7f09e6bc169294eefafd6149aaa272081cbddc7ba4ca3",
+ "sha256:8cf5877f7ed384dabfdcc37922c3191bf27e55b498fecece9fd5c2c7aaa34c33",
+ "sha256:8d2900b7f5318bc7ad8631d3d40190b95ef2aa8cc59473b73b294e4a55e9f30f",
+ "sha256:8d7b4beebb178e9183138f552238f7e6613162a42164233e2bda00cb3afac58f",
+ "sha256:8f52fe6859b9db71ee609b0c0a70fea5f1e71c3462ecf144ca800d3f434f0764",
+ "sha256:98f3f020a2b736566c707c8e034945c02aa94e124c24f77ca097c446f81b01f1",
+ "sha256:9aa543980ab1fbf1720969af1d99095a548ea42e00361e727c58a40832439114",
+ "sha256:9b99f564659cfa704a2dd82d0684207b1aadf7d02d33e54845f9fc78e06b7581",
+ "sha256:9bcf86dfc8ff3e992fed847c077bd875d9e0ba2fa25d859c3a0f0f76f07f0c8d",
+ "sha256:9bd0ae7cc2b85320abd5e0abad5ccee5564ed5f0cc90245d2f9a8ef330a8deae",
+ "sha256:9d3c0f8567ffe7502d969c2c1b809892dc793b5d0665f602aad19895f8d508da",
+ "sha256:9e5ac3437746189a9b4121db2a7b86056ac8786b12e88838696899328fc44bb2",
+ "sha256:a36c506e5f8aeb40680491d39ed94670487ce6614b9d27cabe45d94cd5d63e1e",
+ "sha256:a5ab722ae5a873d8dcee1f5f45ddd93c34210aed44ff2dc643b5025981908cda",
+ "sha256:a96f02ba1bcd330807fc060ed91d1f7a20853da6dd449e5da4b09bfcc08fdcf5",
+ "sha256:acb6b2f96f60f70e7f34efe0c3ea34ca63f19ca63ce90019c6cbca6b676e81fa",
+ "sha256:ae15347a88cf8af0949a9872b57a320d2605ae069bcdf047677318bc0bba45b1",
+ "sha256:af8920ce4a55ff41167ddbc20077f5698c2e710ad3353d32a07d3264f3a2021e",
+ "sha256:afd825e30f8d1f521713a5669b63657bcfe5980a916c95855060048b88e1adb7",
+ "sha256:b21b4031b53d25b0858d4e124f2f9131ffc1530431c6d1321805c90da78388d1",
+ "sha256:b4b68c961b5cc402cbd99cca5eb2547e46ce77260eb705f4d117fd9c3f932b95",
+ "sha256:b66aa6357b265670bb574f050ffceefb98549c721cf28351b748be1ef9577d93",
+ "sha256:b9e240ae0ba96477682aa87899d94ddec1cc7926f9df29b1dd57b39e797d5ab5",
+ "sha256:bc64d1b1dab08f679fb89c368f4c05693f58a9faf744c4d390d7ed1d8223869b",
+ "sha256:bf8443781533b8d37b295016a4b53c1494fa9a03573c09ca5104550c138d5c05",
+ "sha256:c26aab6ea9c54d3bed716b8851c8bfc40cb249b8e9880e250d1eddde9f709bf5",
+ "sha256:c3cd1fc1dc7c376c54440aeaaa0dcc803d2126732ff5c6b68ccd619f2e64be4f",
+ "sha256:c7257171bb8d4432fe9d6fdde4d55fdbe663a63636a17f7f9aaba9bcb3153ad7",
+ "sha256:d42e3a3fc18acc88b838efded0e6ec3edf3e328a58c68fbd36a7263a874906c8",
+ "sha256:d74fcaf87132ffc0447b3c685a9f862ffb5b43e70ea6beec2fb8057d5d2a1fea",
+ "sha256:d8c1d679df4361408b628f42b26a5d62bd3e9ba7f0c0e7969f925021554755aa",
+ "sha256:e856c1c7255c739434489ec9c8aa9cdf5179785d10ff20add308b5d673bed5cd",
+ "sha256:eac68f96539b32fce2c9b47eb7c25bb2582bdaf1bbb360d25f564ee9e04c542b",
+ "sha256:ed7326563024b6e91fef6b6c7a1a2ff0a71b97793ac33dbbcf38f6005e51ff6e",
+ "sha256:ed8c3d2cd329bf779b7ed38db176738f3f8be637bb395ce9629fc76f78afe3d4",
+ "sha256:f4c9bda132ad108b387c33fabfea47866af87f4ea6ffb79418004f0521e63204",
+ "sha256:f643ffd2669ffd4b5a3e9b41c909b72b2a1d5e4915da90a77e119b8d48ce867a"
],
- "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
- "version": "==5.0.1"
+ "markers": "python_version >= '3.6'",
+ "version": "==5.1.0"
},
"markupsafe": {
"hashes": [
@@ -689,6 +695,14 @@
"index": "pypi",
"version": "==1.26.3"
},
+ "opensearch-py": {
+ "hashes": [
+ "sha256:564f175af134aa885f4ced6846eb4532e08b414fff0a7976f76b276fe0e69158",
+ "sha256:7867319132133e2974c09f76a54eb1d502b989229be52da583d93ddc743ea111"
+ ],
+ "index": "pypi",
+ "version": "==2.4.2"
+ },
"packaging": {
"hashes": [
"sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5",
@@ -753,41 +767,49 @@
},
"pycryptodome": {
"hashes": [
- "sha256:08b445799d571041765e7d5c9ca09c5d3866c2f22eeb0dd4394a4169285184f4",
- "sha256:11ddf6c9b52116b62223b6a9f4741bc4f62bb265392a4463282f7f34bb287180",
- "sha256:1c6273ca5a03b672e504995529b8bae56da0ebb691d8ef141c4aa68f60765700",
- "sha256:27e1efcb68993b7ce5d1d047a46a601d41281bba9f1971e6be4aa27c69ab8065",
- "sha256:2c16426ef49d9cba018be2340ea986837e1dfa25c2ea181787971654dd49aadd",
- "sha256:37e531bf896b70fe302f003d3be5a0a8697737a8d177967da7e23eff60d6483c",
- "sha256:420972f9c62978e852c74055d81c354079ce3c3a2213a92c9d7e37bbc63a26e2",
- "sha256:4464b0e8fd5508bff9baf18e6fd4c6548b1ac2ce9862d6965ff6a84ec9cb302a",
- "sha256:4805e053571140cb37cf153b5c72cd324bb1e3e837cbe590a19f69b6cf85fd03",
- "sha256:67939a3adbe637281c611596e44500ff309d547e932c449337649921b17b6297",
- "sha256:694020d2ff985cd714381b9da949a21028c24b86f562526186f6af7c7547e986",
- "sha256:6c3df3613592ea6afaec900fd7189d23c8c28b75b550254f4bd33fe94acb84b9",
- "sha256:6d0d2b97758ebf2f36c39060520447c26455acb3bcff309c28b1c816173a6ff5",
- "sha256:7c9e222d0976f68d0cf6409cfea896676ddc1d98485d601e9508f90f60e2b0a2",
- "sha256:81e9d23c0316fc1b45d984a44881b220062336bbdc340aa9218e8d0656587934",
- "sha256:8ae0dd1bcfada451c35f9e29a3e5db385caabc190f98e4a80ad02a61098fb776",
- "sha256:8cf5d3d6cf921fa81acd1f632f6cedcc03f5f68fc50c364cd39490ba01d17c49",
- "sha256:954d156cd50130afd53f8d77f830fe6d5801bd23e97a69d358fed068f433fbfe",
- "sha256:a470237ee71a1efd63f9becebc0ad84b88ec28e6784a2047684b693f458f41b7",
- "sha256:a991f8ffe8dfe708f86690948ae46442eebdd0fff07dc1b605987939a34ec979",
- "sha256:b0bfe61506795877ff974f994397f0c862d037f6f1c0bfc3572195fc00833b96",
- "sha256:b7efd46b0b4ac869046e814d83244aeab14ef787f4850644119b1c8b0ec2d637",
- "sha256:b8b80ff92049fd042177282917d994d344365ab7e8ec2bc03e853d93d2401786",
- "sha256:c1bc0c49d986a1491d66d2a56570f12e960b12508b7e71f2423f532e28857f36",
- "sha256:c22c80246c3c880c6950d2a8addf156cee74ec0dc5757d01e8e7067a3c7da015",
- "sha256:cd4e7e8bf0fc1ada854688b9b309ee607e2aa85a8b44180f91021a4dd330a928",
- "sha256:cd4e95b0eb4b28251c825fe7aa941fe077f993e5ca9b855665935b86fbb1cc08",
- "sha256:e038ab77fec0956d7aa989a3c647652937fc142ef41c9382c2ebd13c127d5b4a",
- "sha256:e3e6f89480616781d2a7f981472d0cdb09b9da9e8196f43c1234eff45c915766",
- "sha256:e70f5c839c7798743a948efa2a65d1fe96bb397fe6d7f2bde93d869fe4f0ad69",
- "sha256:ed932eb6c2b1c4391e166e1a562c9d2f020bfff44a0e1b108f67af38b390ea89",
- "sha256:f34976c5c8eb79e14c7d970fb097482835be8d410a4220f86260695ede4c3e17"
+ "sha256:06d6de87c19f967f03b4cf9b34e538ef46e99a337e9a61a77dbe44b2cbcf0690",
+ "sha256:09609209ed7de61c2b560cc5c8c4fbf892f8b15b1faf7e4cbffac97db1fffda7",
+ "sha256:210ba1b647837bfc42dd5a813cdecb5b86193ae11a3f5d972b9a0ae2c7e9e4b4",
+ "sha256:2a1250b7ea809f752b68e3e6f3fd946b5939a52eaeea18c73bdab53e9ba3c2dd",
+ "sha256:2ab6ab0cb755154ad14e507d1df72de9897e99fd2d4922851a276ccc14f4f1a5",
+ "sha256:3427d9e5310af6680678f4cce149f54e0bb4af60101c7f2c16fdf878b39ccccc",
+ "sha256:3cd3ef3aee1079ae44afaeee13393cf68b1058f70576b11439483e34f93cf818",
+ "sha256:405002eafad114a2f9a930f5db65feef7b53c4784495dd8758069b89baf68eab",
+ "sha256:417a276aaa9cb3be91f9014e9d18d10e840a7a9b9a9be64a42f553c5b50b4d1d",
+ "sha256:4401564ebf37dfde45d096974c7a159b52eeabd9969135f0426907db367a652a",
+ "sha256:49a4c4dc60b78ec41d2afa392491d788c2e06edf48580fbfb0dd0f828af49d25",
+ "sha256:5601c934c498cd267640b57569e73793cb9a83506f7c73a8ec57a516f5b0b091",
+ "sha256:6e0e4a987d38cfc2e71b4a1b591bae4891eeabe5fa0f56154f576e26287bfdea",
+ "sha256:76658f0d942051d12a9bd08ca1b6b34fd762a8ee4240984f7c06ddfb55eaf15a",
+ "sha256:76cb39afede7055127e35a444c1c041d2e8d2f1f9c121ecef573757ba4cd2c3c",
+ "sha256:8d6b98d0d83d21fb757a182d52940d028564efe8147baa9ce0f38d057104ae72",
+ "sha256:9b3ae153c89a480a0ec402e23db8d8d84a3833b65fa4b15b81b83be9d637aab9",
+ "sha256:a60fedd2b37b4cb11ccb5d0399efe26db9e0dd149016c1cc6c8161974ceac2d6",
+ "sha256:ac1c7c0624a862f2e53438a15c9259d1655325fc2ec4392e66dc46cdae24d044",
+ "sha256:acae12b9ede49f38eb0ef76fdec2df2e94aad85ae46ec85be3648a57f0a7db04",
+ "sha256:acc2614e2e5346a4a4eab6e199203034924313626f9620b7b4b38e9ad74b7e0c",
+ "sha256:acf6e43fa75aca2d33e93409f2dafe386fe051818ee79ee8a3e21de9caa2ac9e",
+ "sha256:baee115a9ba6c5d2709a1e88ffe62b73ecc044852a925dcb67713a288c4ec70f",
+ "sha256:c18b381553638414b38705f07d1ef0a7cf301bc78a5f9bc17a957eb19446834b",
+ "sha256:d29daa681517f4bc318cd8a23af87e1f2a7bad2fe361e8aa29c77d652a065de4",
+ "sha256:d5954acfe9e00bc83ed9f5cb082ed22c592fbbef86dc48b907238be64ead5c33",
+ "sha256:ec0bb1188c1d13426039af8ffcb4dbe3aad1d7680c35a62d8eaf2a529b5d3d4f",
+ "sha256:ec1f93feb3bb93380ab0ebf8b859e8e5678c0f010d2d78367cf6bc30bfeb148e",
+ "sha256:f0e6d631bae3f231d3634f91ae4da7a960f7ff87f2865b2d2b831af1dfb04e9a",
+ "sha256:f35d6cee81fa145333137009d9c8ba90951d7d77b67c79cbe5f03c7eb74d8fe2",
+ "sha256:f47888542a0633baff535a04726948e876bf1ed880fddb7c10a736fa99146ab3",
+ "sha256:fb3b87461fa35afa19c971b0a2b7456a7b1db7b4eba9a8424666104925b78128"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
- "version": "==3.19.1"
+ "version": "==3.20.0"
+ },
+ "pymysql": {
+ "hashes": [
+ "sha256:4f13a7df8bf36a51e81dd9f3605fede45a4878fe02f9236349fd82a3f0612f96",
+ "sha256:8969ec6d763c856f7073c4c64662882675702efcb114b4bcbb955aea3a069fa7"
+ ],
+ "index": "pypi",
+ "version": "==1.1.0"
},
"python-dateutil": {
"hashes": [
@@ -1013,6 +1035,61 @@
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==1.16.0"
},
+ "sqlalchemy": {
+ "hashes": [
+ "sha256:0d3cab3076af2e4aa5693f89622bef7fa770c6fec967143e4da7508b3dceb9b9",
+ "sha256:0dacf67aee53b16f365c589ce72e766efaabd2b145f9de7c917777b575e3659d",
+ "sha256:10331f129982a19df4284ceac6fe87353ca3ca6b4ca77ff7d697209ae0a5915e",
+ "sha256:14a6f68e8fc96e5e8f5647ef6cda6250c780612a573d99e4d881581432ef1669",
+ "sha256:1b1180cda6df7af84fe72e4530f192231b1f29a7496951db4ff38dac1687202d",
+ "sha256:29049e2c299b5ace92cbed0c1610a7a236f3baf4c6b66eb9547c01179f638ec5",
+ "sha256:342d365988ba88ada8af320d43df4e0b13a694dbd75951f537b2d5e4cb5cd002",
+ "sha256:420362338681eec03f53467804541a854617faed7272fe71a1bfdb07336a381e",
+ "sha256:4344d059265cc8b1b1be351bfb88749294b87a8b2bbe21dfbe066c4199541ebd",
+ "sha256:4f7a7d7fcc675d3d85fbf3b3828ecd5990b8d61bd6de3f1b260080b3beccf215",
+ "sha256:555651adbb503ac7f4cb35834c5e4ae0819aab2cd24857a123370764dc7d7e24",
+ "sha256:59a21853f5daeb50412d459cfb13cb82c089ad4c04ec208cd14dddd99fc23b39",
+ "sha256:5fdd402169aa00df3142149940b3bf9ce7dde075928c1886d9a1df63d4b8de62",
+ "sha256:605b6b059f4b57b277f75ace81cc5bc6335efcbcc4ccb9066695e515dbdb3900",
+ "sha256:665f0a3954635b5b777a55111ababf44b4fc12b1f3ba0a435b602b6387ffd7cf",
+ "sha256:6f9e2e59cbcc6ba1488404aad43de005d05ca56e069477b33ff74e91b6319735",
+ "sha256:736ea78cd06de6c21ecba7416499e7236a22374561493b456a1f7ffbe3f6cdb4",
+ "sha256:74b080c897563f81062b74e44f5a72fa44c2b373741a9ade701d5f789a10ba23",
+ "sha256:75432b5b14dc2fff43c50435e248b45c7cdadef73388e5610852b95280ffd0e9",
+ "sha256:75f99202324383d613ddd1f7455ac908dca9c2dd729ec8584c9541dd41822a2c",
+ "sha256:790f533fa5c8901a62b6fef5811d48980adeb2f51f1290ade8b5e7ba990ba3de",
+ "sha256:798f717ae7c806d67145f6ae94dc7c342d3222d3b9a311a784f371a4333212c7",
+ "sha256:7c88f0c7dcc5f99bdb34b4fd9b69b93c89f893f454f40219fe923a3a2fd11625",
+ "sha256:7d505815ac340568fd03f719446a589162d55c52f08abd77ba8964fbb7eb5b5f",
+ "sha256:84daa0a2055df9ca0f148a64fdde12ac635e30edbca80e87df9b3aaf419e144a",
+ "sha256:87d91043ea0dc65ee583026cb18e1b458d8ec5fc0a93637126b5fc0bc3ea68c4",
+ "sha256:87f6e732bccd7dcf1741c00f1ecf33797383128bd1c90144ac8adc02cbb98643",
+ "sha256:884272dcd3ad97f47702965a0e902b540541890f468d24bd1d98bcfe41c3f018",
+ "sha256:8b8cb63d3ea63b29074dcd29da4dc6a97ad1349151f2d2949495418fd6e48db9",
+ "sha256:91f7d9d1c4dd1f4f6e092874c128c11165eafcf7c963128f79e28f8445de82d5",
+ "sha256:a2c69a7664fb2d54b8682dd774c3b54f67f84fa123cf84dda2a5f40dcaa04e08",
+ "sha256:a3be4987e3ee9d9a380b66393b77a4cd6d742480c951a1c56a23c335caca4ce3",
+ "sha256:a86b4240e67d4753dc3092d9511886795b3c2852abe599cffe108952f7af7ac3",
+ "sha256:aa9373708763ef46782d10e950b49d0235bfe58facebd76917d3f5cbf5971aed",
+ "sha256:b64b183d610b424a160b0d4d880995e935208fc043d0302dd29fee32d1ee3f95",
+ "sha256:b801154027107461ee992ff4b5c09aa7cc6ec91ddfe50d02bca344918c3265c6",
+ "sha256:bb209a73b8307f8fe4fe46f6ad5979649be01607f11af1eb94aa9e8a3aaf77f0",
+ "sha256:bc8b7dabe8e67c4832891a5d322cec6d44ef02f432b4588390017f5cec186a84",
+ "sha256:c51db269513917394faec5e5c00d6f83829742ba62e2ac4fa5c98d58be91662f",
+ "sha256:c55731c116806836a5d678a70c84cb13f2cedba920212ba7dcad53260997666d",
+ "sha256:cf18ff7fc9941b8fc23437cc3e68ed4ebeff3599eec6ef5eebf305f3d2e9a7c2",
+ "sha256:d24f571990c05f6b36a396218f251f3e0dda916e0c687ef6fdca5072743208f5",
+ "sha256:db854730a25db7c956423bb9fb4bdd1216c839a689bf9cc15fada0a7fb2f4570",
+ "sha256:dc55990143cbd853a5d038c05e79284baedf3e299661389654551bd02a6a68d7",
+ "sha256:e607cdd99cbf9bb80391f54446b86e16eea6ad309361942bf88318bcd452363c",
+ "sha256:ecf6d4cda1f9f6cb0b45803a01ea7f034e2f1aed9475e883410812d9f9e3cfcf",
+ "sha256:f2a159111a0f58fb034c93eeba211b4141137ec4b0a6e75789ab7a3ef3c7e7e3",
+ "sha256:f37c0caf14b9e9b9e8f6dbc81bc56db06acb4363eba5a633167781a48ef036ed",
+ "sha256:f5693145220517b5f42393e07a6898acdfe820e136c98663b971906120549da5"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==2.0.25"
+ },
"typing-extensions": {
"hashes": [
"sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783",
@@ -1031,11 +1108,11 @@
},
"urllib3": {
"hashes": [
- "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84",
- "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e"
+ "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07",
+ "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"
],
- "markers": "python_version >= '3.7'",
- "version": "==2.0.7"
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
+ "version": "==1.26.18"
},
"webencodings": {
"hashes": [
@@ -1060,6 +1137,14 @@
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
"version": "==2.0.1"
},
+ "zipp": {
+ "hashes": [
+ "sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31",
+ "sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0"
+ ],
+ "markers": "python_version >= '3.8'",
+ "version": "==3.17.0"
+ },
"zope.event": {
"hashes": [
"sha256:2832e95014f4db26c47a13fdaef84cef2f4df37e66b59d8f1f4a8f319a632c26",
@@ -1306,7 +1391,7 @@
"sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519",
"sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"
],
- "markers": "python_full_version >= '3.7.0'",
+ "markers": "python_version >= '3.7'",
"version": "==3.3.2"
},
"coverage": {
@@ -1375,6 +1460,78 @@
"markers": "python_version >= '3.8'",
"version": "==7.0.0"
},
+ "exceptiongroup": {
+ "hashes": [
+ "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14",
+ "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"
+ ],
+ "index": "pypi",
+ "version": "==1.2.0"
+ },
+ "greenlet": {
+ "hashes": [
+ "sha256:01bc7ea167cf943b4c802068e178bbf70ae2e8c080467070d01bfa02f337ee67",
+ "sha256:0448abc479fab28b00cb472d278828b3ccca164531daab4e970a0458786055d6",
+ "sha256:086152f8fbc5955df88382e8a75984e2bb1c892ad2e3c80a2508954e52295257",
+ "sha256:098d86f528c855ead3479afe84b49242e174ed262456c342d70fc7f972bc13c4",
+ "sha256:149e94a2dd82d19838fe4b2259f1b6b9957d5ba1b25640d2380bea9c5df37676",
+ "sha256:1551a8195c0d4a68fac7a4325efac0d541b48def35feb49d803674ac32582f61",
+ "sha256:15d79dd26056573940fcb8c7413d84118086f2ec1a8acdfa854631084393efcc",
+ "sha256:1996cb9306c8595335bb157d133daf5cf9f693ef413e7673cb07e3e5871379ca",
+ "sha256:1a7191e42732df52cb5f39d3527217e7ab73cae2cb3694d241e18f53d84ea9a7",
+ "sha256:1ea188d4f49089fc6fb283845ab18a2518d279c7cd9da1065d7a84e991748728",
+ "sha256:1f672519db1796ca0d8753f9e78ec02355e862d0998193038c7073045899f305",
+ "sha256:2516a9957eed41dd8f1ec0c604f1cdc86758b587d964668b5b196a9db5bfcde6",
+ "sha256:2797aa5aedac23af156bbb5a6aa2cd3427ada2972c828244eb7d1b9255846379",
+ "sha256:2dd6e660effd852586b6a8478a1d244b8dc90ab5b1321751d2ea15deb49ed414",
+ "sha256:3ddc0f794e6ad661e321caa8d2f0a55ce01213c74722587256fb6566049a8b04",
+ "sha256:3ed7fb269f15dc662787f4119ec300ad0702fa1b19d2135a37c2c4de6fadfd4a",
+ "sha256:419b386f84949bf0e7c73e6032e3457b82a787c1ab4a0e43732898a761cc9dbf",
+ "sha256:43374442353259554ce33599da8b692d5aa96f8976d567d4badf263371fbe491",
+ "sha256:52f59dd9c96ad2fc0d5724107444f76eb20aaccb675bf825df6435acb7703559",
+ "sha256:57e8974f23e47dac22b83436bdcf23080ade568ce77df33159e019d161ce1d1e",
+ "sha256:5b51e85cb5ceda94e79d019ed36b35386e8c37d22f07d6a751cb659b180d5274",
+ "sha256:649dde7de1a5eceb258f9cb00bdf50e978c9db1b996964cd80703614c86495eb",
+ "sha256:64d7675ad83578e3fc149b617a444fab8efdafc9385471f868eb5ff83e446b8b",
+ "sha256:68834da854554926fbedd38c76e60c4a2e3198c6fbed520b106a8986445caaf9",
+ "sha256:6b66c9c1e7ccabad3a7d037b2bcb740122a7b17a53734b7d72a344ce39882a1b",
+ "sha256:70fb482fdf2c707765ab5f0b6655e9cfcf3780d8d87355a063547b41177599be",
+ "sha256:7170375bcc99f1a2fbd9c306f5be8764eaf3ac6b5cb968862cad4c7057756506",
+ "sha256:73a411ef564e0e097dbe7e866bb2dda0f027e072b04da387282b02c308807405",
+ "sha256:77457465d89b8263bca14759d7c1684df840b6811b2499838cc5b040a8b5b113",
+ "sha256:7f362975f2d179f9e26928c5b517524e89dd48530a0202570d55ad6ca5d8a56f",
+ "sha256:81bb9c6d52e8321f09c3d165b2a78c680506d9af285bfccbad9fb7ad5a5da3e5",
+ "sha256:881b7db1ebff4ba09aaaeae6aa491daeb226c8150fc20e836ad00041bcb11230",
+ "sha256:894393ce10ceac937e56ec00bb71c4c2f8209ad516e96033e4b3b1de270e200d",
+ "sha256:99bf650dc5d69546e076f413a87481ee1d2d09aaaaaca058c9251b6d8c14783f",
+ "sha256:9da2bd29ed9e4f15955dd1595ad7bc9320308a3b766ef7f837e23ad4b4aac31a",
+ "sha256:afaff6cf5200befd5cec055b07d1c0a5a06c040fe5ad148abcd11ba6ab9b114e",
+ "sha256:b1b5667cced97081bf57b8fa1d6bfca67814b0afd38208d52538316e9422fc61",
+ "sha256:b37eef18ea55f2ffd8f00ff8fe7c8d3818abd3e25fb73fae2ca3b672e333a7a6",
+ "sha256:b542be2440edc2d48547b5923c408cbe0fc94afb9f18741faa6ae970dbcb9b6d",
+ "sha256:b7dcbe92cc99f08c8dd11f930de4d99ef756c3591a5377d1d9cd7dd5e896da71",
+ "sha256:b7f009caad047246ed379e1c4dbcb8b020f0a390667ea74d2387be2998f58a22",
+ "sha256:bba5387a6975598857d86de9eac14210a49d554a77eb8261cc68b7d082f78ce2",
+ "sha256:c5e1536de2aad7bf62e27baf79225d0d64360d4168cf2e6becb91baf1ed074f3",
+ "sha256:c5ee858cfe08f34712f548c3c363e807e7186f03ad7a5039ebadb29e8c6be067",
+ "sha256:c9db1c18f0eaad2f804728c67d6c610778456e3e1cc4ab4bbd5eeb8e6053c6fc",
+ "sha256:d353cadd6083fdb056bb46ed07e4340b0869c305c8ca54ef9da3421acbdf6881",
+ "sha256:d46677c85c5ba00a9cb6f7a00b2bfa6f812192d2c9f7d9c4f6a55b60216712f3",
+ "sha256:d4d1ac74f5c0c0524e4a24335350edad7e5f03b9532da7ea4d3c54d527784f2e",
+ "sha256:d73a9fe764d77f87f8ec26a0c85144d6a951a6c438dfe50487df5595c6373eac",
+ "sha256:da70d4d51c8b306bb7a031d5cff6cc25ad253affe89b70352af5f1cb68e74b53",
+ "sha256:daf3cb43b7cf2ba96d614252ce1684c1bccee6b2183a01328c98d36fcd7d5cb0",
+ "sha256:dca1e2f3ca00b84a396bc1bce13dd21f680f035314d2379c4160c98153b2059b",
+ "sha256:dd4f49ae60e10adbc94b45c0b5e6a179acc1736cf7a90160b404076ee283cf83",
+ "sha256:e1f145462f1fa6e4a4ae3c0f782e580ce44d57c8f2c7aae1b6fa88c0b2efdb41",
+ "sha256:e3391d1e16e2a5a1507d83e4a8b100f4ee626e8eca43cf2cadb543de69827c4c",
+ "sha256:fcd2469d6a2cf298f198f0487e0a5b1a47a42ca0fa4dfd1b6862c999f018ebbf",
+ "sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da",
+ "sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33"
+ ],
+ "index": "pypi",
+ "version": "==3.0.3"
+ },
"idna": {
"hashes": [
"sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca",
@@ -1399,6 +1556,14 @@
"index": "pypi",
"version": "==7.2.3"
},
+ "opensearch-py": {
+ "hashes": [
+ "sha256:564f175af134aa885f4ced6846eb4532e08b414fff0a7976f76b276fe0e69158",
+ "sha256:7867319132133e2974c09f76a54eb1d502b989229be52da583d93ddc743ea111"
+ ],
+ "index": "pypi",
+ "version": "==2.4.2"
+ },
"packaging": {
"hashes": [
"sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5",
@@ -1424,41 +1589,49 @@
},
"pycryptodome": {
"hashes": [
- "sha256:08b445799d571041765e7d5c9ca09c5d3866c2f22eeb0dd4394a4169285184f4",
- "sha256:11ddf6c9b52116b62223b6a9f4741bc4f62bb265392a4463282f7f34bb287180",
- "sha256:1c6273ca5a03b672e504995529b8bae56da0ebb691d8ef141c4aa68f60765700",
- "sha256:27e1efcb68993b7ce5d1d047a46a601d41281bba9f1971e6be4aa27c69ab8065",
- "sha256:2c16426ef49d9cba018be2340ea986837e1dfa25c2ea181787971654dd49aadd",
- "sha256:37e531bf896b70fe302f003d3be5a0a8697737a8d177967da7e23eff60d6483c",
- "sha256:420972f9c62978e852c74055d81c354079ce3c3a2213a92c9d7e37bbc63a26e2",
- "sha256:4464b0e8fd5508bff9baf18e6fd4c6548b1ac2ce9862d6965ff6a84ec9cb302a",
- "sha256:4805e053571140cb37cf153b5c72cd324bb1e3e837cbe590a19f69b6cf85fd03",
- "sha256:67939a3adbe637281c611596e44500ff309d547e932c449337649921b17b6297",
- "sha256:694020d2ff985cd714381b9da949a21028c24b86f562526186f6af7c7547e986",
- "sha256:6c3df3613592ea6afaec900fd7189d23c8c28b75b550254f4bd33fe94acb84b9",
- "sha256:6d0d2b97758ebf2f36c39060520447c26455acb3bcff309c28b1c816173a6ff5",
- "sha256:7c9e222d0976f68d0cf6409cfea896676ddc1d98485d601e9508f90f60e2b0a2",
- "sha256:81e9d23c0316fc1b45d984a44881b220062336bbdc340aa9218e8d0656587934",
- "sha256:8ae0dd1bcfada451c35f9e29a3e5db385caabc190f98e4a80ad02a61098fb776",
- "sha256:8cf5d3d6cf921fa81acd1f632f6cedcc03f5f68fc50c364cd39490ba01d17c49",
- "sha256:954d156cd50130afd53f8d77f830fe6d5801bd23e97a69d358fed068f433fbfe",
- "sha256:a470237ee71a1efd63f9becebc0ad84b88ec28e6784a2047684b693f458f41b7",
- "sha256:a991f8ffe8dfe708f86690948ae46442eebdd0fff07dc1b605987939a34ec979",
- "sha256:b0bfe61506795877ff974f994397f0c862d037f6f1c0bfc3572195fc00833b96",
- "sha256:b7efd46b0b4ac869046e814d83244aeab14ef787f4850644119b1c8b0ec2d637",
- "sha256:b8b80ff92049fd042177282917d994d344365ab7e8ec2bc03e853d93d2401786",
- "sha256:c1bc0c49d986a1491d66d2a56570f12e960b12508b7e71f2423f532e28857f36",
- "sha256:c22c80246c3c880c6950d2a8addf156cee74ec0dc5757d01e8e7067a3c7da015",
- "sha256:cd4e7e8bf0fc1ada854688b9b309ee607e2aa85a8b44180f91021a4dd330a928",
- "sha256:cd4e95b0eb4b28251c825fe7aa941fe077f993e5ca9b855665935b86fbb1cc08",
- "sha256:e038ab77fec0956d7aa989a3c647652937fc142ef41c9382c2ebd13c127d5b4a",
- "sha256:e3e6f89480616781d2a7f981472d0cdb09b9da9e8196f43c1234eff45c915766",
- "sha256:e70f5c839c7798743a948efa2a65d1fe96bb397fe6d7f2bde93d869fe4f0ad69",
- "sha256:ed932eb6c2b1c4391e166e1a562c9d2f020bfff44a0e1b108f67af38b390ea89",
- "sha256:f34976c5c8eb79e14c7d970fb097482835be8d410a4220f86260695ede4c3e17"
+ "sha256:06d6de87c19f967f03b4cf9b34e538ef46e99a337e9a61a77dbe44b2cbcf0690",
+ "sha256:09609209ed7de61c2b560cc5c8c4fbf892f8b15b1faf7e4cbffac97db1fffda7",
+ "sha256:210ba1b647837bfc42dd5a813cdecb5b86193ae11a3f5d972b9a0ae2c7e9e4b4",
+ "sha256:2a1250b7ea809f752b68e3e6f3fd946b5939a52eaeea18c73bdab53e9ba3c2dd",
+ "sha256:2ab6ab0cb755154ad14e507d1df72de9897e99fd2d4922851a276ccc14f4f1a5",
+ "sha256:3427d9e5310af6680678f4cce149f54e0bb4af60101c7f2c16fdf878b39ccccc",
+ "sha256:3cd3ef3aee1079ae44afaeee13393cf68b1058f70576b11439483e34f93cf818",
+ "sha256:405002eafad114a2f9a930f5db65feef7b53c4784495dd8758069b89baf68eab",
+ "sha256:417a276aaa9cb3be91f9014e9d18d10e840a7a9b9a9be64a42f553c5b50b4d1d",
+ "sha256:4401564ebf37dfde45d096974c7a159b52eeabd9969135f0426907db367a652a",
+ "sha256:49a4c4dc60b78ec41d2afa392491d788c2e06edf48580fbfb0dd0f828af49d25",
+ "sha256:5601c934c498cd267640b57569e73793cb9a83506f7c73a8ec57a516f5b0b091",
+ "sha256:6e0e4a987d38cfc2e71b4a1b591bae4891eeabe5fa0f56154f576e26287bfdea",
+ "sha256:76658f0d942051d12a9bd08ca1b6b34fd762a8ee4240984f7c06ddfb55eaf15a",
+ "sha256:76cb39afede7055127e35a444c1c041d2e8d2f1f9c121ecef573757ba4cd2c3c",
+ "sha256:8d6b98d0d83d21fb757a182d52940d028564efe8147baa9ce0f38d057104ae72",
+ "sha256:9b3ae153c89a480a0ec402e23db8d8d84a3833b65fa4b15b81b83be9d637aab9",
+ "sha256:a60fedd2b37b4cb11ccb5d0399efe26db9e0dd149016c1cc6c8161974ceac2d6",
+ "sha256:ac1c7c0624a862f2e53438a15c9259d1655325fc2ec4392e66dc46cdae24d044",
+ "sha256:acae12b9ede49f38eb0ef76fdec2df2e94aad85ae46ec85be3648a57f0a7db04",
+ "sha256:acc2614e2e5346a4a4eab6e199203034924313626f9620b7b4b38e9ad74b7e0c",
+ "sha256:acf6e43fa75aca2d33e93409f2dafe386fe051818ee79ee8a3e21de9caa2ac9e",
+ "sha256:baee115a9ba6c5d2709a1e88ffe62b73ecc044852a925dcb67713a288c4ec70f",
+ "sha256:c18b381553638414b38705f07d1ef0a7cf301bc78a5f9bc17a957eb19446834b",
+ "sha256:d29daa681517f4bc318cd8a23af87e1f2a7bad2fe361e8aa29c77d652a065de4",
+ "sha256:d5954acfe9e00bc83ed9f5cb082ed22c592fbbef86dc48b907238be64ead5c33",
+ "sha256:ec0bb1188c1d13426039af8ffcb4dbe3aad1d7680c35a62d8eaf2a529b5d3d4f",
+ "sha256:ec1f93feb3bb93380ab0ebf8b859e8e5678c0f010d2d78367cf6bc30bfeb148e",
+ "sha256:f0e6d631bae3f231d3634f91ae4da7a960f7ff87f2865b2d2b831af1dfb04e9a",
+ "sha256:f35d6cee81fa145333137009d9c8ba90951d7d77b67c79cbe5f03c7eb74d8fe2",
+ "sha256:f47888542a0633baff535a04726948e876bf1ed880fddb7c10a736fa99146ab3",
+ "sha256:fb3b87461fa35afa19c971b0a2b7456a7b1db7b4eba9a8424666104925b78128"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
- "version": "==3.19.1"
+ "version": "==3.20.0"
+ },
+ "pymysql": {
+ "hashes": [
+ "sha256:4f13a7df8bf36a51e81dd9f3605fede45a4878fe02f9236349fd82a3f0612f96",
+ "sha256:8969ec6d763c856f7073c4c64662882675702efcb114b4bcbb955aea3a069fa7"
+ ],
+ "index": "pypi",
+ "version": "==1.1.0"
},
"pytest": {
"hashes": [
@@ -1468,6 +1641,14 @@
"index": "pypi",
"version": "==7.4.4"
},
+ "python-dateutil": {
+ "hashes": [
+ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
+ "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+ "version": "==2.8.2"
+ },
"requests": {
"hashes": [
"sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
@@ -1476,6 +1657,69 @@
"markers": "python_version >= '3.7'",
"version": "==2.31.0"
},
+ "six": {
+ "hashes": [
+ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
+ "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+ "version": "==1.16.0"
+ },
+ "sqlalchemy": {
+ "hashes": [
+ "sha256:0d3cab3076af2e4aa5693f89622bef7fa770c6fec967143e4da7508b3dceb9b9",
+ "sha256:0dacf67aee53b16f365c589ce72e766efaabd2b145f9de7c917777b575e3659d",
+ "sha256:10331f129982a19df4284ceac6fe87353ca3ca6b4ca77ff7d697209ae0a5915e",
+ "sha256:14a6f68e8fc96e5e8f5647ef6cda6250c780612a573d99e4d881581432ef1669",
+ "sha256:1b1180cda6df7af84fe72e4530f192231b1f29a7496951db4ff38dac1687202d",
+ "sha256:29049e2c299b5ace92cbed0c1610a7a236f3baf4c6b66eb9547c01179f638ec5",
+ "sha256:342d365988ba88ada8af320d43df4e0b13a694dbd75951f537b2d5e4cb5cd002",
+ "sha256:420362338681eec03f53467804541a854617faed7272fe71a1bfdb07336a381e",
+ "sha256:4344d059265cc8b1b1be351bfb88749294b87a8b2bbe21dfbe066c4199541ebd",
+ "sha256:4f7a7d7fcc675d3d85fbf3b3828ecd5990b8d61bd6de3f1b260080b3beccf215",
+ "sha256:555651adbb503ac7f4cb35834c5e4ae0819aab2cd24857a123370764dc7d7e24",
+ "sha256:59a21853f5daeb50412d459cfb13cb82c089ad4c04ec208cd14dddd99fc23b39",
+ "sha256:5fdd402169aa00df3142149940b3bf9ce7dde075928c1886d9a1df63d4b8de62",
+ "sha256:605b6b059f4b57b277f75ace81cc5bc6335efcbcc4ccb9066695e515dbdb3900",
+ "sha256:665f0a3954635b5b777a55111ababf44b4fc12b1f3ba0a435b602b6387ffd7cf",
+ "sha256:6f9e2e59cbcc6ba1488404aad43de005d05ca56e069477b33ff74e91b6319735",
+ "sha256:736ea78cd06de6c21ecba7416499e7236a22374561493b456a1f7ffbe3f6cdb4",
+ "sha256:74b080c897563f81062b74e44f5a72fa44c2b373741a9ade701d5f789a10ba23",
+ "sha256:75432b5b14dc2fff43c50435e248b45c7cdadef73388e5610852b95280ffd0e9",
+ "sha256:75f99202324383d613ddd1f7455ac908dca9c2dd729ec8584c9541dd41822a2c",
+ "sha256:790f533fa5c8901a62b6fef5811d48980adeb2f51f1290ade8b5e7ba990ba3de",
+ "sha256:798f717ae7c806d67145f6ae94dc7c342d3222d3b9a311a784f371a4333212c7",
+ "sha256:7c88f0c7dcc5f99bdb34b4fd9b69b93c89f893f454f40219fe923a3a2fd11625",
+ "sha256:7d505815ac340568fd03f719446a589162d55c52f08abd77ba8964fbb7eb5b5f",
+ "sha256:84daa0a2055df9ca0f148a64fdde12ac635e30edbca80e87df9b3aaf419e144a",
+ "sha256:87d91043ea0dc65ee583026cb18e1b458d8ec5fc0a93637126b5fc0bc3ea68c4",
+ "sha256:87f6e732bccd7dcf1741c00f1ecf33797383128bd1c90144ac8adc02cbb98643",
+ "sha256:884272dcd3ad97f47702965a0e902b540541890f468d24bd1d98bcfe41c3f018",
+ "sha256:8b8cb63d3ea63b29074dcd29da4dc6a97ad1349151f2d2949495418fd6e48db9",
+ "sha256:91f7d9d1c4dd1f4f6e092874c128c11165eafcf7c963128f79e28f8445de82d5",
+ "sha256:a2c69a7664fb2d54b8682dd774c3b54f67f84fa123cf84dda2a5f40dcaa04e08",
+ "sha256:a3be4987e3ee9d9a380b66393b77a4cd6d742480c951a1c56a23c335caca4ce3",
+ "sha256:a86b4240e67d4753dc3092d9511886795b3c2852abe599cffe108952f7af7ac3",
+ "sha256:aa9373708763ef46782d10e950b49d0235bfe58facebd76917d3f5cbf5971aed",
+ "sha256:b64b183d610b424a160b0d4d880995e935208fc043d0302dd29fee32d1ee3f95",
+ "sha256:b801154027107461ee992ff4b5c09aa7cc6ec91ddfe50d02bca344918c3265c6",
+ "sha256:bb209a73b8307f8fe4fe46f6ad5979649be01607f11af1eb94aa9e8a3aaf77f0",
+ "sha256:bc8b7dabe8e67c4832891a5d322cec6d44ef02f432b4588390017f5cec186a84",
+ "sha256:c51db269513917394faec5e5c00d6f83829742ba62e2ac4fa5c98d58be91662f",
+ "sha256:c55731c116806836a5d678a70c84cb13f2cedba920212ba7dcad53260997666d",
+ "sha256:cf18ff7fc9941b8fc23437cc3e68ed4ebeff3599eec6ef5eebf305f3d2e9a7c2",
+ "sha256:d24f571990c05f6b36a396218f251f3e0dda916e0c687ef6fdca5072743208f5",
+ "sha256:db854730a25db7c956423bb9fb4bdd1216c839a689bf9cc15fada0a7fb2f4570",
+ "sha256:dc55990143cbd853a5d038c05e79284baedf3e299661389654551bd02a6a68d7",
+ "sha256:e607cdd99cbf9bb80391f54446b86e16eea6ad309361942bf88318bcd452363c",
+ "sha256:ecf6d4cda1f9f6cb0b45803a01ea7f034e2f1aed9475e883410812d9f9e3cfcf",
+ "sha256:f2a159111a0f58fb034c93eeba211b4141137ec4b0a6e75789ab7a3ef3c7e7e3",
+ "sha256:f37c0caf14b9e9b9e8f6dbc81bc56db06acb4363eba5a633167781a48ef036ed",
+ "sha256:f5693145220517b5f42393e07a6898acdfe820e136c98663b971906120549da5"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==2.0.25"
+ },
"testcontainers-core": {
"hashes": [
"sha256:69a8bf2ddb52ac2d03c26401b12c70db0453cced40372ad783d6dce417e52095"
@@ -1490,6 +1734,28 @@
"index": "pypi",
"version": "==0.0.1rc1"
},
+ "testcontainers-mysql": {
+ "hashes": [
+ "sha256:d22894e0d8c7b4f7424afef99f713aa7e7a19ff987b7723aed863b9c478a2c91"
+ ],
+ "index": "pypi",
+ "version": "==0.0.1rc1"
+ },
+ "testcontainers-opensearch": {
+ "hashes": [
+ "sha256:0bdf270b5b7f53915832f7c31dd2bd3ffdc20b534ea6b32231cc7003049bd0e1"
+ ],
+ "index": "pypi",
+ "version": "==0.0.1rc1"
+ },
+ "tomli": {
+ "hashes": [
+ "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc",
+ "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"
+ ],
+ "markers": "python_version < '3.11'",
+ "version": "==2.0.1"
+ },
"typing-extensions": {
"hashes": [
"sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783",
@@ -1500,11 +1766,11 @@
},
"urllib3": {
"hashes": [
- "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84",
- "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e"
+ "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07",
+ "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"
],
- "markers": "python_version >= '3.7'",
- "version": "==2.0.7"
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
+ "version": "==1.26.18"
},
"wrapt": {
"hashes": [
diff --git a/dbrepo-analyse-service/app.py b/dbrepo-analyse-service/app.py
index 6cd01304782469a3725db0d613b74b52338b3d5a..d7d206d23d5687e7a8e2856c97f0436f7e74bf33 100644
--- a/dbrepo-analyse-service/app.py
+++ b/dbrepo-analyse-service/app.py
@@ -1,57 +1,78 @@
-import os
+import logging
from _csv import Error
-from flask import Flask, request, Response
from json import dumps
-from determine_dt import determine_datatypes
-from determine_pk import determine_pk
-import logging
-from flasgger import Swagger
+from logging.config import dictConfig
+
+from flasgger import LazyJSONEncoder, Swagger
from flasgger.utils import swag_from
-from flasgger import LazyJSONEncoder
+from flask import Flask, Response, abort, request
+from flask_sqlalchemy import SQLAlchemy
from gevent.pywsgi import WSGIServer
+from opensearchpy import OpenSearch
from prometheus_flask_exporter import PrometheusMetrics
+from requests import get
-log_level = os.getenv('LOG_LEVEL', 'INFO')
-
-logging.basicConfig(level=logging.getLevelName(log_level))
+from determine_dt import determine_datatypes
+from determine_pk import determine_pk
+from determine_stats import determine_stats
-from logging.config import dictConfig
+logging.basicConfig(level=logging.DEBUG)
-dictConfig({
- 'version': 1,
- 'formatters': {'default': {
- 'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
- }},
- 'handlers': {'wsgi': {
- 'class': 'logging.StreamHandler',
- 'stream': 'ext://flask.logging.wsgi_errors_stream',
- 'formatter': 'default'
- }},
- 'root': {
- 'level': log_level,
- 'handlers': ['wsgi']
+dictConfig(
+ {
+ "version": 1,
+ "formatters": {
+ "default": {
+ "format": "[%(asctime)s] %(levelname)s in %(module)s: %(message)s",
+ }
+ },
+ "handlers": {
+ "wsgi": {
+ "class": "logging.StreamHandler",
+ "stream": "ext://flask.logging.wsgi_errors_stream",
+ "formatter": "default",
+ }
+ },
+ "root": {"level": "INFO", "handlers": ["wsgi"]},
}
-})
+)
app = Flask(__name__)
+
metrics = PrometheusMetrics(app)
-metrics.info('app_info', 'Application info', version='1.3.0')
+metrics.info("app_info", "Application info", version="1.3.0")
app.config["SWAGGER"] = {"openapi": "3.0.1", "title": "Swagger UI", "uiversion": 3}
+# ========================= DB Config ========================= #
+app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
+app.config[
+ "SQLALCHEMY_DATABASE_URI"
+] = "mysql+pymysql://root:dbrepo@metadata-db:3306/fda"
+db = SQLAlchemy(app)
+
+# ========================= OS Config ========================= #
+opensearch_client = OpenSearch(
+ hosts=["search-db"],
+ port=9200,
+ http_auth=("admin", "admin"),
+ use_ssl=False,
+)
+
+
swagger_config = {
"headers": [],
"specs": [
{
"endpoint": "api-analyse",
"route": "/api-analyse.json",
- "rule_filter": lambda rule: rule.endpoint.startswith('analyze'),
+ "rule_filter": lambda rule: rule.endpoint.startswith("analyze"),
"model_filter": lambda tag: True, # all in
},
{
"endpoint": "api-mdb",
"route": "/api-mdb.json",
- "rule_filter": lambda rule: rule.endpoint.startswith('mdb'),
+ "rule_filter": lambda rule: rule.endpoint.startswith("mdb"),
"model_filter": lambda tag: True, # all in
},
],
@@ -68,97 +89,129 @@ template = {
"version": "$TAG",
"contact": {
"name": "Prof. Andreas Rauber",
- "email": "andreas.rauber@tuwien.ac.at"
+ "email": "andreas.rauber@tuwien.ac.at",
},
"license": {
"name": "Apache 2.0",
- "url": "https://www.apache.org/licenses/LICENSE-2.0"
+ "url": "https://www.apache.org/licenses/LICENSE-2.0",
},
},
"externalDocs": {
"description": "Sourcecode Documentation",
- "url": "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services"
+ "url": "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services",
},
"servers": [
- {
- "url": "http://localhost:5000",
- "description": "Generated server url"
- },
- {
- "url": "https://dbrepo2.tuwien.ac.at",
- "description": "Sandbox"
- }
- ]
+ {"url": "http://localhost:5000", "description": "Generated server url"},
+ {"url": "https://dbrepo2.tuwien.ac.at", "description": "Sandbox"},
+ ],
}
app.json_encoder = LazyJSONEncoder
swagger = Swagger(app, config=swagger_config, template=template)
-@app.route('/health', methods=["GET"], endpoint='analyze_health')
-@swag_from('as-yml/health.yml')
+
+@app.route("/health", methods=["GET"], endpoint="analyze_health")
+@swag_from("as-yml/health.yml")
def health():
- logging.debug('endpoint health, body=%s', request)
+ logging.debug("endpoint health, body=%s", request)
res = dumps({"status": "UP", "message": "Application is up and running"})
return Response(res, mimetype="application/json"), 200
-@app.route('/api/analyse/determinedt', methods=["POST"], endpoint='analyze_determinedt')
-@swag_from('as-yml/determinedt.yml')
+@app.route("/api/analyse/determinedt", methods=["POST"], endpoint="analyze_determinedt")
+@swag_from("as-yml/determinedt.yml")
def determinedt():
- logging.debug('endpoint determine datatype, body=%s', request)
+ logging.debug("endpoint determine datatype, body=%s", request)
input_json = request.get_json()
try:
- filename = str(input_json['filename'])
+ filename = str(input_json["filename"])
enum = False
- if 'enum' in input_json:
- enum = bool(input_json['enum'])
+ if "enum" in input_json:
+ enum = bool(input_json["enum"])
logging.info("Enum is present in payload and set to %s", enum)
enum_tol = 0.001
- if 'enum_tol' in input_json:
- enum_tol = float(input_json['enum_tol'])
- logging.info("Enum toleration is present in payload and set to %s", enum_tol)
+ if "enum_tol" in input_json:
+ enum_tol = float(input_json["enum_tol"])
+ logging.info(
+ "Enum toleration is present in payload and set to %s", enum_tol
+ )
separator = None
- if 'separator' in input_json:
- separator = str(input_json['separator'])
+ if "separator" in input_json:
+ separator = str(input_json["separator"])
logging.info("Seperator is present in payload and set to %s", separator)
res = determine_datatypes(filename, enum, enum_tol, separator)
- logging.debug('determine datatype resulted in datatypes %s', res)
+ logging.debug("determine datatype resulted in datatypes %s", res)
return Response(res, mimetype="application/json"), 200
except OSError as e:
- logging.error('Failed to determine data types: %s', e)
+ logging.error("Failed to determine data types: %s", e)
res = dumps({"success": False, "message": str(e)})
return Response(res, mimetype="application/json"), 409
except Error as e:
- logging.error('Failed to determine separator %s', e)
+ logging.error("Failed to determine separator %s", e)
res = dumps({"success": False, "message": str(e)})
return Response(res, mimetype="application/json"), 422
except Exception as e:
- logging.error('Failed to determine data types: %s', e)
+ logging.error("Failed to determine data types: %s", e)
res = dumps({"success": False, "message": str(e)})
return Response(res, mimetype="application/json"), 500
-@app.route('/api/analyse/determinepk', methods=["POST"], endpoint='analyze_determinepk')
-@swag_from('as-yml/determinepk.yml')
+@app.route("/api/analyse/determinepk", methods=["POST"], endpoint="analyze_determinepk")
+@swag_from("as-yml/determinepk.yml")
def determinepk():
- logging.debug('endpoint determine primary key, body=%s', request)
+ logging.debug("endpoint determine primary key, body=%s", request)
input_json = request.get_json()
try:
- filepath = str(input_json['filepath'])
- seperator = ','
- if 'seperator' in input_json:
- seperator = str(input_json['seperator'])
+ filepath = str(input_json["filepath"])
+ seperator = ","
+ if "seperator" in input_json:
+ seperator = str(input_json["seperator"])
res = determine_pk(filepath, seperator)
- logging.debug('determined list of primary keys: %s', res)
+ logging.debug("determined list of primary keys: %s", res)
return Response(res, mimetype="application/json"), 200
except Exception as e:
- logging.error('Failed to determine primary key: %s', e)
+ logging.error("Failed to determine primary key: %s", e)
res = dumps({"success": False, "message": str(e)})
return Response(res, mimetype="application/json"), 500
+@app.route("/api/analyse/determineStats", methods=["POST"], endpoint="analyze_stats")
+def determineStats():
+ logging.debug(
+ "endpoint to determine the statistical properties, body = %s", request
+ )
+ input_json = request.get_json()
+ if "filepath" not in input_json:
+ return {"message": "Missing 'filepath'", "status": 400}, 400
+
+ filepath = str(input_json["filepath"])
+ separator = str(input_json.get("separator", ","))
+ return determine_stats(filepath, separator)
+
+
+@app.route("/api/analyse/determineStat", methods=["POST"])
+def determineStat():
+ input_json = request.get_json()
+
+ if "database_id" not in input_json:
+ return {"message": "Missing 'database_id'", "status": 400}, 400
+ if "table_id" not in input_json:
+ return {"message": "Missing 'table_id'", "status": 400}, 400
+
+ res = determine_stats(
+ db,
+ opensearch_client,
+ database_id=input_json["database_id"],
+ table_id=input_json["table_id"],
+ )
+ if res:
+ return {"message": "Analysed statistical properties.", "status": 200}
+ else:
+ return {"message": "Database or table does not exist.", "status": 400}, 400
+
+
rest_server_port = 5000
-if __name__ == '__main__':
- http_server = WSGIServer(('', 5000), app)
+if __name__ == "__main__":
+ http_server = WSGIServer(("", 5000), app)
http_server.serve_forever()
diff --git a/dbrepo-analyse-service/build.sh b/dbrepo-analyse-service/build.sh
index 4a0889a662ba1e9d3b704a86c85555c867dfecb9..8460f7b7f5d906a14a2f127ed1c86c3a5637a6de 100755
--- a/dbrepo-analyse-service/build.sh
+++ b/dbrepo-analyse-service/build.sh
@@ -1,4 +1,8 @@
#!/bin/bash
+
+# needed for MariaDB Connector/C
+apt update && apt install -y curl gcc libmariadb3 libmariadb-dev
+
python3 -m venv ./dbrepo-analyse-service/venv
source ./dbrepo-analyse-service/venv/bin/activate
PIPENV_PIPFILE=./dbrepo-analyse-service/Pipfile pipenv install --dev
\ No newline at end of file
diff --git a/dbrepo-analyse-service/config.py b/dbrepo-analyse-service/config.py
index 5eb6e77b5dd5046130938348190b1d778e06d2cd..136153e23f736e0b3b21b523de4db274e3d63833 100644
--- a/dbrepo-analyse-service/config.py
+++ b/dbrepo-analyse-service/config.py
@@ -1,2 +1,2 @@
class Config:
- EUREKA_SERVER = os.environ.get('EUREKA_SERVER')
+ EUREKA_SERVER = os.environ.get("EUREKA_SERVER")
diff --git a/dbrepo-analyse-service/data/test_stats/test_stats_01.csv b/dbrepo-analyse-service/data/test_stats/test_stats_01.csv
new file mode 100644
index 0000000000000000000000000000000000000000..558d3c7e6d0cfe5f794f0bf76d842ebb948b9b66
--- /dev/null
+++ b/dbrepo-analyse-service/data/test_stats/test_stats_01.csv
@@ -0,0 +1,10 @@
+"col1","col2","col3","col4","col5"
+"2021-02-01 00:00:00","Zch_Stampfenbachstrasse",0.44,"150.25","10"
+"2021-04-13 00:00:00","Zch_Stampfenbachstrasse",4.88,"201.5","11"
+"2021-04-01 00:00:00","Zch_Stampfenbachstrasse",29.46,"180.75","12"
+"2021-01-01 00:00:00","Zch_Stampfenbachstrasse",9.85,"170.15","13"
+"2016-06-01 00:00:00","Zch_Stampfenbachstrasse",41.24,"160.35","14"
+"2015-01-01 00:00:00","Zch_Stampfenbachstrasse",8.51,"140.14","15"
+"2022-07-01 00:00:00","Zch_Stampfenbachstrasse",88.34,"130.45","16"
+"2022-01-01 00:00:00","Zch_Stampfenbachstrasse",75.72,"120.55","17"
+"2023-09-01 00:00:00","Zch_Schimmelstrasse",41.66,"110.295","18"
diff --git a/dbrepo-analyse-service/determine_dt.py b/dbrepo-analyse-service/determine_dt.py
index 76d8c850032a71b5c199081584dca4d440fa15dc..75d8c2290f8095b7ce1cd3368e3e003b16e15f20 100644
--- a/dbrepo-analyse-service/determine_dt.py
+++ b/dbrepo-analyse-service/determine_dt.py
@@ -9,8 +9,13 @@ import io
from clients.s3_client import S3Client
import messytables, pandas as pd
-from messytables import CSVTableSet, type_guess, \
- headers_guess, headers_processor, offset_processor
+from messytables import (
+ CSVTableSet,
+ type_guess,
+ headers_guess,
+ headers_processor,
+ offset_processor,
+)
def determine_datatypes(filename, enum=False, enum_tol=0.0001, separator=None) -> {}:
@@ -25,16 +30,16 @@ def determine_datatypes(filename, enum=False, enum_tol=0.0001, separator=None) -
logging.warning(f'Failed to determine data types: file {filename} has empty body')
return json.dumps({'columns': [], 'separator': ','})
if separator is None:
- logging.info('Attempt to guess separator for from first line')
+ logging.info("Attempt to guess separator for from first line")
with io.BytesIO(stream.read()) as fh:
line = next(fh)
- dialect = csv.Sniffer().sniff(line.decode('utf-8'))
+ dialect = csv.Sniffer().sniff(line.decode("utf-8"))
separator = dialect.delimiter
- logging.info('determined separator: %s', separator)
+ logging.info("determined separator: %s", separator)
# Load a file object:
with io.BytesIO(stream.read()) as fh:
- logging.info('Analysing corpus with separator: %s', separator)
+ logging.info("Analysing corpus with separator: %s", separator)
table_set = CSVTableSet(fh, delimiter=separator)
# A table set is a collection of tables:
@@ -58,18 +63,27 @@ def determine_datatypes(filename, enum=False, enum_tol=0.0001, separator=None) -
elif type(types[i]) == messytables.types.IntegerType:
r[headers[i]] = "bigint"
elif type(types[i]) == messytables.types.DateType:
- if "%H" in types[i].format or "%M" in types[i].format or "%S" in types[i].format or "%Z" in types[
- i].format:
- r[headers[i]] = "timestamp" # todo: guesses date format too, return it
+ if (
+ "%H" in types[i].format
+ or "%M" in types[i].format
+ or "%S" in types[i].format
+ or "%Z" in types[i].format
+ ):
+ r[
+ headers[i]
+ ] = "timestamp" # todo: guesses date format too, return it
else:
r[headers[i]] = "date"
- elif type(types[i]) == messytables.types.DecimalType or type(types[i]) == messytables.types.FloatType:
+ elif (
+ type(types[i]) == messytables.types.DecimalType
+ or type(types[i]) == messytables.types.FloatType
+ ):
r[headers[i]] = "decimal"
elif type(types[i]) == messytables.types.StringType:
r[headers[i]] = "varchar"
else:
r[headers[i]] = "text"
fh.close()
- s = {'columns': r, 'separator': separator}
- logging.info('Determined data types %s', s)
+ s = {"columns": r, "separator": separator}
+ logging.info("Determined data types %s", s)
return json.dumps(s)
diff --git a/dbrepo-analyse-service/determine_pk.py b/dbrepo-analyse-service/determine_pk.py
index cc182052891d8bae34c5e8a57217f0843c5dfa1f..6187e0e913fe3338e22c32a7dfd9670c3c1bb16d 100644
--- a/dbrepo-analyse-service/determine_pk.py
+++ b/dbrepo-analyse-service/determine_pk.py
@@ -8,9 +8,9 @@ from determine_dt import determine_datatypes
from clients.s3_client import S3Client
-def determine_pk(filename, separator=','):
+def determine_pk(filename, separator=","):
dt = json.loads(determine_datatypes(filename=filename, separator=separator))
- dt = {k.lower(): v for k, v in dt['columns'].items()}
+ dt = {k.lower(): v for k, v in dt["columns"].items()}
# {k.lower(): v for k, v in dt['columns'].items() if v != 'Numeric'}
colnames = dt.keys()
colindex = list(range(0, len(colnames)))
@@ -27,35 +27,49 @@ def determine_pk(filename, separator=','):
pk = {}
j = 0
k = 0
- logging.info(f'File is {sizeInKb}kb, detection is precise')
+ logging.info(f"File is {sizeInKb}kb, detection is precise")
for item in colnames:
- if item == 'id':
+ if item == "id":
j = j + 1
pk.update({item: j})
colindex.remove(k)
k = k + 1
csvdata = pd.read_csv(stream, sep=separator)
for i in colindex:
- if pd.Series(csvdata.iloc[:, i]).is_unique and pd.Series(csvdata.iloc[:, i]).notnull().values.any():
+ if (
+ pd.Series(csvdata.iloc[:, i]).is_unique
+ and pd.Series(csvdata.iloc[:, i]).notnull().values.any()
+ ):
j = j + 1
pk.update({list(colnames)[i]: j})
else: # stochastic pk determination
pk = {}
j = 0
k = 0
- logging.info(f'File is {sizeInKb}kB (larger than threshold of 400kB), detection is stochastic')
+ logging.info(
+ f"File is {sizeInKb}kB (larger than threshold of 400kB), detection is stochastic"
+ )
for item in colnames:
- if item == 'id':
+ if item == "id":
j = j + 1
pk.update({item: j})
colindex.remove(k)
k = k + 1
- p = np.log10(int(response['ContentLength'])) # logarithmic scaled percentage of random inspected rows
- csvdata = pd.read_csv(filepath_or_buffer=stream, sep=separator, header=0,
- skiprows=lambda k: k > 0 and random.random() > p)
+ p = np.log10(
+ int(response["ContentLength"])
+ ) # logarithmic scaled percentage of random inspected rows
+ csvdata = pd.read_csv(
+ filepath_or_buffer=stream,
+ sep=separator,
+ header=0,
+ skiprows=lambda k: k > 0 and random.random() > p,
+ )
for i in colindex:
- if pd.Series(csvdata.iloc[:, i]).is_unique and pd.Series(csvdata.iloc[:, i]).notnull().values.any():
+ if (
+ pd.Series(csvdata.iloc[:, i]).is_unique
+ and pd.Series(csvdata.iloc[:, i]).notnull().values.any()
+ ):
j = j + 1
pk.update({list(colnames)[i]: j})
- logging.info(f'Determined primary key {pk}')
+ logging.info(f"Determined primary key {pk}")
return json.dumps(pk)
diff --git a/dbrepo-analyse-service/determine_stats.py b/dbrepo-analyse-service/determine_stats.py
new file mode 100644
index 0000000000000000000000000000000000000000..50642ca52b3f3459df47b9a5f18ad473e4bc25fe
--- /dev/null
+++ b/dbrepo-analyse-service/determine_stats.py
@@ -0,0 +1,96 @@
+from pandas import DataFrame
+from sqlalchemy import create_engine, text
+
+
+def determine_stats(db, os, **kwargs):
+ database_id = kwargs.get("database_id")
+ table_id = kwargs.get("table_id")
+
+ with db.engine.connect() as connection:
+ database_name = connection.execute(
+ text(f"SELECT internal_name FROM mdb_databases WHERE id={database_id}")
+ ).fetchone()[0]
+ table_name = connection.execute(
+ text(f"SELECT internal_name FROM mdb_tables WHERE id={table_id}")
+ ).fetchone()[0]
+
+ if not database_name or not table_name:
+ return False
+
+ data_db_host = kwargs.get("data_db_host", "data-db")
+ data_db_port = kwargs.get("data_db_port", 3306)
+ # Generate data db connection on the fly: database name is varying according to the id given
+ data_db_uri = (
+ f"mysql+pymysql://root:dbrepo@{data_db_host}:{data_db_port}/{database_name}"
+ )
+ data_db_engine = create_engine(data_db_uri)
+
+ with data_db_engine.connect() as connection:
+ result = connection.execute(text(f"SELECT * FROM {table_name}"))
+ rows = result.fetchall()
+
+ df = DataFrame(rows, columns=result.keys())
+ for column, dtype in df.dtypes.items():
+ # Check if the column has a numeric data type
+ if dtype.kind in "fi":
+ # Calculate the statistics for the current column
+ column_stats = {
+ "val_min": df[column].min(),
+ "val_max": df[column].max(),
+ "mean": df[column].mean(),
+ "median": df[column].median(),
+ "std_dev": df[column].std(),
+ }
+
+ # Store statistical properties to the metadata db and index to OS
+ # TODO: use prepared statements to eliminate SQL injection
+ update_query = text(
+ f"""
+ UPDATE mdb_columns
+ SET
+ val_min = '{column_stats["val_min"]}',
+ val_max = '{column_stats["val_max"]}',
+ mean = '{column_stats["mean"]}',
+ median = '{column_stats["median"]}',
+ std_dev = '{column_stats["std_dev"]}'
+ WHERE
+ tID = '{table_id}' AND internal_name = '{column}'
+ """
+ )
+ # We need an extra select query to fetch the column ID for OpenSearch
+ select_query = text(
+ f"""
+ SELECT id
+ FROM mdb_columns
+ WHERE tID = '{table_id}' AND internal_name = '{column}'
+ """
+ )
+ with db.engine.begin() as connection:
+ connection.execute(update_query)
+ result = connection.execute(select_query)
+ column_id = result.fetchone()[0]
+ connection.commit()
+
+ # Fetch the existing document
+ existing_document = os.get(index="database", id=database_id)["_source"]
+
+ # Loop over OS response and append the statistics for each column
+ for tidx, table in enumerate(existing_document["tables"]):
+ if table["id"] == table_id:
+ for cidx, column in enumerate(table["columns"]):
+ if column["id"] == column_id:
+ existing_document["tables"][tidx]["columns"][cidx].update(
+ column_stats
+ )
+ # No need to keep searching if column id matches
+ break
+
+ # Index and force refresh
+ os.index(
+ index="database",
+ id=database_id,
+ body=existing_document,
+ refresh=True,
+ )
+
+ return True
diff --git a/dbrepo-analyse-service/pywsgi.py b/dbrepo-analyse-service/pywsgi.py
index fb981f98d5c1a0a3ecb7fe54e370be61d0cdba87..7ef0e4b63ad98d080ea2169e08359a6bc2647213 100644
--- a/dbrepo-analyse-service/pywsgi.py
+++ b/dbrepo-analyse-service/pywsgi.py
@@ -1,9 +1,10 @@
from gevent import monkey
+
monkey.patch_all()
import os
from gevent.pywsgi import WSGIServer
from app import app
-http_server = WSGIServer(('0.0.0.0', int(os.environ['PORT_APP'])), app)
+http_server = WSGIServer(("0.0.0.0", int(os.environ["PORT_APP"])), app)
http_server.serve_forever()
diff --git a/dbrepo-analyse-service/test/conftest.py b/dbrepo-analyse-service/test/conftest.py
index c062a1b9d4a57c1eeb9da77faae0a023fe2a419c..50a2e3ab832e01d1cad63629f3b381677a295c71 100644
--- a/dbrepo-analyse-service/test/conftest.py
+++ b/dbrepo-analyse-service/test/conftest.py
@@ -2,9 +2,12 @@ import os
import pytest
import logging
+import json
from minio.deleteobjects import DeleteObject
from testcontainers.minio import MinioContainer
+from testcontainers.mysql import MySqlContainer
+from testcontainers.opensearch import OpenSearchContainer
@pytest.fixture(scope="session")
@@ -19,13 +22,18 @@ def session(request):
logging.debug("[fixture] starting container")
container.start()
# set the environment for the client
- endpoint = 'http://' + container.get_container_host_ip() + ':' + container.get_exposed_port(9000)
- os.environ['S3_STORAGE_ENDPOINT'] = endpoint
+ endpoint = (
+ "http://"
+ + container.get_container_host_ip()
+ + ":"
+ + container.get_exposed_port(9000)
+ )
+ os.environ["S3_STORAGE_ENDPOINT"] = endpoint
client = container.get_client()
# create buckets
- logging.debug('[fixture] make buckets dbrepo-upload, dbrepo-download')
- client.make_bucket('dbrepo-upload')
- client.make_bucket('dbrepo-download')
+ logging.debug("[fixture] make buckets dbrepo-upload, dbrepo-download")
+ client.make_bucket("dbrepo-upload")
+ client.make_bucket("dbrepo-download")
# destructor
def stop_minio():
@@ -48,7 +56,92 @@ def cleanup(request, session):
objects = []
for obj in session.get_client().list_objects(bucket):
objects.append(DeleteObject(obj.object_name))
- logging.info(f'request to remove objects {objects}')
+ logging.info(f"request to remove objects {objects}")
errors = session.get_client().remove_objects(bucket, objects)
for error in errors:
- raise ConnectionError(f'Failed to delete object with key {error.object_name} of bucket {bucket}')
+ raise ConnectionError(
+ f"Failed to delete object with key {error.object_name} of bucket {bucket}"
+ )
+
+
+@pytest.fixture(scope="function")
+def metadata_db_container():
+ metadata_db = MySqlContainer(
+ "bitnami/mariadb:10.5",
+ MYSQL_USER="root",
+ MYSQL_PASSWORD="dbrepo",
+ MYSQL_DATABASE="fda",
+ )
+ metadata_db._name = "metadata-db-test"
+ metadata_db.ports = {"3306": 33060}
+ metadata_db.env = {
+ "MYSQL_USER": metadata_db.MYSQL_USER,
+ "MARIADB_ROOT_PASSWORD": metadata_db.MYSQL_ROOT_PASSWORD,
+ "MARIADB_DATABASE": metadata_db.MYSQL_DATABASE,
+ }
+ # volume that mounts db schema from metadata-db
+ metadata_db.with_volume_mapping(
+ os.path.abspath("../dbrepo-metadata-db/setup-schema.sql"), "/schema.sql"
+ )
+ # volume for script that initializes schema and inserts test values
+ metadata_db.with_volume_mapping(
+ os.path.abspath("./test/init-db.sh"),
+ "/docker-entrypoint-initdb.d/init-db.sh",
+ )
+
+ # validate creation of schema and data
+ with metadata_db:
+ print(
+ metadata_db.exec(
+ f"mariadb -u{metadata_db.MYSQL_USER} -p{metadata_db.MYSQL_ROOT_PASSWORD} fda -e 'SELECT * FROM mdb_databases;'"
+ )
+ )
+ yield metadata_db
+
+
+@pytest.fixture(scope="function")
+def data_db_container():
+ data_db = MySqlContainer(
+ "bitnami/mariadb:10.5",
+ MYSQL_USER="root",
+ MYSQL_PASSWORD="dbrepo",
+ )
+ data_db._name = "data-db-test"
+ data_db.ports = {"3306": 33061}
+ data_db.env = {
+ "MYSQL_USER": data_db.MYSQL_USER,
+ "MARIADB_ROOT_PASSWORD": data_db.MYSQL_ROOT_PASSWORD,
+ }
+ # volume that mounts csv for data import
+ data_db.with_volume_mapping(
+ os.path.abspath("./data/test_stats/test_stats_01.csv"), "/test_stats_01.csv"
+ )
+ # volume for script to create a test data db and import values from a csv
+ data_db.with_volume_mapping(
+ os.path.abspath("./test/init-data-db.sh"),
+ "/docker-entrypoint-initdb.d/init-data-db.sh",
+ )
+
+ with data_db:
+ yield data_db
+
+
+@pytest.fixture(scope="function")
+def opensearch_container():
+ os_container = OpenSearchContainer("opensearchproject/opensearch:2.10.0")
+
+ with os_container:
+ client = os_container.get_client()
+ index_mapping_path = os.path.join(
+ "..", "dbrepo-search-db", "init", "indices", "database.json"
+ )
+ with open(index_mapping_path, "r") as file:
+ mapping = json.load(file)
+ client.indices.create(index="database", body=mapping)
+
+ yield os_container
+
+
+@pytest.fixture(scope="function")
+def all_containers(opensearch_container, metadata_db_container, data_db_container):
+ yield opensearch_container, metadata_db_container, data_db_container
diff --git a/dbrepo-analyse-service/test/init-data-db.sh b/dbrepo-analyse-service/test/init-data-db.sh
new file mode 100755
index 0000000000000000000000000000000000000000..2ba035d6ed3bf3f07cc01935016d18d142841bb4
--- /dev/null
+++ b/dbrepo-analyse-service/test/init-data-db.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Script to initialize a test data db according to a csv file
+# Intented to be run from pytest inside a mysql/mariadb container
+
+mysql -u"$MYSQL_USER" -p"$MYSQL_ROOT_PASSWORD" <<EOSQL
+CREATE DATABASE internal1;
+
+USE internal1;
+
+CREATE TABLE table1 (
+ col1 DATETIME,
+ col2 VARCHAR(255),
+ col3 FLOAT,
+ col4 FLOAT,
+ col5 INT
+);
+
+LOAD DATA INFILE '/test_stats_01.csv'
+INTO TABLE table1
+FIELDS TERMINATED BY ',' ENCLOSED BY '"'
+LINES TERMINATED BY '\\n'
+IGNORE 1 LINES
+(col1, col2, col3, col4, col5);
+EOSQL
diff --git a/dbrepo-analyse-service/test/init-db.sh b/dbrepo-analyse-service/test/init-db.sh
new file mode 100755
index 0000000000000000000000000000000000000000..c8a8411357f3d6dc83a228c6cd81aa88dad162b7
--- /dev/null
+++ b/dbrepo-analyse-service/test/init-db.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# Script to initialize db schema and insert test values
+# Intented to be run from pytest inside a mysql/mariadb docker container
+
+mysql -u"$MYSQL_USER" -p"$MYSQL_ROOT_PASSWORD" "$MYSQL_DATABASE" <<EOSQL
+source /schema.sql
+
+INSERT INTO mdb_users (id, username, firstname, lastname, email, orcid, affiliation, mariadb_password, theme_dark)
+VALUES
+ ('1', 'user1', 'John', 'Doe', 'user1@example.com', '0000-0001-1234-5678', 'University A', 'password1', TRUE);
+
+INSERT INTO mdb_images (name, version, default_port, dialect, driver_class, jdbc_method, created, last_modified)
+VALUES
+ ('Image1', '1.0', 5432, 'postgresql', 'org.postgresql.Driver', 'jdbc:postgresql', NOW(), NOW()),
+ ('Image2', '2.0', 3306, 'mysql', 'com.mysql.jdbc.Driver', 'jdbc:mysql', NOW(), NOW());
+
+INSERT INTO mdb_containers (internal_name, name, host, port, ui_host, ui_port, ui_additional_flags, sidecar_host, sidecar_port, image_id, created, last_modified, privileged_username, privileged_password)
+VALUES
+ ('container1', 'Container1', 'localhost', 3306, 'localhost', 3306, NULL, 'localhost', 3305, 1, NOW(), NOW(), 'admin', 'admin'),
+ ('container2', 'Container2', 'localhost', 3307, 'localhost', 3307, NULL, 'localhost', 3306, 2, NOW(), NOW(), 'admin', 'admin');
+
+INSERT INTO mdb_databases (cid, name, internal_name, exchange_name, description, engine, is_public, created_by, owned_by, contact_person, created, last_modified)
+VALUES
+ (1, 'Database1', 'internal1', 'Exchange1', 'Description1', 'InnoDB', TRUE, NULL, NULL, NULL, NOW(), NOW()),
+ (2, 'Database2', 'internal2', 'Exchange2', 'Description2', 'MyISAM', FALSE, NULL, NULL, NULL, NOW(), NOW());
+
+INSERT INTO mdb_tables (tDBID, internal_name, queue_name, routing_key, tName, tDescription, num_rows, data_length, max_data_length, avg_row_length, \`separator\`, quote, element_null, skip_lines, element_true, element_false, Version, created, versioned, created_by, owned_by, last_modified)
+VALUES
+ (1, 'table1', 'Queue1', 'RoutingKey1', 'TableName1', 'TableDescription1', 5, 9, 15, 3, ',', '"', NULL, 2, TRUE, FALSE, '1.0', NOW(), TRUE, '1', '1', NOW()),
+ (1, 'table2', 'Queue2', 'RoutingKey2', 'TableName2', 'TableDescription2', 0, 0, 0, 0, ',', "'", 'NA', 0, '1', '0', '2.0', NOW(), TRUE, '1', '1', NOW()),
+ (2, 'table3', 'Queue3', 'RoutingKey3', 'TableName3', 'TableDescription3', 0, 0, 0, 0, ',', '"', 'EMPTY', 5, 'yes', 'no', '1.0', NOW(), TRUE, '1', '1', NOW());
+
+INSERT INTO mdb_columns (tID, dfID, cName, internal_name, alias, Datatype, length, ordinal_position, is_primary_key, index_length, size, d, auto_generated, is_null_allowed, val_min, val_max, mean, median, std_dev, created, last_modified)
+VALUES
+ (1, NULL, 'Column1', 'col1', 'Alias1', 'DATETIME', 255, 1, TRUE, NULL, NULL, NULL, FALSE, TRUE, NULL, NULL, NULL, NULL, NULL, NOW(), NOW()),
+ (1, NULL, 'Column2', 'col2', 'Alias2', 'VARCHAR', NULL, 2, FALSE, NULL, NULL, NULL, FALSE, TRUE, NULL, NULL, NULL, NULL, NULL, NOW(), NOW()),
+ (1, NULL, 'Column3', 'col3', 'Alias3', 'FLOAT', 10, 3, FALSE, NULL, NULL, 2, FALSE, TRUE, 0, 100, NULL, NULL, NULL, NOW(), NOW()),
+ (1, NULL, 'Column4', 'col4', 'Alias4', 'FLOAT', 10, 3, FALSE, NULL, NULL, 2, FALSE, TRUE, 0, 100, NULL, NULL, NULL, NOW(), NOW()),
+ (1, NULL, 'Column5', 'col5', 'Alias5', 'INT', 10, 3, FALSE, NULL, NULL, 2, FALSE, TRUE, 0, 100, NULL, NULL, NULL, NOW(), NOW());
+EOSQL
diff --git a/dbrepo-analyse-service/test/test_determine_dt.py b/dbrepo-analyse-service/test/test_determine_dt.py
index 2acf97aacd300b57bf6e9a5a6a214b7ed9b4bd75..199c94fe91186d44b1cd81e0375d1cf8dc84224d 100644
--- a/dbrepo-analyse-service/test/test_determine_dt.py
+++ b/dbrepo-analyse-service/test/test_determine_dt.py
@@ -15,20 +15,19 @@ from determine_dt import determine_datatypes
class DetermineDatatypesTest(unittest.TestCase):
-
# @Test
def test_determine_datatypesDateTime_succeeds(self):
exp = {
- 'columns': {
- 'Datum': 'timestamp',
- 'Standort': 'varchar',
- 'Parameter': 'varchar',
- 'Intervall': 'varchar',
- 'Einheit': 'varchar',
- 'Wert': 'decimal',
- 'Status': 'varchar'
+ "columns": {
+ "Datum": "timestamp",
+ "Standort": "varchar",
+ "Parameter": "varchar",
+ "Intervall": "varchar",
+ "Einheit": "varchar",
+ "Wert": "decimal",
+ "Status": "varchar",
},
- 'separator': ','
+ "separator": ",",
}
# mock
@@ -41,16 +40,16 @@ class DetermineDatatypesTest(unittest.TestCase):
# @Test
def test_determine_datatypesDateTimeWithTimezone_succeeds(self):
exp = {
- 'columns': {
- 'Datum': 'varchar',
- 'Standort': 'varchar',
- 'Parameter': 'varchar',
- 'Intervall': 'varchar',
- 'Einheit': 'varchar',
- 'Wert': 'decimal',
- 'Status': 'varchar'
+ "columns": {
+ "Datum": "varchar",
+ "Standort": "varchar",
+ "Parameter": "varchar",
+ "Intervall": "varchar",
+ "Einheit": "varchar",
+ "Wert": "decimal",
+ "Status": "varchar",
},
- 'separator': ','
+ "separator": ",",
}
# mock
@@ -63,16 +62,16 @@ class DetermineDatatypesTest(unittest.TestCase):
# @Test
def test_determine_datatypesDateTimeWithT_succeeds(self):
exp = {
- 'columns': {
- 'Datum': 'timestamp',
- 'Standort': 'varchar',
- 'Parameter': 'varchar',
- 'Intervall': 'varchar',
- 'Einheit': 'varchar',
- 'Wert': 'decimal',
- 'Status': 'varchar'
+ "columns": {
+ "Datum": "timestamp",
+ "Standort": "varchar",
+ "Parameter": "varchar",
+ "Intervall": "varchar",
+ "Einheit": "varchar",
+ "Wert": "decimal",
+ "Status": "varchar",
},
- 'separator': ','
+ "separator": ",",
}
# mock
@@ -85,16 +84,16 @@ class DetermineDatatypesTest(unittest.TestCase):
# @Test
def test_determine_datatypes_succeeds(self):
exp = {
- 'columns': {
- 'int': 'bigint',
- 'float': 'decimal',
- 'string': 'varchar',
- 'boolean': 'bool',
- 'date': 'date',
- 'time': 'timestamp',
- 'enum': 'varchar' # currently not used
+ "columns": {
+ "int": "bigint",
+ "float": "decimal",
+ "string": "varchar",
+ "boolean": "bool",
+ "date": "date",
+ "time": "timestamp",
+ "enum": "varchar", # currently not used
},
- 'separator': ','
+ "separator": ",",
}
# mock
@@ -106,62 +105,57 @@ class DetermineDatatypesTest(unittest.TestCase):
# @Test
def test_determine_datatypes_fileDoesNotExist_fails(self):
-
# test
try:
response = determine_datatypes("i_do_not_exist.csv")
except ClientError as e:
pass # s3.head operation
except Exception:
- self.fail('unexpected exception raised')
+ self.fail("unexpected exception raised")
else:
- self.fail('ExpectedException not raised')
+ self.fail("ExpectedException not raised")
# @Test
def test_determine_datatypes_fileEmpty_succeeds(self):
-
# mock
S3Client().upload_file("empty.csv", './data/test_dt/', 'dbrepo-upload')
# test
response = determine_datatypes("empty.csv")
data = json.loads(response)
- self.assertEqual(data['columns'], [])
- self.assertEqual(data['separator'], ',')
+ self.assertEqual(data["columns"], [])
+ self.assertEqual(data["separator"], ",")
# @Test
def test_determine_datatypes_separatorSemicolon_succeeds(self):
-
# mock
S3Client().upload_file("separator.csv", './data/test_dt/', 'dbrepo-upload')
# test
response = determine_datatypes(filename="separator.csv", separator=";")
data = json.loads(response)
- self.assertEqual(data['separator'], ';')
+ self.assertEqual(data["separator"], ";")
# @Test
def test_determine_datatypes_separatorGuess_succeeds(self):
-
# mock
S3Client().upload_file("separator.csv", './data/test_dt/', 'dbrepo-upload')
# test
response = determine_datatypes(filename="separator.csv")
data = json.loads(response)
- self.assertEqual(data['separator'], ';')
+ self.assertEqual(data["separator"], ";")
# @Test
def test_determine_datatypes_separatorGuessLargeDataset_succeeds(self):
-
# mock
S3Client().upload_file("large.csv", './data/test_dt/', 'dbrepo-upload')
# test
response = determine_datatypes(filename="large.csv")
data = json.loads(response)
- self.assertEqual(data['separator'], ',')
+ self.assertEqual(data["separator"], ",")
-if __name__ == '__main__':
+if __name__ == "__main__":
unittest.main()
diff --git a/dbrepo-analyse-service/test/test_determine_pk.py b/dbrepo-analyse-service/test/test_determine_pk.py
index 4fffda96c524d551efe3d8728713547fa7179a8f..d6f1314a2bb1b56bed76a56aa4d18eff3a514a6e 100644
--- a/dbrepo-analyse-service/test/test_determine_pk.py
+++ b/dbrepo-analyse-service/test/test_determine_pk.py
@@ -14,7 +14,6 @@ from clients.s3_client import S3Client
from determine_pk import determine_pk
class DeterminePrimaryKeyTest(unittest.TestCase):
-
# @Test
def test_determine_pk_largeFileIdFirst_succeeds(self):
diff --git a/dbrepo-analyse-service/test/test_determine_stats.py b/dbrepo-analyse-service/test/test_determine_stats.py
new file mode 100644
index 0000000000000000000000000000000000000000..5d82ffedbe65f2f366feac4c317c4b3863f5aa79
--- /dev/null
+++ b/dbrepo-analyse-service/test/test_determine_stats.py
@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Jan 17 17:13:25 2024
+
+@author: Sotiris Tsepelakis
+"""
+
+from opensearchpy import OpenSearch
+from sqlalchemy import create_engine
+
+from determine_stats import determine_stats
+
+
+class TestDetermineStatisticalProperties:
+ # @Test
+ def test_determine_statistical_properties_succeeds(self, all_containers):
+ # mock request
+ payload = {
+ "database_id": 1,
+ "table_id": 1,
+ "data_db_host": "localhost",
+ "data_db_port": 33061,
+ }
+
+ os_host = all_containers[0].get_container_host_ip()
+ os_port = all_containers[0].get_exposed_port("9200/tcp")
+ os = OpenSearch([{"host": os_host, "port": os_port}])
+
+ metadata_db_host = all_containers[1].get_container_host_ip()
+ metadata_db_port = all_containers[1].get_exposed_port(3306)
+ db = create_engine(
+ f"mysql+pymysql://root:dbrepo@{metadata_db_host}:{metadata_db_port}/fda"
+ )
+
+ # index a test document
+ all_containers[0].get_client().index(
+ index="database",
+ id="1",
+ body={
+ "id": 1,
+ "name": "testdb",
+ "exchange_name": "dbrepo",
+ "internal_name": "testdb_ygvq",
+ "tables": [
+ {
+ "id": 1,
+ "database_id": 1,
+ "name": "mytable",
+ "internal_name": "mytable",
+ "queue_name": "dbrepo",
+ "routing_key": "dbrepo.testdb_ygvq.mytable",
+ "columns": [
+ {
+ "id": 1,
+ "database_id": 1,
+ "table_id": 1,
+ "name": "id",
+ "internal_name": "id",
+ "auto_generated": True,
+ "is_primary_key": True,
+ "column_type": "BIGINT",
+ "is_public": True,
+ "is_null_allowed": False,
+ "enums": [],
+ "sets": [],
+ },
+ {
+ "id": 2,
+ "database_id": 1,
+ "table_id": 1,
+ "name": "col1",
+ "internal_name": "col1",
+ "date_format": {
+ "id": 2,
+ "database_format": "%Y-%c-%d %H:%i:%S",
+ "unix_format": "yyyy-MM-dd HH:mm:ss",
+ "has_time": True,
+ "created_at": "2024-01-17T13:22:26.000Z",
+ },
+ "auto_generated": False,
+ "is_primary_key": False,
+ "column_type": "DATETIME",
+ "is_public": True,
+ "is_null_allowed": True,
+ "enums": [],
+ "sets": [],
+ },
+ {
+ "id": 3,
+ "database_id": 1,
+ "table_id": 1,
+ "name": "col2",
+ "internal_name": "col2",
+ "auto_generated": False,
+ "is_primary_key": False,
+ "column_type": "VARCHAR",
+ "size": 255,
+ "is_public": True,
+ "is_null_allowed": True,
+ "enums": [],
+ "sets": [],
+ },
+ {
+ "id": 4,
+ "database_id": 1,
+ "table_id": 1,
+ "name": "col3",
+ "internal_name": "col3",
+ "auto_generated": True,
+ "is_primary_key": True,
+ "column_type": "FLOAT",
+ "size": 24,
+ "is_public": True,
+ "is_null_allowed": True,
+ "enums": [],
+ "sets": [],
+ },
+ {
+ "id": 5,
+ "database_id": 1,
+ "table_id": 1,
+ "name": "col4",
+ "internal_name": "col4",
+ "auto_generated": False,
+ "is_primary_key": False,
+ "column_type": "FLOAT",
+ "size": 24,
+ "is_public": True,
+ "is_null_allowed": True,
+ "enums": [],
+ "sets": [],
+ },
+ {
+ "id": 6,
+ "database_id": 1,
+ "table_id": 1,
+ "name": "col5",
+ "internal_name": "col5",
+ "auto_generated": False,
+ "is_primary_key": False,
+ "column_type": "INT",
+ "size": 255,
+ "is_public": True,
+ "is_null_allowed": True,
+ "enums": [],
+ "sets": [],
+ },
+ ],
+ "constraints": {"uniques": [], "foreign_keys": []},
+ }
+ ],
+ },
+ )
+ # test
+ response = determine_stats(db, os, **payload)
+ assert response