Skip to content
Snippets Groups Projects
Commit a132a136 authored by Sotiris Tsepelakis's avatar Sotiris Tsepelakis :speech_balloon: Committed by Martin Weise
Browse files

Analyse service: Add stats determination functionality

parent 3c2221d6
No related branches found
No related tags found
4 merge requests!231CI: Remove build for log-service,!228Better error message handling in the frontend,!223Release of version 1.4.0,!221Analyse service: Add stats determination functionality
Show whitespace changes
Inline Side-by-side
Showing
with 1104 additions and 332 deletions
.gitlab-ci.yml
+ 1
− 1
View file @ a132a136
...@@ -119,7 +119,7 @@ test-analyse-service: ...@@ -119,7 +119,7 @@ test-analyse-service:
script: script:
- "pip install pipenv" - "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy" - "pipenv install gunicorn && pipenv install --dev --system --deploy"
- "cd ./dbrepo-analyse-service/ && coverage run -m pytest test/test_determine_dt.py test/test_determine_pk.py --junitxml=report.xml && coverage html && coverage report > ./coverage.txt" - "cd ./dbrepo-analyse-service/ && coverage run -m pytest test/test_determine_dt.py test/test_determine_pk.py test/test_determine_stats.py --junitxml=report.xml && coverage html && coverage report > ./coverage.txt"
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'" - "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts: artifacts:
when: always when: always
......
dbrepo-analyse-service/Dockerfile
+ 1
− 1
View file @ a132a136
FROM python:3.9-slim FROM python:3.9-slim
MAINTAINER Martin Weise <martin.weise@tuwien.ac.at> MAINTAINER Martin Weise <martin.weise@tuwien.ac.at>
RUN apt update && apt install -y curl RUN apt update && apt install -y curl gcc libmariadb-dev
WORKDIR /app WORKDIR /app
......
dbrepo-analyse-service/Pipfile
+ 5
− 0
View file @ a132a136
...@@ -16,11 +16,16 @@ numpy = "*" ...@@ -16,11 +16,16 @@ numpy = "*"
pandas = "*" pandas = "*"
messytables = "*" messytables = "*"
minio = "*" minio = "*"
flask-sqlalchemy = "*"
opensearch-py = "*"
pymysql = "*"
[dev-packages] [dev-packages]
coverage = "*" coverage = "*"
pytest = "*" pytest = "*"
testcontainers-minio = "*" testcontainers-minio = "*"
testcontainers-mysql = "*"
testcontainers-opensearch = "*"
[requires] [requires]
python_version = "3.9" python_version = "3.9"
dbrepo-analyse-service/Pipfile.lock
+ 443
− 177
View file @ a132a136
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "fe382056038fea58538a52ab9f8ac7bd80659537f8336a534cb79a71e515bceb" "sha256": "593bb7bb2fab9041c4da489021bb7fe2f857624e18f484d53f1c4f3958b04837"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": { "requires": {
...@@ -69,19 +69,19 @@ ...@@ -69,19 +69,19 @@
}, },
"boto3": { "boto3": {
"hashes": [ "hashes": [
"sha256:1f94042f4efb5133b6b9b8b3243afc01143a81d21b3197a3afadf5780f97b05d", "sha256:1efc02be786884034d503d59c018cf7650d0cff9fcb37cd2eb49b802a6fe6111",
"sha256:5c1bb487c68120aae236354d81b8a1a55d0aa3395d30748a01825ef90891921e" "sha256:8ca248cc84e7e859e4e276eb9c4309fa01a3e58473bf48d6c33448be870c2bb8"
], ],
"index": "pypi", "index": "pypi",
"version": "==1.34.14" "version": "==1.34.17"
}, },
"botocore": { "botocore": {
"hashes": [ "hashes": [
"sha256:041bed0852649cab7e4dcd4d87f9d1cc084467fb846e5b60015e014761d96414", "sha256:7272c39032c6f1d62781e4c8445d9a1d9140c2bf52ba7ee66bf6db559c4b2427",
"sha256:3b592f50f0406e236782a3a0a9ad1c3976060fdb2e04a23d18c3df5b7dfad3e0" "sha256:e48a662f3a6919219276b55085e8f73c3347966675f55e9d448be30cf79678ee"
], ],
"markers": "python_version >= '3.8'", "markers": "python_version >= '3.8'",
"version": "==1.34.14" "version": "==1.34.17"
}, },
"certifi": { "certifi": {
"hashes": [ "hashes": [
...@@ -250,7 +250,7 @@ ...@@ -250,7 +250,7 @@
"sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519", "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519",
"sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561" "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==3.3.2" "version": "==3.3.2"
}, },
"click": { "click": {
...@@ -284,6 +284,14 @@ ...@@ -284,6 +284,14 @@
"index": "pypi", "index": "pypi",
"version": "==3.0.0" "version": "==3.0.0"
}, },
"flask-sqlalchemy": {
"hashes": [
"sha256:4ba4be7f419dc72f4efd8802d69974803c37259dd42f3913b0dcf75c9447e0a0",
"sha256:e4b68bb881802dda1a7d878b2fc84c06d1ee57fb40b874d3dc97dabfa36b8312"
],
"index": "pypi",
"version": "==3.1.1"
},
"gevent": { "gevent": {
"hashes": [ "hashes": [
"sha256:272cffdf535978d59c38ed837916dfd2b5d193be1e9e5dcc60a5f4d5025dd98a", "sha256:272cffdf535978d59c38ed837916dfd2b5d193be1e9e5dcc60a5f4d5025dd98a",
...@@ -391,7 +399,7 @@ ...@@ -391,7 +399,7 @@
"sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da", "sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da",
"sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33" "sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33"
], ],
"markers": "python_version >= '3.11' and platform_python_implementation == 'CPython'", "index": "pypi",
"version": "==3.0.3" "version": "==3.0.3"
}, },
"gunicorn": { "gunicorn": {
...@@ -418,6 +426,14 @@ ...@@ -418,6 +426,14 @@
"markers": "python_version >= '3.5'", "markers": "python_version >= '3.5'",
"version": "==3.6" "version": "==3.6"
}, },
"importlib-metadata": {
"hashes": [
"sha256:4805911c3a4ec7c3966410053e9ec6a1fecd629117df5adee56dfc9432a1081e",
"sha256:f238736bb06590ae52ac1fab06a3a9ef1d8dce2b7a35b5ab329371d6c8f5d2cc"
],
"markers": "python_version < '3.10'",
"version": "==7.0.1"
},
"itsdangerous": { "itsdangerous": {
"hashes": [ "hashes": [
"sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44", "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44",
...@@ -428,11 +444,11 @@ ...@@ -428,11 +444,11 @@
}, },
"jinja2": { "jinja2": {
"hashes": [ "hashes": [
"sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852", "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa",
"sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61" "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"
], ],
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==3.1.2" "version": "==3.1.3"
}, },
"jmespath": { "jmespath": {
"hashes": [ "hashes": [
...@@ -466,97 +482,87 @@ ...@@ -466,97 +482,87 @@
}, },
"lxml": { "lxml": {
"hashes": [ "hashes": [
"sha256:00bfccab28f710bb13f7f644c980ad50ce3e5b6a412b5bb9a6c30136b298fb2c", "sha256:13521a321a25c641b9ea127ef478b580b5ec82aa2e9fc076c86169d161798b01",
"sha256:03c977ffc9a4bf17b3e0f8db0451dc38e9f4ec92cfdb5df462d38fbe6e6e0825", "sha256:14deca1460b4b0f6b01f1ddc9557704e8b365f55c63070463f6c18619ebf964f",
"sha256:0499310df9afc0ce634ce14cacbb333d62f561038ea4db640494e4a22ac4f2e9", "sha256:16018f7099245157564d7148165132c70adb272fb5a17c048ba70d9cc542a1a1",
"sha256:070469a23f2ef3a9e72165af7e0b12eca9a6e47c3a8ec1cad60d14fb2f2c3aa8", "sha256:16dd953fb719f0ffc5bc067428fc9e88f599e15723a85618c45847c96f11f431",
"sha256:0963de4fe463caff48e6ce4d83d19a3c099126874185d10cf490c29057ca518d", "sha256:19a1bc898ae9f06bccb7c3e1dfd73897ecbbd2c96afe9095a6026016e5ca97b8",
"sha256:0a79eca2ef5e032c8ed9da07f84a07a29105f220b777613dfe7fc31445691ee3", "sha256:1ad17c20e3666c035db502c78b86e58ff6b5991906e55bdbef94977700c72623",
"sha256:0dc36ec06514fe8848c4733d47f96a0636f82d9ca3eaa2132373426bc03f178f", "sha256:22b7ee4c35f374e2c20337a95502057964d7e35b996b1c667b5c65c567d2252a",
"sha256:0ecc0f1e1d901b66f2f68edff85b8ff421fa9683d02eaea6742a42c145d741b6", "sha256:24ef5a4631c0b6cceaf2dbca21687e29725b7c4e171f33a8f8ce23c12558ded1",
"sha256:0f70e5de6b3e24ababeca597f776e5f37973f05d28a4d9f467aa5b45745af762", "sha256:25663d6e99659544ee8fe1b89b1a8c0aaa5e34b103fab124b17fa958c4a324a6",
"sha256:0fb55d77e685def5218701d5d296fca62f595752c88402404da685864b06b67e", "sha256:262bc5f512a66b527d026518507e78c2f9c2bd9eb5c8aeeb9f0eb43fcb69dc67",
"sha256:1215c8e57a25ad68488abb83a36734f6c6b3f0ccd880f0c68da98682d463ef09", "sha256:280f3edf15c2a967d923bcfb1f8f15337ad36f93525828b40a0f9d6c2ad24890",
"sha256:1546fa25a6dde5698271e19787062446f433c98ca7eab35545f665dde2c1bd34", "sha256:2ad3a8ce9e8a767131061a22cd28fdffa3cd2dc193f399ff7b81777f3520e372",
"sha256:18b456f1bfb338be94a916166ac5507e73b9eb9f6e1f0fbd1c8caff2f3fa5535", "sha256:2befa20a13f1a75c751f47e00929fb3433d67eb9923c2c0b364de449121f447c",
"sha256:18caa0d3925902949cb060fe5f2da70c953d60bd9ef78657bd389f6db30533cc", "sha256:2f37c6d7106a9d6f0708d4e164b707037b7380fcd0b04c5bd9cae1fb46a856fb",
"sha256:19251f782ea51a4841e747158195312ef63e06b47889e159dc5f1b2e5d668465", "sha256:304128394c9c22b6569eba2a6d98392b56fbdfbad58f83ea702530be80d0f9df",
"sha256:1d78e91cbffe733ff325e0d258bb64702c8d91f8f652db088bd2989c6a8f90cc", "sha256:342e95bddec3a698ac24378d61996b3ee5ba9acfeb253986002ac53c9a5f6f84",
"sha256:2ec9fa65e0638551a5ad31cb9fa160b321f19632e5ec517fe68d7b4110133e69", "sha256:3aeca824b38ca78d9ee2ab82bd9883083d0492d9d17df065ba3b94e88e4d7ee6",
"sha256:3013823b0069cb4bd9b558e87076a18142703c6d2ac3a5d5521dd35734d23a72", "sha256:3d184e0d5c918cff04cdde9dbdf9600e960161d773666958c9d7b565ccc60c45",
"sha256:31362b900b8fd2a026b9d24695ebe5449ea8f0c948af2445d7880b738d9fc368", "sha256:3e3898ae2b58eeafedfe99e542a17859017d72d7f6a63de0f04f99c2cb125936",
"sha256:32a135d4ef8f966bc087d450d641df73fc6874f04cf6608111541b50090e6f13", "sha256:3eea6ed6e6c918e468e693c41ef07f3c3acc310b70ddd9cc72d9ef84bc9564ca",
"sha256:331237209fe76951450c1119af0879f04f32d1b07b21e83a34ba439520492feb", "sha256:3f14a4fb1c1c402a22e6a341a24c1341b4a3def81b41cd354386dcb795f83897",
"sha256:3714f339449d2738b4fadd078a6022704a2af3cab06bec9627b19eaa4205090d", "sha256:436a943c2900bb98123b06437cdd30580a61340fbdb7b28aaf345a459c19046a",
"sha256:387c5416b8bb4b8ad7306797cb2719a841f5f3836b3c39fcaa56b9af5448dd2a", "sha256:4946e7f59b7b6a9e27bef34422f645e9a368cb2be11bf1ef3cafc39a1f6ba68d",
"sha256:39a3cf581e9040e66aaa719b2f338b2b7eb43ce1db059089c82ae72e0fd48b47", "sha256:49a9b4af45e8b925e1cd6f3b15bbba2c81e7dba6dce170c677c9cda547411e14",
"sha256:3ffc56d68b9782919d69ae9a6fac99efd7547f2666ccb7ecfd12871564d16133", "sha256:4f8b0c78e7aac24979ef09b7f50da871c2de2def043d468c4b41f512d831e912",
"sha256:42069ce16dab9755b381b90defd846ca407b9ead05dc20732fd5502b5cc49b87", "sha256:52427a7eadc98f9e62cb1368a5079ae826f94f05755d2d567d93ee1bc3ceb354",
"sha256:430780608d16b3bb96ef99a67a1a0626c8a295193af53ce9c4d5ec3fef2fbc79", "sha256:5e53d7e6a98b64fe54775d23a7c669763451340c3d44ad5e3a3b48a1efbdc96f",
"sha256:4432a1d89a9b340bc6bd1201aef3ba03112f151d3f340d9218247dc0c85028ab", "sha256:5fcfbebdb0c5d8d18b84118842f31965d59ee3e66996ac842e21f957eb76138c",
"sha256:45e747b4e82390ffd802528b9e7b39333c1ce71423057bf5961b30ec0d52f76f", "sha256:601f4a75797d7a770daed8b42b97cd1bb1ba18bd51a9382077a6a247a12aa38d",
"sha256:473f2d0511dd84697326ee9362b0c0c2e9f99a433dcb1fbb5aa8df3d1b2185db", "sha256:61c5a7edbd7c695e54fca029ceb351fc45cd8860119a0f83e48be44e1c464862",
"sha256:47f46a2ebade07f3afa47695882e7725440c49bf77cba39c3762a42597e5aad3", "sha256:6a2a2c724d97c1eb8cf966b16ca2915566a4904b9aad2ed9a09c748ffe14f969",
"sha256:49dc4dcf14a08f160bb3f5f571f63514d918b8933a25c221536571a5fc010271", "sha256:6d48fc57e7c1e3df57be5ae8614bab6d4e7b60f65c5457915c26892c41afc59e",
"sha256:4ae66b6b0f82e7839b6b8d009182c652d48e7d2ea21a6709f3033ce5fbf199c4", "sha256:6f11b77ec0979f7e4dc5ae081325a2946f1fe424148d3945f943ceaede98adb8",
"sha256:4b49a1569ed6d05808f4d163a316e7bf4a230e0c36855b59f56020ae27ae586a", "sha256:704f5572ff473a5f897745abebc6df40f22d4133c1e0a1f124e4f2bd3330ff7e",
"sha256:4d58af4ebd711dad40f1c024a779950d9918d04d74f49230edf5d271dcd33c28", "sha256:725e171e0b99a66ec8605ac77fa12239dbe061482ac854d25720e2294652eeaa",
"sha256:4df6be79be4d7574e9e4002aeb6aa03d3f3809582db07abb166df7fc6e7438af", "sha256:7cfced4a069003d8913408e10ca8ed092c49a7f6cefee9bb74b6b3e860683b45",
"sha256:57f5c362cbd9d688fb1fa07c8955cec26d5c066fbcb4163aa341ff751eba7587", "sha256:7ec465e6549ed97e9f1e5ed51c657c9ede767bc1c11552f7f4d022c4df4a977a",
"sha256:5892cff0a6817743fe470b7402677310ffc8514a74de14d4e591cecdc457ff61", "sha256:82bddf0e72cb2af3cbba7cec1d2fd11fda0de6be8f4492223d4a268713ef2147",
"sha256:60974986aa80a8bb883da5663f90bc632bd4ce0d0508e66a9132412facec65f6", "sha256:82cd34f1081ae4ea2ede3d52f71b7be313756e99b4b5f829f89b12da552d3aa3",
"sha256:62a3c0fdf70f785cd29824666d1dcea88c207f0b73ddbc28fb7a6a1a5bbb1af7", "sha256:843b9c835580d52828d8f69ea4302537337a21e6b4f1ec711a52241ba4a824f3",
"sha256:67ddff0272905a0b78a2c3ea01487e0551cc38094cd5994f73af370f355ecb47", "sha256:877efb968c3d7eb2dad540b6cabf2f1d3c0fbf4b2d309a3c141f79c7e0061324",
"sha256:6af86081c888ce81ca7e361ed7fa2ba1678e2a86eb5a059c96d5a719364d319e", "sha256:8b9f19df998761babaa7f09e6bc169294eefafd6149aaa272081cbddc7ba4ca3",
"sha256:6cdd0fb749c80ffcf408f659b209e82333f10b517786083a3dd3c3b5adc60111", "sha256:8cf5877f7ed384dabfdcc37922c3191bf27e55b498fecece9fd5c2c7aaa34c33",
"sha256:737a4dba9f7ee842c6597f719dda2eabeeaefe42443f7f24de20c262f88527cd", "sha256:8d2900b7f5318bc7ad8631d3d40190b95ef2aa8cc59473b73b294e4a55e9f30f",
"sha256:73e71b95c5215310a92e560369ac1f0e2cd018d5a36be182da88958f3d6084f5", "sha256:8d7b4beebb178e9183138f552238f7e6613162a42164233e2bda00cb3afac58f",
"sha256:745383c124f096cc03fb942c8a05ea1e8cb4f44c5b28887adce6224e4540808e", "sha256:8f52fe6859b9db71ee609b0c0a70fea5f1e71c3462ecf144ca800d3f434f0764",
"sha256:7febf50135363e981097eeada84781eeae92bfc3c203495f63d6b542a7132ba7", "sha256:98f3f020a2b736566c707c8e034945c02aa94e124c24f77ca097c446f81b01f1",
"sha256:821fb791565536617b058c2cd5b8d28a1285b3375eecc5bd6b5c6d87add0d3dd", "sha256:9aa543980ab1fbf1720969af1d99095a548ea42e00361e727c58a40832439114",
"sha256:83ff41e1bc0666f31acda52407d869ea257f232c2d9394806647a0e7454de73e", "sha256:9b99f564659cfa704a2dd82d0684207b1aadf7d02d33e54845f9fc78e06b7581",
"sha256:8991837fdf8086486f1c300d936bacd757e2e5398be84cd54a1fba0e6b6d5591", "sha256:9bcf86dfc8ff3e992fed847c077bd875d9e0ba2fa25d859c3a0f0f76f07f0c8d",
"sha256:8a70c47c14f89b8bfb430f85b608aa460204fe7c005545d79afd31b925cc6669", "sha256:9bd0ae7cc2b85320abd5e0abad5ccee5564ed5f0cc90245d2f9a8ef330a8deae",
"sha256:8ba56c3686fa60cc04191d22d1733aad484c9cbc153cdc3e8eb9bdfcad30f704", "sha256:9d3c0f8567ffe7502d969c2c1b809892dc793b5d0665f602aad19895f8d508da",
"sha256:8de180f748a17382dd695b3999be03a647d09af16ae589c4e9c37138ddb6d4c6", "sha256:9e5ac3437746189a9b4121db2a7b86056ac8786b12e88838696899328fc44bb2",
"sha256:96c2abcbcb24f850f00f279c4660ce6498cae16ff1659845838d752c26890748", "sha256:a36c506e5f8aeb40680491d39ed94670487ce6614b9d27cabe45d94cd5d63e1e",
"sha256:99b5ca5775435aa296d32ea711e194aaae871b21dbf0d57cb7d4431e5d3ad699", "sha256:a5ab722ae5a873d8dcee1f5f45ddd93c34210aed44ff2dc643b5025981908cda",
"sha256:9a4eff4d8ad0bbc9f470a9be19c5e718af4baf47111d7c2d9b036b9986107e7c", "sha256:a96f02ba1bcd330807fc060ed91d1f7a20853da6dd449e5da4b09bfcc08fdcf5",
"sha256:9a99dae826c80cf0a21dd21eb66db16310d1f58ac4c27c804da980212fbcabe2", "sha256:acb6b2f96f60f70e7f34efe0c3ea34ca63f19ca63ce90019c6cbca6b676e81fa",
"sha256:9e8a4782ecaaacf8e9355f39179b1f00e7f27b774306eccbe80f6215724be4cd", "sha256:ae15347a88cf8af0949a9872b57a320d2605ae069bcdf047677318bc0bba45b1",
"sha256:a28eab2d9ea79b830be50e3350d827ae8abf5b23e278e14929824d5ab2069008", "sha256:af8920ce4a55ff41167ddbc20077f5698c2e710ad3353d32a07d3264f3a2021e",
"sha256:ae3a0ec0f1b6cf1e8bca41bc86cd64ba02e31c71716efbf149a0f7ebc168cf0b", "sha256:afd825e30f8d1f521713a5669b63657bcfe5980a916c95855060048b88e1adb7",
"sha256:b005101a257c494e84d36ecb62b02ba195b02b7f8892f57b1f5aaa352ed44467", "sha256:b21b4031b53d25b0858d4e124f2f9131ffc1530431c6d1321805c90da78388d1",
"sha256:b4eef43c5dc5c579d0804e55a32dd1bacbd008c8191ed4d65be278bbb11ddc61", "sha256:b4b68c961b5cc402cbd99cca5eb2547e46ce77260eb705f4d117fd9c3f932b95",
"sha256:b6bb5a0a87ab1e01f086cbb418be9e409719cd216954aa38b1cceee36a561ce1", "sha256:b66aa6357b265670bb574f050ffceefb98549c721cf28351b748be1ef9577d93",
"sha256:b7bb0010c2969c23bf3d2b3892e638a7cb83e7daeb749e3db5f3c002fd191e11", "sha256:b9e240ae0ba96477682aa87899d94ddec1cc7926f9df29b1dd57b39e797d5ab5",
"sha256:b889c0b9be774466308c3590e093ce9a6f9115c78fc8624aa6ba8dfeaf5188ab", "sha256:bc64d1b1dab08f679fb89c368f4c05693f58a9faf744c4d390d7ed1d8223869b",
"sha256:bcded868b05583d41ab0b024f39f90a04e486a2349a9b493d8d17024f1433aa6", "sha256:bf8443781533b8d37b295016a4b53c1494fa9a03573c09ca5104550c138d5c05",
"sha256:c21e60c017cab3a7e7786185cc8853b8614b01ccd69cc8b24608e5356784631b", "sha256:c26aab6ea9c54d3bed716b8851c8bfc40cb249b8e9880e250d1eddde9f709bf5",
"sha256:c32a4fbae218b336d547bc626897325202e4e9f1794ac5f4d0bb3caacf41df21", "sha256:c3cd1fc1dc7c376c54440aeaaa0dcc803d2126732ff5c6b68ccd619f2e64be4f",
"sha256:c4eaa83b610595ef9f20aa69b96053d5b7f3f70c67c7a3af8f433136a9d315ab", "sha256:c7257171bb8d4432fe9d6fdde4d55fdbe663a63636a17f7f9aaba9bcb3153ad7",
"sha256:c5b23f63fcec652bf1e775eca5e03a713a4994d2a7ce2e70a91e964a26220e0d", "sha256:d42e3a3fc18acc88b838efded0e6ec3edf3e328a58c68fbd36a7263a874906c8",
"sha256:cfdac95815d3025e4c9edce2ef2ebe4e034edc35a2c44a606dd846554387ae38", "sha256:d74fcaf87132ffc0447b3c685a9f862ffb5b43e70ea6beec2fb8057d5d2a1fea",
"sha256:d0047c90e0ebd0d8f3c1e6636e10f597b8f25e4ef9e6416dd2e5c4c0960270cc", "sha256:d8c1d679df4361408b628f42b26a5d62bd3e9ba7f0c0e7969f925021554755aa",
"sha256:d05cf827f160340f67c25ce6f271689a844605aa123849f1a80e21c9bd21f00b", "sha256:e856c1c7255c739434489ec9c8aa9cdf5179785d10ff20add308b5d673bed5cd",
"sha256:d64e543b07964ff73b4eb994bee894803a80e19fd3b29a5ffbe3c637fe43e788", "sha256:eac68f96539b32fce2c9b47eb7c25bb2582bdaf1bbb360d25f564ee9e04c542b",
"sha256:d6d3ce5638cd4ed3fa684507f164e7039e1b07475bc8f37ba6e12271c1a2e9e0", "sha256:ed7326563024b6e91fef6b6c7a1a2ff0a71b97793ac33dbbcf38f6005e51ff6e",
"sha256:d80d9f4d986bb6ad65bae86f07391152f7b6c65cfc63d118616b18b0be2e79da", "sha256:ed8c3d2cd329bf779b7ed38db176738f3f8be637bb395ce9629fc76f78afe3d4",
"sha256:dbff288e1869db78f8731ca257553dd699edef07e173b35e71b1122b630d6008", "sha256:f4c9bda132ad108b387c33fabfea47866af87f4ea6ffb79418004f0521e63204",
"sha256:dcb25128c9e7f01c00ad116d2c762c3942724981a35c6e5c551ab55d4c2bfcfe", "sha256:f643ffd2669ffd4b5a3e9b41c909b72b2a1d5e4915da90a77e119b8d48ce867a"
"sha256:dcc7dc4b9b65f185aa8533abc78f0a3b2ac38fe075bb23d3c1590ba0990f6c80",
"sha256:e2388a792f9c239510d62a9e615662b8202e4ca275aafcc9c4af154654462a14",
"sha256:e2bd8faf6a9682ca385e4bca1a38a057be716dc303f16ddec9e4c9cf01b7d722",
"sha256:e97c74725e86d84a477df081eef69b70f048128afee841dbd8c690a9e3d2e8e0",
"sha256:ea5e4b3eff9029a02fe7736540675ab8fca44277232f0027397b0d7111d04b1c",
"sha256:f36c3103a6f2641777b64f1da860f37cbaa718ce3feea621583627269e68eb03",
"sha256:f56e6a38c64a79e228d48344bb3bec265ac035fc1277ce8c049445bb18e4cd41",
"sha256:f9464ff2fd1f2ba4d0d246a560d369ee7e7213c556a30db9ae9426850ce1baf9",
"sha256:feb1102d9e5de08120d46a1011110c43b2547ecb3ae80030801e0e2dacd1ee18",
"sha256:ff29353c12f0abc9cb3395899b7192a970d5a63f80ac1e7f0c3247ed83f5dcd4"
], ],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "markers": "python_version >= '3.6'",
"version": "==5.0.1" "version": "==5.1.0"
}, },
"markupsafe": { "markupsafe": {
"hashes": [ "hashes": [
...@@ -689,6 +695,14 @@ ...@@ -689,6 +695,14 @@
"index": "pypi", "index": "pypi",
"version": "==1.26.3" "version": "==1.26.3"
}, },
"opensearch-py": {
"hashes": [
"sha256:564f175af134aa885f4ced6846eb4532e08b414fff0a7976f76b276fe0e69158",
"sha256:7867319132133e2974c09f76a54eb1d502b989229be52da583d93ddc743ea111"
],
"index": "pypi",
"version": "==2.4.2"
},
"packaging": { "packaging": {
"hashes": [ "hashes": [
"sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5", "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5",
...@@ -753,41 +767,49 @@ ...@@ -753,41 +767,49 @@
}, },
"pycryptodome": { "pycryptodome": {
"hashes": [ "hashes": [
"sha256:08b445799d571041765e7d5c9ca09c5d3866c2f22eeb0dd4394a4169285184f4", "sha256:06d6de87c19f967f03b4cf9b34e538ef46e99a337e9a61a77dbe44b2cbcf0690",
"sha256:11ddf6c9b52116b62223b6a9f4741bc4f62bb265392a4463282f7f34bb287180", "sha256:09609209ed7de61c2b560cc5c8c4fbf892f8b15b1faf7e4cbffac97db1fffda7",
"sha256:1c6273ca5a03b672e504995529b8bae56da0ebb691d8ef141c4aa68f60765700", "sha256:210ba1b647837bfc42dd5a813cdecb5b86193ae11a3f5d972b9a0ae2c7e9e4b4",
"sha256:27e1efcb68993b7ce5d1d047a46a601d41281bba9f1971e6be4aa27c69ab8065", "sha256:2a1250b7ea809f752b68e3e6f3fd946b5939a52eaeea18c73bdab53e9ba3c2dd",
"sha256:2c16426ef49d9cba018be2340ea986837e1dfa25c2ea181787971654dd49aadd", "sha256:2ab6ab0cb755154ad14e507d1df72de9897e99fd2d4922851a276ccc14f4f1a5",
"sha256:37e531bf896b70fe302f003d3be5a0a8697737a8d177967da7e23eff60d6483c", "sha256:3427d9e5310af6680678f4cce149f54e0bb4af60101c7f2c16fdf878b39ccccc",
"sha256:420972f9c62978e852c74055d81c354079ce3c3a2213a92c9d7e37bbc63a26e2", "sha256:3cd3ef3aee1079ae44afaeee13393cf68b1058f70576b11439483e34f93cf818",
"sha256:4464b0e8fd5508bff9baf18e6fd4c6548b1ac2ce9862d6965ff6a84ec9cb302a", "sha256:405002eafad114a2f9a930f5db65feef7b53c4784495dd8758069b89baf68eab",
"sha256:4805e053571140cb37cf153b5c72cd324bb1e3e837cbe590a19f69b6cf85fd03", "sha256:417a276aaa9cb3be91f9014e9d18d10e840a7a9b9a9be64a42f553c5b50b4d1d",
"sha256:67939a3adbe637281c611596e44500ff309d547e932c449337649921b17b6297", "sha256:4401564ebf37dfde45d096974c7a159b52eeabd9969135f0426907db367a652a",
"sha256:694020d2ff985cd714381b9da949a21028c24b86f562526186f6af7c7547e986", "sha256:49a4c4dc60b78ec41d2afa392491d788c2e06edf48580fbfb0dd0f828af49d25",
"sha256:6c3df3613592ea6afaec900fd7189d23c8c28b75b550254f4bd33fe94acb84b9", "sha256:5601c934c498cd267640b57569e73793cb9a83506f7c73a8ec57a516f5b0b091",
"sha256:6d0d2b97758ebf2f36c39060520447c26455acb3bcff309c28b1c816173a6ff5", "sha256:6e0e4a987d38cfc2e71b4a1b591bae4891eeabe5fa0f56154f576e26287bfdea",
"sha256:7c9e222d0976f68d0cf6409cfea896676ddc1d98485d601e9508f90f60e2b0a2", "sha256:76658f0d942051d12a9bd08ca1b6b34fd762a8ee4240984f7c06ddfb55eaf15a",
"sha256:81e9d23c0316fc1b45d984a44881b220062336bbdc340aa9218e8d0656587934", "sha256:76cb39afede7055127e35a444c1c041d2e8d2f1f9c121ecef573757ba4cd2c3c",
"sha256:8ae0dd1bcfada451c35f9e29a3e5db385caabc190f98e4a80ad02a61098fb776", "sha256:8d6b98d0d83d21fb757a182d52940d028564efe8147baa9ce0f38d057104ae72",
"sha256:8cf5d3d6cf921fa81acd1f632f6cedcc03f5f68fc50c364cd39490ba01d17c49", "sha256:9b3ae153c89a480a0ec402e23db8d8d84a3833b65fa4b15b81b83be9d637aab9",
"sha256:954d156cd50130afd53f8d77f830fe6d5801bd23e97a69d358fed068f433fbfe", "sha256:a60fedd2b37b4cb11ccb5d0399efe26db9e0dd149016c1cc6c8161974ceac2d6",
"sha256:a470237ee71a1efd63f9becebc0ad84b88ec28e6784a2047684b693f458f41b7", "sha256:ac1c7c0624a862f2e53438a15c9259d1655325fc2ec4392e66dc46cdae24d044",
"sha256:a991f8ffe8dfe708f86690948ae46442eebdd0fff07dc1b605987939a34ec979", "sha256:acae12b9ede49f38eb0ef76fdec2df2e94aad85ae46ec85be3648a57f0a7db04",
"sha256:b0bfe61506795877ff974f994397f0c862d037f6f1c0bfc3572195fc00833b96", "sha256:acc2614e2e5346a4a4eab6e199203034924313626f9620b7b4b38e9ad74b7e0c",
"sha256:b7efd46b0b4ac869046e814d83244aeab14ef787f4850644119b1c8b0ec2d637", "sha256:acf6e43fa75aca2d33e93409f2dafe386fe051818ee79ee8a3e21de9caa2ac9e",
"sha256:b8b80ff92049fd042177282917d994d344365ab7e8ec2bc03e853d93d2401786", "sha256:baee115a9ba6c5d2709a1e88ffe62b73ecc044852a925dcb67713a288c4ec70f",
"sha256:c1bc0c49d986a1491d66d2a56570f12e960b12508b7e71f2423f532e28857f36", "sha256:c18b381553638414b38705f07d1ef0a7cf301bc78a5f9bc17a957eb19446834b",
"sha256:c22c80246c3c880c6950d2a8addf156cee74ec0dc5757d01e8e7067a3c7da015", "sha256:d29daa681517f4bc318cd8a23af87e1f2a7bad2fe361e8aa29c77d652a065de4",
"sha256:cd4e7e8bf0fc1ada854688b9b309ee607e2aa85a8b44180f91021a4dd330a928", "sha256:d5954acfe9e00bc83ed9f5cb082ed22c592fbbef86dc48b907238be64ead5c33",
"sha256:cd4e95b0eb4b28251c825fe7aa941fe077f993e5ca9b855665935b86fbb1cc08", "sha256:ec0bb1188c1d13426039af8ffcb4dbe3aad1d7680c35a62d8eaf2a529b5d3d4f",
"sha256:e038ab77fec0956d7aa989a3c647652937fc142ef41c9382c2ebd13c127d5b4a", "sha256:ec1f93feb3bb93380ab0ebf8b859e8e5678c0f010d2d78367cf6bc30bfeb148e",
"sha256:e3e6f89480616781d2a7f981472d0cdb09b9da9e8196f43c1234eff45c915766", "sha256:f0e6d631bae3f231d3634f91ae4da7a960f7ff87f2865b2d2b831af1dfb04e9a",
"sha256:e70f5c839c7798743a948efa2a65d1fe96bb397fe6d7f2bde93d869fe4f0ad69", "sha256:f35d6cee81fa145333137009d9c8ba90951d7d77b67c79cbe5f03c7eb74d8fe2",
"sha256:ed932eb6c2b1c4391e166e1a562c9d2f020bfff44a0e1b108f67af38b390ea89", "sha256:f47888542a0633baff535a04726948e876bf1ed880fddb7c10a736fa99146ab3",
"sha256:f34976c5c8eb79e14c7d970fb097482835be8d410a4220f86260695ede4c3e17" "sha256:fb3b87461fa35afa19c971b0a2b7456a7b1db7b4eba9a8424666104925b78128"
], ],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"version": "==3.19.1" "version": "==3.20.0"
},
"pymysql": {
"hashes": [
"sha256:4f13a7df8bf36a51e81dd9f3605fede45a4878fe02f9236349fd82a3f0612f96",
"sha256:8969ec6d763c856f7073c4c64662882675702efcb114b4bcbb955aea3a069fa7"
],
"index": "pypi",
"version": "==1.1.0"
}, },
"python-dateutil": { "python-dateutil": {
"hashes": [ "hashes": [
...@@ -1013,6 +1035,61 @@ ...@@ -1013,6 +1035,61 @@
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==1.16.0" "version": "==1.16.0"
}, },
"sqlalchemy": {
"hashes": [
"sha256:0d3cab3076af2e4aa5693f89622bef7fa770c6fec967143e4da7508b3dceb9b9",
"sha256:0dacf67aee53b16f365c589ce72e766efaabd2b145f9de7c917777b575e3659d",
"sha256:10331f129982a19df4284ceac6fe87353ca3ca6b4ca77ff7d697209ae0a5915e",
"sha256:14a6f68e8fc96e5e8f5647ef6cda6250c780612a573d99e4d881581432ef1669",
"sha256:1b1180cda6df7af84fe72e4530f192231b1f29a7496951db4ff38dac1687202d",
"sha256:29049e2c299b5ace92cbed0c1610a7a236f3baf4c6b66eb9547c01179f638ec5",
"sha256:342d365988ba88ada8af320d43df4e0b13a694dbd75951f537b2d5e4cb5cd002",
"sha256:420362338681eec03f53467804541a854617faed7272fe71a1bfdb07336a381e",
"sha256:4344d059265cc8b1b1be351bfb88749294b87a8b2bbe21dfbe066c4199541ebd",
"sha256:4f7a7d7fcc675d3d85fbf3b3828ecd5990b8d61bd6de3f1b260080b3beccf215",
"sha256:555651adbb503ac7f4cb35834c5e4ae0819aab2cd24857a123370764dc7d7e24",
"sha256:59a21853f5daeb50412d459cfb13cb82c089ad4c04ec208cd14dddd99fc23b39",
"sha256:5fdd402169aa00df3142149940b3bf9ce7dde075928c1886d9a1df63d4b8de62",
"sha256:605b6b059f4b57b277f75ace81cc5bc6335efcbcc4ccb9066695e515dbdb3900",
"sha256:665f0a3954635b5b777a55111ababf44b4fc12b1f3ba0a435b602b6387ffd7cf",
"sha256:6f9e2e59cbcc6ba1488404aad43de005d05ca56e069477b33ff74e91b6319735",
"sha256:736ea78cd06de6c21ecba7416499e7236a22374561493b456a1f7ffbe3f6cdb4",
"sha256:74b080c897563f81062b74e44f5a72fa44c2b373741a9ade701d5f789a10ba23",
"sha256:75432b5b14dc2fff43c50435e248b45c7cdadef73388e5610852b95280ffd0e9",
"sha256:75f99202324383d613ddd1f7455ac908dca9c2dd729ec8584c9541dd41822a2c",
"sha256:790f533fa5c8901a62b6fef5811d48980adeb2f51f1290ade8b5e7ba990ba3de",
"sha256:798f717ae7c806d67145f6ae94dc7c342d3222d3b9a311a784f371a4333212c7",
"sha256:7c88f0c7dcc5f99bdb34b4fd9b69b93c89f893f454f40219fe923a3a2fd11625",
"sha256:7d505815ac340568fd03f719446a589162d55c52f08abd77ba8964fbb7eb5b5f",
"sha256:84daa0a2055df9ca0f148a64fdde12ac635e30edbca80e87df9b3aaf419e144a",
"sha256:87d91043ea0dc65ee583026cb18e1b458d8ec5fc0a93637126b5fc0bc3ea68c4",
"sha256:87f6e732bccd7dcf1741c00f1ecf33797383128bd1c90144ac8adc02cbb98643",
"sha256:884272dcd3ad97f47702965a0e902b540541890f468d24bd1d98bcfe41c3f018",
"sha256:8b8cb63d3ea63b29074dcd29da4dc6a97ad1349151f2d2949495418fd6e48db9",
"sha256:91f7d9d1c4dd1f4f6e092874c128c11165eafcf7c963128f79e28f8445de82d5",
"sha256:a2c69a7664fb2d54b8682dd774c3b54f67f84fa123cf84dda2a5f40dcaa04e08",
"sha256:a3be4987e3ee9d9a380b66393b77a4cd6d742480c951a1c56a23c335caca4ce3",
"sha256:a86b4240e67d4753dc3092d9511886795b3c2852abe599cffe108952f7af7ac3",
"sha256:aa9373708763ef46782d10e950b49d0235bfe58facebd76917d3f5cbf5971aed",
"sha256:b64b183d610b424a160b0d4d880995e935208fc043d0302dd29fee32d1ee3f95",
"sha256:b801154027107461ee992ff4b5c09aa7cc6ec91ddfe50d02bca344918c3265c6",
"sha256:bb209a73b8307f8fe4fe46f6ad5979649be01607f11af1eb94aa9e8a3aaf77f0",
"sha256:bc8b7dabe8e67c4832891a5d322cec6d44ef02f432b4588390017f5cec186a84",
"sha256:c51db269513917394faec5e5c00d6f83829742ba62e2ac4fa5c98d58be91662f",
"sha256:c55731c116806836a5d678a70c84cb13f2cedba920212ba7dcad53260997666d",
"sha256:cf18ff7fc9941b8fc23437cc3e68ed4ebeff3599eec6ef5eebf305f3d2e9a7c2",
"sha256:d24f571990c05f6b36a396218f251f3e0dda916e0c687ef6fdca5072743208f5",
"sha256:db854730a25db7c956423bb9fb4bdd1216c839a689bf9cc15fada0a7fb2f4570",
"sha256:dc55990143cbd853a5d038c05e79284baedf3e299661389654551bd02a6a68d7",
"sha256:e607cdd99cbf9bb80391f54446b86e16eea6ad309361942bf88318bcd452363c",
"sha256:ecf6d4cda1f9f6cb0b45803a01ea7f034e2f1aed9475e883410812d9f9e3cfcf",
"sha256:f2a159111a0f58fb034c93eeba211b4141137ec4b0a6e75789ab7a3ef3c7e7e3",
"sha256:f37c0caf14b9e9b9e8f6dbc81bc56db06acb4363eba5a633167781a48ef036ed",
"sha256:f5693145220517b5f42393e07a6898acdfe820e136c98663b971906120549da5"
],
"markers": "python_version >= '3.7'",
"version": "==2.0.25"
},
"typing-extensions": { "typing-extensions": {
"hashes": [ "hashes": [
"sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783", "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783",
...@@ -1031,11 +1108,11 @@ ...@@ -1031,11 +1108,11 @@
}, },
"urllib3": { "urllib3": {
"hashes": [ "hashes": [
"sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84", "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07",
"sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e" "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"
], ],
"markers": "python_version >= '3.7'", "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
"version": "==2.0.7" "version": "==1.26.18"
}, },
"webencodings": { "webencodings": {
"hashes": [ "hashes": [
...@@ -1060,6 +1137,14 @@ ...@@ -1060,6 +1137,14 @@
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
"version": "==2.0.1" "version": "==2.0.1"
}, },
"zipp": {
"hashes": [
"sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31",
"sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0"
],
"markers": "python_version >= '3.8'",
"version": "==3.17.0"
},
"zope.event": { "zope.event": {
"hashes": [ "hashes": [
"sha256:2832e95014f4db26c47a13fdaef84cef2f4df37e66b59d8f1f4a8f319a632c26", "sha256:2832e95014f4db26c47a13fdaef84cef2f4df37e66b59d8f1f4a8f319a632c26",
...@@ -1306,7 +1391,7 @@ ...@@ -1306,7 +1391,7 @@
"sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519", "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519",
"sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561" "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==3.3.2" "version": "==3.3.2"
}, },
"coverage": { "coverage": {
...@@ -1375,6 +1460,78 @@ ...@@ -1375,6 +1460,78 @@
"markers": "python_version >= '3.8'", "markers": "python_version >= '3.8'",
"version": "==7.0.0" "version": "==7.0.0"
}, },
"exceptiongroup": {
"hashes": [
"sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14",
"sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"
],
"index": "pypi",
"version": "==1.2.0"
},
"greenlet": {
"hashes": [
"sha256:01bc7ea167cf943b4c802068e178bbf70ae2e8c080467070d01bfa02f337ee67",
"sha256:0448abc479fab28b00cb472d278828b3ccca164531daab4e970a0458786055d6",
"sha256:086152f8fbc5955df88382e8a75984e2bb1c892ad2e3c80a2508954e52295257",
"sha256:098d86f528c855ead3479afe84b49242e174ed262456c342d70fc7f972bc13c4",
"sha256:149e94a2dd82d19838fe4b2259f1b6b9957d5ba1b25640d2380bea9c5df37676",
"sha256:1551a8195c0d4a68fac7a4325efac0d541b48def35feb49d803674ac32582f61",
"sha256:15d79dd26056573940fcb8c7413d84118086f2ec1a8acdfa854631084393efcc",
"sha256:1996cb9306c8595335bb157d133daf5cf9f693ef413e7673cb07e3e5871379ca",
"sha256:1a7191e42732df52cb5f39d3527217e7ab73cae2cb3694d241e18f53d84ea9a7",
"sha256:1ea188d4f49089fc6fb283845ab18a2518d279c7cd9da1065d7a84e991748728",
"sha256:1f672519db1796ca0d8753f9e78ec02355e862d0998193038c7073045899f305",
"sha256:2516a9957eed41dd8f1ec0c604f1cdc86758b587d964668b5b196a9db5bfcde6",
"sha256:2797aa5aedac23af156bbb5a6aa2cd3427ada2972c828244eb7d1b9255846379",
"sha256:2dd6e660effd852586b6a8478a1d244b8dc90ab5b1321751d2ea15deb49ed414",
"sha256:3ddc0f794e6ad661e321caa8d2f0a55ce01213c74722587256fb6566049a8b04",
"sha256:3ed7fb269f15dc662787f4119ec300ad0702fa1b19d2135a37c2c4de6fadfd4a",
"sha256:419b386f84949bf0e7c73e6032e3457b82a787c1ab4a0e43732898a761cc9dbf",
"sha256:43374442353259554ce33599da8b692d5aa96f8976d567d4badf263371fbe491",
"sha256:52f59dd9c96ad2fc0d5724107444f76eb20aaccb675bf825df6435acb7703559",
"sha256:57e8974f23e47dac22b83436bdcf23080ade568ce77df33159e019d161ce1d1e",
"sha256:5b51e85cb5ceda94e79d019ed36b35386e8c37d22f07d6a751cb659b180d5274",
"sha256:649dde7de1a5eceb258f9cb00bdf50e978c9db1b996964cd80703614c86495eb",
"sha256:64d7675ad83578e3fc149b617a444fab8efdafc9385471f868eb5ff83e446b8b",
"sha256:68834da854554926fbedd38c76e60c4a2e3198c6fbed520b106a8986445caaf9",
"sha256:6b66c9c1e7ccabad3a7d037b2bcb740122a7b17a53734b7d72a344ce39882a1b",
"sha256:70fb482fdf2c707765ab5f0b6655e9cfcf3780d8d87355a063547b41177599be",
"sha256:7170375bcc99f1a2fbd9c306f5be8764eaf3ac6b5cb968862cad4c7057756506",
"sha256:73a411ef564e0e097dbe7e866bb2dda0f027e072b04da387282b02c308807405",
"sha256:77457465d89b8263bca14759d7c1684df840b6811b2499838cc5b040a8b5b113",
"sha256:7f362975f2d179f9e26928c5b517524e89dd48530a0202570d55ad6ca5d8a56f",
"sha256:81bb9c6d52e8321f09c3d165b2a78c680506d9af285bfccbad9fb7ad5a5da3e5",
"sha256:881b7db1ebff4ba09aaaeae6aa491daeb226c8150fc20e836ad00041bcb11230",
"sha256:894393ce10ceac937e56ec00bb71c4c2f8209ad516e96033e4b3b1de270e200d",
"sha256:99bf650dc5d69546e076f413a87481ee1d2d09aaaaaca058c9251b6d8c14783f",
"sha256:9da2bd29ed9e4f15955dd1595ad7bc9320308a3b766ef7f837e23ad4b4aac31a",
"sha256:afaff6cf5200befd5cec055b07d1c0a5a06c040fe5ad148abcd11ba6ab9b114e",
"sha256:b1b5667cced97081bf57b8fa1d6bfca67814b0afd38208d52538316e9422fc61",
"sha256:b37eef18ea55f2ffd8f00ff8fe7c8d3818abd3e25fb73fae2ca3b672e333a7a6",
"sha256:b542be2440edc2d48547b5923c408cbe0fc94afb9f18741faa6ae970dbcb9b6d",
"sha256:b7dcbe92cc99f08c8dd11f930de4d99ef756c3591a5377d1d9cd7dd5e896da71",
"sha256:b7f009caad047246ed379e1c4dbcb8b020f0a390667ea74d2387be2998f58a22",
"sha256:bba5387a6975598857d86de9eac14210a49d554a77eb8261cc68b7d082f78ce2",
"sha256:c5e1536de2aad7bf62e27baf79225d0d64360d4168cf2e6becb91baf1ed074f3",
"sha256:c5ee858cfe08f34712f548c3c363e807e7186f03ad7a5039ebadb29e8c6be067",
"sha256:c9db1c18f0eaad2f804728c67d6c610778456e3e1cc4ab4bbd5eeb8e6053c6fc",
"sha256:d353cadd6083fdb056bb46ed07e4340b0869c305c8ca54ef9da3421acbdf6881",
"sha256:d46677c85c5ba00a9cb6f7a00b2bfa6f812192d2c9f7d9c4f6a55b60216712f3",
"sha256:d4d1ac74f5c0c0524e4a24335350edad7e5f03b9532da7ea4d3c54d527784f2e",
"sha256:d73a9fe764d77f87f8ec26a0c85144d6a951a6c438dfe50487df5595c6373eac",
"sha256:da70d4d51c8b306bb7a031d5cff6cc25ad253affe89b70352af5f1cb68e74b53",
"sha256:daf3cb43b7cf2ba96d614252ce1684c1bccee6b2183a01328c98d36fcd7d5cb0",
"sha256:dca1e2f3ca00b84a396bc1bce13dd21f680f035314d2379c4160c98153b2059b",
"sha256:dd4f49ae60e10adbc94b45c0b5e6a179acc1736cf7a90160b404076ee283cf83",
"sha256:e1f145462f1fa6e4a4ae3c0f782e580ce44d57c8f2c7aae1b6fa88c0b2efdb41",
"sha256:e3391d1e16e2a5a1507d83e4a8b100f4ee626e8eca43cf2cadb543de69827c4c",
"sha256:fcd2469d6a2cf298f198f0487e0a5b1a47a42ca0fa4dfd1b6862c999f018ebbf",
"sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da",
"sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33"
],
"index": "pypi",
"version": "==3.0.3"
},
"idna": { "idna": {
"hashes": [ "hashes": [
"sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca", "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca",
...@@ -1399,6 +1556,14 @@ ...@@ -1399,6 +1556,14 @@
"index": "pypi", "index": "pypi",
"version": "==7.2.3" "version": "==7.2.3"
}, },
"opensearch-py": {
"hashes": [
"sha256:564f175af134aa885f4ced6846eb4532e08b414fff0a7976f76b276fe0e69158",
"sha256:7867319132133e2974c09f76a54eb1d502b989229be52da583d93ddc743ea111"
],
"index": "pypi",
"version": "==2.4.2"
},
"packaging": { "packaging": {
"hashes": [ "hashes": [
"sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5", "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5",
...@@ -1424,41 +1589,49 @@ ...@@ -1424,41 +1589,49 @@
}, },
"pycryptodome": { "pycryptodome": {
"hashes": [ "hashes": [
"sha256:08b445799d571041765e7d5c9ca09c5d3866c2f22eeb0dd4394a4169285184f4", "sha256:06d6de87c19f967f03b4cf9b34e538ef46e99a337e9a61a77dbe44b2cbcf0690",
"sha256:11ddf6c9b52116b62223b6a9f4741bc4f62bb265392a4463282f7f34bb287180", "sha256:09609209ed7de61c2b560cc5c8c4fbf892f8b15b1faf7e4cbffac97db1fffda7",
"sha256:1c6273ca5a03b672e504995529b8bae56da0ebb691d8ef141c4aa68f60765700", "sha256:210ba1b647837bfc42dd5a813cdecb5b86193ae11a3f5d972b9a0ae2c7e9e4b4",
"sha256:27e1efcb68993b7ce5d1d047a46a601d41281bba9f1971e6be4aa27c69ab8065", "sha256:2a1250b7ea809f752b68e3e6f3fd946b5939a52eaeea18c73bdab53e9ba3c2dd",
"sha256:2c16426ef49d9cba018be2340ea986837e1dfa25c2ea181787971654dd49aadd", "sha256:2ab6ab0cb755154ad14e507d1df72de9897e99fd2d4922851a276ccc14f4f1a5",
"sha256:37e531bf896b70fe302f003d3be5a0a8697737a8d177967da7e23eff60d6483c", "sha256:3427d9e5310af6680678f4cce149f54e0bb4af60101c7f2c16fdf878b39ccccc",
"sha256:420972f9c62978e852c74055d81c354079ce3c3a2213a92c9d7e37bbc63a26e2", "sha256:3cd3ef3aee1079ae44afaeee13393cf68b1058f70576b11439483e34f93cf818",
"sha256:4464b0e8fd5508bff9baf18e6fd4c6548b1ac2ce9862d6965ff6a84ec9cb302a", "sha256:405002eafad114a2f9a930f5db65feef7b53c4784495dd8758069b89baf68eab",
"sha256:4805e053571140cb37cf153b5c72cd324bb1e3e837cbe590a19f69b6cf85fd03", "sha256:417a276aaa9cb3be91f9014e9d18d10e840a7a9b9a9be64a42f553c5b50b4d1d",
"sha256:67939a3adbe637281c611596e44500ff309d547e932c449337649921b17b6297", "sha256:4401564ebf37dfde45d096974c7a159b52eeabd9969135f0426907db367a652a",
"sha256:694020d2ff985cd714381b9da949a21028c24b86f562526186f6af7c7547e986", "sha256:49a4c4dc60b78ec41d2afa392491d788c2e06edf48580fbfb0dd0f828af49d25",
"sha256:6c3df3613592ea6afaec900fd7189d23c8c28b75b550254f4bd33fe94acb84b9", "sha256:5601c934c498cd267640b57569e73793cb9a83506f7c73a8ec57a516f5b0b091",
"sha256:6d0d2b97758ebf2f36c39060520447c26455acb3bcff309c28b1c816173a6ff5", "sha256:6e0e4a987d38cfc2e71b4a1b591bae4891eeabe5fa0f56154f576e26287bfdea",
"sha256:7c9e222d0976f68d0cf6409cfea896676ddc1d98485d601e9508f90f60e2b0a2", "sha256:76658f0d942051d12a9bd08ca1b6b34fd762a8ee4240984f7c06ddfb55eaf15a",
"sha256:81e9d23c0316fc1b45d984a44881b220062336bbdc340aa9218e8d0656587934", "sha256:76cb39afede7055127e35a444c1c041d2e8d2f1f9c121ecef573757ba4cd2c3c",
"sha256:8ae0dd1bcfada451c35f9e29a3e5db385caabc190f98e4a80ad02a61098fb776", "sha256:8d6b98d0d83d21fb757a182d52940d028564efe8147baa9ce0f38d057104ae72",
"sha256:8cf5d3d6cf921fa81acd1f632f6cedcc03f5f68fc50c364cd39490ba01d17c49", "sha256:9b3ae153c89a480a0ec402e23db8d8d84a3833b65fa4b15b81b83be9d637aab9",
"sha256:954d156cd50130afd53f8d77f830fe6d5801bd23e97a69d358fed068f433fbfe", "sha256:a60fedd2b37b4cb11ccb5d0399efe26db9e0dd149016c1cc6c8161974ceac2d6",
"sha256:a470237ee71a1efd63f9becebc0ad84b88ec28e6784a2047684b693f458f41b7", "sha256:ac1c7c0624a862f2e53438a15c9259d1655325fc2ec4392e66dc46cdae24d044",
"sha256:a991f8ffe8dfe708f86690948ae46442eebdd0fff07dc1b605987939a34ec979", "sha256:acae12b9ede49f38eb0ef76fdec2df2e94aad85ae46ec85be3648a57f0a7db04",
"sha256:b0bfe61506795877ff974f994397f0c862d037f6f1c0bfc3572195fc00833b96", "sha256:acc2614e2e5346a4a4eab6e199203034924313626f9620b7b4b38e9ad74b7e0c",
"sha256:b7efd46b0b4ac869046e814d83244aeab14ef787f4850644119b1c8b0ec2d637", "sha256:acf6e43fa75aca2d33e93409f2dafe386fe051818ee79ee8a3e21de9caa2ac9e",
"sha256:b8b80ff92049fd042177282917d994d344365ab7e8ec2bc03e853d93d2401786", "sha256:baee115a9ba6c5d2709a1e88ffe62b73ecc044852a925dcb67713a288c4ec70f",
"sha256:c1bc0c49d986a1491d66d2a56570f12e960b12508b7e71f2423f532e28857f36", "sha256:c18b381553638414b38705f07d1ef0a7cf301bc78a5f9bc17a957eb19446834b",
"sha256:c22c80246c3c880c6950d2a8addf156cee74ec0dc5757d01e8e7067a3c7da015", "sha256:d29daa681517f4bc318cd8a23af87e1f2a7bad2fe361e8aa29c77d652a065de4",
"sha256:cd4e7e8bf0fc1ada854688b9b309ee607e2aa85a8b44180f91021a4dd330a928", "sha256:d5954acfe9e00bc83ed9f5cb082ed22c592fbbef86dc48b907238be64ead5c33",
"sha256:cd4e95b0eb4b28251c825fe7aa941fe077f993e5ca9b855665935b86fbb1cc08", "sha256:ec0bb1188c1d13426039af8ffcb4dbe3aad1d7680c35a62d8eaf2a529b5d3d4f",
"sha256:e038ab77fec0956d7aa989a3c647652937fc142ef41c9382c2ebd13c127d5b4a", "sha256:ec1f93feb3bb93380ab0ebf8b859e8e5678c0f010d2d78367cf6bc30bfeb148e",
"sha256:e3e6f89480616781d2a7f981472d0cdb09b9da9e8196f43c1234eff45c915766", "sha256:f0e6d631bae3f231d3634f91ae4da7a960f7ff87f2865b2d2b831af1dfb04e9a",
"sha256:e70f5c839c7798743a948efa2a65d1fe96bb397fe6d7f2bde93d869fe4f0ad69", "sha256:f35d6cee81fa145333137009d9c8ba90951d7d77b67c79cbe5f03c7eb74d8fe2",
"sha256:ed932eb6c2b1c4391e166e1a562c9d2f020bfff44a0e1b108f67af38b390ea89", "sha256:f47888542a0633baff535a04726948e876bf1ed880fddb7c10a736fa99146ab3",
"sha256:f34976c5c8eb79e14c7d970fb097482835be8d410a4220f86260695ede4c3e17" "sha256:fb3b87461fa35afa19c971b0a2b7456a7b1db7b4eba9a8424666104925b78128"
], ],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"version": "==3.19.1" "version": "==3.20.0"
},
"pymysql": {
"hashes": [
"sha256:4f13a7df8bf36a51e81dd9f3605fede45a4878fe02f9236349fd82a3f0612f96",
"sha256:8969ec6d763c856f7073c4c64662882675702efcb114b4bcbb955aea3a069fa7"
],
"index": "pypi",
"version": "==1.1.0"
}, },
"pytest": { "pytest": {
"hashes": [ "hashes": [
...@@ -1468,6 +1641,14 @@ ...@@ -1468,6 +1641,14 @@
"index": "pypi", "index": "pypi",
"version": "==7.4.4" "version": "==7.4.4"
}, },
"python-dateutil": {
"hashes": [
"sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
"sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==2.8.2"
},
"requests": { "requests": {
"hashes": [ "hashes": [
"sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
...@@ -1476,6 +1657,69 @@ ...@@ -1476,6 +1657,69 @@
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==2.31.0" "version": "==2.31.0"
}, },
"six": {
"hashes": [
"sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
"sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==1.16.0"
},
"sqlalchemy": {
"hashes": [
"sha256:0d3cab3076af2e4aa5693f89622bef7fa770c6fec967143e4da7508b3dceb9b9",
"sha256:0dacf67aee53b16f365c589ce72e766efaabd2b145f9de7c917777b575e3659d",
"sha256:10331f129982a19df4284ceac6fe87353ca3ca6b4ca77ff7d697209ae0a5915e",
"sha256:14a6f68e8fc96e5e8f5647ef6cda6250c780612a573d99e4d881581432ef1669",
"sha256:1b1180cda6df7af84fe72e4530f192231b1f29a7496951db4ff38dac1687202d",
"sha256:29049e2c299b5ace92cbed0c1610a7a236f3baf4c6b66eb9547c01179f638ec5",
"sha256:342d365988ba88ada8af320d43df4e0b13a694dbd75951f537b2d5e4cb5cd002",
"sha256:420362338681eec03f53467804541a854617faed7272fe71a1bfdb07336a381e",
"sha256:4344d059265cc8b1b1be351bfb88749294b87a8b2bbe21dfbe066c4199541ebd",
"sha256:4f7a7d7fcc675d3d85fbf3b3828ecd5990b8d61bd6de3f1b260080b3beccf215",
"sha256:555651adbb503ac7f4cb35834c5e4ae0819aab2cd24857a123370764dc7d7e24",
"sha256:59a21853f5daeb50412d459cfb13cb82c089ad4c04ec208cd14dddd99fc23b39",
"sha256:5fdd402169aa00df3142149940b3bf9ce7dde075928c1886d9a1df63d4b8de62",
"sha256:605b6b059f4b57b277f75ace81cc5bc6335efcbcc4ccb9066695e515dbdb3900",
"sha256:665f0a3954635b5b777a55111ababf44b4fc12b1f3ba0a435b602b6387ffd7cf",
"sha256:6f9e2e59cbcc6ba1488404aad43de005d05ca56e069477b33ff74e91b6319735",
"sha256:736ea78cd06de6c21ecba7416499e7236a22374561493b456a1f7ffbe3f6cdb4",
"sha256:74b080c897563f81062b74e44f5a72fa44c2b373741a9ade701d5f789a10ba23",
"sha256:75432b5b14dc2fff43c50435e248b45c7cdadef73388e5610852b95280ffd0e9",
"sha256:75f99202324383d613ddd1f7455ac908dca9c2dd729ec8584c9541dd41822a2c",
"sha256:790f533fa5c8901a62b6fef5811d48980adeb2f51f1290ade8b5e7ba990ba3de",
"sha256:798f717ae7c806d67145f6ae94dc7c342d3222d3b9a311a784f371a4333212c7",
"sha256:7c88f0c7dcc5f99bdb34b4fd9b69b93c89f893f454f40219fe923a3a2fd11625",
"sha256:7d505815ac340568fd03f719446a589162d55c52f08abd77ba8964fbb7eb5b5f",
"sha256:84daa0a2055df9ca0f148a64fdde12ac635e30edbca80e87df9b3aaf419e144a",
"sha256:87d91043ea0dc65ee583026cb18e1b458d8ec5fc0a93637126b5fc0bc3ea68c4",
"sha256:87f6e732bccd7dcf1741c00f1ecf33797383128bd1c90144ac8adc02cbb98643",
"sha256:884272dcd3ad97f47702965a0e902b540541890f468d24bd1d98bcfe41c3f018",
"sha256:8b8cb63d3ea63b29074dcd29da4dc6a97ad1349151f2d2949495418fd6e48db9",
"sha256:91f7d9d1c4dd1f4f6e092874c128c11165eafcf7c963128f79e28f8445de82d5",
"sha256:a2c69a7664fb2d54b8682dd774c3b54f67f84fa123cf84dda2a5f40dcaa04e08",
"sha256:a3be4987e3ee9d9a380b66393b77a4cd6d742480c951a1c56a23c335caca4ce3",
"sha256:a86b4240e67d4753dc3092d9511886795b3c2852abe599cffe108952f7af7ac3",
"sha256:aa9373708763ef46782d10e950b49d0235bfe58facebd76917d3f5cbf5971aed",
"sha256:b64b183d610b424a160b0d4d880995e935208fc043d0302dd29fee32d1ee3f95",
"sha256:b801154027107461ee992ff4b5c09aa7cc6ec91ddfe50d02bca344918c3265c6",
"sha256:bb209a73b8307f8fe4fe46f6ad5979649be01607f11af1eb94aa9e8a3aaf77f0",
"sha256:bc8b7dabe8e67c4832891a5d322cec6d44ef02f432b4588390017f5cec186a84",
"sha256:c51db269513917394faec5e5c00d6f83829742ba62e2ac4fa5c98d58be91662f",
"sha256:c55731c116806836a5d678a70c84cb13f2cedba920212ba7dcad53260997666d",
"sha256:cf18ff7fc9941b8fc23437cc3e68ed4ebeff3599eec6ef5eebf305f3d2e9a7c2",
"sha256:d24f571990c05f6b36a396218f251f3e0dda916e0c687ef6fdca5072743208f5",
"sha256:db854730a25db7c956423bb9fb4bdd1216c839a689bf9cc15fada0a7fb2f4570",
"sha256:dc55990143cbd853a5d038c05e79284baedf3e299661389654551bd02a6a68d7",
"sha256:e607cdd99cbf9bb80391f54446b86e16eea6ad309361942bf88318bcd452363c",
"sha256:ecf6d4cda1f9f6cb0b45803a01ea7f034e2f1aed9475e883410812d9f9e3cfcf",
"sha256:f2a159111a0f58fb034c93eeba211b4141137ec4b0a6e75789ab7a3ef3c7e7e3",
"sha256:f37c0caf14b9e9b9e8f6dbc81bc56db06acb4363eba5a633167781a48ef036ed",
"sha256:f5693145220517b5f42393e07a6898acdfe820e136c98663b971906120549da5"
],
"markers": "python_version >= '3.7'",
"version": "==2.0.25"
},
"testcontainers-core": { "testcontainers-core": {
"hashes": [ "hashes": [
"sha256:69a8bf2ddb52ac2d03c26401b12c70db0453cced40372ad783d6dce417e52095" "sha256:69a8bf2ddb52ac2d03c26401b12c70db0453cced40372ad783d6dce417e52095"
...@@ -1490,6 +1734,28 @@ ...@@ -1490,6 +1734,28 @@
"index": "pypi", "index": "pypi",
"version": "==0.0.1rc1" "version": "==0.0.1rc1"
}, },
"testcontainers-mysql": {
"hashes": [
"sha256:d22894e0d8c7b4f7424afef99f713aa7e7a19ff987b7723aed863b9c478a2c91"
],
"index": "pypi",
"version": "==0.0.1rc1"
},
"testcontainers-opensearch": {
"hashes": [
"sha256:0bdf270b5b7f53915832f7c31dd2bd3ffdc20b534ea6b32231cc7003049bd0e1"
],
"index": "pypi",
"version": "==0.0.1rc1"
},
"tomli": {
"hashes": [
"sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc",
"sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"
],
"markers": "python_version < '3.11'",
"version": "==2.0.1"
},
"typing-extensions": { "typing-extensions": {
"hashes": [ "hashes": [
"sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783", "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783",
...@@ -1500,11 +1766,11 @@ ...@@ -1500,11 +1766,11 @@
}, },
"urllib3": { "urllib3": {
"hashes": [ "hashes": [
"sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84", "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07",
"sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e" "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"
], ],
"markers": "python_version >= '3.7'", "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
"version": "==2.0.7" "version": "==1.26.18"
}, },
"wrapt": { "wrapt": {
"hashes": [ "hashes": [
......
dbrepo-analyse-service/app.py
+ 122
− 69
View file @ a132a136
import os import logging
from _csv import Error from _csv import Error
from flask import Flask, request, Response
from json import dumps from json import dumps
from determine_dt import determine_datatypes from logging.config import dictConfig
from determine_pk import determine_pk
import logging from flasgger import LazyJSONEncoder, Swagger
from flasgger import Swagger
from flasgger.utils import swag_from from flasgger.utils import swag_from
from flasgger import LazyJSONEncoder from flask import Flask, Response, abort, request
from flask_sqlalchemy import SQLAlchemy
from gevent.pywsgi import WSGIServer from gevent.pywsgi import WSGIServer
from opensearchpy import OpenSearch
from prometheus_flask_exporter import PrometheusMetrics from prometheus_flask_exporter import PrometheusMetrics
from requests import get
log_level = os.getenv('LOG_LEVEL', 'INFO') from determine_dt import determine_datatypes
from determine_pk import determine_pk
from determine_stats import determine_stats
logging.basicConfig(level=logging.getLevelName(log_level)) logging.basicConfig(level=logging.DEBUG)
from logging.config import dictConfig dictConfig(
{
dictConfig({ "version": 1,
'version': 1, "formatters": {
'formatters': {'default': { "default": {
'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s', "format": "[%(asctime)s] %(levelname)s in %(module)s: %(message)s",
}}, }
'handlers': {'wsgi': { },
'class': 'logging.StreamHandler', "handlers": {
'stream': 'ext://flask.logging.wsgi_errors_stream', "wsgi": {
'formatter': 'default' "class": "logging.StreamHandler",
}}, "stream": "ext://flask.logging.wsgi_errors_stream",
'root': { "formatter": "default",
'level': log_level,
'handlers': ['wsgi']
} }
}) },
"root": {"level": "INFO", "handlers": ["wsgi"]},
}
)
app = Flask(__name__) app = Flask(__name__)
metrics = PrometheusMetrics(app) metrics = PrometheusMetrics(app)
metrics.info('app_info', 'Application info', version='1.3.0') metrics.info("app_info", "Application info", version="1.3.0")
app.config["SWAGGER"] = {"openapi": "3.0.1", "title": "Swagger UI", "uiversion": 3} app.config["SWAGGER"] = {"openapi": "3.0.1", "title": "Swagger UI", "uiversion": 3}
# ========================= DB Config ========================= #
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
app.config[
"SQLALCHEMY_DATABASE_URI"
] = "mysql+pymysql://root:dbrepo@metadata-db:3306/fda"
db = SQLAlchemy(app)
# ========================= OS Config ========================= #
opensearch_client = OpenSearch(
hosts=["search-db"],
port=9200,
http_auth=("admin", "admin"),
use_ssl=False,
)
swagger_config = { swagger_config = {
"headers": [], "headers": [],
"specs": [ "specs": [
{ {
"endpoint": "api-analyse", "endpoint": "api-analyse",
"route": "/api-analyse.json", "route": "/api-analyse.json",
"rule_filter": lambda rule: rule.endpoint.startswith('analyze'), "rule_filter": lambda rule: rule.endpoint.startswith("analyze"),
"model_filter": lambda tag: True, # all in "model_filter": lambda tag: True, # all in
}, },
{ {
"endpoint": "api-mdb", "endpoint": "api-mdb",
"route": "/api-mdb.json", "route": "/api-mdb.json",
"rule_filter": lambda rule: rule.endpoint.startswith('mdb'), "rule_filter": lambda rule: rule.endpoint.startswith("mdb"),
"model_filter": lambda tag: True, # all in "model_filter": lambda tag: True, # all in
}, },
], ],
...@@ -68,97 +89,129 @@ template = { ...@@ -68,97 +89,129 @@ template = {
"version": "$TAG", "version": "$TAG",
"contact": { "contact": {
"name": "Prof. Andreas Rauber", "name": "Prof. Andreas Rauber",
"email": "andreas.rauber@tuwien.ac.at" "email": "andreas.rauber@tuwien.ac.at",
}, },
"license": { "license": {
"name": "Apache 2.0", "name": "Apache 2.0",
"url": "https://www.apache.org/licenses/LICENSE-2.0" "url": "https://www.apache.org/licenses/LICENSE-2.0",
}, },
}, },
"externalDocs": { "externalDocs": {
"description": "Sourcecode Documentation", "description": "Sourcecode Documentation",
"url": "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services" "url": "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services",
}, },
"servers": [ "servers": [
{ {"url": "http://localhost:5000", "description": "Generated server url"},
"url": "http://localhost:5000", {"url": "https://dbrepo2.tuwien.ac.at", "description": "Sandbox"},
"description": "Generated server url" ],
},
{
"url": "https://dbrepo2.tuwien.ac.at",
"description": "Sandbox"
}
]
} }
app.json_encoder = LazyJSONEncoder app.json_encoder = LazyJSONEncoder
swagger = Swagger(app, config=swagger_config, template=template) swagger = Swagger(app, config=swagger_config, template=template)
@app.route('/health', methods=["GET"], endpoint='analyze_health')
@swag_from('as-yml/health.yml') @app.route("/health", methods=["GET"], endpoint="analyze_health")
@swag_from("as-yml/health.yml")
def health(): def health():
logging.debug('endpoint health, body=%s', request) logging.debug("endpoint health, body=%s", request)
res = dumps({"status": "UP", "message": "Application is up and running"}) res = dumps({"status": "UP", "message": "Application is up and running"})
return Response(res, mimetype="application/json"), 200 return Response(res, mimetype="application/json"), 200
@app.route('/api/analyse/determinedt', methods=["POST"], endpoint='analyze_determinedt') @app.route("/api/analyse/determinedt", methods=["POST"], endpoint="analyze_determinedt")
@swag_from('as-yml/determinedt.yml') @swag_from("as-yml/determinedt.yml")
def determinedt(): def determinedt():
logging.debug('endpoint determine datatype, body=%s', request) logging.debug("endpoint determine datatype, body=%s", request)
input_json = request.get_json() input_json = request.get_json()
try: try:
filename = str(input_json['filename']) filename = str(input_json["filename"])
enum = False enum = False
if 'enum' in input_json: if "enum" in input_json:
enum = bool(input_json['enum']) enum = bool(input_json["enum"])
logging.info("Enum is present in payload and set to %s", enum) logging.info("Enum is present in payload and set to %s", enum)
enum_tol = 0.001 enum_tol = 0.001
if 'enum_tol' in input_json: if "enum_tol" in input_json:
enum_tol = float(input_json['enum_tol']) enum_tol = float(input_json["enum_tol"])
logging.info("Enum toleration is present in payload and set to %s", enum_tol) logging.info(
"Enum toleration is present in payload and set to %s", enum_tol
)
separator = None separator = None
if 'separator' in input_json: if "separator" in input_json:
separator = str(input_json['separator']) separator = str(input_json["separator"])
logging.info("Seperator is present in payload and set to %s", separator) logging.info("Seperator is present in payload and set to %s", separator)
res = determine_datatypes(filename, enum, enum_tol, separator) res = determine_datatypes(filename, enum, enum_tol, separator)
logging.debug('determine datatype resulted in datatypes %s', res) logging.debug("determine datatype resulted in datatypes %s", res)
return Response(res, mimetype="application/json"), 200 return Response(res, mimetype="application/json"), 200
except OSError as e: except OSError as e:
logging.error('Failed to determine data types: %s', e) logging.error("Failed to determine data types: %s", e)
res = dumps({"success": False, "message": str(e)}) res = dumps({"success": False, "message": str(e)})
return Response(res, mimetype="application/json"), 409 return Response(res, mimetype="application/json"), 409
except Error as e: except Error as e:
logging.error('Failed to determine separator %s', e) logging.error("Failed to determine separator %s", e)
res = dumps({"success": False, "message": str(e)}) res = dumps({"success": False, "message": str(e)})
return Response(res, mimetype="application/json"), 422 return Response(res, mimetype="application/json"), 422
except Exception as e: except Exception as e:
logging.error('Failed to determine data types: %s', e) logging.error("Failed to determine data types: %s", e)
res = dumps({"success": False, "message": str(e)}) res = dumps({"success": False, "message": str(e)})
return Response(res, mimetype="application/json"), 500 return Response(res, mimetype="application/json"), 500
@app.route('/api/analyse/determinepk', methods=["POST"], endpoint='analyze_determinepk') @app.route("/api/analyse/determinepk", methods=["POST"], endpoint="analyze_determinepk")
@swag_from('as-yml/determinepk.yml') @swag_from("as-yml/determinepk.yml")
def determinepk(): def determinepk():
logging.debug('endpoint determine primary key, body=%s', request) logging.debug("endpoint determine primary key, body=%s", request)
input_json = request.get_json() input_json = request.get_json()
try: try:
filepath = str(input_json['filepath']) filepath = str(input_json["filepath"])
seperator = ',' seperator = ","
if 'seperator' in input_json: if "seperator" in input_json:
seperator = str(input_json['seperator']) seperator = str(input_json["seperator"])
res = determine_pk(filepath, seperator) res = determine_pk(filepath, seperator)
logging.debug('determined list of primary keys: %s', res) logging.debug("determined list of primary keys: %s", res)
return Response(res, mimetype="application/json"), 200 return Response(res, mimetype="application/json"), 200
except Exception as e: except Exception as e:
logging.error('Failed to determine primary key: %s', e) logging.error("Failed to determine primary key: %s", e)
res = dumps({"success": False, "message": str(e)}) res = dumps({"success": False, "message": str(e)})
return Response(res, mimetype="application/json"), 500 return Response(res, mimetype="application/json"), 500
@app.route("/api/analyse/determineStats", methods=["POST"], endpoint="analyze_stats")
def determineStats():
logging.debug(
"endpoint to determine the statistical properties, body = %s", request
)
input_json = request.get_json()
if "filepath" not in input_json:
return {"message": "Missing 'filepath'", "status": 400}, 400
filepath = str(input_json["filepath"])
separator = str(input_json.get("separator", ","))
return determine_stats(filepath, separator)
@app.route("/api/analyse/determineStat", methods=["POST"])
def determineStat():
input_json = request.get_json()
if "database_id" not in input_json:
return {"message": "Missing 'database_id'", "status": 400}, 400
if "table_id" not in input_json:
return {"message": "Missing 'table_id'", "status": 400}, 400
res = determine_stats(
db,
opensearch_client,
database_id=input_json["database_id"],
table_id=input_json["table_id"],
)
if res:
return {"message": "Analysed statistical properties.", "status": 200}
else:
return {"message": "Database or table does not exist.", "status": 400}, 400
rest_server_port = 5000 rest_server_port = 5000
if __name__ == '__main__': if __name__ == "__main__":
http_server = WSGIServer(('', 5000), app) http_server = WSGIServer(("", 5000), app)
http_server.serve_forever() http_server.serve_forever()
dbrepo-analyse-service/build.sh
+ 4
− 0
View file @ a132a136
#!/bin/bash #!/bin/bash
# needed for MariaDB Connector/C
apt update && apt install -y curl gcc libmariadb3 libmariadb-dev
python3 -m venv ./dbrepo-analyse-service/venv python3 -m venv ./dbrepo-analyse-service/venv
source ./dbrepo-analyse-service/venv/bin/activate source ./dbrepo-analyse-service/venv/bin/activate
PIPENV_PIPFILE=./dbrepo-analyse-service/Pipfile pipenv install --dev PIPENV_PIPFILE=./dbrepo-analyse-service/Pipfile pipenv install --dev
\ No newline at end of file
dbrepo-analyse-service/config.py
+ 1
− 1
View file @ a132a136
class Config: class Config:
EUREKA_SERVER = os.environ.get('EUREKA_SERVER') EUREKA_SERVER = os.environ.get("EUREKA_SERVER")
dbrepo-analyse-service/data/test_stats/test_stats_01.csv 0 → 100644
+ 10
− 0
View file @ a132a136
"col1","col2","col3","col4","col5"
"2021-02-01 00:00:00","Zch_Stampfenbachstrasse",0.44,"150.25","10"
"2021-04-13 00:00:00","Zch_Stampfenbachstrasse",4.88,"201.5","11"
"2021-04-01 00:00:00","Zch_Stampfenbachstrasse",29.46,"180.75","12"
"2021-01-01 00:00:00","Zch_Stampfenbachstrasse",9.85,"170.15","13"
"2016-06-01 00:00:00","Zch_Stampfenbachstrasse",41.24,"160.35","14"
"2015-01-01 00:00:00","Zch_Stampfenbachstrasse",8.51,"140.14","15"
"2022-07-01 00:00:00","Zch_Stampfenbachstrasse",88.34,"130.45","16"
"2022-01-01 00:00:00","Zch_Stampfenbachstrasse",75.72,"120.55","17"
"2023-09-01 00:00:00","Zch_Schimmelstrasse",41.66,"110.295","18"
dbrepo-analyse-service/determine_dt.py
+ 26
− 12
View file @ a132a136
...@@ -9,8 +9,13 @@ import io ...@@ -9,8 +9,13 @@ import io
from clients.s3_client import S3Client from clients.s3_client import S3Client
import messytables, pandas as pd import messytables, pandas as pd
from messytables import CSVTableSet, type_guess, \ from messytables import (
headers_guess, headers_processor, offset_processor CSVTableSet,
type_guess,
headers_guess,
headers_processor,
offset_processor,
)
def determine_datatypes(filename, enum=False, enum_tol=0.0001, separator=None) -> {}: def determine_datatypes(filename, enum=False, enum_tol=0.0001, separator=None) -> {}:
...@@ -25,16 +30,16 @@ def determine_datatypes(filename, enum=False, enum_tol=0.0001, separator=None) - ...@@ -25,16 +30,16 @@ def determine_datatypes(filename, enum=False, enum_tol=0.0001, separator=None) -
logging.warning(f'Failed to determine data types: file {filename} has empty body') logging.warning(f'Failed to determine data types: file {filename} has empty body')
return json.dumps({'columns': [], 'separator': ','}) return json.dumps({'columns': [], 'separator': ','})
if separator is None: if separator is None:
logging.info('Attempt to guess separator for from first line') logging.info("Attempt to guess separator for from first line")
with io.BytesIO(stream.read()) as fh: with io.BytesIO(stream.read()) as fh:
line = next(fh) line = next(fh)
dialect = csv.Sniffer().sniff(line.decode('utf-8')) dialect = csv.Sniffer().sniff(line.decode("utf-8"))
separator = dialect.delimiter separator = dialect.delimiter
logging.info('determined separator: %s', separator) logging.info("determined separator: %s", separator)
# Load a file object: # Load a file object:
with io.BytesIO(stream.read()) as fh: with io.BytesIO(stream.read()) as fh:
logging.info('Analysing corpus with separator: %s', separator) logging.info("Analysing corpus with separator: %s", separator)
table_set = CSVTableSet(fh, delimiter=separator) table_set = CSVTableSet(fh, delimiter=separator)
# A table set is a collection of tables: # A table set is a collection of tables:
...@@ -58,18 +63,27 @@ def determine_datatypes(filename, enum=False, enum_tol=0.0001, separator=None) - ...@@ -58,18 +63,27 @@ def determine_datatypes(filename, enum=False, enum_tol=0.0001, separator=None) -
elif type(types[i]) == messytables.types.IntegerType: elif type(types[i]) == messytables.types.IntegerType:
r[headers[i]] = "bigint" r[headers[i]] = "bigint"
elif type(types[i]) == messytables.types.DateType: elif type(types[i]) == messytables.types.DateType:
if "%H" in types[i].format or "%M" in types[i].format or "%S" in types[i].format or "%Z" in types[ if (
i].format: "%H" in types[i].format
r[headers[i]] = "timestamp" # todo: guesses date format too, return it or "%M" in types[i].format
or "%S" in types[i].format
or "%Z" in types[i].format
):
r[
headers[i]
] = "timestamp" # todo: guesses date format too, return it
else: else:
r[headers[i]] = "date" r[headers[i]] = "date"
elif type(types[i]) == messytables.types.DecimalType or type(types[i]) == messytables.types.FloatType: elif (
type(types[i]) == messytables.types.DecimalType
or type(types[i]) == messytables.types.FloatType
):
r[headers[i]] = "decimal" r[headers[i]] = "decimal"
elif type(types[i]) == messytables.types.StringType: elif type(types[i]) == messytables.types.StringType:
r[headers[i]] = "varchar" r[headers[i]] = "varchar"
else: else:
r[headers[i]] = "text" r[headers[i]] = "text"
fh.close() fh.close()
s = {'columns': r, 'separator': separator} s = {"columns": r, "separator": separator}
logging.info('Determined data types %s', s) logging.info("Determined data types %s", s)
return json.dumps(s) return json.dumps(s)
dbrepo-analyse-service/determine_pk.py
+ 26
− 12
View file @ a132a136
...@@ -8,9 +8,9 @@ from determine_dt import determine_datatypes ...@@ -8,9 +8,9 @@ from determine_dt import determine_datatypes
from clients.s3_client import S3Client from clients.s3_client import S3Client
def determine_pk(filename, separator=','): def determine_pk(filename, separator=","):
dt = json.loads(determine_datatypes(filename=filename, separator=separator)) dt = json.loads(determine_datatypes(filename=filename, separator=separator))
dt = {k.lower(): v for k, v in dt['columns'].items()} dt = {k.lower(): v for k, v in dt["columns"].items()}
# {k.lower(): v for k, v in dt['columns'].items() if v != 'Numeric'} # {k.lower(): v for k, v in dt['columns'].items() if v != 'Numeric'}
colnames = dt.keys() colnames = dt.keys()
colindex = list(range(0, len(colnames))) colindex = list(range(0, len(colnames)))
...@@ -27,35 +27,49 @@ def determine_pk(filename, separator=','): ...@@ -27,35 +27,49 @@ def determine_pk(filename, separator=','):
pk = {} pk = {}
j = 0 j = 0
k = 0 k = 0
logging.info(f'File is {sizeInKb}kb, detection is precise') logging.info(f"File is {sizeInKb}kb, detection is precise")
for item in colnames: for item in colnames:
if item == 'id': if item == "id":
j = j + 1 j = j + 1
pk.update({item: j}) pk.update({item: j})
colindex.remove(k) colindex.remove(k)
k = k + 1 k = k + 1
csvdata = pd.read_csv(stream, sep=separator) csvdata = pd.read_csv(stream, sep=separator)
for i in colindex: for i in colindex:
if pd.Series(csvdata.iloc[:, i]).is_unique and pd.Series(csvdata.iloc[:, i]).notnull().values.any(): if (
pd.Series(csvdata.iloc[:, i]).is_unique
and pd.Series(csvdata.iloc[:, i]).notnull().values.any()
):
j = j + 1 j = j + 1
pk.update({list(colnames)[i]: j}) pk.update({list(colnames)[i]: j})
else: # stochastic pk determination else: # stochastic pk determination
pk = {} pk = {}
j = 0 j = 0
k = 0 k = 0
logging.info(f'File is {sizeInKb}kB (larger than threshold of 400kB), detection is stochastic') logging.info(
f"File is {sizeInKb}kB (larger than threshold of 400kB), detection is stochastic"
)
for item in colnames: for item in colnames:
if item == 'id': if item == "id":
j = j + 1 j = j + 1
pk.update({item: j}) pk.update({item: j})
colindex.remove(k) colindex.remove(k)
k = k + 1 k = k + 1
p = np.log10(int(response['ContentLength'])) # logarithmic scaled percentage of random inspected rows p = np.log10(
csvdata = pd.read_csv(filepath_or_buffer=stream, sep=separator, header=0, int(response["ContentLength"])
skiprows=lambda k: k > 0 and random.random() > p) ) # logarithmic scaled percentage of random inspected rows
csvdata = pd.read_csv(
filepath_or_buffer=stream,
sep=separator,
header=0,
skiprows=lambda k: k > 0 and random.random() > p,
)
for i in colindex: for i in colindex:
if pd.Series(csvdata.iloc[:, i]).is_unique and pd.Series(csvdata.iloc[:, i]).notnull().values.any(): if (
pd.Series(csvdata.iloc[:, i]).is_unique
and pd.Series(csvdata.iloc[:, i]).notnull().values.any()
):
j = j + 1 j = j + 1
pk.update({list(colnames)[i]: j}) pk.update({list(colnames)[i]: j})
logging.info(f'Determined primary key {pk}') logging.info(f"Determined primary key {pk}")
return json.dumps(pk) return json.dumps(pk)
dbrepo-analyse-service/determine_stats.py 0 → 100644
+ 96
− 0
View file @ a132a136
from pandas import DataFrame
from sqlalchemy import create_engine, text
def determine_stats(db, os, **kwargs):
database_id = kwargs.get("database_id")
table_id = kwargs.get("table_id")
with db.engine.connect() as connection:
database_name = connection.execute(
text(f"SELECT internal_name FROM mdb_databases WHERE id={database_id}")
).fetchone()[0]
table_name = connection.execute(
text(f"SELECT internal_name FROM mdb_tables WHERE id={table_id}")
).fetchone()[0]
if not database_name or not table_name:
return False
data_db_host = kwargs.get("data_db_host", "data-db")
data_db_port = kwargs.get("data_db_port", 3306)
# Generate data db connection on the fly: database name is varying according to the id given
data_db_uri = (
f"mysql+pymysql://root:dbrepo@{data_db_host}:{data_db_port}/{database_name}"
)
data_db_engine = create_engine(data_db_uri)
with data_db_engine.connect() as connection:
result = connection.execute(text(f"SELECT * FROM {table_name}"))
rows = result.fetchall()
df = DataFrame(rows, columns=result.keys())
for column, dtype in df.dtypes.items():
# Check if the column has a numeric data type
if dtype.kind in "fi":
# Calculate the statistics for the current column
column_stats = {
"val_min": df[column].min(),
"val_max": df[column].max(),
"mean": df[column].mean(),
"median": df[column].median(),
"std_dev": df[column].std(),
}
# Store statistical properties to the metadata db and index to OS
# TODO: use prepared statements to eliminate SQL injection
update_query = text(
f"""
UPDATE mdb_columns
SET
val_min = '{column_stats["val_min"]}',
val_max = '{column_stats["val_max"]}',
mean = '{column_stats["mean"]}',
median = '{column_stats["median"]}',
std_dev = '{column_stats["std_dev"]}'
WHERE
tID = '{table_id}' AND internal_name = '{column}'
"""
)
# We need an extra select query to fetch the column ID for OpenSearch
select_query = text(
f"""
SELECT id
FROM mdb_columns
WHERE tID = '{table_id}' AND internal_name = '{column}'
"""
)
with db.engine.begin() as connection:
connection.execute(update_query)
result = connection.execute(select_query)
column_id = result.fetchone()[0]
connection.commit()
# Fetch the existing document
existing_document = os.get(index="database", id=database_id)["_source"]
# Loop over OS response and append the statistics for each column
for tidx, table in enumerate(existing_document["tables"]):
if table["id"] == table_id:
for cidx, column in enumerate(table["columns"]):
if column["id"] == column_id:
existing_document["tables"][tidx]["columns"][cidx].update(
column_stats
)
# No need to keep searching if column id matches
break
# Index and force refresh
os.index(
index="database",
id=database_id,
body=existing_document,
refresh=True,
)
return True
dbrepo-analyse-service/pywsgi.py
+ 2
− 1
View file @ a132a136
from gevent import monkey from gevent import monkey
monkey.patch_all() monkey.patch_all()
import os import os
from gevent.pywsgi import WSGIServer from gevent.pywsgi import WSGIServer
from app import app from app import app
http_server = WSGIServer(('0.0.0.0', int(os.environ['PORT_APP'])), app) http_server = WSGIServer(("0.0.0.0", int(os.environ["PORT_APP"])), app)
http_server.serve_forever() http_server.serve_forever()
dbrepo-analyse-service/test/conftest.py
+ 100
− 7
View file @ a132a136
...@@ -2,9 +2,12 @@ import os ...@@ -2,9 +2,12 @@ import os
import pytest import pytest
import logging import logging
import json
from minio.deleteobjects import DeleteObject from minio.deleteobjects import DeleteObject
from testcontainers.minio import MinioContainer from testcontainers.minio import MinioContainer
from testcontainers.mysql import MySqlContainer
from testcontainers.opensearch import OpenSearchContainer
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
...@@ -19,13 +22,18 @@ def session(request): ...@@ -19,13 +22,18 @@ def session(request):
logging.debug("[fixture] starting container") logging.debug("[fixture] starting container")
container.start() container.start()
# set the environment for the client # set the environment for the client
endpoint = 'http://' + container.get_container_host_ip() + ':' + container.get_exposed_port(9000) endpoint = (
os.environ['S3_STORAGE_ENDPOINT'] = endpoint "http://"
+ container.get_container_host_ip()
+ ":"
+ container.get_exposed_port(9000)
)
os.environ["S3_STORAGE_ENDPOINT"] = endpoint
client = container.get_client() client = container.get_client()
# create buckets # create buckets
logging.debug('[fixture] make buckets dbrepo-upload, dbrepo-download') logging.debug("[fixture] make buckets dbrepo-upload, dbrepo-download")
client.make_bucket('dbrepo-upload') client.make_bucket("dbrepo-upload")
client.make_bucket('dbrepo-download') client.make_bucket("dbrepo-download")
# destructor # destructor
def stop_minio(): def stop_minio():
...@@ -48,7 +56,92 @@ def cleanup(request, session): ...@@ -48,7 +56,92 @@ def cleanup(request, session):
objects = [] objects = []
for obj in session.get_client().list_objects(bucket): for obj in session.get_client().list_objects(bucket):
objects.append(DeleteObject(obj.object_name)) objects.append(DeleteObject(obj.object_name))
logging.info(f'request to remove objects {objects}') logging.info(f"request to remove objects {objects}")
errors = session.get_client().remove_objects(bucket, objects) errors = session.get_client().remove_objects(bucket, objects)
for error in errors: for error in errors:
raise ConnectionError(f'Failed to delete object with key {error.object_name} of bucket {bucket}') raise ConnectionError(
f"Failed to delete object with key {error.object_name} of bucket {bucket}"
)
@pytest.fixture(scope="function")
def metadata_db_container():
metadata_db = MySqlContainer(
"bitnami/mariadb:10.5",
MYSQL_USER="root",
MYSQL_PASSWORD="dbrepo",
MYSQL_DATABASE="fda",
)
metadata_db._name = "metadata-db-test"
metadata_db.ports = {"3306": 33060}
metadata_db.env = {
"MYSQL_USER": metadata_db.MYSQL_USER,
"MARIADB_ROOT_PASSWORD": metadata_db.MYSQL_ROOT_PASSWORD,
"MARIADB_DATABASE": metadata_db.MYSQL_DATABASE,
}
# volume that mounts db schema from metadata-db
metadata_db.with_volume_mapping(
os.path.abspath("../dbrepo-metadata-db/setup-schema.sql"), "/schema.sql"
)
# volume for script that initializes schema and inserts test values
metadata_db.with_volume_mapping(
os.path.abspath("./test/init-db.sh"),
"/docker-entrypoint-initdb.d/init-db.sh",
)
# validate creation of schema and data
with metadata_db:
print(
metadata_db.exec(
f"mariadb -u{metadata_db.MYSQL_USER} -p{metadata_db.MYSQL_ROOT_PASSWORD} fda -e 'SELECT * FROM mdb_databases;'"
)
)
yield metadata_db
@pytest.fixture(scope="function")
def data_db_container():
data_db = MySqlContainer(
"bitnami/mariadb:10.5",
MYSQL_USER="root",
MYSQL_PASSWORD="dbrepo",
)
data_db._name = "data-db-test"
data_db.ports = {"3306": 33061}
data_db.env = {
"MYSQL_USER": data_db.MYSQL_USER,
"MARIADB_ROOT_PASSWORD": data_db.MYSQL_ROOT_PASSWORD,
}
# volume that mounts csv for data import
data_db.with_volume_mapping(
os.path.abspath("./data/test_stats/test_stats_01.csv"), "/test_stats_01.csv"
)
# volume for script to create a test data db and import values from a csv
data_db.with_volume_mapping(
os.path.abspath("./test/init-data-db.sh"),
"/docker-entrypoint-initdb.d/init-data-db.sh",
)
with data_db:
yield data_db
@pytest.fixture(scope="function")
def opensearch_container():
os_container = OpenSearchContainer("opensearchproject/opensearch:2.10.0")
with os_container:
client = os_container.get_client()
index_mapping_path = os.path.join(
"..", "dbrepo-search-db", "init", "indices", "database.json"
)
with open(index_mapping_path, "r") as file:
mapping = json.load(file)
client.indices.create(index="database", body=mapping)
yield os_container
@pytest.fixture(scope="function")
def all_containers(opensearch_container, metadata_db_container, data_db_container):
yield opensearch_container, metadata_db_container, data_db_container
dbrepo-analyse-service/test/init-data-db.sh 0 → 100755
+ 25
− 0
View file @ a132a136
#!/bin/bash
#
# Script to initialize a test data db according to a csv file
# Intented to be run from pytest inside a mysql/mariadb container
mysql -u"$MYSQL_USER" -p"$MYSQL_ROOT_PASSWORD" <<EOSQL
CREATE DATABASE internal1;
USE internal1;
CREATE TABLE table1 (
col1 DATETIME,
col2 VARCHAR(255),
col3 FLOAT,
col4 FLOAT,
col5 INT
);
LOAD DATA INFILE '/test_stats_01.csv'
INTO TABLE table1
FIELDS TERMINATED BY ',' ENCLOSED BY '"'
LINES TERMINATED BY '\\n'
IGNORE 1 LINES
(col1, col2, col3, col4, col5);
EOSQL
dbrepo-analyse-service/test/init-db.sh 0 → 100755
+ 41
− 0
View file @ a132a136
#!/bin/bash
#
# Script to initialize db schema and insert test values
# Intented to be run from pytest inside a mysql/mariadb docker container
mysql -u"$MYSQL_USER" -p"$MYSQL_ROOT_PASSWORD" "$MYSQL_DATABASE" <<EOSQL
source /schema.sql
INSERT INTO mdb_users (id, username, firstname, lastname, email, orcid, affiliation, mariadb_password, theme_dark)
VALUES
('1', 'user1', 'John', 'Doe', 'user1@example.com', '0000-0001-1234-5678', 'University A', 'password1', TRUE);
INSERT INTO mdb_images (name, version, default_port, dialect, driver_class, jdbc_method, created, last_modified)
VALUES
('Image1', '1.0', 5432, 'postgresql', 'org.postgresql.Driver', 'jdbc:postgresql', NOW(), NOW()),
('Image2', '2.0', 3306, 'mysql', 'com.mysql.jdbc.Driver', 'jdbc:mysql', NOW(), NOW());
INSERT INTO mdb_containers (internal_name, name, host, port, ui_host, ui_port, ui_additional_flags, sidecar_host, sidecar_port, image_id, created, last_modified, privileged_username, privileged_password)
VALUES
('container1', 'Container1', 'localhost', 3306, 'localhost', 3306, NULL, 'localhost', 3305, 1, NOW(), NOW(), 'admin', 'admin'),
('container2', 'Container2', 'localhost', 3307, 'localhost', 3307, NULL, 'localhost', 3306, 2, NOW(), NOW(), 'admin', 'admin');
INSERT INTO mdb_databases (cid, name, internal_name, exchange_name, description, engine, is_public, created_by, owned_by, contact_person, created, last_modified)
VALUES
(1, 'Database1', 'internal1', 'Exchange1', 'Description1', 'InnoDB', TRUE, NULL, NULL, NULL, NOW(), NOW()),
(2, 'Database2', 'internal2', 'Exchange2', 'Description2', 'MyISAM', FALSE, NULL, NULL, NULL, NOW(), NOW());
INSERT INTO mdb_tables (tDBID, internal_name, queue_name, routing_key, tName, tDescription, num_rows, data_length, max_data_length, avg_row_length, \`separator\`, quote, element_null, skip_lines, element_true, element_false, Version, created, versioned, created_by, owned_by, last_modified)
VALUES
(1, 'table1', 'Queue1', 'RoutingKey1', 'TableName1', 'TableDescription1', 5, 9, 15, 3, ',', '"', NULL, 2, TRUE, FALSE, '1.0', NOW(), TRUE, '1', '1', NOW()),
(1, 'table2', 'Queue2', 'RoutingKey2', 'TableName2', 'TableDescription2', 0, 0, 0, 0, ',', "'", 'NA', 0, '1', '0', '2.0', NOW(), TRUE, '1', '1', NOW()),
(2, 'table3', 'Queue3', 'RoutingKey3', 'TableName3', 'TableDescription3', 0, 0, 0, 0, ',', '"', 'EMPTY', 5, 'yes', 'no', '1.0', NOW(), TRUE, '1', '1', NOW());
INSERT INTO mdb_columns (tID, dfID, cName, internal_name, alias, Datatype, length, ordinal_position, is_primary_key, index_length, size, d, auto_generated, is_null_allowed, val_min, val_max, mean, median, std_dev, created, last_modified)
VALUES
(1, NULL, 'Column1', 'col1', 'Alias1', 'DATETIME', 255, 1, TRUE, NULL, NULL, NULL, FALSE, TRUE, NULL, NULL, NULL, NULL, NULL, NOW(), NOW()),
(1, NULL, 'Column2', 'col2', 'Alias2', 'VARCHAR', NULL, 2, FALSE, NULL, NULL, NULL, FALSE, TRUE, NULL, NULL, NULL, NULL, NULL, NOW(), NOW()),
(1, NULL, 'Column3', 'col3', 'Alias3', 'FLOAT', 10, 3, FALSE, NULL, NULL, 2, FALSE, TRUE, 0, 100, NULL, NULL, NULL, NOW(), NOW()),
(1, NULL, 'Column4', 'col4', 'Alias4', 'FLOAT', 10, 3, FALSE, NULL, NULL, 2, FALSE, TRUE, 0, 100, NULL, NULL, NULL, NOW(), NOW()),
(1, NULL, 'Column5', 'col5', 'Alias5', 'INT', 10, 3, FALSE, NULL, NULL, 2, FALSE, TRUE, 0, 100, NULL, NULL, NULL, NOW(), NOW());
EOSQL
dbrepo-analyse-service/test/test_determine_dt.py
+ 44
− 50
View file @ a132a136
...@@ -15,20 +15,19 @@ from determine_dt import determine_datatypes ...@@ -15,20 +15,19 @@ from determine_dt import determine_datatypes
class DetermineDatatypesTest(unittest.TestCase): class DetermineDatatypesTest(unittest.TestCase):
# @Test # @Test
def test_determine_datatypesDateTime_succeeds(self): def test_determine_datatypesDateTime_succeeds(self):
exp = { exp = {
'columns': { "columns": {
'Datum': 'timestamp', "Datum": "timestamp",
'Standort': 'varchar', "Standort": "varchar",
'Parameter': 'varchar', "Parameter": "varchar",
'Intervall': 'varchar', "Intervall": "varchar",
'Einheit': 'varchar', "Einheit": "varchar",
'Wert': 'decimal', "Wert": "decimal",
'Status': 'varchar' "Status": "varchar",
}, },
'separator': ',' "separator": ",",
} }
# mock # mock
...@@ -41,16 +40,16 @@ class DetermineDatatypesTest(unittest.TestCase): ...@@ -41,16 +40,16 @@ class DetermineDatatypesTest(unittest.TestCase):
# @Test # @Test
def test_determine_datatypesDateTimeWithTimezone_succeeds(self): def test_determine_datatypesDateTimeWithTimezone_succeeds(self):
exp = { exp = {
'columns': { "columns": {
'Datum': 'varchar', "Datum": "varchar",
'Standort': 'varchar', "Standort": "varchar",
'Parameter': 'varchar', "Parameter": "varchar",
'Intervall': 'varchar', "Intervall": "varchar",
'Einheit': 'varchar', "Einheit": "varchar",
'Wert': 'decimal', "Wert": "decimal",
'Status': 'varchar' "Status": "varchar",
}, },
'separator': ',' "separator": ",",
} }
# mock # mock
...@@ -63,16 +62,16 @@ class DetermineDatatypesTest(unittest.TestCase): ...@@ -63,16 +62,16 @@ class DetermineDatatypesTest(unittest.TestCase):
# @Test # @Test
def test_determine_datatypesDateTimeWithT_succeeds(self): def test_determine_datatypesDateTimeWithT_succeeds(self):
exp = { exp = {
'columns': { "columns": {
'Datum': 'timestamp', "Datum": "timestamp",
'Standort': 'varchar', "Standort": "varchar",
'Parameter': 'varchar', "Parameter": "varchar",
'Intervall': 'varchar', "Intervall": "varchar",
'Einheit': 'varchar', "Einheit": "varchar",
'Wert': 'decimal', "Wert": "decimal",
'Status': 'varchar' "Status": "varchar",
}, },
'separator': ',' "separator": ",",
} }
# mock # mock
...@@ -85,16 +84,16 @@ class DetermineDatatypesTest(unittest.TestCase): ...@@ -85,16 +84,16 @@ class DetermineDatatypesTest(unittest.TestCase):
# @Test # @Test
def test_determine_datatypes_succeeds(self): def test_determine_datatypes_succeeds(self):
exp = { exp = {
'columns': { "columns": {
'int': 'bigint', "int": "bigint",
'float': 'decimal', "float": "decimal",
'string': 'varchar', "string": "varchar",
'boolean': 'bool', "boolean": "bool",
'date': 'date', "date": "date",
'time': 'timestamp', "time": "timestamp",
'enum': 'varchar' # currently not used "enum": "varchar", # currently not used
}, },
'separator': ',' "separator": ",",
} }
# mock # mock
...@@ -106,62 +105,57 @@ class DetermineDatatypesTest(unittest.TestCase): ...@@ -106,62 +105,57 @@ class DetermineDatatypesTest(unittest.TestCase):
# @Test # @Test
def test_determine_datatypes_fileDoesNotExist_fails(self): def test_determine_datatypes_fileDoesNotExist_fails(self):
# test # test
try: try:
response = determine_datatypes("i_do_not_exist.csv") response = determine_datatypes("i_do_not_exist.csv")
except ClientError as e: except ClientError as e:
pass # s3.head operation pass # s3.head operation
except Exception: except Exception:
self.fail('unexpected exception raised') self.fail("unexpected exception raised")
else: else:
self.fail('ExpectedException not raised') self.fail("ExpectedException not raised")
# @Test # @Test
def test_determine_datatypes_fileEmpty_succeeds(self): def test_determine_datatypes_fileEmpty_succeeds(self):
# mock # mock
S3Client().upload_file("empty.csv", './data/test_dt/', 'dbrepo-upload') S3Client().upload_file("empty.csv", './data/test_dt/', 'dbrepo-upload')
# test # test
response = determine_datatypes("empty.csv") response = determine_datatypes("empty.csv")
data = json.loads(response) data = json.loads(response)
self.assertEqual(data['columns'], []) self.assertEqual(data["columns"], [])
self.assertEqual(data['separator'], ',') self.assertEqual(data["separator"], ",")
# @Test # @Test
def test_determine_datatypes_separatorSemicolon_succeeds(self): def test_determine_datatypes_separatorSemicolon_succeeds(self):
# mock # mock
S3Client().upload_file("separator.csv", './data/test_dt/', 'dbrepo-upload') S3Client().upload_file("separator.csv", './data/test_dt/', 'dbrepo-upload')
# test # test
response = determine_datatypes(filename="separator.csv", separator=";") response = determine_datatypes(filename="separator.csv", separator=";")
data = json.loads(response) data = json.loads(response)
self.assertEqual(data['separator'], ';') self.assertEqual(data["separator"], ";")
# @Test # @Test
def test_determine_datatypes_separatorGuess_succeeds(self): def test_determine_datatypes_separatorGuess_succeeds(self):
# mock # mock
S3Client().upload_file("separator.csv", './data/test_dt/', 'dbrepo-upload') S3Client().upload_file("separator.csv", './data/test_dt/', 'dbrepo-upload')
# test # test
response = determine_datatypes(filename="separator.csv") response = determine_datatypes(filename="separator.csv")
data = json.loads(response) data = json.loads(response)
self.assertEqual(data['separator'], ';') self.assertEqual(data["separator"], ";")
# @Test # @Test
def test_determine_datatypes_separatorGuessLargeDataset_succeeds(self): def test_determine_datatypes_separatorGuessLargeDataset_succeeds(self):
# mock # mock
S3Client().upload_file("large.csv", './data/test_dt/', 'dbrepo-upload') S3Client().upload_file("large.csv", './data/test_dt/', 'dbrepo-upload')
# test # test
response = determine_datatypes(filename="large.csv") response = determine_datatypes(filename="large.csv")
data = json.loads(response) data = json.loads(response)
self.assertEqual(data['separator'], ',') self.assertEqual(data["separator"], ",")
if __name__ == '__main__': if __name__ == "__main__":
unittest.main() unittest.main()
dbrepo-analyse-service/test/test_determine_pk.py
+ 0
− 1
View file @ a132a136
...@@ -14,7 +14,6 @@ from clients.s3_client import S3Client ...@@ -14,7 +14,6 @@ from clients.s3_client import S3Client
from determine_pk import determine_pk from determine_pk import determine_pk
class DeterminePrimaryKeyTest(unittest.TestCase): class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test # @Test
def test_determine_pk_largeFileIdFirst_succeeds(self): def test_determine_pk_largeFileIdFirst_succeeds(self):
......
dbrepo-analyse-service/test/test_determine_stats.py 0 → 100644
+ 157
− 0
View file @ a132a136
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jan 17 17:13:25 2024
@author: Sotiris Tsepelakis
"""
from opensearchpy import OpenSearch
from sqlalchemy import create_engine
from determine_stats import determine_stats
class TestDetermineStatisticalProperties:
# @Test
def test_determine_statistical_properties_succeeds(self, all_containers):
# mock request
payload = {
"database_id": 1,
"table_id": 1,
"data_db_host": "localhost",
"data_db_port": 33061,
}
os_host = all_containers[0].get_container_host_ip()
os_port = all_containers[0].get_exposed_port("9200/tcp")
os = OpenSearch([{"host": os_host, "port": os_port}])
metadata_db_host = all_containers[1].get_container_host_ip()
metadata_db_port = all_containers[1].get_exposed_port(3306)
db = create_engine(
f"mysql+pymysql://root:dbrepo@{metadata_db_host}:{metadata_db_port}/fda"
)
# index a test document
all_containers[0].get_client().index(
index="database",
id="1",
body={
"id": 1,
"name": "testdb",
"exchange_name": "dbrepo",
"internal_name": "testdb_ygvq",
"tables": [
{
"id": 1,
"database_id": 1,
"name": "mytable",
"internal_name": "mytable",
"queue_name": "dbrepo",
"routing_key": "dbrepo.testdb_ygvq.mytable",
"columns": [
{
"id": 1,
"database_id": 1,
"table_id": 1,
"name": "id",
"internal_name": "id",
"auto_generated": True,
"is_primary_key": True,
"column_type": "BIGINT",
"is_public": True,
"is_null_allowed": False,
"enums": [],
"sets": [],
},
{
"id": 2,
"database_id": 1,
"table_id": 1,
"name": "col1",
"internal_name": "col1",
"date_format": {
"id": 2,
"database_format": "%Y-%c-%d %H:%i:%S",
"unix_format": "yyyy-MM-dd HH:mm:ss",
"has_time": True,
"created_at": "2024-01-17T13:22:26.000Z",
},
"auto_generated": False,
"is_primary_key": False,
"column_type": "DATETIME",
"is_public": True,
"is_null_allowed": True,
"enums": [],
"sets": [],
},
{
"id": 3,
"database_id": 1,
"table_id": 1,
"name": "col2",
"internal_name": "col2",
"auto_generated": False,
"is_primary_key": False,
"column_type": "VARCHAR",
"size": 255,
"is_public": True,
"is_null_allowed": True,
"enums": [],
"sets": [],
},
{
"id": 4,
"database_id": 1,
"table_id": 1,
"name": "col3",
"internal_name": "col3",
"auto_generated": True,
"is_primary_key": True,
"column_type": "FLOAT",
"size": 24,
"is_public": True,
"is_null_allowed": True,
"enums": [],
"sets": [],
},
{
"id": 5,
"database_id": 1,
"table_id": 1,
"name": "col4",
"internal_name": "col4",
"auto_generated": False,
"is_primary_key": False,
"column_type": "FLOAT",
"size": 24,
"is_public": True,
"is_null_allowed": True,
"enums": [],
"sets": [],
},
{
"id": 6,
"database_id": 1,
"table_id": 1,
"name": "col5",
"internal_name": "col5",
"auto_generated": False,
"is_primary_key": False,
"column_type": "INT",
"size": 255,
"is_public": True,
"is_null_allowed": True,
"enums": [],
"sets": [],
},
],
"constraints": {"uniques": [], "foreign_keys": []},
}
],
},
)
# test
response = determine_stats(db, os, **payload)
assert response
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment