From b3187620a1b06aec63119883626bc39488d4930f Mon Sep 17 00:00:00 2001
From: Martin Weise <martin.weise@tuwien.ac.at>
Date: Fri, 10 Nov 2023 09:40:36 +0100
Subject: [PATCH] Implemented parts of the sidecar
---
dbrepo-data-db/sidecar/Dockerfile | 14 +-
dbrepo-data-db/sidecar/Pipfile | 1 +
dbrepo-data-db/sidecar/Pipfile.lock | 522 ++----------------
dbrepo-data-db/sidecar/app.py | 39 +-
.../sidecar/clients/minio_client.py | 75 +++
dbrepo-data-db/sidecar/ds-yml/export.yml | 21 +
dbrepo-data-db/sidecar/ds-yml/import.yml | 25 +-
dbrepo-gateway-service/dbrepo.conf | 13 -
docker-compose.yml | 7 +-
9 files changed, 182 insertions(+), 535 deletions(-)
create mode 100644 dbrepo-data-db/sidecar/clients/minio_client.py
create mode 100644 dbrepo-data-db/sidecar/ds-yml/export.yml
diff --git a/dbrepo-data-db/sidecar/Dockerfile b/dbrepo-data-db/sidecar/Dockerfile
index eafdd534d8..eb4a1541fd 100644
--- a/dbrepo-data-db/sidecar/Dockerfile
+++ b/dbrepo-data-db/sidecar/Dockerfile
@@ -1,8 +1,8 @@
FROM python:3.10-alpine
-RUN apk add bash curl jq && adduser -D alpine
+RUN apk add bash curl jq
-WORKDIR /home/alpine
+WORKDIR /app
COPY Pipfile Pipfile.lock ./
@@ -10,13 +10,17 @@ RUN pip install pipenv && \
pipenv install gunicorn && \
pipenv install --system --deploy
+USER 1000
+
+COPY ./clients ./clients
COPY ./ds-yml ./ds-yml
COPY ./app.py ./app.py
-ENV UPLOAD_ENDPOINT="http://upload-service:1080/api/upload"
+ENV S3_STORAGE_ENDPOINT="http://storage-service:9000"
+ENV S3_ACCESS_KEY_ID="minioadmin"
+ENV S3_SECRET_ACCESS_KEY="minioadmin"
-RUN chown -R alpine:alpine ./
-USER alpine
+RUN ls -la ./clients
EXPOSE 5000
diff --git a/dbrepo-data-db/sidecar/Pipfile b/dbrepo-data-db/sidecar/Pipfile
index 3d06f782ad..ef26d7ffbc 100644
--- a/dbrepo-data-db/sidecar/Pipfile
+++ b/dbrepo-data-db/sidecar/Pipfile
@@ -13,6 +13,7 @@ prometheus-flask-exporter = "*"
python-dotenv = "~=1.0"
sqlalchemy-utils = "*"
gunicorn = "*"
+boto3 = "*"
[dev-packages]
diff --git a/dbrepo-data-db/sidecar/Pipfile.lock b/dbrepo-data-db/sidecar/Pipfile.lock
index a13787df33..d3008811f8 100644
--- a/dbrepo-data-db/sidecar/Pipfile.lock
+++ b/dbrepo-data-db/sidecar/Pipfile.lock
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
- "sha256": "ffe85da356a65c571d9d36f4cb2c78dcb2cf1e079f121c0ca7f73c447c8ad5ac"
+ "sha256": "1f500b7ec4d31276a0472ffeaebfe17b31945c080b3b5207b9c5703b35322c40"
},
"pipfile-spec": 6,
"requires": {
@@ -16,115 +16,6 @@
]
},
"default": {
- "aiohttp": {
- "hashes": [
- "sha256:002f23e6ea8d3dd8d149e569fd580c999232b5fbc601c48d55398fbc2e582e8c",
- "sha256:01770d8c04bd8db568abb636c1fdd4f7140b284b8b3e0b4584f070180c1e5c62",
- "sha256:0912ed87fee967940aacc5306d3aa8ba3a459fcd12add0b407081fbefc931e53",
- "sha256:0cccd1de239afa866e4ce5c789b3032442f19c261c7d8a01183fd956b1935349",
- "sha256:0fa375b3d34e71ccccf172cab401cd94a72de7a8cc01847a7b3386204093bb47",
- "sha256:13da35c9ceb847732bf5c6c5781dcf4780e14392e5d3b3c689f6d22f8e15ae31",
- "sha256:14cd52ccf40006c7a6cd34a0f8663734e5363fd981807173faf3a017e202fec9",
- "sha256:16d330b3b9db87c3883e565340d292638a878236418b23cc8b9b11a054aaa887",
- "sha256:1bed815f3dc3d915c5c1e556c397c8667826fbc1b935d95b0ad680787896a358",
- "sha256:1d84166673694841d8953f0a8d0c90e1087739d24632fe86b1a08819168b4566",
- "sha256:1f13f60d78224f0dace220d8ab4ef1dbc37115eeeab8c06804fec11bec2bbd07",
- "sha256:229852e147f44da0241954fc6cb910ba074e597f06789c867cb7fb0621e0ba7a",
- "sha256:253bf92b744b3170eb4c4ca2fa58f9c4b87aeb1df42f71d4e78815e6e8b73c9e",
- "sha256:255ba9d6d5ff1a382bb9a578cd563605aa69bec845680e21c44afc2670607a95",
- "sha256:2817b2f66ca82ee699acd90e05c95e79bbf1dc986abb62b61ec8aaf851e81c93",
- "sha256:2b8d4e166e600dcfbff51919c7a3789ff6ca8b3ecce16e1d9c96d95dd569eb4c",
- "sha256:2d5b785c792802e7b275c420d84f3397668e9d49ab1cb52bd916b3b3ffcf09ad",
- "sha256:3161ce82ab85acd267c8f4b14aa226047a6bee1e4e6adb74b798bd42c6ae1f80",
- "sha256:33164093be11fcef3ce2571a0dccd9041c9a93fa3bde86569d7b03120d276c6f",
- "sha256:39a312d0e991690ccc1a61f1e9e42daa519dcc34ad03eb6f826d94c1190190dd",
- "sha256:3b2ab182fc28e7a81f6c70bfbd829045d9480063f5ab06f6e601a3eddbbd49a0",
- "sha256:3c68330a59506254b556b99a91857428cab98b2f84061260a67865f7f52899f5",
- "sha256:3f0e27e5b733803333bb2371249f41cf42bae8884863e8e8965ec69bebe53132",
- "sha256:3f5c7ce535a1d2429a634310e308fb7d718905487257060e5d4598e29dc17f0b",
- "sha256:3fd194939b1f764d6bb05490987bfe104287bbf51b8d862261ccf66f48fb4096",
- "sha256:41bdc2ba359032e36c0e9de5a3bd00d6fb7ea558a6ce6b70acedf0da86458321",
- "sha256:41d55fc043954cddbbd82503d9cc3f4814a40bcef30b3569bc7b5e34130718c1",
- "sha256:42c89579f82e49db436b69c938ab3e1559e5a4409eb8639eb4143989bc390f2f",
- "sha256:45ad816b2c8e3b60b510f30dbd37fe74fd4a772248a52bb021f6fd65dff809b6",
- "sha256:4ac39027011414dbd3d87f7edb31680e1f430834c8cef029f11c66dad0670aa5",
- "sha256:4d4cbe4ffa9d05f46a28252efc5941e0462792930caa370a6efaf491f412bc66",
- "sha256:4fcf3eabd3fd1a5e6092d1242295fa37d0354b2eb2077e6eb670accad78e40e1",
- "sha256:5d791245a894be071d5ab04bbb4850534261a7d4fd363b094a7b9963e8cdbd31",
- "sha256:6c43ecfef7deaf0617cee936836518e7424ee12cb709883f2c9a1adda63cc460",
- "sha256:6c5f938d199a6fdbdc10bbb9447496561c3a9a565b43be564648d81e1102ac22",
- "sha256:6e2f9cc8e5328f829f6e1fb74a0a3a939b14e67e80832975e01929e320386b34",
- "sha256:713103a8bdde61d13490adf47171a1039fd880113981e55401a0f7b42c37d071",
- "sha256:71783b0b6455ac8f34b5ec99d83e686892c50498d5d00b8e56d47f41b38fbe04",
- "sha256:76b36b3124f0223903609944a3c8bf28a599b2cc0ce0be60b45211c8e9be97f8",
- "sha256:7bc88fc494b1f0311d67f29fee6fd636606f4697e8cc793a2d912ac5b19aa38d",
- "sha256:7ee912f7e78287516df155f69da575a0ba33b02dd7c1d6614dbc9463f43066e3",
- "sha256:86f20cee0f0a317c76573b627b954c412ea766d6ada1a9fcf1b805763ae7feeb",
- "sha256:89341b2c19fb5eac30c341133ae2cc3544d40d9b1892749cdd25892bbc6ac951",
- "sha256:8a9b5a0606faca4f6cc0d338359d6fa137104c337f489cd135bb7fbdbccb1e39",
- "sha256:8d399dade330c53b4106160f75f55407e9ae7505263ea86f2ccca6bfcbdb4921",
- "sha256:8e31e9db1bee8b4f407b77fd2507337a0a80665ad7b6c749d08df595d88f1cf5",
- "sha256:90c72ebb7cb3a08a7f40061079817133f502a160561d0675b0a6adf231382c92",
- "sha256:918810ef188f84152af6b938254911055a72e0f935b5fbc4c1a4ed0b0584aed1",
- "sha256:93c15c8e48e5e7b89d5cb4613479d144fda8344e2d886cf694fd36db4cc86865",
- "sha256:96603a562b546632441926cd1293cfcb5b69f0b4159e6077f7c7dbdfb686af4d",
- "sha256:99c5ac4ad492b4a19fc132306cd57075c28446ec2ed970973bbf036bcda1bcc6",
- "sha256:9c19b26acdd08dd239e0d3669a3dddafd600902e37881f13fbd8a53943079dbc",
- "sha256:9de50a199b7710fa2904be5a4a9b51af587ab24c8e540a7243ab737b45844543",
- "sha256:9e2ee0ac5a1f5c7dd3197de309adfb99ac4617ff02b0603fd1e65b07dc772e4b",
- "sha256:a2ece4af1f3c967a4390c284797ab595a9f1bc1130ef8b01828915a05a6ae684",
- "sha256:a3628b6c7b880b181a3ae0a0683698513874df63783fd89de99b7b7539e3e8a8",
- "sha256:ad1407db8f2f49329729564f71685557157bfa42b48f4b93e53721a16eb813ed",
- "sha256:b04691bc6601ef47c88f0255043df6f570ada1a9ebef99c34bd0b72866c217ae",
- "sha256:b0cf2a4501bff9330a8a5248b4ce951851e415bdcce9dc158e76cfd55e15085c",
- "sha256:b2fe42e523be344124c6c8ef32a011444e869dc5f883c591ed87f84339de5976",
- "sha256:b30e963f9e0d52c28f284d554a9469af073030030cef8693106d918b2ca92f54",
- "sha256:bb54c54510e47a8c7c8e63454a6acc817519337b2b78606c4e840871a3e15349",
- "sha256:bd111d7fc5591ddf377a408ed9067045259ff2770f37e2d94e6478d0f3fc0c17",
- "sha256:bdf70bfe5a1414ba9afb9d49f0c912dc524cf60141102f3a11143ba3d291870f",
- "sha256:ca80e1b90a05a4f476547f904992ae81eda5c2c85c66ee4195bb8f9c5fb47f28",
- "sha256:caf486ac1e689dda3502567eb89ffe02876546599bbf915ec94b1fa424eeffd4",
- "sha256:ccc360e87341ad47c777f5723f68adbb52b37ab450c8bc3ca9ca1f3e849e5fe2",
- "sha256:d25036d161c4fe2225d1abff2bd52c34ed0b1099f02c208cd34d8c05729882f0",
- "sha256:d52d5dc7c6682b720280f9d9db41d36ebe4791622c842e258c9206232251ab2b",
- "sha256:d67f8baed00870aa390ea2590798766256f31dc5ed3ecc737debb6e97e2ede78",
- "sha256:d76e8b13161a202d14c9584590c4df4d068c9567c99506497bdd67eaedf36403",
- "sha256:d95fc1bf33a9a81469aa760617b5971331cdd74370d1214f0b3109272c0e1e3c",
- "sha256:de6a1c9f6803b90e20869e6b99c2c18cef5cc691363954c93cb9adeb26d9f3ae",
- "sha256:e1d8cb0b56b3587c5c01de3bf2f600f186da7e7b5f7353d1bf26a8ddca57f965",
- "sha256:e2a988a0c673c2e12084f5e6ba3392d76c75ddb8ebc6c7e9ead68248101cd446",
- "sha256:e3f1e3f1a1751bb62b4a1b7f4e435afcdade6c17a4fd9b9d43607cebd242924a",
- "sha256:e6a00ffcc173e765e200ceefb06399ba09c06db97f401f920513a10c803604ca",
- "sha256:e827d48cf802de06d9c935088c2924e3c7e7533377d66b6f31ed175c1620e05e",
- "sha256:ebf3fd9f141700b510d4b190094db0ce37ac6361a6806c153c161dc6c041ccda",
- "sha256:ec00c3305788e04bf6d29d42e504560e159ccaf0be30c09203b468a6c1ccd3b2",
- "sha256:ec4fd86658c6a8964d75426517dc01cbf840bbf32d055ce64a9e63a40fd7b771",
- "sha256:efd2fcf7e7b9d7ab16e6b7d54205beded0a9c8566cb30f09c1abe42b4e22bdcb",
- "sha256:f0f03211fd14a6a0aed2997d4b1c013d49fb7b50eeb9ffdf5e51f23cfe2c77fa",
- "sha256:f628dbf3c91e12f4d6c8b3f092069567d8eb17814aebba3d7d60c149391aee3a",
- "sha256:f8ef51e459eb2ad8e7a66c1d6440c808485840ad55ecc3cafefadea47d1b1ba2",
- "sha256:fc37e9aef10a696a5a4474802930079ccfc14d9f9c10b4662169671ff034b7df",
- "sha256:fdee8405931b0615220e5ddf8cd7edd8592c606a8e4ca2a00704883c396e4479"
- ],
- "markers": "python_version >= '3.6'",
- "version": "==3.8.6"
- },
- "aiosignal": {
- "hashes": [
- "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc",
- "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"
- ],
- "markers": "python_version >= '3.7'",
- "version": "==1.3.1"
- },
- "async-timeout": {
- "hashes": [
- "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f",
- "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"
- ],
- "markers": "python_version >= '3.7'",
- "version": "==4.0.3"
- },
"attrs": {
"hashes": [
"sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04",
@@ -141,6 +32,23 @@
"markers": "python_version >= '3.8'",
"version": "==1.7.0"
},
+ "boto3": {
+ "hashes": [
+ "sha256:1d10691911c4b8b9443d3060257ba32b68b6e3cad0eebbb9f69fd1c52a78417f",
+ "sha256:489c4967805b677b7a4030460e4c06c0903d6bc0f6834453611bf87efbd8d8a3"
+ ],
+ "index": "pypi",
+ "markers": "python_version >= '3.7'",
+ "version": "==1.28.83"
+ },
+ "botocore": {
+ "hashes": [
+ "sha256:40914b0fb28f13d709e1f8a4481e278350b77a3987be81acd23715ec8d5fedca",
+ "sha256:c742069e8bfd06d212d712228258ff09fb481b6ec02358e539381ce0fcad065a"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==1.31.83"
+ },
"certifi": {
"hashes": [
"sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082",
@@ -149,102 +57,6 @@
"markers": "python_version >= '3.6'",
"version": "==2023.7.22"
},
- "charset-normalizer": {
- "hashes": [
- "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027",
- "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087",
- "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786",
- "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8",
- "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09",
- "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185",
- "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574",
- "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e",
- "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519",
- "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898",
- "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269",
- "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3",
- "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f",
- "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6",
- "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8",
- "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a",
- "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73",
- "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc",
- "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714",
- "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2",
- "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc",
- "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce",
- "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d",
- "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e",
- "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6",
- "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269",
- "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96",
- "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d",
- "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a",
- "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4",
- "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77",
- "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d",
- "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0",
- "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed",
- "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068",
- "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac",
- "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25",
- "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8",
- "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab",
- "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26",
- "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2",
- "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db",
- "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f",
- "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5",
- "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99",
- "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c",
- "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d",
- "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811",
- "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa",
- "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a",
- "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03",
- "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b",
- "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04",
- "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c",
- "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001",
- "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458",
- "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389",
- "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99",
- "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985",
- "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537",
- "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238",
- "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f",
- "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d",
- "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796",
- "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a",
- "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143",
- "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8",
- "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c",
- "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5",
- "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5",
- "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711",
- "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4",
- "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6",
- "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c",
- "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7",
- "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4",
- "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b",
- "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae",
- "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12",
- "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c",
- "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae",
- "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8",
- "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887",
- "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b",
- "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4",
- "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f",
- "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5",
- "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33",
- "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519",
- "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"
- ],
- "markers": "python_full_version >= '3.7.0'",
- "version": "==3.3.2"
- },
"click": {
"hashes": [
"sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28",
@@ -295,80 +107,6 @@
"markers": "python_version >= '3.8'",
"version": "==3.1.1"
},
- "frozenlist": {
- "hashes": [
- "sha256:007df07a6e3eb3e33e9a1fe6a9db7af152bbd8a185f9aaa6ece10a3529e3e1c6",
- "sha256:008eb8b31b3ea6896da16c38c1b136cb9fec9e249e77f6211d479db79a4eaf01",
- "sha256:09163bdf0b2907454042edb19f887c6d33806adc71fbd54afc14908bfdc22251",
- "sha256:0c7c1b47859ee2cac3846fde1c1dc0f15da6cec5a0e5c72d101e0f83dcb67ff9",
- "sha256:0e5c8764c7829343d919cc2dfc587a8db01c4f70a4ebbc49abde5d4b158b007b",
- "sha256:10ff5faaa22786315ef57097a279b833ecab1a0bfb07d604c9cbb1c4cdc2ed87",
- "sha256:17ae5cd0f333f94f2e03aaf140bb762c64783935cc764ff9c82dff626089bebf",
- "sha256:19488c57c12d4e8095a922f328df3f179c820c212940a498623ed39160bc3c2f",
- "sha256:1a0848b52815006ea6596c395f87449f693dc419061cc21e970f139d466dc0a0",
- "sha256:1e78fb68cf9c1a6aa4a9a12e960a5c9dfbdb89b3695197aa7064705662515de2",
- "sha256:261b9f5d17cac914531331ff1b1d452125bf5daa05faf73b71d935485b0c510b",
- "sha256:2b8bcf994563466db019fab287ff390fffbfdb4f905fc77bc1c1d604b1c689cc",
- "sha256:38461d02d66de17455072c9ba981d35f1d2a73024bee7790ac2f9e361ef1cd0c",
- "sha256:490132667476f6781b4c9458298b0c1cddf237488abd228b0b3650e5ecba7467",
- "sha256:491e014f5c43656da08958808588cc6c016847b4360e327a62cb308c791bd2d9",
- "sha256:515e1abc578dd3b275d6a5114030b1330ba044ffba03f94091842852f806f1c1",
- "sha256:556de4430ce324c836789fa4560ca62d1591d2538b8ceb0b4f68fb7b2384a27a",
- "sha256:5833593c25ac59ede40ed4de6d67eb42928cca97f26feea219f21d0ed0959b79",
- "sha256:6221d84d463fb110bdd7619b69cb43878a11d51cbb9394ae3105d082d5199167",
- "sha256:6918d49b1f90821e93069682c06ffde41829c346c66b721e65a5c62b4bab0300",
- "sha256:6c38721585f285203e4b4132a352eb3daa19121a035f3182e08e437cface44bf",
- "sha256:71932b597f9895f011f47f17d6428252fc728ba2ae6024e13c3398a087c2cdea",
- "sha256:7211ef110a9194b6042449431e08c4d80c0481e5891e58d429df5899690511c2",
- "sha256:764226ceef3125e53ea2cb275000e309c0aa5464d43bd72abd661e27fffc26ab",
- "sha256:7645a8e814a3ee34a89c4a372011dcd817964ce8cb273c8ed6119d706e9613e3",
- "sha256:76d4711f6f6d08551a7e9ef28c722f4a50dd0fc204c56b4bcd95c6cc05ce6fbb",
- "sha256:7f4f399d28478d1f604c2ff9119907af9726aed73680e5ed1ca634d377abb087",
- "sha256:88f7bc0fcca81f985f78dd0fa68d2c75abf8272b1f5c323ea4a01a4d7a614efc",
- "sha256:8d0edd6b1c7fb94922bf569c9b092ee187a83f03fb1a63076e7774b60f9481a8",
- "sha256:901289d524fdd571be1c7be054f48b1f88ce8dddcbdf1ec698b27d4b8b9e5d62",
- "sha256:93ea75c050c5bb3d98016b4ba2497851eadf0ac154d88a67d7a6816206f6fa7f",
- "sha256:981b9ab5a0a3178ff413bca62526bb784249421c24ad7381e39d67981be2c326",
- "sha256:9ac08e601308e41eb533f232dbf6b7e4cea762f9f84f6357136eed926c15d12c",
- "sha256:a02eb8ab2b8f200179b5f62b59757685ae9987996ae549ccf30f983f40602431",
- "sha256:a0c6da9aee33ff0b1a451e867da0c1f47408112b3391dd43133838339e410963",
- "sha256:a6c8097e01886188e5be3e6b14e94ab365f384736aa1fca6a0b9e35bd4a30bc7",
- "sha256:aa384489fefeb62321b238e64c07ef48398fe80f9e1e6afeff22e140e0850eef",
- "sha256:ad2a9eb6d9839ae241701d0918f54c51365a51407fd80f6b8289e2dfca977cc3",
- "sha256:b206646d176a007466358aa21d85cd8600a415c67c9bd15403336c331a10d956",
- "sha256:b826d97e4276750beca7c8f0f1a4938892697a6bcd8ec8217b3312dad6982781",
- "sha256:b89ac9768b82205936771f8d2eb3ce88503b1556324c9f903e7156669f521472",
- "sha256:bd7bd3b3830247580de99c99ea2a01416dfc3c34471ca1298bccabf86d0ff4dc",
- "sha256:bdf1847068c362f16b353163391210269e4f0569a3c166bc6a9f74ccbfc7e839",
- "sha256:c11b0746f5d946fecf750428a95f3e9ebe792c1ee3b1e96eeba145dc631a9672",
- "sha256:c5374b80521d3d3f2ec5572e05adc94601985cc526fb276d0c8574a6d749f1b3",
- "sha256:ca265542ca427bf97aed183c1676e2a9c66942e822b14dc6e5f42e038f92a503",
- "sha256:ce31ae3e19f3c902de379cf1323d90c649425b86de7bbdf82871b8a2a0615f3d",
- "sha256:ceb6ec0a10c65540421e20ebd29083c50e6d1143278746a4ef6bcf6153171eb8",
- "sha256:d081f13b095d74b67d550de04df1c756831f3b83dc9881c38985834387487f1b",
- "sha256:d5655a942f5f5d2c9ed93d72148226d75369b4f6952680211972a33e59b1dfdc",
- "sha256:d5a32087d720c608f42caed0ef36d2b3ea61a9d09ee59a5142d6070da9041b8f",
- "sha256:d6484756b12f40003c6128bfcc3fa9f0d49a687e171186c2d85ec82e3758c559",
- "sha256:dd65632acaf0d47608190a71bfe46b209719bf2beb59507db08ccdbe712f969b",
- "sha256:de343e75f40e972bae1ef6090267f8260c1446a1695e77096db6cfa25e759a95",
- "sha256:e29cda763f752553fa14c68fb2195150bfab22b352572cb36c43c47bedba70eb",
- "sha256:e41f3de4df3e80de75845d3e743b3f1c4c8613c3997a912dbf0229fc61a8b963",
- "sha256:e66d2a64d44d50d2543405fb183a21f76b3b5fd16f130f5c99187c3fb4e64919",
- "sha256:e74b0506fa5aa5598ac6a975a12aa8928cbb58e1f5ac8360792ef15de1aa848f",
- "sha256:f0ed05f5079c708fe74bf9027e95125334b6978bf07fd5ab923e9e55e5fbb9d3",
- "sha256:f61e2dc5ad442c52b4887f1fdc112f97caeff4d9e6ebe78879364ac59f1663e1",
- "sha256:fec520865f42e5c7f050c2a79038897b1c7d1595e907a9e08e3353293ffc948e"
- ],
- "markers": "python_version >= '3.8'",
- "version": "==1.4.0"
- },
- "future": {
- "hashes": [
- "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307"
- ],
- "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
- "version": "==0.18.3"
- },
"greenlet": {
"hashes": [
"sha256:0a02d259510b3630f330c86557331a3b0e0c79dac3d166e449a39363beaae174",
@@ -441,14 +179,6 @@
"markers": "python_version >= '3.5'",
"version": "==21.2.0"
},
- "idna": {
- "hashes": [
- "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4",
- "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"
- ],
- "markers": "python_version >= '3.5'",
- "version": "==3.4"
- },
"itsdangerous": {
"hashes": [
"sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44",
@@ -465,6 +195,14 @@
"markers": "python_version >= '3.7'",
"version": "==3.1.2"
},
+ "jmespath": {
+ "hashes": [
+ "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980",
+ "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==1.0.1"
+ },
"jsonschema": {
"hashes": [
"sha256:c9ff4d7447eed9592c23a12ccee508baf0dd0d59650615e847feb6cdca74f392",
@@ -547,6 +285,14 @@
"markers": "python_version >= '3.7'",
"version": "==2.1.3"
},
+ "minio": {
+ "hashes": [
+ "sha256:0aa525d77a3bc61378444c2400b0ba2685ad4cd6ecb3fba4141a0d0765e25f40",
+ "sha256:b0b687c1ec9be422a1f8b04c65fb8e43a1c090f9508178db57c434a17341c404"
+ ],
+ "index": "pypi",
+ "version": "==7.1.17"
+ },
"mistune": {
"hashes": [
"sha256:71481854c30fdbc938963d3605b72501f5c10a9320ecd412c121c163a1c7d205",
@@ -555,86 +301,6 @@
"markers": "python_version >= '3.7'",
"version": "==3.0.2"
},
- "multidict": {
- "hashes": [
- "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9",
- "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8",
- "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03",
- "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710",
- "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161",
- "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664",
- "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569",
- "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067",
- "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313",
- "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706",
- "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2",
- "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636",
- "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49",
- "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93",
- "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603",
- "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0",
- "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60",
- "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4",
- "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e",
- "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1",
- "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60",
- "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951",
- "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc",
- "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe",
- "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95",
- "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d",
- "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8",
- "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed",
- "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2",
- "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775",
- "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87",
- "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c",
- "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2",
- "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98",
- "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3",
- "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe",
- "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78",
- "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660",
- "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176",
- "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e",
- "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988",
- "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c",
- "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c",
- "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0",
- "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449",
- "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f",
- "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde",
- "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5",
- "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d",
- "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac",
- "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a",
- "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9",
- "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca",
- "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11",
- "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35",
- "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063",
- "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b",
- "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982",
- "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258",
- "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1",
- "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52",
- "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480",
- "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7",
- "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461",
- "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d",
- "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc",
- "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779",
- "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a",
- "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547",
- "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0",
- "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171",
- "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf",
- "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d",
- "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba"
- ],
- "markers": "python_version >= '3.7'",
- "version": "==6.0.4"
- },
"packaging": {
"hashes": [
"sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5",
@@ -667,6 +333,14 @@
"markers": "python_version >= '3.7'",
"version": "==2.8.0"
},
+ "python-dateutil": {
+ "hashes": [
+ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
+ "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+ "version": "==2.8.2"
+ },
"python-dotenv": {
"hashes": [
"sha256:a8df96034aae6d2d50a4ebe8216326c61c3eb64836776504fcca410e5937a3ba",
@@ -740,14 +414,6 @@
"markers": "python_version >= '3.8'",
"version": "==0.30.2"
},
- "requests": {
- "hashes": [
- "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
- "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"
- ],
- "markers": "python_version >= '3.7'",
- "version": "==2.31.0"
- },
"rpds-py": {
"hashes": [
"sha256:0525847f83f506aa1e28eb2057b696fe38217e12931c8b1b02198cfe6975e142",
@@ -853,6 +519,14 @@
"markers": "python_version >= '3.8'",
"version": "==0.12.0"
},
+ "s3transfer": {
+ "hashes": [
+ "sha256:10d6923c6359175f264811ef4bf6161a3156ce8e350e705396a7557d6293c33a",
+ "sha256:fd3889a66f5fe17299fe75b82eae6cf722554edca744ca5d5fe308b104883d2e"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==0.7.0"
+ },
"six": {
"hashes": [
"sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
@@ -925,22 +599,6 @@
"markers": "python_version >= '3.6'",
"version": "==0.41.1"
},
- "tinydb": {
- "hashes": [
- "sha256:30c06d12383d7c332e404ca6a6103fb2b32cbf25712689648c39d9a6bd34bd3d",
- "sha256:6dd686a9c5a75dfa9280088fd79a419aefe19cd7f4bd85eba203540ef856d564"
- ],
- "markers": "python_version >= '3.7' and python_version < '4.0'",
- "version": "==4.8.0"
- },
- "tuspy": {
- "hashes": [
- "sha256:09a4ef6e20b05c6f83c53c5839892ead141947afb2f6b7dea5cca45f8d120793",
- "sha256:4c136bb97593367376e77f1d9580c1990bd535698f86765c80db9b9d3864f35f"
- ],
- "index": "pypi",
- "version": "==1.0.1"
- },
"typing-extensions": {
"hashes": [
"sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0",
@@ -964,86 +622,6 @@
],
"markers": "python_version >= '3.8'",
"version": "==3.0.1"
- },
- "yarl": {
- "hashes": [
- "sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571",
- "sha256:066c163aec9d3d073dc9ffe5dd3ad05069bcb03fcaab8d221290ba99f9f69ee3",
- "sha256:13414591ff516e04fcdee8dc051c13fd3db13b673c7a4cb1350e6b2ad9639ad3",
- "sha256:149ddea5abf329752ea5051b61bd6c1d979e13fbf122d3a1f9f0c8be6cb6f63c",
- "sha256:159d81f22d7a43e6eabc36d7194cb53f2f15f498dbbfa8edc8a3239350f59fe7",
- "sha256:1b1bba902cba32cdec51fca038fd53f8beee88b77efc373968d1ed021024cc04",
- "sha256:22a94666751778629f1ec4280b08eb11815783c63f52092a5953faf73be24191",
- "sha256:2a96c19c52ff442a808c105901d0bdfd2e28575b3d5f82e2f5fd67e20dc5f4ea",
- "sha256:2b0738fb871812722a0ac2154be1f049c6223b9f6f22eec352996b69775b36d4",
- "sha256:2c315df3293cd521033533d242d15eab26583360b58f7ee5d9565f15fee1bef4",
- "sha256:32f1d071b3f362c80f1a7d322bfd7b2d11e33d2adf395cc1dd4df36c9c243095",
- "sha256:3458a24e4ea3fd8930e934c129b676c27452e4ebda80fbe47b56d8c6c7a63a9e",
- "sha256:38a3928ae37558bc1b559f67410df446d1fbfa87318b124bf5032c31e3447b74",
- "sha256:3da8a678ca8b96c8606bbb8bfacd99a12ad5dd288bc6f7979baddd62f71c63ef",
- "sha256:494053246b119b041960ddcd20fd76224149cfea8ed8777b687358727911dd33",
- "sha256:50f33040f3836e912ed16d212f6cc1efb3231a8a60526a407aeb66c1c1956dde",
- "sha256:52a25809fcbecfc63ac9ba0c0fb586f90837f5425edfd1ec9f3372b119585e45",
- "sha256:53338749febd28935d55b41bf0bcc79d634881195a39f6b2f767870b72514caf",
- "sha256:5415d5a4b080dc9612b1b63cba008db84e908b95848369aa1da3686ae27b6d2b",
- "sha256:5610f80cf43b6202e2c33ba3ec2ee0a2884f8f423c8f4f62906731d876ef4fac",
- "sha256:566185e8ebc0898b11f8026447eacd02e46226716229cea8db37496c8cdd26e0",
- "sha256:56ff08ab5df8429901ebdc5d15941b59f6253393cb5da07b4170beefcf1b2528",
- "sha256:59723a029760079b7d991a401386390c4be5bfec1e7dd83e25a6a0881859e716",
- "sha256:5fcd436ea16fee7d4207c045b1e340020e58a2597301cfbcfdbe5abd2356c2fb",
- "sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18",
- "sha256:63c48f6cef34e6319a74c727376e95626f84ea091f92c0250a98e53e62c77c72",
- "sha256:646d663eb2232d7909e6601f1a9107e66f9791f290a1b3dc7057818fe44fc2b6",
- "sha256:662e6016409828ee910f5d9602a2729a8a57d74b163c89a837de3fea050c7582",
- "sha256:674ca19cbee4a82c9f54e0d1eee28116e63bc6fd1e96c43031d11cbab8b2afd5",
- "sha256:6a5883464143ab3ae9ba68daae8e7c5c95b969462bbe42e2464d60e7e2698368",
- "sha256:6e7221580dc1db478464cfeef9b03b95c5852cc22894e418562997df0d074ccc",
- "sha256:75df5ef94c3fdc393c6b19d80e6ef1ecc9ae2f4263c09cacb178d871c02a5ba9",
- "sha256:783185c75c12a017cc345015ea359cc801c3b29a2966c2655cd12b233bf5a2be",
- "sha256:822b30a0f22e588b32d3120f6d41e4ed021806418b4c9f0bc3048b8c8cb3f92a",
- "sha256:8288d7cd28f8119b07dd49b7230d6b4562f9b61ee9a4ab02221060d21136be80",
- "sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8",
- "sha256:832b7e711027c114d79dffb92576acd1bd2decc467dec60e1cac96912602d0e6",
- "sha256:838162460b3a08987546e881a2bfa573960bb559dfa739e7800ceeec92e64417",
- "sha256:83fcc480d7549ccebe9415d96d9263e2d4226798c37ebd18c930fce43dfb9574",
- "sha256:84e0b1599334b1e1478db01b756e55937d4614f8654311eb26012091be109d59",
- "sha256:891c0e3ec5ec881541f6c5113d8df0315ce5440e244a716b95f2525b7b9f3608",
- "sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82",
- "sha256:8c56986609b057b4839968ba901944af91b8e92f1725d1a2d77cbac6972b9ed1",
- "sha256:8ea48e0a2f931064469bdabca50c2f578b565fc446f302a79ba6cc0ee7f384d3",
- "sha256:8ec53a0ea2a80c5cd1ab397925f94bff59222aa3cf9c6da938ce05c9ec20428d",
- "sha256:95d2ecefbcf4e744ea952d073c6922e72ee650ffc79028eb1e320e732898d7e8",
- "sha256:9b3152f2f5677b997ae6c804b73da05a39daa6a9e85a512e0e6823d81cdad7cc",
- "sha256:9bf345c3a4f5ba7f766430f97f9cc1320786f19584acc7086491f45524a551ac",
- "sha256:a60347f234c2212a9f0361955007fcf4033a75bf600a33c88a0a8e91af77c0e8",
- "sha256:a74dcbfe780e62f4b5a062714576f16c2f3493a0394e555ab141bf0d746bb955",
- "sha256:a83503934c6273806aed765035716216cc9ab4e0364f7f066227e1aaea90b8d0",
- "sha256:ac9bb4c5ce3975aeac288cfcb5061ce60e0d14d92209e780c93954076c7c4367",
- "sha256:aff634b15beff8902d1f918012fc2a42e0dbae6f469fce134c8a0dc51ca423bb",
- "sha256:b03917871bf859a81ccb180c9a2e6c1e04d2f6a51d953e6a5cdd70c93d4e5a2a",
- "sha256:b124e2a6d223b65ba8768d5706d103280914d61f5cae3afbc50fc3dfcc016623",
- "sha256:b25322201585c69abc7b0e89e72790469f7dad90d26754717f3310bfe30331c2",
- "sha256:b7232f8dfbd225d57340e441d8caf8652a6acd06b389ea2d3222b8bc89cbfca6",
- "sha256:b8cc1863402472f16c600e3e93d542b7e7542a540f95c30afd472e8e549fc3f7",
- "sha256:b9a4e67ad7b646cd6f0938c7ebfd60e481b7410f574c560e455e938d2da8e0f4",
- "sha256:be6b3fdec5c62f2a67cb3f8c6dbf56bbf3f61c0f046f84645cd1ca73532ea051",
- "sha256:bf74d08542c3a9ea97bb8f343d4fcbd4d8f91bba5ec9d5d7f792dbe727f88938",
- "sha256:c027a6e96ef77d401d8d5a5c8d6bc478e8042f1e448272e8d9752cb0aff8b5c8",
- "sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9",
- "sha256:c1012fa63eb6c032f3ce5d2171c267992ae0c00b9e164efe4d73db818465fac3",
- "sha256:c3a53ba34a636a256d767c086ceb111358876e1fb6b50dfc4d3f4951d40133d5",
- "sha256:d4e2c6d555e77b37288eaf45b8f60f0737c9efa3452c6c44626a5455aeb250b9",
- "sha256:de119f56f3c5f0e2fb4dee508531a32b069a5f2c6e827b272d1e0ff5ac040333",
- "sha256:e65610c5792870d45d7b68c677681376fcf9cc1c289f23e8e8b39c1485384185",
- "sha256:e9fdc7ac0d42bc3ea78818557fab03af6181e076a2944f43c38684b4b6bed8e3",
- "sha256:ee4afac41415d52d53a9833ebae7e32b344be72835bbb589018c9e938045a560",
- "sha256:f364d3480bffd3aa566e886587eaca7c8c04d74f6e8933f3f2c996b7f09bee1b",
- "sha256:f3b078dbe227f79be488ffcfc7a9edb3409d018e0952cf13f15fd6512847f3f7",
- "sha256:f4e2d08f07a3d7d3e12549052eb5ad3eab1c349c53ac51c209a0e5991bbada78",
- "sha256:f7a3d8146575e08c29ed1cd287068e6d02f1c7bdff8970db96683b9591b86ee7"
- ],
- "markers": "python_version >= '3.7'",
- "version": "==1.9.2"
}
},
"develop": {}
diff --git a/dbrepo-data-db/sidecar/app.py b/dbrepo-data-db/sidecar/app.py
index c56eba4c3f..105f3f03f2 100644
--- a/dbrepo-data-db/sidecar/app.py
+++ b/dbrepo-data-db/sidecar/app.py
@@ -1,11 +1,11 @@
+import json
import os
import logging
-from urllib.error import URLError, ContentTooShortError, HTTPError
from flasgger import LazyJSONEncoder, Swagger
from flask import Flask, request, Response
from flasgger.utils import swag_from
-import urllib.request
+from clients.minio_client import MinioClient
from prometheus_flask_exporter import PrometheusMetrics
logging.basicConfig(level=logging.DEBUG)
@@ -99,33 +99,26 @@ app.json_encoder = LazyJSONEncoder
@app.route("/health", methods=["GET"], endpoint="actuator_health")
@swag_from("ds-yml/health.yml")
def health():
- return Response({"status": "UP"}, mimetype="application/json"), 200
+ return Response(json.dumps({"status": "UP"}), mimetype="application/json"), 200
-@app.route("/sidecar/import", methods=["POST"], endpoint="sidecar_import")
+@app.route("/sidecar/import/<string:filename>", methods=["POST"], endpoint="sidecar_import")
@swag_from("ds-yml/import.yml")
-def import_csv():
- logging.debug('endpoint import csv, body=%s', request)
- input_json = request.get_json()
- filepath = str(input_json['filepath'])
- api = os.getenv("UPLOAD_ENDPOINT", "http://localhost:1080/api/upload")
- try:
- urllib.request.urlretrieve(api + "/files/" + filepath, "/tmp/" + filepath)
- except URLError as e:
- logging.error('Failed to import .csv: %s', e)
- return Response(), 503
- return Response(), 202
+def import_csv(filename):
+ logging.debug('endpoint import csv, filename=%s, body=%s', filename, request)
+ minio_client = MinioClient()
+ response = minio_client.download_file(filename)
+ if response is False:
+ return Response(), 400
+ return Response(json.dumps(response)), 202
-@app.route("/sidecar/export/<string:filename>", methods=["PUT"], endpoint="sidecar_export")
+@app.route("/sidecar/export/<string:filename>", methods=["POST"], endpoint="sidecar_export")
@swag_from("ds-yml/export.yml")
def import_csv(filename):
logging.debug('endpoint export csv, filename=%s, body=%s', filename, request)
- api = os.getenv("UPLOAD_ENDPOINT", "http://localhost:1080/api/upload")
- try:
- # upload
- urllib.request.urlretrieve(api + "/files/" + filepath, "/tmp/" + filepath)
- except URLError as e:
- logging.error('Failed to import .csv: %s', e)
- return Response(), 503
+ minio_client = MinioClient()
+ response = minio_client.upload_file(filename)
+ if response is False:
+ return Response(), 400
return Response(), 202
diff --git a/dbrepo-data-db/sidecar/clients/minio_client.py b/dbrepo-data-db/sidecar/clients/minio_client.py
new file mode 100644
index 0000000000..a3f27dbb07
--- /dev/null
+++ b/dbrepo-data-db/sidecar/clients/minio_client.py
@@ -0,0 +1,75 @@
+import os
+import boto3
+import logging
+import sys
+
+from botocore.exceptions import ClientError
+
+
+class MinioClient:
+
+ def __init__(self):
+ endpoint_url = os.getenv('S3_STORAGE_ENDPOINT', 'http://localhost:9000')
+ aws_access_key_id = os.getenv('S3_ACCESS_KEY_ID', 'minioadmin')
+ aws_secret_access_key = os.getenv('S3_SECRET_ACCESS_KEY', 'minioadmin')
+ logging.info("retrieve file from S3, endpoint_url=%s, aws_access_key_id=%s, aws_secret_access_key=(hidden)",
+ endpoint_url, aws_access_key_id)
+ self.client = boto3.client(service_name='s3', endpoint_url=endpoint_url, aws_access_key_id=aws_access_key_id,
+ aws_secret_access_key=aws_secret_access_key)
+ self.bucket_exists_or_exit("dbrepo-upload")
+ self.bucket_exists_or_exit("dbrepo-download")
+
+ def upload_file(self, filename) -> bool:
+ """
+ Uploads a file to the blob storage.
+ Follows the official API https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html.
+ :param filename: The filename.
+ :return: True if the file was uploaded.
+ """
+ filepath = os.path.join("/tmp/", filename)
+ try:
+ self.client.upload_file(filepath, "dbrepo-download", filename)
+ logging.info("Uploaded .csv %s with key %s", filepath, filename)
+ return True
+ except ClientError as e:
+ logging.error(e)
+ return False
+
+ def download_file(self, filename) -> bool:
+ """
+ Downloads a file from the blob storage.
+ Follows the official API https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-example-download-file.html
+ :param filename: The filename.
+ :return: True if the file was downloaded and saved.
+ """
+ self.file_exists("dbrepo-upload", filename)
+ filepath = os.path.join("/tmp/", filename)
+ try:
+ self.client.download_file("dbrepo-upload", filename, filepath)
+ logging.info("Downloaded .csv with key %s into %s", filename, filepath)
+ return True
+ except ClientError as e:
+ logging.error(e)
+ return False
+
+ def file_exists(self, bucket, filename):
+ try:
+ self.client.head_object(Bucket=bucket, Key=filename)
+ except ClientError as e:
+ if e.response["Error"]["Code"] == "404":
+ logging.error("Failed to find key %s in bucket %s", filename, bucket)
+ else:
+ logging.error("Unexpected error when finding key %s in bucket %s: %s", filename, bucket,
+ e.response["Error"]["Code"])
+ raise e
+
+ def bucket_exists_or_exit(self, bucket):
+ try:
+ self.client.head_bucket(Bucket=bucket)
+ except ClientError as e:
+ if e.response["Error"]["Code"] == "404":
+ logging.error("Failed to find bucket %s", bucket)
+ else:
+ logging.error("Unexpected error when finding bucket %s: %s", bucket,
+ e.response["Error"]["Code"])
+ sys.exit(1)
diff --git a/dbrepo-data-db/sidecar/ds-yml/export.yml b/dbrepo-data-db/sidecar/ds-yml/export.yml
new file mode 100644
index 0000000000..4a212d023d
--- /dev/null
+++ b/dbrepo-data-db/sidecar/ds-yml/export.yml
@@ -0,0 +1,21 @@
+summary: Exports a .csv to the Storage Service
+description: |
+ Exports a specific .csv file to the Storage Service via S3
+consumes:
+ - application/json
+produces:
+ - application/json
+parameters:
+ - in: path
+ name: filename
+ description: Name of the object file to export to the Storage Service
+ required: true
+
+responses:
+ 202:
+ description: Exported the .csv
+ content: { }
+ 400:
+ description: The Storage Service could not be contacted or .csv was not found.
+tags:
+ - sidecar
diff --git a/dbrepo-data-db/sidecar/ds-yml/import.yml b/dbrepo-data-db/sidecar/ds-yml/import.yml
index 0eba6e63a0..ad2d68b304 100644
--- a/dbrepo-data-db/sidecar/ds-yml/import.yml
+++ b/dbrepo-data-db/sidecar/ds-yml/import.yml
@@ -1,32 +1,21 @@
-summary: Imports a .csv from the Upload Service
+summary: Imports a .csv from the Storage Service
description: |
- Imports a specific .csv file from the Upload Service via HTTP
+ Imports a specific .csv file from the Storage Service via S3
consumes:
- application/json
produces:
- application/json
parameters:
- - in: "body"
- name: "body"
- description: "Payload to import the .csv"
+ - in: path
+ name: filename
+ description: Name of the object file to import from the Storage Service
required: true
- schema:
- $ref: "#/components/schemas/Import"
responses:
202:
description: Imported the .csv
content: { }
- 503:
- description: The Upload Service could not be contacted or .csv was not found.
+ 400:
+ description: The Storage Service could not be contacted or .csv was not found.
tags:
- sidecar
-
-components:
- schemas:
- Import:
- type: "object"
- properties:
- filepath:
- type: "string"
- example: "sample.csv"
\ No newline at end of file
diff --git a/dbrepo-gateway-service/dbrepo.conf b/dbrepo-gateway-service/dbrepo.conf
index 47bff96a1b..a9d996358b 100644
--- a/dbrepo-gateway-service/dbrepo.conf
+++ b/dbrepo-gateway-service/dbrepo.conf
@@ -26,10 +26,6 @@ upstream ui {
server ui:3000;
}
-upstream upload {
- server upload-service:1080;
-}
-
upstream search-db-dashboard {
server search-db-dashboard:5601;
}
@@ -95,15 +91,6 @@ server {
proxy_read_timeout 90;
}
- location /api/upload {
- proxy_set_header Host $host;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_set_header X-Forwarded-Proto $scheme;
- proxy_pass http://upload;
- proxy_read_timeout 90;
- }
-
location /broker {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
diff --git a/docker-compose.yml b/docker-compose.yml
index d06bb911dc..3a51aa40cb 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -264,7 +264,9 @@ services:
- "3600:5000"
environment:
FLASK_DEBUG: ${SEARCH_DEBUG_MODE:-true}
- UPLOAD_ENDPOINT: "${UPLOAD_ENDPOINT:-http://upload-service:1080/api/upload/files}"
+ S3_STORAGE_ENDPOINT: "${STORAGE_ENDPOINT:-http://storage-service:9000}"
+ S3_ACCESS_KEY_ID: "${STORAGE_USERNAME:-minioadmin}"
+ S3_SECRET_ACCESS_KEY: ${STORAGE_PASSWORD:-minioadmin}
volumes:
- "${SHARED_FILESYSTEM:-/tmp}:/tmp"
healthcheck:
@@ -377,9 +379,6 @@ services:
MINIO_BROWSER_REDIRECT_URL: "${STORAGE_BASE_URL:-http://localhost/admin/storage/}"
ports:
- 9000:9000
- depends_on:
- dbrepo-authentication-service:
- condition: service_healthy
healthcheck:
test: [ "CMD", "mc", "ready", "local" ]
interval: 5s
--
GitLab