Skip to content
Snippets Groups Projects
Commit c974fb2b authored by Martin Weise's avatar Martin Weise
Browse files

Merge branch 'dev'

parents 46b9cac7 05c709f2
No related branches found
No related tags found
No related merge requests found
Expand all Show whitespace changes
Inline Side-by-side
Showing
with 2979 additions and 213551 deletions
.docs/changelog.md
+ 16
− 0
View file @ c974fb2b
...@@ -2,6 +2,22 @@ ...@@ -2,6 +2,22 @@
author: Martin Weise author: Martin Weise
--- ---
## v1.6.1 (2025-01-21)
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.1)
### What's Changed
#### Changes
* Added privacy feature for hidden databases (and optionally tables, views, subsets) that hides them completely from
e.g. the search in [#482](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/issues/482).
#### Fixes
* Added init container that adds the admin user to the Metadata Database
in [#480](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/issues/480).
## v1.6.0 (2025-01-07) ## v1.6.0 (2025-01-07)
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.0) [:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.0)
......
.gitlab-ci.yml
+ 58
− 1
View file @ c974fb2b
...@@ -6,6 +6,7 @@ variables: ...@@ -6,6 +6,7 @@ variables:
ALPINE_VERSION: "3.21" ALPINE_VERSION: "3.21"
PYTHON_VERSION: "3.11" PYTHON_VERSION: "3.11"
JAVA_VERSION: "17" JAVA_VERSION: "17"
NODE_VERSION: "18"
SONARQUBE_VERSION: "10.0" SONARQUBE_VERSION: "10.0"
BUN_VERSION: "1.1.40" BUN_VERSION: "1.1.40"
DOC_VERSION: "1.6" DOC_VERSION: "1.6"
...@@ -50,6 +51,7 @@ lint-docker-compose: ...@@ -50,6 +51,7 @@ lint-docker-compose:
- 'wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /usr/bin/yq && chmod +x /usr/bin/yq' - 'wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /usr/bin/yq && chmod +x /usr/bin/yq'
- 'ls -la .scripts' - 'ls -la .scripts'
script: script:
- "bash .scripts/check-compose.sh"
- "yq compare -P docker-compose.yml .docker/docker-compose.yml 'volumes.*'" - "yq compare -P docker-compose.yml .docker/docker-compose.yml 'volumes.*'"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-analyse-service'" - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-analyse-service'"
- "bash .scripts/check-service.sh 'dbrepo-auth-db'" - "bash .scripts/check-service.sh 'dbrepo-auth-db'"
...@@ -75,12 +77,45 @@ lint-helm-chart: ...@@ -75,12 +77,45 @@ lint-helm-chart:
image: docker.io/alpine:${ALPINE_VERSION} image: docker.io/alpine:${ALPINE_VERSION}
stage: lint stage: lint
before_script: before_script:
- apk add helm - apk --no-cache add helm git bash
- cp ./helm/dbrepo/values.schema.json ./CI_values.schema.json
- helm plugin install https://github.com/losisin/helm-values-schema-json.git
- helm dependency update ./helm/seaweedfs - helm dependency update ./helm/seaweedfs
- helm package ./helm/seaweedfs --destination ./build - helm package ./helm/seaweedfs --destination ./build
- helm dependency update ./helm/dbrepo - helm dependency update ./helm/dbrepo
script: script:
- bash .scripts/check-helm.sh
- helm lint ./helm/dbrepo - helm lint ./helm/dbrepo
- helm schema -input ./helm/dbrepo/values.yaml -output ./helm/dbrepo/values.schema.json
- diff ./CI_values.schema.json ./helm/dbrepo/values.schema.json
artifacts:
when: always
paths:
- ./helm/dbrepo/values.schema.json
expire_in: 1 days
lint-helm-readme:
image: docker.io/node:${NODE_VERSION}-alpine${ALPINE_VERSION}
stage: lint
before_script:
- apk --no-cache add alpine-sdk bash git
- cp ./helm/dbrepo/README.md ./CI_README.md
- git clone https://github.com/bitnami/readme-generator-for-helm
- (cd ./readme-generator-for-helm && npm install && npm install -g pkg && pkg . -o /usr/local/sbin/readme-generator)
script:
- readme-generator --readme ./helm/dbrepo/README.md --values ./helm/dbrepo/values.yaml
- diff ./CI_README.md ./helm/dbrepo/README.md
artifacts:
when: always
paths:
- ./helm/dbrepo/README.md
expire_in: 1 days
lint-metadata-schema:
image: docker.io/alpine:${ALPINE_VERSION}
stage: lint
script:
- diff dbrepo-metadata-db/1_setup-schema.sql helm/dbrepo/files/01-setup-schema.sql
build-metadata-service: build-metadata-service:
image: maven:3-openjdk-${JAVA_VERSION} image: maven:3-openjdk-${JAVA_VERSION}
...@@ -262,6 +297,28 @@ test-analyse-service: ...@@ -262,6 +297,28 @@ test-analyse-service:
junit: ./dbrepo-analyse-service/report.xml junit: ./dbrepo-analyse-service/report.xml
coverage: '/TOTAL.*?([0-9]{1,3})%/' coverage: '/TOTAL.*?([0-9]{1,3})%/'
test-auth-service-init:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: test
variables:
PIPENV_PIPFILE: "./dbrepo-auth-service/init/Pipfile"
before_script:
- "apk add --no-cache alpine-sdk gcc python3-dev mariadb-connector-c-dev"
- "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy"
script:
- cd ./dbrepo-auth-service/init/ && coverage run -m pytest test/test_unit_app.py --junitxml=report.xml && coverage html --omit="test/*" && coverage report --omit="test/*" > ./coverage.txt
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts:
when: always
paths:
- ./dbrepo-auth-service/init/report.xml
- ./dbrepo-auth-service/init/coverage.txt
expire_in: 1 days
reports:
junit: ./dbrepo-auth-service/init/report.xml
coverage: '/TOTAL.*?([0-9]{1,3})%/'
test-search-service: test-search-service:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: test stage: test
......
.scripts/check-compose.sh 0 → 100644
+ 6
− 0
View file @ c974fb2b
#!/bin/bash
OUTPUT=$(cat .docker/docker-compose.yml | grep "registry.datalab.tuwien.ac.at/.*" | grep -v "$APP_VERSION")
if [ $? -ne 1 ]; then
echo "[ERROR] Some image version(s) differ from APP_VERSION=${APP_VERSION}: ${OUTPUT}"
exit 1
fi
\ No newline at end of file
.scripts/check-helm.sh
+ 5
− 0
View file @ c974fb2b
...@@ -4,3 +4,8 @@ if [ $? -ne 0 ]; then ...@@ -4,3 +4,8 @@ if [ $? -ne 0 ]; then
echo "!!! Please install the Helm values schema plugin first\n" echo "!!! Please install the Helm values schema plugin first\n"
echo " https://github.com/losisin/helm-values-schema-json" echo " https://github.com/losisin/helm-values-schema-json"
fi fi
OUTPUT=$(cat helm/dbrepo/values.yaml | grep "registry.datalab.tuwien.ac.at/.*" | grep -v "$APP_VERSION")
if [ $? -ne 1 ]; then
echo "[ERROR] Some image version(s) differ from APP_VERSION=${APP_VERSION} in Helm Char: ${OUTPUT}"
exit 1
fi
\ No newline at end of file
dbrepo-analyse-service/Pipfile.lock
+ 91
− 98
View file @ c974fb2b
...@@ -175,20 +175,20 @@ ...@@ -175,20 +175,20 @@
}, },
"boto3": { "boto3": {
"hashes": [ "hashes": [
"sha256:258ab77225a81d3cf3029c9afe9920cd9dec317689dfadec6f6f0a23130bb60a", "sha256:7d398f66a11e67777c189d1f58c0a75d9d60f98d0ee51b8817e828930bf19e4e",
"sha256:eb21380d73fec6645439c0d802210f72a0cdb3295b02953f246ff53f512faa8f" "sha256:8e49416216a6e3a62c2a0c44fba4dd2852c85472e7b702516605b1363867d220"
], ],
"index": "pypi", "index": "pypi",
"markers": "python_version >= '3.8'", "markers": "python_version >= '3.8'",
"version": "==1.36.1" "version": "==1.35.97"
}, },
"botocore": { "botocore": {
"hashes": [ "hashes": [
"sha256:dec513b4eb8a847d79bbefdcdd07040ed9d44c20b0001136f0890a03d595705a", "sha256:88f2fab29192ffe2f2115d5bafbbd823ff4b6eb2774296e03ec8b5b0fe074f61",
"sha256:f789a6f272b5b3d8f8756495019785e33868e5e00dd9662a3ee7959ac939bb12" "sha256:fed4f156b1a9b8ece53738f702ba5851b8c6216b4952de326547f349cc494f14"
], ],
"markers": "python_version >= '3.8'", "markers": "python_version >= '3.8'",
"version": "==1.36.1" "version": "==1.35.97"
}, },
"certifi": { "certifi": {
"hashes": [ "hashes": [
...@@ -268,7 +268,7 @@ ...@@ -268,7 +268,7 @@
"sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87", "sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87",
"sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b" "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b"
], ],
"markers": "python_version >= '3.8'", "markers": "platform_python_implementation != 'PyPy'",
"version": "==1.17.1" "version": "==1.17.1"
}, },
"charset-normalizer": { "charset-normalizer": {
...@@ -412,7 +412,7 @@ ...@@ -412,7 +412,7 @@
}, },
"dbrepo": { "dbrepo": {
"hashes": [ "hashes": [
"sha256:251f3c2088bbd289cee86d5394b1e62e29aa081f994dd0845d895e3330f6a106" "sha256:7cddcbdcb3eade84f67db01fa32e0649ecc01d4c3cc5e7542d3c402ad52efc19"
], ],
"path": "./lib/dbrepo-1.6.1.tar.gz" "path": "./lib/dbrepo-1.6.1.tar.gz"
}, },
...@@ -1427,11 +1427,11 @@ ...@@ -1427,11 +1427,11 @@
}, },
"referencing": { "referencing": {
"hashes": [ "hashes": [
"sha256:363d9c65f080d0d70bc41c721dce3c7f3e77fc09f269cd5c8813da18069a6794", "sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c",
"sha256:ca2e6492769e3602957e9b831b94211599d2aade9477f5d44110d2530cf9aade" "sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de"
], ],
"markers": "python_version >= '3.9'", "markers": "python_version >= '3.8'",
"version": "==0.36.1" "version": "==0.35.1"
}, },
"requests": { "requests": {
"hashes": [ "hashes": [
...@@ -1553,11 +1553,11 @@ ...@@ -1553,11 +1553,11 @@
}, },
"s3transfer": { "s3transfer": {
"hashes": [ "hashes": [
"sha256:3f25c900a367c8b7f7d8f9c34edc87e300bde424f779dc9f0a8ae4f9df9264f6", "sha256:244a76a24355363a68164241438de1b72f8781664920260c48465896b712a41e",
"sha256:8fa0aa48177be1f3425176dfe1ab85dcd3d962df603c3dbfc585e6bf857ef0ff" "sha256:29edc09801743c21eb5ecbc617a152df41d3c287f67b615f73e5f750583666a7"
], ],
"markers": "python_version >= '3.8'", "markers": "python_version >= '3.8'",
"version": "==0.11.1" "version": "==0.10.4"
}, },
"setuptools": { "setuptools": {
"hashes": [ "hashes": [
...@@ -1612,7 +1612,7 @@ ...@@ -1612,7 +1612,7 @@
"sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df", "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df",
"sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d" "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d"
], ],
"markers": "python_version >= '3.9'", "markers": "python_version >= '3.10'",
"version": "==2.3.0" "version": "==2.3.0"
}, },
"werkzeug": { "werkzeug": {
...@@ -1877,7 +1877,7 @@ ...@@ -1877,7 +1877,7 @@
"sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87", "sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87",
"sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b" "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b"
], ],
"markers": "python_version >= '3.8'", "markers": "platform_python_implementation != 'PyPy'",
"version": "==1.17.1" "version": "==1.17.1"
}, },
"charset-normalizer": { "charset-normalizer": {
...@@ -2236,93 +2236,86 @@ ...@@ -2236,93 +2236,86 @@
"sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df", "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df",
"sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d" "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d"
], ],
"markers": "python_version >= '3.9'", "markers": "python_version >= '3.10'",
"version": "==2.3.0" "version": "==2.3.0"
}, },
"wrapt": { "wrapt": {
"hashes": [ "hashes": [
"sha256:08e7ce672e35efa54c5024936e559469436f8b8096253404faeb54d2a878416f", "sha256:09f5141599eaf36d6cc0b760ad87c2ab6b8618d009b2922639266676775a73a6",
"sha256:0a6e821770cf99cc586d33833b2ff32faebdbe886bd6322395606cf55153246c", "sha256:0aad4f54b3155d673a5c4706a71a0a84f3d415b2fc8a2a399a964d70f18846a2",
"sha256:0b929ac182f5ace000d459c59c2c9c33047e20e935f8e39371fa6e3b85d56f4a", "sha256:0eb33799b7582bb73787b9903b70595f8eff67eecc9455f668ed01adf53f9eea",
"sha256:129a150f5c445165ff941fc02ee27df65940fcb8a22a61828b1853c98763a64b", "sha256:0ee037e4cc9d039efe712b13c483f4efa2c3499642369e01570b3bb1842eea3f",
"sha256:13e6afb7fe71fe7485a4550a8844cc9ffbe263c0f1a1eea569bc7091d4898555", "sha256:0fdc4e73a3fa0c25eed4d836d9732226f0326957cb075044a7f252b465299433",
"sha256:1473400e5b2733e58b396a04eb7f35f541e1fb976d0c0724d0223dd607e0f74c", "sha256:13887d1415dc0e213a9adeb9026ae1f427023f77110d988fbd478643490aa40c",
"sha256:18983c537e04d11cf027fbb60a1e8dfd5190e2b60cc27bc0808e653e7b218d1b", "sha256:144ed42a4ec3aca5d6f1524f99ee49493bbd0d9c66c24da7ec44b4661dca4dcc",
"sha256:1a7ed2d9d039bd41e889f6fb9364554052ca21ce823580f6a07c4ec245c1f5d6", "sha256:14f78f8c313884f889c6696af62aa881af302a989a7c0df398d2b541fa53e8a9",
"sha256:1e1fe0e6ab7775fd842bc39e86f6dcfc4507ab0ffe206093e76d61cde37225c8", "sha256:15f96fe5e2efdc613983327240ae89cf6368c07eeb0f194d240e9549aa1ea739",
"sha256:1fb5699e4464afe5c7e65fa51d4f99e0b2eadcc176e4aa33600a3df7801d6662", "sha256:162d5f15bdd3b8037e06540902227ef9e0f298496c0afaadd9e2875851446693",
"sha256:2696993ee1eebd20b8e4ee4356483c4cb696066ddc24bd70bcbb80fa56ff9061", "sha256:169033329022739c6f0d8cd3031a113953b0ba500f3d5978904bdd40baec4568",
"sha256:35621ae4c00e056adb0009f8e86e28eb4a41a4bfa8f9bfa9fca7d343fe94f998", "sha256:16b2fdfa09a74a3930175b6d9d7d008022aa72a4f02de2b3eecafcc1adfd3cfe",
"sha256:36ccae62f64235cf8ddb682073a60519426fdd4725524ae38874adf72b5f2aeb", "sha256:181a844005c9818792212a32e004cb4c6bd8e35cae8e97b1a39a1918d95cef58",
"sha256:3cedbfa9c940fdad3e6e941db7138e26ce8aad38ab5fe9dcfadfed9db7a54e62", "sha256:18fb16fb6bb75f4ec6272829007f3129a9a5264d0230372f9651e5f75cfec552",
"sha256:3d57c572081fed831ad2d26fd430d565b76aa277ed1d30ff4d40670b1c0dd984", "sha256:1c119802ae432b8c5d55dd5253825d09c1dca1c97ffc7b32c53ecdb348712f64",
"sha256:3fc7cb4c1c744f8c05cd5f9438a3caa6ab94ce8344e952d7c45a8ed59dd88392", "sha256:20888d886186d19eab53816db2e615950b1ce7dbd5c239107daf2c8a6a4a03c6",
"sha256:4011d137b9955791f9084749cba9a367c68d50ab8d11d64c50ba1688c9b457f2", "sha256:21ffcf16f5c243a626b0f8da637948e3d5984e3bc0c1bc500ad990e88e974e3b",
"sha256:40d615e4fe22f4ad3528448c193b218e077656ca9ccb22ce2cb20db730f8d306", "sha256:27a49f217839bf559d436308bae8fc4a9dd0ac98ffdb9d6aeb3f00385b0fb72c",
"sha256:410a92fefd2e0e10d26210e1dfb4a876ddaf8439ef60d6434f21ef8d87efc5b7", "sha256:2b20fcef5a3ee410671a5a59472e1ff9dda21cfbe5dfd15e23ee4b99ac455c8e",
"sha256:41388e9d4d1522446fe79d3213196bd9e3b301a336965b9e27ca2788ebd122f3", "sha256:2c160bb8815787646b27a0c8575a26a4d6bf6abd7c5eb250ad3f2d38b29cb2cb",
"sha256:468090021f391fe0056ad3e807e3d9034e0fd01adcd3bdfba977b6fdf4213ea9", "sha256:2f1bc359f6c52e53565e7af24b423e7a1eea97d155f38ac9e90e95303514710b",
"sha256:49703ce2ddc220df165bd2962f8e03b84c89fee2d65e1c24a7defff6f988f4d6", "sha256:30c0c08434fe2af6e40c5c75c036d7e3c7e7f499079fc479e740d9586b09fb0d",
"sha256:4a721d3c943dae44f8e243b380cb645a709ba5bd35d3ad27bc2ed947e9c68192", "sha256:3260178f3bc006acae93378bfd6dbf33c9249de93cc1b78d8cc7b7416f4ea99a",
"sha256:4afd5814270fdf6380616b321fd31435a462019d834f83c8611a0ce7484c7317", "sha256:3dfd4738a630eddfcb7ff6c8e9fe863df3821f9c991dec73821e05450074ae09",
"sha256:4c82b8785d98cdd9fed4cac84d765d234ed3251bd6afe34cb7ac523cb93e8b4f", "sha256:50a4e3b45e62b1ccb96b3fc0e427f1b458ff2e0def34ae084de88418157a09d1",
"sha256:4db983e7bca53819efdbd64590ee96c9213894272c776966ca6306b73e4affda", "sha256:50bbfa7a92da7540426c774e09d6901e44d8f9b513b276ebae03ae244f0c6dbf",
"sha256:582530701bff1dec6779efa00c516496968edd851fba224fbd86e46cc6b73563", "sha256:52f0907287d9104112dbebda46af4db0793fcc4c64c8a867099212d116b6db64",
"sha256:58455b79ec2661c3600e65c0a716955adc2410f7383755d537584b0de41b1d8a", "sha256:53e2986a65eba7c399d7ad1ccd204562d4ffe6e937344fe5a49eb5a83858f797",
"sha256:58705da316756681ad3c9c73fd15499aa4d8c69f9fd38dc8a35e06c12468582f", "sha256:5660e470edfa15ae7ef407272c642d29e9962777a6b30bfa8fc0da2173dc9afd",
"sha256:5bb1d0dbf99411f3d871deb6faa9aabb9d4e744d67dcaaa05399af89d847a91d", "sha256:57e932ad1908b53e9ad67a746432f02bc8473a9ee16e26a47645a2b224fba5fd",
"sha256:5c803c401ea1c1c18de70a06a6f79fcc9c5acfc79133e9869e730ad7f8ad8ef9", "sha256:589f24449fd58508533c4a69b2a0f45e9e3419b86b43a0607e2fdb989c6f2552",
"sha256:5cbabee4f083b6b4cd282f5b817a867cf0b1028c54d445b7ec7cfe6505057cf8", "sha256:5c2e24ba455af4b0a237a890ea6ed9bafd01fac2c47095f87c53ea3344215d43",
"sha256:612dff5db80beef9e649c6d803a8d50c409082f1fedc9dbcdfde2983b2025b82", "sha256:5ebea3ebb6a394f50f150a52e279508e91c8770625ac8fcb5d8cf35995a320f2",
"sha256:62c2caa1585c82b3f7a7ab56afef7b3602021d6da34fbc1cf234ff139fed3cd9", "sha256:67c30d3fe245adb0eb1061a0e526905970a0dabe7c5fba5078e0ee9d19f28167",
"sha256:69606d7bb691b50a4240ce6b22ebb319c1cfb164e5f6569835058196e0f3a845", "sha256:6bb82447ddae4e3d9b51f40c494f66e6cbd8fb0e8e8b993678416535c67f9a0d",
"sha256:6d9187b01bebc3875bac9b087948a2bccefe464a7d8f627cf6e48b1bbae30f82", "sha256:6ce4cff3922707048d754e365c4ebf41a3bcbf29b329349bf85d51873c7c7e9e",
"sha256:6ed6ffac43aecfe6d86ec5b74b06a5be33d5bb9243d055141e8cabb12aa08125", "sha256:6d44b14f3a2f6343a07c90344850b7af5515538ce3a5d01f9c87d8bae9bd8724",
"sha256:703919b1633412ab54bcf920ab388735832fdcb9f9a00ae49387f0fe67dad504", "sha256:6fd88935b12b59a933ef45facb57575095f205d30d0ae8dd1a3b485bc4fa2fbd",
"sha256:766d8bbefcb9e00c3ac3b000d9acc51f1b399513f44d77dfe0eb026ad7c9a19b", "sha256:78da796b74f2c8e0af021ee99feb3bff7cb46f8e658fe25c20e66be1080db4a2",
"sha256:80dd7db6a7cb57ffbc279c4394246414ec99537ae81ffd702443335a61dbf3a7", "sha256:7966f98fa36933333d8a1c3d8552aa3d0735001901a4aabcfbd5a502b4ef14fe",
"sha256:8112e52c5822fc4253f3901b676c55ddf288614dc7011634e2719718eaa187dc", "sha256:7eca3a1afa9820785b79cb137c68ca38c2f77cfedc3120115da42e1d5800907e",
"sha256:8c8b293cd65ad716d13d8dd3624e42e5a19cc2a2f1acc74b30c2c13f15cb61a6", "sha256:823a262d967cbdf835787039b873ff551e36c14658bdc2e43267968b67f61f88",
"sha256:8fdbdb757d5390f7c675e558fd3186d590973244fab0c5fe63d373ade3e99d40", "sha256:88623fd957ba500d8bb0f7427a76496d99313ca2f9e932481c0882e034cf1add",
"sha256:91bd7d1773e64019f9288b7a5101f3ae50d3d8e6b1de7edee9c2ccc1d32f0c0a", "sha256:889587664d245dae75c752b643061f922e8a590d43a4cd088eca415ca83f2d13",
"sha256:95c658736ec15602da0ed73f312d410117723914a5c91a14ee4cdd72f1d790b3", "sha256:9176057c60438c2ce2284cdefc2b3ee5eddc8c87cd6e24c558d9f5c64298fa4a",
"sha256:99039fa9e6306880572915728d7f6c24a86ec57b0a83f6b2491e1d8ab0235b9a", "sha256:93018dbb956e0ad99ea2fa2c3c22f033549dcb1f56ad9f4555dfe25e49688c5d",
"sha256:9a2bce789a5ea90e51a02dfcc39e31b7f1e662bc3317979aa7e5538e3a034f72", "sha256:97eaff096fcb467e0f486f3bf354c1072245c2045859d71ba71158717ec97dcc",
"sha256:9a7d15bbd2bc99e92e39f49a04653062ee6085c0e18b3b7512a4f2fe91f2d681", "sha256:997e8f9b984e4263993d3baf3329367e7c7673b63789bc761718a6f9ed68653d",
"sha256:9abc77a4ce4c6f2a3168ff34b1da9b0f311a8f1cfd694ec96b0603dff1c79438", "sha256:99e544e6ce26f89ad5acc6f407bc4daf7c1d42321e836f5c768f834100bdf35c",
"sha256:9e8659775f1adf02eb1e6f109751268e493c73716ca5761f8acb695e52a756ae", "sha256:9e04f3bd30e0b23c0ca7e1d4084e7d28b6d7d2feb8b7bc69b496fe881280579b",
"sha256:9fee687dce376205d9a494e9c121e27183b2a3df18037f89d69bd7b35bcf59e2", "sha256:a7aa07603d67007c15b33d20095cc9276f3e127bfb1b8106b3e84ec6907d137e",
"sha256:a5aaeff38654462bc4b09023918b7f21790efb807f54c000a39d41d69cf552cb", "sha256:a992f9e019145e84616048556546edeaba68e05e1c1ffbe8391067a63cdadb0c",
"sha256:a604bf7a053f8362d27eb9fefd2097f82600b856d5abe996d623babd067b1ab5", "sha256:b1a4c8edd038fee0ce67bf119b16eaa45d22a52bbaf7d0a17d2312eb0003b1bb",
"sha256:abbb9e76177c35d4e8568e58650aa6926040d6a9f6f03435b7a522bf1c487f9a", "sha256:b8bd35c15bc82c5cbe397e8196fa57a17ce5d3f30e925a6fd39e4c5bb02fdcff",
"sha256:acc130bc0375999da18e3d19e5a86403667ac0c4042a094fefb7eec8ebac7cf3", "sha256:b9a58a1cbdc0588ed4c8ab0c191002d5d831a58c3bad88523fe471ea97eaf57d",
"sha256:b18f2d1533a71f069c7f82d524a52599053d4c7166e9dd374ae2136b7f40f7c8", "sha256:bac64f57a5a7926ebc9ab519fb9eba1fc6dcd1f65d7f45937b2ce38da65c2270",
"sha256:b4e42a40a5e164cbfdb7b386c966a588b1047558a990981ace551ed7e12ca9c2", "sha256:bca1c0824f824bcd97b4b179dd55dcad1dab419252be2b2faebbcacefa3b27b2",
"sha256:b5e251054542ae57ac7f3fba5d10bfff615b6c2fb09abeb37d2f1463f841ae22", "sha256:bdf7b0e3d3713331c0bb9daac47cd10e5aa60d060e53696f50de4e560bd5617f",
"sha256:b60fb58b90c6d63779cb0c0c54eeb38941bae3ecf7a73c764c52c88c2dcb9d72", "sha256:c53ef8936c4d587cb96bb1cf0d076e822fa38266c2b646837ef60465da8db22e",
"sha256:b870b5df5b71d8c3359d21be8f0d6c485fa0ebdb6477dda51a1ea54a9b558061", "sha256:cbead724daa13cae46e8ab3bb24938d8514d123f34345535b184f3eb1b7ad717",
"sha256:ba0f0eb61ef00ea10e00eb53a9129501f52385c44853dbd6c4ad3f403603083f", "sha256:cd7649f0c493d35f9aad9790bbecd7b6fd2e2f7141f6cb1e1e9bb7a681d6d0a4",
"sha256:bb87745b2e6dc56361bfde481d5a378dc314b252a98d7dd19a651a3fa58f24a9", "sha256:d609f0ab0603bbcbf2de906b366b9f9bec75c32b4493550a940de658cc2ce512",
"sha256:bb90fb8bda722a1b9d48ac1e6c38f923ea757b3baf8ebd0c82e09c5c1a0e7a04", "sha256:d792631942a102d6d4f71e4948aceb307310ac0a0af054be6d28b4f79583e0f1",
"sha256:bc570b5f14a79734437cb7b0500376b6b791153314986074486e0b0fa8d71d98", "sha256:d87334b521ab0e2564902c0b10039dee8670485e9d397fe97c34b88801f474f7",
"sha256:c86563182421896d73858e08e1db93afdd2b947a70064b813d515d66549e15f9", "sha256:da0d0c1c4bd55f9ace919454776dbf0821f537b9a77f739f0c3e34b14728b3b3",
"sha256:c958bcfd59bacc2d0249dcfe575e71da54f9dcf4a8bdf89c4cb9a68a1170d73f", "sha256:e0f0e731e0ca1583befd3af71b9f90d64ded1535da7b80181cb9e907cc10bbae",
"sha256:d18a4865f46b8579d44e4fe1e2bcbc6472ad83d98e22a26c963d46e4c125ef0b", "sha256:e5bd9186d52cf3d36bf1823be0e85297e4dbad909bc6dd495ce0d272806d84a7",
"sha256:d5e2439eecc762cd85e7bd37161d4714aa03a33c5ba884e26c81559817ca0925", "sha256:e72053cc4706dac537d5a772135dc3e1de5aff52883f49994c1757c1b2dc9db2",
"sha256:e3890b508a23299083e065f435a492b5435eba6e304a7114d2f919d400888cc6", "sha256:e8a7b0699a381226d81d75b48ea58414beb5891ba8982bdc8e42912f766de074",
"sha256:e496a8ce2c256da1eb98bd15803a79bee00fc351f5dfb9ea82594a3f058309e0", "sha256:ec3e763e7ca8dcba0792fc3e8ff7061186f59e9aafe4438e6bb1f635a6ab0901",
"sha256:e8b2816ebef96d83657b56306152a93909a83f23994f4b30ad4573b00bd11bb9", "sha256:f17e8d926f63aed65ff949682c922f96d00f65c2e852c24272232313fa7823d5",
"sha256:eaf675418ed6b3b31c7a989fd007fa7c3be66ce14e5c3b27336383604c9da85c", "sha256:f3117feb1fc479eaf84b549d3f229d5d2abdb823f003bc2a1c6dd70072912fa0"
"sha256:ec89ed91f2fa8e3f52ae53cd3cf640d6feff92ba90d62236a81e4e563ac0e991",
"sha256:ecc840861360ba9d176d413a5489b9a0aff6d6303d7e733e2c4623cfa26904a6",
"sha256:f09b286faeff3c750a879d336fb6d8713206fc97af3adc14def0cdd349df6000",
"sha256:f393cda562f79828f38a819f4788641ac7c4085f30f1ce1a68672baa686482bb",
"sha256:f917c1180fdb8623c2b75a99192f4025e412597c50b2ac870f156de8fb101119",
"sha256:fc78a84e2dfbc27afe4b2bd7c80c8db9bca75cc5b85df52bfe634596a1da846b",
"sha256:ff04ef6eec3eee8a5efef2401495967a916feaa353643defcc03fc74fe213b58"
], ],
"markers": "python_version >= '3.8'", "markers": "python_version >= '3.8'",
"version": "==1.17.2" "version": "==1.17.1"
} }
} }
} }
dbrepo-analyse-service/data/test_dt/large.csv deleted 100644 → 0
+ 0
− 210193
View file @ 46b9cac7
This diff is collapsed.
dbrepo-analyse-service/test/test_determine_dt.py
+ 0
− 9
View file @ c974fb2b
...@@ -234,15 +234,6 @@ class DetermineDatatypesTest(unittest.TestCase): ...@@ -234,15 +234,6 @@ class DetermineDatatypesTest(unittest.TestCase):
response = determine_datatypes(filename="separator.csv") response = determine_datatypes(filename="separator.csv")
self.assertEqual(";", response.separator) self.assertEqual(";", response.separator)
def test_determine_datatypes_separatorGuessLargeDataset_succeeds(self):
# mock
S3Client().upload_file("large.csv", './data/test_dt/', 'dbrepo')
# test
response = determine_datatypes(filename="large.csv")
self.assertEqual(",", response.separator)
def test_determine_datatypes_separatorGuessText_succeeds(self): def test_determine_datatypes_separatorGuessText_succeeds(self):
exp = AnalysisDto(separator=";", line_termination="\n", columns={ exp = AnalysisDto(separator=";", line_termination="\n", columns={
"id": { "id": {
......
dbrepo-auth-service/dbrepo-realm.json
+ 82
− 84
View file @ c974fb2b
This diff is collapsed.
dbrepo-auth-service/init/.gitignore 0 → 100644
+ 5
− 0
View file @ c974fb2b
# IDE
.idea/
# artifacts
.coverage
report.xml
\ No newline at end of file
dbrepo-auth-service/init/Dockerfile
+ 4
− 1
View file @ c974fb2b
FROM python:3.11-alpine FROM python:3.11-alpine
LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at" LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at"
RUN apk add --no-cache alpine-sdk gcc python3-dev mariadb-connector-c-dev RUN apk add --no-cache alpine-sdk \
gcc \
python3-dev \
mariadb-connector-c-dev
WORKDIR /home/alpine WORKDIR /home/alpine
......
dbrepo-auth-service/init/Pipfile
+ 6
− 1
View file @ c974fb2b
...@@ -4,8 +4,13 @@ verify_ssl = true ...@@ -4,8 +4,13 @@ verify_ssl = true
name = "pypi" name = "pypi"
[packages] [packages]
python-keycloak = "*" requests = "*"
mariadb = "*" mariadb = "*"
[dev-packages]
coverage = "*"
pytest = "*"
requests-mock = "*"
[requires] [requires]
python_version = "3.11" python_version = "3.11"
dbrepo-auth-service/init/Pipfile.lock
+ 326
− 302
View file @ c974fb2b
This diff is collapsed.
dbrepo-auth-service/init/app.py
+ 40
− 33
View file @ c974fb2b
import logging
import os import os
import sys
import mariadb import mariadb
from keycloak import KeycloakAdmin from requests import post, get
endpoint = os.getenv('AUTH_SERVICE_ENDPOINT', 'http://localhost:8080')
system_username = os.getenv('SYSTEM_USERNAME', 'admin') system_username = os.getenv('SYSTEM_USERNAME', 'admin')
admin = KeycloakAdmin(server_url=os.getenv('AUTH_SERVICE_ENDPOINT', 'http://localhost:8080'),
username=os.getenv('AUTH_SERVICE_ADMIN', 'admin'), def fetch() -> str:
password=os.getenv('AUTH_SERVICE_ADMIN_PASSWORD', 'admin'), print(f'Fetching user id of internal user with username: {system_username}')
verify=True) response = post(url=f'{endpoint}/realms/master/protocol/openid-connect/token', data=dict({
keycloak_user_id = admin.get_user_id(username=system_username) 'username': os.getenv('AUTH_SERVICE_ADMIN', 'admin'),
logging.info(f'Successfully fetched keycloak user id: {keycloak_user_id}') 'password': os.getenv('AUTH_SERVICE_ADMIN_PASSWORD', 'admin'),
ldap_user = admin.get_user(user_id=keycloak_user_id) 'grant_type': 'password',
if ldap_user is None: 'client_id': 'admin-cli'
logging.error(f'Failed to obtain user') }))
sys.exit(1)
ldap_user_attrs = ldap_user.get('attributes') if response.status_code != 200:
if ldap_user_attrs is None: raise IOError(f'Failed to obtain admin token: {response.status_code}')
logging.error(f'Failed to obtain user attributes')
sys.exit(1) response = get(url=f'{endpoint}/admin/realms/dbrepo/users/?username={system_username}', headers=dict({
'Authorization': f'Bearer {response.json()["access_token"]}'
}))
if response.status_code != 200 or len(response.json()) != 1:
raise FileNotFoundError(f'Failed to obtain user')
ldap_user = response.json()[0]
print(f'Successfully fetched user id: {ldap_user["id"]}')
if 'attributes' not in ldap_user or ldap_user['attributes'] is None:
raise ModuleNotFoundError(f'Failed to obtain user attributes: {ldap_user}')
ldap_user_attrs = ldap_user['attributes']
if 'LDAP_ID' not in ldap_user_attrs: if 'LDAP_ID' not in ldap_user_attrs:
logging.error(f'Failed to obtain ldap id: LDAP_ID not in attributes {ldap_user_attrs}') raise ImportError(f'Failed to obtain ldap id: LDAP_ID not in attributes {ldap_user_attrs}')
sys.exit(1)
if len(ldap_user_attrs['LDAP_ID']) != 1: if len(ldap_user_attrs['LDAP_ID']) != 1:
logging.error(f'Failed to obtain ldap id: wrong length {len(ldap_user_attrs["LDAP_ID"])} != 1') raise EnvironmentError(f'Failed to obtain ldap id: wrong length {len(ldap_user_attrs["LDAP_ID"])} != 1')
sys.exit(1)
ldap_user_id = ldap_user_attrs['LDAP_ID'][0] ldap_user_id = ldap_user_attrs['LDAP_ID'][0]
logging.info(f'Successfully fetched ldap user id: {ldap_user_id}') print(f'Successfully fetched ldap user id: {ldap_user_id}')
return ldap_user_id
try:
def save(user_id: str) -> None:
conn = mariadb.connect(user=os.getenv('METADATA_USERNAME', 'root'), conn = mariadb.connect(user=os.getenv('METADATA_USERNAME', 'root'),
password=os.getenv('METADATA_DB_PASSWORD', 'dbrepo'), password=os.getenv('METADATA_DB_PASSWORD', 'dbrepo'),
host="metadata-db", host="metadata-db",
...@@ -39,12 +47,11 @@ try: ...@@ -39,12 +47,11 @@ try:
cursor = conn.cursor() cursor = conn.cursor()
cursor.execute( cursor.execute(
"INSERT IGNORE INTO `mdb_users` (`id`, `username`, `email`, `mariadb_password`, `is_internal`) VALUES (?, ?, LEFT(UUID(), 20), PASSWORD(LEFT(UUID(), 20)), true)", "INSERT IGNORE INTO `mdb_users` (`id`, `username`, `email`, `mariadb_password`, `is_internal`) VALUES (?, ?, LEFT(UUID(), 20), PASSWORD(LEFT(UUID(), 20)), true)",
(ldap_user_id, system_username)) (user_id, system_username))
conn.commit() conn.commit()
conn.close() conn.close()
except mariadb.Error as e:
logging.info(f"Error connecting to MariaDB Platform: {e}")
exit(1)
logging.info(f'Successfully inserted user')
exit(0) if __name__ == '__main__':
save(fetch())
print(f'Successfully inserted user')
dbrepo-auth-service/init/test/test_unit_app.py 0 → 100644
+ 135
− 0
View file @ c974fb2b
import unittest
import requests_mock
from app import endpoint, fetch
class AppUnitTest(unittest.TestCase):
token_res = {
"access_token": "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJETU9zNU5UamEyTjZOQW4tX080QjdqeGdHNTBhbGY0bDRkWWRVWm14SUprIn0.eyJleHAiOjE3MzczODI1MjQsImlhdCI6MTczNzM4MjQ2NCwianRpIjoiNzUxZGY2ZDAtZWYzMy00MDkzLTgyYzgtNGU2MTRiMjc5NmMzIiwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgwL3JlYWxtcy9tYXN0ZXIiLCJzdWIiOiJkMzFjN2VlMS1iODI3LTQ5ZGMtYmZhYS04ZjljZmU4ZDY4NWQiLCJ0eXAiOiJCZWFyZXIiLCJhenAiOiJhZG1pbi1jbGkiLCJzZXNzaW9uX3N0YXRlIjoiYWU2NGQyYmQtMzIyNS00ZTA1LTk5NDMtMmJiOTFmYjhmZTUyIiwic2NvcGUiOiJwcm9maWxlIGVtYWlsIiwic2lkIjoiYWU2NGQyYmQtMzIyNS00ZTA1LTk5NDMtMmJiOTFmYjhmZTUyIn0.hBFe36TbbHLbvYckWYwVfs1AHrenRu2sC-GPjoJFb_5vKrmJBibnHQ8B3ZS2Y2NdsmWgZUwCgu-t6rZ7boAq1K3AuLQgAUQjr0W63aVwTgg-RAnUX5QV2PZO39U9QPuB_EgfKyeqTlw-9wKLC1zWaqHc1JwaudlaoreKnRVhoy-Y6lhglyqgVWmvoEPRDjBx7Q4JQQEIuRwLuYT-c6TTtginMdwSMWZ3cTv9tXf23pUUUkugtAA0Z9cCQxuD9zeKG2YunWjDic1ZIIEwO0IxD-lP_yvtB2expolcyzgPe80Z8Psigtnoaoiti_ERqqP8DRcPVdsXEpBrQhlC8Wj86A",
"expires_in": 60,
"refresh_expires_in": 1800,
"refresh_token": "eyJhbGciOiJIUzUxMiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJkZmY2Y2NjNS1iZDliLTQ2N2MtOGU5My0wZmU1MWQzNTNlYWYifQ.eyJleHAiOjE3MzczODQyNjQsImlhdCI6MTczNzM4MjQ2NCwianRpIjoiMGU3NGQ1YzctNTg3ZC00ZTc0LTljN2UtNjIzNDhjNjEzZmYyIiwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgwL3JlYWxtcy9tYXN0ZXIiLCJhdWQiOiJodHRwOi8vbG9jYWxob3N0OjgwODAvcmVhbG1zL21hc3RlciIsInN1YiI6ImQzMWM3ZWUxLWI4MjctNDlkYy1iZmFhLThmOWNmZThkNjg1ZCIsInR5cCI6IlJlZnJlc2giLCJhenAiOiJhZG1pbi1jbGkiLCJzZXNzaW9uX3N0YXRlIjoiYWU2NGQyYmQtMzIyNS00ZTA1LTk5NDMtMmJiOTFmYjhmZTUyIiwic2NvcGUiOiJwcm9maWxlIGVtYWlsIiwic2lkIjoiYWU2NGQyYmQtMzIyNS00ZTA1LTk5NDMtMmJiOTFmYjhmZTUyIn0.iwmBDa5K5PKxxB8YT5VhA4gS_EVgaGa5gmZ-1ySAPX36jxPpaVGb905ApFZxIQMYMYjo3D1w0H2Uoij8eOWiPw",
"token_type": "Bearer",
"not-before-policy": 0,
"session_state": "ae64d2bd-3225-4e05-9943-2bb91fb8fe52",
"scope": "profile email"
}
user_res = [
{"id": "5b516520-67cb-4aa0-86a6-d12f8b8f1a20",
"username": "admin",
"firstName": "User1",
"lastName": "Bar1",
"emailVerified": False,
"attributes": {"LDAP_ENTRY_DN": ["cn=admin,ou=users,dc=dbrepo,dc=at"],
"createTimestamp": ["20250120141013Z"],
"modifyTimestamp": ["20250120141013Z"],
"LDAP_ID": ["02b6e096-6b84-103f-81f6-1f6da137f2bb"]},
"createdTimestamp": 1737382606939,
"enabled": True,
"totp": False,
"federationLink": "c109d473-5ce1-4032-af7b-02e5442f5c07",
"disableableCredentialTypes": [],
"requiredActions": [],
"notBefore": 0,
"access": {"manageGroupMembership": True,
"view": True,
"mapRoles": True,
"impersonate": True,
"manage": True}}]
def test_fetch_succeeds(self):
with requests_mock.Mocker() as mock:
# mock
mock.post(f'{endpoint}/realms/master/protocol/openid-connect/token', json=self.token_res, status_code=200)
mock.get(f'{endpoint}/admin/realms/dbrepo/users/?username=admin', json=self.user_res, status_code=200)
# test
user_id = fetch()
self.assertEqual("02b6e096-6b84-103f-81f6-1f6da137f2bb", user_id)
def test_fetch_token_bad_request_fails(self):
with requests_mock.Mocker() as mock:
# mock
mock.post(f'{endpoint}/realms/master/protocol/openid-connect/token', json=self.token_res, status_code=400)
# test
try:
fetch()
except IOError:
pass
def test_fetch_token_unauthorized_fails(self):
with requests_mock.Mocker() as mock:
# mock
mock.post(f'{endpoint}/realms/master/protocol/openid-connect/token', json=self.token_res, status_code=401)
# test
try:
fetch()
except IOError:
pass
def test_fetch_user_not_found_fails(self):
with requests_mock.Mocker() as mock:
# mock
mock.post(f'{endpoint}/realms/master/protocol/openid-connect/token', json=self.token_res, status_code=200)
mock.get(f'{endpoint}/admin/realms/dbrepo/users/?username=admin', json=[], status_code=200)
# test
try:
fetch()
except FileNotFoundError:
pass
def test_fetch_user_too_much_fails(self):
with requests_mock.Mocker() as mock:
# mock
mock.post(f'{endpoint}/realms/master/protocol/openid-connect/token', json=self.token_res, status_code=200)
mock.get(f'{endpoint}/admin/realms/dbrepo/users/?username=admin', json=[{}, {}], status_code=200)
# test
try:
fetch()
except FileNotFoundError:
pass
def test_fetch_user_not_ok_fails(self):
with requests_mock.Mocker() as mock:
# mock
mock.post(f'{endpoint}/realms/master/protocol/openid-connect/token', json=self.token_res, status_code=200)
mock.get(f'{endpoint}/admin/realms/dbrepo/users/?username=admin', json=[{}], status_code=202)
# test
try:
fetch()
except FileNotFoundError:
pass
def test_fetch_user_no_attrs_fails(self):
with requests_mock.Mocker() as mock:
# mock
mock.post(f'{endpoint}/realms/master/protocol/openid-connect/token', json=self.token_res, status_code=200)
mock.get(f'{endpoint}/admin/realms/dbrepo/users/?username=admin', json=[{
"id": "5b516520-67cb-4aa0-86a6-d12f8b8f1a20"
}], status_code=200)
# test
try:
fetch()
except ModuleNotFoundError:
pass
def test_fetch_user_empty_attrs_fails(self):
with requests_mock.Mocker() as mock:
# mock
mock.post(f'{endpoint}/realms/master/protocol/openid-connect/token', json=self.token_res, status_code=200)
mock.get(f'{endpoint}/admin/realms/dbrepo/users/?username=admin', json=[{
"id": "5b516520-67cb-4aa0-86a6-d12f8b8f1a20",
"attributes": {}
}], status_code=200)
# test
try:
fetch()
except ImportError:
pass
dbrepo-auth-service/master-realm.json
+ 2179
− 2800
View file @ c974fb2b
This diff is collapsed.
dbrepo-data-service/rest-service/src/main/resources/init/querystore.sql
+ 0
− 2
View file @ c974fb2b
CREATE SEQUENCE `qs_queries_seq` NOCACHE;
CREATE TABLE `qs_queries` ( `id` bigint not null primary key default nextval(`qs_queries_seq`), `created` datetime not null default now(), `executed` datetime not null default now(), `created_by` varchar(36), `query` text not null, `query_normalized` text not null, `is_persisted` boolean not null, `query_hash` varchar(255) not null, `result_hash` varchar(255), `result_number` bigint ) WITH SYSTEM VERSIONING;
CREATE PROCEDURE hash_table(IN name VARCHAR(255), OUT hash VARCHAR(255), OUT count BIGINT) BEGIN DECLARE _sql TEXT; SELECT CONCAT('SELECT SHA2(GROUP_CONCAT(CONCAT_WS(\'\',', GROUP_CONCAT(CONCAT('`', column_name, '`') ORDER BY column_name), ') SEPARATOR \',\'), 256) AS hash, COUNT(*) AS count FROM `', name, '` INTO @hash, @count;') FROM `information_schema`.`columns` WHERE `table_schema` = DATABASE() AND `table_name` = name INTO _sql; PREPARE stmt FROM _sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; SET hash = @hash; SET count = @count; END; CREATE PROCEDURE hash_table(IN name VARCHAR(255), OUT hash VARCHAR(255), OUT count BIGINT) BEGIN DECLARE _sql TEXT; SELECT CONCAT('SELECT SHA2(GROUP_CONCAT(CONCAT_WS(\'\',', GROUP_CONCAT(CONCAT('`', column_name, '`') ORDER BY column_name), ') SEPARATOR \',\'), 256) AS hash, COUNT(*) AS count FROM `', name, '` INTO @hash, @count;') FROM `information_schema`.`columns` WHERE `table_schema` = DATABASE() AND `table_name` = name INTO _sql; PREPARE stmt FROM _sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; SET hash = @hash; SET count = @count; END;
CREATE PROCEDURE store_query(IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _username varchar(255) DEFAULT REGEXP_REPLACE(current_user(), '@.*', ''); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END; CREATE PROCEDURE store_query(IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _username varchar(255) DEFAULT REGEXP_REPLACE(current_user(), '@.*', ''); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END;
CREATE DEFINER = 'root' PROCEDURE _store_query(IN _username VARCHAR(255), IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END; CREATE DEFINER = 'root' PROCEDURE _store_query(IN _username VARCHAR(255), IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END;
\ No newline at end of file
docker-compose.yml
+ 2
− 3
View file @ c974fb2b
...@@ -80,7 +80,7 @@ services: ...@@ -80,7 +80,7 @@ services:
restart: "no" restart: "no"
container_name: dbrepo-auth-service container_name: dbrepo-auth-service
hostname: auth-service hostname: auth-service
image: bitnami/keycloak:26.0.0-debian-12-r1 image: bitnami/keycloak:24.0.5-debian-12-r8
volumes: volumes:
- ./dbrepo-auth-service/import-realms.sh:/docker-entrypoint-initdb.d/import-realms.sh - ./dbrepo-auth-service/import-realms.sh:/docker-entrypoint-initdb.d/import-realms.sh
- ./dbrepo-auth-service/master-realm.json:/opt/keycloak/data/import/master-realm.json - ./dbrepo-auth-service/master-realm.json:/opt/keycloak/data/import/master-realm.json
...@@ -90,13 +90,12 @@ services: ...@@ -90,13 +90,12 @@ services:
environment: environment:
KEYCLOAK_ENABLE_HTTPS: "false" KEYCLOAK_ENABLE_HTTPS: "false"
KEYCLOAK_ENABLE_STATISTICS: "true" KEYCLOAK_ENABLE_STATISTICS: "true"
KEYCLOAK_ENABLE_HEALTH_ENDPOINTS: "true"
KEYCLOAK_DATABASE_HOST: "auth-db" KEYCLOAK_DATABASE_HOST: "auth-db"
KEYCLOAK_DATABASE_NAME: "${AUTH_DB_NAME:-keycloak}" KEYCLOAK_DATABASE_NAME: "${AUTH_DB_NAME:-keycloak}"
KEYCLOAK_DATABASE_USER: "${AUTH_DB_USERNAME:-keycloak}" KEYCLOAK_DATABASE_USER: "${AUTH_DB_USERNAME:-keycloak}"
KEYCLOAK_DATABASE_PASSWORD: "${AUTH_DB_PASSWORD:-dbrepo}" KEYCLOAK_DATABASE_PASSWORD: "${AUTH_DB_PASSWORD:-dbrepo}"
healthcheck: healthcheck:
test: curl --head -fsS http://localhost:9000/health/ready test: curl -fsS http://localhost:8080/realms/master
interval: 10s interval: 10s
timeout: 5s timeout: 5s
retries: 12 retries: 12
......
helm/dbrepo/Chart.lock
+ 8
− 8
View file @ c974fb2b
...@@ -6,25 +6,25 @@ dependencies: ...@@ -6,25 +6,25 @@ dependencies:
repository: https://charts.bitnami.com/bitnami repository: https://charts.bitnami.com/bitnami
version: 21.6.1 version: 21.6.1
- name: mariadb-galera - name: mariadb-galera
repository: oci://registry-1.docker.io/bitnamicharts repository: https://charts.bitnami.com/bitnami
version: 13.2.7 version: 13.2.7
- name: mariadb-galera - name: mariadb-galera
repository: oci://registry-1.docker.io/bitnamicharts repository: https://charts.bitnami.com/bitnami
version: 13.2.7 version: 13.2.7
- name: rabbitmq - name: rabbitmq
repository: oci://registry-1.docker.io/bitnamicharts repository: https://charts.bitnami.com/bitnami
version: 14.0.0 version: 14.0.0
- name: seaweedfs - name: seaweedfs
repository: file://../seaweedfs repository: file://../seaweedfs
version: 4.2.1 version: 4.2.1
- name: grafana - name: grafana
repository: oci://registry-1.docker.io/bitnamicharts repository: https://charts.bitnami.com/bitnami
version: 11.4.2 version: 11.4.2
- name: prometheus - name: prometheus
repository: oci://registry-1.docker.io/bitnamicharts repository: https://charts.bitnami.com/bitnami
version: 1.3.22 version: 1.3.22
- name: nginx - name: nginx
repository: oci://registry-1.docker.io/bitnamicharts repository: https://charts.bitnami.com/bitnami
version: 18.3.1 version: 18.3.1
digest: sha256:f244730fab10d52050634ce3286413d378b92e4dc97b0ad2951295c0d2971146 digest: sha256:414c043a3751945d7bd5b02fa00ee0464bee7f08efb469e00a5f059cdbff03b5
generated: "2025-01-14T16:48:00.637443736+01:00" generated: "2025-01-19T17:22:48.686050629+01:00"
helm/dbrepo/README.md
+ 16
− 16
View file @ c974fb2b
...@@ -65,7 +65,7 @@ The command removes all the Kubernetes components associated with the chart and ...@@ -65,7 +65,7 @@ The command removes all the Kubernetes components associated with the chart and
### Metadata Database ### Metadata Database
| Name | Description | Value | | Name | Description | Value |
| ---------------------------------------- |----------------------------------------------------------------------------------------------------------------------------------------| ---------------------------------------------------------------------- | | ---------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- |
| `metadatadb.enabled` | Enable the Metadata datadb. | `true` | | `metadatadb.enabled` | Enable the Metadata datadb. | `true` |
| `metadatadb.host` | The hostname for the microservices. | `metadata-db` | | `metadatadb.host` | The hostname for the microservices. | `metadata-db` |
| `metadatadb.extraFlags` | Extra flags to ensure the query store works as intended, ref https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.6/api/data-db/#data | `--character-set-server=utf8mb4 --collation-server=utf8mb4_general_ci` | | `metadatadb.extraFlags` | Extra flags to ensure the query store works as intended, ref https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.6/api/data-db/#data | `--character-set-server=utf8mb4 --collation-server=utf8mb4_general_ci` |
...@@ -82,7 +82,7 @@ The command removes all the Kubernetes components associated with the chart and ...@@ -82,7 +82,7 @@ The command removes all the Kubernetes components associated with the chart and
### Auth Service ### Auth Service
| Name | Description | Value | | Name | Description | Value |
| ---------------------------------- | ----------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | -------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `authservice.enabled` | Enable the Auth Service. | `true` | | `authservice.enabled` | Enable the Auth Service. | `true` |
| `authservice.image.debug` | Set the logging level to `trace`. Otherwise, set to `info`. | `false` | | `authservice.image.debug` | Set the logging level to `trace`. Otherwise, set to `info`. | `false` |
| `authservice.endpoint` | The hostname for the microservices. | `http://auth-service` | | `authservice.endpoint` | The hostname for the microservices. | `http://auth-service` |
...@@ -92,13 +92,13 @@ The command removes all the Kubernetes components associated with the chart and ...@@ -92,13 +92,13 @@ The command removes all the Kubernetes components associated with the chart and
| `authservice.tls.existingSecret` | The secret containing the `tls.crt`, `tls.key` and `ca.crt`. | `ingress-cert` | | `authservice.tls.existingSecret` | The secret containing the `tls.crt`, `tls.key` and `ca.crt`. | `ingress-cert` |
| `authservice.client.id` | The client id for the microservices. | `dbrepo-client` | | `authservice.client.id` | The client id for the microservices. | `dbrepo-client` |
| `authservice.client.secret` | The client secret for the microservices. | `MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG` | | `authservice.client.secret` | The client secret for the microservices. | `MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG` |
| `authservice.init.resourcesPreset` | The container resource preset | `nano` | | `authservice.setupJob.resourcesPreset` | The container resource preset | `nano` |
| `authservice.init.resources` | Set container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` | | `authservice.setupJob.resources` | Set container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` |
### Data Database ### Data Database
| Name | Description | Value | | Name | Description | Value |
| ------------------------------------ |----------------------------------------------------------------------------------------------------------------------------------------| ---------------------------------------------------------------------- | | ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- |
| `datadb.host` | The hostname for the microservices. | `data-db` | | `datadb.host` | The hostname for the microservices. | `data-db` |
| `datadb.extraFlags` | Extra flags to ensure the query store works as intended, ref https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.6/api/data-db/#data | `--character-set-server=utf8mb4 --collation-server=utf8mb4_general_ci` | | `datadb.extraFlags` | Extra flags to ensure the query store works as intended, ref https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.6/api/data-db/#data | `--character-set-server=utf8mb4 --collation-server=utf8mb4_general_ci` |
| `datadb.rootUser.user` | The root username. | `root` | | `datadb.rootUser.user` | The root username. | `root` |
...@@ -224,7 +224,7 @@ mqtt.prefetch = 10 ...@@ -224,7 +224,7 @@ mqtt.prefetch = 10
| `metadataservice.containerSecurityContext.readOnlyRootFilesystem` | Set container's Security Context readOnlyRootFilesystem | `false` | | `metadataservice.containerSecurityContext.readOnlyRootFilesystem` | Set container's Security Context readOnlyRootFilesystem | `false` |
| `metadataservice.containerSecurityContext.capabilities.drop` | Set container's Security Context runAsNonRoot | `["ALL"]` | | `metadataservice.containerSecurityContext.capabilities.drop` | Set container's Security Context runAsNonRoot | `["ALL"]` |
| `metadataservice.containerSecurityContext.seccompProfile.type` | Set container's Security Context seccomp profile | `RuntimeDefault` | | `metadataservice.containerSecurityContext.seccompProfile.type` | Set container's Security Context seccomp profile | `RuntimeDefault` |
| `metadataservice.resourcesPreset` | The container resource preset | `micro` | | `metadataservice.resourcesPreset` | The container resource preset | `small` |
| `metadataservice.resources` | Set container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` | | `metadataservice.resources` | Set container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` |
| `metadataservice.endpoint` | The Metadata Service endpoint. | `http://metadata-service` | | `metadataservice.endpoint` | The Metadata Service endpoint. | `http://metadata-service` |
| `metadataservice.crossref.endpoint` | The CrossRef endpoint. | `http://data.crossref.org` | | `metadataservice.crossref.endpoint` | The CrossRef endpoint. | `http://data.crossref.org` |
......
helm/dbrepo/charts/mariadb-galera-13.2.7.tgz
+ 0
− 0
View file @ c974fb2b
No preview for this file type
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment