diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fd7117f1c3064cccc916cf768a31e9ce95a13205..f571fba1d16722ed56e3152dbe634272ebcbd284 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -266,7 +266,7 @@ test-search-service:
script:
- "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy"
- - cd ./dbrepo-search-service/ && coverage run -m pytest test/test_opensearch_client.py --junitxml=report.xml && coverage html && coverage report > ./coverage.txt
+ - cd ./dbrepo-search-service/ && coverage run -m pytest test/test_app.py test/test_jwt.py test/test_opensearch_client.py test/test_keycloak_client.py --junitxml=report.xml && coverage html && coverage report > ./coverage.txt
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts:
when: always
@@ -290,7 +290,7 @@ test-search-service-init:
script:
- "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy"
- - cd ./dbrepo-search-service/init/ && coverage run -m pytest ./test/test_app.py --junitxml=report.xml && coverage html && coverage report > ./coverage.txt
+ - cd ./dbrepo-search-service/init/ && coverage run -m pytest test/test_app.py --junitxml=report.xml && coverage html && coverage report > ./coverage.txt
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts:
when: always
diff --git a/dbrepo-search-service/.coveragerc b/dbrepo-search-service/.coveragerc
index 97830be8d6132cc05a9e96bea0f8a7a7bc8f8365..4683a93d3748d16ab20a61f318e3016d3f4a8e09 100644
--- a/dbrepo-search-service/.coveragerc
+++ b/dbrepo-search-service/.coveragerc
@@ -1,4 +1,5 @@
[report]
omit =
*/test/*
- */omlib/*
\ No newline at end of file
+ */omlib/*
+ */init/*
\ No newline at end of file
diff --git a/dbrepo-search-service/.gitignore b/dbrepo-search-service/.gitignore
index 3795a9e85157cb0caaed500eb74e852b14fd11e7..78c8fdf6e56a9e006504b1f2345abdacbf6e7bb4 100644
--- a/dbrepo-search-service/.gitignore
+++ b/dbrepo-search-service/.gitignore
@@ -9,8 +9,8 @@ __pycache__/
# Generated
coverage.txt
report.xml
-^clients/
-^omlib/
+clients/
+omlib/
# Libraries
./lib/dbrepo-1.4.4*
diff --git a/dbrepo-search-service/Pipfile b/dbrepo-search-service/Pipfile
index 298815c4b21885d64092b9f736fe4fb9516c8b1b..e74391ce665a39201bd480819e31a712fe82a6c7 100644
--- a/dbrepo-search-service/Pipfile
+++ b/dbrepo-search-service/Pipfile
@@ -24,6 +24,7 @@ gunicorn = "*"
[dev-packages]
coverage = "*"
pytest = "*"
+requests-mock = "*"
[requires]
python_version = "3.11"
diff --git a/dbrepo-search-service/Pipfile.lock b/dbrepo-search-service/Pipfile.lock
index 507877dba8a2d664e5ade5fe58431e08831418bf..123e864f6dd050a838ae2c7e9137a4d7b3f1e27f 100644
--- a/dbrepo-search-service/Pipfile.lock
+++ b/dbrepo-search-service/Pipfile.lock
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
- "sha256": "e82f3e09ac14fafd1be12c5b57dfcbfafecfd7c8ac2d098ce101264da2d22d42"
+ "sha256": "491e5f6ada48e8af417dfa7d6a0b4d98ccf9b9072df53b44d8de014b687fc80c"
},
"pipfile-spec": 6,
"requires": {
@@ -390,7 +390,6 @@
"hashes": [
"sha256:84607677b0826bb9b2fa120aacdf56d16c8d9ae423f435b2bd2c22b1c965a33c"
],
- "markers": "python_version >= '3.11'",
"path": "./lib/dbrepo-1.4.7.tar.gz"
},
"docker": {
@@ -616,7 +615,7 @@
"sha256:f406b22b7c9a9b4f8aa9d2ab13d6ae0ac3e85c9a809bd590ad53fed2bf70dc79",
"sha256:f6ff3b14f2df4c41660a7dec01045a045653998784bf8cfcb5a525bdffffbc8f"
],
- "markers": "python_version < '3.13' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))",
+ "markers": "python_version < '3.13' and (platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32'))))))",
"version": "==3.1.1"
},
"gunicorn": {
@@ -1265,7 +1264,7 @@
"sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3",
"sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"
],
- "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
"version": "==2.9.0.post0"
},
"python-dotenv": {
@@ -1482,7 +1481,7 @@
"sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
"sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
],
- "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
"version": "==1.16.0"
},
"sqlalchemy": {
@@ -1794,6 +1793,125 @@
}
},
"develop": {
+ "certifi": {
+ "hashes": [
+ "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8",
+ "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==2024.8.30"
+ },
+ "charset-normalizer": {
+ "hashes": [
+ "sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621",
+ "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6",
+ "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8",
+ "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912",
+ "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c",
+ "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b",
+ "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d",
+ "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d",
+ "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95",
+ "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e",
+ "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565",
+ "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64",
+ "sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab",
+ "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be",
+ "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e",
+ "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907",
+ "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0",
+ "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2",
+ "sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62",
+ "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62",
+ "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23",
+ "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc",
+ "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284",
+ "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca",
+ "sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455",
+ "sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858",
+ "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b",
+ "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594",
+ "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc",
+ "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db",
+ "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b",
+ "sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea",
+ "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6",
+ "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920",
+ "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749",
+ "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7",
+ "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd",
+ "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99",
+ "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242",
+ "sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee",
+ "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129",
+ "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2",
+ "sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51",
+ "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee",
+ "sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8",
+ "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b",
+ "sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613",
+ "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742",
+ "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe",
+ "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3",
+ "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5",
+ "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631",
+ "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7",
+ "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15",
+ "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c",
+ "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea",
+ "sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417",
+ "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250",
+ "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88",
+ "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca",
+ "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa",
+ "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99",
+ "sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149",
+ "sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41",
+ "sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574",
+ "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0",
+ "sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f",
+ "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d",
+ "sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654",
+ "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3",
+ "sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19",
+ "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90",
+ "sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578",
+ "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9",
+ "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1",
+ "sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51",
+ "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719",
+ "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236",
+ "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a",
+ "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c",
+ "sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade",
+ "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944",
+ "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc",
+ "sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6",
+ "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6",
+ "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27",
+ "sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6",
+ "sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2",
+ "sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12",
+ "sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf",
+ "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114",
+ "sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7",
+ "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf",
+ "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d",
+ "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b",
+ "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed",
+ "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03",
+ "sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4",
+ "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67",
+ "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365",
+ "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a",
+ "sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748",
+ "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b",
+ "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079",
+ "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482"
+ ],
+ "markers": "python_full_version >= '3.7.0'",
+ "version": "==3.4.0"
+ },
"coverage": {
"hashes": [
"sha256:04f2189716e85ec9192df307f7c255f90e78b6e9863a03223c3b998d24a3c6c6",
@@ -1863,6 +1981,14 @@
"markers": "python_version >= '3.9'",
"version": "==7.6.3"
},
+ "idna": {
+ "hashes": [
+ "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9",
+ "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==3.10"
+ },
"iniconfig": {
"hashes": [
"sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3",
@@ -1895,6 +2021,31 @@
"index": "pypi",
"markers": "python_version >= '3.8'",
"version": "==8.3.3"
+ },
+ "requests": {
+ "hashes": [
+ "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760",
+ "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"
+ ],
+ "markers": "python_version >= '3.8'",
+ "version": "==2.32.3"
+ },
+ "requests-mock": {
+ "hashes": [
+ "sha256:b1e37054004cdd5e56c84454cc7df12b25f90f382159087f4b6915aaeef39563",
+ "sha256:e9e12e333b525156e82a3c852f22016b9158220d2f47454de9cae8a77d371401"
+ ],
+ "index": "pypi",
+ "markers": "python_version >= '3.5'",
+ "version": "==1.12.1"
+ },
+ "urllib3": {
+ "hashes": [
+ "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac",
+ "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"
+ ],
+ "markers": "python_version >= '3.10'",
+ "version": "==2.2.3"
}
}
}
diff --git a/dbrepo-search-service/app.py b/dbrepo-search-service/app.py
index 56d2c711c0b7ac0218a1bf7d65f86888e18266d4..7566178526d97cd7e21a288edd1d9881e49ad9cc 100644
--- a/dbrepo-search-service/app.py
+++ b/dbrepo-search-service/app.py
@@ -2,6 +2,7 @@ import math
import os
import logging
from ast import literal_eval
+from json import dumps
from typing import List, Any
import requests
@@ -10,6 +11,7 @@ from flasgger import LazyJSONEncoder, Swagger, swag_from
from flask import Flask, request
from flask_cors import CORS
from flask_httpauth import HTTPTokenAuth, HTTPBasicAuth, MultiAuth
+from jwt.exceptions import JWTDecodeError
from opensearchpy import TransportError, NotFoundError
from prometheus_flask_exporter import PrometheusMetrics
from pydantic import ValidationError
@@ -206,9 +208,6 @@ app.config["OPENSEARCH_PASSWORD"] = os.getenv('OPENSEARCH_PASSWORD', 'admin')
app.json_encoder = LazyJSONEncoder
-available_types = literal_eval(
- os.getenv("COLLECTION", "['database','table','column','identifier','unit','concept','user','view']"))
-
@token_auth.verify_token
def verify_token(token: str):
@@ -217,7 +216,7 @@ def verify_token(token: str):
try:
client = KeycloakClient()
return client.verify_jwt(access_token=token)
- except AssertionError:
+ except JWTDecodeError as error:
return False
@@ -268,8 +267,7 @@ def general_filter(index, results):
"view": ["id", "name", "creator", " created"],
}
if index not in important_keys.keys():
- error_msg = "the keys to be returned to the user for your index aren't specified in the important Keys dict"
- raise KeyError(error_msg)
+ raise KeyError(f"Failed to find index {index} in: {important_keys.keys()}")
for result in results:
result_keys_copy = tuple(result.keys())
for key in result_keys_copy:
@@ -294,35 +292,37 @@ def get_index(index: str):
:return: list of the results
"""
logging.info(f'Searching for index: {index}')
- if index not in available_types:
- return ApiError(status='NOT_FOUND', message='Failed to find index',
- code='search.index.missing').model_dump(), 404
results = OpenSearchClient().query_index_by_term_opensearch("*", "contains")
- results = general_filter(index, results)
-
- results_per_page = min(request.args.get("results_per_page", 50, type=int), 500)
- max_pages = math.ceil(len(results) / results_per_page)
- page = min(request.args.get("page", 1, type=int), max_pages)
- results = results[(results_per_page * (page - 1)): (results_per_page * page)]
- return dict({"results": results}), 200
+ try:
+ results = general_filter(index, results)
+
+ results_per_page = min(request.args.get("results_per_page", 50, type=int), 500)
+ max_pages = math.ceil(len(results) / results_per_page)
+ page = min(request.args.get("page", 1, type=int), max_pages)
+ results = results[(results_per_page * (page - 1)): (results_per_page * page)]
+ return dict({"results": results}), 200
+ except KeyError:
+ return ApiError(status='NOT_FOUND', message=f'Failed to find get index: {index}',
+ code='search.index.missing').model_dump(), 404
-@app.route("/api/search/<string:type>/fields", methods=["GET"], endpoint="search_get_index_fields")
+@app.route("/api/search/<string:field_type>/fields", methods=["GET"], endpoint="search_get_index_fields")
@metrics.gauge(name='dbrepo_search_type_list', description='Time needed to list search types')
@swag_from("os-yml/get_fields.yml")
-def get_fields(type: str):
+def get_fields(field_type: str):
"""
returns a list of attributes of the data for a specific index.
- :param type: The search type
+ :param field_type: The search type
:return:
"""
- logging.info(f'Searching in index database for type: {type}')
- if type not in available_types:
- return ApiError(status='NOT_FOUND', message='Failed to find type',
+ logging.info(f'Searching in index database for type: {field_type}')
+ try:
+ fields = OpenSearchClient().get_fields_for_index(field_type)
+ logging.debug(f'get fields for field_type {field_type} resulted in {len(fields)} field(s)')
+ return fields, 200
+ except NotFoundError:
+ return ApiError(status='NOT_FOUND', message=f'Failed to find fields for search type {field_type}',
code='search.type.missing').model_dump(), 404
- fields = OpenSearchClient().get_fields_for_index(type)
- logging.debug(f'get fields for type {type} resulted in {len(fields)} field(s)')
- return fields, 200
@app.route("/api/search", methods=["GET"], endpoint="search_fuzzy_search")
@@ -344,10 +344,10 @@ def get_fuzzy_search():
return dict({"results": results}), 200
-@app.route("/api/search/<string:type>", methods=["POST"], endpoint="search_post_general_search")
+@app.route("/api/search/<string:field_type>", methods=["POST"], endpoint="search_post_general_search")
@metrics.gauge(name='dbrepo_search_type', description='Time needed to search by type')
@swag_from("os-yml/post_general_search.yml")
-def post_general_search(type):
+def post_general_search(field_type):
"""
Main endpoint for fuzzy searching.
:return:
@@ -356,11 +356,7 @@ def post_general_search(type):
return ApiError(status='UNSUPPORTED_MEDIA_TYPE', message='Content type needs to be application/json',
code='search.general.media').model_dump(), 415
req_body = request.json
- logging.info(f'Searching in index database for type: {type}')
- logging.debug(f"search request body: {req_body}")
- if type is not None and type not in available_types:
- return ApiError(status='NOT_FOUND', message=f'Type {type} is not in collection: {available_types}',
- code='search.general.missing').model_dump(), 404
+ logging.info(f'Searching in index database for type: {field_type}')
t1 = request.args.get("t1")
if not str(t1).isdigit():
t1 = None
@@ -370,9 +366,9 @@ def post_general_search(type):
if t1 is not None and t2 is not None and "unit.uri" in req_body and "concept.uri" in req_body:
response = OpenSearchClient().unit_independent_search(t1, t2, req_body)
else:
- response = OpenSearchClient().general_search(type, req_body)
+ response = OpenSearchClient().general_search(field_type, req_body)
# filter by type
- if type == 'table':
+ if field_type == 'table':
tmp = []
for database in response:
if database["tables"] is not None:
@@ -380,7 +376,7 @@ def post_general_search(type):
table["is_public"] = database["is_public"]
tmp.append(table)
response = tmp
- if type == 'identifier':
+ if field_type == 'identifier':
tmp = []
for database in response:
if database["identifiers"] is not None:
@@ -398,30 +394,30 @@ def post_general_search(type):
if 'identifier' in view:
tmp.append(view['identifier'])
response = tmp
- elif type == 'column':
+ elif field_type == 'column':
response = [x for xs in response for x in xs["tables"]]
for table in response:
for column in table["columns"]:
column["table_id"] = table["id"]
column["database_id"] = table["database_id"]
response = [x for xs in response for x in xs["columns"]]
- elif type == 'concept':
+ elif field_type == 'concept':
tmp = []
tables = [x for xs in response for x in xs["tables"]]
for column in [x for xs in tables for x in xs["columns"]]:
if 'concept' in column and column["concept"] is not None:
tmp.append(column["concept"])
response = tmp
- elif type == 'unit':
+ elif field_type == 'unit':
tmp = []
tables = [x for xs in response for x in xs["tables"]]
for column in [x for xs in tables for x in xs["columns"]]:
if 'unit' in column and column["unit"] is not None:
tmp.append(column["unit"])
response = tmp
- elif type == 'view':
+ elif field_type == 'view':
response = [x for xs in response for x in xs["views"]]
- return dict({'results': response, 'type': type}), 200
+ return dict({'results': response, 'type': field_type}), 200
@app.route("/api/search/database/<int:database_id>", methods=["PUT"], endpoint="search_put_database")
@@ -436,16 +432,9 @@ def update_database(database_id: int) -> Database | ApiError:
logging.error(f"Failed to validate: {e}")
return ApiError(status='BAD_REQUEST', message=f'Malformed payload: {e}',
code='search.general.missing').model_dump(), 400
- try:
- database = OpenSearchClient().update_database(database_id, payload)
- logging.info(f"Updated database with id : {database_id}")
- return database.model_dump(), 202
- except NotFoundError:
- return ApiError(status='NOT_FOUND', message='Failed to find database',
- code='search.database.missing').model_dump(), 404
- except TransportError:
- return ApiError(status='BAD_REQUEST', message='Failed to update database',
- code='search.database.invalid').model_dump(), 400
+ database = OpenSearchClient().update_database(database_id, payload)
+ logging.info(f"Updated database with id : {database_id}")
+ return database.model_dump(), 202
@app.route("/api/search/database/<int:database_id>", methods=["DELETE"], endpoint="database_delete_database")
@@ -455,7 +444,7 @@ def update_database(database_id: int) -> Database | ApiError:
def delete_database(database_id: int):
try:
OpenSearchClient().delete_database(database_id)
- return None, 202
+ return dumps({}), 202
except NotFoundError:
return ApiError(status='NOT_FOUND', message='Failed to find database',
code='search.database.missing').model_dump(), 404
diff --git a/dbrepo-search-service/init/clients/opensearch_client.py b/dbrepo-search-service/init/clients/opensearch_client.py
index 85bcedc03466165dba5b7721ab5c932bea616eea..7d25fcded5a29e87524523785133d7aaa56f314d 100644
--- a/dbrepo-search-service/init/clients/opensearch_client.py
+++ b/dbrepo-search-service/init/clients/opensearch_client.py
@@ -8,7 +8,7 @@ import logging
from dbrepo.api.dto import Database
from collections.abc import MutableMapping
-from opensearchpy import OpenSearch, TransportError, RequestError
+from opensearchpy import OpenSearch, TransportError, RequestError, NotFoundError
from omlib.measure import om
from omlib.constants import OM_IDS
@@ -67,16 +67,8 @@ class OpenSearchClient:
@throws: opensearchpy.exceptions.NotFoundError If the database was not found in the Search Database.
"""
logging.debug(f"updating database with id: {database_id} in search database")
- try:
- self._instance().index(index="database", id=database_id, body=dumps(data.model_dump()))
- except RequestError as e:
- logging.error(f"Failed to update in search database: {e.info}")
- raise e
- try:
- response: dict = self._instance().get(index="database", id=database_id)
- except TransportError as e:
- logging.error(f"Failed to get updated database in search database: {e.status_code}")
- raise e
+ self._instance().index(index="database", id=database_id, body=dumps(data.model_dump()))
+ response: dict = self._instance().get(index="database", id=database_id)
database = Database.parse_obj(response["_source"])
logging.info(f"Updated database with id {database_id} in index 'database'")
return database
@@ -118,10 +110,10 @@ class OpenSearchClient:
results = [hit["_source"] for hit in response["hits"]["hits"]]
return results
- def get_fields_for_index(self, type: str):
+ def get_fields_for_index(self, field_type: str):
"""
returns a list of attributes of the data for a specific index.
- :param type: The search type
+ :param field_type: The search type
:return: list of fields
"""
fields = {
@@ -134,8 +126,10 @@ class OpenSearchClient:
"view": "views.*",
"user": "creator.*",
}
- logging.debug(f'requesting field(s) {fields[type]} for filter: {type}')
- fields = self._instance().indices.get_field_mapping(fields[type])
+ if field_type not in fields.keys():
+ raise NotFoundError(f"Failed to find field type: {field_type}")
+ logging.debug(f'requesting field(s) {fields[field_type]} for filter: {field_type}')
+ fields = self._instance().indices.get_field_mapping(fields[field_type])
fields_list = []
fd = flatten_dict(fields)
for key in fd.keys():
@@ -169,13 +163,13 @@ class OpenSearchClient:
logging.info(f"Found {len(response['hits']['hits'])} result(s)")
return response
- def general_search(self, type: str = None, field_value_pairs: dict = None):
+ def general_search(self, field_type: str = None, field_value_pairs: dict = None):
"""
Main method for searching stuff in the opensearch db
all parameters are optional
- :param type: The index to be searched. Optional.
+ :param field_type: The index to be searched. Optional.
:param field_value_pairs: The key-value pair of properties that need to match. Optional.
:return: The object of results and HTTP status code. e.g. { "hits": { "hits": [] } }, 200
"""
@@ -204,7 +198,7 @@ class OpenSearchClient:
body = {
"query": {"bool": {"must": musts}}
}
- logging.debug(f'search in index database for type: {type}')
+ logging.debug(f'search in index database for type: {field_type}')
logging.debug(f'search body: {dumps(body)}')
response = self._instance().search(
index="database",
@@ -213,12 +207,10 @@ class OpenSearchClient:
results = [hit["_source"] for hit in response["hits"]["hits"]]
return results
- def unit_independent_search(self, t1=None, t2=None, field_value_pairs=None):
+ def unit_independent_search(self, t1: float, t2: float, field_value_pairs):
"""
Main method for searching stuff in the opensearch db
- all parameters are optional
-
:param t1: start value
:param t2: end value
:param field_value_pairs: the key-value pairs
@@ -240,6 +232,8 @@ class OpenSearchClient:
)
unit_uris = [hit["key"] for hit in response["aggregations"]["units"]["buckets"]]
logging.debug(f"found {len(unit_uris)} unit(s) in column index")
+ if len(unit_uris) == 0:
+ raise NotFoundError("Failed to search: no unit assigned")
base_unit = unit_uri_to_unit(field_value_pairs["unit.uri"])
for unit_uri in unit_uris:
gte = t1
diff --git a/dbrepo-search-service/test.sh b/dbrepo-search-service/test.sh
deleted file mode 100644
index b7d1ea07b7383f61390ddf602b11c0d0b6bca04f..0000000000000000000000000000000000000000
--- a/dbrepo-search-service/test.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash
-cd ./dbrepo-search-service
-pip install pipenv
-pipenv install --dev
-coverage run -m pytest ./test/test_opensearch_client.py
-echo "[INFO] Start testing ./init"
-(cd ./init && pipenv install --dev)
-coverage run -m pytest ./init/test/test_app.py
\ No newline at end of file
diff --git a/dbrepo-search-service/test/conftest.py b/dbrepo-search-service/test/conftest.py
index ab6b7e3a661b78d74545d30285c94608ce5bd662..1d603685d63e464a4ffe2f4aac005c2af319fc5a 100644
--- a/dbrepo-search-service/test/conftest.py
+++ b/dbrepo-search-service/test/conftest.py
@@ -1,7 +1,8 @@
import logging
+import os
import pytest
-import os
+import json
from testcontainers.opensearch import OpenSearchContainer
@@ -30,20 +31,17 @@ def session(request):
request.addfinalizer(stop_opensearch)
return container
-# @pytest.fixture(scope="function", autouse=True)
-# def cleanup(request, session):
-# """
-# Clean up after each test by removing the buckets and re-adding them (=so they are empty again)
-# :param request: /
-# :param session: /
-# :return:
-# """
-# logging.info("[fixture] truncate buckets")
-# for bucket in ["dbrepo-upload", "dbrepo-download"]:
-# objects = []
-# for obj in session.get_client().list_objects(bucket):
-# objects.append(DeleteObject(obj.object_name))
-# logging.info(f'request to remove objects {objects}')
-# errors = session.get_client().remove_objects(bucket, objects)
-# for error in errors:
-# raise ConnectionError(f'Failed to delete object with key {error.object_name} of bucket {bucket}')
+
+@pytest.fixture(scope="function", autouse=True)
+def cleanup(request, session):
+ """
+ Clean up after each test by removing the index and re-adding it (=so it's empty again)
+ :param request: /
+ :param session: /
+ :return:
+ """
+ logging.info("[fixture] clean schema")
+ with open('./init/database.json', 'r') as f:
+ if session.get_client().indices.exists(index="database"):
+ session.get_client().indices.delete(index="database")
+ session.get_client().indices.create(index="database", body=json.load(f))
diff --git a/dbrepo-search-service/test/rsa/rs256.key b/dbrepo-search-service/test/rsa/rs256.key
new file mode 100644
index 0000000000000000000000000000000000000000..86b3eaf5c6c4c6b83071b6d1e9d69cb22bcd4085
--- /dev/null
+++ b/dbrepo-search-service/test/rsa/rs256.key
@@ -0,0 +1,3 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQABAoIBADNcMt6hAHub4JTAYS6Mra0EPRBO2XhWmACBrv3+8ETClXd5475KPLDewgRVtlmtbwU8G8awUXESQgPS9lfiqvQhPreA3cHlm6oP2WMKOEtakr2s8I+frsTBLCo0Ini9RaSzjoVVgS0zofyhASKi+T970MafSj5P3XNb8YBFdXgoYDiA7FXLH6a/+m7LScL+wGcFMAAeYESxZbMQLfH3v8L+4EcTraiwjLG17ZdlF3dpybMyUSse6ZQ/PdlyvBuzzLXhN6Ce2gd9ATfS+YWTzo7Yf+GU+ex5bIpVOfHqtuM/hyq7YGKENClsXwNZIAoFnvGCbvECAfgyapVrD30IfykCgYEA0rgsSZ82pxT40NxwgBD1g9lbNVBKXphRB/3S078qusUzJjT7AldEj4imGPhAbI7bI8gAeWJsp1XJWkjM8ktaVrh+NQl7p8e9OPh0pQF/5Bdg8ajbjXESpjnaU66pVYRQy/d+jNli/YRAHX5RUfsBl+6W4+WSVMGmKBiqJsur+ecCgYEAz1YVXClcmUnyZem5B+2E9noIzjF6ROE+jIb6rawM85P3Xd0lXtECQavtxw+Qk7I32qOwrxl1UpK2foVel3pazi+4OpMfmqtYGenRP1Zk1cZwrDo0cIemTDGjj3kJ8tYn12CGolFQpJZgK6OHzvG0tOxI5VZgjIViWNPe1PGWXtUCgYEAxXGNDe8BZs1f11S2lUlOw5yGug3hoYFXbAWJ5p7Ziuf8ZXB/QlJDC7se54a11wKEk6Jzz0lKRgE8CjzszJuOqnN0zn10QGIIC7nCklo1W6QMUmPGVWH994N976tZP6gbjQL6sT+AYcvpx7j0ubxYYeRNvnz+ACzzY964kGGHY0ECgYEAumlwPPNnMN7+VEjGNm2D7UMdJZ3wi3tkjF5ThdA5uMohTsAk+FG80KSu3RmOaGyEsUwY7+VYyYvlDm4E9PZqLBVVczyR3rMNPAcwPd0EPfvzk7WlLkOX7ct3fehaXH3VRlyfz9KCSeh1wOZ/lT1VtpD2nVOC7PSDzs92+kfXZZ0CgYAnrD1y4skgXkdwolZ3unn3EFyGm2d+X5aMTHwQPdWxqoNIAl/9wdghlzihwnPhhsxq1WzlxuC3V2IMrNPtRx70Mi+FbSmR5m4Xx5RptgMtMlwno+L40PzNJgMjHGjt0wcx3Vel8wuohDtnqMyS7P5nG1/TQx0Cyzwn7QOXlNpgbQ==
+-----END RSA PRIVATE KEY-----
\ No newline at end of file
diff --git a/dbrepo-search-service/test/rsa/rsa256.pkey b/dbrepo-search-service/test/rsa/rsa256.pkey
new file mode 100644
index 0000000000000000000000000000000000000000..857dfb22beeac202c2955d7cc4f782b787492beb
--- /dev/null
+++ b/dbrepo-search-service/test/rsa/rsa256.pkey
@@ -0,0 +1,3 @@
+-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB
+-----END PUBLIC KEY-----
diff --git a/dbrepo-search-service/test/test_app.py b/dbrepo-search-service/test/test_app.py
new file mode 100644
index 0000000000000000000000000000000000000000..1b1af020987175fcd8894e61fd2e85519f42c998
--- /dev/null
+++ b/dbrepo-search-service/test/test_app.py
@@ -0,0 +1,300 @@
+import json
+import time
+import unittest
+import datetime
+
+import jwt
+from dbrepo.api.dto import Database, User, UserAttributes, Container, Image, Table, Constraints, Column, ColumnType, \
+ Concept, Unit
+
+from app import app
+
+req = Database(id=1,
+ name="Test",
+ internal_name="test_tuw1",
+ creator=User(id="c6b71ef5-2d2f-48b2-9d79-b8f23a3a0502",
+ username="foo",
+ attributes=UserAttributes(theme="dark")),
+ owner=User(id="c6b71ef5-2d2f-48b2-9d79-b8f23a3a0502",
+ username="foo",
+ attributes=UserAttributes(theme="dark")),
+ contact=User(id="c6b71ef5-2d2f-48b2-9d79-b8f23a3a0502",
+ username="foo",
+ attributes=UserAttributes(theme="dark")),
+ created=datetime.datetime(2024, 3, 25, 16, tzinfo=datetime.timezone.utc),
+ exchange_name="dbrepo",
+ is_public=True,
+ container=Container(id=1,
+ name="MariaDB",
+ internal_name="mariadb",
+ host="data-db",
+ port="3306",
+ created=datetime.datetime(2024, 3, 1, 10, tzinfo=datetime.timezone.utc),
+ sidecar_host="data-db-sidecar",
+ sidecar_port=3305,
+ image=Image(id=1,
+ registry="docker.io",
+ name="mariadb",
+ version="11.1.3",
+ dialect="org.hibernate.dialect.MariaDBDialect",
+ driver_class="org.mariadb.jdbc.Driver",
+ jdbc_method="mariadb",
+ default_port=3306)),
+ tables=[Table(id=1, database_id=1, name="Data", internal_name="data",
+ creator=User(id="c6b71ef5-2d2f-48b2-9d79-b8f23a3a0502",
+ username="foo",
+ attributes=UserAttributes(theme="dark")),
+ owner=User(id="c6b71ef5-2d2f-48b2-9d79-b8f23a3a0502",
+ username="foo",
+ attributes=UserAttributes(theme="dark")),
+ created=datetime.datetime(2024, 3, 1, 10, tzinfo=datetime.timezone.utc),
+ constraints=Constraints(uniques=[], foreign_keys=[], checks=[], primary_key=[]),
+ is_versioned=False,
+ created_by="c6b71ef5-2d2f-48b2-9d79-b8f23a3a0502",
+ queue_name="dbrepo",
+ routing_key="dbrepo.1.1",
+ is_public=True,
+ columns=[Column(id=1, database_id=1, table_id=1, name="ID", internal_name="id",
+ column_type=ColumnType.BIGINT, is_public=True, is_null_allowed=False,
+ size=20, d=0,
+ concept=Concept(id=1, uri="http://www.wikidata.org/entity/Q2221906",
+ created=datetime.datetime(2024, 3, 1, 10,
+ tzinfo=datetime.timezone.utc)),
+ unit=Unit(id=1,
+ uri="http://www.ontology-of-units-of-measure.org/resource/om-2/degreeCelsius",
+ created=datetime.datetime(2024, 3, 1, 10,
+ tzinfo=datetime.timezone.utc)),
+ val_min=0,
+ val_max=10)]
+ )])
+
+
+class JwtTest(unittest.TestCase):
+
+ def token(self, roles: [str], iat: int = int(time.time())):
+ claims = {
+ 'iat': iat,
+ 'realm_access': {
+ 'roles': roles
+ }
+ }
+ with open('test/rsa/rs256.key', 'rb') as fh:
+ return jwt.JWT().encode(claims, jwt.jwk_from_pem(fh.read()), alg='RS256')
+
+ def test_update_database_media_type_fails(self):
+ with app.test_client() as test_client:
+ # test
+ response = test_client.put('/api/search/database/1',
+ headers={'Authorization': f'Bearer {self.token(["update-search-index"])}'})
+ self.assertEqual(415, response.status_code)
+
+ def test_health_succeeds(self):
+ with app.test_client() as test_client:
+ # test
+ response = test_client.get('/health')
+ self.assertEqual(200, response.status_code)
+
+ def test_update_database_no_auth_fails(self):
+ with app.test_client() as test_client:
+ # test
+ response = test_client.put('/api/search/database/1')
+ self.assertEqual(401, response.status_code)
+
+ def test_update_database_no_body_fails(self):
+ with app.test_client() as test_client:
+ # test
+ response = test_client.put('/api/search/database/1',
+ headers={'Authorization': f'Bearer {self.token(["update-search-index"])}',
+ 'Content-Type': 'application/json'})
+ self.assertEqual(400, response.status_code)
+
+ def test_update_database_empty_body_fails(self):
+ with app.test_client() as test_client:
+ # test
+ response = test_client.put('/api/search/database/1',
+ headers={'Authorization': f'Bearer {self.token(["update-search-index"])}',
+ 'Content-Type': 'application/json'},
+ data={})
+ self.assertEqual(400, response.status_code)
+
+ def test_update_database_malformed_body_fails(self):
+ with app.test_client() as test_client:
+ # test
+ response = test_client.put('/api/search/database/1',
+ headers={'Authorization': f'Bearer {self.token(["update-search-index"])}',
+ 'Content-Type': 'application/json'},
+ data=dict({"id": 1}))
+ self.assertEqual(400, response.status_code)
+
+ def test_update_database_succeeds(self):
+ with app.test_client() as test_client:
+ # test
+ response = test_client.put('/api/search/database/1',
+ headers={'Authorization': f'Bearer {self.token(["update-search-index"])}',
+ 'Content-Type': 'application/json'},
+ data=req.model_dump_json())
+ self.assertEqual(202, response.status_code)
+
+ def test_get_fields_succeeds(self):
+ with app.test_client() as test_client:
+ # test
+ response = test_client.get('/api/search/database/fields', headers={'Content-Type': 'application/json'})
+ self.assertEqual(200, response.status_code)
+
+ def test_get_fields_fails(self):
+ with app.test_client() as test_client:
+ # test
+ response = test_client.get('/api/search/unknown/fields', headers={'Content-Type': 'application/json'})
+ self.assertEqual(404, response.status_code)
+
+ def test_delete_database_no_auth_fails(self):
+ with app.test_client() as test_client:
+ # test
+ response = test_client.delete('/api/search/database/1')
+ self.assertEqual(401, response.status_code)
+
+ def test_delete_database_no_role_fails(self):
+ with app.test_client() as test_client:
+ # test
+ response = test_client.delete('/api/search/database/1',
+ headers={'Authorization': f'Bearer {self.token([])}'})
+ self.assertEqual(403, response.status_code)
+
+ def test_delete_database_succeeds(self):
+ with app.test_client() as test_client:
+ # mock
+ test_client.put('/api/search/database/1',
+ headers={'Authorization': f'Bearer {self.token(["update-search-index"])}',
+ 'Content-Type': 'application/json'},
+ data=req.model_dump_json())
+ # test
+ response = test_client.delete('/api/search/database/1',
+ headers={'Authorization': f'Bearer {self.token(["admin"])}'})
+ self.assertEqual(202, response.status_code)
+
+ def test_delete_database_not_found_fails(self):
+ with app.test_client() as test_client:
+ # test
+ response = test_client.delete('/api/search/database/1',
+ headers={'Authorization': f'Bearer {self.token(["admin"])}'})
+ self.assertEqual(404, response.status_code)
+
+ def test_get_fuzzy_search_succeeds(self):
+ with app.test_client() as test_client:
+ # test
+ response = test_client.get('/api/search?q=test')
+ self.assertEqual(200, response.status_code)
+
+ def test_get_fuzzy_search_no_query_fails(self):
+ with app.test_client() as test_client:
+ # test
+ response = test_client.get('/api/search')
+ self.assertEqual(400, response.status_code)
+
+ def test_get_index_succeeds(self):
+ with app.test_client() as test_client:
+ # test
+ response = test_client.get('/api/search/table')
+ self.assertEqual(200, response.status_code)
+
+ def test_get_index_fails(self):
+ with app.test_client() as test_client:
+ # test
+ response = test_client.get('/api/search/unknown')
+ self.assertEqual(404, response.status_code)
+
+ def test_post_general_search_media_type_fails(self):
+ with app.test_client() as test_client:
+ # test
+ response = test_client.post('/api/search/database')
+ self.assertEqual(415, response.status_code)
+
+ def test_post_general_search_no_body_fails(self):
+ with app.test_client() as test_client:
+ # test
+ response = test_client.post('/api/search/database', headers={'Content-Type': 'application/json'})
+ self.assertEqual(400, response.status_code)
+
+ def test_post_general_search_succeeds(self):
+ with app.test_client() as test_client:
+ # mock
+ test_client.put('/api/search/database/1',
+ headers={'Authorization': f'Bearer {self.token(["update-search-index"])}',
+ 'Content-Type': 'application/json'},
+ data=req.model_dump_json())
+ # test
+ response = test_client.post('/api/search/database', headers={'Content-Type': 'application/json'},
+ data=json.dumps({'id': 1}))
+ self.assertEqual(200, response.status_code)
+
+ def test_post_general_search_table_succeeds(self):
+ with app.test_client() as test_client:
+ # mock
+ test_client.put('/api/search/database/1',
+ headers={'Authorization': f'Bearer {self.token(["update-search-index"])}',
+ 'Content-Type': 'application/json'},
+ data=req.model_dump_json())
+ # test
+ response = test_client.post('/api/search/table', headers={'Content-Type': 'application/json'},
+ data=json.dumps({'id': 1}))
+ self.assertEqual(200, response.status_code)
+
+ def test_post_general_search_column_succeeds(self):
+ with app.test_client() as test_client:
+ # mock
+ test_client.put('/api/search/database/1',
+ headers={'Authorization': f'Bearer {self.token(["update-search-index"])}',
+ 'Content-Type': 'application/json'},
+ data=req.model_dump_json())
+ # test
+ response = test_client.post('/api/search/column', headers={'Content-Type': 'application/json'},
+ data=json.dumps({'id': 1}))
+ self.assertEqual(200, response.status_code)
+
+ def test_post_general_search_identifier_succeeds(self):
+ with app.test_client() as test_client:
+ # mock
+ test_client.put('/api/search/database/1',
+ headers={'Authorization': f'Bearer {self.token(["update-search-index"])}',
+ 'Content-Type': 'application/json'},
+ data=req.model_dump_json())
+ # test
+ response = test_client.post('/api/search/identifier', headers={'Content-Type': 'application/json'},
+ data=json.dumps({'id': 1}))
+ self.assertEqual(200, response.status_code)
+
+ def test_post_general_search_concept_succeeds(self):
+ with app.test_client() as test_client:
+ # mock
+ test_client.put('/api/search/database/1',
+ headers={'Authorization': f'Bearer {self.token(["update-search-index"])}',
+ 'Content-Type': 'application/json'},
+ data=req.model_dump_json())
+ # test
+ response = test_client.post('/api/search/concept', headers={'Content-Type': 'application/json'},
+ data=json.dumps({'id': 1}))
+ self.assertEqual(200, response.status_code)
+
+ def test_post_general_search_unit_succeeds(self):
+ with app.test_client() as test_client:
+ # mock
+ test_client.put('/api/search/database/1',
+ headers={'Authorization': f'Bearer {self.token(["update-search-index"])}',
+ 'Content-Type': 'application/json'},
+ data=req.model_dump_json())
+ # test
+ response = test_client.post('/api/search/unit', headers={'Content-Type': 'application/json'},
+ data=json.dumps({'id': 1}))
+ self.assertEqual(200, response.status_code)
+
+ def test_post_general_search_view_succeeds(self):
+ with app.test_client() as test_client:
+ # mock
+ test_client.put('/api/search/database/1',
+ headers={'Authorization': f'Bearer {self.token(["update-search-index"])}',
+ 'Content-Type': 'application/json'},
+ data=req.model_dump_json())
+ # test
+ response = test_client.post('/api/search/view', headers={'Content-Type': 'application/json'},
+ data=json.dumps({'id': 1}))
+ self.assertEqual(200, response.status_code)
diff --git a/dbrepo-search-service/test/test_jwt.py b/dbrepo-search-service/test/test_jwt.py
new file mode 100644
index 0000000000000000000000000000000000000000..59cd4ee1168117d0aeb6bf3549fe5088edc379b9
--- /dev/null
+++ b/dbrepo-search-service/test/test_jwt.py
@@ -0,0 +1,97 @@
+import time
+import unittest
+
+import jwt
+import requests_mock
+
+from app import verify_token, app, verify_password, get_user_roles
+from clients.keycloak_client import User
+
+
+class JwtTest(unittest.TestCase):
+
+ def response(self, roles: [str]) -> dict:
+ return dict({
+ "client_id": "username",
+ "realm_access": {
+ "roles": roles
+ }
+ })
+
+ def token(self, roles: [str], iat: int = int(time.time())) -> str:
+ claims = {
+ 'iat': iat,
+ 'realm_access': {
+ 'roles': roles
+ }
+ }
+ with open('test/rsa/rs256.key', 'rb') as fh:
+ return jwt.JWT().encode(claims, jwt.jwk_from_pem(fh.read()), alg='RS256')
+
+ def test_verify_token_no_token_fails(self):
+ with app.app_context():
+ # test
+ user = verify_token(None)
+ self.assertFalse(user)
+
+ def test_verify_token_empty_token_fails(self):
+ with app.app_context():
+ # test
+ user = verify_token("")
+ self.assertFalse(user)
+
+ def test_verify_token_malformed_token_fails(self):
+ with app.app_context():
+ # test
+ user = verify_token("eyEYEY12345")
+ self.assertFalse(user)
+
+ def test_verify_token_succeeds(self):
+ with app.app_context():
+ with requests_mock.Mocker() as mock:
+ # mock
+ mock.post('http://auth-service:8080/api/auth/realms/dbrepo/protocol/openid-connect/token',
+ json=self.response([]))
+ # test
+ user = verify_token(self.token([]))
+ self.assertEqual([], user.roles)
+
+ def test_verify_password_no_username_fails(self):
+ with app.app_context():
+ # test
+ user = verify_password(None, "pass")
+ self.assertFalse(user)
+
+ def test_verify_password_empty_username_fails(self):
+ with app.app_context():
+ # test
+ user = verify_password("", "pass")
+ self.assertFalse(user)
+
+ def test_verify_password_no_password_fails(self):
+ with app.app_context():
+ # test
+ user = verify_password("username", None)
+ self.assertFalse(user)
+
+ def test_verify_password_empty_password_fails(self):
+ with app.app_context():
+ # test
+ user = verify_password("username", "")
+ self.assertFalse(user)
+
+ def test_verify_password_succeeds(self):
+ with app.app_context():
+ with requests_mock.Mocker() as mock:
+ # mock
+ mock.post('http://auth-service:8080/api/auth/realms/dbrepo/protocol/openid-connect/token',
+ json=self.response([]))
+ # test
+ user = verify_password("username", "password")
+ self.assertIsNotNone(user)
+
+ def test_get_user_roles_succeeds(self):
+ with app.app_context():
+ # test
+ roles: [str] = get_user_roles(User(username="username", roles=[]))
+ self.assertEqual([], roles)
diff --git a/dbrepo-search-service/test/test_keycloak_client.py b/dbrepo-search-service/test/test_keycloak_client.py
new file mode 100644
index 0000000000000000000000000000000000000000..453a9b802be9885daa8e87afe265c272ee1ca211
--- /dev/null
+++ b/dbrepo-search-service/test/test_keycloak_client.py
@@ -0,0 +1,57 @@
+import time
+import unittest
+
+import jwt
+import requests_mock
+
+from app import app
+from clients.keycloak_client import KeycloakClient
+
+
+class JwtTest(unittest.TestCase):
+
+ def response(self, username) -> dict:
+ return dict({
+ "client_id": username,
+ "access_token": "eyEY1234"
+ })
+
+ def token(self, username: str, roles: [str], iat: int = int(time.time())) -> str:
+ claims = {
+ 'iat': iat,
+ 'client_id': username,
+ 'realm_access': {
+ 'roles': roles
+ }
+ }
+ with open('test/rsa/rs256.key', 'rb') as fh:
+ return jwt.JWT().encode(claims, jwt.jwk_from_pem(fh.read()), alg='RS256')
+
+ def test_obtain_user_token_succeeds(self):
+ with app.app_context():
+ with requests_mock.Mocker() as mock:
+ # mock
+ mock.post('http://auth-service:8080/api/auth/realms/dbrepo/protocol/openid-connect/token',
+ json=self.response("username"))
+ # test
+ token = KeycloakClient().obtain_user_token("username", "password")
+ self.assertEqual("eyEY1234", token)
+
+ def test_obtain_user_token_malformed_fails(self):
+ with app.app_context():
+ with requests_mock.Mocker() as mock:
+ # mock
+ mock.post('http://auth-service:8080/api/auth/realms/dbrepo/protocol/openid-connect/token',
+ json={"client_id": "username"})
+ # test
+ try:
+ KeycloakClient().obtain_user_token("username", "password")
+ self.fail()
+ except AssertionError:
+ pass
+
+ def test_verify_jwt_succeeds(self):
+ with app.app_context():
+ # test
+ user = KeycloakClient().verify_jwt(self.token("username", []))
+ self.assertEqual("username", user.username)
diff --git a/dbrepo-search-service/test/test_opensearch_client.py b/dbrepo-search-service/test/test_opensearch_client.py
index f100927e6e02e2af4cb7f9a4401618d497103346..581e5f8c5d94435c4344b4e8478ec09b116fd735 100644
--- a/dbrepo-search-service/test/test_opensearch_client.py
+++ b/dbrepo-search-service/test/test_opensearch_client.py
@@ -4,6 +4,7 @@ import unittest
import opensearchpy
from dbrepo.api.dto import Database, User, UserAttributes, Container, Image, Table, Column, ColumnType, Constraints, \
PrimaryKey, TableMinimal, ColumnMinimal, Concept, Unit
+from opensearchpy import NotFoundError
from app import app
@@ -73,9 +74,8 @@ class OpenSearchClientTest(unittest.TestCase):
def test_update_database_succeeds(self):
with app.app_context():
- client = OpenSearchClient()
# mock
- client.update_database(database_id=1, data=req)
+ OpenSearchClient().update_database(database_id=req.id, data=req)
# test
req.tables = [Table(id=1,
@@ -87,9 +87,10 @@ class OpenSearchClientTest(unittest.TestCase):
database_id=req.id,
constraints=Constraints(uniques=[], foreign_keys=[], checks=[],
primary_key=[PrimaryKey(id=1,
- table=TableMinimal(id=1, database_id=1),
+ table=TableMinimal(id=1,
+ database_id=req.id),
column=ColumnMinimal(id=1, table_id=1,
- database_id=1))]),
+ database_id=req.id))]),
is_versioned=True,
created_by="c6b71ef5-2d2f-48b2-9d79-b8f23a3a0502",
creator=User(id="c6b71ef5-2d2f-48b2-9d79-b8f23a3a0502",
@@ -107,7 +108,7 @@ class OpenSearchClientTest(unittest.TestCase):
column_type=ColumnType.BIGINT,
is_public=True,
is_null_allowed=False)])]
- database = client.update_database(database_id=1, data=req)
+ database = OpenSearchClient().update_database(database_id=req.id, data=req)
self.assertEqual(1, database.id)
self.assertEqual("Test", database.name)
self.assertEqual("test_tuw1", database.internal_name)
@@ -157,10 +158,8 @@ class OpenSearchClientTest(unittest.TestCase):
def test_update_database_create_succeeds(self):
with app.app_context():
- client = OpenSearchClient()
-
# test
- database = client.update_database(database_id=1, data=req)
+ database = OpenSearchClient().update_database(database_id=req.id, data=req)
self.assertEqual(1, database.id)
self.assertEqual("Test", database.name)
self.assertEqual("test_tuw1", database.internal_name)
@@ -185,124 +184,87 @@ class OpenSearchClientTest(unittest.TestCase):
def test_update_database_malformed_fails(self):
with app.app_context():
app.config['OPENSEARCH_USERNAME'] = 'i_do_not_exist'
- client = OpenSearchClient()
# test
try:
- database = client.update_database(database_id=1, data=req)
+ database = OpenSearchClient().update_database(database_id=req.id, data=req)
except opensearchpy.exceptions.TransportError:
pass
def test_delete_database_fails(self):
with app.app_context():
- client = OpenSearchClient()
# test
try:
- client.delete_database(database_id=9999)
+ OpenSearchClient().delete_database(database_id=9999)
except opensearchpy.exceptions.NotFoundError:
pass
def test_delete_database_succeeds(self):
with app.app_context():
- client = OpenSearchClient()
-
# mock
- client.update_database(database_id=req.id, data=req)
+ OpenSearchClient().update_database(database_id=req.id, data=req)
# test
- client.delete_database(database_id=req.id)
+ OpenSearchClient().delete_database(database_id=req.id)
- def test_find_database_succeeds(self):
+ def test_get_database_succeeds(self):
with app.app_context():
- client = OpenSearchClient()
-
# mock
- client.update_database(database_id=req.id, data=req)
+ OpenSearchClient().update_database(database_id=req.id, data=req)
# test
- client.get_database(database_id=req.id)
+ database = OpenSearchClient().get_database(database_id=req.id)
+ self.assertEqual(req.id, database.id)
- def test_find_database_fails(self):
+ def test_get_database_fails(self):
with app.app_context():
- client = OpenSearchClient()
# mock
- client.update_database(database_id=1, data=req)
+ OpenSearchClient().update_database(database_id=req.id, data=req)
# test
try:
- client.get_database(database_id=1)
+ OpenSearchClient().get_database(database_id=req.id)
except opensearchpy.exceptions.NotFoundError:
pass
- # def test_query_index_by_term_opensearch_contains_succeeds(self):
- # with app.app_context():
- # client = OpenSearchClient()
- #
- # # mock
- # client.update_database(database_id=1, data=req)
- #
- # # test
- # response = client.query_index_by_term_opensearch(term="test", mode="contains")
- # self.assertEqual(1, len(response))
- # self.assertEqual(1, response[0]['id'])
- # self.assertEqual('Test', response[0]['name'])
-
- # def test_query_index_by_term_opensearch_exact_succeeds(self):
- # with app.app_context():
- # client = OpenSearchClient()
- #
- # # mock
- # client.update_database(database_id=1, data=req)
- #
- # # test
- # response = client.query_index_by_term_opensearch(term="test", mode="exact")
- # self.assertEqual(1, len(response))
- # self.assertEqual(1, response[0]['id'])
- # self.assertEqual('Test', response[0]['name'])
-
def test_get_fields_for_index_database_succeeds(self):
with app.app_context():
- client = OpenSearchClient()
-
# mock
- client.update_database(database_id=1, data=req)
+ OpenSearchClient().update_database(database_id=req.id, data=req)
# test
- response = client.get_fields_for_index(type="database")
+ response = OpenSearchClient().get_fields_for_index(field_type="database")
self.assertTrue(len(response) > 0)
def test_get_fields_for_index_user_succeeds(self):
with app.app_context():
- client = OpenSearchClient()
-
# mock
- client.update_database(database_id=1, data=req)
+ OpenSearchClient().update_database(database_id=req.id, data=req)
# test
- response = client.get_fields_for_index(type="user")
+ response = OpenSearchClient().get_fields_for_index(field_type="user")
self.assertTrue(len(response) > 0)
def test_fuzzy_search_succeeds(self):
with app.app_context():
- client = OpenSearchClient()
-
# mock
- client.update_database(database_id=1, data=req)
+ OpenSearchClient().update_database(database_id=req.id, data=req)
# test
- response = client.fuzzy_search(search_term="test")
+ response = OpenSearchClient().fuzzy_search(search_term="test")
self.assertTrue(len(response) > 0)
- # def test_general_search_succeeds(self):
- # with app.app_context():
- # client = OpenSearchClient()
- #
- # # mock
- # client.update_database(database_id=1, data=req)
- #
- # # test
- # response = client.general_search(type="database", field_value_pairs={"name": "Test",
- # "id": None})
- # self.assertTrue(len(response) > 0)
+ def test_unit_independent_search_fails(self):
+ with app.app_context():
+ # mock
+ OpenSearchClient().update_database(database_id=req.id, data=req)
+
+ # test
+ try:
+ OpenSearchClient().unit_independent_search(0, 100, {
+ "unit.uri": "http://www.ontology-of-units-of-measure.org/resource/om-2/degreeCelsius"})
+ self.fail()
+ except NotFoundError:
+ pass