diff --git a/dbrepo-analyse-service/Pipfile.lock b/dbrepo-analyse-service/Pipfile.lock
index ae160f77733183750c665ce9c9053d76b34a18e0..c668b400e4fa26855881c6b78400b2c66becbc0f 100644
--- a/dbrepo-analyse-service/Pipfile.lock
+++ b/dbrepo-analyse-service/Pipfile.lock
@@ -175,20 +175,20 @@
},
"boto3": {
"hashes": [
- "sha256:159898f51c2997a12541c0e02d6e5a8fe2993ddb307b9478fd9a339f98b57e00",
- "sha256:d0ca7a58ce25701a52232cc8df9d87854824f1f2964b929305722ebc7959d5a9"
+ "sha256:258ab77225a81d3cf3029c9afe9920cd9dec317689dfadec6f6f0a23130bb60a",
+ "sha256:eb21380d73fec6645439c0d802210f72a0cdb3295b02953f246ff53f512faa8f"
],
"index": "pypi",
"markers": "python_version >= '3.8'",
- "version": "==1.36.0"
+ "version": "==1.36.1"
},
"botocore": {
"hashes": [
- "sha256:0232029ff9ae3f5b50cdb25cbd257c16f87402b6d31a05bd6483638ee6434c4b",
- "sha256:b54b11f0cfc47fc1243ada0f7f461266c279968487616720fa8ebb02183917d7"
+ "sha256:dec513b4eb8a847d79bbefdcdd07040ed9d44c20b0001136f0890a03d595705a",
+ "sha256:f789a6f272b5b3d8f8756495019785e33868e5e00dd9662a3ee7959ac939bb12"
],
"markers": "python_version >= '3.8'",
- "version": "==1.36.0"
+ "version": "==1.36.1"
},
"certifi": {
"hashes": [
@@ -268,7 +268,7 @@
"sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87",
"sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b"
],
- "markers": "platform_python_implementation != 'PyPy'",
+ "markers": "python_version >= '3.8'",
"version": "==1.17.1"
},
"charset-normalizer": {
@@ -412,7 +412,7 @@
},
"dbrepo": {
"hashes": [
- "sha256:0d11a0e0ec942d5b0ddfadd9e9007ce6dab9c5b9cc433e0f53b4fafcfc597bef"
+ "sha256:251f3c2088bbd289cee86d5394b1e62e29aa081f994dd0845d895e3330f6a106"
],
"path": "./lib/dbrepo-1.6.1.tar.gz"
},
@@ -1427,11 +1427,11 @@
},
"referencing": {
"hashes": [
- "sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c",
- "sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de"
+ "sha256:363d9c65f080d0d70bc41c721dce3c7f3e77fc09f269cd5c8813da18069a6794",
+ "sha256:ca2e6492769e3602957e9b831b94211599d2aade9477f5d44110d2530cf9aade"
],
- "markers": "python_version >= '3.8'",
- "version": "==0.35.1"
+ "markers": "python_version >= '3.9'",
+ "version": "==0.36.1"
},
"requests": {
"hashes": [
@@ -1553,11 +1553,11 @@
},
"s3transfer": {
"hashes": [
- "sha256:6563eda054c33bdebef7cbf309488634651c47270d828e594d151cd289fb7cf7",
- "sha256:f43b03931c198743569bbfb6a328a53f4b2b4ec723cd7c01fab68e3119db3f8b"
+ "sha256:3f25c900a367c8b7f7d8f9c34edc87e300bde424f779dc9f0a8ae4f9df9264f6",
+ "sha256:8fa0aa48177be1f3425176dfe1ab85dcd3d962df603c3dbfc585e6bf857ef0ff"
],
"markers": "python_version >= '3.8'",
- "version": "==0.11.0"
+ "version": "==0.11.1"
},
"setuptools": {
"hashes": [
@@ -1877,7 +1877,7 @@
"sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87",
"sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b"
],
- "markers": "platform_python_implementation != 'PyPy'",
+ "markers": "python_version >= '3.8'",
"version": "==1.17.1"
},
"charset-normalizer": {
diff --git a/dbrepo-analyse-service/lib/dbrepo-1.6.1.tar.gz b/dbrepo-analyse-service/lib/dbrepo-1.6.1.tar.gz
index 5ce8fdab038ca28aa52e5c8544ce3bcfee7ca3fa..7914db1bb84dddf85611cda3b766c0c0cdc094c7 100644
Binary files a/dbrepo-analyse-service/lib/dbrepo-1.6.1.tar.gz and b/dbrepo-analyse-service/lib/dbrepo-1.6.1.tar.gz differ
diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java
index 22099eafae1353d6061fc881812078ff212382f4..8a08f8231fe0de7babf1db995dc60786407425f8 100644
--- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java
+++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java
@@ -91,21 +91,6 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
});
}
- @Test
- @WithMockUser(username = USER_4_USERNAME)
- public void list_publicDataPrivateSchemaNoRole_fails() throws QueryNotFoundException, DatabaseNotFoundException,
- RemoteUnavailableException, SQLException, MetadataServiceException {
-
- /* mock */
- when(subsetService.findAll(DATABASE_3_PRIVILEGED_DTO, null))
- .thenReturn(List.of(QUERY_1_DTO, QUERY_2_DTO, QUERY_3_DTO, QUERY_4_DTO, QUERY_5_DTO, QUERY_6_DTO));
-
- /* test */
- assertThrows(NotAllowedException.class, () -> {
- generic_list(DATABASE_3_ID, DATABASE_3_PRIVILEGED_DTO, USER_4_PRINCIPAL);
- });
- }
-
@Test
@WithMockUser(username = USER_3_USERNAME)
public void list_publicDataPrivateSchema_succeeds() throws DatabaseUnavailableException, NotAllowedException,
@@ -131,21 +116,6 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
});
}
- @Test
- @WithMockUser(username = USER_4_USERNAME)
- public void list_publicDataAndPrivateSchemaNoRole_fails() throws DatabaseNotFoundException,
- RemoteUnavailableException, MetadataServiceException {
-
- /* mock */
- when(credentialService.getDatabase(DATABASE_3_ID))
- .thenReturn(DATABASE_3_PRIVILEGED_DTO);
-
- /* test */
- assertThrows(NotAllowedException.class, () -> {
- generic_list(DATABASE_3_ID, DATABASE_3_PRIVILEGED_DTO, USER_4_PRINCIPAL);
- });
- }
-
@Test
@WithMockUser(username = USER_3_USERNAME)
public void list_publicDataAndPrivateSchemaUnavailable_fails() throws SQLException, QueryNotFoundException,
@@ -316,21 +286,6 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
});
}
- @Test
- @WithMockUser(username = USER_4_USERNAME)
- public void findById_publicDataAndPrivateSchemaNoRole_fails() throws DatabaseNotFoundException,
- RemoteUnavailableException, MetadataServiceException {
-
- /* mock */
- when(credentialService.getDatabase(DATABASE_3_ID))
- .thenReturn(DATABASE_3_PRIVILEGED_DTO);
-
- /* test */
- assertThrows(NotAllowedException.class, () -> {
- generic_findById(DATABASE_3_ID, QUERY_5_ID, "application/json", null, USER_4_PRINCIPAL);
- });
- }
-
@Test
@WithMockUser(username = USER_3_USERNAME)
public void findById_publicDataAndPrivateSchemaUnavailable_fails() throws DatabaseNotFoundException, RemoteUnavailableException,
diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/DatabaseBriefDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/DatabaseBriefDto.java
index 707df1600c8252b80c4e091e8dfb945e0975c1d4..46072e83dc14af22d09923d3e9462a53506aa4f8 100644
--- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/DatabaseBriefDto.java
+++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/DatabaseBriefDto.java
@@ -10,6 +10,7 @@ import lombok.*;
import lombok.extern.jackson.Jacksonized;
import java.util.List;
+import java.util.UUID;
@Getter
@Setter
@@ -53,6 +54,6 @@ public class DatabaseBriefDto {
@NotNull
@JsonProperty("owner_id")
- private UserBriefDto ownerId;
+ private UUID ownerId;
}
diff --git a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java
index fe8e4385e2857be757ebc72904ef76c988b8fa57..c5482f70411433f5df08bd0281a60c75d132bf26 100644
--- a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java
+++ b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java
@@ -907,6 +907,9 @@ public interface MetadataMapper {
return database;
}
+ @Mappings({
+ @Mapping(target = "ownerId", source = "owner.id")
+ })
DatabaseBriefDto databaseToDatabaseBriefDto(Database data);
AccessType accessTypeDtoToAccessType(AccessTypeDto data);
diff --git a/dbrepo-search-service/Pipfile.lock b/dbrepo-search-service/Pipfile.lock
index e700161ce55394f4a9edf485b4f10c7c00c49572..c0508dd3daf66ff03c848411ae47f1698da81014 100644
--- a/dbrepo-search-service/Pipfile.lock
+++ b/dbrepo-search-service/Pipfile.lock
@@ -360,7 +360,7 @@
},
"dbrepo": {
"hashes": [
- "sha256:0d11a0e0ec942d5b0ddfadd9e9007ce6dab9c5b9cc433e0f53b4fafcfc597bef"
+ "sha256:a08b6eb49c108466b231c1b2cae5be501043fe4208a782899ce103105e22e3c6"
],
"path": "./lib/dbrepo-1.6.1.tar.gz"
},
@@ -1330,11 +1330,11 @@
},
"referencing": {
"hashes": [
- "sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c",
- "sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de"
+ "sha256:363d9c65f080d0d70bc41c721dce3c7f3e77fc09f269cd5c8813da18069a6794",
+ "sha256:ca2e6492769e3602957e9b831b94211599d2aade9477f5d44110d2530cf9aade"
],
- "markers": "python_version >= '3.8'",
- "version": "==0.35.1"
+ "markers": "python_version >= '3.9'",
+ "version": "==0.36.1"
},
"requests": {
"hashes": [
diff --git a/dbrepo-search-service/app.py b/dbrepo-search-service/app.py
index e4e8581a9ee484c198f5a00a76a58b5296dd4188..f9e2dbcc77ac11a6f7c8175e2b0d868f848c98af 100644
--- a/dbrepo-search-service/app.py
+++ b/dbrepo-search-service/app.py
@@ -5,17 +5,19 @@ from json import dumps
from typing import List, Any
import requests
-from clients.keycloak_client import User, KeycloakClient
-from clients.opensearch_client import OpenSearchClient
from dbrepo.api.dto import Database, ApiError
from flasgger import LazyJSONEncoder, Swagger, swag_from
-from flask import Flask, request
+from flask import Flask, request, Response
from flask_cors import CORS
from flask_httpauth import HTTPTokenAuth, HTTPBasicAuth, MultiAuth
from jwt.exceptions import JWTDecodeError
from opensearchpy import NotFoundError
from prometheus_flask_exporter import PrometheusMetrics
from pydantic import ValidationError
+from pydantic.deprecated.json import pydantic_encoder
+
+from clients.keycloak_client import User, KeycloakClient
+from clients.opensearch_client import OpenSearchClient, flatten
logging.addLevelName(level=logging.NOTSET, levelName='TRACE')
logging.basicConfig(level=logging.DEBUG)
@@ -123,18 +125,6 @@ template = {
}
}
},
- "SearchResultDto": {
- "required": ["results"],
- "type": "object",
- "properties": {
- "results": {
- "type": "array",
- "items": {
- "type": "object"
- }
- }
- }
- },
"SearchRequestDto": {
"required": ["search_term", "field_value_pairs"],
"type": "object",
@@ -208,13 +198,13 @@ app.json_encoder = LazyJSONEncoder
@token_auth.verify_token
-def verify_token(token: str):
+def verify_token(token: str) -> bool | User:
if token is None or token == "":
return False
try:
client = KeycloakClient()
return client.verify_jwt(access_token=token)
- except JWTDecodeError as error:
+ except JWTDecodeError:
return False
@@ -259,10 +249,10 @@ def general_filter(index, results):
"table": ["id", "name", "description"],
"identifier": ["id", "type", "creator"],
"user": ["id", "username"],
- "database": ["id", "name", "is_public", "details"],
+ "database": ["id", "name", "is_public", "is_schema_public", "details"],
"concept": ["uri", "name"],
"unit": [],
- "view": ["id", "name", "creator", " created"],
+ "view": ["id", "name", "creator"],
}
if index not in important_keys.keys():
raise KeyError(f"Failed to find index {index} in: {important_keys.keys()}")
@@ -289,7 +279,7 @@ def get_index(index: str):
:param index: desired index
:return: list of the results
"""
- logging.info(f'Searching for index: {index}')
+ logging.debug(f'endpoint get search type: {index}')
results = OpenSearchClient().query_index_by_term_opensearch("*", "contains")
try:
results = general_filter(index, results)
@@ -298,7 +288,7 @@ def get_index(index: str):
max_pages = math.ceil(len(results) / results_per_page)
page = min(request.args.get("page", 1, type=int), max_pages)
results = results[(results_per_page * (page - 1)): (results_per_page * page)]
- return dict({"results": results}), 200
+ return Response(dumps(results, default=pydantic_encoder)), 200, {'Content-Type': 'application/json'}
except KeyError:
return ApiError(status='NOT_FOUND', message=f'Failed to find get index: {index}',
code='search.index.missing').model_dump(), 404
@@ -313,11 +303,11 @@ def get_fields(field_type: str):
:param field_type: The search type
:return:
"""
- logging.info(f'Searching in index database for type: {field_type}')
+ logging.debug(f'endpoint get search type fields: {field_type}')
try:
fields = OpenSearchClient().get_fields_for_index(field_type)
logging.debug(f'get fields for field_type {field_type} resulted in {len(fields)} field(s)')
- return fields, 200
+ return Response(dumps(fields, default=pydantic_encoder)), 200, {'Content-Type': 'application/json'}
except NotFoundError:
return ApiError(status='NOT_FOUND', message=f'Failed to find fields for search type {field_type}',
code='search.type.missing').model_dump(), 404
@@ -331,15 +321,19 @@ def get_fuzzy_search():
Main endpoint for fuzzy searching.
:return:
"""
- search_term: str = request.args.get('q')
+ search_term: str | None = request.args.get('q')
+ logging.debug(f'endpoint get fuzzy search, q={search_term}')
if search_term is None or len(search_term) == 0:
return ApiError(status='BAD_REQUEST', message='Provide a search term with ?q=term',
code='search.fuzzy.invalid').model_dump(), 400
logging.debug(f"search request query: {search_term}")
- results = OpenSearchClient().fuzzy_search(search_term)
- if "hits" in results and "hits" in results["hits"]:
- results = [hit["_source"] for hit in results["hits"]["hits"]]
- return dict({"results": results}), 200
+ user_id, error, status = KeycloakClient().userId(request.headers.get('Authorization'))
+ if error is not None and status is not None:
+ return error, status
+ results: [Database] = OpenSearchClient().fuzzy_search(search_term=search_term,
+ user_id=user_id,
+ user_token=request.headers.get('Authorization'))
+ return Response(dumps(results, default=pydantic_encoder)), 200, {'Content-Type': 'application/json'}
@app.route("/api/search/<string:field_type>", methods=["POST"], endpoint="search_post_general_search")
@@ -353,27 +347,32 @@ def post_general_search(field_type):
if request.content_type != "application/json":
return ApiError(status='UNSUPPORTED_MEDIA_TYPE', message='Content type needs to be application/json',
code='search.general.media').model_dump(), 415
- req_body = request.json
- logging.info(f'Searching in index database for type: {field_type}')
+ value_pairs = request.json
+ logging.debug(f'endpoint get general search, field_type={field_type}, value_pairs={value_pairs}')
t1 = request.args.get("t1")
if not str(t1).isdigit():
t1 = None
t2 = request.args.get("t2")
if not str(t2).isdigit():
t2 = None
- if t1 is not None and t2 is not None and "unit.uri" in req_body and "concept.uri" in req_body:
- response = OpenSearchClient().unit_independent_search(t1, t2, req_body)
+ user_id, error, status = KeycloakClient().userId(request.headers.get('Authorization'))
+ if error is not None and status is not None:
+ return error, status
+ if t1 is not None and t2 is not None and "unit.uri" in value_pairs and "concept.uri" in value_pairs:
+ response: [Database] = OpenSearchClient().unit_independent_search(t1, t2, value_pairs, user_id)
else:
- response = OpenSearchClient().general_search(field_type, req_body)
+ response: [Database] = OpenSearchClient().general_search(field_type=field_type,
+ field_value_pairs=value_pairs,
+ user_id=user_id,
+ user_token=request.headers.get('Authorization'))
# filter by type
+ tables = [table for table in flatten([database.tables for database in response]) if
+ table.is_public or table.is_schema_public or (user_id is not None and table.owner.id == user_id)]
+ views = [view for view in flatten([database.views for database in response]) if
+ view.is_public or view.is_schema_public or (user_id is not None and view.owner.id == user_id)]
if field_type == 'table':
- tmp = []
- for database in response:
- if database["tables"] is not None:
- for table in database["tables"]:
- table["is_public"] = database["is_public"]
- tmp.append(table)
- response = tmp
+ logging.debug(f'filtered to {len(tables)} tables')
+ response = tables
if field_type == 'identifier':
tmp = []
for database in response:
@@ -393,12 +392,7 @@ def post_general_search(field_type):
tmp.append(view['identifier'])
response = tmp
elif field_type == 'column':
- response = [x for xs in response for x in xs["tables"]]
- for table in response:
- for column in table["columns"]:
- column["table_id"] = table["id"]
- column["database_id"] = table["database_id"]
- response = [x for xs in response for x in xs["columns"]]
+ response = flatten([table.columns for table in tables])
elif field_type == 'concept':
tmp = []
tables = [x for xs in response for x in xs["tables"]]
@@ -414,15 +408,15 @@ def post_general_search(field_type):
tmp.append(column["unit"])
response = tmp
elif field_type == 'view':
- response = [x for xs in response for x in xs["views"]]
- return dict({'results': response, 'type': field_type}), 200
+ response = views
+ return Response(dumps(response, default=pydantic_encoder)), 200, {'Content-Type': 'application/json'}
@app.route("/api/search/database/<int:database_id>", methods=["PUT"], endpoint="search_put_database")
@metrics.gauge(name='dbrepo_search_update_database',
description='Time needed to update a database in the search database')
@auth.login_required(role=['update-search-index'])
-def update_database(database_id: int) -> Database | ApiError:
+def update_database(database_id: int):
logging.debug(f"updating database with id: {database_id}")
try:
payload: Database = Database.model_validate(request.json)
@@ -431,7 +425,7 @@ def update_database(database_id: int) -> Database | ApiError:
return ApiError(status='BAD_REQUEST', message=f'Malformed payload: {e}',
code='search.general.missing').model_dump(), 400
database = OpenSearchClient().update_database(database_id, payload)
- logging.info(f"Updated database with id : {database_id}")
+ logging.info(f"Updated database with id: {database_id}")
return database.model_dump(), 202
@@ -442,7 +436,7 @@ def update_database(database_id: int) -> Database | ApiError:
def delete_database(database_id: int):
try:
OpenSearchClient().delete_database(database_id)
- return dumps({}), 202
+ return Response(dumps({})), 202
except NotFoundError:
return ApiError(status='NOT_FOUND', message='Failed to find database',
code='search.database.missing').model_dump(), 404
diff --git a/dbrepo-search-service/init/Pipfile.lock b/dbrepo-search-service/init/Pipfile.lock
index a8257b53df3cc300043d7aeb6412481708b9b1ff..bf53ace7e7551b8e0961373d6ab23e49ff53f300 100644
--- a/dbrepo-search-service/init/Pipfile.lock
+++ b/dbrepo-search-service/init/Pipfile.lock
@@ -254,9 +254,10 @@
},
"dbrepo": {
"hashes": [
- "sha256:0d11a0e0ec942d5b0ddfadd9e9007ce6dab9c5b9cc433e0f53b4fafcfc597bef"
+ "sha256:251f3c2088bbd289cee86d5394b1e62e29aa081f994dd0845d895e3330f6a106"
],
- "path": "./lib/dbrepo-1.6.1.tar.gz"
+ "path": "./lib/dbrepo-1.6.1.tar.gz",
+ "version": "==1.6.1"
},
"docker": {
"hashes": [
@@ -278,7 +279,6 @@
"sha256:f69fcd559dc907ed196ab9df0e48471709175e696d6e698dd4dbe940f96ce66b"
],
"index": "pypi",
- "markers": "python_version >= '3.8'",
"version": "==2.3.3"
},
"frozenlist": {
@@ -643,7 +643,6 @@
"sha256:6598df0bc7a003294edd0ba88a331e0793acbb8c910c43edf398791e3b2eccda"
],
"index": "pypi",
- "markers": "python_version >= '3.8' and python_version < '4'",
"version": "==2.8.0"
},
"packaging": {
@@ -934,7 +933,6 @@
"sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761"
],
"index": "pypi",
- "markers": "python_version >= '3.8'",
"version": "==8.3.4"
},
"python-dateutil": {
@@ -951,7 +949,6 @@
"sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"
],
"index": "pypi",
- "markers": "python_version >= '3.8'",
"version": "==1.0.1"
},
"pytz": {
@@ -967,7 +964,6 @@
"sha256:5a694a64f48a751079999c37dccf91a6210077d845d09adf7c3ce23a876265a7"
],
"index": "pypi",
- "markers": "python_version >= '3.9' and python_version < '4'",
"version": "==7.1.2"
},
"requests": {
@@ -998,7 +994,6 @@
"sha256:0bdf270b5b7f53915832f7c31dd2bd3ffdc20b534ea6b32231cc7003049bd0e1"
],
"index": "pypi",
- "markers": "python_version >= '3.7'",
"version": "==0.0.1rc1"
},
"tinydb": {
@@ -1290,7 +1285,6 @@
"sha256:fd34e7b3405f0cc7ab03d54a334c17a9e802897580d964bd8c2001f4b9fd488f"
],
"index": "pypi",
- "markers": "python_version >= '3.9'",
"version": "==7.6.10"
},
"iniconfig": {
@@ -1323,7 +1317,6 @@
"sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761"
],
"index": "pypi",
- "markers": "python_version >= '3.8'",
"version": "==8.3.4"
}
}
diff --git a/dbrepo-search-service/init/app.py b/dbrepo-search-service/init/app.py
index 9fe915f92c50d2b712058783d4eecf1b087cc8f7..f8f671bade77541508aec72cd19066157cd162f3 100644
--- a/dbrepo-search-service/init/app.py
+++ b/dbrepo-search-service/init/app.py
@@ -1,12 +1,11 @@
import json
-import os
import logging
+import os
+from logging.config import dictConfig
from typing import List
import opensearchpy.exceptions
from dbrepo.RestClient import RestClient
-from logging.config import dictConfig
-
from dbrepo.api.dto import Database
from opensearchpy import OpenSearch
@@ -46,6 +45,8 @@ class App:
search_username: str = None
search_password: str = None
search_instance: OpenSearch = None
+ system_username: str = None
+ system_password: str = None
def __init__(self):
self.metadata_service_endpoint = os.getenv("METADATA_SERVICE_ENDPOINT", "http://metadata-service:8080")
@@ -53,6 +54,8 @@ class App:
self.search_port = int(os.getenv("OPENSEARCH_PORT", "9200"))
self.search_username = os.getenv("OPENSEARCH_USERNAME", "admin")
self.search_password = os.getenv("OPENSEARCH_PASSWORD", "admin")
+ self.system_username = os.getenv("SYSTEM_USERNAME", "admin")
+ self.system_password = os.getenv("SYSTEM_PASSWORD", "admin")
def _instance(self) -> OpenSearch:
"""
@@ -84,7 +87,8 @@ class App:
def fetch_databases(self) -> List[Database]:
logging.debug(f"fetching database from endpoint: {self.metadata_service_endpoint}")
- client = RestClient(endpoint=self.metadata_service_endpoint)
+ client = RestClient(endpoint=self.metadata_service_endpoint, username=self.system_username,
+ password=self.system_password)
databases = []
for index, database in enumerate(client.get_databases()):
logging.debug(f"fetching database {index}/{len(databases)} details for database id: {database.id}")
@@ -93,16 +97,17 @@ class App:
return databases
def save_databases(self, databases: List[Database]):
- logging.debug(f"save {len(databases)} database(s)")
+ index = f'database'
+ logging.debug(f"save {len(databases)} database(s) in index: {index}")
for doc in databases:
doc: Database = doc
try:
- self._instance().delete(index="database", id=doc.id)
- logging.debug(f"deleted database with id {doc.id}")
+ self._instance().delete(index=index, id=doc.id)
+ logging.debug(f"truncated database with id {doc.id} in index: {index}")
except opensearchpy.NotFoundError:
- logging.warning(f"Database with id {doc.id} does not exist, skip.")
- self._instance().create(index="database", id=doc.id, body=doc.model_dump())
- logging.debug(f"created database with id {doc.id}")
+ pass
+ self._instance().create(index=index, id=doc.id, body=doc.model_dump())
+ logging.info(f"Saved database with id {doc.id} in index: {index}")
if __name__ == "__main__":
diff --git a/dbrepo-search-service/init/clients/keycloak_client.py b/dbrepo-search-service/init/clients/keycloak_client.py
index afa36a1112ce41b5686641f5691df3f44075cf2f..2e15d00a9b272233feb7bab4cf6166c63b151e04 100644
--- a/dbrepo-search-service/init/clients/keycloak_client.py
+++ b/dbrepo-search-service/init/clients/keycloak_client.py
@@ -1,14 +1,17 @@
import logging
from dataclasses import dataclass
-import requests
-from flask import current_app
from typing import List
+import requests
+from dbrepo.api.dto import ApiError
+from flask import current_app
from jwt import jwk_from_pem, JWT
+from jwt.exceptions import JWTDecodeError
@dataclass(init=True, eq=True)
class User:
+ id: str
username: str
roles: List[str]
@@ -30,8 +33,22 @@ class KeycloakClient:
raise AssertionError("Failed to obtain user token(s)")
return response.json()["access_token"]
- def verify_jwt(self, access_token: str) -> User:
+ def verify_jwt(self, access_token: str) -> ApiError | User:
public_key = jwk_from_pem(str(current_app.config["JWT_PUBKEY"]).encode('utf-8'))
payload = JWT().decode(message=access_token, key=public_key, do_time_check=True)
- logging.debug(f"JWT token client_id={payload.get('client_id')} and realm_access={payload.get('realm_access')}")
- return User(username=payload.get('client_id'), roles=payload.get('realm_access')["roles"])
+ return User(id=payload.get('uid'), username=payload.get('client_id'),
+ roles=payload.get('realm_access')["roles"])
+
+ def userId(self, auth_header: str | None) -> (str | None, ApiError, int):
+ if auth_header is None:
+ return None, None, None
+ try:
+ user = self.verify_jwt(auth_header.split(" ")[1])
+ logging.debug(f'mapped JWT to user.id {user.id}')
+ return user.id, None, None
+ except JWTDecodeError as e:
+ logging.error(f'Failed to decode JWT: {e}')
+ if str(e) == 'JWT Expired':
+ return None, ApiError(status='UNAUTHORIZED', message=f'Token expired',
+ code='search.user.unauthorized').model_dump(), 401
+ return None, ApiError(status='FORBIDDEN', message=str(e), code='search.user.forbidden').model_dump(), 403
diff --git a/dbrepo-search-service/init/clients/opensearch_client.py b/dbrepo-search-service/init/clients/opensearch_client.py
index bf782a7a7e8245c5cd57eea5a641b7f14c46df07..35c26f03f5f684adc2652c199db7f99afd6cfc13 100644
--- a/dbrepo-search-service/init/clients/opensearch_client.py
+++ b/dbrepo-search-service/init/clients/opensearch_client.py
@@ -7,7 +7,9 @@ from collections.abc import MutableMapping
from json import dumps, load
from dbrepo.api.dto import Database
+from dbrepo.api.exceptions import ForbiddenError, NotExistsError
from opensearchpy import OpenSearch, NotFoundError
+from requests import head
from omlib.constants import OM_IDS
from omlib.measure import om
@@ -20,16 +22,22 @@ class OpenSearchClient:
The client to communicate with the OpenSearch database.
"""
host: str = None
+ instance: OpenSearch = None
+ metadata_endpoint: str = None
+ password: str = None
port: int = None
+ system_username: str = None
+ system_password: str = None
username: str = None
- password: str = None
- instance: OpenSearch = None
def __init__(self, host: str = None, port: int = None, username: str = None, password: str = None):
self.host = os.getenv('OPENSEARCH_HOST', host)
+ self.metadata_endpoint = os.getenv('METADATA_SERVICE_ENDPOINT', 'http://metadata-service:8080')
+ self.password = os.getenv('OPENSEARCH_PASSWORD', password)
self.port = int(os.getenv('OPENSEARCH_PORT', port))
+ self.system_username = os.getenv('SYSTEM_USERNAME', 'admin')
+ self.system_password = os.getenv('SYSTEM_PASSWORD', 'admin')
self.username = os.getenv('OPENSEARCH_USERNAME', username)
- self.password = os.getenv('OPENSEARCH_PASSWORD', password)
def _instance(self) -> OpenSearch:
"""
@@ -43,18 +51,6 @@ class OpenSearchClient:
http_auth=(self.username, self.password))
return self.instance
- def get_database(self, database_id: int) -> Database:
- """
- Gets a database by given id.
-
- @param database_id: The database id.
-
- @returns: The database, if successful.
- @throws: opensearchpy.exceptions.NotFoundError If the database was not found in the Search Database.
- """
- response: dict = self._instance().get(index="database", id=database_id)
- return Database.model_validate(response["_source"])
-
def update_database(self, database_id: int, data: Database) -> Database:
"""
Updates the database data with given id.
@@ -68,9 +64,7 @@ class OpenSearchClient:
logging.debug(f"updating database with id: {database_id} in search database")
self._instance().index(index="database", id=database_id, body=dumps(data.model_dump()))
response: dict = self._instance().get(index="database", id=database_id)
- database = Database.model_validate(response["_source"])
- logging.info(f"Updated database with id {database_id} in index 'database'")
- return database
+ return Database.model_validate(response["_source"])
def delete_database(self, database_id: int) -> None:
"""
@@ -142,27 +136,50 @@ class OpenSearchClient:
fields_list.append(entry)
return fields_list
- def fuzzy_search(self, search_term=None):
- logging.info(f"Performing fuzzy search")
- fuzzy_body = {
- "query": {
- "multi_match": {
- "query": search_term,
- "fuzziness": "AUTO",
- "fuzzy_transpositions": True,
- "minimum_should_match": 3
- }
- }
- }
- logging.debug(f'search body: {fuzzy_body}')
+ def fuzzy_search(self, search_term: str, user_id: str | None = None, user_token: str | None = None) -> [Database]:
response = self._instance().search(
index="database",
- body=fuzzy_body
+ body={
+ "query": {
+ "multi_match": {
+ "query": search_term,
+ "fuzziness": "AUTO",
+ "prefix_length": 2
+ }
+ }
+ }
)
- logging.info(f"Found {len(response['hits']['hits'])} result(s)")
- return response
-
- def general_search(self, field_type: str = None, field_value_pairs: dict = None):
+ results: [Database] = []
+ if "hits" in response and "hits" in response["hits"]:
+ results = [Database.model_validate(hit["_source"]) for hit in response["hits"]["hits"]]
+ logging.debug(f'found {len(results)} results')
+ return self.filter_results(results, user_id, user_token)
+
+ def filter_results(self, results: [Database], user_id: str | None = None, user_token: str | None = None) -> [
+ Database]:
+ filtered: [Database] = []
+ for database in results:
+ if database.is_public or database.is_schema_public:
+ logging.debug(f'database with id {database.id} is public or has public schema')
+ filtered.append(database)
+ elif user_id is not None and user_token is not None:
+ try:
+ url = f'{self.metadata_endpoint}/api/database/{database.id}/access/{user_id}'
+ logging.debug(f'requesting access from url: {url}')
+ response = head(url=url, auth=(self.system_username, self.system_password))
+ if response.status_code == 200:
+ logging.debug(f'database with id {database.id} is draft and access was found')
+ filtered.append(database)
+ else:
+ logging.warning(
+ f'database with id {database.id} is not accessible: code {response.status_code}')
+ except (ForbiddenError, NotExistsError) as e:
+ logging.warning(f'database with id {database.id} is draft but no access was found')
+ logging.debug(f'filtered {len(filtered)} results')
+ return filtered
+
+ def general_search(self, field_type: str = None, field_value_pairs: dict = None, user_id: str | None = None,
+ user_token: str | None = None) -> [Database]:
"""
Main method for searching stuff in the opensearch db
@@ -203,10 +220,14 @@ class OpenSearchClient:
index="database",
body=dumps(body)
)
- results = [hit["_source"] for hit in response["hits"]["hits"]]
- return results
-
- def unit_independent_search(self, t1: float, t2: float, field_value_pairs):
+ results: [Database] = []
+ if "hits" in response and "hits" in response["hits"]:
+ results = [Database.model_validate(hit["_source"]) for hit in response["hits"]["hits"]]
+ logging.debug(f'found {len(results)} results')
+ return self.filter_results(results, user_id, user_token)
+
+ def unit_independent_search(self, t1: float, t2: float, field_value_pairs: dict, userId: str | None = None) -> [
+ Database]:
"""
Main method for searching stuff in the opensearch db
@@ -287,16 +308,12 @@ class OpenSearchClient:
body = ''
for search in searches:
body += '%s \n' % dumps(search)
- responses = self._instance().msearch(
+ response = self._instance().msearch(
body=dumps(body)
)
- response = {
- "hits": {
- "hits": flatten([hits["hits"]["hits"] for hits in responses["responses"]])
- },
- "took": responses["took"]
- }
- return response
+ results = flatten([hits["hits"]["hits"] for hits in response["responses"]])
+ return [database for database in results if
+ database.is_public or database.is_schema_public or (userId is not None and database.owner.id == userId)]
def key_to_attr_name(key: str) -> str:
diff --git a/dbrepo-search-service/init/database.json b/dbrepo-search-service/init/database.json
index 363624ff059daa02d4edb58f3f6bce1b5b4664dd..59cbd78438a5fff391d1237c70a0384d7b397a83 100644
--- a/dbrepo-search-service/init/database.json
+++ b/dbrepo-search-service/init/database.json
@@ -572,37 +572,6 @@
}
}
},
- "owner": {
- "properties": {
- "id": {
- "type": "text",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "qualified_name": {
- "type": "text",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "username": {
- "type": "text",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- }
- },
"tables": {
"properties": {
"columns": {
@@ -640,6 +609,12 @@
"is_null_allowed": {
"type": "boolean"
},
+ "is_public": {
+ "type": "boolean"
+ },
+ "is_schema_public": {
+ "type": "boolean"
+ },
"mean": {
"type": "float"
},
@@ -827,55 +802,6 @@
"num_rows": {
"type": "long"
},
- "owner": {
- "properties": {
- "id": {
- "type": "text",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "qualified_name": {
- "type": "text",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "name": {
- "type": "text",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "orcid": {
- "type": "text",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "username": {
- "type": "text",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- }
- },
"queue_name": {
"type": "text",
"fields": {
@@ -933,6 +859,9 @@
"is_public": {
"type": "boolean"
},
+ "is_schema_public": {
+ "type": "boolean"
+ },
"name": {
"type": "text",
"fields": {
diff --git a/dbrepo-search-service/init/lib/dbrepo-1.6.1.tar.gz b/dbrepo-search-service/init/lib/dbrepo-1.6.1.tar.gz
index 5ce8fdab038ca28aa52e5c8544ce3bcfee7ca3fa..7914db1bb84dddf85611cda3b766c0c0cdc094c7 100644
Binary files a/dbrepo-search-service/init/lib/dbrepo-1.6.1.tar.gz and b/dbrepo-search-service/init/lib/dbrepo-1.6.1.tar.gz differ
diff --git a/dbrepo-search-service/lib/dbrepo-1.6.1.tar.gz b/dbrepo-search-service/lib/dbrepo-1.6.1.tar.gz
index 5ce8fdab038ca28aa52e5c8544ce3bcfee7ca3fa..7914db1bb84dddf85611cda3b766c0c0cdc094c7 100644
Binary files a/dbrepo-search-service/lib/dbrepo-1.6.1.tar.gz and b/dbrepo-search-service/lib/dbrepo-1.6.1.tar.gz differ
diff --git a/dbrepo-search-service/os-yml/get_fuzzy_search.yml b/dbrepo-search-service/os-yml/get_fuzzy_search.yml
index bc54419eb9735fe731fb12a5e911070ebc29f80e..db2ef87b3268f10329ba11117ee323208d940af5 100644
--- a/dbrepo-search-service/os-yml/get_fuzzy_search.yml
+++ b/dbrepo-search-service/os-yml/get_fuzzy_search.yml
@@ -19,6 +19,9 @@ responses:
content:
application/json:
schema:
- $ref: '#/components/schemas/SearchResultDto'
+ type: array
+ properties:
+ id:
+ type: string
415:
description: Wrong accept type
diff --git a/dbrepo-search-service/test/test_jwt.py b/dbrepo-search-service/test/test_jwt.py
index 59cd4ee1168117d0aeb6bf3549fe5088edc379b9..96ce8410da4be0598fd6d635e7decc9950bd42e8 100644
--- a/dbrepo-search-service/test/test_jwt.py
+++ b/dbrepo-search-service/test/test_jwt.py
@@ -12,9 +12,9 @@ class JwtTest(unittest.TestCase):
def response(self, roles: [str]) -> dict:
return dict({
- "client_id": "username",
- "realm_access": {
- "roles": roles
+ 'client_id': 'username',
+ 'realm_access': {
+ 'roles': roles
}
})
@@ -37,13 +37,13 @@ class JwtTest(unittest.TestCase):
def test_verify_token_empty_token_fails(self):
with app.app_context():
# test
- user = verify_token("")
+ user = verify_token('')
self.assertFalse(user)
def test_verify_token_malformed_token_fails(self):
with app.app_context():
# test
- user = verify_token("eyEYEY12345")
+ user = verify_token('eyEYEY12345')
self.assertFalse(user)
def test_verify_token_succeeds(self):
@@ -59,25 +59,25 @@ class JwtTest(unittest.TestCase):
def test_verify_password_no_username_fails(self):
with app.app_context():
# test
- user = verify_password(None, "pass")
+ user = verify_password(None, 'pass')
self.assertFalse(user)
def test_verify_password_empty_username_fails(self):
with app.app_context():
# test
- user = verify_password("", "pass")
+ user = verify_password('', 'pass')
self.assertFalse(user)
def test_verify_password_no_password_fails(self):
with app.app_context():
# test
- user = verify_password("username", None)
+ user = verify_password('username', None)
self.assertFalse(user)
def test_verify_password_empty_password_fails(self):
with app.app_context():
# test
- user = verify_password("username", "")
+ user = verify_password('username', '')
self.assertFalse(user)
def test_verify_password_succeeds(self):
@@ -87,11 +87,12 @@ class JwtTest(unittest.TestCase):
mock.post('http://auth-service:8080/api/auth/realms/dbrepo/protocol/openid-connect/token',
json=self.response([]))
# test
- user = verify_password("username", "password")
+ user = verify_password('username', 'password')
self.assertIsNotNone(user)
def test_get_user_roles_succeeds(self):
with app.app_context():
# test
- roles: [str] = get_user_roles(User(username="username", roles=[]))
+ roles: [str] = get_user_roles(
+ User(id='b98415d8-28bc-4472-84ff-3d09cc79aff6', username='username', roles=[]))
self.assertEqual([], roles)
diff --git a/dbrepo-search-service/test/test_opensearch_client.py b/dbrepo-search-service/test/test_opensearch_client.py
index e37a96db10aaf44c9f588907295a2622c1fc0d38..9da77adfde53e155dddc36f364bea9d974964125 100644
--- a/dbrepo-search-service/test/test_opensearch_client.py
+++ b/dbrepo-search-service/test/test_opensearch_client.py
@@ -165,27 +165,6 @@ class OpenSearchClientTest(unittest.TestCase):
# test
OpenSearchClient().delete_database(database_id=req.id)
- def test_get_database_succeeds(self):
- with app.app_context():
- # mock
- OpenSearchClient().update_database(database_id=req.id, data=req)
-
- # test
- database = OpenSearchClient().get_database(database_id=req.id)
- self.assertEqual(req.id, database.id)
-
- def test_get_database_fails(self):
- with app.app_context():
-
- # mock
- OpenSearchClient().update_database(database_id=req.id, data=req)
-
- # test
- try:
- OpenSearchClient().get_database(database_id=req.id)
- except opensearchpy.exceptions.NotFoundError:
- pass
-
def test_get_fields_for_index_database_succeeds(self):
with app.app_context():
# mock
@@ -210,8 +189,7 @@ class OpenSearchClientTest(unittest.TestCase):
OpenSearchClient().update_database(database_id=req.id, data=req)
# test
- response = OpenSearchClient().fuzzy_search(search_term="test")
- self.assertTrue(len(response) > 0)
+ OpenSearchClient().fuzzy_search(search_term="test_tuw")
def test_unit_independent_search_fails(self):
with app.app_context():
diff --git a/dbrepo-ui/components/search/AdvancedSearch.vue b/dbrepo-ui/components/search/AdvancedSearch.vue
index b312b2dc5291812fd98930e8585b50e389c341c5..8197cd3fb8a960f44440cc8238c0a1a42cb7f57e 100644
--- a/dbrepo-ui/components/search/AdvancedSearch.vue
+++ b/dbrepo-ui/components/search/AdvancedSearch.vue
@@ -384,8 +384,8 @@ export default {
this.loading = true
const searchService = useSearchService()
searchService.general_search(this.searchType, this.advancedSearchData)
- .then(({results, type}) => {
- this.$emit('search-result', {results, type})
+ .then((results) => {
+ this.$emit('search-result', results)
})
.finally(() => {
this.loading = false
@@ -443,7 +443,7 @@ export default {
return
}
this.resetAdvancedSearchFields()
- this.$emit('search-result', { results: [], type: this.searchType })
+ this.$emit('search-result', [])
const searchService = useSearchService()
this.loadingFields = true
searchService.fields(this.searchType)
diff --git a/dbrepo-ui/composables/search-service.ts b/dbrepo-ui/composables/search-service.ts
index 62be8b9bc7160f70e33969970fbbba1c500d288f..b61f8358cf74167d1fef5c6f2de54ef017d4c7aa 100644
--- a/dbrepo-ui/composables/search-service.ts
+++ b/dbrepo-ui/composables/search-service.ts
@@ -18,11 +18,11 @@ export const useSearchService = (): any => {
})
}
- async function fuzzy_search(term: string): Promise<SearchResultDto> {
+ async function fuzzy_search(term: string): Promise<DatabaseDto[]> {
const axios = useAxiosInstance()
console.debug('fuzzy search for term', term)
- return new Promise<SearchResultDto>((resolve, reject) => {
- axios.get<SearchResultDto>(`/api/search?q=${term}`)
+ return new Promise<DatabaseDto[]>((resolve, reject) => {
+ axios.get<DatabaseDto[]>(`/api/search?q=${term}`)
.then((response) => {
console.info('Searched for term', term)
resolve(response.data)
diff --git a/dbrepo-ui/nuxt.config.ts b/dbrepo-ui/nuxt.config.ts
index b8e55da8023daf790048ab9bfd5f48a0b82d2ba3..4bce6ec5c5b3dc7f025734d9bf15d8e914123f79 100644
--- a/dbrepo-ui/nuxt.config.ts
+++ b/dbrepo-ui/nuxt.config.ts
@@ -75,8 +75,8 @@ export default defineNuxtConfig({
}
},
api: {
- client: 'https://s155.datalab.tuwien.ac.at',
- server: 'https://s155.datalab.tuwien.ac.at',
+ client: 'http://localhost',
+ server: 'http://gateway-service',
},
upload: {
client: 'http://localhost/api/upload/files',
diff --git a/dbrepo-ui/pages/search.vue b/dbrepo-ui/pages/search.vue
index b13a0f0fc2d7985775fe8c3c450c0acb8785bbd0..b23c896448e12845d0d4b3f92ce0284b5886bd55 100644
--- a/dbrepo-ui/pages/search.vue
+++ b/dbrepo-ui/pages/search.vue
@@ -27,7 +27,8 @@
v-if="isDatabaseSearch"
:loading="loading"
:databases="results" />
- <div>
+ <div
+ v-else>
<v-card
v-for="(result, idx) in results"
:key="idx"
@@ -38,10 +39,13 @@
<v-divider class="mx-4" />
<v-card-title
class="text-primary text-decoration-underline">
- <a v-if="link(result)" :href="link(result)">
+ <a
+ v-if="link(result)"
+ :href="link(result)">
{{ title(result) }}
</a>
- <span v-else>
+ <span
+ v-else>
{{ title(result) }}
</span>
</v-card-title>
@@ -66,23 +70,15 @@
</v-card-text>
</v-card>
</div>
- <v-dialog
- v-model="createDbDialog"
- persistent
- max-width="640">
- <DatabaseCreate @close="closed" />
- </v-dialog>
</div>
</template>
<script>
-import DatabaseCreate from '@/components/database/DatabaseCreate.vue'
import AdvancedSearch from '@/components/search/AdvancedSearch.vue'
import { useUserStore } from '@/stores/user'
export default {
components: {
- DatabaseCreate,
AdvancedSearch
},
data () {
@@ -90,7 +86,6 @@ export default {
results: [],
type: 'database',
loading: false,
- createDbDialog: null,
userStore: useUserStore()
}
},
@@ -136,10 +131,13 @@ export default {
if (!queryKeys || queryKeys.length !== 1 || !queryKeys.includes('q')) {
return
}
+ if (!this.q) {
+ return
+ }
this.loading = true
const searchService = useSearchService()
searchService.fuzzy_search(this.q)
- .then(({results}) => {
+ .then((results) => {
this.results = results
this.loading = false
})
@@ -294,19 +292,8 @@ export default {
}
return tags
},
- closed (event) {
- this.dialog = false
- if (event.success) {
- this.$router.push(`/database/${event.database_id}/info`)
- }
- },
- onSearchResult ({results, type}) {
+ onSearchResult (results) {
this.results = results
- if (!type) {
- return
- }
- console.debug('search for type', type, ':', results)
- this.type = type
},
capitalizeFirstLetter(string) {
if (!string) {
diff --git a/docker-compose.yml b/docker-compose.yml
index 7159ff58cbb59eec4d114603c71c5ea4c52d51b1..315f6bf1884c1e49d2ac870ab53f50d7a6b8c449 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -303,12 +303,14 @@ services:
AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT_SECRET:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
AUTH_SERVICE_ENDPOINT: ${AUTH_SERVICE_ENDPOINT:-http://auth-service:8080}
COLLECTION: ${COLLECTION:-['database','table','column','identifier','unit','concept','user','view']}
+ LOG_LEVEL: ${LOG_LEVEL:-info}
METADATA_SERVICE_ENDPOINT: ${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080}
OPENSEARCH_HOST: ${OPENSEARCH_HOST:-search-db}
OPENSEARCH_PORT: ${OPENSEARCH_PORT:-9200}
OPENSEARCH_USERNAME: ${SEARCH_DB_USERNAME:-admin}
OPENSEARCH_PASSWORD: ${SEARCH_DB_PASSWORD:-admin}
- LOG_LEVEL: ${LOG_LEVEL:-info}
+ SYSTEM_USERNAME: "${SYSTEM_USERNAME:-admin}"
+ SYSTEM_PASSWORD: "${SYSTEM_PASSWORD:-admin}"
healthcheck:
test: curl -sSL localhost:8080/health | grep 'UP' || exit 1
interval: 10s
@@ -402,11 +404,14 @@ services:
context: ./dbrepo-search-service/init
network: host
environment:
+ LOG_LEVEL: ${LOG_LEVEL:-info}
METADATA_SERVICE_ENDPOINT: ${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080}
OPENSEARCH_HOST: ${OPENSEARCH_HOST:-search-db}
OPENSEARCH_PORT: ${OPENSEARCH_PORT:-9200}
OPENSEARCH_USERNAME: ${SEARCH_DB_USERNAME:-admin}
OPENSEARCH_PASSWORD: ${SEARCH_DB_PASSWORD:-admin}
+ SYSTEM_USERNAME: "${SYSTEM_USERNAME:-admin}"
+ SYSTEM_PASSWORD: "${SYSTEM_PASSWORD:-admin}"
depends_on:
dbrepo-search-db:
condition: service_healthy
diff --git a/helm/dbrepo/Chart.yaml b/helm/dbrepo/Chart.yaml
index d2ff855534c9bbea91e408d2fee96a566994647d..22d1865df5866213c931df8b2b522af33137c1e6 100644
--- a/helm/dbrepo/Chart.yaml
+++ b/helm/dbrepo/Chart.yaml
@@ -30,17 +30,17 @@ dependencies:
- name: mariadb-galera
alias: datadb
version: 13.2.7
- repository: oci://registry-1.docker.io/bitnamicharts
+ repository: https://charts.bitnami.com/bitnami
condition: datadb.enabled
- name: mariadb-galera
alias: metadatadb
version: 13.2.7
- repository: oci://registry-1.docker.io/bitnamicharts
+ repository: https://charts.bitnami.com/bitnami
condition: metadatadb.enabled
- name: rabbitmq
alias: brokerservice
version: 14.0.0
- repository: oci://registry-1.docker.io/bitnamicharts
+ repository: https://charts.bitnami.com/bitnami
condition: brokerservice.enabled
- name: seaweedfs
alias: storageservice
@@ -50,15 +50,15 @@ dependencies:
- name: grafana
alias: dashboardservice
version: 11.4.2
- repository: oci://registry-1.docker.io/bitnamicharts
+ repository: https://charts.bitnami.com/bitnami
condition: dashboardservice.enabled
- name: prometheus
alias: metricdb
version: 1.3.22
- repository: oci://registry-1.docker.io/bitnamicharts
+ repository: https://charts.bitnami.com/bitnami
condition: metricdb.enabled
- name: nginx
alias: gatewayservice
version: 18.3.1
- repository: oci://registry-1.docker.io/bitnamicharts
+ repository: https://charts.bitnami.com/bitnami
condition: gatewayservice.enabled
\ No newline at end of file
diff --git a/helm/seaweedfs/Chart.lock b/helm/seaweedfs/Chart.lock
index 1195241a0686ef99b5cde492d2191b3417193279..edcc38c41f0c6b2b35f8d740566918ddb16f17fc 100644
--- a/helm/seaweedfs/Chart.lock
+++ b/helm/seaweedfs/Chart.lock
@@ -4,9 +4,9 @@ dependencies:
version: 20.2.1
- name: postgresql
repository: oci://registry-1.docker.io/bitnamicharts
- version: 16.4.2
+ version: 16.4.3
- name: common
repository: oci://registry-1.docker.io/bitnamicharts
version: 2.29.0
-digest: sha256:b9f5516ae118fdca61c19cfe79cee8bd77ad96978dd8c06ad003646d5691aca5
-generated: "2025-01-14T10:19:53.265063382+01:00"
+digest: sha256:4c967f771b303ca0db9ba2e355790152448c77a05d3f6c69eda6c234bc3f60c6
+generated: "2025-01-17T15:24:18.141765362+01:00"
diff --git a/helm/seaweedfs/charts/postgresql-16.4.2.tgz b/helm/seaweedfs/charts/postgresql-16.4.2.tgz
deleted file mode 100644
index aaa8899b4dad449edbaa98aefefbce4fa4de756d..0000000000000000000000000000000000000000
Binary files a/helm/seaweedfs/charts/postgresql-16.4.2.tgz and /dev/null differ
diff --git a/helm/seaweedfs/charts/postgresql-16.4.3.tgz b/helm/seaweedfs/charts/postgresql-16.4.3.tgz
new file mode 100644
index 0000000000000000000000000000000000000000..429f7ed063f6655796792fbe711b027e147ddda4
Binary files /dev/null and b/helm/seaweedfs/charts/postgresql-16.4.3.tgz differ
diff --git a/lib/python/dbrepo/api/dto.py b/lib/python/dbrepo/api/dto.py
index dc7f0e191210b70978ec7b26834293705d3012e9..fa7eb063fc453c3dd8fabef04f141d07012c94a3 100644
--- a/lib/python/dbrepo/api/dto.py
+++ b/lib/python/dbrepo/api/dto.py
@@ -5,7 +5,7 @@ from dataclasses import field
from enum import Enum
from typing import List, Optional, Annotated
-from pydantic import BaseModel, PlainSerializer, Field
+from pydantic import BaseModel, PlainSerializer
Timestamp = Annotated[
datetime.datetime, PlainSerializer(lambda v: v.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z', return_type=str)
@@ -987,9 +987,10 @@ class DatabaseBrief(BaseModel):
internal_name: str
description: Optional[str] = None
is_public: bool
+ is_schema_public: bool
identifiers: Optional[List[Identifier]] = field(default_factory=list)
contact: UserBrief
- owner: UserBrief
+ owner_id: str
class Unique(BaseModel):
diff --git a/lib/python/tests/test_unit_database.py b/lib/python/tests/test_unit_database.py
index affa94b496b1a8bf104fb12264256ad91e6fc8fb..eeeea68832ac33e2f013a3c2deaa3d4eec33122c 100644
--- a/lib/python/tests/test_unit_database.py
+++ b/lib/python/tests/test_unit_database.py
@@ -24,10 +24,11 @@ class DatabaseUnitTest(unittest.TestCase):
DatabaseBrief(
id=1,
name='test',
- owner=UserBrief(id='8638c043-5145-4be8-a3e4-4b79991b0a16', username='mweise'),
+ owner_id='8638c043-5145-4be8-a3e4-4b79991b0a16',
contact=UserBrief(id='8638c043-5145-4be8-a3e4-4b79991b0a16', username='mweise'),
internal_name='test_abcd',
- is_public=True)
+ is_public=True,
+ is_schema_public=True)
]
with requests_mock.Mocker() as mock:
# mock