From 99e2c83a94de9fcca22a63495adde6bc0b9df647 Mon Sep 17 00:00:00 2001
From: Martin Weise <martin.weise@tuwien.ac.at>
Date: Tue, 9 Apr 2024 18:06:36 +0200
Subject: [PATCH] Fixed the python library deserializer
---
.../templates/broker-service/secret.yaml | 4 +++-
helm-charts/dbrepo/templates/data-db/pvc.yaml | 7 ++++++-
.../templates/metadata-db/configmap.yaml | 4 +++-
.../templates/search-db-dashboard/secret.yaml | 3 +++
.../dbrepo/templates/storage-service/job.yaml | 3 +++
.../templates/storage-service/secret.yaml | 4 +++-
lib/python/dbrepo/RestClient.py | 20 +++++++++----------
lib/python/debug.py | 5 ++---
lib/python/tests/test_container.py | 6 ++----
lib/python/tests/test_database.py | 6 ++----
lib/python/tests/test_identifier.py | 4 +---
lib/python/tests/test_license.py | 9 +++------
lib/python/tests/test_query.py | 6 ++----
lib/python/tests/test_table.py | 6 ++----
lib/python/tests/test_user.py | 6 ++----
lib/python/tests/test_view.py | 6 ++----
16 files changed, 48 insertions(+), 51 deletions(-)
diff --git a/helm-charts/dbrepo/templates/broker-service/secret.yaml b/helm-charts/dbrepo/templates/broker-service/secret.yaml
index c1d158e5cf..de0fb00e96 100644
--- a/helm-charts/dbrepo/templates/broker-service/secret.yaml
+++ b/helm-charts/dbrepo/templates/broker-service/secret.yaml
@@ -1,3 +1,4 @@
+{{- if .Values.brokerService.enabled }}
---
apiVersion: v1
kind: Secret
@@ -84,4 +85,5 @@ stringData:
pubkey.pem: |
-----BEGIN RSA PUBLIC KEY-----
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB
- -----END RSA PUBLIC KEY-----
\ No newline at end of file
+ -----END RSA PUBLIC KEY-----
+{{- end }}
diff --git a/helm-charts/dbrepo/templates/data-db/pvc.yaml b/helm-charts/dbrepo/templates/data-db/pvc.yaml
index dd0dc69aea..b730f78e16 100644
--- a/helm-charts/dbrepo/templates/data-db/pvc.yaml
+++ b/helm-charts/dbrepo/templates/data-db/pvc.yaml
@@ -1,3 +1,6 @@
+{{- if .Values.dataDb.enabled }}
+{{- if .Values.dataDb.persistence.enabled }}
+---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
@@ -10,4 +13,6 @@ spec:
- ReadWriteMany
resources:
requests:
- storage: 10Gi
\ No newline at end of file
+ storage: 8Gi
+{{- end }}
+{{- end }}
\ No newline at end of file
diff --git a/helm-charts/dbrepo/templates/metadata-db/configmap.yaml b/helm-charts/dbrepo/templates/metadata-db/configmap.yaml
index 8385c1be49..0ed3ff6aef 100644
--- a/helm-charts/dbrepo/templates/metadata-db/configmap.yaml
+++ b/helm-charts/dbrepo/templates/metadata-db/configmap.yaml
@@ -1,3 +1,5 @@
+{{- if .Values.metadataDb.enabled }}
+---
apiVersion: v1
kind: ConfigMap
metadata:
@@ -569,4 +571,4 @@ data:
('prov', 'http://www.w3.org/ns/prov#', null, null, null),
('db', 'http://dbpedia.org', 'http://dbpedia.org/ontology/.*', 'http://dbpedia.org/sparql', null);
COMMIT;
-
+{{- end }}
diff --git a/helm-charts/dbrepo/templates/search-db-dashboard/secret.yaml b/helm-charts/dbrepo/templates/search-db-dashboard/secret.yaml
index 9150e8e1d6..cb6da449d2 100644
--- a/helm-charts/dbrepo/templates/search-db-dashboard/secret.yaml
+++ b/helm-charts/dbrepo/templates/search-db-dashboard/secret.yaml
@@ -1,3 +1,5 @@
+{{- if .Values.searchDbDashboard.enabled }}
+---
apiVersion: v1
kind: Secret
metadata:
@@ -19,3 +21,4 @@ stringData:
verificationMode: none
username: {{ .Values.searchdb.username }}
password: {{ .Values.searchdb.password }}
+{{- end }}
diff --git a/helm-charts/dbrepo/templates/storage-service/job.yaml b/helm-charts/dbrepo/templates/storage-service/job.yaml
index f9048a69c8..b732c616b5 100644
--- a/helm-charts/dbrepo/templates/storage-service/job.yaml
+++ b/helm-charts/dbrepo/templates/storage-service/job.yaml
@@ -1,3 +1,5 @@
+{{- if .Values.storageservice.enabled }}
+---
apiVersion: batch/v1
kind: Job
metadata:
@@ -49,3 +51,4 @@ spec:
secret:
defaultMode: 420
secretName: seaweedfs-s3-secret
+{{- end }}
diff --git a/helm-charts/dbrepo/templates/storage-service/secret.yaml b/helm-charts/dbrepo/templates/storage-service/secret.yaml
index fb63db7f72..faaf302f7d 100644
--- a/helm-charts/dbrepo/templates/storage-service/secret.yaml
+++ b/helm-charts/dbrepo/templates/storage-service/secret.yaml
@@ -1,3 +1,5 @@
+{{- if .Values.storageservice.enabled }}
+---
apiVersion: v1
kind: Secret
metadata:
@@ -28,4 +30,4 @@ stringData:
}
]
}
-
+{{- end }}
diff --git a/lib/python/dbrepo/RestClient.py b/lib/python/dbrepo/RestClient.py
index bc08c92826..a114dcc988 100644
--- a/lib/python/dbrepo/RestClient.py
+++ b/lib/python/dbrepo/RestClient.py
@@ -1,5 +1,3 @@
-import dataclasses
-import datetime
import sys
import os
import logging
@@ -48,7 +46,7 @@ class RestClient:
else:
self.secure = secure
- def _wrapper(self, method: str, url: str, params: [(str,)] = None, payload=None, headers=None,
+ def _wrapper(self, method: str, url: str, params: [(str,)] = None, payload=None, headers: dict = None,
force_auth: bool = False, stream: bool = False) -> requests.Response:
if force_auth and (self.username is None or self.password is None):
raise AuthenticationError(f"Failed to perform request: authentication required")
@@ -110,7 +108,7 @@ class RestClient:
response = self._wrapper(method="get", url=url)
if response.status_code == 200:
body = response.json()
- return TypeAdapter(List[User]).validate_json(body)
+ return TypeAdapter(List[User]).validate_python(body)
raise ResponseCodeError(f'Failed to find users: response code: {response.status_code} is not 200 (OK)')
def get_user(self, user_id: str) -> User:
@@ -272,7 +270,7 @@ class RestClient:
response = self._wrapper(method="get", url=url)
if response.status_code == 200:
body = response.json()
- return TypeAdapter(List[ContainerBrief]).validate_json(body)
+ return TypeAdapter(List[ContainerBrief]).validate_python(body)
raise ResponseCodeError(f'Failed to find containers: response code: {response.status_code} is not 200 (OK)')
def get_container(self, container_id: int) -> Container:
@@ -305,7 +303,7 @@ class RestClient:
response = self._wrapper(method="get", url=url)
if response.status_code == 200:
body = response.json()
- return TypeAdapter(List[Database]).validate_json(body)
+ return TypeAdapter(List[Database]).validate_python(body)
raise ResponseCodeError(f'Failed to find databases: response code: {response.status_code} is not 200 (OK)')
def get_databases_count(self) -> int:
@@ -470,7 +468,7 @@ class RestClient:
response = self._wrapper(method="get", url=url)
if response.status_code == 200:
body = response.json()
- return TypeAdapter(List[Table]).validate_json(body)
+ return TypeAdapter(List[Table]).validate_python(body)
raise ResponseCodeError(f'Failed to find tables: response code: {response.status_code} is not 200 (OK)')
def get_table(self, database_id: int, table_id: int) -> Table:
@@ -534,7 +532,7 @@ class RestClient:
response = self._wrapper(method="get", url=url)
if response.status_code == 200:
body = response.json()
- return TypeAdapter(List[View]).validate_json(body)
+ return TypeAdapter(List[View]).validate_python(body)
if response.status_code == 403:
raise ForbiddenError(f'Failed to find views: not allowed')
if response.status_code == 404:
@@ -1265,7 +1263,7 @@ class RestClient:
response = self._wrapper(method="get", url=url)
if response.status_code == 200:
body = response.json()
- return TypeAdapter(List[Query]).validate_json(body)
+ return TypeAdapter(List[Query]).validate_python(body)
if response.status_code == 403 or response.status_code == 405:
raise ForbiddenError(f'Failed to find queries: not allowed')
if response.status_code == 404:
@@ -1393,7 +1391,7 @@ class RestClient:
response = self._wrapper(method="get", url=url)
if response.status_code == 200:
body = response.json()
- return TypeAdapter(List[License]).validate_json(body)
+ return TypeAdapter(List[License]).validate_python(body)
raise ResponseCodeError(f'Failed to get licenses: response code: {response.status_code} is not 200 (OK)')
def get_identifiers(self, ld: bool = False) -> List[Identifier] | str:
@@ -1417,7 +1415,7 @@ class RestClient:
return response.json()
else:
body = response.json()
- return TypeAdapter(List[Identifier]).validate_json(body)
+ return TypeAdapter(List[Identifier]).validate_python(body)
if response.status_code == 406:
raise MalformedError(
f'Failed to get identifiers: accept header must be application/json or application/ld+json')
diff --git a/lib/python/debug.py b/lib/python/debug.py
index 433a9e57ee..afcb586090 100644
--- a/lib/python/debug.py
+++ b/lib/python/debug.py
@@ -1,6 +1,5 @@
from dbrepo.RestClient import RestClient
-from python.dbrepo.api.dto import AccessType
-client = RestClient(endpoint="https://test.dbrepo.tuwien.ac.at", username="foo",
+client = RestClient(endpoint="https://dbrepo1.ec.tuwien.ac.at", username="foo",
password="bar")
-client.delete_database_access()
+client.get_licenses()
diff --git a/lib/python/tests/test_container.py b/lib/python/tests/test_container.py
index de658e7db5..d948e337db 100644
--- a/lib/python/tests/test_container.py
+++ b/lib/python/tests/test_container.py
@@ -1,6 +1,4 @@
-import datetime
import unittest
-from json import dumps
import requests_mock
import datetime
@@ -17,7 +15,7 @@ class ContainerTest(unittest.TestCase):
def test_get_containers_empty_succeeds(self):
with requests_mock.Mocker() as mock:
# mock
- mock.get('/api/container', json=dumps([]))
+ mock.get('/api/container', json=[])
# test
response = RestClient().get_containers()
self.assertEqual([], response)
@@ -37,7 +35,7 @@ class ContainerTest(unittest.TestCase):
hash="f829dd8a884182d0da846f365dee1221fd16610a14c81b8f9f295ff162749e50")
]
# mock
- mock.get('/api/container', json=dumps([exp[0].model_dump()]))
+ mock.get('/api/container', json=[exp[0].model_dump()])
# test
response = RestClient().get_containers()
self.assertEqual(exp, response)
diff --git a/lib/python/tests/test_database.py b/lib/python/tests/test_database.py
index 352836a0be..9df3a11d5a 100644
--- a/lib/python/tests/test_database.py
+++ b/lib/python/tests/test_database.py
@@ -1,6 +1,4 @@
-import datetime
import unittest
-from json import dumps
import requests_mock
import datetime
@@ -19,7 +17,7 @@ class DatabaseTest(unittest.TestCase):
def test_get_databases_empty_succeeds(self):
with requests_mock.Mocker() as mock:
# mock
- mock.get('/api/database', json=dumps([]))
+ mock.get('/api/database', json=[])
# test
response = RestClient().get_databases()
self.assertEqual([], response)
@@ -70,7 +68,7 @@ class DatabaseTest(unittest.TestCase):
]
with requests_mock.Mocker() as mock:
# mock
- mock.get('/api/database', json=dumps([exp[0].model_dump()]))
+ mock.get('/api/database', json=[exp[0].model_dump()])
# test
response = RestClient().get_databases()
self.assertEqual(exp, response)
diff --git a/lib/python/tests/test_identifier.py b/lib/python/tests/test_identifier.py
index 3b338eb20c..c39e7a99a5 100644
--- a/lib/python/tests/test_identifier.py
+++ b/lib/python/tests/test_identifier.py
@@ -1,7 +1,5 @@
import unittest
-from json import dumps
-
import requests_mock
import datetime
@@ -174,7 +172,7 @@ class IdentifierTest(unittest.TestCase):
type=RelatedIdentifierType.DOI)],
creators=[IdentifierCreator(id=5, creator_name='Carberry, Josiah')])]
# mock
- mock.get('/api/pid', json=dumps([exp[0].model_dump()]), headers={"Accept": "application/json"})
+ mock.get('/api/pid', json=[exp[0].model_dump()], headers={"Accept": "application/json"})
# test
response = RestClient().get_identifiers()
self.assertEqual(exp, response)
diff --git a/lib/python/tests/test_license.py b/lib/python/tests/test_license.py
index 5cc3b54ce7..bddb99213a 100644
--- a/lib/python/tests/test_license.py
+++ b/lib/python/tests/test_license.py
@@ -1,13 +1,10 @@
import unittest
-from json import dumps
import requests_mock
-import dataclasses
from dbrepo.RestClient import RestClient
-from dbrepo.api.dto import Database, User, Container, Image, UserAttributes, DatabaseAccess, AccessType, License
-from dbrepo.api.exceptions import ResponseCodeError, NotExistsError, ForbiddenError, MalformedError
+from dbrepo.api.dto import License
class DatabaseTest(unittest.TestCase):
@@ -15,7 +12,7 @@ class DatabaseTest(unittest.TestCase):
def test_get_licenses_empty_succeeds(self):
with requests_mock.Mocker() as mock:
# mock
- mock.get('/api/database/license', json=dumps([]))
+ mock.get('/api/database/license', json=[])
# test
response = RestClient().get_licenses()
self.assertEqual([], response)
@@ -25,7 +22,7 @@ class DatabaseTest(unittest.TestCase):
exp = [License(identifier='CC-BY-4.0', uri='https://creativecommons.org/licenses/by/4.0/',
description='The Creative Commons Attribution license allows re-distribution and re-use of a licensed work on the condition that the creator is appropriately credited.')]
# mock
- mock.get('/api/database/license', json=dumps([exp[0].model_dump()]))
+ mock.get('/api/database/license', json=[exp[0].model_dump()])
# test
response = RestClient().get_licenses()
self.assertEqual(exp, response)
diff --git a/lib/python/tests/test_query.py b/lib/python/tests/test_query.py
index f38a06d573..76bd95d8d3 100644
--- a/lib/python/tests/test_query.py
+++ b/lib/python/tests/test_query.py
@@ -1,7 +1,5 @@
import unittest
-from json import dumps
-
import requests_mock
import datetime
@@ -165,7 +163,7 @@ class QueryTest(unittest.TestCase):
with requests_mock.Mocker() as mock:
exp = []
# mock
- mock.get('/api/database/1/query', json=dumps([]))
+ mock.get('/api/database/1/query', json=[])
# test
response = RestClient().get_queries(database_id=1)
self.assertEqual(exp, response)
@@ -188,7 +186,7 @@ class QueryTest(unittest.TestCase):
result_number=None,
identifiers=[])]
# mock
- mock.get('/api/database/1/query', json=dumps([exp[0].model_dump()]))
+ mock.get('/api/database/1/query', json=[exp[0].model_dump()])
# test
response = RestClient().get_queries(database_id=1)
self.assertEqual(exp, response)
diff --git a/lib/python/tests/test_table.py b/lib/python/tests/test_table.py
index b90d41a612..68f947386e 100644
--- a/lib/python/tests/test_table.py
+++ b/lib/python/tests/test_table.py
@@ -1,7 +1,5 @@
import unittest
-from json import dumps
-
import requests_mock
import datetime
@@ -113,7 +111,7 @@ class TableTest(unittest.TestCase):
def test_get_tables_empty_succeeds(self):
with requests_mock.Mocker() as mock:
# mock
- mock.get('/api/database/1/table', json=dumps([]))
+ mock.get('/api/database/1/table', json=[])
# test
response = RestClient().get_tables(database_id=1)
self.assertEqual([], response)
@@ -147,7 +145,7 @@ class TableTest(unittest.TestCase):
is_public=True,
is_null_allowed=False)])]
# mock
- mock.get('/api/database/1/table', json=dumps([exp[0].model_dump()]))
+ mock.get('/api/database/1/table', json=[exp[0].model_dump()])
# test
response = RestClient().get_tables(database_id=1)
self.assertEqual(exp, response)
diff --git a/lib/python/tests/test_user.py b/lib/python/tests/test_user.py
index 4ef48c4d44..d5afaf2d01 100644
--- a/lib/python/tests/test_user.py
+++ b/lib/python/tests/test_user.py
@@ -1,6 +1,4 @@
-import dataclasses
import unittest
-from json import dumps
import requests_mock
@@ -24,7 +22,7 @@ class UserTest(unittest.TestCase):
def test_get_users_empty_succeeds(self):
with requests_mock.Mocker() as mock:
# mock
- mock.get('http://gateway-service/api/user', json=dumps([]))
+ mock.get('http://gateway-service/api/user', json=[])
# test
response = RestClient().get_users()
self.assertEqual([], response)
@@ -36,7 +34,7 @@ class UserTest(unittest.TestCase):
attributes=UserAttributes(theme='dark'))
]
# mock
- mock.get('http://gateway-service/api/user', json=dumps([exp[0].model_dump()]))
+ mock.get('http://gateway-service/api/user', json=[exp[0].model_dump()])
# test
response = RestClient().get_users()
self.assertEqual(exp, response)
diff --git a/lib/python/tests/test_view.py b/lib/python/tests/test_view.py
index 8baf8d8afd..16eb6f4c51 100644
--- a/lib/python/tests/test_view.py
+++ b/lib/python/tests/test_view.py
@@ -1,7 +1,5 @@
import unittest
-from json import dumps
-
import requests_mock
import datetime
@@ -16,7 +14,7 @@ class ViewTest(unittest.TestCase):
def test_get_views_empty_succeeds(self):
with requests_mock.Mocker() as mock:
# mock
- mock.get('/api/database/1/view', json=dumps([]))
+ mock.get('/api/database/1/view', json=[])
# test
response = RestClient().get_views(database_id=1)
self.assertEqual([], response)
@@ -37,7 +35,7 @@ class ViewTest(unittest.TestCase):
last_modified=datetime.datetime(2024, 1, 1, 0, 0, 0, 0, datetime.timezone.utc),
identifiers=[])]
# mock
- mock.get('/api/database/1/view', json=dumps([exp[0].model_dump()]))
+ mock.get('/api/database/1/view', json=[exp[0].model_dump()])
# test
response = RestClient().get_views(database_id=1)
self.assertEqual(exp, response)
--
GitLab