Merge branch 'dev' into 527-get_identifier_data-with-adjustable-data-size

Signed-off-by: Martin Weise <martin.weise@tuwien.ac.at>

dbrepo-analyse-service/Pipfile

@@ -23,6 +23,7 @@ minio = "*"
 pydantic = "*"
 dbrepo = {path = "./lib/dbrepo-1.9.0.tar.gz"}
 opensearch-py = "*"
+ecs_logging = "*"
 
 [dev-packages]
 coverage = "*"

dbrepo-analyse-service/Pipfile.lock

 {
     "_meta": {
         "hash": {
-            "sha256": "c030ceb850cff7827f79ed1dfba622c39597c78fec39d4acddc80d57efa1da49"
+            "sha256": "bb1e86635cba8bfa5ad0a45abade8841ba91778d2ff0bcac1f455e8f17838baa"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -329,6 +329,14 @@
             "path": "./lib/dbrepo-1.9.0.tar.gz",
             "version": "==1.9.0"
         },
+        "ecs-logging": {
+            "hashes": [
+                "sha256:1dc9e216f614129db0e6a2f9f926da4e4cf8edf8de16d1045a20aa8e950291d3",
+                "sha256:f6e22d267770b06f797076f49b5fcc9d97108b22f452f5f9ed4b5367b1e61b5b"
+            ],
+            "index": "pypi",
+            "version": "==2.2.0"
+        },
         "events": {
             "hashes": [
                 "sha256:a7286af378ba3e46640ac9825156c93bdba7502174dd696090fdfcd4d80a1abd"

dbrepo-analyse-service/app.py

@@ -30,9 +30,8 @@ dictConfig({
         'simple': {
             'format': '[%(asctime)s] [%(levelname)s] %(message)s',
         },
-        'ecs': {
-            'format': '{"@timestamp": "%(asctime)s", "log.level": "%(levelname)s", "log.logger": "%(module)s", "message": "%(message)s", "service_name": "analyse-service", "service_version": "1.9.0"}',
-            'datefmt': '%Y-%m-%dT%H:%M:%S'
+        "ecs": {
+            "()": "ecs_logging.StdlibFormatter"
         },
     },
     'handlers': {

dbrepo-auth-service/init/Pipfile

@@ -6,6 +6,8 @@ name = "pypi"
 [packages]
 requests = "*"
 mariadb = "*"
+ecs_logging = "*"
+dbrepo = {path = "./lib/dbrepo-1.9.0.tar.gz"}
 
 [dev-packages]
 coverage = "*"

dbrepo-auth-service/init/app.py

+import logging
 import os
 
 import mariadb
 from requests import post, get
 
+logging.addLevelName(level=logging.NOTSET, levelName='TRACE')
+logging.basicConfig(level=logging.DEBUG)
+
+from logging.config import dictConfig
+
+# logging configuration
+dictConfig({
+    'version': 1,
+    'formatters': {
+        'default': {
+            'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
+        },
+        'simple': {
+            'format': '[%(asctime)s] [%(levelname)s] %(message)s',
+        },
+        "ecs": {
+            "()": "ecs_logging.StdlibFormatter"
+        },
+    },
+    'handlers': {
+        'wsgi': {
+            'class': 'logging.StreamHandler',
+            'stream': 'ext://flask.logging.wsgi_errors_stream',
+            'formatter': 'simple'
+        },
+        'file': {
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'formatter': 'ecs',
+            'filename': '/var/log/app/service/auth/init.log',
+            'when': 'm',
+            'interval': 1,
+            'backupCount': 5,
+            'encoding': 'utf8'
+        },
+    },
+    'root': {
+        'level': 'DEBUG',
+        'handlers': ['wsgi', 'file']
+    }
+})
+
 
 def fetch_keycloak_master_access_token() -> str:
     """
@@ -22,7 +64,7 @@ def fetch_keycloak_master_access_token() -> str:
 
 
 def fetch(username) -> (str, str):
-    print(f'Fetching user id of internal user with username: {username}')
+    logging.debug(f'fetching user id of internal user with username: {username}')
     endpoint = os.getenv('AUTH_SERVICE_ENDPOINT', 'http://localhost:8080')
     response = get(url=f'{endpoint}/admin/realms/dbrepo/users/?username={username}', headers=dict({
         'Authorization': f'Bearer {fetch_keycloak_master_access_token()}'
@@ -31,7 +73,7 @@ def fetch(username) -> (str, str):
         raise FileNotFoundError(f'Failed to obtain user')
     ldap_user = response.json()[0]
     user_id = ldap_user["id"]
-    print(f'Successfully fetched user id: {user_id}')
+    logging.debug(f'obtained user id for username {username} from auth service: {user_id}')
     if 'attributes' not in ldap_user or ldap_user['attributes'] is None:
         raise ModuleNotFoundError(f'Failed to obtain user attributes: {ldap_user}')
     ldap_user_attrs = ldap_user['attributes']
@@ -40,7 +82,6 @@ def fetch(username) -> (str, str):
     if len(ldap_user_attrs['LDAP_ID']) != 1:
         raise EnvironmentError(f'Failed to obtain ldap id: wrong length {len(ldap_user_attrs["LDAP_ID"])} != 1')
     ldap_user_id = ldap_user_attrs['LDAP_ID'][0]
-    print(f'Successfully fetched ldap user id: {ldap_user_id}')
     return (ldap_user_id, user_id)
 
 
@@ -56,7 +97,7 @@ def save(user_id: str, keycloak_id: str, username: str) -> None:
         (user_id, keycloak_id, username))
     conn.commit()
     conn.close()
-    print(f'Successfully inserted user: {username}')
+    logging.info(f'Successfully inserted user: {username}')
 
 
 if __name__ == '__main__':

dbrepo-dashboard-service/Pipfile

@@ -17,6 +17,7 @@ gunicorn = "*"
 pydantic = "*"
 flask_httpauth = "*"
 grafana-client = "*"
+ecs_logging = "*"
 
 [dev-packages]
 coverage = "*"

dbrepo-dashboard-service/Pipfile.lock

 {
     "_meta": {
         "hash": {
-            "sha256": "0e291266f11c024d2367e2ba20e2afd15a7225424e04ef2bc4e9f9f5d36f18b6"
+            "sha256": "3b53fdc68c4705c6d377819efb99c8c60517b7d75ae1a40b3f9be1240d175dc0"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -278,6 +278,14 @@
             "path": "./lib/dbrepo-1.9.0.tar.gz",
             "version": "==1.9.0"
         },
+        "ecs-logging": {
+            "hashes": [
+                "sha256:1dc9e216f614129db0e6a2f9f926da4e4cf8edf8de16d1045a20aa8e950291d3",
+                "sha256:f6e22d267770b06f797076f49b5fcc9d97108b22f452f5f9ed4b5367b1e61b5b"
+            ],
+            "index": "pypi",
+            "version": "==2.2.0"
+        },
         "flasgger": {
             "hashes": [
                 "sha256:ca098e10bfbb12f047acc6299cc70a33851943a746e550d86e65e60d4df245fb"

dbrepo-dashboard-service/app.py

@@ -31,9 +31,8 @@ dictConfig({
         'simple': {
             'format': '[%(asctime)s] [%(levelname)s] %(message)s',
         },
-        'ecs': {
-            'format': '{"@timestamp": "%(asctime)s", "log.level": "%(levelname)s", "log.logger": "%(module)s", "message": "%(message)s", "service_name": "dashboard-service", "service_version": "1.9.0"}',
-            'datefmt': '%Y-%m-%dT%H:%M:%S'
+        "ecs": {
+            "()": "ecs_logging.StdlibFormatter"
         },
     },
     'handlers': {
@@ -247,8 +246,18 @@ def update_dashboard(uid: str):
                                  code='error.database.malformed').model_dump_json(), 400, headers)
     try:
         dashboard_client().update(database)
-    except DashboardNotFound as e:
+    except DashboardNotFound:
         return Response(ApiError(status='NOT_FOUND', message=f"Failed to update dashboard: not found",
                                  code="error.dashboard.missing").model_dump_json(), 404, headers)
     dashboard_client().update_anonymous_read_access(uid, database.is_public, database.is_schema_public)
     return Response(), 202, headers
+
+
+@app.route("/api/dashboard/<string:uid>/access/<string:username>", methods=["PUT"], endpoint="update_dashboard_access")
+@metrics.gauge(name='dbrepo_update_dashboard_access', description='Time needed to update dashboard access')
+@swag_from("/app/ds-yml/update_dashboard_access.yml")
+@auth.login_required(role=['system'])
+def update_dashboard(uid: str, username: str):
+    logging.debug(f'endpoint update dashboard access, uid={uid}, username={username}')
+    # not implemented
+    return Response(), 202, headers

dbrepo-dashboard-service/ds-yml/update_dashboard_access.yml

+tags:
+  - dashboard-endpoint
+summary: "Update dashboard access"
+operationId: update_dashboard_access
+description: "Updates a dashboard access in the Dashboard UI. Requires role `system`."
+consumes:
+  - "application/json"
+produces:
+  - "application/json"
+parameters:
+  - name: uid
+    in: path
+    required: true
+    schema:
+      type: string
+      format: uuid
+  - name: username
+    in: path
+    required: true
+    schema:
+      type: string
+responses:
+  202:
+    description: Updated dashboard access successfully
+    content:
+      application/json:
+        schema:
+          type: object
+security:
+  - bearerAuth: [ ]
+  - basicAuth: [ ]

dbrepo-dashboard-service/init.py

@@ -20,9 +20,8 @@ dictConfig({
         'simple': {
             'format': '[%(asctime)s] [%(levelname)s] %(message)s',
         },
-        'ecs': {
-            'format': '{"@timestamp": "%(asctime)s", "log.level": "%(levelname)s", "log.logger": "%(module)s", "message": "%(message)s", "service_name": "dashboard-service-init", "service_version": "1.9.0"}',
-            'datefmt': '%Y-%m-%dT%H:%M:%S'
+        "ecs": {
+            "()": "ecs_logging.StdlibFormatter"
         },
     },
     'handlers': {

dbrepo-search-service/Pipfile

@@ -19,6 +19,7 @@ rdflib = "*"
 grafana-client = "*"
 dbrepo = {path = "./lib/dbrepo-1.9.0.tar.gz"}
 gunicorn = "*"
+ecs_logging = "*"
 
 [dev-packages]
 coverage = "*"

dbrepo-search-service/Pipfile.lock

 {
     "_meta": {
         "hash": {
-            "sha256": "3a91c289059bef3bbb9dc4f0686dd2c378e3c3cb19b003d4f0e10de9fd741b55"
+            "sha256": "56c33b1336dadb699cf5c5b2892e926bdad0ac127a68922188b78bb89dec922d"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -278,6 +278,14 @@
             "path": "./lib/dbrepo-1.9.0.tar.gz",
             "version": "==1.9.0"
         },
+        "ecs-logging": {
+            "hashes": [
+                "sha256:1dc9e216f614129db0e6a2f9f926da4e4cf8edf8de16d1045a20aa8e950291d3",
+                "sha256:f6e22d267770b06f797076f49b5fcc9d97108b22f452f5f9ed4b5367b1e61b5b"
+            ],
+            "index": "pypi",
+            "version": "==2.2.0"
+        },
         "events": {
             "hashes": [
                 "sha256:a7286af378ba3e46640ac9825156c93bdba7502174dd696090fdfcd4d80a1abd"

dbrepo-search-service/app.py

@@ -31,9 +31,8 @@ dictConfig({
         'simple': {
             'format': '[%(asctime)s] [%(levelname)s] %(message)s',
         },
-        'ecs': {
-            'format': '{"@timestamp": "%(asctime)s", "log.level": "%(levelname)s", "log.logger": "%(module)s", "message": "%(message)s", "service_name": "search-service", "service_version": "1.9.0"}',
-            'datefmt': '%Y-%m-%dT%H:%M:%S'
+        "ecs": {
+            "()": "ecs_logging.StdlibFormatter"
         },
     },
     'handlers': {

dbrepo-search-service/init.py

@@ -21,9 +21,8 @@ dictConfig({
         'simple': {
             'format': '[%(asctime)s] [%(levelname)s] %(message)s',
         },
-        'ecs': {
-            'format': '{"@timestamp": "%(asctime)s", "log.level": "%(levelname)s", "log.logger": "%(module)s", "message": "%(message)s", "service_name": "search-service-init", "service_version": "1.9.0"}',
-            'datefmt': '%Y-%m-%dT%H:%M:%S'
+        "ecs": {
+            "()": "ecs_logging.StdlibFormatter"
         },
     },
     'handlers': {