.docker/.env

+# Example values to override for non-test deployments
+
+#BASE_URL=https://example.com
+#ADMIN_EMAIL=noreply@example.com
+#LOG_LEVEL=info
+#IDENTITY_SERVICE_ADMIN_PASSWORD=admin
+#AUTH_SERVICE_ADMIN_PASSWORD=admin
+#METADATA_DB_PASSWORD=dbrepo
+#DATA_DB_PASSWORD=dbrepo
+#AUTH_DB_PASSWORD=dbrepo
+#S3_ACCESS_KEY_ID=seaweedfsadmin
+#S3_SECRET_ACCESS_KEY=seaweedfsadmin
+#SYSTEM_PASSWORD=admin

.docker/docker-compose.yml

@@ -80,6 +80,8 @@ services:
     container_name: dbrepo-auth-service
     hostname: auth-service
     image: bitnami/keycloak:26.0.0-debian-12-r1
+    ports:
+      - "8080:8080"
     volumes:
       - ./config/import-realms.sh:/docker-entrypoint-initdb.d/import-realms.sh
       - ./config/master-realm.json:/opt/keycloak/data/import/master-realm.json
@@ -109,7 +111,7 @@ services:
     restart: "no"
     container_name: dbrepo-metadata-service
     hostname: metadata-service
-    image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.5
+    image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.5.0
     volumes:
       - "${SHARED_VOLUME:-/tmp}:/tmp"
     environment:
@@ -172,7 +174,7 @@ services:
     restart: "no"
     container_name: dbrepo-analyse-service
     hostname: analyse-service
-    image: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.5
+    image: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.5.0
     environment:
       AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client}
       AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
@@ -180,8 +182,10 @@ services:
       JWT_PUBKEY: "${JWT_PUBKEY:-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB}"
       S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}"
       S3_BUCKET: "${S3_BUCKET:-dbrepo}"
-      S3_ENDPOINT: "${S3_ENDPOINT:-http://storage-service:9000}"
+      S3_ENDPOINT: "${S3_ENDPOINT:-storage-service:9000}"
+      S3_PROTO: "${S3_PROTO:-http}"
       S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}"
+      METADATA_SERVICE_ENDPOINT: ${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080}
       SYSTEM_USERNAME: "${SYSTEM_USERNAME:-admin}"
       SYSTEM_PASSWORD: "${SYSTEM_PASSWORD:-admin}"
     volumes:
@@ -225,7 +229,7 @@ services:
     restart: "no"
     container_name: dbrepo-search-db
     hostname: search-db
-    image: registry.datalab.tuwien.ac.at/dbrepo/search-db:1.5
+    image: registry.datalab.tuwien.ac.at/dbrepo/search-db:1.5.0
     healthcheck:
       test: curl -sSL localhost:9200/_plugins/_security/health | jq .status | grep UP
       interval: 10s
@@ -249,7 +253,7 @@ services:
     restart: "no"
     container_name: dbrepo-search-service
     hostname: search-service
-    image: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.5
+    image: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.5.0
     environment:
       AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client}
       AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT_SECRET:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
@@ -267,32 +271,11 @@ services:
       timeout: 5s
       retries: 12
 
-  dbrepo-data-db-sidecar:
-    restart: "no"
-    container_name: dbrepo-data-db-sidecar
-    hostname: data-db-sidecar
-    image: registry.datalab.tuwien.ac.at/dbrepo/data-db-sidecar:1.5
-    environment:
-      S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}"
-      S3_BUCKET: "${S3_BUCKET:-dbrepo}"
-      S3_ENDPOINT: "${S3_ENDPOINT:-http://storage-service:9000}"
-      S3_FILE_PATH: "${S3_FILE_PATH:-/tmp}"
-      S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}"
-    volumes:
-      - "${SHARED_FILESYSTEM:-/tmp}:/tmp"
-    healthcheck:
-      test: curl -sSL localhost:8080/health | grep 'UP' || exit 1
-      interval: 10s
-      timeout: 5s
-      retries: 12
-    logging:
-      driver: json-file
-
   dbrepo-ui:
     restart: "no"
     container_name: dbrepo-ui
     hostname: ui
-    image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.5
+    image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.5.0
     environment:
       NUXT_PUBLIC_API_CLIENT: "${BASE_URL:-http://localhost}"
       NUXT_PUBLIC_API_SERVER: "${BASE_URL:-http://localhost}"
@@ -329,6 +312,8 @@ services:
         condition: service_healthy
       dbrepo-search-db:
         condition: service_healthy
+      dbrepo-dashboard-service:
+        condition: service_healthy
       dbrepo-ui:
         condition: service_healthy
     logging:
@@ -360,7 +345,7 @@ services:
     init: true
     container_name: dbrepo-search-service-init
     hostname: search-service-init
-    image: registry.datalab.tuwien.ac.at/dbrepo/search-service-init:1.5
+    image: registry.datalab.tuwien.ac.at/dbrepo/search-service-init:1.5.0
     environment:
       METADATA_SERVICE_ENDPOINT: ${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080}
       OPENSEARCH_HOST: ${OPENSEARCH_HOST:-search-db}
@@ -386,6 +371,7 @@ services:
       - storage-service-data:/data
     ports:
       - "9000:9000"
+      - "8888:8888"
     healthcheck:
       test: echo "cluster.check" | weed shell | grep "checking master.*ok" || exit 1
       interval: 10s
@@ -414,8 +400,15 @@ services:
     restart: "no"
     container_name: dbrepo-dashboard-service
     hostname: dashboard-service
-    image: registry.datalab.tuwien.ac.at/dbrepo/dashboard-service:1.5
+    image: registry.datalab.tuwien.ac.at/dbrepo/dashboard-service:1.5.0
+    ports:
+      - "3000:3000"
+    volumes:
+      - dashboard-service-data:/opt/bitnami/grafana/data
     environment:
+      GF_SERVER_DOMAIN: "dashboard-service"
+      GF_SERVER_ROOT_URL: "${BASE_URL:-http://localhost}"
+      GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION: "true"
       LDAP_ADMIN_USERNAME: "${IDENTITY_SERVICE_ADMIN_USERNAME:-admin}"
       LDAP_ADMIN_PASSWORD: "${IDENTITY_SERVICE_ADMIN_PASSWORD:-admin}"
       LDAP_ROOT: "${IDENTITY_SERVICE_ROOT:-dc=dbrepo,dc=at}"
@@ -435,7 +428,7 @@ services:
     init: true
     container_name: dbrepo-storage-service-init
     hostname: storage-service-init
-    image: registry.datalab.tuwien.ac.at/dbrepo/storage-service-init:1.5
+    image: registry.datalab.tuwien.ac.at/dbrepo/storage-service-init:1.5.0
     environment:
       WEED_CLUSTER_SW_MASTER: "${STORAGE_SERVICE_MASTER_ENDPOINT:-storage-service:9333}"
       S3_BUCKET: "${S3_BUCKET:-dbrepo}"
@@ -475,7 +468,7 @@ services:
     restart: "no"
     container_name: dbrepo-data-service
     hostname: data-service
-    image: registry.datalab.tuwien.ac.at/dbrepo/data-service:1.5
+    image: registry.datalab.tuwien.ac.at/dbrepo/data-service:1.5.0
     volumes:
       - "${SHARED_VOLUME:-/tmp}:/tmp"
     environment:
@@ -493,6 +486,7 @@ services:
       BROKER_USERNAME: "${SYSTEM_USERNAME:-admin}"
       BROKER_VIRTUALHOST: "${BROKER_VIRTUALHOST:-dbrepo}"
       CONNECTION_TIMEOUT: ${CONNECTION_TIMEOUT:-60000}
+      COMPUTE_SERVICE_ENDPOINT: "${COMPUTE_SERVICE_ENDPOINT:-local[2]}"
       EXCHANGE_NAME: "${EXCHANGE_NAME:-dbrepo}"
       METADATA_SERVICE_ENDPOINT: "${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080}"
       GRANT_DEFAULT_READ: "${GRANT_DEFAULT_READ:-SELECT}"
@@ -510,6 +504,7 @@ services:
       S3_FILE_PATH: "${S3_FILE_PATH:-/tmp}"
       S3_IMPORT_BUCKET: "${S3_IMPORT_BUCKET:-dbrepo-upload}"
       S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}"
+      SPARK_USER: "${COMPUTE_SERVICE_USERNAME:-spark}"
       SYSTEM_USERNAME: "${SYSTEM_USERNAME:-admin}"
       SYSTEM_PASSWORD: "${SYSTEM_PASSWORD:-admin}"
     healthcheck:

.docs/api/dashboard-service.md

@@ -8,13 +8,15 @@ author: Martin Weise
 
     Image: [`docker.io/bitnami/grafana:10.4.9-debian-12-r0`](https://hub.docker.com/r/bitnami/grafana)
 
-    * Ports: `http://<hostname>:3000`
-    * Prometheus: `http://<hostname>:3000/prometheus`
+    * Ports: 3000/tcp
+    * UI: `http://<hostname>/dashboard`
+        * Management UI: `http://<hostname>:3000` (see [Management](#management))
+    * Prometheus: `http://<hostname>/dashboard/metrics`
 
     To directly access in Kubernetes (for e.g. debugging), forward the svc port to your local machine:
 
     ```shell
-    kubectl [-n namespace] port-forward svc/broker-service 3000:3000
+    kubectl [-n namespace] port-forward svc/dashboard-service 3000:3000
     ```
 
 ## Overview
@@ -23,8 +25,21 @@ The Dashboard Service is visualizing the status of DBRepo with charts. The defau
 `/etc/grafana/provisioning/dashboards/provider.yaml` checks for new `JSON` dashboard files in `/app/dashboards` every 10
 seconds and makes the available in the Dashboard Service.
 
+## Management
+
+The Dashboard Service can be accessed with admin users (see [Identity Service](../../api/identity-service)). In this
+case, access the UI via the port `3000` directly to avoid UI Redirects.
+
+!!! bug "UI Redirects"
+
+    It is a known bug [#460](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/issues/460)
+    that when being logged-in, the UI randomly redirects when being accessed from the Gateway Service, we therefore
+    recommend to access port `3000` directly: `http://<hostname>:3000`. Anonmyous users are not affected.
+
 ## Limitations
 
+* Unintended redirects when being logged-in (see above).
+
 !!! question "Do you miss functionality? Do these limitations affect you?"
 
     We strongly encourage you to help us implement it as we are welcoming contributors to open-source software and get

.docs/api/data-db.md

@@ -24,24 +24,10 @@ Library.
 
 ## Data
 
-The procedures require the user-generated databases to have the same collation (because of comparison operations).
-Ensure that the Data Database has the character set `utf8mb4` and collation `utf8mb4_general_ci` in your `my.cfg`:
-
-```ini
-[mysqld]
-character_set_server=utf8mb4
-collation_server=utf8mb4_general_ci
-```
-
-We observed this unexpected behavior for
+The procedures requires the in parameter of the `hash_table` stored procedure to have the same collation as the
+`information_schema.columns` table. We observed this unexpected behavior for
 the [MariaDB Galera chart](https://artifacthub.io/packages/helm/bitnami/mariadb-galera) powered by Bitnami and had to
-set extra flags. We could not observe this behavior with
-the [MariaDB Galera container image](https://hub.docker.com/r/bitnami/mariadb-galera) itself.
-
-```yaml
-mariadb-galera:
-  extraFlags: "--character-set-server=utf8mb4 --collation-server=utf8mb4_general_ci"
-```
+set extra flags.
 
 ### Backup
 

.docs/changelog.md

@@ -8,7 +8,9 @@ author: Martin Weise
 
 !!! warning "Contains Breaking Changes"
 
-    This release updates the Metadata Database schema which is incompatible to v1.4.6!
+    This release updates the Metadata Database schema which is incompatible to v1.4.6! Use the migration
+    script [`schema_1.4.5-to-1.5.0.sql`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.5/dbrepo-metadata-db/migration/schema_1.4.5-to-1.5.0.sql)
+    to apply the changes manually.
 
 ### What's Changed
 

.docs/images/coverage.svg

@@ -9,7 +9,7 @@
     </mask>
     <g mask="url(#anybadge_1)">
         <path fill="#555" d="M0 0h65v20H0z"/>
-        <path fill="#A9A9A9" d="M65 0h59v20H65z"/>
+        <path fill="#C0C0C0" d="M65 0h59v20H65z"/>
         <path fill="url(#b)" d="M0 0h124v20H0z"/>
     </g>
     <g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11">

.docs/images/maintainability.svg

+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" width="158" height="20">
+    <linearGradient id="b" x2="0" y2="100%">
+        <stop offset="0" stop-color="#bbb" stop-opacity=".1"/>
+        <stop offset="1" stop-opacity=".1"/>
+    </linearGradient>
+    <mask id="anybadge_1">
+        <rect width="158" height="20" rx="3" fill="#fff"/>
+    </mask>
+    <g mask="url(#anybadge_1)">
+        <path fill="#555" d="M0 0h99v20H0z"/>
+        <path fill="#C0C0C0" d="M99 0h59v20H99z"/>
+        <path fill="url(#b)" d="M0 0h158v20H0z"/>
+    </g>
+    <g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11">
+        <text x="50.5" y="15" fill="#010101" fill-opacity=".3">maintainability</text>
+        <text x="49.5" y="14">maintainability</text>
+    </g>
+    <g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11">
+        <text x="129.5" y="15" fill="#010101" fill-opacity=".3">unknown</text>
+        <text x="128.5" y="14">unknown</text>
+    </g>
+</svg>
+

.docs/images/pipeline.svg

@@ -9,7 +9,7 @@
     </mask>
     <g mask="url(#anybadge_1)">
         <path fill="#555" d="M0 0h57v20H0z"/>
-        <path fill="#A9A9A9" d="M57 0h59v20H57z"/>
+        <path fill="#C0C0C0" d="M57 0h59v20H57z"/>
         <path fill="url(#b)" d="M0 0h116v20H0z"/>
     </g>
     <g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11">

.docs/images/security.svg

+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" width="119" height="20">
+    <linearGradient id="b" x2="0" y2="100%">
+        <stop offset="0" stop-color="#bbb" stop-opacity=".1"/>
+        <stop offset="1" stop-opacity=".1"/>
+    </linearGradient>
+    <mask id="anybadge_1">
+        <rect width="119" height="20" rx="3" fill="#fff"/>
+    </mask>
+    <g mask="url(#anybadge_1)">
+        <path fill="#555" d="M0 0h60v20H0z"/>
+        <path fill="#C0C0C0" d="M60 0h59v20H60z"/>
+        <path fill="url(#b)" d="M0 0h119v20H0z"/>
+    </g>
+    <g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11">
+        <text x="31.0" y="15" fill="#010101" fill-opacity=".3">security</text>
+        <text x="30.0" y="14">security</text>
+    </g>
+    <g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11">
+        <text x="90.5" y="15" fill="#010101" fill-opacity=".3">unknown</text>
+        <text x="89.5" y="14">unknown</text>
+    </g>
+</svg>
+

.docs/index.md

@@ -11,6 +11,8 @@ author: Martin Weise
 [![Image Pulls](https://img.shields.io/docker/pulls/dbrepo/data-service?style=flat)](https://hub.docker.com/u/dbrepo){ tabindex=-1 }
 [![Helm Chart version](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/dbrepo)](https://artifacthub.io/packages/helm/dbrepo/dbrepo){ tabindex=-1 }
 [![GitLab License](https://img.shields.io/gitlab/license/fair-data-austria-db-repository%2Ffda-services?gitlab_url=https%3A%2F%2Fgitlab.phaidra.org%2F&style=flat&cacheSeconds=3600)](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services){ tabindex=-1 }
+![Maintainability Rating](./images/maintainability.svg)
+![Security Rating](./images/security.svg)
 
 Documentation for version: [v1.5.0](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/releases).
 

.gitignore

@@ -61,7 +61,7 @@ ready
 *.p12
 
 # Environment
-.env
+.docker/.env
 
 # X509
 root.crt

.gitlab-ci.yml

@@ -360,9 +360,10 @@ docs-registry:
     refs:
       - /^release-.*/
   before_script:
+    - "pip install pipenv"
+    - "pipenv install --dev --system --deploy"
     - "apt-get update && apt-get install -y sed"
   script:
-    - pip install -r ./requirements.txt
     - python3 .docs/docker/release.py
 
 release-images:
@@ -414,10 +415,11 @@ release-docs:
     refs:
       - /^release-.*/
   before_script:
-    - "apk add --update alpine-sdk bash sed wget openssh jq curl"
+    - "apk add --no-cache alpine-sdk bash sed wget openssh jq curl"
     - "pip install pipenv"
-    - "pip install -r ./requirements.txt"
+    - "pipenv install --dev --system --deploy"
     - "mkdir -p ./final/${DOC_VERSION}/rest"
+    - "mkdir -p ./final/${APP_VERSION}"
   script:
     - "make gen-lib-doc gen-docs-doc package-config"
     - "cp -r ./lib/python/docs/build/html ./final/${DOC_VERSION}/python" # sphinx
@@ -426,6 +428,7 @@ release-docs:
     - "cp .docs/.swagger/custom.css ./final/${DOC_VERSION}/rest/custom.css" # swagger
     - "cp -r ./site/* ./final/${DOC_VERSION}" # mkdocs
     - "cp .docker/dist.tar.gz ./final/${DOC_VERSION}/dist.tar.gz" # dist
+    - "cp .docs/redirect.html ./final/${APP_VERSION}/index.html" # redirect patch docs
     - "bash ./.gitlab/gen-badge.sh"
     - eval $(ssh-agent -s)
     - "mkdir -p /root/.ssh"

.gitlab/gen-badge.sh

 #!/bin/bash
 GITLAB_URL="https://gitlab.phaidra.org"
 # if we reached this script, all the tests have passed
-anybadge --label pipeline --value "passed" failed=red passed=green canceled=darkgray > "./final/${APP_VERSION}/images/pipeline.svg"
+anybadge --label pipeline --value "passed" failed=red passed=green canceled=darkgray > "./final/${DOC_VERSION}/images/pipeline.svg"
 PIPELINE_COVERAGE=$(curl -fsSL -H "PRIVATE-TOKEN: ${CI_TOKEN}" "${GITLAB_URL}/api/v4/projects/450/pipelines/latest?ref=${CI_COMMIT_BRANCH}" | jq --raw-output .coverage)
 echo "[INFO] pipeline coverage: ${PIPELINE_COVERAGE}"
 if [ "${PIPELINE_COVERAGE}" != "null" ]; then
-  anybadge --label coverage --value "${PIPELINE_COVERAGE}" coverage > "./final/${APP_VERSION}/images/coverage.svg"
+  anybadge --label coverage --value "${PIPELINE_COVERAGE}" coverage > "./final/${DOC_VERSION}/images/coverage.svg"
 else
   echo "[WARNING] Skipping badge generation, displaying default badge text: unknown"
 fi
+curl "${SONARQUBE_URL}/api/project_badges/measure?project=fair-data-austria-db-repository_fda-services_a57fa043-ab99-4cdd-a721-162d9a916d77&metric=sqale_rating&token=${CI_SONAR_TOKEN}" > "./final/${DOC_VERSION}/images/maintainability.svg"
+curl "${SONARQUBE_URL}/api/project_badges/measure?project=fair-data-austria-db-repository_fda-services_a57fa043-ab99-4cdd-a721-162d9a916d77&metric=security_rating&token=${CI_SONAR_TOKEN}" > "./final/${DOC_VERSION}/images/security.svg"
+echo "[INFO] retrieved SonarQube badges"
\ No newline at end of file

README.md

-![Java 21](https://img.shields.io/badge/Java-21-white?style=flat)
-![Python 3.11](https://img.shields.io/badge/Python-3.11-white?style=flat)
-![RabbitMQ 3.12](https://img.shields.io/badge/RabbitMQ-3.12-white?style=flat)
-![MariaDB 11.2](https://img.shields.io/badge/MariaDB-11.2-white?style=flat)
-![OpenSearch 2.10](https://img.shields.io/badge/OpenSearch-2.10-white?style=flat)
-![SeaweedFS 3.59](https://img.shields.io/badge/SeaweedFS-3.59-white?style=flat)
-![OpenLDAP 2.6](https://img.shields.io/badge/OpenLDAP-2.6-white?style=flat)
-![Spark 3.4](https://img.shields.io/badge/Spark-3.4-white?style=flat)
-![Keycloak 24.0](https://img.shields.io/badge/Keycloak-24.0-white?style=flat)
+[![CI/CD Status](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/badges/master/pipeline.svg)](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services)
+[![CI/CD Coverage](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/badges/master/coverage.svg)](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services)
+[![Latest Release](https://img.shields.io/gitlab/v/release/fair-data-austria-db-repository%2Ffda-services?gitlab_url=https%3A%2F%2Fgitlab.phaidra.org&display_name=release&style=flat)](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services)
+[![PyPI Library version](https://img.shields.io/pypi/v/dbrepo)](https://pypi.org/project/dbrepo/)
+[![Image Pulls](https://img.shields.io/docker/pulls/dbrepo/data-service?style=flat)](https://hub.docker.com/u/dbrepo)
+[![Helm Chart version](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/dbrepo)](https://artifacthub.io/packages/helm/dbrepo/dbrepo)
+[![GitLab License](https://img.shields.io/gitlab/license/fair-data-austria-db-repository%2Ffda-services?gitlab_url=https%3A%2F%2Fgitlab.phaidra.org%2F&style=flat&cacheSeconds=3600)](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services)
 
 <img src="./dbrepo-ui/public/logo.png" width="200" alt="DBRepo &mdash; Repository for Data in Databases" />
 

dbrepo-analyse-service/api/dto.py

@@ -44,7 +44,6 @@ class ColumnAnalysisDto(BaseModel):
     null_allowed: bool
     size: Optional[int] = None
     d: Optional[int] = None
-    dfid: Optional[int] = None
     enums: Optional[list] = None
     sets: Optional[list] = None
 

dbrepo-analyse-service/determine_dt.py

@@ -74,6 +74,8 @@ def determine_datatypes(filename, enum=False, enum_tol=0.0001, separator=',') ->
                 if pandas.to_numeric(df[name], errors='coerce').notnull().all():
                     logging.debug(f"mapped column {name} from float64 to decimal")
                     col.type = DataTypeDto.DECIMAL
+                    col.size = 40
+                    col.d = 20
                 else:
                     logging.debug(f"mapped column {name} from float64 to text")
                     col.type = DataTypeDto.TEXT

dbrepo-analyse-service/test/test_determine_dt.py

@@ -36,6 +36,8 @@ class DetermineDatatypesTest(unittest.TestCase):
             },
             "Wert": {
                 "type": "decimal",
+                "size": 40,
+                "d": 20,
                 "null_allowed": False,
             },
             "Status": {
@@ -81,6 +83,8 @@ class DetermineDatatypesTest(unittest.TestCase):
             },
             "Wert": {
                 "type": "decimal",
+                "size": 40,
+                "d": 20,
                 "null_allowed": False,
             },
             "Status": {
@@ -125,6 +129,8 @@ class DetermineDatatypesTest(unittest.TestCase):
             },
             "Wert": {
                 "type": "decimal",
+                "size": 40,
+                "d": 20,
                 "null_allowed": False,
             },
             "Status": {
@@ -149,6 +155,8 @@ class DetermineDatatypesTest(unittest.TestCase):
             },
             "float": {
                 "type": "decimal",
+                "size": 40,
+                "d": 20,
                 "null_allowed": False,
             },
             "string": {

dbrepo-dashboard-service/grafana.ini

 [server]
+protocol = http
+domain = localhost
+root_url = http://%(domain)s/dashboard/
 http_port = 3000
 
 [security]

dbrepo-gateway-service/dbrepo.conf

@@ -36,6 +36,10 @@ upstream upload {
     server upload-service:8080;
 }
 
+upstream dashboard-service {
+    server dashboard-service:3000;
+}
+
 server {
     listen 80 default_server;
     server_name _;
@@ -44,6 +48,29 @@ server {
         stub_status;
     }
 
+    location /dashboard/ {
+        rewrite  ^/dashboard/(.*)  /$1 break;
+        proxy_set_header        Host $host;
+        proxy_set_header        X-Real-IP $remote_addr;
+        proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header        X-Forwarded-Proto $scheme;
+        proxy_pass              http://dashboard-service;
+        proxy_read_timeout      90;
+    }
+
+    # Proxy Grafana Live WebSocket connections.
+    location /dashboard/api/live/ {
+        proxy_set_header        Host $host;
+        proxy_set_header        X-Real-IP $remote_addr;
+        proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header        X-Forwarded-Proto $scheme;
+        proxy_set_header        Upgrade $http_upgrade;
+        proxy_set_header        Connection $connection_upgrade;
+        proxy_http_version      1.1;
+        proxy_pass              http://dashboard-service;
+        proxy_read_timeout      90;
+    }
+
     location /api/search {
         proxy_set_header        Host $host;
         proxy_set_header        X-Real-IP $remote_addr;

dbrepo-metadata-db/migration/schema_1.4.5-to-1.5.0.sql

+ALTER TABLE mdb_image_types
+    DROP SYSTEM VERSIONING;
+TRUNCATE mdb_image_types;
+ALTER TABLE mdb_image_types
+    DROP
+        COLUMN hint;
+ALTER TABLE mdb_image_types
+    ADD COLUMN type_hint TEXT;
+ALTER TABLE mdb_image_types
+    ADD COLUMN data_hint TEXT;
+ALTER TABLE mdb_image_types
+    ADD COLUMN is_generated BOOLEAN NOT NULL;
+ALTER TABLE mdb_image_types
+    ADD SYSTEM VERSIONING;
+
+INSERT INTO `mdb_image_types` (image_id, display_name, value, size_min, size_max, size_default, size_required,
+                               size_step, d_min, d_max, d_default, d_required, d_step, type_hint, data_hint,
+                               documentation, is_quoted, is_buildable, is_generated)
+VALUES (1, 'BIGINT(size)', 'bigint', 0, null, null, false, 1, null, null, null, null, null, null, null,
+        'https://mariadb.com/kb/en/bigint/', false, true, false),
+       (1, 'BINARY(size)', 'binary', 0, 255, 255, true, 1, null, null, null, null, null, 'size in Bytes', null,
+        'https://mariadb.com/kb/en/binary/', false, true, false),
+       (1, 'BIT(size)', 'bit', 0, 64, null, false, 1, null, null, null, null, null, null, null,
+        'https://mariadb.com/kb/en/bit/', false, true, false),
+       (1, 'BLOB(size)', 'blob', 0, 65535, null, false, 1, null, null, null, null, null, 'size in Bytes', null,
+        'https://mariadb.com/kb/en/blob/', false, false, false),
+       (1, 'BOOL', 'bool', null, null, null, null, null, null, null, null, null, null, null, null,
+        'https://mariadb.com/kb/en/bool/', false, true, false),
+       (1, 'CHAR(size)', 'char', 0, 255, 255, false, 1, null, null, null, null, null, null, null,
+        'https://mariadb.com/kb/en/char/', false, true, false),
+       (1, 'DATE', 'date', null, null, null, null, null, null, null, null, null, null,
+        'min. 1000-01-01, max. 9999-12-31', 'e.g. YYYY-MM-DD, YY-MM-DD, YYMMDD, YYYY/MM/DD',
+        'https://mariadb.com/kb/en/date/', true, true, false),
+       (1, 'DATETIME(fsp)', 'datetime', 0, 6, null, null, 1, null, null, null, null, null,
+        'fsp=microsecond precision, min. 1000-01-01 00:00:00.0, max. 9999-12-31 23:59:59.9',
+        'e.g. YYYY-MM-DD HH:MM:SS, YY-MM-DD HH:MM:SS, YYYYMMDDHHMMSS, YYMMDDHHMMSS, YYYYMMDD, YYMMDD',
+        'https://mariadb.com/kb/en/datetime/', true, true, false),
+       (1, 'DECIMAL(size, d)', 'decimal', 0, 65, null, false, 1, 0, 38, null, false, null, null, null,
+        'https://mariadb.com/kb/en/decimal/', false, true, false),
+       (1, 'DOUBLE(size, d)', 'double', null, null, null, false, null, null, null, null, false, null, null, null,
+        'https://mariadb.com/kb/en/double/', false, true, false),
+       (1, 'ENUM(v1,v2,...)', 'enum', null, null, null, null, null, null, null, null, null, null, null,
+        'e.g. value1, value2, ...', 'https://mariadb.com/kb/en/enum/', true, true, false),
+       (1, 'FLOAT(size)', 'float', null, null, null, false, null, null, null, null, null, null, null, null,
+        'https://mariadb.com/kb/en/float/', false, true, false),
+       (1, 'INT(size)', 'int', null, null, null, false, null, null, null, null, null, null, 'size in Bytes', null,
+        'https://mariadb.com/kb/en/int/', false, true, false),
+       (1, 'LONGBLOB', 'longblob', null, null, null, null, null, null, null, null, null, null, 'max. 3.999 GiB', null,
+        'https://mariadb.com/kb/en/longblob/', false, true, false),
+       (1, 'LONGTEXT', 'longtext', null, null, null, null, null, null, null, null, null, null, 'max. 3.999 GiB', null,
+        'https://mariadb.com/kb/en/longtext/', true, true, false),
+       (1, 'MEDIUMBLOB', 'mediumblob', null, null, null, null, null, null, null, null, null, null, 'max. 15.999 MiB',
+        null, 'https://mariadb.com/kb/en/mediumblob/', false, true, false),
+       (1, 'MEDIUMINT', 'mediumint', null, null, null, null, null, null, null, null, null, null, 'size in Bytes', null,
+        'https://mariadb.com/kb/en/mediumint/', false, true, false),
+       (1, 'MEDIUMTEXT', 'mediumtext', null, null, null, null, null, null, null, null, null, null, 'size in Bytes',
+        null, 'https://mariadb.com/kb/en/mediumtext/', true, true, false),
+       (1, 'SERIAL', 'serial', null, null, null, null, null, null, null, null, null, null, null,
+        null, 'https://mariadb.com/kb/en/bigint/', true, true, true),
+       (1, 'SET(v1,v2,...)', 'set', null, null, null, null, null, null, null, null, null, null, null,
+        'e.g. value1, value2, ...', 'https://mariadb.com/kb/en/set/', true, true, false),
+       (1, 'SMALLINT(size)', 'smallint', 0, null, null, false, null, null, null, null, null, null, 'size in Bytes',
+        null, 'https://mariadb.com/kb/en/smallint/', false, true, false),
+       (1, 'TEXT(size)', 'text', 0, null, null, false, null, null, null, null, null, null, 'size in Bytes', null,
+        'https://mariadb.com/kb/en/text/', true, true, false),
+       (1, 'TIME(fsp)', 'time', 0, 6, 0, false, null, null, null, null, null, null,
+        'fsp=microsecond precision, min. 0, max. 6', 'e.g. HH:MM:SS, HH:MM, HHMMSS, H:M:S',
+        'https://mariadb.com/kb/en/time/', true, true, false),
+       (1, 'TIMESTAMP(fsp)', 'timestamp', 0, 6, 0, false, null, null, null, null, null, null,
+        'fsp=microsecond precision, min. 0, max. 6',
+        'e.g. YYYY-MM-DD HH:MM:SS, YY-MM-DD HH:MM:SS, YYYYMMDDHHMMSS, YYMMDDHHMMSS, YYYYMMDD, YYMMDD',
+        'https://mariadb.com/kb/en/timestamp/', true, true, false),
+       (1, 'TINYBLOB', 'tinyblob', null, null, null, null, null, null, null, null, null, null, null,
+        'fsp=microsecond precision, min. 0, max. 6', 'https://mariadb.com/kb/en/timestamp/', false, true, false),
+       (1, 'TINYINT(size)', 'tinyint', 0, null, null, false, null, null, null, null, null, null, null,
+        'size in Bytes', 'https://mariadb.com/kb/en/tinyint/', false, true, false),
+       (1, 'TINYTEXT', 'tinytext', null, null, null, null, null, null, null, null, null, null, null,
+        'max. 255 characters', 'https://mariadb.com/kb/en/tinytext/', true, true, false),
+       (1, 'YEAR', 'year', 2, 4, null, false, 2, null, null, null, null, null, 'min. 1901, max. 2155', 'e.g. YYYY, YY',
+        'https://mariadb.com/kb/en/year/', false, true, false),
+       (1, 'VARBINARY(size)', 'varbinary', 0, null, null, true, null, null, null, null, null, null, null,
+        null, 'https://mariadb.com/kb/en/varbinary/', false, true, false),
+       (1, 'VARCHAR(size)', 'varchar', 0, 65532, 255, true, null, null, null, null, null, null, null,
+        null, 'https://mariadb.com/kb/en/varchar/', false, true, false);
+
+ALTER TABLE mdb_related_identifiers
+    DROP SYSTEM VERSIONING;
+ALTER TABLE mdb_related_identifiers
+    MODIFY type ENUM ('DOI','URL','URN','ARK','ARXIV','BIBCODE','EAN13','EISSN','HANDLE','IGSN','ISBN','ISTC','LISSN','LSID','PMID','PURL','UPC','W3ID') NOT NULL;
+ALTER TABLE mdb_related_identifiers
+    MODIFY relation ENUM ('IS_CITED_BY','CITES','IS_SUPPLEMENT_TO','IS_SUPPLEMENTED_BY','IS_CONTINUED_BY','CONTINUES','IS_DESCRIBED_BY','DESCRIBES','HAS_METADATA','IS_METADATA_FOR','HAS_VERSION','IS_VERSION_OF','IS_NEW_VERSION_OF','IS_PREVIOUS_VERSION_OF','IS_PART_OF','HAS_PART','IS_PUBLISHED_IN','IS_REFERENCED_BY','REFERENCES','IS_DOCUMENTED_BY','DOCUMENTS','IS_COMPILED_BY','COMPILES','IS_VARIANT_FORM_OF','IS_ORIGINAL_FORM_OF','IS_IDENTICAL_TO','IS_REVIEWED_BY','REVIEWS','IS_DERIVED_FROM','IS_SOURCE_OF','IS_REQUIRED_BY','REQUIRES','IS_OBSOLETED_BY','OBSOLETES') NOT NULL;
+ALTER TABLE mdb_related_identifiers
+    ADD SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_image_operators`
+(
+    id
+                  SERIAL,
+    image_id
+                  BIGINT
+                               NOT
+                                   NULL,
+    display_name
+                  varchar(255) NOT NULL,
+    value         varchar(255) NOT NULL,
+    documentation TEXT         NOT NULL,
+    PRIMARY KEY
+        (
+         id
+            ),
+    FOREIGN KEY
+        (
+         image_id
+            ) REFERENCES `mdb_images`
+        (
+         `id`
+            ),
+    UNIQUE
+        (
+         value
+            )
+) WITH SYSTEM VERSIONING;
+
+INSERT INTO `mdb_image_operators` (image_id, display_name, value, documentation)
+VALUES (1, 'Equal operator', '=', 'https://mariadb.com/kb/en/assignment-operators-assignment-operator/'),
+       (1, 'NULL-safe equal operator', '<=>', 'https://mariadb.com/kb/en/null-safe-equal/'),
+       (1, 'Less-than operator', '<', 'https://mariadb.com/kb/en/less-than/'),
+       (1, 'Less than or equal operator', '<=', 'https://mariadb.com/kb/en/less-than-or-equal/'),
+       (1, 'Greater-than operator', '>', 'https://mariadb.com/kb/en/greater-than/'),
+       (1, 'Greater than or equal operator', '>=', 'https://mariadb.com/kb/en/greater-than-or-equal/'),
+       (1, 'Not equal operator', '!=', 'https://mariadb.com/kb/en/not-equal/'),
+       (1, 'Addition operator', '+', 'https://mariadb.com/kb/en/addition-operator/'),
+       (1, 'Division operator', '/', 'https://mariadb.com/kb/en/division-operator/'),
+       (1, 'Modulo operator', '%', 'https://mariadb.com/kb/en/modulo-operator/'),
+       (1, 'Multiplication operator', '*', 'https://mariadb.com/kb/en/multiplication-operator/'),
+       (1, 'Subtraction operator', '-', 'https://mariadb.com/kb/en/subtraction-operator-/'),
+       (1, 'LIKE', 'LIKE', 'https://mariadb.com/kb/en/like/'),
+       (1, 'NOT LIKE', 'NOT LIKE', 'https://mariadb.com/kb/en/not-like/'),
+       (1, 'IN', 'IN', 'https://mariadb.com/kb/en/in/'),
+       (1, 'NOT IN', 'NOT IN', 'https://mariadb.com/kb/en/not-in/'),
+       (1, 'IS', 'IS', 'https://mariadb.com/kb/en/is/'),
+       (1, 'IS NOT', 'IS NOT', 'https://mariadb.com/kb/en/is-not/'),
+       (1, 'IS NOT NULL', 'IS NOT NULL', 'https://mariadb.com/kb/en/is-not-null/'),
+       (1, 'IS NULL', 'IS NULL', 'https://mariadb.com/kb/en/is-null/'),
+       (1, 'ISNULL', 'ISNULL', 'https://mariadb.com/kb/en/isnull/'),
+       (1, 'REGEXP', 'REGEXP', 'https://mariadb.com/kb/en/regexp/'),
+       (1, 'NOT REGEXP', 'NOT REGEXP', 'https://mariadb.com/kb/en/not-regexp/'),
+       (1, 'Bitwise AND', '&', 'https://mariadb.com/kb/en/bitwise_and/'),
+       (1, 'Bitwise OR', '|', 'https://mariadb.com/kb/en/bitwise-or/'),
+       (1, 'Bitwise XOR', '^', 'https://mariadb.com/kb/en/bitwise-xor/'),
+       (1, 'Bitwise NOT', '~', 'https://mariadb.com/kb/en/bitwise-not/'),
+       (1, 'Left shift', '<<', 'https://mariadb.com/kb/en/shift-left/'),
+       (1, 'Right shift', '>>', 'https://mariadb.com/kb/en/shift-right/');
\ No newline at end of file