Merge branch 'dev' into 'master'

Fixed mapping problem where UK and FK share columns they are inserted

See merge request !341

.docker/.env

+# Example values to override for non-test deployments
+
+#BASE_URL=https://example.com
+#ADMIN_EMAIL=noreply@example.com
+#LOG_LEVEL=info
+#IDENTITY_SERVICE_ADMIN_PASSWORD=admin
+#AUTH_SERVICE_ADMIN_PASSWORD=admin
+#METADATA_DB_PASSWORD=dbrepo
+#DATA_DB_PASSWORD=dbrepo
+#AUTH_DB_PASSWORD=dbrepo
+#S3_ACCESS_KEY_ID=seaweedfsadmin
+#S3_SECRET_ACCESS_KEY=seaweedfsadmin
+#SYSTEM_PASSWORD=admin

.docs/api/data-db.md

@@ -24,24 +24,10 @@ Library.
 
 ## Data
 
-The procedures require the user-generated databases to have the same collation (because of comparison operations).
-Ensure that the Data Database has the character set `utf8mb4` and collation `utf8mb4_general_ci` in your `my.cfg`:
-
-```ini
-[mysqld]
-character_set_server=utf8mb4
-collation_server=utf8mb4_general_ci
-```
-
-We observed this unexpected behavior for
+The procedures requires the in parameter of the `hash_table` stored procedure to have the same collation as the
+`information_schema.columns` table. We observed this unexpected behavior for
 the [MariaDB Galera chart](https://artifacthub.io/packages/helm/bitnami/mariadb-galera) powered by Bitnami and had to
-set extra flags. We could not observe this behavior with
-the [MariaDB Galera container image](https://hub.docker.com/r/bitnami/mariadb-galera) itself.
-
-```yaml
-mariadb-galera:
-  extraFlags: "--character-set-server=utf8mb4 --collation-server=utf8mb4_general_ci"
-```
+set extra flags.
 
 ### Backup
 

.docs/images/coverage.svg

@@ -9,7 +9,7 @@
     </mask>
     <g mask="url(#anybadge_1)">
         <path fill="#555" d="M0 0h65v20H0z"/>
-        <path fill="#A9A9A9" d="M65 0h59v20H65z"/>
+        <path fill="#C0C0C0" d="M65 0h59v20H65z"/>
         <path fill="url(#b)" d="M0 0h124v20H0z"/>
     </g>
     <g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11">

.docs/images/maintainability.svg

+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" width="158" height="20">
+    <linearGradient id="b" x2="0" y2="100%">
+        <stop offset="0" stop-color="#bbb" stop-opacity=".1"/>
+        <stop offset="1" stop-opacity=".1"/>
+    </linearGradient>
+    <mask id="anybadge_1">
+        <rect width="158" height="20" rx="3" fill="#fff"/>
+    </mask>
+    <g mask="url(#anybadge_1)">
+        <path fill="#555" d="M0 0h99v20H0z"/>
+        <path fill="#C0C0C0" d="M99 0h59v20H99z"/>
+        <path fill="url(#b)" d="M0 0h158v20H0z"/>
+    </g>
+    <g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11">
+        <text x="50.5" y="15" fill="#010101" fill-opacity=".3">maintainability</text>
+        <text x="49.5" y="14">maintainability</text>
+    </g>
+    <g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11">
+        <text x="129.5" y="15" fill="#010101" fill-opacity=".3">unknown</text>
+        <text x="128.5" y="14">unknown</text>
+    </g>
+</svg>
+

.docs/images/pipeline.svg

@@ -9,7 +9,7 @@
     </mask>
     <g mask="url(#anybadge_1)">
         <path fill="#555" d="M0 0h57v20H0z"/>
-        <path fill="#A9A9A9" d="M57 0h59v20H57z"/>
+        <path fill="#C0C0C0" d="M57 0h59v20H57z"/>
         <path fill="url(#b)" d="M0 0h116v20H0z"/>
     </g>
     <g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11">

.docs/images/security.svg

+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" width="119" height="20">
+    <linearGradient id="b" x2="0" y2="100%">
+        <stop offset="0" stop-color="#bbb" stop-opacity=".1"/>
+        <stop offset="1" stop-opacity=".1"/>
+    </linearGradient>
+    <mask id="anybadge_1">
+        <rect width="119" height="20" rx="3" fill="#fff"/>
+    </mask>
+    <g mask="url(#anybadge_1)">
+        <path fill="#555" d="M0 0h60v20H0z"/>
+        <path fill="#C0C0C0" d="M60 0h59v20H60z"/>
+        <path fill="url(#b)" d="M0 0h119v20H0z"/>
+    </g>
+    <g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11">
+        <text x="31.0" y="15" fill="#010101" fill-opacity=".3">security</text>
+        <text x="30.0" y="14">security</text>
+    </g>
+    <g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11">
+        <text x="90.5" y="15" fill="#010101" fill-opacity=".3">unknown</text>
+        <text x="89.5" y="14">unknown</text>
+    </g>
+</svg>
+

.docs/index.md

@@ -11,6 +11,8 @@ author: Martin Weise
 [![Image Pulls](https://img.shields.io/docker/pulls/dbrepo/data-service?style=flat)](https://hub.docker.com/u/dbrepo){ tabindex=-1 }
 [![Helm Chart version](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/dbrepo)](https://artifacthub.io/packages/helm/dbrepo/dbrepo){ tabindex=-1 }
 [![GitLab License](https://img.shields.io/gitlab/license/fair-data-austria-db-repository%2Ffda-services?gitlab_url=https%3A%2F%2Fgitlab.phaidra.org%2F&style=flat&cacheSeconds=3600)](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services){ tabindex=-1 }
+![Maintainability Rating](./images/maintainability.svg)
+![Security Rating](./images/security.svg)
 
 Documentation for version: [v1.5.0](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/releases).
 

.gitignore

@@ -61,7 +61,7 @@ ready
 *.p12
 
 # Environment
-.env
+.docker/.env
 
 # X509
 root.crt

.gitlab-ci.yml

@@ -360,9 +360,10 @@ docs-registry:
     refs:
       - /^release-.*/
   before_script:
+    - "pip install pipenv"
+    - "pipenv install --dev --system --deploy"
     - "apt-get update && apt-get install -y sed"
   script:
-    - pip install -r ./requirements.txt
     - python3 .docs/docker/release.py
 
 release-images:
@@ -414,9 +415,9 @@ release-docs:
     refs:
       - /^release-.*/
   before_script:
-    - "apk add --update alpine-sdk bash sed wget openssh jq curl"
+    - "apk add --no-cache alpine-sdk bash sed wget openssh jq curl"
     - "pip install pipenv"
-    - "pip install -r ./requirements.txt"
+    - "pipenv install --dev --system --deploy"
     - "mkdir -p ./final/${DOC_VERSION}/rest"
   script:
     - "make gen-lib-doc gen-docs-doc package-config"

.gitlab/gen-badge.sh

 #!/bin/bash
 GITLAB_URL="https://gitlab.phaidra.org"
 # if we reached this script, all the tests have passed
-anybadge --label pipeline --value "passed" failed=red passed=green canceled=darkgray > "./final/${APP_VERSION}/images/pipeline.svg"
+anybadge --label pipeline --value "passed" failed=red passed=green canceled=darkgray > "./final/${DOC_VERSION}/images/pipeline.svg"
 PIPELINE_COVERAGE=$(curl -fsSL -H "PRIVATE-TOKEN: ${CI_TOKEN}" "${GITLAB_URL}/api/v4/projects/450/pipelines/latest?ref=${CI_COMMIT_BRANCH}" | jq --raw-output .coverage)
 echo "[INFO] pipeline coverage: ${PIPELINE_COVERAGE}"
 if [ "${PIPELINE_COVERAGE}" != "null" ]; then
-  anybadge --label coverage --value "${PIPELINE_COVERAGE}" coverage > "./final/${APP_VERSION}/images/coverage.svg"
+  anybadge --label coverage --value "${PIPELINE_COVERAGE}" coverage > "./final/${DOC_VERSION}/images/coverage.svg"
 else
   echo "[WARNING] Skipping badge generation, displaying default badge text: unknown"
 fi
+curl "${SONARQUBE_URL}/api/project_badges/measure?project=fair-data-austria-db-repository_fda-services_a57fa043-ab99-4cdd-a721-162d9a916d77&metric=sqale_rating&token=${CI_SONAR_TOKEN}" > "./final/${DOC_VERSION}/images/maintainability.svg"
+curl "${SONARQUBE_URL}/api/project_badges/measure?project=fair-data-austria-db-repository_fda-services_a57fa043-ab99-4cdd-a721-162d9a916d77&metric=security_rating&token=${CI_SONAR_TOKEN}" > "./final/${DOC_VERSION}/images/security.svg"
+echo "[INFO] retrieved SonarQube badges"
\ No newline at end of file

README.md

-![Java 21](https://img.shields.io/badge/Java-21-white?style=flat)
-![Python 3.11](https://img.shields.io/badge/Python-3.11-white?style=flat)
-![RabbitMQ 3.12](https://img.shields.io/badge/RabbitMQ-3.12-white?style=flat)
-![MariaDB 11.2](https://img.shields.io/badge/MariaDB-11.2-white?style=flat)
-![OpenSearch 2.10](https://img.shields.io/badge/OpenSearch-2.10-white?style=flat)
-![SeaweedFS 3.59](https://img.shields.io/badge/SeaweedFS-3.59-white?style=flat)
-![OpenLDAP 2.6](https://img.shields.io/badge/OpenLDAP-2.6-white?style=flat)
-![Spark 3.4](https://img.shields.io/badge/Spark-3.4-white?style=flat)
-![Keycloak 24.0](https://img.shields.io/badge/Keycloak-24.0-white?style=flat)
+[![CI/CD Status](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/badges/master/pipeline.svg)](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services)
+[![CI/CD Coverage](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/badges/master/coverage.svg)](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services)
+[![Latest Release](https://img.shields.io/gitlab/v/release/fair-data-austria-db-repository%2Ffda-services?gitlab_url=https%3A%2F%2Fgitlab.phaidra.org&display_name=release&style=flat)](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services)
+[![PyPI Library version](https://img.shields.io/pypi/v/dbrepo)](https://pypi.org/project/dbrepo/)
+[![Image Pulls](https://img.shields.io/docker/pulls/dbrepo/data-service?style=flat)](https://hub.docker.com/u/dbrepo)
+[![Helm Chart version](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/dbrepo)](https://artifacthub.io/packages/helm/dbrepo/dbrepo)
+[![GitLab License](https://img.shields.io/gitlab/license/fair-data-austria-db-repository%2Ffda-services?gitlab_url=https%3A%2F%2Fgitlab.phaidra.org%2F&style=flat&cacheSeconds=3600)](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services)
 
 <img src="./dbrepo-ui/public/logo.png" width="200" alt="DBRepo &mdash; Repository for Data in Databases" />
 

dbrepo-analyse-service/api/dto.py

@@ -44,7 +44,6 @@ class ColumnAnalysisDto(BaseModel):
     null_allowed: bool
     size: Optional[int] = None
     d: Optional[int] = None
-    dfid: Optional[int] = None
     enums: Optional[list] = None
     sets: Optional[list] = None
 

dbrepo-analyse-service/determine_dt.py

@@ -74,6 +74,8 @@ def determine_datatypes(filename, enum=False, enum_tol=0.0001, separator=',') ->
                 if pandas.to_numeric(df[name], errors='coerce').notnull().all():
                     logging.debug(f"mapped column {name} from float64 to decimal")
                     col.type = DataTypeDto.DECIMAL
+                    col.size = 40
+                    col.d = 20
                 else:
                     logging.debug(f"mapped column {name} from float64 to text")
                     col.type = DataTypeDto.TEXT

dbrepo-analyse-service/test/test_determine_dt.py

@@ -36,6 +36,8 @@ class DetermineDatatypesTest(unittest.TestCase):
             },
             "Wert": {
                 "type": "decimal",
+                "size": 40,
+                "d": 20,
                 "null_allowed": False,
             },
             "Status": {
@@ -81,6 +83,8 @@ class DetermineDatatypesTest(unittest.TestCase):
             },
             "Wert": {
                 "type": "decimal",
+                "size": 40,
+                "d": 20,
                 "null_allowed": False,
             },
             "Status": {
@@ -125,6 +129,8 @@ class DetermineDatatypesTest(unittest.TestCase):
             },
             "Wert": {
                 "type": "decimal",
+                "size": 40,
+                "d": 20,
                 "null_allowed": False,
             },
             "Status": {
@@ -149,6 +155,8 @@ class DetermineDatatypesTest(unittest.TestCase):
             },
             "float": {
                 "type": "decimal",
+                "size": 40,
+                "d": 20,
                 "null_allowed": False,
             },
             "string": {

dbrepo-ui/components/table/TableSchema.vue

@@ -4,6 +4,16 @@
       ref="form"
       v-model="valid"
       :disabled="disabled">
+      <v-row
+        v-if="showPrimaryKeyWarning">
+        <v-col md="8">
+          <v-alert
+            border="start"
+            color="warning">
+            {{ $t('pages.table.subpages.import.schema.primary.warn') }}
+          </v-alert>
+        </v-col>
+      </v-row>
       <v-row
         v-for="(c, idx) in columns"
         :key="`r-${idx}`"
@@ -79,7 +89,6 @@
             :min="columnType(c).size_min"
             :max="columnType(c).size_max"
             :step="columnType(c).size_step"
-            :value="columnType(c).size_required === true ? columnType(c).size_default : null"
             :hint="sizeHint(c)"
             :clearable="!columnType(c).size_required"
             persistent-hint
@@ -163,16 +172,6 @@
             @click="addColumn()" />
         </v-col>
       </v-row>
-      <v-row
-        v-if="showPrimaryKeyWarning">
-        <v-col md="8">
-          <v-alert
-            border="start"
-            color="warning">
-            {{ $t('pages.table.subpages.import.schema.primary.warn') }}
-          </v-alert>
-        </v-col>
-      </v-row>
       <v-row>
         <v-col>
           <v-btn
@@ -300,7 +299,7 @@ export default {
       this.columns.splice(idx, 1)
     },
     addColumn (name = '', type = null, null_allowed = true, primary_key = false, unique = false) {
-      this.columns.push({
+      const column = {
         name,
         type,
         null_allowed,
@@ -309,9 +308,10 @@ export default {
         sets_values: null,
         enums: [],
         enums_values: null,
-        size: 0,
         d: 0
-      })
+      }
+      column.size = this.columnType(column).size_required === true ? this.columnType(column).size_default : null
+      this.columns.push()
       this.$refs.form.validate()
     },
     formatValues (column) {

dbrepo-ui/locales/en-US.json

@@ -370,7 +370,7 @@
             "hint": "Required. First line contains only data"
           },
           "storage": {
-            "text": "Dataset from Storage Service"
+            "text": "Uploaded Dataset"
           },
           "quote": {
             "label": "Quote Encoding",

dbrepo-ui/nuxt.config.ts

 import {transformAssetUrls} from 'vite-plugin-vuetify'
+import vuetify from 'vite-plugin-vuetify'
 
 const proxy: any = {}
 
@@ -117,7 +118,12 @@ export default defineNuxtConfig({
     '@artmizu/nuxt-prometheus',
     '@nuxtjs/i18n',
     '@pinia/nuxt',
-    '@pinia-plugin-persistedstate/nuxt'
+    '@pinia-plugin-persistedstate/nuxt',
+    async (options, nuxt) => {
+      nuxt.hooks.hook('vite:extendConfig', config => config.plugins.push(
+        vuetify()
+      ))
+    },
   ],
 
   pinia: {

dbrepo-ui/package.json

@@ -45,7 +45,7 @@
     "@pinia-plugin-persistedstate/nuxt": "^1.2.0",
     "@types/qs": "^6.9.12",
     "sass": "^1.71.0",
-    "vite-plugin-vuetify": "^2.0.1"
+    "vite-plugin-vuetify": "^2.0.4"
   },
   "browserslist": [
     "defaults and fully supports es6-module",

dbrepo-ui/pages/login.vue

@@ -137,18 +137,12 @@ export default {
             })
             .catch(({code}) => {
               const toast = useToastInstance()
-              if (typeof code !== 'string') {
-                return
-              }
               toast.error(this.$t(code))
             })
         })
         .catch(({code}) => {
           this.loading = false
           const toast = useToastInstance()
-          if (typeof code !== 'string') {
-            return
-          }
           toast.error(this.$t(code))
         })
         .finally(() => {