WIP

.docs/.swagger/api-analyse.yaml

 components:
-  schemas: {}
+  schemas:
+    DataTypesDto:
+      properties:
+        columns:
+          items:
+            $ref: '#/components/schemas/SuggestedColumnDto'
+          type: array
+        line_termination:
+          example: "\r\n"
+          type: string
+        separator:
+          example: ','
+          type: string
+      type: object
+    DetermineDataTypesDto:
+      properties:
+        enum:
+          example: false
+          type: boolean
+        enum_tol:
+          example: 0.01
+          type: double
+        filename:
+          example: s3-key-from-seaweedfs
+          type: string
+        separator:
+          example: ','
+          type: string
+      type: object
+    SuggestedColumnDto:
+      properties:
+        column_name:
+          type: string
+      type: object
 externalDocs:
   description: Sourcecode Documentation
   url: https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services
@@ -27,24 +60,17 @@ paths:
           name: body
           required: true
           schema:
-            properties:
-              enum:
-                example: true
-                type: boolean
-              enum_tol:
-                example: 0.1
-              filename:
-                example: sample.csv
-                type: string
-              separator:
-                example: ','
-                type: string
+            $ref: '#/components/schemas/DetermineDataTypesDto'
             type: object
       produces:
         - application/json
       responses:
         '200':
-          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/DataTypesDto'
+          description: Determined data types successfully
         '405':
           description: Invalid input
       summary: Determine datatypes

.docs/system-other-ui.md

@@ -10,8 +10,7 @@ author: Martin Weise
 
     Image: [`dbrepo/ui:__APPVERSION__`](https://hub.docker.com/r/dbrepo/ui)
 
-    * Ports: 3000/tcp, 9100/tcp
-    * Prometheus: `http://<hostname>:9100/metrics`
+    * Ports: 3000/tcp
     * UI: `http://<hostname>/`
 
 ## Overview
@@ -28,11 +27,10 @@ users, databases and how to import your data.
 
 ### Settings
 
-The User Interface can be configured extensively with 
-the [`dbrepo.config.json`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/master/dbrepo-ui/dbrepo.config.json)
-configuration file, mounted directly into the container with e.g. docker compose. As a small example, you can configure
-the logo :material-numeric-1-circle-outline: in Figure 2. Make sure you mount the logo as image as well, in this example
-we want to mount a custom logo `my_logo.png` into the container and specify the name.
+The User Interface is configured in the `runtimeConfig` section of the `nuxt.config.ts` file during build time. For the
+runtime, you need to override those values through environment variables or by mounting a `.env` file. As a small
+example, you can configure the logo :material-numeric-1-circle-outline: in Figure 2. Make sure you mount the logo as
+image as well, in this example we want to mount a custom logo `my_logo.png` into the container and specify the name.
 
 <figure markdown>
 ![Architecture of the UI microservice](images/screenshots/ui-config-step-1.png){ .img-border }
@@ -40,22 +38,12 @@ we want to mount a custom logo `my_logo.png` into the container and specify the
 </figure>
 
 Text values like the version :material-numeric-2-circle-outline: and title :material-numeric-3-circle-outline: can be
-configured as well via the `dbrepo.config.json` values file. The important links section 
-:material-numeric-4-circle-outline: can be modified or removed entirely by setting `page.information.links` to `[]`.
-
-```json title="dbrepo.config.json"
-{
-    "title": "Database Repository",
-    "version": "__APPVERSION___DOCKER-COMPOSE",
-    "logo": {
-      "path": "/my_logo.png"
-    },
-    "page": {
-    "information": {
-      "links": []
-    },
+configured as well via the Nuxt runtime configuration through single environment variables or `.env` files
+
+```yaml title=".env"
+NUXT_PUBLIC_TITLE="My overriden title"
+NUXT_PUBLIC_LOGO="/app/.output/public/my_logo.png"
 ...
-}
 ```
 
 To work, you need to mount the `my_logo.png` file into the `dbrepo-ui` container via the `docker-compose.yml` file (or
@@ -66,8 +54,7 @@ services:
   dbrepo-ui:
     image: docker.io/dbrepo/ui:__APPVERSION__
     volumes:
-      - ./my_logo.png:/app/static/my_logo.png
-      - ./dbrepo.conf.json:/app/dbrepo.conf.json
+      - ./my_logo.png:/app/.output/public/my_logo.png
   ...
 ```
 
@@ -81,6 +68,13 @@ User Interface on development.
 <figcaption>Figure 3: Architecture of the User Interface</figcaption>
 </figure>
 
+* Runtime: [Bun 1+](https://bun.sh/) (preferred), *alternatively* Node.js 18+
+* Builder: [Vite](https://vitejs.dev/)
+* Server: [Nuxt.js 3+](https://nuxt.com/)
+* Components: [Vue.js 3+](https://vuejs.org/)
+* Frontend: [Vuetify 3+](https://vuetifyjs.com/en/)
+* State: [Pinia](https://pinia.vuejs.org/)
+
 ### Example
 
 See the [Usage Overview](../usage-overview/) page for detailed examples.

.gitlab-ci.yml

@@ -57,16 +57,14 @@ build-data-service:
     - "mvn -f ./dbrepo-metadata-service/pom.xml clean install -Dstyle.color=always -DskipTests"
     - "mvn -f ./dbrepo-data-service/pom.xml clean package -Dstyle.color=always -DskipTests"
 
-build-frontend:
-  image: node:14-alpine
+build-ui:
+  image: oven/bun:1.0.26-alpine
   stage: build
   except:
     refs:
       - /^release-.*/
   script:
-    - "yarn config set network-timeout 600000 -g"
-    - "yarn --cwd ./dbrepo-ui install --legacy-peer-deps"
-    - "yarn --cwd ./dbrepo-ui run build"
+    - "cd ./dbrepo-ui && bun install && bun run build"
 
 build-search-service:
   image: python:3.10-alpine
@@ -176,30 +174,6 @@ test-analyse-service:
       junit: ./dbrepo-analyse-service/report.xml
   coverage: '/TOTAL.*?([0-9]{1,3})%/'
 
-test-frontend:
-  image: node:14-alpine
-  stage: test
-  except:
-    refs:
-      - /^release-.*/
-  needs:
-    - build-frontend
-  script:
-    - "yarn --cwd ./dbrepo-ui install"
-    - "yarn --cwd ./dbrepo-ui run test:unit || true"
-    - "yarn --cwd ./dbrepo-ui run coverage || true"
-    - "cat ./dbrepo-ui/coverage/cobertura-coverage.xml | grep -o 'line-rate=\"[0-9.]*' | head -1 || true"
-  artifacts:
-    when: always
-    paths:
-      - ./dbrepo-ui/coverage/
-    expire_in: 1 days
-    reports:
-      coverage_report:
-        coverage_format: cobertura
-        path: ./dbrepo-ui/coverage/cobertura-coverage.xml
-  coverage: '/TOTAL.*?([0-9]{1,3})%/'
-
 scan-analyse-service:
   image: bitnami/trivy:latest
   stage: scan
@@ -497,7 +471,6 @@ release-images:
     - test-metadata-service
     - test-data-service
     - test-analyse-service
-    - test-frontend
   only:
     refs:
       - /^release-.*/

dbrepo-analyse-service/Pipfile

@@ -6,8 +6,9 @@ name = "pypi"
 [packages]
 boto3 = "*"
 exceptiongroup = "*"
-flask = "*"
 flasgger = "*"
+flask = "~=2.0"
+flask-cors = "~=4.0"
 gevent = "*"
 gunicorn = "*"
 greenlet = "*"

dbrepo-analyse-service/app.py

@@ -6,12 +6,12 @@ from logging.config import dictConfig
 
 from flasgger import LazyJSONEncoder, Swagger
 from flasgger.utils import swag_from
-from flask import Flask, Response, abort, request
+from flask import Flask, Response, request
+from flask_cors import CORS
 from flask_sqlalchemy import SQLAlchemy
 from gevent.pywsgi import WSGIServer
 from opensearchpy import OpenSearch
 from prometheus_flask_exporter import PrometheusMetrics
-from requests import get
 
 from determine_dt import determine_datatypes
 from determine_pk import determine_pk
@@ -40,6 +40,8 @@ dictConfig(
 
 app = Flask(__name__)
 
+cors = CORS(app, resources={r"/api/*": {"origins": "*"}})
+
 metrics = PrometheusMetrics(app)
 metrics.info("app_info", "Application info", version="1.3.0")
 app.config["SWAGGER"] = {"openapi": "3.0.1", "title": "Swagger UI", "uiversion": 3}

dbrepo-analyse-service/as-yml/determinedt.yml

@@ -13,20 +13,48 @@ parameters:
     required: true
     schema:
       type: "object"
+      $ref: '#/components/schemas/DetermineDataTypesDto'
+responses:
+  200:
+    description: Determined data types successfully
+    content:
+      application/json:
+        schema:
+          $ref: '#/components/schemas/DataTypesDto'
+  405:
+    description: "Invalid input"
+components:
+  schemas:
+    DetermineDataTypesDto:
+      type: object
       properties:
-        filename:
-          type: "string"
-          example: "sample.csv"
         enum:
-          type: "boolean"
-          example: true
+          type: boolean
+          example: false
         enum_tol:
-          example: 0.1
+          type: double
+          example: 0.01
+        filename:
+          type: string
+          example: s3-key-from-seaweedfs
         separator:
-          type: "string"
+          type: string
           example: ","
-responses:
-  200:
-    description: "OK"
-  405:
-    description: "Invalid input"
\ No newline at end of file
+    DataTypesDto:
+      type: object
+      properties:
+        columns:
+          type: array
+          items:
+            $ref: '#/components/schemas/SuggestedColumnDto'
+        line_termination:
+          type: string
+          example: "\r\n"
+        separator:
+          type: string
+          example: ","
+    SuggestedColumnDto:
+      type: object
+      properties:
+        column_name:
+          type: string
\ No newline at end of file

dbrepo-analyse-service/determine_dt.py

@@ -39,8 +39,15 @@ def determine_datatypes(filename, enum=False, enum_tol=0.0001, separator=None) -
 
     # Load a file object:
     with io.BytesIO(stream.read()) as fh:
+        line_terminator = None
+        if b"\n" in fh.readline():
+            line_terminator = "\n"
+        elif b"\r" in fh.readline():
+            line_terminator = "\r"
+        elif b"\r\n" in fh.readline():
+            line_terminator = "\r\n"
         logging.info("Analysing corpus with separator: %s", separator)
-        table_set = CSVTableSet(fh, delimiter=separator)
+        table_set = CSVTableSet(fh, delimiter=separator, lineterminator=line_terminator)
 
         # A table set is a collection of tables:
         row_set = table_set.tables[0]
@@ -83,7 +90,6 @@ def determine_datatypes(filename, enum=False, enum_tol=0.0001, separator=None) -
                 r[headers[i]] = "varchar"
             else:
                 r[headers[i]] = "text"
-        fh.close()
-        s = {"columns": r, "separator": separator}
+        s = {"columns": r, "separator": separator, "line_termination": line_terminator}
         logging.info("Determined data types %s", s)
     return json.dumps(s)

dbrepo-analyse-service/test/test_determine_dt.py

@@ -28,6 +28,7 @@ class DetermineDatatypesTest(unittest.TestCase):
                 "Status": "varchar",
             },
             "separator": ",",
+            "line_termination": "\n"
         }
 
         # mock
@@ -50,6 +51,7 @@ class DetermineDatatypesTest(unittest.TestCase):
                 "Status": "varchar",
             },
             "separator": ",",
+            "line_termination": "\n"
         }
 
         # mock
@@ -72,6 +74,7 @@ class DetermineDatatypesTest(unittest.TestCase):
                 "Status": "varchar",
             },
             "separator": ",",
+            "line_termination": "\n"
         }
 
         # mock
@@ -94,6 +97,7 @@ class DetermineDatatypesTest(unittest.TestCase):
                 "enum": "varchar",  # currently not used
             },
             "separator": ",",
+            "line_termination": "\n"
         }
 
         # mock

dbrepo-metadata-db/Dockerfile

@@ -3,4 +3,4 @@ FROM bitnami/mariadb:11.2.2-debian-11-r0 as runtime
 ENV MARIADB_DATABASE=fda
 ENV MARIADB_ROOT_PASSWORD=dbrepo
 
-COPY ./setup-schema.sql /docker-entrypoint-initdb.d/setup-schema.sql
\ No newline at end of file
+COPY 1_setup-schema.sql /docker-entrypoint-initdb.d/1_setup-schema.sql
\ No newline at end of file

dbrepo-metadata-db/migrate_1.4.0-1.4.1.sql

+ALTER TABLE mdb_databases DROP SYSTEM VERSIONING;
+ALTER TABLE mdb_databases ADD COLUMN image longblob;
+ALTER TABLE mdb_databases ADD SYSTEM VERSIONING;
+ALTER TABLE mdb_tables DROP SYSTEM VERSIONING;
+ALTER TABLE mdb_tables ADD COLUMN processed_constraints BOOLEAN NOT NULL DEFAULT false;
+ALTER TABLE mdb_tables ADD SYSTEM VERSIONING;
+ALTER TABLE mdb_columns DROP SYSTEM VERSIONING;
+ALTER TABLE mdb_columns DROP COLUMN alias;
+ALTER TABLE mdb_columns ADD SYSTEM VERSIONING;
+ALTER TABLE mdb_constraints_foreign_key DROP SYSTEM VERSIONING;
+ALTER TABLE mdb_constraints_foreign_key ADD COLUMN name VARCHAR(255) NOT NULL;
+ALTER TABLE mdb_constraints_foreign_key ADD SYSTEM VERSIONING;
+ALTER TABLE mdb_constraints_unique DROP SYSTEM VERSIONING;
+ALTER TABLE mdb_constraints_unique ADD COLUMN name VARCHAR(255) NOT NULL;
+ALTER TABLE mdb_constraints_unique ADD SYSTEM VERSIONING;
+ALTER TABLE mdb_view_columns DROP SYSTEM VERSIONING;
+ALTER TABLE mdb_view_columns ADD COLUMN alias VARCHAR(100);
+ALTER TABLE mdb_view_columns CHANGE COLUMN position ordinal_position INTEGER;
+ALTER TABLE mdb_view_columns ADD SYSTEM VERSIONING;
\ No newline at end of file

dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/DatabaseTransferDto.java

@@ -5,6 +5,8 @@ import lombok.*;
 import jakarta.validation.constraints.NotNull;
 import lombok.extern.jackson.Jacksonized;
 
+import java.util.UUID;
+
 @Getter
 @Setter
 @Builder
@@ -15,6 +17,6 @@ import lombok.extern.jackson.Jacksonized;
 public class DatabaseTransferDto {
 
     @NotNull
-    private String username;
+    private UUID id;
 
 }

dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/LicenseDto.java

@@ -28,4 +28,7 @@ public class LicenseDto {
     @Field(name = "uri", type = FieldType.Keyword)
     private String uri;
 
+    @Schema(example = "A short and simple permissive license with conditions only requiring preservation of copyright and license notices. Licensed works, modifications, and larger works may be distributed under different terms and without source code.")
+    private String description;
+
 }
\ No newline at end of file