Generate metadata

.docs/examples/manufacturing.md

@@ -2,6 +2,10 @@
 author: Martin Weise
 ---
 
+## tl;dr
+
+tbd
+
 ## Description
 
 TBD
@@ -9,3 +13,7 @@ TBD
 ## Solution
 
 TBD
+
+## DBRepo Features
+
+- [x] TBD

.docs/examples/power.md

@@ -2,6 +2,10 @@
 author: Martin Weise
 ---
 
+## tl;dr
+
+tbd
+
 ## Description
 
 TBD
@@ -9,3 +13,9 @@ TBD
 ## Solution
 
 TBD
+
+## DBRepo Features
+
+- [x] High-throughput real-time data import (MQTT)
+- [x] Private database
+- [x] Public embargoed data view

.docs/index.md

@@ -7,18 +7,18 @@ author: Martin Weise
 [![GitLab Release](https://img.shields.io/gitlab/v/release/fair-data-austria-db-repository%2Ffda-services?gitlab_url=https%3A%2F%2Fgitlab.phaidra.org&display_name=release&style=flat&cacheSeconds=3600)](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services){ tabindex=-1 }
 [![GitLab License](https://img.shields.io/gitlab/license/fair-data-austria-db-repository%2Ffda-services?gitlab_url=https%3A%2F%2Fgitlab.phaidra.org%2F&style=flat&cacheSeconds=3600)](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services){ tabindex=-1 }
 
-Documentation for version: [v1.4.3](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/releases).
+Documentation for version: [v1.4.4](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/releases).
 
-DBRepo is a repository for data in databases that are used from the beginning of a research project supporting data 
-evolution, -citation and -versioning. It implements the 
-[RDA WGDC recommendations](https://doi.org/10.1162/99608f92.be565013) on precisely identifying arbitrary subsets of 
-data.
+DBRepo is a repository for data in databases that are used from the beginning until the end of a research 
+project supporting data evolution, -citation and -versioning. It implements the query store of the 
+[RDA WGDC](https://doi.org/10.1162/99608f92.be565013) on precisely identifying arbitrary subsets of data.
 
 ## Why use DBRepo?
 
 * **aaa** bbbb
 
-Installing DBRepo is very easy or just [give it a try online](https://test.dbrepo.tuwien.ac.at).
+Installing DBRepo is very easy or
+[give it a try online](https://test.dbrepo.tuwien.ac.at){ target="_blank" }.
 
 ## Who is using DBRepo?
 
@@ -26,4 +26,4 @@ TBD
 
 ## How can I try DBRepo
 
-[:fontawesome-solid-flask:  Demonstration Instance](https://test.dbrepo.tuwien.ac.at){ .md-button .md-button--primary }
\ No newline at end of file
+[:fontawesome-solid-flask:  Demonstration Instance](https://test.dbrepo.tuwien.ac.at){ .md-button .md-button--primary target="_blank" }
\ No newline at end of file

.gitignore

@@ -9,8 +9,9 @@ ready
 schema.xsd
 final/
 build/
-swagger/
 *.tar
+tmp.yaml
+.docs/.swagger/api-*
 
 # docs
 .docs/.swagger/dist/

.gitlab-ci.yml

@@ -267,7 +267,7 @@ scan-analyse-service:
       - master
   allow_failure: true
   script:
-    - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-analyse-service-report.json dbrepo-analyse-service:latest
+    - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-analyse-service-report.json dbrepo-analyse-service:latest
     - trivy image --insecure --exit-code 0 dbrepo-analyse-service:latest
     - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-analyse-service:latest
   cache:
@@ -287,7 +287,7 @@ scan-auth-service:
       - master
   allow_failure: true
   script:
-    - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-authentication-service-report.json dbrepo-auth-service:latest
+    - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-authentication-service-report.json dbrepo-auth-service:latest
     - trivy image --insecure --exit-code 0 dbrepo-auth-service:latest
     - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-auth-service:latest
   cache:
@@ -307,7 +307,7 @@ scan-broker-service:
       - master
   allow_failure: true
   script:
-    - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-broker-service-report.json dbrepo-broker-service:latest
+    - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-broker-service-report.json dbrepo-broker-service:latest
     - trivy image --insecure --exit-code 0 dbrepo-broker-service:latest
     - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-broker-service:latest
   cache:
@@ -327,7 +327,7 @@ scan-data-db-sidecar:
       - master
   allow_failure: true
   script:
-    - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-data-db-sidecar-report.json dbrepo-data-db-sidecar:latest
+    - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-data-db-sidecar-report.json dbrepo-data-db-sidecar:latest
     - trivy image --insecure --exit-code 0 data-db-sidecar:latest
     - trivy image --insecure --exit-code 1 --severity CRITICAL data-db-sidecar:latest
   cache:
@@ -347,7 +347,7 @@ scan-gateway-service:
       - master
   allow_failure: true
   script:
-    - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-gateway-service-report.json docker.io/nginx:1.25.0-alpine-slim
+    - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-gateway-service-report.json docker.io/nginx:1.25.0-alpine-slim
     - trivy image --insecure --exit-code 0 docker.io/nginx:1.25.0-alpine-slim
     - trivy image --insecure --exit-code 1 --severity CRITICAL docker.io/nginx:1.25.0-alpine-slim
   cache:
@@ -367,7 +367,7 @@ scan-metadata-service:
       - master
   allow_failure: true
   script:
-    - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-metadata-service-report.json dbrepo-metadata-service:latest
+    - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-metadata-service-report.json dbrepo-metadata-service:latest
     - trivy image --insecure --exit-code 0 dbrepo-metadata-service:latest
     - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-metadata-service:latest
   cache:
@@ -387,7 +387,7 @@ scan-data-service:
       - master
   allow_failure: true
   script:
-    - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-data-service-report.json dbrepo-data-service:latest
+    - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-data-service-report.json dbrepo-data-service:latest
     - trivy image --insecure --exit-code 0 dbrepo-data-service:latest
     - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-data-service:latest
   cache:
@@ -407,7 +407,7 @@ scan-search-db:
       - master
   allow_failure: true
   script:
-    - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-search-db-report.json dbrepo-search-db:latest
+    - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-search-db-report.json dbrepo-search-db:latest
     - trivy image --insecure --exit-code 0 dbrepo-search-db:latest
     - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-search-db:latest
   cache:
@@ -427,7 +427,7 @@ scan-search-service-init:
       - master
   allow_failure: true
   script:
-    - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-search-service-init-report.json dbrepo-search-service-init:latest
+    - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-search-service-init-report.json dbrepo-search-service-init:latest
     - trivy image --insecure --exit-code 0 dbrepo-search-service-init:latest
     - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-search-service-init:latest
   cache:
@@ -447,7 +447,7 @@ scan-data-db:
       - master
   allow_failure: true
   script:
-    - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-data-db-report.json docker.io/bitnami/mariadb:11.2.2-debian-11-r0
+    - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-data-db-report.json docker.io/bitnami/mariadb:11.2.2-debian-11-r0
     - trivy image --insecure --exit-code 0 docker.io/bitnami/mariadb:11.2.2-debian-11-r0
     - trivy image --insecure --exit-code 1 --severity CRITICAL docker.io/bitnami/mariadb:11.2.2-debian-11-r0
   cache:
@@ -467,7 +467,7 @@ scan-metadata-db:
       - master
   allow_failure: true
   script:
-    - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-metadata-db-report.json dbrepo-metadata-db:latest
+    - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-metadata-db-report.json dbrepo-metadata-db:latest
     - trivy image --insecure --exit-code 0 dbrepo-metadata-db:latest
     - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-metadata-db:latest
   cache:
@@ -487,7 +487,7 @@ scan-ui:
       - master
   allow_failure: true
   script:
-    - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-ui-report.json dbrepo-ui:latest
+    - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-ui-report.json dbrepo-ui:latest
     - trivy image --insecure --exit-code 0 dbrepo-ui:latest
     - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-ui:latest
   cache:
@@ -507,7 +507,7 @@ scan-search-service:
       - master
   allow_failure: true
   script:
-    - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-ui-report.json dbrepo-search-service:latest
+    - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-ui-report.json dbrepo-search-service:latest
     - trivy image --insecure --exit-code 0 dbrepo-search-service:latest
     - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-search-service:latest
   cache:
@@ -527,7 +527,7 @@ scan-storage-service:
       - master
   allow_failure: true
   script:
-    - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-storage-service-report.json docker.io/chrislusf/seaweedfs:3.59
+    - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-storage-service-report.json docker.io/chrislusf/seaweedfs:3.59
     - trivy image --insecure --exit-code 0 docker.io/chrislusf/seaweedfs:3.59
     - trivy image --insecure --exit-code 1 --severity CRITICAL docker.io/chrislusf/seaweedfs:3.59
   cache:
@@ -547,7 +547,7 @@ scan-storage-service-init:
       - master
   allow_failure: true
   script:
-    - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-storage-service-init-report.json dbrepo-storage-service-init:latest
+    - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-storage-service-init-report.json dbrepo-storage-service-init:latest
     - trivy image --insecure --exit-code 0 dbrepo-storage-service-init:latest
     - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-storage-service-init:latest
   cache:
@@ -609,13 +609,23 @@ release-chart:
 release-docs:
   stage: release
   image: docker.io/python:3.11-alpine
+  variables:
+    VERSION: "${CI_COMMIT_BRANCH:8:8}"
   only:
     refs:
       - /^release-.*/
   before_script:
+    - "wget https://github.com/mikefarah/yq/releases/download/v4.2.0/yq_linux_amd64 -O /usr/bin/yq"
+    - "chmod +x /usr/bin/yq"
     - "apt-get update && apt-get install -y git make sed wget ssh"
-  script:
-    - make docs
+    - "mkdir -p ./final/${VERSION}/swagger"
+  script:
+    - "make gen-swagger-doc gen-lib-doc gen-docs-doc"
+    - "cp -r ./lib/python/docs/build/html ./final/${VERSION}/sphinx" # sphinx
+    - "cp .docs/.swagger/api.yaml ./final/${VERSION}/swagger/api.yaml" # swagger
+    - "cp .docs/.swagger/swagger-ui.html ./final/${VERSION}/swagger/index.html" # swagger
+    - "cp .docs/.swagger/custom.css ./final/${VERSION}/swagger/custom.css" # swagger
+    - "cp -r ./site ./final/${VERSION}" # mkdocs
     - eval $(ssh-agent -s)
     - echo "$CI_KEY_PRIVATE" > /root/.ssh/id_rsa && chmod 0600 /root/.ssh/id_rsa
     - echo "$CI_KEY_PUBLIC" > /root/.ssh/id_rsa.pub

bin/teardown.sh

-#!/bin/bash
-echo "=== [ Stopping dbrepo-* ] ==="
-docker container stop $(docker container ls -aq -f name=^/dbrepo-.*) || true
-echo "=== [ Removing dbrepo-* ] ==="
-docker container rm $(docker container ls -aq -f name=^/dbrepo-.*) || true
-docker volume rm $(docker volume ls -q -f name=^dbrepo-.*) || true
-docker network rm $(docker network ls -q -f name=^dbrepo-.*) || true
-echo "=== [ Stopping * ] ==="
-docker container stop $(docker container ls -aq -f name=.*-service) || true
-docker container stop ui ui-proxy metadata-db || true
-echo "=== [ Removing * ] ==="
-docker container rm $(docker container ls -aq -f name=.*-service) || true
-docker container rm ui ui-proxy metadata-db || true
-docker volume rm $(docker volume ls -q) || true
-echo "=== [ Stopping fda-* ] ==="
-docker container stop $(docker container ls -aq -f name=^/fda-.*) || true
-echo "=== [ Removing fda-* ] ==="
-docker container rm $(docker container ls -aq -f name=^/fda-.*) || true
-docker volume rm $(docker volume ls -q -f name=^fda-.*) || true
-docker network rm $(docker network ls -q -f name=^fda-.*) || true
-echo "=== [ Stopping tuw-* ] ==="
-docker container stop $(docker container ls -aq -f name=^/tuw-.*) || true
-echo "=== [ Removing tuw-* ] ==="
-docker container rm $(docker container ls -aq -f name=^/tuw-.*) || true
-docker volume rm $(docker volume ls -q -f name=^tuw-.*) || true
-docker network rm $(docker network ls -q -f name=^tuw-.*) || true
\ No newline at end of file

dbrepo-analyse-service/Pipfile

@@ -21,7 +21,7 @@ numpy = "*"
 pandas = "*"
 minio = "*"
 pydantic = "*"
-dbrepo = {path = "./lib/dbrepo-1.4.3.tar.gz"}
+dbrepo = {path = "./lib/dbrepo-1.4.4.tar.gz"}
 opensearch-py = "*"
 
 [dev-packages]

dbrepo-analyse-service/app.py

@@ -187,6 +187,7 @@ def get_health():
 
 
 @app.route("/api/analyse/datatypes", methods=["GET"], endpoint="analyse_analyse_datatypes")
+@metrics.gauge(name='dbrepo_analyse_datatypes', description='Time needed to analyse datatypes of dataset')
 @swag_from("as-yml/analyse_datatypes.yml")
 def analyse_datatypes():
     filename: str = request.args.get('filename')
@@ -212,6 +213,7 @@ def analyse_datatypes():
 
 
 @app.route("/api/analyse/keys", methods=["GET"], endpoint="analyse_analyse_keys")
+@metrics.gauge(name='dbrepo_analyse_keys', description='Time needed to analyse keys of dataset')
 @swag_from("as-yml/analyse_keys.yml")
 def analyse_keys():
     filename: str = request.args.get("filename")
@@ -234,6 +236,7 @@ def analyse_keys():
 @app.route("/api/analyse/database/<database_id>/table/<table_id>/statistics", methods=["GET"],
            endpoint="analyse_analyse_table_stat")
 @auth.login_required(role=['admin', 'export-query-data', 'export-table-data'])
+@metrics.gauge(name='dbrepo_analyse_table_stat', description='Time needed to analyse table statistics')
 @swag_from("as-yml/analyse_table_stat.yml")
 def analyse_table_stat(database_id: int = None, table_id: int = None):
     if database_id is None:

dbrepo-analyse-service/metrics.md

+| **Metric**                  | **Description**                             |
+|-----------------------------|---------------------------------------------|
+| `dbrepo_analyse_datatypes`  | Time needed to analyse datatypes of dataset |
+| `dbrepo_analyse_keys`       | Time needed to analyse keys of dataset      |
+| `dbrepo_analyse_table_stat` | Time needed to analyse table statistics     |
+| `dbrepo_analyse_table_stat` | Time needed to analyse table statistics     |
\ No newline at end of file

dbrepo-data-db/sidecar/app.py

@@ -130,6 +130,7 @@ app.config["S3_IMPORT_BUCKET"] = os.getenv('S3_IMPORT_BUCKET', 'dbrepo-upload')
 
 app.json_encoder = LazyJSONEncoder
 
+
 @token_auth.verify_token
 def verify_token(token: str):
     if token is None or token == "":
@@ -177,6 +178,7 @@ def health():
 
 
 @app.route("/sidecar/import/<string:filename>", methods=["POST"], endpoint="sidecar_import")
+@metrics.gauge(name='dbrepo_sidecar_import_dataset', description='Time needed to import dataset from S3')
 @auth.login_required(role=['admin', 'import-database-data'])
 @swag_from("ds-yml/import.yml")
 def import_csv(filename):
@@ -190,6 +192,7 @@ def import_csv(filename):
 
 
 @app.route("/sidecar/export/<string:filename>", methods=["POST"], endpoint="sidecar_export")
+@metrics.gauge(name='dbrepo_sidecar_export_dataset', description='Time needed to export dataset to S3')
 @auth.login_required(role=['admin', 'export-query-data', 'export-table-data'])
 @swag_from("ds-yml/export.yml")
 def import_csv(filename):

dbrepo-data-db/sidecar/metrics.md

+| **Metric**                      | **Description**                       |
+|---------------------------------|---------------------------------------|
+| `dbrepo_sidecar_import_dataset` | Time needed to import dataset from S3 |
+| `dbrepo_sidecar_export_dataset` | Time needed to export dataset to S3   |
\ No newline at end of file

dbrepo-data-service/metrics.md

+| **Metric**                  | **Description**                           |
+|-----------------------------|-------------------------------------------|
+| `dbrepo_message_receive`    | Received AMQP message from Broker Service |
+| `dbrepo_subset_create`      | Create subset                             |
+| `dbrepo_subset_data`        | Retrieved subset data                     |
+| `dbrepo_subset_find`        | Find subset                               |
+| `dbrepo_subset_list`        | Find subsets                              |
+| `dbrepo_subset_persist`     | Persist subset                            |
+| `dbrepo_table_data_create`  | Create table data                         |
+| `dbrepo_table_data_delete`  | Delete table data                         |
+| `dbrepo_table_data_export`  | Export table data                         |
+| `dbrepo_table_data_history` | Find table history                        |
+| `dbrepo_table_data_import`  | Import dataset                            |
+| `dbrepo_table_data_list`    | Retrieve table data                       |
+| `dbrepo_table_data_update`  | Update table data                         |
+| `dbrepo_view_data`          | Retrieve view data                        |
+| `dbrepo_view_schema_list`   | Find view schemas                         |

dbrepo-data-service/pom.xml

@@ -11,7 +11,7 @@
     <groupId>at.tuwien</groupId>
     <artifactId>dbrepo-data-service</artifactId>
     <name>dbrepo-data-service</name>
-    <version>1.4.3</version>
+    <version>1.4.4</version>
 
     <description>Service that manages the data</description>