From 6b0bbac756895a3fda6f32c1165fa81015218247 Mon Sep 17 00:00:00 2001
From: Martin Weise <martin.weise@tuwien.ac.at>
Date: Thu, 13 Jun 2024 08:52:12 +0000
Subject: [PATCH] Dev
---
.docs/.swagger/swagger-site.sh | 21 ----------
.docs/api/data-db.md | 5 +--
.docs/api/index.md | 66 ++++++++++++++++----------------
.docs/api/metadata-db.md | 30 ++++++++++++++-
.docs/api/open-api.md | 2 +-
.docs/api/python.md | 2 +-
.docs/api/ui.md | 2 +-
.docs/concepts/authentication.md | 26 +++++++++++--
.docs/concepts/databases.md | 13 +++++--
.docs/concepts/messaging.md | 21 +++++++---
.docs/index.md | 2 +-
.docs/publications.md | 4 +-
.gitlab-ci.yml | 8 ++--
make/gen.mk | 7 +---
mkdocs.yml | 2 +-
15 files changed, 123 insertions(+), 88 deletions(-)
delete mode 100644 .docs/.swagger/swagger-site.sh
diff --git a/.docs/.swagger/swagger-site.sh b/.docs/.swagger/swagger-site.sh
deleted file mode 100644
index d8d91a73cb..0000000000
--- a/.docs/.swagger/swagger-site.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-declare -A services
-#services[1080]=upload
-services[4000]=search
-services[5000]=analyse
-services[9093]=data
-services[9099]=metadata
-services[3305]=sidecar
-
-rm -f ./tmp.yaml
-mkdir -p ./site/swagger
-touch ./tmp.yaml
-
-# -> build paths: map
-for key in "${!services[@]}"; do
- cat .docs/.swagger/api-${services[$key]}.yaml | yq .paths >> ./tmp.yaml
-done
-
-# -> merge with api.base.yaml into final api.yaml
-yq ".paths *= load(\"tmp.yaml\")" .docs/.swagger/api.base.yaml > .docs/.swagger/api.yaml
-
diff --git a/.docs/api/data-db.md b/.docs/api/data-db.md
index 859264a53a..c91d230be7 100644
--- a/.docs/api/data-db.md
+++ b/.docs/api/data-db.md
@@ -11,10 +11,9 @@ author: Martin Weise
!!! debug "Debug Information"
- Image: [`dbrepo/data-db-sidecar:__APPVERSION__`](https://hub.docker.com/r/dbrepo/data-db-sidecar)
+ Image: [`dbrepo/data-db-sidecar:1.4.4`](https://hub.docker.com/r/dbrepo/data-db-sidecar)
- * Ports: 3305/tcp
- * Swagger UI: `http://<hostname>:1080/swagger-ui/` <a href="../swagger/sidecar" target="_blank">:fontawesome-solid-square-up-right: view online</a>
+ * Ports: 8080/tcp
## Overview
diff --git a/.docs/api/index.md b/.docs/api/index.md
index 1468c2b20f..47d5ba14d1 100644
--- a/.docs/api/index.md
+++ b/.docs/api/index.md
@@ -5,7 +5,7 @@ author: Martin Weise
# Overview
We developed a Python Library for communicating with DBRepo from e.g. Jupyter Notebooks. See
-the [Python Library](./usage-python) page for more details.
+the [Python Library](./open-api) page for more details.
We give usage examples of the most important use-cases we identified.
@@ -34,7 +34,7 @@ A user wants to create an account in DBRepo.
that your administrator has assigned.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 1: Create user account.</figcaption>
</figure>
@@ -44,7 +44,7 @@ A user wants to create an account in DBRepo.
authenticated with DBRepo in Figure 2.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 2: Login to the user account.</figcaption>
</figure>
@@ -54,7 +54,7 @@ A user wants to create an account in DBRepo.
identically in field :material-numeric-4-circle-outline:.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 3: Change user account password.</figcaption>
</figure>
@@ -134,7 +134,7 @@ A user wants to create a database in DBRepo.
Login and press the ":material-plus: DATABASE" button on the top right :material-numeric-1-circle-outline: as seen in Figure 4.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 4: Open the create database dialog.</figcaption>
</figure>
@@ -143,14 +143,14 @@ A user wants to create a database in DBRepo.
finally create the database, press "Create" :material-numeric-3-circle-outline: as seen in Figure 5.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 5: Create database form.</figcaption>
</figure>
After a few seconds, you can see the created database in the "Recent Databases" list, as seen in Figure 6.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 6: View the created database.</figcaption>
</figure>
@@ -213,7 +213,7 @@ access to. This is the default for self-created databases like above in [Create
Figure 7.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 7: Open the import CSV form.</figcaption>
</figure>
@@ -221,7 +221,7 @@ access to. This is the default for self-created databases like above in [Create
as seen in Figure 8.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 8: Basic table information.</figcaption>
</figure>
@@ -238,7 +238,7 @@ access to. This is the default for self-created databases like above in [Create
provide this information in :material-numeric-6-circle-outline:.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 9: Dataset metadata necessary for import.</figcaption>
</figure>
@@ -246,7 +246,7 @@ access to. This is the default for self-created databases like above in [Create
dataset file onto the field in Figure 10.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 10: Dataset import file.</figcaption>
</figure>
@@ -261,7 +261,7 @@ access to. This is the default for self-created databases like above in [Create
definitions by clicking the "ADD COLUMN" button in Figure 11.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 11: Confirm the table schema and provide missing information.</figcaption>
</figure>
@@ -270,7 +270,7 @@ access to. This is the default for self-created databases like above in [Create
dataset as seen in Figure 12.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 12: Confirm the table schema and provide missing information.</figcaption>
</figure>
@@ -284,7 +284,7 @@ access to. This is the default for self-created databases like above in [Create
13.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 13: Table data.</figcaption>
</figure>
@@ -432,7 +432,7 @@ A user wants to import a database dump in `.sql` (or in `.sql.gz`) format into D
clicking the "OK" button :material-numeric-4-circle-outline:.
<figure markdown>
- 
+ 
<figcaption>Figure 14: Setup New Connection in MySQL Workbench.</figcaption>
</figure>
@@ -440,7 +440,7 @@ A user wants to import a database dump in `.sql` (or in `.sql.gz`) format into D
:material-numeric-1-circle-outline: and basic connection and version information :material-numeric-2-circle-outline:.
<figure markdown>
- 
+ 
<figcaption>Figure 15: Server status of the Data Database in MySQL Workbench.</figcaption>
</figure>
@@ -451,7 +451,7 @@ A user wants to import a database dump in `.sql` (or in `.sql.gz`) format into D
The import starts after clicking "Start Import" :material-numeric-5-circle-outline:.
<figure markdown>
- 
+ 
<figcaption>Figure 16: Data Import/Restore in MySQL Workbench.</figcaption>
</figure>
@@ -510,7 +510,7 @@ A user wants to import live data from e.g. sensor measurements fast and without
database via JDBC, you can obtain the connection string in the UI under the database info (c.f. Figure 14).
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 14: JDBC connection information.</figcaption>
</figure>
@@ -526,7 +526,7 @@ A user wants to import live data from e.g. sensor measurements fast and without
database via AMQP, you can obtain the connection string in the UI under the table info (c.f. Figure 14).
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 14: AMQP connection information.</figcaption>
</figure>
@@ -550,7 +550,7 @@ A user wants to create a subset and export it as csv file.
Figure 17.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 17: Open the create subset form.</figcaption>
</figure>
@@ -562,7 +562,7 @@ A user wants to create a subset and export it as csv file.
Once you are confident the query covers the desired result, click ":material-run: Create".
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 18: Subset query building.</figcaption>
</figure>
@@ -571,7 +571,7 @@ A user wants to create a subset and export it as csv file.
View" on the top (c.f. Figure 19).
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 19: Subset result set.</figcaption>
</figure>
@@ -584,7 +584,7 @@ A user wants to create a subset and export it as csv file.
a csv file by clicking the ":material-download: DATA .CSV" button :material-numeric-2-circle-outline:.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 20: Subset information.</figcaption>
</figure>
@@ -648,7 +648,7 @@ A user wants to create a subset and export it as csv file.
under the database info (c.f. Figure 20).
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 20: JDBC connection information.</figcaption>
</figure>
@@ -728,7 +728,7 @@ A user wants to assign a persistent identifier to a database owned by them.
the ":material-identifier: GET PID" button :material-numeric-1-circle-outline: as seen in Figure 21.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 21: Open the get persisent identifier form.</figcaption>
</figure>
@@ -743,7 +743,7 @@ A user wants to assign a persistent identifier to a database owned by them.
:material-numeric-8-circle-outline:.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 22: Identifier creator fields.</figcaption>
</figure>
@@ -753,7 +753,7 @@ A user wants to assign a persistent identifier to a database owned by them.
again :material-numeric-4-circle-outline: if they are not needed in Figure 23.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 23: Identifier title fields.</figcaption>
</figure>
@@ -764,7 +764,7 @@ A user wants to assign a persistent identifier to a database owned by them.
publication month and publication day in Figure 24.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 24: Identifier description fields and publishing information.</figcaption>
</figure>
@@ -775,7 +775,7 @@ A user wants to assign a persistent identifier to a database owned by them.
your data.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 25: Related identifiers, license and language of the identifier.</figcaption>
</figure>
@@ -787,7 +787,7 @@ A user wants to assign a persistent identifier to a database owned by them.
language on the identifier summary page (c.f. Figure 26).
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 26: Identifier funder information.</figcaption>
</figure>
@@ -795,7 +795,7 @@ A user wants to assign a persistent identifier to a database owned by them.
result is displayed in Figure 27.
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 27: Identifier summary page.</figcaption>
</figure>
@@ -884,7 +884,7 @@ A user wants a public database to be private and only give specific users access
the dialog (c.f. Figure 28).
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 28: Database settings for visibility and access.</figcaption>
</figure>
@@ -893,7 +893,7 @@ A user wants a public database to be private and only give specific users access
allowing users to view the private data (c.f. Figure 29).
<figure markdown>
- { .img-border }
+ { .img-border }
<figcaption>Figure 29: Database acccess dialog.</figcaption>
</figure>
diff --git a/.docs/api/metadata-db.md b/.docs/api/metadata-db.md
index e56b88ab8b..38cbe3f127 100644
--- a/.docs/api/metadata-db.md
+++ b/.docs/api/metadata-db.md
@@ -2,4 +2,32 @@
author: Martin Weise
---
-TBD
\ No newline at end of file
+!!! debug "Debug Information"
+
+ Image: [`bitnami/mariadb-galera:11.2.2-debian-11-r0`](https://hub.docker.com/r/bitnami/mariadb-galera)
+
+ * Ports: 3306/tcp
+ * JDBC: `jdbc://mariadb:<hostname>:3306`
+
+## Overview
+
+The metadata database is the single, central source of truth within DBRepo and holds all metadata information for
+interaction between the services and displaying information in the UI.
+
+On the first start, the schema is generated by the file `/docker-entrypoint-initdb.d/setup-schema.sql` within the
+container. You can add custom files that should be executed on the first start by placing them into the
+`/docker-entrypoint-initdb.d/` folder as well. For example:
+
+```yaml
+services:
+ dbrepo-metadata-db:
+ ...
+ volumes:
+ - /path/to/setup-some-data.sql:/docker-entrypoint-initdb.d/setup-some-data.sql
+ ...
+```
+
+!!! warning "Alphabetic Filename Sorting"
+
+ Beware that the init script provided by Bitnami executes files in alphabetic order! For example: the file
+ `setup-schema.sql` is executed **after** the file `setup-data.sql`!
\ No newline at end of file
diff --git a/.docs/api/open-api.md b/.docs/api/open-api.md
index 72395271de..575ce9483d 100644
--- a/.docs/api/open-api.md
+++ b/.docs/api/open-api.md
@@ -6,7 +6,7 @@ author: Martin Weise
## tl;dr
-[:simple-swagger: View Swagger-UI](../../swagger/){ .md-button .md-button--primary tabindex=-1 }
+[:simple-swagger: View Swagger-UI](../../rest/){ .md-button .md-button--primary tabindex=-1 }
## Overview
diff --git a/.docs/api/python.md b/.docs/api/python.md
index 7289639bfa..a48fb53d8d 100644
--- a/.docs/api/python.md
+++ b/.docs/api/python.md
@@ -6,7 +6,7 @@ author: Martin Weise
## tl;dr
-[:fontawesome-solid-cube: View Docs](../sphinx){ .md-button .md-button--primary }
+[:fontawesome-solid-cube: View Docs](../../python){ .md-button .md-button--primary }
## Installing
diff --git a/.docs/api/ui.md b/.docs/api/ui.md
index 659b21ea90..2acc439097 100644
--- a/.docs/api/ui.md
+++ b/.docs/api/ui.md
@@ -52,7 +52,7 @@ User Interface on development.
### Example
-See the [Usage Overview](../usage-overview/) page for detailed examples.
+See the [API Overview](..) page for detailed examples.
## Limitations
diff --git a/.docs/concepts/authentication.md b/.docs/concepts/authentication.md
index 5919b25723..c49abdf86e 100644
--- a/.docs/concepts/authentication.md
+++ b/.docs/concepts/authentication.md
@@ -2,14 +2,32 @@
author: Martin Weise
---
+This is a short conceptional overview on the authentication mechanisms provided by Keycloak. We use Keycloak as the
+single source of truth for authorization and user management.
+
+## Basic Authentication
+
+DBRepo supports `Basic` authentication (that is with username and password) in the REST API. When requesting a resource
+from e.g. the [Metadata Service](../../api/metadata-service), the service internally retrieves a
+[Bearer Token](#bearer-authentication) from the [Auth Service](../../api/auth-service) and checks if a certain role
+is present to perform the desired action. This entails a small overhead on each request, since the service does not
+store anything.
+
## Bearer Authentication
-TBD
+DBRepo supports `Bearer` authentication by accepting JWT tokens in the
+[`Authorization`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Authorization) header of HTTP requests to
+any service. There are two (major) types of tokens:
-## Basic Authentication
+* Access tokens who are short lived (e.g. 15 minutes) to access resources, and
+* Refresh token who are long lived (e.g. 30 days) to request new access tokens without having to provide username and
+ password again.
-TBD
+The [User Interface](../../api/ui) for example refreshes the token on-the-fly by intercepting each request and, in case
+of an expired access token, requests a new one without having to terminate the request. This happens only once after the
+access token has expired (after e.g. 15 minutes).
## OpenID Connect (OIDC)
-TBD
\ No newline at end of file
+We use the widely accepted authentication protocol OIDC for client authentication. Other protocols are e.g. SAML2 which
+are not used by DBRepo.
\ No newline at end of file
diff --git a/.docs/concepts/databases.md b/.docs/concepts/databases.md
index 1076f775ef..ecf1025b1d 100644
--- a/.docs/concepts/databases.md
+++ b/.docs/concepts/databases.md
@@ -4,15 +4,22 @@ author: Martin Weise
## Relational Database
-TBD
+DBRepo manages relational databases that store information relations in tables.
## Query
-TBD
+A query is the method to interact with a relational database and is used to read/write data or to create/change/delete
+schema information e.g. tables. DBRepo uses a query store to store certain (important) queries that generate subsets
+to restore the exact same subset at a later point.
## System Versioning
-TBD
+DBRepo uses a mechanism offered by SQL:2013 to version tables with the system (=server) time. When inserting a tuple
+into a system-versioned table, the database engine maintains invisible `ROW_START` and `ROW_END` timestamp columns to
+denote a tuple validity. When deleting a tuple, the database engine actually just marks the tuple as `ROW_END = NOW()`
+and does not delete the tuple.
+
+At a later point in time, the (historic) tuple can still be queried using system versioning.
## Data Ingest
diff --git a/.docs/concepts/messaging.md b/.docs/concepts/messaging.md
index 448c7a3f63..c19e1fc2a5 100644
--- a/.docs/concepts/messaging.md
+++ b/.docs/concepts/messaging.md
@@ -2,14 +2,23 @@
author: Martin Weise
---
-## Tuple
+We use the message broker RabbitMQ and a single exchange with distributed (`quorum`) queue. Both are named `dbrepo`.
-TBD
+<figure markdown>
+<img src="../../images/exchange-binding.png" />
+<caption>Figure 1: Exchange binding in DBRepo.</caption>
+</figure>
-## AMQP
+## Tuple
-TBD
+A tuple is the atomic granularity of information when transmitting data through message-queue systems. Represented as
+JSON, a tuple looks like this in RabbitMQ:
-## MQTT
+```JSON
+{"firstname": "foo", "lastname": "bar"}
+```
+
+## AMQP
-TBD
\ No newline at end of file
+DBRepo uses AMQP to route messages which allows for both Basic/Bearer authentication. For more information please
+consult the [RabbitMQ AMQP](https://www.rabbitmq.com/tutorials/amqp-concepts) documentation.
\ No newline at end of file
diff --git a/.docs/index.md b/.docs/index.md
index aa65b7d939..a6cfdea09b 100644
--- a/.docs/index.md
+++ b/.docs/index.md
@@ -29,7 +29,7 @@ Installing DBRepo is very easy or
## Who is using DBRepo?
-
+
## How can I try DBRepo
diff --git a/.docs/publications.md b/.docs/publications.md
index 4b64d9a574..b62114dc0f 100644
--- a/.docs/publications.md
+++ b/.docs/publications.md
@@ -20,8 +20,8 @@ hide:
DBRepo logo in various formats:
-* PNG: [bigger](./images/logo/logo.png) ([smaller](./images/logo/favicon.png))
-* SVG: [bigger](./images/logo/logo.svg) ([smaller](./images/logo/favicon.svg))
+* PNG: [bigger](../images/logos/logo.png) ([smaller](../images/logos/favicon.png))
+* SVG: [bigger](../images/logos/logo.svg) ([smaller](../images/logos/favicon.svg))
## Refereed
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 813ca707d1..d2552eb054 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -619,10 +619,10 @@ release-docs:
- "mkdir -p ./final/${APP_VERSION}/swagger"
script:
- "make gen-lib-doc gen-docs-doc"
- - "cp -r ./lib/python/docs/build/html ./final/${APP_VERSION}/sphinx" # sphinx
- - "cp .docs/.swagger/api.yaml ./final/${APP_VERSION}/swagger/api.yaml" # swagger
- - "cp .docs/.swagger/swagger-ui.html ./final/${APP_VERSION}/swagger/index.html" # swagger
- - "cp .docs/.swagger/custom.css ./final/${APP_VERSION}/swagger/custom.css" # swagger
+ - "cp -r ./lib/python/docs/build/html ./final/${APP_VERSION}/python" # sphinx
+ - "cp .docs/.swagger/api.yaml ./final/${APP_VERSION}/rest/api.yaml" # swagger
+ - "cp .docs/.swagger/swagger-ui.html ./final/${APP_VERSION}/rest/index.html" # swagger
+ - "cp .docs/.swagger/custom.css ./final/${APP_VERSION}/rest/custom.css" # swagger
- "cp -r ./site/* ./final/${APP_VERSION}" # mkdocs
- eval $(ssh-agent -s)
- "mkdir -p /root/.ssh"
diff --git a/make/gen.mk b/make/gen.mk
index 14206d6633..dac662c136 100644
--- a/make/gen.mk
+++ b/make/gen.mk
@@ -1,14 +1,9 @@
##@ Generate
-.PHONY: gen-swagger-doc
-gen-swagger-doc: ## Generate Swagger documentation.
- bash .docs/.swagger/swagger-site.sh
-
.PHONY: gen-swagger-doc-fe
-gen-swagger-doc-fe: build-images ## Generate Swagger documentation and fetch.
+gen-swagger-doc: build-images ## Generate Swagger documentation and fetch.
docker compose up -d
bash .docs/.swagger/swagger-generate.sh
- bash .docs/.swagger/swagger-site.sh
docker compose down
openapi-merge-cli --config .docs/.swagger/openapi-merge.json
diff --git a/mkdocs.yml b/mkdocs.yml
index 86b83c4eeb..ada8171b50 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -1,5 +1,5 @@
site_name: Database Repository
-site_url: https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/
+site_url: https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.4/
repo_url: https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services
repo_name: fda-services
site_author: Research Unit Data Science, Technische Universität Wien
--
GitLab