Skip to content
Snippets Groups Projects
Commit 749de1a5 authored by Martin Weise's avatar Martin Weise
Browse files

Merge branch 'version/keycloak' into 'dev' 

Version/keycloak

See merge request !329
parents a58b2ffc 67ea672e
No related branches found
No related tags found
7 merge requests!345Updated docs and endpoints:,!341Fixed mapping problem where UK and FK share columns they are inserted,!339Fixed mapping problem where UK and FK share columns they are inserted,!338Fixed mapping problem where UK and FK share columns they are inserted,!334Fixed mapping problem where UK and FK share columns they are inserted,!333Fixed mapping problem where UK and FK share columns they are inserted,!329Version/keycloak
Expand all Show whitespace changes
Inline Side-by-side
Showing
with 193 additions and 1125 deletions
.docker/.env deleted 100644 → 0
+ 0
− 12
View file @ a58b2ffc
# UNCOMMENT THE LINES BELOW TO OVERRIDE
#BASE_URL=http://example.com
#ADMIN_EMAIL=noreply@example.com
#LOG_LEVEL=debug
#IDENTITY_SERVICE_ADMIN_PASSWORD=admin
#AUTH_SERVICE_ADMIN_PASSWORD=admin
#METADATA_DB_PASSWORD=dbrepo
#DATA_DB_PASSWORD=dbrepo
#AUTH_DB_PASSWORD=dbrepo
#S3_ACCESS_KEY_ID=seaweedfsadmin
#S3_SECRET_ACCESS_KEY=seaweedfsadmin
#SYSTEM_PASSWORD=admin
.docker/config/1_setup-schema.sql deleted 100644 → 0
+ 0
− 564
View file @ a58b2ffc
This diff is collapsed.
.docker/config/2_setup-data.sql deleted 100644 → 0
+ 0
− 8
View file @ a58b2ffc
BEGIN;
INSERT INTO `mdb_containers` (name, internal_name, image_id, host, port, ui_host, ui_port, sidecar_host, sidecar_port,
privileged_username, privileged_password)
VALUES ('mariadb:11.1.3-debian-11-r6', 'mariadb_11_1_3', 1, 'data-db', 3306, 'localhost', 3306, 'data-db-sidecar', 8080,
'root', 'dbrepo');
COMMIT;
.docker/config/advanced.config deleted 100644 → 0
+ 0
− 17
View file @ a58b2ffc
[
{
rabbitmq_auth_backend_ldap,
[
{
tag_queries, [
{
administrator, {in_group_nested, "cn=system,ou=users,dc=dbrepo,dc=at", "member"}
},
{
management, {constant, true}
}
]
}
]
}
].
\ No newline at end of file
.docker/config/dbrepo.conf deleted 100644 → 0
+ 0
− 188
View file @ a58b2ffc
# This is required to proxy Grafana Live WebSocket connections.
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
client_max_body_size 20G;
resolver 127.0.0.11 valid=30s; # docker dns
upstream auth {
server auth-service:8080;
}
upstream broker {
server broker-service:15672;
}
upstream analyse {
server analyse-service:8080;
}
upstream data {
server data-service:8080;
}
upstream metadata {
server metadata-service:8080;
}
upstream search {
server search-service:8080;
}
upstream ui {
server ui:3000;
}
upstream upload {
server upload-service:8080;
}
upstream dashboard-service {
server dashboard-service:3000;
}
server {
listen 80 default_server;
server_name _;
location /dashboard {
rewrite ^/dashboard/(.*) /$1 break;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://dashboard-service;
proxy_read_timeout 90;
}
# Proxy Grafana Live WebSocket connections.
location /dashboard/api/live {
rewrite ^/dashboard/(.*) /$1 break;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Host $host;
proxy_pass http://dashboard-service;
}
location /api/search {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://search;
proxy_read_timeout 90;
}
location /api/broker {
rewrite /api/broker/(.*) /api/$1 break;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://broker;
proxy_read_timeout 90;
}
location /api/upload {
# allow 128.130.0.0/16;
# deny all;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $host;
proxy_pass http://upload;
proxy_read_timeout 90;
# Disable request and response buffering
proxy_request_buffering off;
proxy_buffering off;
proxy_http_version 1.1;
}
location /api/analyse {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://analyse;
proxy_read_timeout 90;
}
location /api/auth {
rewrite /api/auth/(.*) /$1 break;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://auth;
proxy_read_timeout 90;
}
location ~ /api/database/([0-9]+)/table/([0-9]+)/(data|history|export|statistic) {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://data;
proxy_read_timeout 90;
}
location ~ /api/database/([0-9]+)/view/([0-9]+)/data {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://data;
proxy_read_timeout 90;
}
location ~ /api/database/([0-9]+)/view {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://metadata;
proxy_read_timeout 90;
}
location ~ /api/database/([0-9]+)/subset {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://data;
proxy_read_timeout 600;
}
location ~ /api/(database|concept|container|identifier|image|message|license|oai|ontology|unit|user) {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://metadata;
proxy_read_timeout 90;
}
location ~ /pid/([0-9]+) {
rewrite /pid/(.*) /api/identifier/$1 break;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://metadata;
proxy_read_timeout 90;
}
location / {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://ui;
proxy_read_timeout 90;
}
}
.docker/config/definitions.json deleted 100644 → 0
+ 0
− 52
View file @ a58b2ffc
{
"bindings": [
{
"arguments": {},
"destination": "dbrepo",
"destination_type": "queue",
"routing_key": "dbrepo.#",
"source": "dbrepo",
"vhost": "dbrepo"
}
],
"exchanges": [
{
"arguments": {},
"auto_delete": false,
"durable": true,
"name": "dbrepo",
"type": "topic",
"vhost": "dbrepo"
}
],
"global_parameters": [],
"parameters": [],
"permissions": [],
"policies": [],
"queues": [
{
"arguments": {
"x-queue-type": "quorum"
},
"auto_delete": false,
"durable": true,
"name": "dbrepo",
"type": "quorum",
"vhost": "dbrepo"
}
],
"rabbit_version": "3.10.25",
"rabbitmq_version": "3.10.25",
"topic_permissions": [],
"users": [],
"vhosts": [
{
"limits": [],
"metadata": {
"description": "Default virtual host",
"tags": []
},
"name": "dbrepo"
}
]
}
\ No newline at end of file
.docker/config/enabled_plugins deleted 100644 → 0
+ 0
− 1
View file @ a58b2ffc
[rabbitmq_prometheus,rabbitmq_auth_backend_ldap,rabbitmq_auth_mechanism_ssl,rabbitmq_management,rabbitmq_mqtt].
\ No newline at end of file
.docker/config/rabbitmq.conf deleted 100644 → 0
+ 0
− 34
View file @ a58b2ffc
# user
default_vhost = dbrepo
default_user_tags.administrator = false
# enable http outside localhost
listeners.tcp.1 = 0.0.0.0:5672
# management prefix (https://www.rabbitmq.com/management.html#path-prefix)
management.load_definitions = /app/definitions.json
# logging
log.console = true
log.console.level = warning
auth_ldap.log = true
# MQTT
mqtt.vhost = dbrepo
mqtt.exchange = dbrepo
mqtt.prefetch = 10
# Obviously your authentication server cannot vouch for itself, so you'll need another backend with at least one user in
# it. You should probably use the internal database
auth_backends.1.authn = ldap
auth_backends.1.authz = ldap
auth_backends.2 = internal
# LDAP
auth_ldap.servers.1 = identity-service
auth_ldap.port = 1389
auth_ldap.user_dn_pattern = ${username}
auth_ldap.dn_lookup_base = dc=dbrepo,dc=at
auth_ldap.dn_lookup_attribute = uid
auth_ldap.dn_lookup_bind.user_dn = cn=admin,dc=dbrepo,dc=at
auth_ldap.dn_lookup_bind.password = admin
.docker/config/s3_config.json deleted 100644 → 0
+ 0
− 20
View file @ a58b2ffc
{
"identities": [
{
"name": "admin",
"credentials": [
{
"accessKey": "seaweedfsadmin",
"secretKey": "seaweedfsadmin"
}
],
"actions": [
"Read",
"Write",
"List",
"Tagging",
"Admin"
]
}
]
}
\ No newline at end of file
.docker/docker-compose.yml
+ 35
− 34
View file @ 749de1a5
......@@ -59,17 +59,16 @@ services:
restart: "no"
container_name: dbrepo-auth-db
hostname: auth-db
image: docker.io/bitnami/mariadb:11.1.3-debian-11-r6
image: docker.io/bitnami/postgresql:17.0.0-debian-12-r1
volumes:
- auth-db-data:/bitnami/mariadb
ports:
- "3308:3306"
- auth-db-data:/bitnami/postgresql
environment:
MARIADB_DATABASE: "${AUTH_DB_NAME:-keycloak}"
MARIADB_ROOT_PASSWORD: "${AUTH_DB_PASSWORD:-dbrepo}"
MARIADB_GALERA_MARIABACKUP_PASSWORD: "${AUTH_DB_BACKUP_PASSWORD:-dbrepobackup}"
POSTGRESQL_DATABASE: "${AUTH_DB_NAME:-keycloak}"
POSTGRESQL_USERNAME: "${AUTH_DB_USERNAME:-keycloak}"
POSTGRESQL_PASSWORD: "${AUTH_DB_PASSWORD:-dbrepo}"
PGPASSWORD: "${AUTH_DB_PASSWORD:-dbrepo}"
healthcheck:
test: mysqladmin ping --user=root --password="${AUTH_DB_PASSWORD:-dbrepo}" --silent
test: "psql -U ${AUTH_DB_USERNAME:-keycloak} -h 127.0.0.1 -p 5432 -d ${AUTH_DB_NAME:-keycloak} -c 'select version();'"
interval: 15s
timeout: 5s
retries: 12
......@@ -80,18 +79,25 @@ services:
restart: "no"
container_name: dbrepo-auth-service
hostname: auth-service
image: registry.datalab.tuwien.ac.at/dbrepo/auth-service:1.4.6
image: bitnami/keycloak:26.0.0-debian-12-r1
volumes:
- ./config/import-realms.sh:/docker-entrypoint-initdb.d/import-realms.sh
- ./config/master-realm.json:/opt/keycloak/data/import/master-realm.json
- ./config/dbrepo-realm.json:/opt/keycloak/data/import/dbrepo-realm.json
environment:
BITNAMI_DEBUG: "true"
KEYCLOAK_ENABLE_HTTPS: "false"
KEYCLOAK_ENABLE_STATISTICS: "true"
KEYCLOAK_ENABLE_HEALTH_ENDPOINTS: "true"
KEYCLOAK_DATABASE_HOST: "auth-db"
KEYCLOAK_DATABASE_NAME: "${AUTH_DB_NAME:-keycloak}"
KEYCLOAK_DATABASE_USER: "${AUTH_DB_USERNAME:-keycloak}"
KEYCLOAK_DATABASE_PASSWORD: "${AUTH_DB_PASSWORD:-dbrepo}"
healthcheck:
test: curl -sSL 'http://0.0.0.0:8080/realms/dbrepo' | grep "dbrepo" || exit 1
interval: 15s
test: curl --head -fsS http://localhost:9000/health/ready
interval: 10s
timeout: 5s
retries: 12
environment:
AUTH_DB: "${AUTH_DB:-keycloak}"
KC_DB_USERNAME: root
KC_DB_PASSWORD: "${AUTH_DB_PASSWORD:-dbrepo}"
KEYCLOAK_ADMIN: "${AUTH_SERVICE_ADMIN_USERNAME:-admin}"
KEYCLOAK_ADMIN_PASSWORD: "${AUTH_SERVICE_ADMIN_PASSWORD:-admin}"
depends_on:
dbrepo-identity-service:
condition: service_healthy
......@@ -104,7 +110,7 @@ services:
restart: "no"
container_name: dbrepo-metadata-service
hostname: metadata-service
image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.4.6
image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.4.7
volumes:
- "${SHARED_VOLUME:-/tmp}:/tmp"
environment:
......@@ -167,7 +173,7 @@ services:
restart: "no"
container_name: dbrepo-analyse-service
hostname: analyse-service
image: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.4.6
image: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.4.7
environment:
AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client}
AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
......@@ -220,7 +226,7 @@ services:
restart: "no"
container_name: dbrepo-search-db
hostname: search-db
image: registry.datalab.tuwien.ac.at/dbrepo/search-db:1.4.6
image: registry.datalab.tuwien.ac.at/dbrepo/search-db:1.4.7
healthcheck:
test: curl -sSL localhost:9200/_plugins/_security/health | jq .status | grep UP
interval: 10s
......@@ -244,7 +250,7 @@ services:
restart: "no"
container_name: dbrepo-search-service
hostname: search-service
image: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.4.6
image: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.4.7
environment:
AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client}
AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT_SECRET:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
......@@ -266,7 +272,7 @@ services:
restart: "no"
container_name: dbrepo-data-db-sidecar
hostname: data-db-sidecar
image: registry.datalab.tuwien.ac.at/dbrepo/data-db-sidecar:1.4.6
image: registry.datalab.tuwien.ac.at/dbrepo/data-db-sidecar:1.4.7
environment:
S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}"
S3_BUCKET: "${S3_BUCKET:-dbrepo}"
......@@ -287,7 +293,7 @@ services:
restart: "no"
container_name: dbrepo-ui
hostname: ui
image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.4.6
image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.4.7
environment:
NUXT_PUBLIC_API_CLIENT: "${BASE_URL:-http://localhost}"
NUXT_PUBLIC_API_SERVER: "${BASE_URL:-http://localhost}"
......@@ -318,12 +324,8 @@ services:
depends_on:
dbrepo-analyse-service:
condition: service_healthy
dbrepo-auth-service:
condition: service_healthy
dbrepo-broker-service:
condition: service_healthy
dbrepo-dashboard-service:
condition: service_healthy
dbrepo-metadata-service:
condition: service_healthy
dbrepo-search-db:
......@@ -356,9 +358,10 @@ services:
dbrepo-search-service-init:
restart: "no"
init: true
container_name: dbrepo-search-service-init
hostname: search-service-init
image: registry.datalab.tuwien.ac.at/dbrepo/search-service-init:1.4.6
image: registry.datalab.tuwien.ac.at/dbrepo/search-service-init:1.4.7
environment:
METADATA_SERVICE_ENDPOINT: ${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080}
OPENSEARCH_HOST: ${OPENSEARCH_HOST:-search-db}
......@@ -412,11 +415,8 @@ services:
restart: "no"
container_name: dbrepo-dashboard-service
hostname: dashboard-service
image: registry.datalab.tuwien.ac.at/dbrepo/dashboard-service:1.4.6
image: registry.datalab.tuwien.ac.at/dbrepo/dashboard-service:1.4.7
environment:
GF_SERVER_DOMAIN: "dashboard-service"
GF_SERVER_ROOT_URL: "${BASE_URL:-http://localhost}/dashboard/"
GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION: "true"
LDAP_ADMIN_USERNAME: "${IDENTITY_SERVICE_ADMIN_USERNAME:-admin}"
LDAP_ADMIN_PASSWORD: "${IDENTITY_SERVICE_ADMIN_PASSWORD:-admin}"
LDAP_ROOT: "${IDENTITY_SERVICE_ROOT:-dc=dbrepo,dc=at}"
......@@ -433,9 +433,10 @@ services:
dbrepo-storage-service-init:
restart: "no"
init: true
container_name: dbrepo-storage-service-init
hostname: storage-service-init
image: registry.datalab.tuwien.ac.at/dbrepo/storage-service-init:1.4.6
image: registry.datalab.tuwien.ac.at/dbrepo/storage-service-init:1.4.7
environment:
WEED_CLUSTER_SW_MASTER: "${STORAGE_SERVICE_MASTER_ENDPOINT:-storage-service:9333}"
S3_BUCKET: "${S3_BUCKET:-dbrepo}"
......@@ -475,7 +476,7 @@ services:
restart: "no"
container_name: dbrepo-data-service
hostname: data-service
image: registry.datalab.tuwien.ac.at/dbrepo/data-service:1.4.6
image: registry.datalab.tuwien.ac.at/dbrepo/data-service:1.4.7
volumes:
- "${SHARED_VOLUME:-/tmp}:/tmp"
environment:
......
......
.docs/api/auth-service.md
+ 2
− 2
View file @ 749de1a5
......@@ -6,10 +6,10 @@ author: Martin Weise
!!! debug "Debug Information"
Image: [`quay.io/keycloak/keycloak:24.0`](quay.io/keycloak/keycloak)
Image: [`docker.io/bitnami/keycloak:26.0.0-debian-12-r1`](https://hub.docker.com/r/bitnami/keycloak)
* Ports: 8080/tcp
* UI: `http://<hostname>/api/auth/`
* UI: `http://<hostname>:8080/`
## Overview
......
......
.docs/api/dashboard-service.md
+ 0
− 15
View file @ 749de1a5
......@@ -17,21 +17,6 @@ The Dashboard Service is visualizing the status of DBRepo with charts. The defau
`/etc/grafana/provisioning/dashboards/provider.yaml` checks for new `JSON` dashboard files in `/app/dashboards` every 10
seconds and makes the available in the Dashboard Service.
<figure markdown>
![DBRepo Dashboard](../images/screenshots/grafana4.png)
<figcaption>Figure 1: DBRepo Dashboard</figcaption>
</figure>
<figure markdown>
![MariaDB Galera Dashboard](../images/screenshots/grafana3.png)
<figcaption>Figure 2: Database Dashboard (Kubernetes deployment only)</figcaption>
</figure>
<figure markdown>
![RabbitMQ Dashboard](../images/screenshots/grafana5.png)
<figcaption>Figure 3: Broker Service Dashboard</figcaption>
</figure>
## Limitations
!!! question "Do you miss functionality? Do these limitations affect you?"
......
......
.docs/api/metric-db.md
+ 131
− 1
View file @ 749de1a5
......@@ -10,7 +10,137 @@ author: Martin Weise
## Overview
The Metric Database is responsible for saving time-series data for the [Dashboard Service](../../api/dashboard-service)
The Metric Database is responsible for saving time-series data for the [Dashboard Service](../../api/dashboard-service).
## Metrics
## Analyse Service
| **Metric** | **Description** |
|-----------------------------|---------------------------------------------|
| `dbrepo_analyse_datatypes` | Time needed to analyse datatypes of dataset |
| `dbrepo_analyse_keys` | Time needed to analyse keys of dataset |
| `dbrepo_analyse_table_stat` | Time needed to analyse table statistics |
| `dbrepo_analyse_table_stat` | Time needed to analyse table statistics |
## Auth Service
See [Keycloak documentation](https://www.keycloak.org/server/configuration-metrics).
## Broker Service
See [RabbitMQ documentation](https://www.rabbitmq.com/docs/prometheus).
## Databases
See [MariaDB Galera documentation](https://galeracluster.com/documentation/html_docs_20210213-1355-master/documentation/galera-manager-monitoring-clusters.html).
The [Data Database Sidecar](#) metrics are:
| **Metric** | **Description** |
|---------------------------------|---------------------------------------|
| `dbrepo_sidecar_import_dataset` | Time needed to import dataset from S3 |
| `dbrepo_sidecar_export_dataset` | Time needed to export dataset to S3 |
## Data Service
| **Metric** | **Description** |
|-----------------------------|-------------------------------------------|
| `dbrepo_message_receive` | Received AMQP message from Broker Service |
| `dbrepo_subset_create` | Create subset |
| `dbrepo_subset_data` | Retrieved subset data |
| `dbrepo_subset_find` | Find subset |
| `dbrepo_subset_list` | Find subsets |
| `dbrepo_subset_persist` | Persist subset |
| `dbrepo_table_data_create` | Create table data |
| `dbrepo_table_data_delete` | Delete table data |
| `dbrepo_table_data_export` | Export table data |
| `dbrepo_table_data_history` | Find table history |
| `dbrepo_table_data_import` | Import dataset |
| `dbrepo_table_data_list` | Retrieve table data |
| `dbrepo_table_data_update` | Update table data |
| `dbrepo_view_data` | Retrieve view data |
| `dbrepo_view_schema_list` | Find view schemas |
## Metadata Service
| **Metric** | **Description** |
|------------------------------------|---------------------------------------------------|
| `dbrepo_database_count` | The total number of managed research databases |
| `dbrepo_view_count` | The total number of available view data sources |
| `dbrepo_subset_count` | The total number of available subset data sources |
| `dbrepo_table_count` | The total number of available table data sources |
| `dbrepo_volume_sum` | The total volume of available research data |
| `dbrepo_user_refresh_token` | Refresh user token |
| `dbrepo_identifier_save` | Save identifier |
| `dbrepo_oai_record_get` | Get the record |
| `dbrepo_access_give` | Give access to some database |
| `dbrepo_ontologies_find` | Find one ontology |
| `dbrepo_database_findall` | List databases |
| `dbrepo_tables_refresh` | Refresh database tables metadata |
| `dbrepo_license_findall` | Get all licenses |
| `dbrepo_user_modify` | Modify user information |
| `dbrepo_container_findall` | Find all containers |
| `dbrepo_maintenance_delete` | Delete maintenance message |
| `dbrepo_maintenance_update` | Update maintenance message |
| `dbrepo_ontologies_create` | Register a new ontology |
| `dbrepo_identifier_delete` | Delete some identifier |
| `dbrepo_oai_identify` | Identify the repository |
| `dbrepo_database_create` | Create database |
| `dbrepo_oai_metadataformats_list` | List the metadata formats |
| `dbrepo_user_password_modify` | Modify user password |
| `dbrepo_semantic_concepts_findall` | List semantic concepts |
| `dbrepo_identifier_retrieve` | Retrieve metadata from identifier |
| `dbrepo_identifier_list` | Find all identifiers |
| `dbrepo_views_findall` | Find all views |
| `dbrepo_identifier_create` | Draft identifier |
| `dbrepo_oai_identifiers_list` | List the identifiers |
| `dbrepo_image_findall` | Find all images |
| `dbrepo_database_visibility` | Update database visibility |
| `dbrepo_container_create` | Create container |
| `dbrepo_views_refresh` | Refresh database views metadata |
| `dbrepo_database_find` | Find some database |
| `dbrepo_access_get` | Check access to some database |
| `dbrepo_identifier_find` | Find some identifier |
| `dbrepo_maintenance_create` | Create maintenance message |
| `dbrepo_container_delete` | Delete some container |
| `dbrepo_ontologies_delete` | Delete an ontology |
| `dbrepo_ontologies_findall` | List all ontologies |
| `dbrepo_user_token` | Obtain user token |
| `dbrepo_view_find` | Find one view |
| `dbrepo_user_create` | Create user |
| `dbrepo_ontologies_update` | Update an ontology |
| `dbrepo_maintenance_findall` | Find maintenance messages |
| `dbrepo_users_list` | Find all users |
| `dbrepo_image_find` | Find some image |
| `dbrepo_user_find` | Get a user info |
| `dbrepo_image_delete` | Delete some image |
| `dbrepo_identifier_publish` | Publish identifier |
| `dbrepo_image_update` | Update some image |
| `dbrepo_view_create` | Create a view |
| `dbrepo_semantic_units_findall` | List semantic units |
| `dbrepo_image_create` | Create image |
| `dbrepo_database_image` | Update database image |
| `dbrepo_view_delete` | Delete one view |
| `dbrepo_database_transfer` | Update database owner |
| `dbrepo_maintenance_find` | Find one maintenance message |
| `dbrepo_access_modify` | Modify access to some database |
| `dbrepo_ontologies_entities_find` | Find entities |
| `dbrepo_access_delete` | Revoke access to some database |
| `dbrepo_container_find` | Find some container |
## Search Service
| **Metric** | **Description** |
|---------------------------------|---------------------------------------------------------|
| `dbrepo_search_index_list` | Time needed to list search index |
| `dbrepo_search_type_list` | Time needed to list search types |
| `dbrepo_search_fuzzy` | Time needed to search fuzzy |
| `dbrepo_search_type` | Time needed to search by type |
| `dbrepo_search_update_database` | Time needed to update a database in the search database |
| `dbrepo_search_delete_database` | Time needed to delete a database in the search database |
## Limitations
......
......
.docs/api/ui.md
+ 2
− 2
View file @ 749de1a5
......@@ -6,7 +6,7 @@ author: Martin Weise
!!! debug "Debug Information"
Image: [`registry.datalab.tuwien.ac.at/dbrepo/ui:1.4.6`](https://hub.docker.com/r/dbrepo/ui)
Image: [`registry.datalab.tuwien.ac.at/dbrepo/ui:1.4.7`](https://hub.docker.com/r/dbrepo/ui)
* Ports: 3000/tcp
......@@ -37,7 +37,7 @@ image as well, in this example we want to mount a custom logo `my_logo.png` into
```yaml title="docker-compose.yml"
services:
dbrepo-ui:
image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.4.6
image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.4.7
volumes:
- ./my_logo.png:/app/.output/public/my_logo.png
- ./favicon.ico:/app/.output/public/favicon.ico
......
......
.docs/concepts/database.md deleted 100644 → 0
+ 0
− 40
View file @ a58b2ffc
---
author: Martin Weise
---
## Relational Database
DBRepo manages relational databases that store information relations in tables.
## Query
A query is the method to interact with a relational database and is used to read/write data or to create/change/delete
schema information e.g. tables. DBRepo uses a query store to store certain (important) queries that generate subsets
to restore the exact same subset at a later point.
## System Versioning
DBRepo uses a mechanism offered by SQL:2013 to version tables with the system (=server) time. When inserting a tuple
into a system-versioned table, the database engine maintains invisible `ROW_START` and `ROW_END` timestamp columns to
denote a tuple validity. When deleting a tuple, the database engine actually just marks the tuple as `ROW_END = NOW()`
and does not delete the tuple.
At a later point in time, the (historic) tuple can still be queried using system versioning.
## Data Ingest
<figure markdown>
![Data ingest](../images/data-ingest.svg)
<figcaption>Figure 1: Modes of data ingest</figcaption>
</figure>
More [usage examples](../usage-overview/) include how to ingest datasets, data dumps, live data, etc.
### Generation of Metadata in DBRepo
You can generate metadata e.g. UI tbd
!!! warning "Limitation"
Only system-versioned tables are considered when generating metadata to tables. If your table is not system-versioned
e.g. a base table, it will not be visible in the UI.
\ No newline at end of file
.docs/concepts/monitoring.md
+ 16
− 128
View file @ 749de1a5
......@@ -11,131 +11,19 @@ instance is started.
You need can setup Prometheus in a few minutes using
a [Docker container](https://prometheus.io/docs/prometheus/latest/installation/).
## Example
## Analyse Service
| **Metric** | **Description** |
|-----------------------------|---------------------------------------------|
| `dbrepo_analyse_datatypes` | Time needed to analyse datatypes of dataset |
| `dbrepo_analyse_keys` | Time needed to analyse keys of dataset |
| `dbrepo_analyse_table_stat` | Time needed to analyse table statistics |
| `dbrepo_analyse_table_stat` | Time needed to analyse table statistics |
## Auth Service
See [Keycloak documentation](https://www.keycloak.org/server/configuration-metrics).
## Broker Service
See [RabbitMQ documentation](https://www.rabbitmq.com/docs/prometheus).
## Databases
See [MariaDB Galera documentation](https://galeracluster.com/documentation/html_docs_20210213-1355-master/documentation/galera-manager-monitoring-clusters.html).
The [Data Database Sidecar](#) metrics are:
| **Metric** | **Description** |
|---------------------------------|---------------------------------------|
| `dbrepo_sidecar_import_dataset` | Time needed to import dataset from S3 |
| `dbrepo_sidecar_export_dataset` | Time needed to export dataset to S3 |
## Data Service
| **Metric** | **Description** |
|-----------------------------|-------------------------------------------|
| `dbrepo_message_receive` | Received AMQP message from Broker Service |
| `dbrepo_subset_create` | Create subset |
| `dbrepo_subset_data` | Retrieved subset data |
| `dbrepo_subset_find` | Find subset |
| `dbrepo_subset_list` | Find subsets |
| `dbrepo_subset_persist` | Persist subset |
| `dbrepo_table_data_create` | Create table data |
| `dbrepo_table_data_delete` | Delete table data |
| `dbrepo_table_data_export` | Export table data |
| `dbrepo_table_data_history` | Find table history |
| `dbrepo_table_data_import` | Import dataset |
| `dbrepo_table_data_list` | Retrieve table data |
| `dbrepo_table_data_update` | Update table data |
| `dbrepo_view_data` | Retrieve view data |
| `dbrepo_view_schema_list` | Find view schemas |
## Metadata Service
| **Metric** | **Description** |
|------------------------------------|---------------------------------------------------|
| `dbrepo_database_count` | The total number of managed research databases |
| `dbrepo_view_count` | The total number of available view data sources |
| `dbrepo_subset_count` | The total number of available subset data sources |
| `dbrepo_table_count` | The total number of available table data sources |
| `dbrepo_volume_sum` | The total volume of available research data |
| `dbrepo_user_refresh_token` | Refresh user token |
| `dbrepo_identifier_save` | Save identifier |
| `dbrepo_oai_record_get` | Get the record |
| `dbrepo_access_give` | Give access to some database |
| `dbrepo_ontologies_find` | Find one ontology |
| `dbrepo_database_findall` | List databases |
| `dbrepo_tables_refresh` | Refresh database tables metadata |
| `dbrepo_license_findall` | Get all licenses |
| `dbrepo_user_modify` | Modify user information |
| `dbrepo_container_findall` | Find all containers |
| `dbrepo_maintenance_delete` | Delete maintenance message |
| `dbrepo_maintenance_update` | Update maintenance message |
| `dbrepo_ontologies_create` | Register a new ontology |
| `dbrepo_identifier_delete` | Delete some identifier |
| `dbrepo_oai_identify` | Identify the repository |
| `dbrepo_database_create` | Create database |
| `dbrepo_oai_metadataformats_list` | List the metadata formats |
| `dbrepo_user_password_modify` | Modify user password |
| `dbrepo_semantic_concepts_findall` | List semantic concepts |
| `dbrepo_identifier_retrieve` | Retrieve metadata from identifier |
| `dbrepo_identifier_list` | Find all identifiers |
| `dbrepo_views_findall` | Find all views |
| `dbrepo_identifier_create` | Draft identifier |
| `dbrepo_oai_identifiers_list` | List the identifiers |
| `dbrepo_image_findall` | Find all images |
| `dbrepo_database_visibility` | Update database visibility |
| `dbrepo_container_create` | Create container |
| `dbrepo_views_refresh` | Refresh database views metadata |
| `dbrepo_database_find` | Find some database |
| `dbrepo_access_get` | Check access to some database |
| `dbrepo_identifier_find` | Find some identifier |
| `dbrepo_maintenance_create` | Create maintenance message |
| `dbrepo_container_delete` | Delete some container |
| `dbrepo_ontologies_delete` | Delete an ontology |
| `dbrepo_ontologies_findall` | List all ontologies |
| `dbrepo_user_token` | Obtain user token |
| `dbrepo_view_find` | Find one view |
| `dbrepo_user_create` | Create user |
| `dbrepo_ontologies_update` | Update an ontology |
| `dbrepo_maintenance_findall` | Find maintenance messages |
| `dbrepo_users_list` | Find all users |
| `dbrepo_image_find` | Find some image |
| `dbrepo_user_find` | Get a user info |
| `dbrepo_image_delete` | Delete some image |
| `dbrepo_identifier_publish` | Publish identifier |
| `dbrepo_image_update` | Update some image |
| `dbrepo_view_create` | Create a view |
| `dbrepo_semantic_units_findall` | List semantic units |
| `dbrepo_image_create` | Create image |
| `dbrepo_database_image` | Update database image |
| `dbrepo_view_delete` | Delete one view |
| `dbrepo_database_transfer` | Update database owner |
| `dbrepo_maintenance_find` | Find one maintenance message |
| `dbrepo_access_modify` | Modify access to some database |
| `dbrepo_ontologies_entities_find` | Find entities |
| `dbrepo_access_delete` | Revoke access to some database |
| `dbrepo_container_find` | Find some container |
## Search Service
| **Metric** | **Description** |
|---------------------------------|---------------------------------------------------------|
| `dbrepo_search_index_list` | Time needed to list search index |
| `dbrepo_search_type_list` | Time needed to list search types |
| `dbrepo_search_fuzzy` | Time needed to search fuzzy |
| `dbrepo_search_type` | Time needed to search by type |
| `dbrepo_search_update_database` | Time needed to update a database in the search database |
| `dbrepo_search_delete_database` | Time needed to delete a database in the search database |
## Dashboards
<figure markdown>
![DBRepo Dashboard](../images/screenshots/grafana4.png)
<figcaption>Figure 1: DBRepo Dashboard</figcaption>
</figure>
<figure markdown>
![MariaDB Galera Dashboard](../images/screenshots/grafana3.png)
<figcaption>Figure 2: Database Dashboard (Kubernetes deployment only)</figcaption>
</figure>
<figure markdown>
![RabbitMQ Dashboard](../images/screenshots/grafana5.png)
<figcaption>Figure 3: Broker Service Dashboard</figcaption>
</figure>
\ No newline at end of file
.docs/images/screenshots/grafana4.png
+ 0
− 0
View replaced file @ a58b2ffc View file @ 749de1a5
.docs/images/screenshots/grafana4.png

563 KiB | W: | H:

.docs/images/screenshots/grafana4.png

185 KiB | W: | H:

.docs/images/screenshots/grafana4.png
.docs/images/screenshots/grafana4.png
.docs/images/screenshots/grafana4.png
.docs/images/screenshots/grafana4.png
  • 2-up
  • Swipe
  • Onion skin
.docs/index.md
+ 1
− 1
View file @ 749de1a5
......@@ -8,7 +8,7 @@ author: Martin Weise
[![Image Pulls](https://img.shields.io/docker/pulls/dbrepo/data-service?style=flat&cacheSeconds=3600)](https://hub.docker.com/u/dbrepo){ tabindex=-1 }
[![GitLab License](https://img.shields.io/gitlab/license/fair-data-austria-db-repository%2Ffda-services?gitlab_url=https%3A%2F%2Fgitlab.phaidra.org%2F&style=flat&cacheSeconds=3600)](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services){ tabindex=-1 }
Documentation for version: [v1.4.6](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/releases).
Documentation for version: [v1.4.7](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/releases).
DBRepo is a repository for data in databases that are used from the beginning until the end of a research
project supporting data evolution, -citation and -versioning. It implements the query store of the
......
......
.docs/installation.md
+ 3
− 3
View file @ 749de1a5
......@@ -11,7 +11,7 @@ author: Martin Weise
If you have [Docker](https://docs.docker.com/engine/install/) already installed on your system, you can install DBRepo with:
```shell
curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.6/install.sh | bash
curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.7/install.sh | bash
```
!!! bug "Default installation security disclaimer"
......@@ -39,7 +39,7 @@ SSL/TLS certificate is recommended. Follow the [secure install](#secure-install)
Execute the install script to download only the environment and save it to `dist`.
```shell
curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.6/install.sh | DOWNLOAD_ONLY=1 bash
curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.7/install.sh | DOWNLOAD_ONLY=1 bash
```
### Static Configuration
......@@ -81,7 +81,7 @@ the variable `IDENTITY_SERVICE_ADMIN_PASSWORD` in `.env`.
Update the client secret of the `dbrepo-client`:
```bash
curl -sSL "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.6/.scripts/reg-client-secret.sh" | bash
curl -sSL "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.7/.scripts/reg-client-secret.sh" | bash
```
Also, update the JWT key according to the
......
......
.docs/kubernetes.md
+ 3
− 3
View file @ 749de1a5
......@@ -7,7 +7,7 @@ author: Martin Weise
## TL;DR
To install DBRepo in your existing cluster, download the
sample [`values.yaml`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.4.6/helm/dbrepo/values.yaml)
sample [`values.yaml`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.4.7/helm/dbrepo/values.yaml)
for your deployment and update the variables, especially `hostname`.
```shell
......@@ -15,7 +15,7 @@ helm upgrade --install dbrepo \
-n dbrepo \
"oci://registry.datalab.tuwien.ac.at/dbrepo/helm/dbrepo" \
--values ./values.yaml \
--version "1.4.6" \
--version "1.4.7" \
--create-namespace \
--cleanup-on-fail
```
......@@ -36,7 +36,7 @@ brokerservice:
The `brokerservice.auth.passwordHash` field is the RabbitMQ SHA512-hash of the `brokerservice.auth.password` field and
can be obtained with
the [`generate-rabbitmq-pw.sh`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.4.6/helm/dbrepo/hack/generate-rabbitmq-pw.sh)
the [`generate-rabbitmq-pw.sh`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.4.7/helm/dbrepo/hack/generate-rabbitmq-pw.sh)
script:
```console
......
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment