Skip to content
Snippets Groups Projects
Commit 35e69074 authored by Martin Weise's avatar Martin Weise
Browse files

Hotfix secrets 2

parent 42fdf99d
No related branches found
No related tags found
2 merge requests!302Dev,!301Hotfix secrets 2
Show whitespace changes
Inline Side-by-side
Showing
with 117 additions and 113 deletions
.docker/docker-compose.yml
+ 47
− 39
View file @ 35e69074
......@@ -8,6 +8,7 @@ volumes:
upload-service-data:
search-db-data:
storage-service-data:
identity-service-data:
services:
dbrepo-metadata-db:
......@@ -66,7 +67,7 @@ services:
MARIADB_ROOT_PASSWORD: "${AUTH_PASSWORD:-dbrepo}"
healthcheck:
test: mysqladmin ping --user="${AUTH_USERNAME:-root}" --password="${AUTH_PASSWORD:-dbrepo}" --silent
interval: 10s
interval: 15s
timeout: 5s
retries: 12
logging:
......@@ -76,19 +77,21 @@ services:
restart: "no"
container_name: dbrepo-auth-service
hostname: auth-service
image: registry.datalab.tuwien.ac.at/dbrepo/auth-service:1.4.4
image: registry.datalab.tuwien.ac.at/dbrepo/auth-service:1.4.5
healthcheck:
test: curl -sSL 'http://0.0.0.0:8080/realms/dbrepo' | grep "dbrepo" || exit 1
interval: 10s
interval: 15s
timeout: 5s
retries: 12
environment:
AUTH_DB: "${AUTH_DB:-keycloak}"
KC_DB_USERNAME: "${AUTH_USERNAME:-root}"
KC_DB_PASSWORD: "${AUTH_PASSWORD:-dbrepo}"
KEYCLOAK_ADMIN: "${KEYCLOAK_ADMIN:-fda}"
KEYCLOAK_ADMIN_PASSWORD: "${KEYCLOAK_ADMIN_PASSWORD:-fda}"
KEYCLOAK_ADMIN: "${KEYCLOAK_ADMIN:-admin}"
KEYCLOAK_ADMIN_PASSWORD: "${KEYCLOAK_ADMIN_PASSWORD:-admin}"
depends_on:
dbrepo-identity-service:
condition: service_healthy
dbrepo-auth-db:
condition: service_healthy
logging:
......@@ -98,16 +101,14 @@ services:
restart: "no"
container_name: dbrepo-metadata-service
hostname: metadata-service
image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.4.4
image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.4.5
volumes:
- "${SHARED_VOLUME:-/tmp}:/tmp"
environment:
ADMIN_EMAIL: "${ADMIN_EMAIL:-noreply@localhost}"
ADMIN_PASSWORD: "${ADMIN_PASSWORD:-admin}"
ADMIN_USERNAME: "${ADMIN_USERNAME:-admin}"
ANALYSE_SERVICE_ENDPOINT: "${ANALYSE_SERVICE_ENDPOINT:-http://gateway-service}"
AUTH_SERVICE_ADMIN: ${AUTH_SERVICE_ADMIN:-fda}
AUTH_SERVICE_ADMIN_PASSWORD: ${AUTH_SERVICE_ADMIN_PASSWORD:-fda}
AUTH_SERVICE_ADMIN: ${AUTH_SERVICE_ADMIN:-admin}
AUTH_SERVICE_ADMIN_PASSWORD: ${AUTH_SERVICE_ADMIN_PASSWORD:-admin}
AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client}
AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
AUTH_SERVICE_ENDPOINT: ${AUTH_SERVICE_ENDPOINT:-http://gateway-service/api/auth}
......@@ -115,11 +116,12 @@ services:
BROKER_EXCHANGE_NAME: ${BROKER_EXCHANGE_NAME:-dbrepo}
BROKER_QUEUE_NAME: ${BROKER_QUEUE_NAME:-dbrepo}
BROKER_HOST: "${BROKER_ENDPOINT:-broker-service}"
BROKER_PASSWORD: ${BROKER_PASSWORD:-fda}
BROKER_PASSWORD: ${BROKER_PASSWORD:-admin}
BROKER_PORT: ${BROKER_PORT:-5672}
BROKER_SERVICE_ENDPOINT: ${BROKER_SERVICE_ENDPOINT:-http://gateway-service/admin/broker}
BROKER_USERNAME: ${BROKER_USERNAME:-fda}
BROKER_USERNAME: ${BROKER_USERNAME:-admin}
BROKER_VIRTUALHOST: "${BROKER_VIRTUALHOST:-dbrepo}"
CROSSREF_ENDPOINT: "${CROSSREF_ENDPOINT:-http://data.crossref.org}"
DATA_SERVICE_ENDPOINT: ${DATA_SERVICE_ENDPOINT:-http://data-service:8080}
DELETED_RECORD: "${DELETED_RECORD:-persistent}"
GRANULARITY: "${GRANULARITY:-YYYY-MM-DDThh:mm:ssZ}"
......@@ -132,13 +134,15 @@ services:
METADATA_PASSWORD: "${METADATA_PASSWORD:-dbrepo}"
PID_BASE: ${PID_BASE:-http://localhost/pid/}
REPOSITORY_NAME: "${REPOSITORY_NAME:-Database Repository}"
ROR_ENDPOINT: "${ROR_ENDPOINT:-https://api.ror.org}"
SEARCH_SERVICE_ENDPOINT: "${SEARCH_SERVICE_ENDPOINT:-http://gateway-service}"
S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}"
S3_BUCKET: "${S3_BUCKET:-dbrepo}"
S3_ENDPOINT: "${S3_ENDPOINT:-http://storage-service:9000}"
S3_EXPORT_BUCKET: "${S3_EXPORT_BUCKET:-dbrepo-download}"
S3_IMPORT_BUCKET: "${S3_IMPORT_BUCKET:-dbrepo-upload}"
S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}"
SPARQL_CONNECTION_TIMEOUT: "${SPARQL_CONNECTION_TIMEOUT:-10000}"
SYSTEM_USERNAME: "${SYSTEM_USERNAME:-admin}"
SYSTEM_PASSWORD: "${SYSTEM_PASSWORD:-admin}"
healthcheck:
test: curl -sSL localhost:8080/actuator/health/liveness | grep 'UP' || exit 1
interval: 10s
......@@ -160,7 +164,7 @@ services:
restart: "no"
container_name: dbrepo-analyse-service
hostname: analyse-service
image: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.4.4
image: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.4.5
environment:
ADMIN_PASSWORD: "${ADMIN_PASSWORD:-admin}"
ADMIN_USERNAME: "${ADMIN_USERNAME:-admin}"
......@@ -170,9 +174,8 @@ services:
GATEWAY_SERVICE_ENDPOINT: ${GATEWAY_SERVICE_ENDPOINT:-http://gateway-service}
JWT_PUBKEY: "${JWT_PUBKEY:-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB}"
S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}"
S3_BUCKET: "${S3_BUCKET:-dbrepo}"
S3_ENDPOINT: "${S3_ENDPOINT:-http://storage-service:9000}"
S3_EXPORT_BUCKET: "${S3_EXPORT_BUCKET:-dbrepo-download}"
S3_IMPORT_BUCKET: "${S3_IMPORT_BUCKET:-dbrepo-upload}"
S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}"
volumes:
- "${SHARED_FILESYSTEM:-/tmp}:/tmp"
......@@ -189,13 +192,16 @@ services:
container_name: dbrepo-broker-service
hostname: broker-service
image: docker.io/bitnami/rabbitmq:3.12-debian-12
ports:
- 5672:5672
volumes:
- ./dist/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf
- ./dist/advanced.config:/etc/rabbitmq/advanced.config
- ./dist/enabled_plugins:/etc/rabbitmq/enabled_plugins
- ./dist/definitions.json:/app/definitions.json
- broker-service-data:/bitnami/rabbitmq/mnesia
depends_on:
dbrepo-auth-service:
dbrepo-identity-service:
condition: service_healthy
healthcheck:
test: rabbitmq-diagnostics -q is_running | grep 'is fully booted and running'
......@@ -209,7 +215,7 @@ services:
restart: "no"
container_name: dbrepo-search-db
hostname: search-db
image: registry.datalab.tuwien.ac.at/dbrepo/search-db:1.4.4
image: registry.datalab.tuwien.ac.at/dbrepo/search-db:1.4.5
healthcheck:
test: curl -sSL localhost:9200/_plugins/_security/health | jq .status | grep UP
interval: 10s
......@@ -233,10 +239,8 @@ services:
restart: "no"
container_name: dbrepo-search-service
hostname: search-service
image: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.4.4
image: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.4.5
environment:
ADMIN_PASSWORD: "${ADMIN_PASSWORD:-admin}"
ADMIN_USERNAME: "${ADMIN_USERNAME:-admin}"
AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client}
AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
AUTH_SERVICE_ENDPOINT: ${AUTH_SERVICE_ENDPOINT:-http://auth-service:8080}
......@@ -252,13 +256,12 @@ services:
restart: "no"
container_name: dbrepo-data-db-sidecar
hostname: data-db-sidecar
image: registry.datalab.tuwien.ac.at/dbrepo/data-db-sidecar:1.4.4
image: registry.datalab.tuwien.ac.at/dbrepo/data-db-sidecar:1.4.5
environment:
S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}"
S3_BUCKET: "${S3_BUCKET:-dbrepo}"
S3_ENDPOINT: "${S3_ENDPOINT:-http://storage-service:9000}"
S3_EXPORT_BUCKET: "${S3_EXPORT_BUCKET:-dbrepo-download}"
S3_FILE_PATH: "${S3_FILE_PATH:-/tmp}"
S3_IMPORT_BUCKET: "${S3_IMPORT_BUCKET:-dbrepo-upload}"
S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}"
volumes:
- "${SHARED_FILESYSTEM:-/tmp}:/tmp"
......@@ -274,7 +277,7 @@ services:
restart: "no"
container_name: dbrepo-ui
hostname: ui
image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.4.4
image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.4.5
depends_on:
dbrepo-search-service:
condition: service_started
......@@ -318,7 +321,7 @@ services:
restart: "no"
container_name: dbrepo-search-service-init
hostname: search-service-init
image: registry.datalab.tuwien.ac.at/dbrepo/search-service-init:1.4.4
image: registry.datalab.tuwien.ac.at/dbrepo/search-service-init:1.4.5
environment:
GATEWAY_SERVICE_ENDPOINT: ${GATEWAY_SERVICE_ENDPOINT:-http://gateway-service}
OPENSEARCH_HOST: ${OPENSEARCH_HOST:-search-db}
......@@ -341,6 +344,8 @@ services:
volumes:
- ./dist/s3_config.json:/app/s3_config.json
- storage-service-data:/data
ports:
- "9000:9000"
healthcheck:
test: echo "cluster.check" | weed shell | grep "checking master.*ok" || exit 1
interval: 10s
......@@ -353,9 +358,10 @@ services:
restart: "no"
container_name: dbrepo-storage-service-init
hostname: storage-service-init
image: registry.datalab.tuwien.ac.at/dbrepo/storage-service-init:1.4.4
image: registry.datalab.tuwien.ac.at/dbrepo/storage-service-init:1.4.5
environment:
SEAWEEDFS_ENDPOINT: "${STORAGE_SEAWEEDFS_ENDPOINT:-storage-service:9333}"
WEED_CLUSTER_SW_MASTER: "${STORAGE_SERVICE_MASTER_ENDPOINT:-storage-service:9333}"
S3_BUCKET: "${S3_BUCKET:-dbrepo}"
depends_on:
dbrepo-storage-service:
condition: service_healthy
......@@ -368,9 +374,11 @@ services:
hostname: upload-service
image: docker.io/tusproject/tusd:v2.4.0
command:
- "--base-path=/api/upload/files/"
- "-behind-proxy"
- "-max-size=2000000000"
- "-base-path=/api/upload/files/"
- "-s3-endpoint=${STORAGE_ENDPOINT:-http://storage-service:9000}"
- "-s3-bucket=dbrepo-upload"
- "-s3-bucket=dbrepo"
environment:
AWS_ACCESS_KEY_ID: "${STORAGE_USERNAME:-seaweedfsadmin}"
AWS_SECRET_ACCESS_KEY: "${STORAGE_PASSWORD:-seaweedfsadmin}"
......@@ -390,24 +398,22 @@ services:
restart: "no"
container_name: dbrepo-data-service
hostname: data-service
image: registry.datalab.tuwien.ac.at/dbrepo/data-service:1.4.4
image: registry.datalab.tuwien.ac.at/dbrepo/data-service:1.4.5
volumes:
- "${SHARED_VOLUME:-/tmp}:/tmp"
environment:
ADMIN_PASSWORD: "${ADMIN_PASSWORD:-admin}"
ADMIN_USERNAME: "${ADMIN_USERNAME:-admin}"
AUTH_SERVICE_ADMIN: ${AUTH_SERVICE_ADMIN:-fda}
AUTH_SERVICE_ADMIN_PASSWORD: ${AUTH_SERVICE_ADMIN_PASSWORD:-fda}
AUTH_SERVICE_ADMIN: ${AUTH_SERVICE_ADMIN:-admin}
AUTH_SERVICE_ADMIN_PASSWORD: ${AUTH_SERVICE_ADMIN_PASSWORD:-admin}
AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client}
AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
AUTH_SERVICE_ENDPOINT: ${AUTH_SERVICE_ENDPOINT:-http://auth-service:8080}
BROKER_EXCHANGE_NAME: ${BROKER_EXCHANGE_NAME:-dbrepo}
BROKER_QUEUE_NAME: ${BROKER_QUEUE_NAME:-dbrepo}
BROKER_HOST: "${BROKER_ENDPOINT:-broker-service}"
BROKER_PASSWORD: ${BROKER_PASSWORD:-fda}
BROKER_PASSWORD: ${BROKER_PASSWORD:-admin}
BROKER_PORT: ${BROKER_PORT:-5672}
BROKER_SERVICE_ENDPOINT: ${BROKER_SERVICE_ENDPOINT:-http://gateway-service/admin/broker}
BROKER_USERNAME: ${BROKER_USERNAME:-fda}
BROKER_USERNAME: ${BROKER_USERNAME:-admin}
BROKER_VIRTUALHOST: "${BROKER_VIRTUALHOST:-dbrepo}"
CONNECTION_TIMEOUT: ${CONNECTION_TIMEOUT:-60000}
EXCHANGE_NAME: ${EXCHANGE_NAME:-dbrepo}
......@@ -422,11 +428,13 @@ services:
REQUEUE_REJECTED: ${REQUEUE_REJECTED:-false}
ROUTING_KEY: "${ROUTING_KEY:-dbrepo.#}"
S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}"
S3_BUCKET: "${S3_BUCKET:-dbrepo}"
S3_ENDPOINT: "${S3_ENDPOINT:-http://storage-service:9000}"
S3_EXPORT_BUCKET: "${S3_EXPORT_BUCKET:-dbrepo-download}"
S3_FILE_PATH: "${S3_FILE_PATH:-/tmp}"
S3_IMPORT_BUCKET: "${S3_IMPORT_BUCKET:-dbrepo-upload}"
S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}"
SYSTEM_USERNAME: "${SYSTEM_USERNAME:-admin}"
SYSTEM_PASSWORD: "${SYSTEM_PASSWORD:-admin}"
healthcheck:
test: curl -sSL localhost:8080/actuator/health/liveness | grep 'UP' || exit 1
interval: 10s
......
.docs/api/analyse-service.md
+ 1
− 1
View file @ 35e69074
......@@ -6,7 +6,7 @@ author: Martin Weise
!!! debug "Debug Information"
Image: [`registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.4.4`](https://hub.docker.com/r/dbrepo/analyse-service)
Image: [`registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.4.5`](https://hub.docker.com/r/dbrepo/analyse-service)
* Ports: 5000/tcp
* Prometheus: `http://<hostname>:5000/metrics`
......
.docs/api/auth-service.md
+ 1
− 1
View file @ 35e69074
......@@ -6,7 +6,7 @@ author: Martin Weise
!!! debug "Debug Information"
Image: [`registry.datalab.tuwien.ac.at/dbrepo/authentication-service:1.4.4`](https://hub.docker.com/r/dbrepo/authentication-service)
Image: [`quay.io/keycloak/keycloak:24.0`](quay.io/keycloak/keycloak)
* Ports: 8080/tcp
* UI: `http://<hostname>/api/auth/`
......
.docs/api/data-db.md
+ 1
− 1
View file @ 35e69074
......@@ -11,7 +11,7 @@ author: Martin Weise
!!! debug "Debug Information"
Image: [`dbrepo/data-db-sidecar:1.4.4`](https://hub.docker.com/r/dbrepo/data-db-sidecar)
Image: [`dbrepo/data-db-sidecar:1.4.5`](https://hub.docker.com/r/dbrepo/data-db-sidecar)
* Ports: 8080/tcp
......
.docs/api/data-service.md
+ 1
− 1
View file @ 35e69074
......@@ -6,7 +6,7 @@ author: Martin Weise
!!! debug "Debug Information"
Image: [`registry.datalab.tuwien.ac.at/dbrepo/data-service:1.4.4`](https://hub.docker.com/r/dbrepo/data-service)
Image: [`registry.datalab.tuwien.ac.at/dbrepo/data-service:1.4.5`](https://hub.docker.com/r/dbrepo/data-service)
* Ports: 9093/tcp
* Info: `http://<hostname>:9093/actuator/info`
......
.docs/api/metadata-service.md
+ 2
− 2
View file @ 35e69074
......@@ -6,7 +6,7 @@ author: Martin Weise
!!! debug "Debug Information"
Image: [`registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.4.4`](https://hub.docker.com/r/dbrepo/metadata-service)
Image: [`registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.4.5`](https://hub.docker.com/r/dbrepo/metadata-service)
* Ports: 9099/tcp
* Info: `http://<hostname>:9099/actuator/info`
......@@ -46,7 +46,7 @@ To activate DOI minting, pass your DataCite Fabrica credentials in the environme
```yaml title="docker-compose.yml"
services:
dbrepo-metadata-service:
image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.4.4
image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.4.5
environment:
spring_profiles_active: doi
DATACITE_URL: https://api.datacite.org
......
.docs/api/python.md
+ 5
− 5
View file @ 35e69074
......@@ -66,17 +66,17 @@ client.import_table_data(database_id=7, table_id=13, file_name_or_data_frame=df)
## Supported Features & Best-Practices
- Manage user
account ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.4/api/#create-user-account))
account ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.5/api/#create-user-account))
- Manage
databases ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo//usage-overview/#create-database))
- Manage database access &
visibility ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.4/api/#create-database))
visibility ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.5/api/#create-database))
- Import
dataset ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.4/api/#import-dataset))
dataset ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.5/api/#import-dataset))
- Create persistent
identifiers ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.4/api/#assign-database-pid))
identifiers ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.5/api/#assign-database-pid))
- Execute
queries ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.4/api/#export-subset))
queries ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.5/api/#export-subset))
- Get data from tables/views/subsets
## Configure
......
.docs/api/search-service.md
+ 1
− 1
View file @ 35e69074
......@@ -6,7 +6,7 @@ author: Martin Weise
!!! debug "Debug Information"
Image: [`registry.datalab.tuwien.ac.at/dbrepo/search-service:1.4.4`](https://hub.docker.com/r/dbrepo/search-service)
Image: [`registry.datalab.tuwien.ac.at/dbrepo/search-service:1.4.5`](https://hub.docker.com/r/dbrepo/search-service)
* Ports: 4000/tcp
* Health: `http://<hostname>:4000/api/search/health`
......
.docs/api/ui.md
+ 2
− 2
View file @ 35e69074
......@@ -6,7 +6,7 @@ author: Martin Weise
!!! debug "Debug Information"
Image: [`registry.datalab.tuwien.ac.at/dbrepo/ui:1.4.4`](https://hub.docker.com/r/dbrepo/ui)
Image: [`registry.datalab.tuwien.ac.at/dbrepo/ui:1.4.5`](https://hub.docker.com/r/dbrepo/ui)
* Ports: 3000/tcp
......@@ -35,7 +35,7 @@ if you use a Kubernetes deployment via ConfigMap and Volumes).
```yaml title="docker-compose.yml"
services:
dbrepo-ui:
image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.4.4
image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.4.5
volumes:
- ./my_logo.png:/app/.output/public/my_logo.png
...
......
.docs/installation.md
+ 9
− 9
View file @ 35e69074
......@@ -11,7 +11,7 @@ author: Martin Weise
If you have [Docker](https://docs.docker.com/engine/install/) already installed on your system, you can install DBRepo with:
```shell
curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.4/install.sh | bash
curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.5/install.sh | bash
```
Or perform a [custom install](#custom-install).
......@@ -47,7 +47,7 @@ are *not* recommended and not tested.
In case you prefer a customized install, start by downloading the `docker-compose.yml` file used to define the services:
```bash
curl -O docker-compose.yml -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.4/.docker/docker-compose.yml
curl -O docker-compose.yml -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.5/.docker/docker-compose.yml
```
Create the folder `dist/` that hold necessary configuration files and download the Metadata Database schema and initial
......@@ -55,16 +55,16 @@ data to display the created Data Database container:
```bash
mkdir -p dist
curl -O dist/setup-schema.sql -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.4/dbrepo-metadata-db/setup-schema.sql
curl -O dist/setup-data.sql -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.4/dbrepo-metadata-db/setup-data.sql
curl -O dist/setup-schema.sql -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.5/dbrepo-metadata-db/setup-schema.sql
curl -O dist/setup-data.sql -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.5/dbrepo-metadata-db/setup-data.sql
```
Download the Broker Service configuration files:
```bash
curl -O dist/rabbitmq.conf -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.4/dbrepo-broker-service/rabbitmq.conf
curl -O dist/enabled_plugins -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.4/dbrepo-broker-service/enabled_plugins
curl -O dist/definitions.json -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.4/dbrepo-broker-service/definitions.json
curl -O dist/rabbitmq.conf -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.5/dbrepo-broker-service/rabbitmq.conf
curl -O dist/enabled_plugins -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.5/dbrepo-broker-service/enabled_plugins
curl -O dist/definitions.json -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.5/dbrepo-broker-service/definitions.json
```
!!! warning "Default admin user credentials"
......@@ -76,13 +76,13 @@ curl -O dist/definitions.json -sSL https://gitlab.phaidra.org/fair-data-austria-
Download the Gateway Service configuration file (or integrate it into your existing NGINX reverse proxy config):
```bash
curl -O dist/dbrepo.conf -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.4/dbrepo-gateway-service/dbrepo.conf
curl -O dist/dbrepo.conf -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.5/dbrepo-gateway-service/dbrepo.conf
```
Download the S3 configuration for the Storage Service:
```bash
curl -O dist/s3_config.conf -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.4/dbrepo-storage-service/s3_config.conf
curl -O dist/s3_config.conf -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.5/dbrepo-storage-service/s3_config.conf
```
Continue the custom install by customizing the [User Interface](../api/ui).
......
.docs/kubernetes.md
+ 3
− 3
View file @ 35e69074
......@@ -7,7 +7,7 @@ author: Martin Weise
## TL;DR
To install DBRepo in your existing cluster, download the
sample [`values.yaml`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.4.4/helm/dbrepo/values.yaml)
sample [`values.yaml`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.4.5/helm/dbrepo/values.yaml)
for your deployment and update the variables, especially `hostname`.
```shell
......@@ -15,7 +15,7 @@ helm upgrade --install dbrepo \
-n dbrepo \
"oci://registry.datalab.tuwien.ac.at/dbrepo/helm/dbrepo" \
--values ./values.yaml \
--version "1.4.4" \
--version "1.4.5" \
--create-namespace \
--cleanup-on-fail
```
......@@ -36,7 +36,7 @@ brokerservice:
The `brokerservice.auth.passwordHash` field is the RabbitMQ SHA512-hash of the `brokerservice.auth.password` field and
can be obtained with
the [`generate-rabbitmq-pw.sh`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.4.4/helm/dbrepo/hack/generate-rabbitmq-pw.sh)
the [`generate-rabbitmq-pw.sh`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.4.5/helm/dbrepo/hack/generate-rabbitmq-pw.sh)
script:
```console
......
README.md
+ 1
− 0
View file @ 35e69074
......@@ -5,6 +5,7 @@
![OpenSearch 2.10](https://img.shields.io/badge/OpenSearch-2.10-white?style=flat)
![SeaweedFS 3.59](https://img.shields.io/badge/SeaweedFS-3.59-white?style=flat)
![OpenLDAP 2.6](https://img.shields.io/badge/OpenLDAP-2.6-white?style=flat)
![Keycloak 24.0](https://img.shields.io/badge/Keycloak-24.0-white?style=flat)
<img src="./dbrepo-ui/public/logo.png" width="200" alt="DBRepo &mdash; Repository for Data in Databases" />
......
dbrepo-analyse-service/app.py
+ 1
− 2
View file @ 35e69074
......@@ -186,9 +186,8 @@ app.config["AUTH_SERVICE_ENDPOINT"] = os.getenv("AUTH_SERVICE_ENDPOINT", "http:/
app.config["AUTH_SERVICE_CLIENT"] = os.getenv("AUTH_SERVICE_CLIENT", "dbrepo-client")
app.config["AUTH_SERVICE_CLIENT_SECRET"] = os.getenv("AUTH_SERVICE_CLIENT_SECRET", "MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG")
app.config["S3_ACCESS_KEY_ID"] = os.getenv('S3_ACCESS_KEY_ID', 'seaweedfsadmin')
app.config["S3_BUCKET"] = os.getenv('S3_BUCKET', 'dbrepo')
app.config["S3_ENDPOINT"] = os.getenv('S3_ENDPOINT', 'http://localhost:9000')
app.config["S3_EXPORT_BUCKET"] = os.getenv('S3_EXPORT_BUCKET', 'dbrepo-download')
app.config["S3_IMPORT_BUCKET"] = os.getenv('S3_IMPORT_BUCKET', 'dbrepo-upload')
app.config["S3_SECRET_ACCESS_KEY"] = os.getenv('S3_SECRET_ACCESS_KEY', 'seaweedfsadmin')
app.json_encoder = LazyJSONEncoder
......
dbrepo-analyse-service/clients/s3_client.py
+ 3
− 4
View file @ 35e69074
......@@ -17,10 +17,9 @@ class S3Client:
endpoint_url, aws_access_key_id)
self.client = boto3.client(service_name='s3', endpoint_url=endpoint_url, aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
self.bucket_exists_or_exit(current_app.config['S3_EXPORT_BUCKET'])
self.bucket_exists_or_exit(current_app.config['S3_IMPORT_BUCKET'])
self.bucket_exists_or_exit(current_app.config['S3_BUCKET'])
def upload_file(self, filename: str, path: str = "/tmp", bucket: str = "dbrepo-upload") -> bool:
def upload_file(self, filename: str, path: str = "/tmp", bucket: str = "dbrepo") -> bool:
"""
Uploads a file to the blob storage.
Follows the official API https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html.
......@@ -43,7 +42,7 @@ class S3Client:
logging.warning(f"Failed to upload file with key {filename}")
raise ConnectionRefusedError(f"Failed to upload file with key {filename}", e)
def download_file(self, filename: str, path: str = "/tmp", bucket: str = "dbrepo-download"):
def download_file(self, filename: str, path: str = "/tmp", bucket: str = "dbrepo"):
"""
Downloads a file from the blob storage.
Follows the official API https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-example-download-file.html
......
dbrepo-analyse-service/determine_dt.py
+ 2
− 2
View file @ 35e69074
......@@ -18,8 +18,8 @@ def determine_datatypes(filename, enum=False, enum_tol=0.0001, separator=None) -
# Enum is not SQL standard, hence, it might not be supported by all db-engines.
# However, it can be used in Postgres and MySQL.
s3_client = S3Client()
s3_client.file_exists(current_app.config['S3_IMPORT_BUCKET'], filename)
response = s3_client.get_file(current_app.config['S3_IMPORT_BUCKET'], filename)
s3_client.file_exists(current_app.config['S3_BUCKET'], filename)
response = s3_client.get_file(current_app.config['S3_BUCKET'], filename)
stream = response['Body']
if response['ContentLength'] == 0:
logging.warning(f'Failed to determine data types: file {filename} has empty body')
......
dbrepo-analyse-service/determine_pk.py
+ 2
− 2
View file @ 35e69074
......@@ -16,8 +16,8 @@ def determine_pk(filename, separator=","):
colindex = list(range(0, len(colnames)))
s3_client = S3Client()
s3_client.file_exists('dbrepo-upload', filename)
response = s3_client.get_file('dbrepo-upload', filename)
s3_client.file_exists('dbrepo', filename)
response = s3_client.get_file('dbrepo', filename)
stream = response['Body']
if response['ContentLength'] == 0:
raise OSError(f'Failed to determine primary key: file {filename} has empty body')
......
dbrepo-analyse-service/test/conftest.py
+ 11
− 14
View file @ 35e69074
......@@ -41,9 +41,8 @@ def session(request, app_context):
app.config["S3_ENDPOINT"] = endpoint
client = container.get_client()
# create buckets
logging.debug("[fixture] make buckets dbrepo-upload, dbrepo-download")
client.make_bucket("dbrepo-upload")
client.make_bucket("dbrepo-download")
logging.debug("[fixture] make bucket dbrepo")
client.make_bucket("dbrepo")
# destructor
def stop_minio():
......@@ -61,17 +60,15 @@ def cleanup(request, session):
:param session: /
:return:
"""
logging.info("[fixture] truncate buckets")
for bucket in ["dbrepo-upload", "dbrepo-download"]:
bucket = "dbrepo"
logging.info(f"[fixture] truncate bucket: {bucket}")
objects = []
for obj in session.get_client().list_objects(bucket):
objects.append(DeleteObject(obj.object_name))
logging.info(f"request to remove objects {objects}")
errors = session.get_client().remove_objects(bucket, objects)
for error in errors:
raise ConnectionError(
f"Failed to delete object with key {error.object_name} of bucket {bucket}"
)
raise ConnectionError(f"Failed to delete object with key {error.object_name} of bucket: {bucket}")
@pytest.fixture(scope="function")
......
dbrepo-analyse-service/test/test_determine_dt.py
+ 9
− 9
View file @ 35e69074
......@@ -24,7 +24,7 @@ class DetermineDatatypesTest(unittest.TestCase):
}
# mock
S3Client().upload_file("datetime.csv", './data/test_dt/', 'dbrepo-upload')
S3Client().upload_file("datetime.csv", './data/test_dt/', 'dbrepo')
# test
response = determine_datatypes(filename="datetime.csv", separator=",")
......@@ -47,7 +47,7 @@ class DetermineDatatypesTest(unittest.TestCase):
}
# mock
S3Client().upload_file("datetime_tz.csv", './data/test_dt/', 'dbrepo-upload')
S3Client().upload_file("datetime_tz.csv", './data/test_dt/', 'dbrepo')
# test
response = determine_datatypes(filename="datetime_tz.csv", separator=",")
......@@ -70,7 +70,7 @@ class DetermineDatatypesTest(unittest.TestCase):
}
# mock
S3Client().upload_file("datetime_t.csv", './data/test_dt/', 'dbrepo-upload')
S3Client().upload_file("datetime_t.csv", './data/test_dt/', 'dbrepo')
# test
response = determine_datatypes(filename="datetime_t.csv", separator=",")
......@@ -94,7 +94,7 @@ class DetermineDatatypesTest(unittest.TestCase):
}
# mock
S3Client().upload_file("datatypes.csv", './data/test_dt/', 'dbrepo-upload')
S3Client().upload_file("datatypes.csv", './data/test_dt/', 'dbrepo')
# test
response = determine_datatypes(filename="datatypes.csv", separator=",")
......@@ -117,7 +117,7 @@ class DetermineDatatypesTest(unittest.TestCase):
def test_determine_datatypes_fileEmpty_succeeds(self):
# mock
S3Client().upload_file("empty.csv", './data/test_dt/', 'dbrepo-upload')
S3Client().upload_file("empty.csv", './data/test_dt/', 'dbrepo')
# test
response = determine_datatypes("empty.csv")
......@@ -129,7 +129,7 @@ class DetermineDatatypesTest(unittest.TestCase):
def test_determine_datatypes_separatorSemicolon_succeeds(self):
# mock
S3Client().upload_file("separator.csv", './data/test_dt/', 'dbrepo-upload')
S3Client().upload_file("separator.csv", './data/test_dt/', 'dbrepo')
# test
response = determine_datatypes(filename="separator.csv", separator=";")
......@@ -140,7 +140,7 @@ class DetermineDatatypesTest(unittest.TestCase):
def test_determine_datatypes_separatorGuess_succeeds(self):
# mock
S3Client().upload_file("separator.csv", './data/test_dt/', 'dbrepo-upload')
S3Client().upload_file("separator.csv", './data/test_dt/', 'dbrepo')
# test
response = determine_datatypes(filename="separator.csv")
......@@ -151,7 +151,7 @@ class DetermineDatatypesTest(unittest.TestCase):
def test_determine_datatypes_separatorGuessLargeDataset_succeeds(self):
# mock
S3Client().upload_file("large.csv", './data/test_dt/', 'dbrepo-upload')
S3Client().upload_file("large.csv", './data/test_dt/', 'dbrepo')
# test
response = determine_datatypes(filename="large.csv")
......@@ -171,7 +171,7 @@ class DetermineDatatypesTest(unittest.TestCase):
}
# mock
S3Client().upload_file("novel.csv", './data/test_dt/', 'dbrepo-upload')
S3Client().upload_file("novel.csv", './data/test_dt/', 'dbrepo')
# test
response = determine_datatypes(filename="novel.csv", separator=";")
......
dbrepo-analyse-service/test/test_determine_pk.py
+ 8
− 8
View file @ 35e69074
......@@ -7,7 +7,7 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test
def test_determine_pk_largeFileIdFirst_succeeds(self):
# mock
S3Client().upload_file("largefile_idfirst.csv", './data/test_pk/', 'dbrepo-upload')
S3Client().upload_file("largefile_idfirst.csv", './data/test_pk/', 'dbrepo')
# test
response = determine_pk('largefile_idfirst.csv')
......@@ -16,7 +16,7 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test
def test_determine_pk_largeFileIdInBetween_succeeds(self):
# mock
S3Client().upload_file("largefile_idinbtw.csv", './data/test_pk/', 'dbrepo-upload')
S3Client().upload_file("largefile_idinbtw.csv", './data/test_pk/', 'dbrepo')
# test
response = determine_pk('largefile_idinbtw.csv')
......@@ -25,7 +25,7 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test
def test_determine_pk_largeFileNoPrimaryKey_fails(self):
# mock
S3Client().upload_file("largefile_no_pk.csv", './data/test_pk/', 'dbrepo-upload')
S3Client().upload_file("largefile_no_pk.csv", './data/test_pk/', 'dbrepo')
# test
response = determine_pk('largefile_no_pk.csv')
......@@ -34,7 +34,7 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test
def test_determine_pk_largeFileNullInUnique_fails(self):
# mock
S3Client().upload_file("largefile_nullinunique.csv", './data/test_pk/', 'dbrepo-upload')
S3Client().upload_file("largefile_nullinunique.csv", './data/test_pk/', 'dbrepo')
# test
response = determine_pk('largefile_nullinunique.csv')
......@@ -43,7 +43,7 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test
def test_determine_pk_smallFileIdFirst_fails(self):
# mock
S3Client().upload_file("smallfile_idfirst.csv", './data/test_pk/', 'dbrepo-upload')
S3Client().upload_file("smallfile_idfirst.csv", './data/test_pk/', 'dbrepo')
# test
response = determine_pk('smallfile_idfirst.csv')
......@@ -52,7 +52,7 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test
def test_determine_pk_smallFileIdIntBetween_fails(self):
# mock
S3Client().upload_file("smallfile_idinbtw.csv", './data/test_pk/', 'dbrepo-upload')
S3Client().upload_file("smallfile_idinbtw.csv", './data/test_pk/', 'dbrepo')
# test
response = determine_pk('smallfile_idinbtw.csv')
......@@ -61,7 +61,7 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test
def test_determine_pk_smallFileNoPrimaryKey_fails(self):
# mock
S3Client().upload_file("smallfile_no_pk.csv", './data/test_pk/', 'dbrepo-upload')
S3Client().upload_file("smallfile_no_pk.csv", './data/test_pk/', 'dbrepo')
# test
response = determine_pk('smallfile_no_pk.csv')
......@@ -70,7 +70,7 @@ class DeterminePrimaryKeyTest(unittest.TestCase):
# @Test
def test_determine_pk_smallFileNullInUnique_fails(self):
# mock
S3Client().upload_file("smallfile_nullinunique.csv", './data/test_pk/', 'dbrepo-upload')
S3Client().upload_file("smallfile_nullinunique.csv", './data/test_pk/', 'dbrepo')
# test
response = determine_pk('smallfile_nullinunique.csv')
......
dbrepo-analyse-service/test/test_s3_client.py
+ 7
− 7
View file @ 35e69074
......@@ -43,17 +43,17 @@ class S3ClientTest(unittest.TestCase):
def test_download_file_succeeds(self):
# mock
S3Client().upload_file(filename="testdt01.csv", path="./data/", bucket="dbrepo-upload")
S3Client().upload_file(filename="testdt01.csv", path="./data/", bucket="dbrepo")
# test
S3Client().download_file(filename="testdt01.csv", bucket="dbrepo-upload")
S3Client().download_file(filename="testdt01.csv", bucket="dbrepo")
# @Test
def test_download_file_notFound_fails(self):
# test
try:
S3Client().download_file(filename="testdt01.csv", bucket="dbrepo-upload")
S3Client().download_file(filename="testdt01.csv", bucket="dbrepo")
except ClientError:
pass
except Exception:
......@@ -78,10 +78,10 @@ class S3ClientTest(unittest.TestCase):
def test_get_file_succeeds(self):
# mock
S3Client().upload_file(filename="testdt01.csv", path="./data/", bucket="dbrepo-upload")
S3Client().upload_file(filename="testdt01.csv", path="./data/", bucket="dbrepo")
# test
response = S3Client().get_file(bucket="dbrepo-upload", filename="testdt01.csv")
response = S3Client().get_file(bucket="dbrepo", filename="testdt01.csv")
self.assertIsNotNone(response)
# @Test
......@@ -89,7 +89,7 @@ class S3ClientTest(unittest.TestCase):
# test
try:
S3Client().get_file(bucket="dbrepo-upload", filename="idonotexist.csv")
S3Client().get_file(bucket="dbrepo", filename="idonotexist.csv")
except ClientError:
pass
except Exception:
......@@ -101,7 +101,7 @@ class S3ClientTest(unittest.TestCase):
def test_bucket_exists_succeeds(self):
# test
response = S3Client().bucket_exists_or_exit("dbrepo-upload")
response = S3Client().bucket_exists_or_exit("dbrepo")
self.assertIsNotNone(response)
# @Test
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment