diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml index d51b00551a3a6af30bbb7d454f6fea00c17e7d65..a7496bc4f47838b9944b1c844a3b7d98e32cf1f3 100644 --- a/.docker/docker-compose.yml +++ b/.docker/docker-compose.yml @@ -14,10 +14,11 @@ services: restart: "no" container_name: dbrepo-metadata-db hostname: metadata-db - image: docker.io/dbrepo/metadata-db:1.4.4 + image: docker.io/bitnami/mariadb:11.1.3-debian-11-r6 volumes: - metadata-db-data:/bitnami/mariadb - - ./dist/2_setup-data.sql:/docker-entrypoint-initdb.d/2_setup-data.sql + - ./dbrepo-metadata-db/setup-schema.sql:/docker-entrypoint-initdb.d/1_setup-schema.sql + - ./dbrepo-metadata-db/setup-data.sql:/docker-entrypoint-initdb.d/2_setup-data.sql ports: - "3306:3306" environment: @@ -35,7 +36,7 @@ services: restart: "no" container_name: dbrepo-data-db hostname: data-db - image: docker.io/bitnami/mariadb-galera:11.2.2-debian-11-r0 + image: docker.io/bitnami/mariadb:11.1.3-debian-11-r6 volumes: - data-db-data:/bitnami/mariadb - "${SHARED_VOLUME:-/tmp}:/tmp" @@ -43,7 +44,6 @@ services: - "3307:3306" environment: MARIADB_ROOT_PASSWORD: "${USER_DB_PASSWORD:-dbrepo}" - MARIADB_GALERA_MARIABACKUP_PASSWORD: "${USER_DB_BACKUP_PASSWORD:-dbrepo}" healthcheck: test: mysqladmin ping --user="${USER_DB_USERNAME:-root}" --password="${USER_DB_PASSWORD:-dbrepo}" --silent interval: 10s @@ -56,7 +56,7 @@ services: restart: "no" container_name: dbrepo-auth-db hostname: auth-db - image: docker.io/bitnami/mariadb:11.2.2-debian-11-r0 + image: docker.io/bitnami/mariadb:11.1.3-debian-11-r6 volumes: - auth-db-data:/bitnami/mariadb ports: @@ -76,7 +76,7 @@ services: restart: "no" container_name: dbrepo-auth-service hostname: auth-service - image: docker.io/dbrepo/auth-service:1.4.4 + image: registry.datalab.tuwien.ac.at/dbrepo/auth-service:1.4.4 healthcheck: test: curl -sSL 'http://0.0.0.0:8080/realms/dbrepo' | grep "dbrepo" || exit 1 interval: 10s @@ -98,7 +98,7 @@ services: restart: "no" container_name: dbrepo-metadata-service hostname: metadata-service - image: docker.io/dbrepo/metadata-service:1.4.4 + image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.4.4 volumes: - "${SHARED_VOLUME:-/tmp}:/tmp" environment: @@ -124,7 +124,7 @@ services: DELETED_RECORD: "${DELETED_RECORD:-persistent}" GRANULARITY: "${GRANULARITY:-YYYY-MM-DDThh:mm:ssZ}" JWT_PUBKEY: "${JWT_PUBKEY:-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB}" - LOG_LEVEL: "${LOG_LEVEL:-info}" + LOG_LEVEL: ${LOG_LEVEL:-info} METADATA_DB: "${METADATA_DB:-dbrepo}" METADATA_HOST: "${METADATA_HOST:-metadata-db}" METADATA_JDBC_EXTRA_ARGS: "${METADATA_JDBC_EXTRA_ARGS:-}" @@ -134,13 +134,13 @@ services: REPOSITORY_NAME: "${REPOSITORY_NAME:-Database Repository}" SEARCH_SERVICE_ENDPOINT: "${SEARCH_SERVICE_ENDPOINT:-http://gateway-service}" S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}" - S3_ENDPOINT: "${S3_ENDPOINT:-http://gateway-service/api/storage}" + S3_ENDPOINT: "${S3_ENDPOINT:-http://storage-service:9000}" S3_EXPORT_BUCKET: "${S3_EXPORT_BUCKET:-dbrepo-download}" S3_IMPORT_BUCKET: "${S3_IMPORT_BUCKET:-dbrepo-upload}" S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}" SPARQL_CONNECTION_TIMEOUT: "${SPARQL_CONNECTION_TIMEOUT:-10000}" healthcheck: - test: wget -qO- localhost:8080/actuator/health/readiness | grep -q "UP" || exit 1 + test: curl -sSL localhost:8080/actuator/health/liveness | grep 'UP' || exit 1 interval: 10s timeout: 5s retries: 12 @@ -160,7 +160,7 @@ services: restart: "no" container_name: dbrepo-analyse-service hostname: analyse-service - image: docker.io/dbrepo/analyse-service:1.4.4 + image: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.4.4 environment: ADMIN_PASSWORD: "${ADMIN_PASSWORD:-admin}" ADMIN_USERNAME: "${ADMIN_USERNAME:-admin}" @@ -211,7 +211,7 @@ services: restart: "no" container_name: dbrepo-search-db hostname: search-db - image: docker.io/dbrepo/search-db:1.4.4 + image: registry.datalab.tuwien.ac.at/dbrepo/search-db:1.4.4 healthcheck: test: curl -sSL localhost:9200/_plugins/_security/health | jq .status | grep UP interval: 10s @@ -235,7 +235,7 @@ services: restart: "no" container_name: dbrepo-search-service hostname: search-service - image: docker.io/dbrepo/search-service:1.4.4 + image: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.4.4 environment: ADMIN_PASSWORD: "${ADMIN_PASSWORD:-admin}" ADMIN_USERNAME: "${ADMIN_USERNAME:-admin}" @@ -243,6 +243,7 @@ services: AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG} AUTH_SERVICE_ENDPOINT: ${AUTH_SERVICE_ENDPOINT:-http://auth-service:8080} COLLECTION: ${COLLECTION:-['database','table','column','identifier','unit','concept','user','view']} + GATEWAY_SERVICE_ENDPOINT: ${GATEWAY_SERVICE_ENDPOINT:-http://gateway-service} OPENSEARCH_HOST: ${OPENSEARCH_HOST:-search-db} OPENSEARCH_PORT: ${OPENSEARCH_PORT:-9200} OPENSEARCH_USERNAME: ${OPENSEARCH_USERNAME:-admin} @@ -253,7 +254,7 @@ services: restart: "no" container_name: dbrepo-data-db-sidecar hostname: data-db-sidecar - image: docker.io/dbrepo/data-db-sidecar:1.4.4 + image: registry.datalab.tuwien.ac.at/dbrepo/data-db-sidecar:1.4.4 environment: S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}" S3_ENDPOINT: "${S3_ENDPOINT:-http://storage-service:9000}" @@ -275,7 +276,7 @@ services: restart: "no" container_name: dbrepo-ui hostname: ui - image: docker.io/dbrepo/ui:1.4.4 + image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.4.4 depends_on: dbrepo-search-service: condition: service_started @@ -293,7 +294,7 @@ services: restart: "no" container_name: dbrepo-gateway-service hostname: gateway-service - image: docker.io/nginx:1.25-alpine-slim + image: docker.io/nginx:1.27.0-alpine3.19-slim ports: - "80:80" - "443:443" @@ -319,7 +320,7 @@ services: restart: "no" container_name: dbrepo-search-service-init hostname: search-service-init - image: docker.io/dbrepo/search-service-init:1.4.4 + image: registry.datalab.tuwien.ac.at/dbrepo/search-service-init:1.4.4 environment: GATEWAY_SERVICE_ENDPOINT: ${GATEWAY_SERVICE_ENDPOINT:-http://gateway-service} OPENSEARCH_HOST: ${OPENSEARCH_HOST:-search-db} @@ -354,7 +355,7 @@ services: restart: "no" container_name: dbrepo-storage-service-init hostname: storage-service-init - image: docker.io/dbrepo/storage-service-init:1.4.4 + image: registry.datalab.tuwien.ac.at/dbrepo/storage-service-init:1.4.4 environment: SEAWEEDFS_ENDPOINT: "${STORAGE_SEAWEEDFS_ENDPOINT:-storage-service:9333}" depends_on: @@ -391,7 +392,7 @@ services: restart: "no" container_name: dbrepo-data-service hostname: data-service - image: docker.io/dbrepo/data-service:1.4.4 + image: registry.datalab.tuwien.ac.at/dbrepo/data-service:1.4.4 volumes: - "${SHARED_VOLUME:-/tmp}:/tmp" environment: @@ -429,7 +430,7 @@ services: S3_IMPORT_BUCKET: "${S3_IMPORT_BUCKET:-dbrepo-upload}" S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}" healthcheck: - test: wget -qO- localhost:8080/actuator/health/readiness | grep -q "UP" || exit 1 + test: curl -sSL localhost:8080/actuator/health/liveness | grep 'UP' || exit 1 interval: 10s timeout: 5s retries: 12 diff --git a/.docs/deployment-docker-compose.md b/.docs/installation.md similarity index 77% rename from .docs/deployment-docker-compose.md rename to .docs/installation.md index 870ea0c142a07af408176625dae5225f7e4a637d..2561b4944b65de05a6f79b55502ad2d61ad15906 100644 --- a/.docs/deployment-docker-compose.md +++ b/.docs/installation.md @@ -2,7 +2,9 @@ author: Martin Weise --- -# Docker Compose +# Installation + +[](https://hub.docker.com/u/dbrepo){ tabindex=-1 } ## TL;DR @@ -12,6 +14,8 @@ If you have [Docker](https://docs.docker.com/engine/install/) already installed curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.4/install.sh | bash ``` +Or perform a [custom install](#custom-install). + ## Requirements ### Hardware @@ -26,7 +30,7 @@ the following settings. *Optional*: public IP-address if you want to secure the deployment with a (free) TLS-certificate from Let's Encrypt. -!!! tip "Resource Consumption" +!!! info "Resource Consumption" Note that most of the vCPU and RAM resources will be needed for starting the infrastructure, this is because of Docker. During operation and especially idle times, the deployment will use significantly less resources. @@ -38,9 +42,11 @@ official [Docker Engine](https://docs.docker.com/engine/install/debian/) install a [Debian](https://www.debian.org/)-based operating system. Other software deployments (e.g. Docker Desktop on Windows) are *not* recommended and not tested. -## Architecture +## Custom Install -### Overview +TBD + +## Architecture The repository is designed as a service-based architecture to ensure scalability and the utilization of various technologies. The conceptualized microservices operate the basic database operations, data versioning as well as @@ -51,8 +57,6 @@ technologies. The conceptualized microservices operate the basic database operat <figcaption>Architecture of the services deployed via Docker Compose</figcaption> </figure> -### Notes - Please note that we only save the state of the databases as well as the [Broker Service](../broker-service) since RabbitMQ maintains state inside the container. @@ -61,43 +65,6 @@ since RabbitMQ maintains state inside the container. We maintain a rapid prototype deployment option through Docker Compose (v2.17.0 and newer). This deployment creates the core infrastructure and a single Docker container for all user-generated databases. -=== "Linux" - - Download and install [Docker Engine](https://docs.docker.com/desktop/install/linux-install/) for your Linux - distribution. Although the installation might work, we *do not* recommend Docker Desktop. - - Ensure the Docker daemon is running at all times: - - systemctl enable docker --now - - Install DBRepo with the default configuration: - - curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/install.sh | bash - -=== "Windows" - - Open `cmd.exe` as administrator and install WSL2 and the Debian subsystem: - - wsl --install Debian - - Open `optionalfeatures` by typing into the open terminal window or searching for it and enable "Windows Subsystem - for Linux": - - <figure markdown> - { .img-border } - <figcaption>Enable Subsystem for Linux in Windows Features</figcaption> - </figure> - - Install [Docker Desktop](https://docs.docker.com/desktop/install/windows-install/) on the Windows host machine. - Open Docker Desktop and go to settings (:fontawesome-solid-gear:) > General > Tick "Use WSL2 based engine" if not - already ticked. - - Open the Debian container by typing "Debian" into the search, you should see a terminal window. - - Install DBRepo with the default configuration from the Debian container: - - curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/master/install.sh | bash - View the logs: docker compose logs -f diff --git a/.docs/deployment-helm.md b/.docs/kubernetes.md similarity index 63% rename from .docs/deployment-helm.md rename to .docs/kubernetes.md index 86f1257f1dc01c8c3f2fc9bd307afdec24c05a7f..608502e320b56ac829d92b781ea19c8128b927c5 100644 --- a/.docs/deployment-helm.md +++ b/.docs/kubernetes.md @@ -21,7 +21,34 @@ helm upgrade --install dbrepo \ ``` This chart is also on [Artifact Hub](https://artifacthub.io/packages/helm/dbrepo/dbrepo) with a full documentation -about values, etc. +about values, etc. Before installing, you need to change credentials, e.g. the Broker Service administrator user +password: + +```yaml title="values.yaml" +brokerservice: + ... + auth: + ... + username: broker + password: broker + passwordHash: 1gwjNNTBPKLgyzbsUykfR0JIFC6nNqbNJaxzZ14uPT8JGcTZ +``` + +The `brokerservice.auth.passwordHash` field is the RabbitMQ SHA512-hash of the `brokerservice.auth.password` field and +can be obtained with +the [`generate-rabbitmq-pw.sh`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.4.4/helm/dbrepo/hack/generate-rabbitmq-pw.sh) +script: + +```console +$ ./generate-rabbitmq-pw.sh my_password +klPdmv4dgnRH64czHolIHAfXvc0G9hc24FQmPlI6eeI1NOf9 +``` + +The script needs the package `xxd` for generation of the random salt. If you don't have `xxd` installed, install it: + +* Debian/Ubuntu: `apt install xxd` +* Windows: `choco install xxd` +* MacOS: `brew install coreutils` ## Prerequisites diff --git a/helm/dbrepo/hack/generate-rabbitmq-pw.sh b/helm/dbrepo/hack/generate-rabbitmq-pw.sh new file mode 100755 index 0000000000000000000000000000000000000000..4bccc93d7b0b9dc8a9e8b4116bcd6826f83f3f71 --- /dev/null +++ b/helm/dbrepo/hack/generate-rabbitmq-pw.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# https://stackoverflow.com/a/53175209/2634294 +# THIS SCRIPT REQUIRES xxd TO BE INSTALLED: +# DEBIAN: apt install xxd +# MACOS: brew install coreutils +function encode_password() +{ + SALT=$(od -A n -t x -N 4 /dev/urandom) + PASS=$SALT$(echo -n $1 | xxd -ps | tr -d '\n' | tr -d ' ') + PASS=$(echo -n $PASS | xxd -r -p | sha256sum | head -c 128) + PASS=$(echo -n $SALT$PASS | xxd -r -p | base64 | tr -d '\n') + echo $PASS +} +encode_password $1 \ No newline at end of file diff --git a/helm/dbrepo/templates/broker-secret.yaml b/helm/dbrepo/templates/broker-secret.yaml index 9291cdbead49275baa472b9aecd9f7a83dc407d2..84a3088663e0eae7f937781f453fafa873437d10 100644 --- a/helm/dbrepo/templates/broker-secret.yaml +++ b/helm/dbrepo/templates/broker-secret.yaml @@ -34,7 +34,7 @@ stringData: { "configure": ".*", "read": ".*", - "user": "broker", + "user": "{{ .Values.brokerservice.auth.username }}", "vhost": "dbrepo", "write": ".*" } @@ -60,8 +60,8 @@ stringData: { "hashing_algorithm": "rabbit_password_hashing_sha256", "limits": {}, - "name": "broker", - "password_hash": "Sek6WxpX2L6UhxlwRkD0cnYAH5GbtTcCFq1yY/SCc1mAa0gB", + "name": "{{ .Values.brokerservice.auth.username }}", + "password_hash": "{{ .Values.brokerservice.auth.passwordHash }}", "tags": [ "administrator" ] diff --git a/helm/dbrepo/values.yaml b/helm/dbrepo/values.yaml index eba2674db01a2f239b8df0393494b584af4b1aea..1942c37da95ec8af21990235806a61d8a4819781 100644 --- a/helm/dbrepo/values.yaml +++ b/helm/dbrepo/values.yaml @@ -323,6 +323,7 @@ brokerservice: existingSecret: ingress-cert username: broker password: broker + passwordHash: 1gwjNNTBPKLgyzbsUykfR0JIFC6nNqbNJaxzZ14uPT8JGcTZ extraConfiguration: |- default_vhost = dbrepo default_user_tags.administrator = true diff --git a/install.sh b/install.sh index 9850ccd35eaae4c71032655bbe72e97892aba0f1..a11a7c3b2380bca183e2a7987df405eef8212259 100644 --- a/install.sh +++ b/install.sh @@ -1,7 +1,7 @@ #!/bin/bash # preset -VERSION="latest" +VERSION="1.4.4" MIN_CPU=8 MIN_RAM=8 MIN_MAP_COUNT=262144 @@ -59,7 +59,8 @@ fi echo "[🚀] Gathering environment ..." mkdir -p ./dist curl -sSL -o ./docker-compose.yml "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/.docker/docker-compose.yml" -curl -sSL -o ./dist/2_setup-data.sql "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-metadata-db/2_setup-data.sql" +curl -sSL -o ./dist/1_setup-schema.sql "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-metadata-db/setup-schema.sql" +curl -sSL -o ./dist/2_setup-data.sql "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-metadata-db/setup-data.sql" curl -sSL -o ./dist/rabbitmq.conf "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-broker-service/rabbitmq.conf" curl -sSL -o ./dist/enabled_plugins "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-broker-service/enabled_plugins" curl -sSL -o ./dist/cert.pem "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-broker-service/cert.pem" diff --git a/mkdocs.yml b/mkdocs.yml index ada8171b5076dc755d32c4fee00ca586ef997826..e2627e6b0d2368bd47dbb1511f8b7fc8a72a68d4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -10,8 +10,8 @@ nav: - Welcome to DBRepo: index.md - Why use DBRepo: why.md - Help with DBRepo: help.md - - Installation: deployment-docker-compose.md - - Kubernetes: deployment-helm.md + - Installation: installation.md + - Kubernetes: kubernetes.md - Migration Guide: migration.md - contributing.md - Concepts: