From 9a05dea2c12b2afd51627b9008f4e5c1ac6dde97 Mon Sep 17 00:00:00 2001
From: Martin Weise <martin.weise@tuwien.ac.at>
Date: Sat, 15 Jun 2024 12:22:51 +0200
Subject: [PATCH] Added the password hash script and fixed some paths
---
.docker/docker-compose.yml | 41 +++++++-------
...ment-docker-compose.md => installation.md} | 53 ++++---------------
.docs/{deployment-helm.md => kubernetes.md} | 29 +++++++++-
helm/dbrepo/hack/generate-rabbitmq-pw.sh | 14 +++++
helm/dbrepo/templates/broker-secret.yaml | 6 +--
helm/dbrepo/values.yaml | 1 +
install.sh | 5 +-
mkdocs.yml | 4 +-
8 files changed, 82 insertions(+), 71 deletions(-)
rename .docs/{deployment-docker-compose.md => installation.md} (77%)
rename .docs/{deployment-helm.md => kubernetes.md} (63%)
create mode 100755 helm/dbrepo/hack/generate-rabbitmq-pw.sh
diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml
index d51b00551a..a7496bc4f4 100644
--- a/.docker/docker-compose.yml
+++ b/.docker/docker-compose.yml
@@ -14,10 +14,11 @@ services:
restart: "no"
container_name: dbrepo-metadata-db
hostname: metadata-db
- image: docker.io/dbrepo/metadata-db:1.4.4
+ image: docker.io/bitnami/mariadb:11.1.3-debian-11-r6
volumes:
- metadata-db-data:/bitnami/mariadb
- - ./dist/2_setup-data.sql:/docker-entrypoint-initdb.d/2_setup-data.sql
+ - ./dbrepo-metadata-db/setup-schema.sql:/docker-entrypoint-initdb.d/1_setup-schema.sql
+ - ./dbrepo-metadata-db/setup-data.sql:/docker-entrypoint-initdb.d/2_setup-data.sql
ports:
- "3306:3306"
environment:
@@ -35,7 +36,7 @@ services:
restart: "no"
container_name: dbrepo-data-db
hostname: data-db
- image: docker.io/bitnami/mariadb-galera:11.2.2-debian-11-r0
+ image: docker.io/bitnami/mariadb:11.1.3-debian-11-r6
volumes:
- data-db-data:/bitnami/mariadb
- "${SHARED_VOLUME:-/tmp}:/tmp"
@@ -43,7 +44,6 @@ services:
- "3307:3306"
environment:
MARIADB_ROOT_PASSWORD: "${USER_DB_PASSWORD:-dbrepo}"
- MARIADB_GALERA_MARIABACKUP_PASSWORD: "${USER_DB_BACKUP_PASSWORD:-dbrepo}"
healthcheck:
test: mysqladmin ping --user="${USER_DB_USERNAME:-root}" --password="${USER_DB_PASSWORD:-dbrepo}" --silent
interval: 10s
@@ -56,7 +56,7 @@ services:
restart: "no"
container_name: dbrepo-auth-db
hostname: auth-db
- image: docker.io/bitnami/mariadb:11.2.2-debian-11-r0
+ image: docker.io/bitnami/mariadb:11.1.3-debian-11-r6
volumes:
- auth-db-data:/bitnami/mariadb
ports:
@@ -76,7 +76,7 @@ services:
restart: "no"
container_name: dbrepo-auth-service
hostname: auth-service
- image: docker.io/dbrepo/auth-service:1.4.4
+ image: registry.datalab.tuwien.ac.at/dbrepo/auth-service:1.4.4
healthcheck:
test: curl -sSL 'http://0.0.0.0:8080/realms/dbrepo' | grep "dbrepo" || exit 1
interval: 10s
@@ -98,7 +98,7 @@ services:
restart: "no"
container_name: dbrepo-metadata-service
hostname: metadata-service
- image: docker.io/dbrepo/metadata-service:1.4.4
+ image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.4.4
volumes:
- "${SHARED_VOLUME:-/tmp}:/tmp"
environment:
@@ -124,7 +124,7 @@ services:
DELETED_RECORD: "${DELETED_RECORD:-persistent}"
GRANULARITY: "${GRANULARITY:-YYYY-MM-DDThh:mm:ssZ}"
JWT_PUBKEY: "${JWT_PUBKEY:-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB}"
- LOG_LEVEL: "${LOG_LEVEL:-info}"
+ LOG_LEVEL: ${LOG_LEVEL:-info}
METADATA_DB: "${METADATA_DB:-dbrepo}"
METADATA_HOST: "${METADATA_HOST:-metadata-db}"
METADATA_JDBC_EXTRA_ARGS: "${METADATA_JDBC_EXTRA_ARGS:-}"
@@ -134,13 +134,13 @@ services:
REPOSITORY_NAME: "${REPOSITORY_NAME:-Database Repository}"
SEARCH_SERVICE_ENDPOINT: "${SEARCH_SERVICE_ENDPOINT:-http://gateway-service}"
S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}"
- S3_ENDPOINT: "${S3_ENDPOINT:-http://gateway-service/api/storage}"
+ S3_ENDPOINT: "${S3_ENDPOINT:-http://storage-service:9000}"
S3_EXPORT_BUCKET: "${S3_EXPORT_BUCKET:-dbrepo-download}"
S3_IMPORT_BUCKET: "${S3_IMPORT_BUCKET:-dbrepo-upload}"
S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}"
SPARQL_CONNECTION_TIMEOUT: "${SPARQL_CONNECTION_TIMEOUT:-10000}"
healthcheck:
- test: wget -qO- localhost:8080/actuator/health/readiness | grep -q "UP" || exit 1
+ test: curl -sSL localhost:8080/actuator/health/liveness | grep 'UP' || exit 1
interval: 10s
timeout: 5s
retries: 12
@@ -160,7 +160,7 @@ services:
restart: "no"
container_name: dbrepo-analyse-service
hostname: analyse-service
- image: docker.io/dbrepo/analyse-service:1.4.4
+ image: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.4.4
environment:
ADMIN_PASSWORD: "${ADMIN_PASSWORD:-admin}"
ADMIN_USERNAME: "${ADMIN_USERNAME:-admin}"
@@ -211,7 +211,7 @@ services:
restart: "no"
container_name: dbrepo-search-db
hostname: search-db
- image: docker.io/dbrepo/search-db:1.4.4
+ image: registry.datalab.tuwien.ac.at/dbrepo/search-db:1.4.4
healthcheck:
test: curl -sSL localhost:9200/_plugins/_security/health | jq .status | grep UP
interval: 10s
@@ -235,7 +235,7 @@ services:
restart: "no"
container_name: dbrepo-search-service
hostname: search-service
- image: docker.io/dbrepo/search-service:1.4.4
+ image: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.4.4
environment:
ADMIN_PASSWORD: "${ADMIN_PASSWORD:-admin}"
ADMIN_USERNAME: "${ADMIN_USERNAME:-admin}"
@@ -243,6 +243,7 @@ services:
AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
AUTH_SERVICE_ENDPOINT: ${AUTH_SERVICE_ENDPOINT:-http://auth-service:8080}
COLLECTION: ${COLLECTION:-['database','table','column','identifier','unit','concept','user','view']}
+ GATEWAY_SERVICE_ENDPOINT: ${GATEWAY_SERVICE_ENDPOINT:-http://gateway-service}
OPENSEARCH_HOST: ${OPENSEARCH_HOST:-search-db}
OPENSEARCH_PORT: ${OPENSEARCH_PORT:-9200}
OPENSEARCH_USERNAME: ${OPENSEARCH_USERNAME:-admin}
@@ -253,7 +254,7 @@ services:
restart: "no"
container_name: dbrepo-data-db-sidecar
hostname: data-db-sidecar
- image: docker.io/dbrepo/data-db-sidecar:1.4.4
+ image: registry.datalab.tuwien.ac.at/dbrepo/data-db-sidecar:1.4.4
environment:
S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}"
S3_ENDPOINT: "${S3_ENDPOINT:-http://storage-service:9000}"
@@ -275,7 +276,7 @@ services:
restart: "no"
container_name: dbrepo-ui
hostname: ui
- image: docker.io/dbrepo/ui:1.4.4
+ image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.4.4
depends_on:
dbrepo-search-service:
condition: service_started
@@ -293,7 +294,7 @@ services:
restart: "no"
container_name: dbrepo-gateway-service
hostname: gateway-service
- image: docker.io/nginx:1.25-alpine-slim
+ image: docker.io/nginx:1.27.0-alpine3.19-slim
ports:
- "80:80"
- "443:443"
@@ -319,7 +320,7 @@ services:
restart: "no"
container_name: dbrepo-search-service-init
hostname: search-service-init
- image: docker.io/dbrepo/search-service-init:1.4.4
+ image: registry.datalab.tuwien.ac.at/dbrepo/search-service-init:1.4.4
environment:
GATEWAY_SERVICE_ENDPOINT: ${GATEWAY_SERVICE_ENDPOINT:-http://gateway-service}
OPENSEARCH_HOST: ${OPENSEARCH_HOST:-search-db}
@@ -354,7 +355,7 @@ services:
restart: "no"
container_name: dbrepo-storage-service-init
hostname: storage-service-init
- image: docker.io/dbrepo/storage-service-init:1.4.4
+ image: registry.datalab.tuwien.ac.at/dbrepo/storage-service-init:1.4.4
environment:
SEAWEEDFS_ENDPOINT: "${STORAGE_SEAWEEDFS_ENDPOINT:-storage-service:9333}"
depends_on:
@@ -391,7 +392,7 @@ services:
restart: "no"
container_name: dbrepo-data-service
hostname: data-service
- image: docker.io/dbrepo/data-service:1.4.4
+ image: registry.datalab.tuwien.ac.at/dbrepo/data-service:1.4.4
volumes:
- "${SHARED_VOLUME:-/tmp}:/tmp"
environment:
@@ -429,7 +430,7 @@ services:
S3_IMPORT_BUCKET: "${S3_IMPORT_BUCKET:-dbrepo-upload}"
S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}"
healthcheck:
- test: wget -qO- localhost:8080/actuator/health/readiness | grep -q "UP" || exit 1
+ test: curl -sSL localhost:8080/actuator/health/liveness | grep 'UP' || exit 1
interval: 10s
timeout: 5s
retries: 12
diff --git a/.docs/deployment-docker-compose.md b/.docs/installation.md
similarity index 77%
rename from .docs/deployment-docker-compose.md
rename to .docs/installation.md
index 870ea0c142..2561b4944b 100644
--- a/.docs/deployment-docker-compose.md
+++ b/.docs/installation.md
@@ -2,7 +2,9 @@
author: Martin Weise
---
-# Docker Compose
+# Installation
+
+[](https://hub.docker.com/u/dbrepo){ tabindex=-1 }
## TL;DR
@@ -12,6 +14,8 @@ If you have [Docker](https://docs.docker.com/engine/install/) already installed
curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.4/install.sh | bash
```
+Or perform a [custom install](#custom-install).
+
## Requirements
### Hardware
@@ -26,7 +30,7 @@ the following settings.
*Optional*: public IP-address if you want to secure the deployment with a (free) TLS-certificate from Let's Encrypt.
-!!! tip "Resource Consumption"
+!!! info "Resource Consumption"
Note that most of the vCPU and RAM resources will be needed for starting the infrastructure, this is because of
Docker. During operation and especially idle times, the deployment will use significantly less resources.
@@ -38,9 +42,11 @@ official [Docker Engine](https://docs.docker.com/engine/install/debian/) install
a [Debian](https://www.debian.org/)-based operating system. Other software deployments (e.g. Docker Desktop on Windows)
are *not* recommended and not tested.
-## Architecture
+## Custom Install
-### Overview
+TBD
+
+## Architecture
The repository is designed as a service-based architecture to ensure scalability and the utilization of various
technologies. The conceptualized microservices operate the basic database operations, data versioning as well as
@@ -51,8 +57,6 @@ technologies. The conceptualized microservices operate the basic database operat
<figcaption>Architecture of the services deployed via Docker Compose</figcaption>
</figure>
-### Notes
-
Please note that we only save the state of the databases as well as the [Broker Service](../broker-service)
since RabbitMQ maintains state inside the container.
@@ -61,43 +65,6 @@ since RabbitMQ maintains state inside the container.
We maintain a rapid prototype deployment option through Docker Compose (v2.17.0 and newer). This deployment creates the
core infrastructure and a single Docker container for all user-generated databases.
-=== "Linux"
-
- Download and install [Docker Engine](https://docs.docker.com/desktop/install/linux-install/) for your Linux
- distribution. Although the installation might work, we *do not* recommend Docker Desktop.
-
- Ensure the Docker daemon is running at all times:
-
- systemctl enable docker --now
-
- Install DBRepo with the default configuration:
-
- curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/install.sh | bash
-
-=== "Windows"
-
- Open `cmd.exe` as administrator and install WSL2 and the Debian subsystem:
-
- wsl --install Debian
-
- Open `optionalfeatures` by typing into the open terminal window or searching for it and enable "Windows Subsystem
- for Linux":
-
- <figure markdown>
- { .img-border }
- <figcaption>Enable Subsystem for Linux in Windows Features</figcaption>
- </figure>
-
- Install [Docker Desktop](https://docs.docker.com/desktop/install/windows-install/) on the Windows host machine.
- Open Docker Desktop and go to settings (:fontawesome-solid-gear:) > General > Tick "Use WSL2 based engine" if not
- already ticked.
-
- Open the Debian container by typing "Debian" into the search, you should see a terminal window.
-
- Install DBRepo with the default configuration from the Debian container:
-
- curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/master/install.sh | bash
-
View the logs:
docker compose logs -f
diff --git a/.docs/deployment-helm.md b/.docs/kubernetes.md
similarity index 63%
rename from .docs/deployment-helm.md
rename to .docs/kubernetes.md
index 86f1257f1d..608502e320 100644
--- a/.docs/deployment-helm.md
+++ b/.docs/kubernetes.md
@@ -21,7 +21,34 @@ helm upgrade --install dbrepo \
```
This chart is also on [Artifact Hub](https://artifacthub.io/packages/helm/dbrepo/dbrepo) with a full documentation
-about values, etc.
+about values, etc. Before installing, you need to change credentials, e.g. the Broker Service administrator user
+password:
+
+```yaml title="values.yaml"
+brokerservice:
+ ...
+ auth:
+ ...
+ username: broker
+ password: broker
+ passwordHash: 1gwjNNTBPKLgyzbsUykfR0JIFC6nNqbNJaxzZ14uPT8JGcTZ
+```
+
+The `brokerservice.auth.passwordHash` field is the RabbitMQ SHA512-hash of the `brokerservice.auth.password` field and
+can be obtained with
+the [`generate-rabbitmq-pw.sh`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.4.4/helm/dbrepo/hack/generate-rabbitmq-pw.sh)
+script:
+
+```console
+$ ./generate-rabbitmq-pw.sh my_password
+klPdmv4dgnRH64czHolIHAfXvc0G9hc24FQmPlI6eeI1NOf9
+```
+
+The script needs the package `xxd` for generation of the random salt. If you don't have `xxd` installed, install it:
+
+* Debian/Ubuntu: `apt install xxd`
+* Windows: `choco install xxd`
+* MacOS: `brew install coreutils`
## Prerequisites
diff --git a/helm/dbrepo/hack/generate-rabbitmq-pw.sh b/helm/dbrepo/hack/generate-rabbitmq-pw.sh
new file mode 100755
index 0000000000..4bccc93d7b
--- /dev/null
+++ b/helm/dbrepo/hack/generate-rabbitmq-pw.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+# https://stackoverflow.com/a/53175209/2634294
+# THIS SCRIPT REQUIRES xxd TO BE INSTALLED:
+# DEBIAN: apt install xxd
+# MACOS: brew install coreutils
+function encode_password()
+{
+ SALT=$(od -A n -t x -N 4 /dev/urandom)
+ PASS=$SALT$(echo -n $1 | xxd -ps | tr -d '\n' | tr -d ' ')
+ PASS=$(echo -n $PASS | xxd -r -p | sha256sum | head -c 128)
+ PASS=$(echo -n $SALT$PASS | xxd -r -p | base64 | tr -d '\n')
+ echo $PASS
+}
+encode_password $1
\ No newline at end of file
diff --git a/helm/dbrepo/templates/broker-secret.yaml b/helm/dbrepo/templates/broker-secret.yaml
index 9291cdbead..84a3088663 100644
--- a/helm/dbrepo/templates/broker-secret.yaml
+++ b/helm/dbrepo/templates/broker-secret.yaml
@@ -34,7 +34,7 @@ stringData:
{
"configure": ".*",
"read": ".*",
- "user": "broker",
+ "user": "{{ .Values.brokerservice.auth.username }}",
"vhost": "dbrepo",
"write": ".*"
}
@@ -60,8 +60,8 @@ stringData:
{
"hashing_algorithm": "rabbit_password_hashing_sha256",
"limits": {},
- "name": "broker",
- "password_hash": "Sek6WxpX2L6UhxlwRkD0cnYAH5GbtTcCFq1yY/SCc1mAa0gB",
+ "name": "{{ .Values.brokerservice.auth.username }}",
+ "password_hash": "{{ .Values.brokerservice.auth.passwordHash }}",
"tags": [
"administrator"
]
diff --git a/helm/dbrepo/values.yaml b/helm/dbrepo/values.yaml
index eba2674db0..1942c37da9 100644
--- a/helm/dbrepo/values.yaml
+++ b/helm/dbrepo/values.yaml
@@ -323,6 +323,7 @@ brokerservice:
existingSecret: ingress-cert
username: broker
password: broker
+ passwordHash: 1gwjNNTBPKLgyzbsUykfR0JIFC6nNqbNJaxzZ14uPT8JGcTZ
extraConfiguration: |-
default_vhost = dbrepo
default_user_tags.administrator = true
diff --git a/install.sh b/install.sh
index 9850ccd35e..a11a7c3b23 100644
--- a/install.sh
+++ b/install.sh
@@ -1,7 +1,7 @@
#!/bin/bash
# preset
-VERSION="latest"
+VERSION="1.4.4"
MIN_CPU=8
MIN_RAM=8
MIN_MAP_COUNT=262144
@@ -59,7 +59,8 @@ fi
echo "[🚀] Gathering environment ..."
mkdir -p ./dist
curl -sSL -o ./docker-compose.yml "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/.docker/docker-compose.yml"
-curl -sSL -o ./dist/2_setup-data.sql "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-metadata-db/2_setup-data.sql"
+curl -sSL -o ./dist/1_setup-schema.sql "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-metadata-db/setup-schema.sql"
+curl -sSL -o ./dist/2_setup-data.sql "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-metadata-db/setup-data.sql"
curl -sSL -o ./dist/rabbitmq.conf "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-broker-service/rabbitmq.conf"
curl -sSL -o ./dist/enabled_plugins "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-broker-service/enabled_plugins"
curl -sSL -o ./dist/cert.pem "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-broker-service/cert.pem"
diff --git a/mkdocs.yml b/mkdocs.yml
index ada8171b50..e2627e6b0d 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -10,8 +10,8 @@ nav:
- Welcome to DBRepo: index.md
- Why use DBRepo: why.md
- Help with DBRepo: help.md
- - Installation: deployment-docker-compose.md
- - Kubernetes: deployment-helm.md
+ - Installation: installation.md
+ - Kubernetes: kubernetes.md
- Migration Guide: migration.md
- contributing.md
- Concepts:
--
GitLab