Added the password hash script and fixed some paths

9a05dea2 · Martin Weise · 70d86142 · 9a05dea2 · 9a05dea2 · 9a05dea2
Verified Commit 9a05dea2 authored 11 months ago by Martin Weise
--- a/.docker/docker-compose.yml
+++ b/.docker/docker-compose.yml
@@ -14,10 +14,11 @@ services:
    restart: "no"
    container_name: dbrepo-metadata-db
    hostname: metadata-db
-    image: docker.io/dbrepo/metadata-db:1.4.4
+    image: docker.io/bitnami/mariadb:11.1.3-debian-11-r6
    volumes:
      - metadata-db-data:/bitnami/mariadb
-      - ./dist/2_setup-data.sql:/docker-entrypoint-initdb.d/2_setup-data.sql
+      - ./dbrepo-metadata-db/setup-schema.sql:/docker-entrypoint-initdb.d/1_setup-schema.sql
+      - ./dbrepo-metadata-db/setup-data.sql:/docker-entrypoint-initdb.d/2_setup-data.sql
    ports:
      - "3306:3306"
    environment:
@@ -35,7 +36,7 @@ services:
    restart: "no"
    container_name: dbrepo-data-db
    hostname: data-db
-    image: docker.io/bitnami/mariadb-galera:11.2.2-debian-11-r0
+    image: docker.io/bitnami/mariadb:11.1.3-debian-11-r6
    volumes:
      - data-db-data:/bitnami/mariadb
      - "${SHARED_VOLUME:-/tmp}:/tmp"
@@ -43,7 +44,6 @@ services:
      - "3307:3306"
    environment:
      MARIADB_ROOT_PASSWORD: "${USER_DB_PASSWORD:-dbrepo}"
-      MARIADB_GALERA_MARIABACKUP_PASSWORD: "${USER_DB_BACKUP_PASSWORD:-dbrepo}"
    healthcheck:
      test: mysqladmin ping --user="${USER_DB_USERNAME:-root}" --password="${USER_DB_PASSWORD:-dbrepo}" --silent
      interval: 10s
@@ -56,7 +56,7 @@ services:
    restart: "no"
    container_name: dbrepo-auth-db
    hostname: auth-db
-    image: docker.io/bitnami/mariadb:11.2.2-debian-11-r0
+    image: docker.io/bitnami/mariadb:11.1.3-debian-11-r6
    volumes:
      - auth-db-data:/bitnami/mariadb
    ports:
@@ -76,7 +76,7 @@ services:
    restart: "no"
    container_name: dbrepo-auth-service
    hostname: auth-service
-    image: docker.io/dbrepo/auth-service:1.4.4
+    image: registry.datalab.tuwien.ac.at/dbrepo/auth-service:1.4.4
    healthcheck:
      test: curl -sSL 'http://0.0.0.0:8080/realms/dbrepo' | grep "dbrepo" || exit 1
      interval: 10s
@@ -98,7 +98,7 @@ services:
    restart: "no"
    container_name: dbrepo-metadata-service
    hostname: metadata-service
-    image: docker.io/dbrepo/metadata-service:1.4.4
+    image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.4.4
    volumes:
      - "${SHARED_VOLUME:-/tmp}:/tmp"
    environment:
@@ -124,7 +124,7 @@ services:
      DELETED_RECORD: "${DELETED_RECORD:-persistent}"
      GRANULARITY: "${GRANULARITY:-YYYY-MM-DDThh:mm:ssZ}"
      JWT_PUBKEY: "${JWT_PUBKEY:-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB}"
-      LOG_LEVEL: "${LOG_LEVEL:-info}"
+      LOG_LEVEL: ${LOG_LEVEL:-info}
      METADATA_DB: "${METADATA_DB:-dbrepo}"
      METADATA_HOST: "${METADATA_HOST:-metadata-db}"
      METADATA_JDBC_EXTRA_ARGS: "${METADATA_JDBC_EXTRA_ARGS:-}"
@@ -134,13 +134,13 @@ services:
      REPOSITORY_NAME: "${REPOSITORY_NAME:-Database Repository}"
      SEARCH_SERVICE_ENDPOINT: "${SEARCH_SERVICE_ENDPOINT:-http://gateway-service}"
      S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}"
-      S3_ENDPOINT: "${S3_ENDPOINT:-http://gateway-service/api/storage}"
+      S3_ENDPOINT: "${S3_ENDPOINT:-http://storage-service:9000}"
      S3_EXPORT_BUCKET: "${S3_EXPORT_BUCKET:-dbrepo-download}"
      S3_IMPORT_BUCKET: "${S3_IMPORT_BUCKET:-dbrepo-upload}"
      S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}"
      SPARQL_CONNECTION_TIMEOUT: "${SPARQL_CONNECTION_TIMEOUT:-10000}"
    healthcheck:
-      test: wget -qO- localhost:8080/actuator/health/readiness | grep -q "UP" || exit 1
+      test: curl -sSL localhost:8080/actuator/health/liveness | grep 'UP' || exit 1
      interval: 10s
      timeout: 5s
      retries: 12
@@ -160,7 +160,7 @@ services:
    restart: "no"
    container_name: dbrepo-analyse-service
    hostname: analyse-service
-    image: docker.io/dbrepo/analyse-service:1.4.4
+    image: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.4.4
    environment:
      ADMIN_PASSWORD: "${ADMIN_PASSWORD:-admin}"
      ADMIN_USERNAME: "${ADMIN_USERNAME:-admin}"
@@ -211,7 +211,7 @@ services:
    restart: "no"
    container_name: dbrepo-search-db
    hostname: search-db
-    image: docker.io/dbrepo/search-db:1.4.4
+    image: registry.datalab.tuwien.ac.at/dbrepo/search-db:1.4.4
    healthcheck:
      test: curl -sSL localhost:9200/_plugins/_security/health | jq .status | grep UP
      interval: 10s
@@ -235,7 +235,7 @@ services:
    restart: "no"
    container_name: dbrepo-search-service
    hostname: search-service
-    image: docker.io/dbrepo/search-service:1.4.4
+    image: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.4.4
    environment:
      ADMIN_PASSWORD: "${ADMIN_PASSWORD:-admin}"
      ADMIN_USERNAME: "${ADMIN_USERNAME:-admin}"
@@ -243,6 +243,7 @@ services:
      AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
      AUTH_SERVICE_ENDPOINT: ${AUTH_SERVICE_ENDPOINT:-http://auth-service:8080}
      COLLECTION: ${COLLECTION:-['database','table','column','identifier','unit','concept','user','view']}
+      GATEWAY_SERVICE_ENDPOINT: ${GATEWAY_SERVICE_ENDPOINT:-http://gateway-service}
      OPENSEARCH_HOST: ${OPENSEARCH_HOST:-search-db}
      OPENSEARCH_PORT: ${OPENSEARCH_PORT:-9200}
      OPENSEARCH_USERNAME: ${OPENSEARCH_USERNAME:-admin}
@@ -253,7 +254,7 @@ services:
    restart: "no"
    container_name: dbrepo-data-db-sidecar
    hostname: data-db-sidecar
-    image: docker.io/dbrepo/data-db-sidecar:1.4.4
+    image: registry.datalab.tuwien.ac.at/dbrepo/data-db-sidecar:1.4.4
    environment:
      S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}"
      S3_ENDPOINT: "${S3_ENDPOINT:-http://storage-service:9000}"
@@ -275,7 +276,7 @@ services:
    restart: "no"
    container_name: dbrepo-ui
    hostname: ui
-    image: docker.io/dbrepo/ui:1.4.4
+    image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.4.4
    depends_on:
      dbrepo-search-service:
        condition: service_started
@@ -293,7 +294,7 @@ services:
    restart: "no"
    container_name: dbrepo-gateway-service
    hostname: gateway-service
-    image: docker.io/nginx:1.25-alpine-slim
+    image: docker.io/nginx:1.27.0-alpine3.19-slim
    ports:
      - "80:80"
      - "443:443"
@@ -319,7 +320,7 @@ services:
    restart: "no"
    container_name: dbrepo-search-service-init
    hostname: search-service-init
-    image: docker.io/dbrepo/search-service-init:1.4.4
+    image: registry.datalab.tuwien.ac.at/dbrepo/search-service-init:1.4.4
    environment:
      GATEWAY_SERVICE_ENDPOINT: ${GATEWAY_SERVICE_ENDPOINT:-http://gateway-service}
      OPENSEARCH_HOST: ${OPENSEARCH_HOST:-search-db}
@@ -354,7 +355,7 @@ services:
    restart: "no"
    container_name: dbrepo-storage-service-init
    hostname: storage-service-init
-    image: docker.io/dbrepo/storage-service-init:1.4.4
+    image: registry.datalab.tuwien.ac.at/dbrepo/storage-service-init:1.4.4
    environment:
      SEAWEEDFS_ENDPOINT: "${STORAGE_SEAWEEDFS_ENDPOINT:-storage-service:9333}"
    depends_on:
@@ -391,7 +392,7 @@ services:
    restart: "no"
    container_name: dbrepo-data-service
    hostname: data-service
-    image: docker.io/dbrepo/data-service:1.4.4
+    image: registry.datalab.tuwien.ac.at/dbrepo/data-service:1.4.4
    volumes:
      - "${SHARED_VOLUME:-/tmp}:/tmp"
    environment:
@@ -429,7 +430,7 @@ services:
      S3_IMPORT_BUCKET: "${S3_IMPORT_BUCKET:-dbrepo-upload}"
      S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}"
    healthcheck:
-      test: wget -qO- localhost:8080/actuator/health/readiness | grep -q "UP" || exit 1
+      test: curl -sSL localhost:8080/actuator/health/liveness | grep 'UP' || exit 1
      interval: 10s
      timeout: 5s
      retries: 12

--- a/.docs/deployment-docker-compose.md
+++ b/.docs/deployment-docker-compose.md
@@ -2,7 +2,9 @@
 author: Martin Weise
 ---
-# Docker Compose
+# Installation
+[![Image Pulls](https://img.shields.io/docker/pulls/dbrepo/data-service?style=flat&cacheSeconds=3600)](https://hub.docker.com/u/dbrepo){ tabindex=-1 }
 ## TL;DR
@@ -12,6 +14,8 @@ If you have [Docker](https://docs.docker.com/engine/install/) already installed
 curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.4/install.sh | bash
 ```
+Or perform a [custom install](#custom-install).
 ## Requirements
 ### Hardware
@@ -26,7 +30,7 @@ the following settings.
 *Optional*: public IP-address if you want to secure the deployment with a (free) TLS-certificate from Let's Encrypt.
-!!! tip "Resource Consumption"
+!!! info "Resource Consumption"
    Note that most of the vCPU and RAM resources will be needed for starting the infrastructure, this is because of
    Docker. During operation and especially idle times, the deployment will use significantly less resources.
@@ -38,9 +42,11 @@ official [Docker Engine](https://docs.docker.com/engine/install/debian/) install
 a [Debian](https://www.debian.org/)-based operating system. Other software deployments (e.g. Docker Desktop on Windows)
 are *not* recommended and not tested.
-## Architecture
+## Custom Install
-### Overview
+TBD
+## Architecture
 The repository is designed as a service-based architecture to ensure scalability and the utilization of various
 technologies. The conceptualized microservices operate the basic database operations, data versioning as well as
@@ -51,8 +57,6 @@ technologies. The conceptualized microservices operate the basic database operat
 <figcaption>Architecture of the services deployed via Docker Compose</figcaption>
 </figure>
-### Notes
 Please note that we only save the state of the databases as well as the [Broker Service](../broker-service)
 since RabbitMQ maintains state inside the container.
@@ -61,43 +65,6 @@ since RabbitMQ maintains state inside the container.
 We maintain a rapid prototype deployment option through Docker Compose (v2.17.0 and newer). This deployment creates the
 core infrastructure and a single Docker container for all user-generated databases.
-=== "Linux"
-    Download and install [Docker Engine](https://docs.docker.com/desktop/install/linux-install/) for your Linux
-    distribution. Although the installation might work, we *do not* recommend Docker Desktop.
-    Ensure the Docker daemon is running at all times:
-        systemctl enable docker --now
-    Install DBRepo with the default configuration:
-        curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/install.sh | bash
-=== "Windows"
-    Open `cmd.exe` as administrator and install WSL2 and the Debian subsystem:
-        wsl --install Debian
-    Open `optionalfeatures` by typing into the open terminal window or searching for it and enable "Windows Subsystem 
-    for Linux":
-    <figure markdown>
-    ![Data ingest](images/optionalfeatures.png){ .img-border }
-       <figcaption>Enable Subsystem for Linux in Windows Features</figcaption>
-    </figure>
-    Install [Docker Desktop](https://docs.docker.com/desktop/install/windows-install/) on the Windows host machine.
-    Open Docker Desktop and go to settings (:fontawesome-solid-gear:) > General > Tick "Use WSL2 based engine" if not
-    already ticked.
-    Open the Debian container by typing "Debian" into the search, you should see a terminal window.
-    Install DBRepo with the default configuration from the Debian container:
-        curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/master/install.sh | bash
 View the logs:
    docker compose logs -f

--- a/.docs/deployment-helm.md
+++ b/.docs/deployment-helm.md
@@ -21,7 +21,34 @@ helm upgrade --install dbrepo \
 ```
 This chart is also on [Artifact Hub](https://artifacthub.io/packages/helm/dbrepo/dbrepo) with a full documentation
-about values, etc.
+about values, etc. Before installing, you need to change credentials, e.g. the Broker Service administrator user
+password:
+```yaml title="values.yaml"
+brokerservice:
+  ...
+  auth:
+    ...
+    username: broker
+    password: broker
+    passwordHash: 1gwjNNTBPKLgyzbsUykfR0JIFC6nNqbNJaxzZ14uPT8JGcTZ
+```
+The `brokerservice.auth.passwordHash` field is the RabbitMQ SHA512-hash of the `brokerservice.auth.password` field and
+can be obtained with
+the [`generate-rabbitmq-pw.sh`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.4.4/helm/dbrepo/hack/generate-rabbitmq-pw.sh)
+script:
+```console
+$ ./generate-rabbitmq-pw.sh my_password
+klPdmv4dgnRH64czHolIHAfXvc0G9hc24FQmPlI6eeI1NOf9
+```
+The script needs the package `xxd` for generation of the random salt. If you don't have `xxd` installed, install it:
+* Debian/Ubuntu: `apt install xxd`
+* Windows: `choco install xxd`
+* MacOS: `brew install coreutils`
 ## Prerequisites

--- a/helm/dbrepo/hack/generate-rabbitmq-pw.sh
+++ b/helm/dbrepo/hack/generate-rabbitmq-pw.sh
+#!/bin/bash
+# https://stackoverflow.com/a/53175209/2634294
+# THIS SCRIPT REQUIRES xxd TO BE INSTALLED:
+#     DEBIAN: apt install xxd
+#     MACOS: brew install coreutils
+function encode_password()
+{
+    SALT=$(od -A n -t x -N 4 /dev/urandom)
+    PASS=$SALT$(echo -n $1 | xxd -ps | tr -d '\n' | tr -d ' ')
+    PASS=$(echo -n $PASS | xxd -r -p | sha256sum | head -c 128)
+    PASS=$(echo -n $SALT$PASS | xxd -r -p | base64 | tr -d '\n')
+    echo $PASS
+}
+encode_password $1
\ No newline at end of file
--- a/helm/dbrepo/templates/broker-secret.yaml
+++ b/helm/dbrepo/templates/broker-secret.yaml
@@ -34,7 +34,7 @@ stringData:
        {
          "configure": ".*",
          "read": ".*",
-          "user": "broker",
+          "user": "{{ .Values.brokerservice.auth.username }}",
          "vhost": "dbrepo",
          "write": ".*"
        }
@@ -60,8 +60,8 @@ stringData:
        {
          "hashing_algorithm": "rabbit_password_hashing_sha256",
          "limits": {},
-          "name": "broker",
+          "name": "{{ .Values.brokerservice.auth.username }}",
-          "password_hash": "Sek6WxpX2L6UhxlwRkD0cnYAH5GbtTcCFq1yY/SCc1mAa0gB",
+          "password_hash": "{{ .Values.brokerservice.auth.passwordHash }}",
          "tags": [
            "administrator"
          ]

--- a/helm/dbrepo/values.yaml
+++ b/helm/dbrepo/values.yaml
@@ -323,6 +323,7 @@ brokerservice:
      existingSecret: ingress-cert
    username: broker
    password: broker
+    passwordHash: 1gwjNNTBPKLgyzbsUykfR0JIFC6nNqbNJaxzZ14uPT8JGcTZ
  extraConfiguration: |-
    default_vhost = dbrepo
    default_user_tags.administrator = true

--- a/install.sh
+++ b/install.sh
 #!/bin/bash
 # preset
-VERSION="latest"
+VERSION="1.4.4"
 MIN_CPU=8
 MIN_RAM=8
 MIN_MAP_COUNT=262144
@@ -59,7 +59,8 @@ fi
 echo "[🚀] Gathering environment ..."
 mkdir -p ./dist
 curl -sSL -o ./docker-compose.yml "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/.docker/docker-compose.yml"
-curl -sSL -o ./dist/2_setup-data.sql "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-metadata-db/2_setup-data.sql"
+curl -sSL -o ./dist/1_setup-schema.sql "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-metadata-db/setup-schema.sql"
+curl -sSL -o ./dist/2_setup-data.sql "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-metadata-db/setup-data.sql"
 curl -sSL -o ./dist/rabbitmq.conf "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-broker-service/rabbitmq.conf"
 curl -sSL -o ./dist/enabled_plugins "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-broker-service/enabled_plugins"
 curl -sSL -o ./dist/cert.pem "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-broker-service/cert.pem"

--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -10,8 +10,8 @@ nav:
    - Welcome to DBRepo: index.md
    - Why use DBRepo: why.md
    - Help with DBRepo: help.md
-    - Installation: deployment-docker-compose.md
+    - Installation: installation.md
-    - Kubernetes: deployment-helm.md
+    - Kubernetes: kubernetes.md
    - Migration Guide: migration.md
    - contributing.md
  - Concepts: