diff --git a/.docs/requirements.txt b/.docs/requirements.txt deleted file mode 100644 index 1df697b8b2fc6e9019545dffdb813e149bbf823f..0000000000000000000000000000000000000000 --- a/.docs/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -mkdocs==1.4.3 -mkdocs-material==9.1.17 -mkdocs-with-pdf==0.9.3 -mkdocs-material-extensions>=1.0.3 -requests>=2.27.0 -py-dotenv>=0.1 -python-dotenv==1.0.0 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 84dafc88bfad06d716e956f47289f287d6af5865..6950219ea5e072b382dee782c5f32356e0ea13bb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,8 +4,8 @@ variables: TRIVY_CACHE_DIR: ".trivycache/" DOCKER_HOST: "unix:///var/run/dind/docker.sock" TESTCONTAINERS_RYUK_DISABLED: "false" - APP_VERSION: "1.4.1" - CHART_VERSION: "1.4.1" + APP_VERSION: "1.4.0" + CHART_VERSION: "1.4.0" image: debian:12-slim @@ -71,6 +71,18 @@ build-docker: - "docker build -t dbrepo-data-service:build --target build dbrepo-data-service" - "docker compose build --parallel" +build-helm: + image: docker.io/docker:24-dind + stage: build + before_script: + - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL + script: + - apk add sed helm curl + - 'sed -i -e "s/^version:.*/version: \"${CHART_VERSION}\"/g" ./helm-charts/dbrepo/Chart.yaml' + - 'sed -i -e "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/g" ./helm-charts/dbrepo/Chart.yaml' + - find ./helm-charts -type f -exec sed -i -e "s/__CHARTVERSION__/${CHART_VERSION}/g" {} \; + - helm package ./helm-charts/dbrepo --destination ./build + test-metadata-service: image: maven:3-openjdk-17 stage: test @@ -598,7 +610,7 @@ docs-registry: - release-v1.3 - release-v1.4 script: - - pip install pipenv && pipenv install --dev --system --deploy + - pip install -r ./requirements.txt - python3 .docs/docker/release.py cache: paths: @@ -624,7 +636,7 @@ docs-latest: script: - apt-get update && apt-get install -y git make sed - git fetch && git checkout master - - pip install pipenv && pipenv install --dev --system --deploy + - pip install -r ./requirements.txt - mkdir -p ./final - sed -i -e "s/__APPVERSION__/${APP_VERSION}/g" .docs/redirect.html - cp ./.docs/redirect.html ./final/index.html @@ -655,7 +667,7 @@ docs-1.3: script: - apt-get update && apt-get install -y git make wget - git fetch && git checkout release-v1.3 - - pip install -r ./.docs/requirements.txt + - pip install -r ./requirements.txt - wget https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/.docs/overrides/main.html -O .docs/overrides/main.html -q - mkdir -p ./final - mkdocs build && cp -r ./site ./final/1.3.0 @@ -684,7 +696,7 @@ docs-1.4: script: - apt-get update && apt-get install -y git make sed wget - git fetch && git checkout release-v1.4 - - pip install pipenv && pipenv install --dev --system --deploy + - pip install -r ./requirements.txt - wget https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/.docs/overrides/main.html -O .docs/overrides/main.html -q - mkdir -p ./final - find .docs/ -type f -exec sed -i -e "s/__APPVERSION__/${APP_VERSION}/g" {} \; diff --git a/helm-charts/dbrepo/Chart.yaml b/helm-charts/dbrepo/Chart.yaml index 79a5fd1fa2cac445afa3c7d661a17589c7999129..fe4c5cf04a7f1637d6fff38a3311851131a9ce92 100644 --- a/helm-charts/dbrepo/Chart.yaml +++ b/helm-charts/dbrepo/Chart.yaml @@ -16,7 +16,7 @@ home: https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/ icon: https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/master/.docs/images/signet_white.png dependencies: - name: opensearch - alias: searchDb + alias: searchdb version: 2.15.0 # app version 2.10.0 repository: https://opensearch-project.github.io/helm-charts/ - name: opensearch-dashboards @@ -44,10 +44,10 @@ dependencies: version: 12.5.1 repository: https://charts.bitnami.com/bitnami - name: fluent-bit - alias: logService + alias: logservice version: 0.40.0 repository: https://fluent.github.io/helm-charts - name: seaweedfs - alias: storageService + alias: storageservice version: 3.59.4 repository: https://seaweedfs.github.io/seaweedfs/helm diff --git a/helm-charts/dbrepo/README.md b/helm-charts/dbrepo/README.md index bfd84c76f8626453ddafa56e703b5e42f8c64327..d7af72f404d5d894ce0965435be4ebbbef6c1138 100644 --- a/helm-charts/dbrepo/README.md +++ b/helm-charts/dbrepo/README.md @@ -58,30 +58,30 @@ The command removes all the Kubernetes components associated with the chart and The Metadata Database uses the [Bitnami MariaDB Galera](https://artifacthub.io/packages/helm/bitnami/mariadb-galera) Helm chart. See their documentation for the remaining overridden values. -| Name | Description | Value | -|-----------------------------|-------------------------------------------|---------------| -| `metadata-db.host` | Hostname. | `metadata-db` | -| `metadata-db.jdbcExtraArgs` | Extra arguments for the JDBC connections. | `""` | +| Name | Description | Value | +|----------------------------|-------------------------------------------|---------------| +| `metadataDb.host` | Hostname. | `metadata-db` | +| `metadataDb.jdbcExtraArgs` | Extra arguments for the JDBC connections. | `""` | ### Authentication Service The Auth Service uses the [Bitnami Keycloak](https://artifacthub.io/packages/helm/bitnami/keycloak) Helm chart. See their documentation for the remaining overridden values. -| Name | Description | Value | -|------------------------------|-----------------------------------------------------------------|------------------------------------| -| `auth-service.client.id` | Client id. This value is publicly known. | `dbrepo-client` | -| `auth-service.client.secret` | Client secret. This value should never be known outside DBRepo. | `MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG` | +| Name | Description | Value | +|-----------------------------|-----------------------------------------------------------------|------------------------------------| +| `authService.client.id` | Client id. This value is publicly known. | `dbrepo-client` | +| `authService.client.secret` | Client secret. This value should never be known outside DBRepo. | `MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG` | ### Auth Database The Auth Database uses the [Bitnami PostgreSQL HA](https://artifacthub.io/packages/helm/bitnami/postgresql-ha) Helm chart. See their documentation for the remaining overridden values. -| Name | Description | Value | -|----------------|--------------------------------------|------------------| -| `auth-db.host` | Hostname. Needed for other services. | `auth-db-pgpool` | -| `auth-db.port` | Port. Needed for other services. | `5432` | +| Name | Description | Value | +|---------------|--------------------------------------|------------------| +| `authDb.host` | Hostname. Needed for other services. | `auth-db-pgpool` | +| `authDb.port` | Port. Needed for other services. | `5432` | ### Data Database @@ -95,12 +95,12 @@ The Search Database uses the [OpenSearch](https://artifacthub.io/packages/helm/opensearch-project-helm-charts/opensearch) Helm chart. See their documentation for the remaining overridden values. -| Name | Description | Value | -|----------------------|--------------------------------------|-------------| -| `search-db.host` | Hostname. Needed for other services. | `search-db` | -| `search-db.port` | Port. Needed for other services. | `9200` | -| `search-db.username` | Username. Needed for other services. | `admin` | -| `search-db.password` | Password. Needed for other services. | `admin` | +| Name | Description | Value | +|---------------------|--------------------------------------|-------------| +| `searchdb.host` | Hostname. Needed for other services. | `search-db` | +| `searchdb.port` | Port. Needed for other services. | `9200` | +| `searchdb.username` | Username. Needed for other services. | `admin` | +| `searchdb.password` | Password. Needed for other services. | `admin` | ### Search Database Dashboard @@ -110,29 +110,29 @@ chart. See their documentation for the remaining overridden values. ### Upload Service -| Name | Description | Value | -|-----------------------------------|----------------------------------------|-------------------| -| `upload-service.enabled` | Enables/disabled the deployment. | `true` | -| `upload-service.image.registry` | Registry to pull the image | `docker.io` | -| `upload-service.image.repository` | Repository to pull the image | `tusproject/tusd` | -| `upload-service.image.tag` | Tag of the image. | `v1.12` | -| `upload-service.replicaCount` | Number of replicas for the deployment. | `2` | +| Name | Description | Value | +|----------------------------------|----------------------------------------|-------------------| +| `uploadService.enabled` | Enables/disabled the deployment. | `true` | +| `uploadService.image.registry` | Registry to pull the image | `docker.io` | +| `uploadService.image.repository` | Repository to pull the image | `tusproject/tusd` | +| `uploadService.image.tag` | Tag of the image. | `v1.12` | +| `uploadService.replicaCount` | Number of replicas for the deployment. | `2` | ### Broker Service The Broker Service uses the [Bitnami RabbitMQ](https://artifacthub.io/packages/helm/bitnami/rabbitmq) Helm chart. See their documentation for the remaining overridden values. -| Name | Description | Value | -|------------------------------------|-------------------------------------------------------------------------|-------------------------------| -| `broker-service.url` | Admin API endpoint. Needed for other services. | `http://broker-service:15672` | -| `broker-service.host` | Service hostname. Needed for other services. | `broker-service` | -| `broker-service.port` | Service port. Needed for other services. | `5672` | -| `broker-service.virtualHost` | Virtual host on RabbitMQ. Needed for other services. | `dbrepo` | -| `broker-service.queueName` | Queue name on RabbitMQ. Needed for other services. | `dbrepo` | -| `broker-service.exchangeName` | Exchange name on RabbitMQ. Needed for other services. | `dbrepo` | -| `broker-service.routingKey` | Route binding for queue to exchange defined. Needed for other services. | `dbrepo.#` | -| `broker-service.connectionTimeout` | Connection timeout. Needed for other services. | `60000` | +| Name | Description | Value | +|-----------------------------------|-------------------------------------------------------------------------|-------------------------------| +| `brokerService.url` | Admin API endpoint. Needed for other services. | `http://broker-service:15672` | +| `brokerService.host` | Service hostname. Needed for other services. | `broker-service` | +| `brokerService.port` | Service port. Needed for other services. | `5672` | +| `brokerService.virtualHost` | Virtual host on RabbitMQ. Needed for other services. | `dbrepo` | +| `brokerService.queueName` | Queue name on RabbitMQ. Needed for other services. | `dbrepo` | +| `brokerService.exchangeName` | Exchange name on RabbitMQ. Needed for other services. | `dbrepo` | +| `brokerService.routingKey` | Route binding for queue to exchange defined. Needed for other services. | `dbrepo.#` | +| `brokerService.connectionTimeout` | Connection timeout. Needed for other services. | `60000` | ### Analyse Service @@ -204,10 +204,10 @@ Helm chart. See their documentation for the remaining overridden values. The Storage Service uses the [SeaweedFS](https://artifacthub.io/packages/helm/seaweedfs/seaweedfs) Helm chart. See their documentation for the remaining overridden values. -| Name | Description | Value | -|---------------------------------|---------------------------------------------|------------------| -| `storage-service.auth.username` | Username for S3. Needed for other services. | `seaweedfsadmin` | -| `storage-service.auth.password` | Password for S3. Needed for other services. | `seaweedfsadmin` | +| Name | Description | Value | +|--------------------------------|---------------------------------------------|------------------| +| `storageservice.auth.username` | Username for S3. Needed for other services. | `seaweedfsadmin` | +| `storageservice.auth.password` | Password for S3. Needed for other services. | `seaweedfsadmin` | ### User Interface diff --git a/helm-charts/dbrepo/templates/data-service/secret.yaml b/helm-charts/dbrepo/templates/data-service/secret.yaml index 3386171d49f7e942130842cbfdb1864125fe6c27..8bfa60c1eaa069e517b23670f6e400ded2639b03 100644 --- a/helm-charts/dbrepo/templates/data-service/secret.yaml +++ b/helm-charts/dbrepo/templates/data-service/secret.yaml @@ -12,8 +12,8 @@ stringData: metadata-username: "{{ .Values.metadataDb.rootUser.user }}" metadata-password: "{{ .Values.metadataDb.rootUser.password }}" metadata-jdbc-extra-args: "{{ .Values.metadataDb.jdbcExtraArgs }}" - search-username: "{{ .Values.searchDb.username }}" - search-password: "{{ .Values.searchDb.password }}" + search-username: "{{ .Values.searchdb.username }}" + search-password: "{{ .Values.searchdb.password }}" jwt-issuer: "{{ $jwtIssuer }}" jwt-pubkey: "{{ .Values.dataService.jwt.pubkey }}" broker-username: "{{ .Values.brokerService.auth.username }}" diff --git a/helm-charts/dbrepo/templates/metadata-service/secret.yaml b/helm-charts/dbrepo/templates/metadata-service/secret.yaml index 374381bab36c01217c0ae4add61307df53baff3d..0976f3b63c1d5148590718b155ae4b3fbc31c977 100644 --- a/helm-charts/dbrepo/templates/metadata-service/secret.yaml +++ b/helm-charts/dbrepo/templates/metadata-service/secret.yaml @@ -9,25 +9,25 @@ metadata: stringData: admin-email: "{{ .Values.metadataService.adminEmail }}" base-url: "{{ .Values.hostname }}" - broker-endpoint: "{{ index .Values "broker-service" "url" }}" - broker-host: "{{ index .Values "broker-service" "host" }}" - broker-port: "{{ index .Values "broker-service" "port" }}" + broker-endpoint: "{{ .Values.brokerService.url }}" + broker-host: "{{ .Values.brokerService.host }}" + broker-port: "{{ .Values.brokerService.port }}" gateway-endpoint: "{{ .Values.hostname }}" website: "{{ .Values.metadataService.website }}" - search-username: "{{ index .Values "search-db" "username" }}" - search-password: "{{ index .Values "search-db" "password" }}" - broker-username: "{{ index .Values "broker-service" "auth" "username" }}" - broker-password: "{{ index .Values "broker-service" "auth" "password" }}" + search-username: "{{ .Values.searchdb.username }}" + search-password: "{{ .Values.searchdb.password }}" + broker-username: "{{ .Values.brokerService.auth.username }}" + broker-password: "{{ .Values.brokerService.auth.password }}" log-level: "{{ ternary "trace" "info" .Values.metadataService.image.debug }}" - metadata-db: "{{ index .Values "metadata-db" "db" "name" }}" - metadata-host: "{{ index .Values "metadata-db" "host" }}" - metadata-username: "{{ index .Values "metadata-db" "rootUser" "user" }}" - metadata-password: "{{ index .Values "metadata-db" "rootUser" "password" }}" - metadata-jdbc-extra-args: "{{ index .Values "metadata-db" "jdbcExtraArgs" }}" + metadata-db: "{{ .Values.metadataDb.db.name }}" + metadata-host: "{{ .Values.metadataDb.host }}" + metadata-username: "{{ .Values.metadataDb.rootUser.user }}" + metadata-password: "{{ .Values.metadataDb.rootUser.password }}" + metadata-jdbc-extra-args: "{{ .Values.metadataDb.jdbcExtraArgs }}" keycloak-host: "{{ .Values.metadataService.authService.url }}" - keycloak-admin: "{{ index .Values "auth-service" "auth" "adminUser" }}" - keycloak-admin-password: "{{ index .Values "auth-service" "auth" "adminPassword" }}" - keycloak-client-secret: "{{ index .Values "auth-service" "client" "secret" }}" + keycloak-admin: "{{ .Values.authService.auth.adminUser }}" + keycloak-admin-password: "{{ .Values.authService.auth.adminPassword }}" + keycloak-client-secret: "{{ .Values.authService.client.secret }}" datacite-url: "{{ .Values.metadataService.datacite.url }}" datacite-prefix: "{{ .Values.metadataService.datacite.prefix | toString }}" datacite-username: "{{ .Values.metadataService.datacite.username }}" @@ -35,17 +35,17 @@ stringData: repository-name: "{{ .Values.metadataService.repositoryName }}" pid-base: "{{ $pidBase }}" jwt-issuer: "{{ $jwtIssuer }}" - broker-virtualhost: "{{ index .Values "broker-service" "virtualHost" }}" - queue-name: "{{ index .Values "broker-service" "queueName" }}" - exchange-name: "{{ index .Values "broker-service" "exchangeName" }}" - routing-key: "{{ index .Values "broker-service" "routingKey" }}" - connection-timeout: "{{ index .Values "broker-service" "connectionTimeout" }}" + broker-virtualhost: "{{ .Values.brokerService.virtualHost }}" + queue-name: "{{ .Values.brokerService.queueName }}" + exchange-name: "{{ .Values.brokerService.exchangeName }}" + routing-key: "{{ .Values.brokerService.routingKey }}" + connection-timeout: "{{ .Values.brokerService.connectionTimeout }}" min-concurrent-consumers: "{{ .Values.dataService.consumerConcurrentMin }}" max-concurrent-consumers: "{{ .Values.dataService.consumerConcurrentMax }}" requeue-rejected: "{{ .Values.dataService.requeueRejected }}" s3-storage-endpoint: http://storage-service-s3:9000 - s3-access-key-id: "{{ index .Values "storage-service" "s3" "auth" "username" }}" - s3-secret-access-key: "{{ index .Values "storage-service" "s3" "auth" "password" }}" + s3-access-key-id: "{{ .Values.storageservice.s3.auth.username }}" + s3-secret-access-key: "{{ .Values.storageservice.s3.auth.password }}" s3-import-bucket: "dbrepo-upload" s3-export-bucket: "dbrepo-download" delete-stale-files-rate: {{ .Values.metadataService.rates.deleteStaleFiles | quote }} diff --git a/helm-charts/dbrepo/templates/search-db-dashboard/secret.yaml b/helm-charts/dbrepo/templates/search-db-dashboard/secret.yaml index f7caf1d29290312437cf014e2766d8b8eabcd226..9150e8e1d6b48d0168558fb2662ca04f2a1d74e6 100644 --- a/helm-charts/dbrepo/templates/search-db-dashboard/secret.yaml +++ b/helm-charts/dbrepo/templates/search-db-dashboard/secret.yaml @@ -17,5 +17,5 @@ stringData: opensearch: ssl: verificationMode: none - username: {{ .Values.searchDb.username }} - password: {{ .Values.searchDb.password }} + username: {{ .Values.searchdb.username }} + password: {{ .Values.searchdb.password }} diff --git a/helm-charts/dbrepo/templates/search-service/secret.yaml b/helm-charts/dbrepo/templates/search-service/secret.yaml index de636d510424d2250fe598463c4eb4b6b6be53b2..834c319e93f2219025e8c8dc27893561109e450c 100644 --- a/helm-charts/dbrepo/templates/search-service/secret.yaml +++ b/helm-charts/dbrepo/templates/search-service/secret.yaml @@ -5,8 +5,8 @@ metadata: name: search-service-secret namespace: {{ .Values.namespace }} stringData: - opensearch-host: "{{ .Values.searchDb.host }}" - opensearch-port: "{{ .Values.searchDb.port }}" - opensearch-username: "{{ .Values.searchDb.username }}" - opensearch-password: "{{ .Values.searchDb.password }}" + opensearch-host: "{{ .Values.searchdb.host }}" + opensearch-port: "{{ .Values.searchdb.port }}" + opensearch-username: "{{ .Values.searchdb.username }}" + opensearch-password: "{{ .Values.searchdb.password }}" log-level: "{{ ternary "DEBUG" "INFO" .Values.searchService.image.debug }}" diff --git a/helm-charts/dbrepo/templates/storage-service/secret.yaml b/helm-charts/dbrepo/templates/storage-service/secret.yaml index baf13a16ad937e93c364e6fbd4e1a594c10c227c..fb63db7f721a7fa356044df5c8aa41db7d6f9fd0 100644 --- a/helm-charts/dbrepo/templates/storage-service/secret.yaml +++ b/helm-charts/dbrepo/templates/storage-service/secret.yaml @@ -14,8 +14,8 @@ stringData: "name": "admin", "credentials": [ { - "accessKey": "{{ .Values.storageService.s3.auth.username }}", - "secretKey": "{{ .Values.storageService.s3.auth.password }}" + "accessKey": "{{ .Values.storageservice.s3.auth.username }}", + "secretKey": "{{ .Values.storageservice.s3.auth.password }}" } ], "actions": [ diff --git a/helm-charts/dbrepo/templates/upload-service/secret.yaml b/helm-charts/dbrepo/templates/upload-service/secret.yaml index ba309cc7ec41b7390f46c5e12d0a2893e31bee18..64d24c4396c7d6a84c2b6aef0da4b3d1c1043161 100644 --- a/helm-charts/dbrepo/templates/upload-service/secret.yaml +++ b/helm-charts/dbrepo/templates/upload-service/secret.yaml @@ -6,7 +6,7 @@ metadata: name: upload-service-secret namespace: {{ .Values.namespace }} stringData: - aws-access-key-id: "{{ .Values.storageService.s3.auth.username }}" - aws-secret-access-key: "{{ .Values.storageService.s3.auth.password }}" + aws-access-key-id: "{{ .Values.storageservice.s3.auth.username }}" + aws-secret-access-key: "{{ .Values.storageservice.s3.auth.password }}" aws-region: "default" {{- end }} \ No newline at end of file diff --git a/helm-charts/dbrepo/values.yaml b/helm-charts/dbrepo/values.yaml index ad8e6da8272de514cd310ea6ce2a0413054d4812..bad06f041513082f949e67553bb3aa88c5a7fed2 100644 --- a/helm-charts/dbrepo/values.yaml +++ b/helm-charts/dbrepo/values.yaml @@ -1,6 +1,6 @@ -namespace: +namespace: "" -hostname: +hostname: "" strategyType: RollingUpdate @@ -156,7 +156,7 @@ dataDb: sharedStorageClass: default replicaCount: 3 # uneven -searchDb: +searchdb: fullnameOverride: search-db host: search-db port: 9200 @@ -397,8 +397,7 @@ searchService: debug: false replicaCount: 2 -storageService: - fullnameOverride: storage-service +storageservice: master: enabled: true filer: @@ -428,7 +427,7 @@ storageService: username: seaweedfsadmin password: seaweedfsadmin -logService: +logservice: fullnameOverride: log-service config: outputs: |