From 32146b9424530c2e2cbc6b7d075d6d7027cf1998 Mon Sep 17 00:00:00 2001
From: Martin Weise <martin.weise@tuwien.ac.at>
Date: Fri, 27 Dec 2024 11:27:31 +0100
Subject: [PATCH] First attempt in GitOps pipeline
---
.gitlab-ci.yml | 184 ++++++++++++++++++---------------
.gitlab/agents/dev/config.yaml | 0
.gitlab/agents/dev/values.yaml | 161 +++++++++++++++++++++++++++++
3 files changed, 261 insertions(+), 84 deletions(-)
create mode 100644 .gitlab/agents/dev/config.yaml
create mode 100644 .gitlab/agents/dev/values.yaml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index aba94058e0..03f3c6b903 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -28,14 +28,55 @@ cache:
- .m2/repository
stages:
- - build
- lint
+ - build
+ - deploy
- test
- docs
- release
- verify
- scan
+lint-docker-compose:
+ image: docker.io/alpine:${ALPINE_VERSION}
+ stage: lint
+ variables:
+ VERSION: 3.3.0
+ BINARY: yq_linux_amd64
+ before_script:
+ - 'apk --no-cache add bash wget'
+ - 'wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /usr/bin/yq && chmod +x /usr/bin/yq'
+ - 'ls -la .scripts'
+ script:
+ - "yq compare -P docker-compose.yml .docker/docker-compose.yml 'volumes.*'"
+ - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-analyse-service'"
+ - "bash .scripts/check-service.sh 'dbrepo-auth-db'"
+ - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-auth-service'"
+ - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-broker-service'"
+ - "IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-dashboard-service'"
+ - "bash .scripts/check-service.sh 'dbrepo-data-db'"
+ - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-data-service'"
+ - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-gateway-service'"
+ - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-identity-service'"
+ - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-metadata-db'"
+ - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-metadata-service'"
+ - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-metric-db'"
+ - "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-search-db'"
+ - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-search-service'"
+ - "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-search-service-init'"
+ - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-storage-service'"
+ - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-storage-service-init'"
+ - "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-ui'"
+ - "bash .scripts/check-service.sh 'dbrepo-upload-service'"
+
+lint-helm-chart:
+ image: docker.io/alpine:3.20
+ stage: lint
+ before_script:
+ - apk add helm
+ script:
+ - helm lint ./helm/dbrepo
+
build-metadata-service:
image: maven:3-openjdk-${JAVA_VERSION}
stage: build
@@ -125,73 +166,24 @@ build-helm:
- apk add sed helm curl
- helm package ./helm/dbrepo --destination ./build
-lint-docker-compose:
+deploy-staging:
image: docker.io/alpine:${ALPINE_VERSION}
- stage: lint
- variables:
- VERSION: 3.3.0
- BINARY: yq_linux_amd64
+ stage: deploy
+ environment:
+ name: staging/datalab
+ url: ${CI_ENV_STAGING_URL}
before_script:
- - 'apk --no-cache add bash wget'
- - 'wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /usr/bin/yq && chmod +x /usr/bin/yq'
- - 'ls -la .scripts'
+ - apk add --no-cache helm make
+ - echo ${CI_K8S_CONFIG} | base64 -d > ./kubecfg
+ - make build-helm
+ - helm -n ${CI_ENV_STAGING_NAMESPACE} uninstall ${CI_ENV_STAGING_RELEASE_NAME}
+ - kubectl -n ${CI_ENV_STAGING_NAMESPACE} delete pvc --all
script:
- - "yq compare -P docker-compose.yml .docker/docker-compose.yml 'volumes.*'"
- - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-analyse-service'"
- - "bash .scripts/check-service.sh 'dbrepo-auth-db'"
- - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-auth-service'"
- - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-broker-service'"
- - "IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-dashboard-service'"
- - "bash .scripts/check-service.sh 'dbrepo-data-db'"
- - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-data-service'"
- - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-gateway-service'"
- - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-identity-service'"
- - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-metadata-db'"
- - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-metadata-service'"
- - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-metric-db'"
- - "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-search-db'"
- - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-search-service'"
- - "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-search-service-init'"
- - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-storage-service'"
- - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-storage-service-init'"
- - "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-ui'"
- - "bash .scripts/check-service.sh 'dbrepo-upload-service'"
-
-verify-install-script:
- image: docker.io/docker:24-dind
- stage: verify
+ - helm -n ${CI_ENV_STAGING_NAMESPACE} upgrade --install ${CI_ENV_STAGING_RELEASE_NAME} ./build/${CI_ENV_STAGING_RELEASE_NAME}-${CHART_VERSION}.tgz --create-namespace -f ./.gitlab/agents/dev/values.yaml
only:
refs:
- - /^release-.*/
- variables:
- SKIP_CHECKS: 1
- before_script:
- - "apk add bash curl"
- script:
- - "curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${DOC_VERSION}/install.sh | bash | grep 'Success!'"
-
-verify-dist:
- image: docker.io/alpine:${ALPINE_VERSION}
- stage: verify
- only:
- refs:
- - /^release-.*/
- before_script:
- - "apk add curl"
- script:
- - "curl -v --output /dev/null --fail https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/${APP_VERSION}/dist.tar.gz"
-
-lint-helm-chart:
- image: docker.io/alpine:3.20
- stage: lint
- needs:
- - build-metadata-service
- dependencies:
- - build-metadata-service
- before_script:
- - apk add helm
- script:
- - helm lint ./helm/dbrepo
+ - dev
+ when: manual
test-metadata-service:
image: maven:3-openjdk-${JAVA_VERSION}
@@ -355,27 +347,6 @@ test-ui:
- "sleep 30"
- "ENDPOINT=http://localhost:3000 bash ./dbrepo-ui/test/test_heap.sh"
-scan-sonarqube:
- image: sonarsource/sonar-scanner-cli:10.0
- stage: scan
- only:
- refs:
- - master
- needs:
- - build-data-service
- - build-metadata-service
- dependencies:
- - build-data-service
- - build-metadata-service
- script:
- - 'sonar-scanner -Dsonar.token="${CI_SONAR_TOKEN}"'
- allow_failure: true
- cache:
- policy: pull
- key: "${CI_COMMIT_SHORT_SHA}"
- paths:
- - sonar-scanner/
-
release-images:
stage: release
image: docker:24-dind
@@ -472,3 +443,48 @@ release-libs:
script:
- bash ./lib/python/package.sh
- bash ./lib/python/release.sh
+
+verify-install-script:
+ image: docker.io/docker:24-dind
+ stage: verify
+ only:
+ refs:
+ - /^release-.*/
+ variables:
+ SKIP_CHECKS: 1
+ before_script:
+ - "apk add bash curl"
+ script:
+ - "curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${DOC_VERSION}/install.sh | bash | grep 'Success!'"
+
+verify-dist:
+ image: docker.io/alpine:${ALPINE_VERSION}
+ stage: verify
+ only:
+ refs:
+ - /^release-.*/
+ before_script:
+ - "apk add curl"
+ script:
+ - "curl -v --output /dev/null --fail https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/${APP_VERSION}/dist.tar.gz"
+
+scan-sonarqube:
+ image: sonarsource/sonar-scanner-cli:10.0
+ stage: scan
+ only:
+ refs:
+ - master
+ needs:
+ - build-data-service
+ - build-metadata-service
+ dependencies:
+ - build-data-service
+ - build-metadata-service
+ script:
+ - 'sonar-scanner -Dsonar.token="${CI_SONAR_TOKEN}"'
+ allow_failure: true
+ cache:
+ policy: pull
+ key: "${CI_COMMIT_SHORT_SHA}"
+ paths:
+ - sonar-scanner/
\ No newline at end of file
diff --git a/.gitlab/agents/dev/config.yaml b/.gitlab/agents/dev/config.yaml
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/.gitlab/agents/dev/values.yaml b/.gitlab/agents/dev/values.yaml
new file mode 100644
index 0000000000..e1641b5077
--- /dev/null
+++ b/.gitlab/agents/dev/values.yaml
@@ -0,0 +1,161 @@
+hostname: s155.datalab.tuwien.ac.at
+gateway: https://s155.datalab.tuwien.ac.at
+
+metadatadb:
+ enabled: true
+ rootUser:
+ user: root
+ password: da19c7cf5c0deba7bd47c174a0eb273b
+ galera:
+ mariabackup:
+ user: mariabackup
+ password: 9e447eeaf3e4b6aa26ea01582f0e8a54
+ persistence:
+ enabled: true
+
+authservice:
+ enabled: true
+ auth:
+ adminUser: admin
+ adminPassword: ea72038fa14b968fc0ed09e182ecf624
+ postgresql:
+ auth:
+ postgresPassword: 129d5b888b8df271fa482da39f15c513
+ jwt:
+ pubkey: "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB"
+ client:
+ id: dbrepo-client
+ secret: MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG
+ persistence:
+ enabled: true
+
+brokerservice:
+ enabled: true
+ ldap:
+ bindpw: b8534187c9adf9618e7bd1c79c7f4639
+identityservice:
+ enabled: true
+ global:
+ adminUser: admin
+ adminPassword: b8534187c9adf9618e7bd1c79c7f4639
+ users: admin
+ userPasswords: ea72038fa14b968fc0ed09e182ecf624
+
+datadb:
+ enabled: true
+ rootUser:
+ user: root
+ password: fdf8578499b2083eb3aa03a861ac7912
+ galera:
+ mariabackup:
+ user: mariabackup
+ password: ef60e32e3217525474635cd28422c829
+ replicaCount: 3
+ persistence:
+ enabled: true
+
+searchdb:
+ enabled: true
+ security:
+ enabled: false
+ extraEnvs:
+ - name: DISABLE_INSTALL_DEMO_CONFIG
+ value: "true"
+ persistence:
+ enabled: true
+
+analyseservice:
+ enabled: true
+
+metadataservice:
+ enabled: true
+ admin:
+ email: noreply@example.com
+ deletedRecord: permanent
+ repositoryName: Database Repository
+ granularity: YYYY-MM-DDThh:mm:ssZ
+ datacite:
+ enabled: false
+ url: https://api.datacite.org
+ prefix: ""
+ username: ""
+ password: ""
+
+dataservice:
+ enabled: true
+ rabbitmq:
+ consumer:
+ username: admin
+ password: ea72038fa14b968fc0ed09e182ecf624
+ s3:
+ auth:
+ username: a45e7a77607a8906e92237f00ea72f58
+ password: e2c4303dcbfd3a2c606fe30d19fcb82b
+ filePath: /s3
+
+searchservice:
+ enabled: true
+
+storageservice:
+ enabled: true
+
+uploadservice:
+ enabled: true
+
+dashboardservice:
+ enabled: true
+
+metricdb:
+ enabled: true
+ server:
+ rbac:
+ create: false
+
+ui:
+ enabled: true
+ public:
+ api:
+ client: https://s155.datalab.tuwien.ac.at
+ server: https://s155.datalab.tuwien.ac.at
+ title: "Database Repository"
+ logo: "https://s155.datalab.tuwien.ac.at/assets/logo.png"
+ icon: "https://s155.datalab.tuwien.ac.at/assets/favicon.png"
+ touch: "https://s155.datalab.tuwien.ac.at/assets/favicon.png"
+ broker:
+ host: s155.datalab.tuwien.ac.at
+ extra: "128.130.0.0/15"
+ database:
+ extra: "128.130.0.0/15"
+ pid:
+ default:
+ publisher: "TU Wien"
+ doi:
+ enabled: false
+ endpoint: https://doi.org
+ extraVolumes: [ ]
+ # - name: images-map
+ # configMap:
+ # name: ui-config
+ extraVolumeMounts: [ ]
+ # - name: images-map
+ # mountPath: /static/logo.svg
+ # subPath: logo.svg
+
+gatewayservice:
+ extraVolumes:
+ - name: config-map
+ configMap:
+ name: gateway-service-config
+ extraVolumeMounts:
+ - name: config-map
+ mountPath: /etc/nginx/assets/assets
+
+ingress:
+ enabled: true
+ className: nginx
+ tls:
+ enabled: true
+ secretName: ingress-cert
+ annotations:
+ cert-manager.io/cluster-issuer: letsencrypt-cluster-issuer
+# nginx.ingress.kubernetes.io/whitelist-source-range: 128.130.0.0/15
--
GitLab