diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e5545013834cec57068909664ed0f632329ceff5..cb06af28364f2560a9aeca24163e21a120afbb1f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,288 +24,288 @@ cache: - .m2/repository stages: -# - build -# - lint + - build + - lint - test -# - docs -# - release -# - verify -# - scan -# -#build-metadata-service: -# image: maven:3-openjdk-17 -# stage: build -# script: -# - "mvn -f ./dbrepo-metadata-service/pom.xml clean install $MAVEN_OPTS -DskipTests" -# # Compiled classes are needed for SonarQube in later stages -# artifacts: -# when: always -# paths: -# - ./dbrepo-metadata-service/test/target/classes -# - ./dbrepo-metadata-service/services/target/classes -# - ./dbrepo-metadata-service/repositories/target/classes -# - ./dbrepo-metadata-service/rest-service/target/classes -# - ./dbrepo-metadata-service/api/target/classes -# - ./dbrepo-metadata-service/oai/target/classes -# - ./dbrepo-metadata-service/entities/target/classes -# expire_in: 1 days -# -#build-analyse-service: -# image: docker.io/python:3.11-alpine -# stage: build -# variables: -# PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile" -# script: -# - "pip install pipenv" -# - "pipenv install gunicorn && pipenv install --dev --system --deploy" -# -#build-data-db-sidecar: -# image: docker.io/python:3.11-alpine -# stage: build -# variables: -# PIPENV_PIPFILE: "./dbrepo-data-db/sidecar/Pipfile" -# script: -# - "pip install pipenv" -# - "pipenv install gunicorn && pipenv install --dev --system --deploy" -# -#build-lib: -# image: docker.io/python:3.11-alpine -# stage: build -# variables: -# PIPENV_PIPFILE: "./lib/python/Pipfile" -# script: -# - "pip install pipenv" -# - "pipenv install gunicorn && pipenv install --dev --system --deploy" -# -#build-data-service: -# image: maven:3-openjdk-17 -# stage: build -# needs: -# - build-metadata-service -# dependencies: -# - build-metadata-service -# script: -# - "mvn -f ./dbrepo-metadata-service/pom.xml clean install $MAVEN_OPTS -DskipTests" -# - "mvn -f ./dbrepo-data-service/pom.xml clean package $MAVEN_OPTS -DskipTests" -# # Compiled classes are needed for SonarQube in later stages -# artifacts: -# when: always -# paths: -# - ./dbrepo-data-service/services/target/classes -# - ./dbrepo-data-service/rest-service/target/classes -# - ./dbrepo-data-service/querystore/target/classes -# expire_in: 1 days -# -#build-ui: -# image: oven/bun:1.1.20-alpine -# stage: build -# script: -# - "cd ./dbrepo-ui && bun install && bun run build" -# -#build-search-service: -# image: docker.io/python:3.11-alpine -# stage: build -# script: -# - "pip install pipenv" -# - "cd dbrepo-search-service && pipenv install --system --deploy" -# -#build-docker: -# image: docker.io/docker:24-dind -# stage: build -# before_script: -# - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL -# script: -# - "docker build -t dbrepo-metadata-service:build --target build dbrepo-metadata-service" -# - "docker build -t dbrepo-data-service:build --target build dbrepo-data-service" -# - "docker compose build --parallel" -# -#build-helm: -# image: docker.io/docker:24-dind -# stage: build -# before_script: -# - echo "$CI_GPG_KEYRING" | base64 -d > ./secring.gpg -# - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL -# script: -# - apk add sed helm curl -# - helm package ./helm/dbrepo --destination ./build -# -#lint-docker-compose: -# image: docker.io/alpine:3.18 -# stage: lint -# variables: -# VERSION: 3.3.0 -# BINARY: yq_linux_amd64 -# before_script: -# - 'apk --no-cache add bash wget' -# - 'wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /usr/bin/yq && chmod +x /usr/bin/yq' -# - 'ls -la .scripts' -# script: -# - "yq compare -P docker-compose.yml .docker/docker-compose.yml 'volumes.*'" -# - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-analyse-service'" -# - "bash .scripts/check-service.sh 'dbrepo-auth-db'" -# - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-auth-service'" -# - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-broker-service'" -# - "IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-dashboard-service'" -# - "bash .scripts/check-service.sh 'dbrepo-data-db'" -# - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-data-db-sidecar'" -# - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-data-service'" -# - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-gateway-service'" -# - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-identity-service'" -# - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-metadata-db'" -# - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-metadata-service'" -# - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-metric-db'" -# - "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-search-db'" -# - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-search-service'" -# - "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-search-service-init'" -# - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-storage-service'" -# - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-storage-service-init'" -# - "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-ui'" -# - "bash .scripts/check-service.sh 'dbrepo-upload-service'" -# -#verify-install-script: -# image: docker.io/docker:24-dind -# stage: verify -# only: -# refs: -# - /^release-.*/ -# variables: -# SKIP_CHECKS: 1 -# script: -# - apk add bash curl -# - bash install.sh -# - exit 0 -# -#lint-helm-chart: -# image: docker.io/alpine:3.20 -# stage: lint -# needs: -# - build-metadata-service -# dependencies: -# - build-metadata-service -# before_script: -# - apk add helm -# script: -# - helm lint ./helm/dbrepo -# -#test-metadata-service: -# image: maven:3-openjdk-17 -# stage: test -# needs: -# - build-metadata-service -# dependencies: -# - build-metadata-service -# script: -# - "mvn -f ./dbrepo-metadata-service/pom.xml clean test $MAVEN_OPTS verify" -# - "cat ./dbrepo-metadata-service/report/target/site/jacoco-aggregate/index.html | grep -o 'Total[^%]*%' | sed 's/<.*>/ /; s/Total/Jacoco Coverage Total:/'" -# artifacts: -# when: always -# paths: -# - ./dbrepo-metadata-service/report/target/site/jacoco-aggregate/ -# - ./dbrepo-metadata-service/rest-service/target/surefire-reports/ -# expire_in: 1 days -# reports: -# junit: ./dbrepo-metadata-service/rest-service/target/surefire-reports/TEST-*.xml -# coverage: '/Total.*?([0-9]{1,3})%/' -# -#test-data-service: -# image: maven:3-openjdk-17 -# stage: test -# needs: -# - build-data-service -# dependencies: -# - build-data-service -# script: -# - "mvn -f ./dbrepo-data-service/pom.xml clean test verify $MAVEN_OPTS" -# - "cat ./dbrepo-data-service/report/target/site/jacoco-aggregate/index.html | grep -o 'Total[^%]*%' | sed 's/<.*>/ /; s/Total/Jacoco Coverage Total:/'" -# artifacts: -# when: always -# paths: -# - ./dbrepo-data-service/report/target/site/jacoco-aggregate/ -# - ./dbrepo-data-service/rest-service/target/surefire-reports/ -# expire_in: 1 days -# reports: -# junit: ./dbrepo-data-service/rest-service/target/surefire-reports/TEST-*.xml -# coverage: '/Total.*?([0-9]{1,3})%/' -# -#test-analyse-service: -# image: docker.io/python:3.11-alpine -# stage: test -# variables: -# PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile" -# needs: -# - build-analyse-service -# dependencies: -# - build-analyse-service -# script: -# - "pip install pipenv" -# - "pipenv install gunicorn && pipenv install --dev --system --deploy" -# - cd ./dbrepo-analyse-service/ && coverage run -m pytest test/test_determine_dt.py test/test_determine_pk.py test/test_s3_client.py --junitxml=report.xml && coverage html --omit="test/*" && coverage report --omit="test/*" > ./coverage.txt -# - "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'" -# artifacts: -# when: always -# paths: -# - ./dbrepo-analyse-service/report.xml -# - ./dbrepo-analyse-service/coverage.txt -# expire_in: 1 days -# reports: -# junit: ./dbrepo-analyse-service/report.xml -# coverage: '/TOTAL.*?([0-9]{1,3})%/' -# -#test-search-service: -# image: docker.io/python:3.11-alpine -# stage: test -# variables: -# PIPENV_PIPFILE: "./dbrepo-search-service/Pipfile" -# needs: -# - build-search-service -# dependencies: -# - build-search-service -# script: -# - "pip install pipenv" -# - "pipenv install gunicorn && pipenv install --dev --system --deploy" -# - cd ./dbrepo-search-service/ && coverage run -m pytest test/test_opensearch_client.py --junitxml=report.xml && coverage html --omit="test/*,omlib/*" && coverage report --omit="test/*,omlib/*" > ./coverage.txt -# - "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'" -# artifacts: -# when: always -# paths: -# - ./dbrepo-search-service/report.xml -# - ./dbrepo-search-service/coverage.txt -# expire_in: 1 days -# reports: -# junit: ./dbrepo-search-service/report.xml -# coverage: '/TOTAL.*?([0-9]{1,3})%/' -# -#test-lib: -# image: docker.io/python:3.11-alpine -# stage: test -# variables: -# PIPENV_PIPFILE: "./lib/python/Pipfile" -# needs: -# - build-lib -# dependencies: -# - build-lib -# script: -# - "pip install pipenv" -# - "pipenv install gunicorn && pipenv install --dev --system --deploy" -# - cd ./lib/python/ && coverage run -m pytest tests/test_unit_analyse.py tests/test_unit_container.py tests/test_unit_database.py tests/test_unit_identifier.py tests/test_unit_license.py tests/test_unit_query.py tests/test_unit_rest_client.py tests/test_unit_table.py tests/test_unit_user.py tests/test_unit_view.py --junitxml=report.xml && coverage html --omit="test/*" && coverage report --omit="test/*" > ./coverage.txt -# - "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'" -# artifacts: -# when: always -# paths: -# - ./lib/python/report.xml -# - ./lib/python/coverage.txt -# expire_in: 1 days -# reports: -# junit: ./lib/python/report.xml -# coverage: '/TOTAL.*?([0-9]{1,3})%/' + - docs + - release + - verify + - scan + +build-metadata-service: + image: maven:3-openjdk-17 + stage: build + script: + - "mvn -f ./dbrepo-metadata-service/pom.xml clean install $MAVEN_OPTS -DskipTests" + # Compiled classes are needed for SonarQube in later stages + artifacts: + when: always + paths: + - ./dbrepo-metadata-service/test/target/classes + - ./dbrepo-metadata-service/services/target/classes + - ./dbrepo-metadata-service/repositories/target/classes + - ./dbrepo-metadata-service/rest-service/target/classes + - ./dbrepo-metadata-service/api/target/classes + - ./dbrepo-metadata-service/oai/target/classes + - ./dbrepo-metadata-service/entities/target/classes + expire_in: 1 days + +build-analyse-service: + image: docker.io/python:3.11-alpine + stage: build + variables: + PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile" + script: + - "pip install pipenv" + - "pipenv install gunicorn && pipenv install --dev --system --deploy" + +build-data-db-sidecar: + image: docker.io/python:3.11-alpine + stage: build + variables: + PIPENV_PIPFILE: "./dbrepo-data-db/sidecar/Pipfile" + script: + - "pip install pipenv" + - "pipenv install gunicorn && pipenv install --dev --system --deploy" + +build-lib: + image: docker.io/python:3.11-alpine + stage: build + variables: + PIPENV_PIPFILE: "./lib/python/Pipfile" + script: + - "pip install pipenv" + - "pipenv install gunicorn && pipenv install --dev --system --deploy" + +build-data-service: + image: maven:3-openjdk-17 + stage: build + needs: + - build-metadata-service + dependencies: + - build-metadata-service + script: + - "mvn -f ./dbrepo-metadata-service/pom.xml clean install $MAVEN_OPTS -DskipTests" + - "mvn -f ./dbrepo-data-service/pom.xml clean package $MAVEN_OPTS -DskipTests" + # Compiled classes are needed for SonarQube in later stages + artifacts: + when: always + paths: + - ./dbrepo-data-service/services/target/classes + - ./dbrepo-data-service/rest-service/target/classes + - ./dbrepo-data-service/querystore/target/classes + expire_in: 1 days + +build-ui: + image: oven/bun:1.1.20-alpine + stage: build + script: + - "cd ./dbrepo-ui && bun install && bun run build" + +build-search-service: + image: docker.io/python:3.11-alpine + stage: build + script: + - "pip install pipenv" + - "cd dbrepo-search-service && pipenv install --system --deploy" + +build-docker: + image: docker.io/docker:24-dind + stage: build + before_script: + - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL + script: + - "docker build -t dbrepo-metadata-service:build --target build dbrepo-metadata-service" + - "docker build -t dbrepo-data-service:build --target build dbrepo-data-service" + - "docker compose build --parallel" + +build-helm: + image: docker.io/docker:24-dind + stage: build + before_script: + - echo "$CI_GPG_KEYRING" | base64 -d > ./secring.gpg + - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL + script: + - apk add sed helm curl + - helm package ./helm/dbrepo --destination ./build + +lint-docker-compose: + image: docker.io/alpine:3.18 + stage: lint + variables: + VERSION: 3.3.0 + BINARY: yq_linux_amd64 + before_script: + - 'apk --no-cache add bash wget' + - 'wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /usr/bin/yq && chmod +x /usr/bin/yq' + - 'ls -la .scripts' + script: + - "yq compare -P docker-compose.yml .docker/docker-compose.yml 'volumes.*'" + - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-analyse-service'" + - "bash .scripts/check-service.sh 'dbrepo-auth-db'" + - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-auth-service'" + - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-broker-service'" + - "IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-dashboard-service'" + - "bash .scripts/check-service.sh 'dbrepo-data-db'" + - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-data-db-sidecar'" + - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-data-service'" + - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-gateway-service'" + - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-identity-service'" + - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-metadata-db'" + - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-metadata-service'" + - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-metric-db'" + - "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-search-db'" + - "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-search-service'" + - "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-search-service-init'" + - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-storage-service'" + - "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-storage-service-init'" + - "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-ui'" + - "bash .scripts/check-service.sh 'dbrepo-upload-service'" + +verify-install-script: + image: docker.io/docker:24-dind + stage: verify + only: + refs: + - /^release-.*/ + variables: + SKIP_CHECKS: 1 + script: + - apk add bash curl + - bash install.sh + - exit 0 + +lint-helm-chart: + image: docker.io/alpine:3.20 + stage: lint + needs: + - build-metadata-service + dependencies: + - build-metadata-service + before_script: + - apk add helm + script: + - helm lint ./helm/dbrepo + +test-metadata-service: + image: maven:3-openjdk-17 + stage: test + needs: + - build-metadata-service + dependencies: + - build-metadata-service + script: + - "mvn -f ./dbrepo-metadata-service/pom.xml clean test $MAVEN_OPTS verify" + - "cat ./dbrepo-metadata-service/report/target/site/jacoco-aggregate/index.html | grep -o 'Total[^%]*%' | sed 's/<.*>/ /; s/Total/Jacoco Coverage Total:/'" + artifacts: + when: always + paths: + - ./dbrepo-metadata-service/report/target/site/jacoco-aggregate/ + - ./dbrepo-metadata-service/rest-service/target/surefire-reports/ + expire_in: 1 days + reports: + junit: ./dbrepo-metadata-service/rest-service/target/surefire-reports/TEST-*.xml + coverage: '/Total.*?([0-9]{1,3})%/' + +test-data-service: + image: maven:3-openjdk-17 + stage: test + needs: + - build-data-service + dependencies: + - build-data-service + script: + - "mvn -f ./dbrepo-data-service/pom.xml clean test verify $MAVEN_OPTS" + - "cat ./dbrepo-data-service/report/target/site/jacoco-aggregate/index.html | grep -o 'Total[^%]*%' | sed 's/<.*>/ /; s/Total/Jacoco Coverage Total:/'" + artifacts: + when: always + paths: + - ./dbrepo-data-service/report/target/site/jacoco-aggregate/ + - ./dbrepo-data-service/rest-service/target/surefire-reports/ + expire_in: 1 days + reports: + junit: ./dbrepo-data-service/rest-service/target/surefire-reports/TEST-*.xml + coverage: '/Total.*?([0-9]{1,3})%/' + +test-analyse-service: + image: docker.io/python:3.11-alpine + stage: test + variables: + PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile" + needs: + - build-analyse-service + dependencies: + - build-analyse-service + script: + - "pip install pipenv" + - "pipenv install gunicorn && pipenv install --dev --system --deploy" + - cd ./dbrepo-analyse-service/ && coverage run -m pytest test/test_determine_dt.py test/test_determine_pk.py test/test_s3_client.py --junitxml=report.xml && coverage html --omit="test/*" && coverage report --omit="test/*" > ./coverage.txt + - "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'" + artifacts: + when: always + paths: + - ./dbrepo-analyse-service/report.xml + - ./dbrepo-analyse-service/coverage.txt + expire_in: 1 days + reports: + junit: ./dbrepo-analyse-service/report.xml + coverage: '/TOTAL.*?([0-9]{1,3})%/' + +test-search-service: + image: docker.io/python:3.11-alpine + stage: test + variables: + PIPENV_PIPFILE: "./dbrepo-search-service/Pipfile" + needs: + - build-search-service + dependencies: + - build-search-service + script: + - "pip install pipenv" + - "pipenv install gunicorn && pipenv install --dev --system --deploy" + - cd ./dbrepo-search-service/ && coverage run -m pytest test/test_opensearch_client.py --junitxml=report.xml && coverage html --omit="test/*,omlib/*" && coverage report --omit="test/*,omlib/*" > ./coverage.txt + - "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'" + artifacts: + when: always + paths: + - ./dbrepo-search-service/report.xml + - ./dbrepo-search-service/coverage.txt + expire_in: 1 days + reports: + junit: ./dbrepo-search-service/report.xml + coverage: '/TOTAL.*?([0-9]{1,3})%/' + +test-lib: + image: docker.io/python:3.11-alpine + stage: test + variables: + PIPENV_PIPFILE: "./lib/python/Pipfile" + needs: + - build-lib + dependencies: + - build-lib + script: + - "pip install pipenv" + - "pipenv install gunicorn && pipenv install --dev --system --deploy" + - cd ./lib/python/ && coverage run -m pytest tests/test_unit_analyse.py tests/test_unit_container.py tests/test_unit_database.py tests/test_unit_identifier.py tests/test_unit_license.py tests/test_unit_query.py tests/test_unit_rest_client.py tests/test_unit_table.py tests/test_unit_user.py tests/test_unit_view.py --junitxml=report.xml && coverage html --omit="test/*" && coverage report --omit="test/*" > ./coverage.txt + - "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'" + artifacts: + when: always + paths: + - ./lib/python/report.xml + - ./lib/python/coverage.txt + expire_in: 1 days + reports: + junit: ./lib/python/report.xml + coverage: '/TOTAL.*?([0-9]{1,3})%/' test-ui: stage: test image: docker.io/docker:24-dind -# needs: -# - build-ui -# dependencies: -# - build-ui + needs: + - build-ui + dependencies: + - build-ui before_script: - "apk add bash apache2-utils" - "docker compose build dbrepo-ui" @@ -314,125 +314,125 @@ test-ui: - "sleep 30" - "ENDPOINT=http://localhost:3000 bash ./dbrepo-ui/test/test_heap.sh" -#scan-sonarqube: -# image: sonarsource/sonar-scanner-cli:10.0 -# stage: scan -# only: -# refs: -# - master -# needs: -# - build-data-service -# - build-metadata-service -# dependencies: -# - build-data-service -# - build-metadata-service -# script: -# - 'sonar-scanner -Dsonar.token="${CI_SONAR_TOKEN}"' -# allow_failure: true -# cache: -# policy: pull -# key: "${CI_COMMIT_SHORT_SHA}" -# paths: -# - sonar-scanner/ -# -#docs-registry: -# stage: docs -# image: docker.io/python:3.11-slim -# only: -# refs: -# - /^release-.*/ -# before_script: -# - "apt-get update && apt-get install -y sed" -# script: -# - pip install -r ./requirements.txt -# - python3 .docs/docker/release.py -# -#release-images: -# stage: release -# image: docker:24-dind -# dependencies: -# - test-metadata-service -# - test-data-service -# - test-analyse-service -# only: -# refs: -# - /^release-.*/ -# before_script: -# - "docker logout ${CI_REGISTRY_URL}" -# - "echo ${CI_REGISTRY_PASSWORD} | docker login --username ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY_URL}" -# - "docker logout ${CI_REGISTRY2_URL}" -# - "echo ${CI_REGISTRY2_PASSWORD} | docker login --username ${CI_REGISTRY2_USER} --password-stdin ${CI_REGISTRY2_URL}" -# - "ifconfig eth0 mtu 1450 up" -# - "apk add make bash" -# script: -# - "make release-images" -# -#release-helm: -# stage: release -# image: docker:24-dind -# only: -# refs: -# - /^release-.*/ -# when: manual -# before_script: -# - "docker logout ${CI_REGISTRY_URL}" -# - "echo ${CI_REGISTRY_PASSWORD} | docker login --username ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY_URL}" -# - "docker logout ${CI_REGISTRY2_URL}" -# - "echo ${CI_REGISTRY2_PASSWORD} | docker login --username ${CI_REGISTRY2_USER} --password-stdin ${CI_REGISTRY2_URL}" -# - "apk add sed helm curl" -# - "mkdir -p ~/.gnupg" -# - echo "$CI_GPG_KEYRING" | base64 -d > ~/.gnupg/secring.gpg -# - echo "$CI_GPG_KEYRING2" | base64 -d > ~/.gnupg/pubring.gpg -# - "helm package ./helm/dbrepo --sign --key 'Martin Weise' --keyring ~/.gnupg/secring.gpg --destination ./build" -# - "helm plugin install https://github.com/sigstore/helm-sigstore" -# script: -# - "helm push ./build/dbrepo-${CHART_VERSION}.tgz oci://${CI_REGISTRY2_URL}/helm" -# - "helm sigstore upload ./build/dbrepo-${CHART_VERSION}.tgz" -# -#release-docs: -# stage: release -# image: docker.io/python:3.11-alpine -# only: -# refs: -# - /^release-.*/ -# before_script: -# - "apk add --update alpine-sdk bash sed wget openssh jq curl" -# - "pip install pipenv" -# - "pip install -r ./requirements.txt" -# - "mkdir -p ./final/${APP_VERSION}/rest" -# script: -# - "make gen-lib-doc gen-docs-doc package-config" -# - "cp -r ./lib/python/docs/build/html ./final/${APP_VERSION}/python" # sphinx -# - "cp .docs/.swagger/api.yaml ./final/${APP_VERSION}/rest/api.yaml" # swagger -# - "cp .docs/.swagger/swagger-ui.html ./final/${APP_VERSION}/rest/index.html" # swagger -# - "cp .docs/.swagger/custom.css ./final/${APP_VERSION}/rest/custom.css" # swagger -# - "cp -r ./site/* ./final/${APP_VERSION}" # mkdocs -# - "cp .docker/dist.tar.gz ./final/${APP_VERSION}/dist.tar.gz" # dist -# - "bash ./.gitlab/gen-badge.sh" -# - eval $(ssh-agent -s) -# - "mkdir -p /root/.ssh" -# - echo "$CI_KEY_PRIVATE" > /root/.ssh/id_rsa && chmod 0600 /root/.ssh/id_rsa -# - echo "$CI_KEY_PUBLIC" > /root/.ssh/id_rsa.pub -# - echo "$CI_DOC_ID" > ~/.ssh/known_hosts -# - tar czf ./final.tar.gz ./final -# - "scp -oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedAlgorithms=+ssh-rsa final.tar.gz $CI_DOC_USER@$CI_DOC_IP:final.tar.gz" -# - "scp -oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedAlgorithms=+ssh-rsa versions.json $CI_DOC_USER@$CI_DOC_IP:/system/user/ifs/infrastructures/public_html/dbrepo/versions.json" -# - "scp -oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedAlgorithms=+ssh-rsa .docs/redirect.html $CI_DOC_USER@$CI_DOC_IP:/system/user/ifs/infrastructures/public_html/dbrepo/index.html" -# - 'ssh -oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedAlgorithms=+ssh-rsa $CI_DOC_USER@$CI_DOC_IP "rm -rf /system/user/ifs/infrastructures/public_html/dbrepo/${APP_VERSION}; tar xzf ./final.tar.gz; rm -f ./final.tar.gz; cp -r ./final/* /system/user/ifs/infrastructures/public_html/dbrepo/${APP_VERSION}; rm -rf ./final"' -# -#release-libs: -# stage: release -# image: docker.io/python:3.11-alpine -# when: manual -# only: -# refs: -# - /^release-.*/ -# variables: -# PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile" -# before_script: -# - "apk add sed bash" -# - "pip install pipenv twine build" -# - "pipenv install gunicorn && pipenv install --dev --system --deploy" -# script: -# - bash ./lib/python/package.sh -# - bash ./lib/python/release.sh +scan-sonarqube: + image: sonarsource/sonar-scanner-cli:10.0 + stage: scan + only: + refs: + - master + needs: + - build-data-service + - build-metadata-service + dependencies: + - build-data-service + - build-metadata-service + script: + - 'sonar-scanner -Dsonar.token="${CI_SONAR_TOKEN}"' + allow_failure: true + cache: + policy: pull + key: "${CI_COMMIT_SHORT_SHA}" + paths: + - sonar-scanner/ + +docs-registry: + stage: docs + image: docker.io/python:3.11-slim + only: + refs: + - /^release-.*/ + before_script: + - "apt-get update && apt-get install -y sed" + script: + - pip install -r ./requirements.txt + - python3 .docs/docker/release.py + +release-images: + stage: release + image: docker:24-dind + dependencies: + - test-metadata-service + - test-data-service + - test-analyse-service + only: + refs: + - /^release-.*/ + before_script: + - "docker logout ${CI_REGISTRY_URL}" + - "echo ${CI_REGISTRY_PASSWORD} | docker login --username ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY_URL}" + - "docker logout ${CI_REGISTRY2_URL}" + - "echo ${CI_REGISTRY2_PASSWORD} | docker login --username ${CI_REGISTRY2_USER} --password-stdin ${CI_REGISTRY2_URL}" + - "ifconfig eth0 mtu 1450 up" + - "apk add make bash" + script: + - "make release-images" + +release-helm: + stage: release + image: docker:24-dind + only: + refs: + - /^release-.*/ + when: manual + before_script: + - "docker logout ${CI_REGISTRY_URL}" + - "echo ${CI_REGISTRY_PASSWORD} | docker login --username ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY_URL}" + - "docker logout ${CI_REGISTRY2_URL}" + - "echo ${CI_REGISTRY2_PASSWORD} | docker login --username ${CI_REGISTRY2_USER} --password-stdin ${CI_REGISTRY2_URL}" + - "apk add sed helm curl" + - "mkdir -p ~/.gnupg" + - echo "$CI_GPG_KEYRING" | base64 -d > ~/.gnupg/secring.gpg + - echo "$CI_GPG_KEYRING2" | base64 -d > ~/.gnupg/pubring.gpg + - "helm package ./helm/dbrepo --sign --key 'Martin Weise' --keyring ~/.gnupg/secring.gpg --destination ./build" + - "helm plugin install https://github.com/sigstore/helm-sigstore" + script: + - "helm push ./build/dbrepo-${CHART_VERSION}.tgz oci://${CI_REGISTRY2_URL}/helm" + - "helm sigstore upload ./build/dbrepo-${CHART_VERSION}.tgz" + +release-docs: + stage: release + image: docker.io/python:3.11-alpine + only: + refs: + - /^release-.*/ + before_script: + - "apk add --update alpine-sdk bash sed wget openssh jq curl" + - "pip install pipenv" + - "pip install -r ./requirements.txt" + - "mkdir -p ./final/${APP_VERSION}/rest" + script: + - "make gen-lib-doc gen-docs-doc package-config" + - "cp -r ./lib/python/docs/build/html ./final/${APP_VERSION}/python" # sphinx + - "cp .docs/.swagger/api.yaml ./final/${APP_VERSION}/rest/api.yaml" # swagger + - "cp .docs/.swagger/swagger-ui.html ./final/${APP_VERSION}/rest/index.html" # swagger + - "cp .docs/.swagger/custom.css ./final/${APP_VERSION}/rest/custom.css" # swagger + - "cp -r ./site/* ./final/${APP_VERSION}" # mkdocs + - "cp .docker/dist.tar.gz ./final/${APP_VERSION}/dist.tar.gz" # dist + - "bash ./.gitlab/gen-badge.sh" + - eval $(ssh-agent -s) + - "mkdir -p /root/.ssh" + - echo "$CI_KEY_PRIVATE" > /root/.ssh/id_rsa && chmod 0600 /root/.ssh/id_rsa + - echo "$CI_KEY_PUBLIC" > /root/.ssh/id_rsa.pub + - echo "$CI_DOC_ID" > ~/.ssh/known_hosts + - tar czf ./final.tar.gz ./final + - "scp -oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedAlgorithms=+ssh-rsa final.tar.gz $CI_DOC_USER@$CI_DOC_IP:final.tar.gz" + - "scp -oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedAlgorithms=+ssh-rsa versions.json $CI_DOC_USER@$CI_DOC_IP:/system/user/ifs/infrastructures/public_html/dbrepo/versions.json" + - "scp -oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedAlgorithms=+ssh-rsa .docs/redirect.html $CI_DOC_USER@$CI_DOC_IP:/system/user/ifs/infrastructures/public_html/dbrepo/index.html" + - 'ssh -oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedAlgorithms=+ssh-rsa $CI_DOC_USER@$CI_DOC_IP "rm -rf /system/user/ifs/infrastructures/public_html/dbrepo/${APP_VERSION}; tar xzf ./final.tar.gz; rm -f ./final.tar.gz; cp -r ./final/* /system/user/ifs/infrastructures/public_html/dbrepo/${APP_VERSION}; rm -rf ./final"' + +release-libs: + stage: release + image: docker.io/python:3.11-alpine + when: manual + only: + refs: + - /^release-.*/ + variables: + PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile" + before_script: + - "apk add sed bash" + - "pip install pipenv twine build" + - "pipenv install gunicorn && pipenv install --dev --system --deploy" + script: + - bash ./lib/python/package.sh + - bash ./lib/python/release.sh