From ba2d4cdffabb471c2c27940946c226613e0b03bd Mon Sep 17 00:00:00 2001
From: Martin Weise <martin.weise@tuwien.ac.at>
Date: Thu, 7 Mar 2024 11:51:45 +0000
Subject: [PATCH] Updated the CI/CD pipeline
---
.docs/build-website.sh | 6 +-
.gitlab-ci.yml | 284 ++++++-----------------------------------
Dockerfile | 20 ---
Makefile | 3 +
docker-compose.yml | 9 --
5 files changed, 43 insertions(+), 279 deletions(-)
delete mode 100644 Dockerfile
diff --git a/.docs/build-website.sh b/.docs/build-website.sh
index 5020643f39..477eff8e7c 100755
--- a/.docs/build-website.sh
+++ b/.docs/build-website.sh
@@ -43,11 +43,11 @@ function generate_api {
}
# usage
-if [ -z "$VERSIONS" ]; then
- echo "Variable VERSIONS not set"
+if [ -z "$DOC_VERSIONS" ]; then
+ echo "Variable DOC_VERSIONS not set"
exit 1
fi
-versions=(${VERSIONS//,/ })
+versions=(${DOC_VERSIONS//,/ })
# usage
if [ -z "$APP_VERSION" ]; then
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c25100d5bb..8e2be77ab7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,8 +4,9 @@ variables:
TRIVY_CACHE_DIR: ".trivycache/"
DOCKER_HOST: "unix:///var/run/dind/docker.sock"
TESTCONTAINERS_RYUK_DISABLED: "false"
- APP_VERSION: "1.4.0"
- CHART_VERSION: "1.4.0"
+ DOC_VERSIONS: "latest,1.4.1,1.4.0,1.3.0"
+ APP_VERSION: "1.4.1"
+ CHART_VERSION: "1.4.1"
image: debian:12-slim
@@ -458,182 +459,12 @@ scan-storage-service-init:
reports:
container_scanning: ./.trivy/trivy-storage-service-init-report.json
-release-latest:
- stage: release
- image: docker:24-dind
- needs:
- - test-metadata-service
- - test-data-service
- - test-analyse-service
- - test-frontend
- only:
- refs:
- - master
- before_script:
- - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL
- - echo "$CI_REGISTRY2_PASSWORD" | docker login --username "$CI_REGISTRY2_USER" --password-stdin $CI_REGISTRY2_URL
- script:
- - "ifconfig eth0 mtu 1450 up"
- - "apk add make bash"
- - TAG=latest make release
-
-release-1.3:
- stage: release
- image: docker:24-dind
- needs:
- - test-metadata-service
- - test-data-service
- - test-analyse-service
- - test-frontend
- only:
- refs:
- - release-v1.3
- before_script:
- - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL
- - echo "$CI_REGISTRY2_PASSWORD" | docker login --username "$CI_REGISTRY2_USER" --password-stdin $CI_REGISTRY2_URL
- script:
- - "ifconfig eth0 mtu 1450 up"
- - "apk add make bash"
- - "TAG=1.3.0 make release"
-
-release-1.4:
- stage: release
- image: docker:24-dind
- needs:
- - test-metadata-service
- - test-data-service
- - test-analyse-service
- - test-frontend
- only:
- refs:
- - release-v1.4
- before_script:
- - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL
- - echo "$CI_REGISTRY2_PASSWORD" | docker login --username "$CI_REGISTRY2_USER" --password-stdin $CI_REGISTRY2_URL
- script:
- - "ifconfig eth0 mtu 1450 up"
- - "apk add make bash"
- - "TAG=${APP_VERSION} make release"
-
-release-helm-1.4:
- stage: release
- image: docker:24-dind
- only:
- refs:
- - release-v1.4
- before_script:
- - echo "$CI_REGISTRY2_PASSWORD" | docker login --username "$CI_REGISTRY2_USER" --password-stdin $CI_REGISTRY2_URL
- script:
- - apk add sed helm curl
- - 'sed -i -e "s/^version:.*/version: \"${CHART_VERSION}\"/g" ./helm-charts/dbrepo/Chart.yaml'
- - 'sed -i -e "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/g" ./helm-charts/dbrepo/Chart.yaml'
- - find ./helm-charts -type f -exec sed -i -e "s/__CHARTVERSION__/${CHART_VERSION}/g" {} \;
- - helm package ./helm-charts/dbrepo --destination ./build
- - helm push "./build/dbrepo-${CHART_VERSION}.tgz" "oci://${CI_REGISTRY2_URL}/helm"
-
-build-api-latest:
- stage: build
- image: docker.io/alpine:3.18
- only:
- refs:
- - master
- - release-v1.3
- - release-v1.4
- script:
- - apk add bash git
- - git fetch && git checkout master
- - bash .docs/.swagger/swagger-site.sh
- - find ./site -type f -exec sed -i -e "s/__APPVERSION__/latest/g" {} \;
- - mkdir -p ./swagger/latest
- - cp -r ./site/* ./swagger/latest/
- cache:
- paths:
- - ./swagger/latest
- artifacts:
- when: always
- paths:
- - ./swagger/latest
- expire_in: 1 days
-
-build-api-1.3:
- stage: build
- image: docker.io/alpine:3.18
- only:
- refs:
- - master
- - release-v1.3
- - release-v1.4
- script:
- - apk add bash git maven
- - git fetch && git checkout release-v1.3
- - bash .docs/.swagger/generate.sh
- - mkdir -p ./swagger/1.3.0
- - cp -r ./site/* ./swagger/1.3.0/
- cache:
- paths:
- - ./swagger/1.3.0
- artifacts:
- when: always
- paths:
- - ./swagger/1.3.0
- expire_in: 1 days
-
-build-api-1.4.0:
- stage: build
- image: docker.io/alpine:3.18
- only:
- refs:
- - master
- - release-v1.3
- - release-v1.4
- script:
- - apk add bash git maven
- - git fetch --tags && git checkout v1.4.0
- - bash .docs/.swagger/swagger-site.sh
- - find ./site -type f -exec sed -i -e "s/__APPVERSION__/1.4.0/g" {} \;
- - mkdir -p ./swagger/1.4.0
- - cp -r ./site/* ./swagger/1.4.0/
- cache:
- paths:
- - ./swagger/1.4.0
- artifacts:
- when: always
- paths:
- - ./swagger/1.4.0
- expire_in: 1 days
-
-build-api-1.4:
- stage: build
- image: docker.io/alpine:3.18
- only:
- refs:
- - master
- - release-v1.3
- - release-v1.4
- script:
- - apk add bash git maven
- - git fetch && git checkout release-v1.4
- - bash .docs/.swagger/swagger-site.sh
- - find ./site -type f -exec sed -i -e "s/__APPVERSION__/${APP_VERSION}/g" {} \;
- - mkdir -p ./swagger/${APP_VERSION}
- - cp -r ./site/* ./swagger/${APP_VERSION}/
- cache:
- paths:
- - ./swagger/${APP_VERSION}
- artifacts:
- when: always
- paths:
- - ./swagger/${APP_VERSION}
- expire_in: 1 days
-
docs-registry:
stage: docs
image: docker.io/python:3.11-slim
only:
refs:
- - master
- - release-v1.3
- - release-v1.4
+ - /^release-.*/
script:
- pip install -r ./requirements.txt
- python3 .docs/docker/release.py
@@ -646,25 +477,15 @@ docs-registry:
- ./final
expire_in: 1 days
-docs-1.3:
+docs-docs:
stage: docs
- image: docker.io/python:3.9-slim
- needs:
- - build-api-latest
- - build-api-1.3
+ image: docker.io/python:3.11-slim
only:
refs:
- - master
- - release-v1.3
- - release-v1.4
+ - /^release-.*/
script:
- - apt-get update && apt-get install -y git make wget
- - git fetch && git checkout release-v1.3
- - pip install -r ./requirements.txt
- - wget https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/.docs/overrides/main.html -O .docs/overrides/main.html -q
- - mkdir -p ./final
- - mkdocs build && cp -r ./site ./final/1.3.0
- - cp -r ./swagger/1.3.0 ./final/1.3.0/swagger
+ - apt-get update && apt-get install -y git make sed wget
+ - make docs
cache:
paths:
- ./final
@@ -674,80 +495,49 @@ docs-1.3:
- ./final
expire_in: 1 days
-docs-1.4.0:
- stage: docs
- image: docker.io/python:3.11-slim
+release-images:
+ stage: release
+ image: docker:24-dind
needs:
- - build-api-latest
- - build-api-1.3
- - build-api-1.4
+ - test-metadata-service
+ - test-data-service
+ - test-analyse-service
+ - test-frontend
only:
refs:
- - master
- - release-v1.3
- - release-v1.4
+ - /^release-.*/
+ before_script:
+ - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL
+ - echo "$CI_REGISTRY2_PASSWORD" | docker login --username "$CI_REGISTRY2_USER" --password-stdin $CI_REGISTRY2_URL
script:
- - apt-get update && apt-get install -y git
- - git fetch --tags && git checkout v1.4.0
- - pip install -r ./requirements.txt
- - wget https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/.docs/overrides/main.html -O .docs/overrides/main.html -q
- - mkdir -p ./final
- - find .docs/ -type f -exec sed -i -e "s/__APPVERSION__/1.4.0/g" {} \;
- - find .docs/ -type f -exec sed -i -e "s/__CHARTVERSION__/1.4.0/g" {} \;
- - mkdocs build && cp -r ./site ./final/1.4.0
- - cp -r ./swagger/${APP_VERSION} ./final/1.4.0/swagger
- cache:
- paths:
- - ./final
- artifacts:
- when: always
- paths:
- - ./final
- expire_in: 1 days
+ - "ifconfig eth0 mtu 1450 up"
+ - "apk add make bash"
+ - "TAG=${APP_VERSION} make release"
-docs-1.4:
- stage: docs
- image: docker.io/python:3.11-slim
- needs:
- - build-api-latest
- - build-api-1.3
- - build-api-1.4
+release-chart:
+ stage: release
+ image: docker:24-dind
only:
refs:
- - master
- - release-v1.3
- - release-v1.4
+ - /^release-.*/
+ before_script:
+ - echo "$CI_REGISTRY2_PASSWORD" | docker login --username "$CI_REGISTRY2_USER" --password-stdin $CI_REGISTRY2_URL
script:
- - apt-get update && apt-get install -y git make sed wget
- - git fetch && git checkout release-v1.4
- - pip install -r ./requirements.txt
- - wget https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/.docs/overrides/main.html -O .docs/overrides/main.html -q
- - mkdir -p ./final
- - find .docs/ -type f -exec sed -i -e "s/__APPVERSION__/${APP_VERSION}/g" {} \;
- - find .docs/ -type f -exec sed -i -e "s/__CHARTVERSION__/${CHART_VERSION}/g" {} \;
- - mkdocs build && cp -r ./site ./final/${APP_VERSION}
- - cp -r ./swagger/${APP_VERSION} ./final/${APP_VERSION}/swagger
- cache:
- paths:
- - ./final
- artifacts:
- when: always
- paths:
- - ./final
- expire_in: 1 days
+ - apk add sed helm curl
+ - 'sed -i -e "s/^version:.*/version: \"${CHART_VERSION}\"/g" ./helm-charts/dbrepo/Chart.yaml'
+ - 'sed -i -e "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/g" ./helm-charts/dbrepo/Chart.yaml'
+ - find ./helm-charts -type f -exec sed -i -e "s/__CHARTVERSION__/${CHART_VERSION}/g" {} \;
+ - helm package ./helm-charts/dbrepo --destination ./build
+ - helm push "./build/dbrepo-${CHART_VERSION}.tgz" "oci://${CI_REGISTRY2_URL}/helm"
release-docs:
stage: release
image: docker.io/finalgene/openssh:9.1
needs:
- - docs-latest
- - docs-1.3
- - docs-1.4
+ - docs-docs
only:
refs:
- - master
- - release-v1.3
- - release-v1.4
+ - /^release-.*/
script:
- eval $(ssh-agent -s)
- echo "$CI_KEY_PRIVATE" > /root/.ssh/id_rsa && chmod 0600 /root/.ssh/id_rsa
diff --git a/Dockerfile b/Dockerfile
deleted file mode 100644
index 0030985143..0000000000
--- a/Dockerfile
+++ /dev/null
@@ -1,20 +0,0 @@
-FROM python:3.11-slim as build
-
-WORKDIR /app
-
-RUN apt-get update && apt-get install -y git
-
-ENV VERSIONS="1.4.1,1.4.0,1.3.0"
-ENV APP_VERSION="1.4.1"
-
-COPY .git/ .git/
-COPY .docs/ .docs/
-COPY ./requirements.txt ./requirements.txt
-
-RUN bash .docs/build-website.sh
-
-FROM nginx as runtime
-
-WORKDIR /usr/share/nginx/html/infrastructures/dbrepo/
-
-COPY --from=build /app/final /usr/share/nginx/html/infrastructures/dbrepo/
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 2b2e6c4ed1..6187dc1136 100644
--- a/Makefile
+++ b/Makefile
@@ -224,3 +224,6 @@ teardown:
build-api:
bash .docs/.swagger/swagger-generate.sh
+
+docs:
+ bash .docs/build-website.sh
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
index 0380fd2e92..1fb934e775 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -10,15 +10,6 @@ volumes:
storage-service-data:
services:
- website:
- restart: "no"
- container_name: website
- hostname: website
- image: website
- build: .
- ports:
- - 8000:80
-
dbrepo-metadata-db:
restart: "no"
container_name: dbrepo-metadata-db
--
GitLab