diff --git a/.docs/.swagger/swagger-site.sh b/.docs/.swagger/swagger-site.sh index b9a0d163e9649e1766a2efed232ea634a45c5916..f6f91bc9109786bf5c8341c548e49fca5ed59cf4 100644 --- a/.docs/.swagger/swagger-site.sh +++ b/.docs/.swagger/swagger-site.sh @@ -7,10 +7,6 @@ services[9093]=data services[9099]=metadata services[3305]=sidecar -# clean up -echo "clean up ./site" -rm -rf ./site - # ensure target directories are present echo "ensure target directory ./site are present" mkdir -p ./site diff --git a/.docs/build-website.sh b/.docs/build-website.sh new file mode 100644 index 0000000000000000000000000000000000000000..6e2fb29e7959e6f65ed85f73e5e217c889db1ead --- /dev/null +++ b/.docs/build-website.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +OVERRIDES_MAIN_HTML="" +SCRIPTS_EXTRA_JS="" + +function clean_cache { + echo "Removing cache from directory ./site" + rm -rf ./site +} + +function generate_docs { + BRANCH="release-$1" + echo "===================================================" + echo "Building DOCS for version $1 on branch $BRANCH" + echo "===================================================" + git reset --hard && git checkout "$BRANCH" + pip install -r ./requirements.txt > /dev/null + mkdir -p ./final + if [ "$1" = "latest" ]; then + OVERRIDES_MAIN_HTML=$(cat .docs/overrides/main.html) + sed -i -e "s/__APPVERSION__/${APP_VERSION}/g" .docs/scripts/extra.js + SCRIPTS_EXTRA_JS=$(cat .docs/scripts/extra.js) + else + echo $OVERRIDES_MAIN_HTML > .docs/overrides/main.html + mkdir -p .docs/scripts + echo $SCRIPTS_EXTRA_JS > .docs/scripts/extra.js + fi + find .docs/ -type f -exec sed -i -e "s/__APPVERSION__/$1/g" {} \; + find .docs/ -type f -exec sed -i -e "s/__CHARTVERSION__/$1/g" {} \; + mkdocs build > /dev/null && cp -r ./site "./final/$1" + cp -r "./swagger/$1" "./final/$1/swagger" + clean_cache +} + +function generate_api { + BRANCH="release-$1" + echo "===================================================" + echo "Building API for version $1 on branch $BRANCH" + echo "===================================================" + git reset --hard && git checkout "$BRANCH" + bash .docs/.swagger/swagger-site.sh + find ./site -type f -exec sed -i -e "s/__APPVERSION__/$1/g" {} \; + mkdir -p "./swagger/$1" + cp -r ./site/* "./swagger/$1/" + clean_cache +} + +# usage +if [ -z "$DOC_VERSIONS" ]; then + echo "Variable DOC_VERSIONS not set" + exit 1 +fi +versions=(${DOC_VERSIONS//,/ }) + +# usage +if [ -z "$APP_VERSION" ]; then + echo "Variable APP_VERSION not set" + exit 2 +fi +echo "===================================================" +echo "APP_VERSION=$APP_VERSION" +echo "===================================================" + +# ensure branches exist on machine +git fetch + +generate_api "latest" +generate_docs "latest" + +# versions +for i in "${!versions[@]}"; do + version="${versions[i]}" + generate_api "$version" + generate_docs "$version" +done + + +# finalization +echo "===================================================" +echo "Moving default version $APP_VERSION docs to /" +cp -r ./final/${APP_VERSION}/* ./final/ +echo "===================================================" diff --git a/.docs/overrides/main.html b/.docs/overrides/main.html index bc23fb67ea28eedf751c5cda48461513524cd46d..9f9a6daf6fa57add34aa8e9170bc55315f5ed3b1 100644 --- a/.docs/overrides/main.html +++ b/.docs/overrides/main.html @@ -3,24 +3,9 @@ {% block announce %} <label for="version">Version</label> <select style="margin-left:8px;" id="version" name="version" onchange="switchVersion(this.options[this.selectedIndex].value)"> + <option value="latest">latest</option> + <option value="1.4.1">1.4.1</option> <option value="1.4.0">1.4.0</option> <option value="1.3.0">1.3.0</option> - <option value="latest">latest</option> </select> -<script> - function getVersion() { - const segments = location.pathname.split('/') - console.debug('segments', segments) - if (segments.length >= 4) { - return segments[3] - } else { - return '__APPVERSION__' - } - } - function switchVersion(value) { - if (!value) { return } - this.location.href = location.protocol + '//' + location.host + '/infrastructures/dbrepo/' + value + '/' - } - document.getElementById("version").value = getVersion() -</script> {% endblock %} diff --git a/.docs/redirect.html b/.docs/redirect.html index 053a5581bfeeb797c2d571a1dd378b1ed88df305..a3737a67b6655b56c4c31f35f8bfc279a72b5e0f 100644 --- a/.docs/redirect.html +++ b/.docs/redirect.html @@ -5,7 +5,7 @@ <head> <meta charset="UTF-8"> <title>Redirect Notice</title> - <meta http-equiv="Refresh" content="0; url='https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/__APPVERSION__/'" /> + <meta http-equiv="Refresh" content="0; url='/infrastructures/dbrepo/__APPVERSION__/'" /> </head> <body> <h1>Redirect Notice</h1> @@ -14,7 +14,7 @@ available at: </p> <p> - <a href="https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/__APPVERSION__/">https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/__APPVERSION__/</a> + <a href="/infrastructures/dbrepo/__APPVERSION__/">/infrastructures/dbrepo/__APPVERSION__/</a> </p> </body> </html> \ No newline at end of file diff --git a/.docs/scripts/extra.js b/.docs/scripts/extra.js new file mode 100644 index 0000000000000000000000000000000000000000..2360d0d2db8579cfbd71d6077fe894cc46056c61 --- /dev/null +++ b/.docs/scripts/extra.js @@ -0,0 +1,14 @@ +function getVersion() { + const segments = location.pathname.split('/'); + if (segments.length > 4 && (segments[3].match(/[0-9]\.[0-9]\.[0-9]/g) || segments[3] == 'latest')) { + console.debug('version', segments[3]); + return segments[3]; + } + console.debug('default version', '__APPVERSION__'); + return '__APPVERSION__'; +} +function switchVersion(value) { + if (!value) { return } + this.location.href = location.protocol + '//' + location.host + '/infrastructures/dbrepo/' + value + '/'; +} +document.getElementById("version").value = getVersion(); \ No newline at end of file diff --git a/.gitignore b/.gitignore index a8f907321af8b733fbac77271140b4c069910b4d..2ae0c2d6d11336d2645da6a8d55eb4c8df68a907 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ ready schema.xsd final/ build/ +swagger/ # docs .docs/.swagger/dist/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index edce253766e65fa113ab011223c93288bb5a3ba8..f88822aca28d17298646022ed6d4456e6da99da4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,15 +4,18 @@ variables: TRIVY_CACHE_DIR: ".trivycache/" DOCKER_HOST: "unix:///var/run/dind/docker.sock" TESTCONTAINERS_RYUK_DISABLED: "false" - APP_VERSION: "1.4.0" - CHART_VERSION: "1.4.0" + DOC_VERSIONS: "latest,1.4.1,1.4.0,1.3.0" + APP_VERSION: "1.4.2" + CHART_VERSION: "1.4.2" image: debian:12-slim cache: + policy: pull + key: ${CI_BUILD_REF_NAME} paths: - - /root/.m2/repository/ - - /root/.npm/ + - final/ + - .m2/ stages: - build @@ -24,12 +27,18 @@ stages: build-metadata-service: image: maven:3-openjdk-17 stage: build + except: + refs: + - /^release-.*/ script: - "mvn -f ./dbrepo-metadata-service/pom.xml clean install -Dstyle.color=always -DskipTests" build-analyse-service: image: python:3.9-slim stage: build + except: + refs: + - /^release-.*/ variables: PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile" script: @@ -39,6 +48,9 @@ build-analyse-service: build-data-service: image: maven:3-openjdk-17 stage: build + except: + refs: + - /^release-.*/ needs: - build-metadata-service script: @@ -48,6 +60,9 @@ build-data-service: build-frontend: image: node:14-alpine stage: build + except: + refs: + - /^release-.*/ script: - "yarn config set network-timeout 600000 -g" - "yarn --cwd ./dbrepo-ui install --legacy-peer-deps" @@ -56,6 +71,9 @@ build-frontend: build-search-service: image: python:3.10-alpine stage: build + except: + refs: + - /^release-.*/ script: - "pip install pipenv" - "cd dbrepo-search-service && pipenv install --system --deploy" @@ -63,6 +81,9 @@ build-search-service: build-docker: image: docker.io/docker:24-dind stage: build + except: + refs: + - /^release-.*/ before_script: - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL script: @@ -74,6 +95,9 @@ build-docker: build-helm: image: docker.io/docker:24-dind stage: build + except: + refs: + - /^release-.*/ before_script: - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL script: @@ -86,6 +110,9 @@ build-helm: test-metadata-service: image: maven:3-openjdk-17 stage: test + except: + refs: + - /^release-.*/ needs: - build-metadata-service script: @@ -105,6 +132,9 @@ test-metadata-service: test-data-service: image: maven:3-openjdk-17 stage: test + except: + refs: + - /^release-.*/ needs: - build-data-service script: @@ -124,6 +154,9 @@ test-data-service: test-analyse-service: image: python:3.9-slim stage: test + except: + refs: + - /^release-.*/ variables: PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile" needs: @@ -146,6 +179,9 @@ test-analyse-service: test-frontend: image: node:14-alpine stage: test + except: + refs: + - /^release-.*/ needs: - build-frontend script: @@ -170,7 +206,6 @@ scan-analyse-service: only: refs: - master - - release-v1.4 allow_failure: true script: - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-analyse-service-report.json docker.io/dbrepo/analyse-service:latest @@ -191,7 +226,6 @@ scan-authentication-service: only: refs: - master - - release-v1.4 allow_failure: true script: - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-authentication-service-report.json docker.io/dbrepo/authentication-service:latest @@ -212,7 +246,6 @@ scan-broker-service: only: refs: - master - - release-v1.4 allow_failure: true script: - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-broker-service-report.json docker.io/dbrepo/broker-service:latest @@ -233,7 +266,6 @@ scan-gateway-service: only: refs: - master - - release-v1.4 allow_failure: true script: - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-gateway-service-report.json docker.io/nginx:1.25.0-alpine-slim @@ -254,7 +286,6 @@ scan-metadata-service: only: refs: - master - - release-v1.4 allow_failure: true script: - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-metadata-service-report.json docker.io/dbrepo/metadata-service:latest @@ -275,7 +306,6 @@ scan-data-service: only: refs: - master - - release-v1.4 allow_failure: true script: - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-data-service-report.json docker.io/dbrepo/data-service:latest @@ -296,7 +326,6 @@ scan-search-db: only: refs: - master - - release-v1.4 allow_failure: true script: - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-search-db-report.json docker.io/dbrepo/search-db:latest @@ -317,7 +346,6 @@ scan-search-dashboard: only: refs: - master - - release-v1.4 allow_failure: true script: - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-search-dashboard-report.json docker.io/opensearchproject/opensearch-dashboards:2.10.0 @@ -338,7 +366,6 @@ scan-search-db-init: only: refs: - master - - release-v1.4 allow_failure: true script: - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-search-db-init-report.json docker.io/dbrepo/search-db-init:latest @@ -359,7 +386,6 @@ scan-data-db: only: refs: - master - - release-v1.4 allow_failure: true script: - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-data-db-report.json docker.io/bitnami/mariadb:11.2.2-debian-11-r0 @@ -380,7 +406,6 @@ scan-metadata-db: only: refs: - master - - release-v1.4 allow_failure: true script: - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-metadata-db-report.json docker.io/dbrepo/metadata-db:latest @@ -401,7 +426,6 @@ scan-ui: only: refs: - master - - release-v1.4 allow_failure: true script: - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-ui-report.json docker.io/dbrepo/ui:latest @@ -422,7 +446,6 @@ scan-storage-service: only: refs: - master - - release-v1.4 allow_failure: true script: - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-storage-service-report.json docker.io/chrislusf/seaweedfs:3.59 @@ -443,7 +466,6 @@ scan-storage-service-init: only: refs: - master - - release-v1.4 allow_failure: true script: - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-storage-service-init-report.json docker.io/dbrepo/storage-service-init:latest @@ -458,55 +480,27 @@ scan-storage-service-init: reports: container_scanning: ./.trivy/trivy-storage-service-init-report.json -release-latest: - stage: release - image: docker:24-dind - needs: - - test-metadata-service - - test-data-service - - test-analyse-service - - test-frontend - only: - refs: - - master - before_script: - - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL - - echo "$CI_REGISTRY2_PASSWORD" | docker login --username "$CI_REGISTRY2_USER" --password-stdin $CI_REGISTRY2_URL - script: - - "ifconfig eth0 mtu 1450 up" - - "apk add make bash" - - TAG=latest make release - -release-1.3: - stage: release - image: docker:24-dind - needs: - - test-metadata-service - - test-data-service - - test-analyse-service - - test-frontend +docs-registry: + stage: docs + image: docker.io/python:3.11-slim only: refs: - - release-v1.3 - before_script: - - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL - - echo "$CI_REGISTRY2_PASSWORD" | docker login --username "$CI_REGISTRY2_USER" --password-stdin $CI_REGISTRY2_URL + - /^release-.*/ script: - - "ifconfig eth0 mtu 1450 up" - - "apk add make bash" - - "TAG=1.3.0 make release" + - pip install -r ./requirements.txt + - python3 .docs/docker/release.py -release-1.4: +release-images: stage: release image: docker:24-dind - needs: + dependencies: - test-metadata-service - test-data-service - test-analyse-service - test-frontend only: refs: - - release-v1.4 + - /^release-.*/ before_script: - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL - echo "$CI_REGISTRY2_PASSWORD" | docker login --username "$CI_REGISTRY2_USER" --password-stdin $CI_REGISTRY2_URL @@ -515,12 +509,12 @@ release-1.4: - "apk add make bash" - "TAG=${APP_VERSION} make release" -release-helm-1.4: +release-chart: stage: release image: docker:24-dind only: refs: - - release-v1.4 + - /^release-.*/ before_script: - echo "$CI_REGISTRY2_PASSWORD" | docker login --username "$CI_REGISTRY2_USER" --password-stdin $CI_REGISTRY2_URL script: @@ -531,205 +525,19 @@ release-helm-1.4: - helm package ./helm-charts/dbrepo --destination ./build - helm push "./build/dbrepo-${CHART_VERSION}.tgz" "oci://${CI_REGISTRY2_URL}/helm" -build-api-latest: - stage: build - image: docker.io/alpine:3.18 - only: - refs: - - master - - release-v1.3 - - release-v1.4 - script: - - apk add bash git - - git fetch && git checkout master - - bash .docs/.swagger/swagger-site.sh - - find ./site -type f -exec sed -i -e "s/__APPVERSION__/latest/g" {} \; - - mkdir -p ./swagger/latest - - cp -r ./site/* ./swagger/latest/ - cache: - paths: - - ./swagger/latest - artifacts: - when: always - paths: - - ./swagger/latest - expire_in: 1 days - -build-api-1.3: - stage: build - image: docker.io/alpine:3.18 - only: - refs: - - master - - release-v1.3 - - release-v1.4 - script: - - apk add bash git maven - - git fetch && git checkout release-v1.3 - - bash .docs/.swagger/generate.sh - - mkdir -p ./swagger/1.3.0 - - cp -r ./site/* ./swagger/1.3.0/ - cache: - paths: - - ./swagger/1.3.0 - artifacts: - when: always - paths: - - ./swagger/1.3.0 - expire_in: 1 days - -build-api-1.4: - stage: build - image: docker.io/alpine:3.18 - only: - refs: - - master - - release-v1.3 - - release-v1.4 - script: - - apk add bash git maven - - git fetch && git checkout release-v1.4 - - bash .docs/.swagger/swagger-site.sh - - find ./site -type f -exec sed -i -e "s/__APPVERSION__/${APP_VERSION}/g" {} \; - - mkdir -p ./swagger/${APP_VERSION} - - cp -r ./site/* ./swagger/${APP_VERSION}/ - cache: - paths: - - ./swagger/${APP_VERSION} - artifacts: - when: always - paths: - - ./swagger/${APP_VERSION} - expire_in: 1 days - -docs-registry: - stage: docs - image: docker.io/python:3.11-slim - only: - refs: - - master - - release-v1.3 - - release-v1.4 - script: - - pip install -r ./requirements.txt - - python3 .docs/docker/release.py - cache: - paths: - - ./final - artifacts: - when: always - paths: - - ./final - expire_in: 1 days - -docs-latest: - stage: docs - image: docker.io/python:3.9-slim - needs: - - build-api-latest - - build-api-1.3 - - build-api-1.4 - only: - refs: - - master - - release-v1.3 - - release-v1.4 - script: - - apt-get update && apt-get install -y git make sed - - git fetch && git checkout master - - pip install -r ./requirements.txt - - mkdir -p ./final - - sed -i -e "s/__APPVERSION__/${APP_VERSION}/g" .docs/redirect.html - - cp ./.docs/redirect.html ./final/index.html - - find .docs/ -type f -exec sed -i -e "s/__APPVERSION__/latest/g" {} \; - - find .docs/ -type f -exec sed -i -e "s/__CHARTVERSION__/${CHART_VERSION}/g" {} \; - - mkdocs build && cp -r ./site ./final/latest - - cp -r ./swagger/latest ./final/latest/swagger - cache: - paths: - - ./final - artifacts: - when: always - paths: - - ./final - expire_in: 1 days - -docs-1.3: - stage: docs - image: docker.io/python:3.9-slim - needs: - - build-api-latest - - build-api-1.3 - only: - refs: - - master - - release-v1.3 - - release-v1.4 - script: - - apt-get update && apt-get install -y git make wget - - git fetch && git checkout release-v1.3 - - pip install -r ./requirements.txt - - wget https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/.docs/overrides/main.html -O .docs/overrides/main.html -q - - mkdir -p ./final - - mkdocs build && cp -r ./site ./final/1.3.0 - - cp -r ./swagger/1.3.0 ./final/1.3.0/swagger - cache: - paths: - - ./final - artifacts: - when: always - paths: - - ./final - expire_in: 1 days - -docs-1.4: - stage: docs - image: docker.io/python:3.11-slim - needs: - - build-api-latest - - build-api-1.3 - - build-api-1.4 - only: - refs: - - master - - release-v1.3 - - release-v1.4 - script: - - apt-get update && apt-get install -y git make sed wget - - git fetch && git checkout release-v1.4 - - pip install -r ./requirements.txt - - wget https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/.docs/overrides/main.html -O .docs/overrides/main.html -q - - mkdir -p ./final - - find .docs/ -type f -exec sed -i -e "s/__APPVERSION__/${APP_VERSION}/g" {} \; - - find .docs/ -type f -exec sed -i -e "s/__CHARTVERSION__/${CHART_VERSION}/g" {} \; - - mkdocs build && cp -r ./site ./final/${APP_VERSION} - - cp -r ./swagger/${APP_VERSION} ./final/${APP_VERSION}/swagger - cache: - paths: - - ./final - artifacts: - when: always - paths: - - ./final - expire_in: 1 days - release-docs: stage: release - image: docker.io/finalgene/openssh:9.1 - needs: - - docs-latest - - docs-1.3 - - docs-1.4 + image: docker.io/python:3.11-slim only: refs: - - master - - release-v1.3 - - release-v1.4 + - /^release-.*/ script: + - apt-get update && apt-get install -y git make sed wget ssh + - make docs - eval $(ssh-agent -s) - echo "$CI_KEY_PRIVATE" > /root/.ssh/id_rsa && chmod 0600 /root/.ssh/id_rsa - echo "$CI_KEY_PUBLIC" > /root/.ssh/id_rsa.pub - echo "$CI_DOC_ID" > ~/.ssh/known_hosts - - tar czfv final.tar.gz ./final + - tar czfv ./final.tar.gz ./final - "scp -oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedAlgorithms=+ssh-rsa final.tar.gz $CI_DOC_USER@$CI_DOC_IP:final.tar.gz" - "ssh -oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedAlgorithms=+ssh-rsa $CI_DOC_USER@$CI_DOC_IP 'rm -rf /system/user/ifs/infrastructures/public_html/dbrepo/*; tar xzfv ./final.tar.gz; rm -f ./final.tar.gz; cp -r ./final/* /system/user/ifs/infrastructures/public_html/dbrepo; rm -rf ./final'" diff --git a/Makefile b/Makefile index 45e493ffa90202c9268578eeb2213fc712c74cd4..6187dc11366577f2e08dfe2f71430d04d14ea7ed 100644 --- a/Makefile +++ b/Makefile @@ -222,5 +222,8 @@ test: test-backend test-frontend teardown: ./bin/teardown.sh -build-api: build-docker +build-api: bash .docs/.swagger/swagger-generate.sh + +docs: + bash .docs/build-website.sh \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 14affa17a9a3a1ebb0687936f79c7895f73c3b19..60f1c85e8897ec1936c405ab6d72fe7fa855b456 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -43,6 +43,8 @@ nav: - contact.md extra_css: - stylesheets/extra.css +extra_javascript: + - scripts/extra.js theme: favicon: images/signet_white.png custom_dir: .docs/overrides