Skip to content
Snippets Groups Projects
Verified Commit eef50499 authored by Martin Weise's avatar Martin Weise
Browse files

Merge pipeline and docs

parent c2ee81c9
No related branches found
No related tags found
4 merge requests!256Master,!254Master,!253Dev,!252Dev
Show whitespace changes
Inline Side-by-side
.docs/.swagger/swagger-site.sh
+ 0
− 4
View file @ eef50499
......@@ -7,10 +7,6 @@ services[9093]=data
services[9099]=metadata
services[3305]=sidecar
# clean up
echo "clean up ./site"
rm -rf ./site
# ensure target directories are present
echo "ensure target directory ./site are present"
mkdir -p ./site
......
.docs/build-website.sh 0 → 100644
+ 82
− 0
View file @ eef50499
#!/bin/bash
OVERRIDES_MAIN_HTML=""
SCRIPTS_EXTRA_JS=""
function clean_cache {
echo "Removing cache from directory ./site"
rm -rf ./site
}
function generate_docs {
BRANCH="release-$1"
echo "==================================================="
echo "Building DOCS for version $1 on branch $BRANCH"
echo "==================================================="
git reset --hard && git checkout "$BRANCH"
pip install -r ./requirements.txt > /dev/null
mkdir -p ./final
if [ "$1" = "latest" ]; then
OVERRIDES_MAIN_HTML=$(cat .docs/overrides/main.html)
sed -i -e "s/__APPVERSION__/${APP_VERSION}/g" .docs/scripts/extra.js
SCRIPTS_EXTRA_JS=$(cat .docs/scripts/extra.js)
else
echo $OVERRIDES_MAIN_HTML > .docs/overrides/main.html
mkdir -p .docs/scripts
echo $SCRIPTS_EXTRA_JS > .docs/scripts/extra.js
fi
find .docs/ -type f -exec sed -i -e "s/__APPVERSION__/$1/g" {} \;
find .docs/ -type f -exec sed -i -e "s/__CHARTVERSION__/$1/g" {} \;
mkdocs build > /dev/null && cp -r ./site "./final/$1"
cp -r "./swagger/$1" "./final/$1/swagger"
clean_cache
}
function generate_api {
BRANCH="release-$1"
echo "==================================================="
echo "Building API for version $1 on branch $BRANCH"
echo "==================================================="
git reset --hard && git checkout "$BRANCH"
bash .docs/.swagger/swagger-site.sh
find ./site -type f -exec sed -i -e "s/__APPVERSION__/$1/g" {} \;
mkdir -p "./swagger/$1"
cp -r ./site/* "./swagger/$1/"
clean_cache
}
# usage
if [ -z "$DOC_VERSIONS" ]; then
echo "Variable DOC_VERSIONS not set"
exit 1
fi
versions=(${DOC_VERSIONS//,/ })
# usage
if [ -z "$APP_VERSION" ]; then
echo "Variable APP_VERSION not set"
exit 2
fi
echo "==================================================="
echo "APP_VERSION=$APP_VERSION"
echo "==================================================="
# ensure branches exist on machine
git fetch
generate_api "latest"
generate_docs "latest"
# versions
for i in "${!versions[@]}"; do
version="${versions[i]}"
generate_api "$version"
generate_docs "$version"
done
# finalization
echo "==================================================="
echo "Moving default version $APP_VERSION docs to /"
cp -r ./final/${APP_VERSION}/* ./final/
echo "==================================================="
.docs/overrides/main.html
+ 2
− 17
View file @ eef50499
......@@ -3,24 +3,9 @@
{% block announce %}
<label for="version">Version</label>
<select style="margin-left:8px;" id="version" name="version" onchange="switchVersion(this.options[this.selectedIndex].value)">
<option value="latest">latest</option>
<option value="1.4.1">1.4.1</option>
<option value="1.4.0">1.4.0</option>
<option value="1.3.0">1.3.0</option>
<option value="latest">latest</option>
</select>
<script>
function getVersion() {
const segments = location.pathname.split('/')
console.debug('segments', segments)
if (segments.length >= 4) {
return segments[3]
} else {
return '__APPVERSION__'
}
}
function switchVersion(value) {
if (!value) { return }
this.location.href = location.protocol + '//' + location.host + '/infrastructures/dbrepo/' + value + '/'
}
document.getElementById("version").value = getVersion()
</script>
{% endblock %}
.docs/redirect.html
+ 2
− 2
View file @ eef50499
......@@ -5,7 +5,7 @@
<head>
<meta charset="UTF-8">
<title>Redirect Notice</title>
<meta http-equiv="Refresh" content="0; url='https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/__APPVERSION__/'" />
<meta http-equiv="Refresh" content="0; url='/infrastructures/dbrepo/__APPVERSION__/'" />
</head>
<body>
<h1>Redirect Notice</h1>
......@@ -14,7 +14,7 @@
available at:
</p>
<p>
<a href="https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/__APPVERSION__/">https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/__APPVERSION__/</a>
<a href="/infrastructures/dbrepo/__APPVERSION__/">/infrastructures/dbrepo/__APPVERSION__/</a>
</p>
</body>
</html>
\ No newline at end of file
.docs/scripts/extra.js 0 → 100644
+ 14
− 0
View file @ eef50499
function getVersion() {
const segments = location.pathname.split('/');
if (segments.length > 4 && (segments[3].match(/[0-9]\.[0-9]\.[0-9]/g) || segments[3] == 'latest')) {
console.debug('version', segments[3]);
return segments[3];
}
console.debug('default version', '__APPVERSION__');
return '__APPVERSION__';
}
function switchVersion(value) {
if (!value) { return }
this.location.href = location.protocol + '//' + location.host + '/infrastructures/dbrepo/' + value + '/';
}
document.getElementById("version").value = getVersion();
\ No newline at end of file
.gitignore
+ 1
− 0
View file @ eef50499
......@@ -9,6 +9,7 @@ ready
schema.xsd
final/
build/
swagger/
# docs
.docs/.swagger/dist/
......
.gitlab-ci.yml
+ 56
− 248
View file @ eef50499
......@@ -4,15 +4,18 @@ variables:
TRIVY_CACHE_DIR: ".trivycache/"
DOCKER_HOST: "unix:///var/run/dind/docker.sock"
TESTCONTAINERS_RYUK_DISABLED: "false"
APP_VERSION: "1.4.0"
CHART_VERSION: "1.4.0"
DOC_VERSIONS: "latest,1.4.1,1.4.0,1.3.0"
APP_VERSION: "1.4.2"
CHART_VERSION: "1.4.2"
image: debian:12-slim
cache:
policy: pull
key: ${CI_BUILD_REF_NAME}
paths:
- /root/.m2/repository/
- /root/.npm/
- final/
- .m2/
stages:
- build
......@@ -24,12 +27,18 @@ stages:
build-metadata-service:
image: maven:3-openjdk-17
stage: build
except:
refs:
- /^release-.*/
script:
- "mvn -f ./dbrepo-metadata-service/pom.xml clean install -Dstyle.color=always -DskipTests"
build-analyse-service:
image: python:3.9-slim
stage: build
except:
refs:
- /^release-.*/
variables:
PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile"
script:
......@@ -39,6 +48,9 @@ build-analyse-service:
build-data-service:
image: maven:3-openjdk-17
stage: build
except:
refs:
- /^release-.*/
needs:
- build-metadata-service
script:
......@@ -48,6 +60,9 @@ build-data-service:
build-frontend:
image: node:14-alpine
stage: build
except:
refs:
- /^release-.*/
script:
- "yarn config set network-timeout 600000 -g"
- "yarn --cwd ./dbrepo-ui install --legacy-peer-deps"
......@@ -56,6 +71,9 @@ build-frontend:
build-search-service:
image: python:3.10-alpine
stage: build
except:
refs:
- /^release-.*/
script:
- "pip install pipenv"
- "cd dbrepo-search-service && pipenv install --system --deploy"
......@@ -63,6 +81,9 @@ build-search-service:
build-docker:
image: docker.io/docker:24-dind
stage: build
except:
refs:
- /^release-.*/
before_script:
- echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL
script:
......@@ -74,6 +95,9 @@ build-docker:
build-helm:
image: docker.io/docker:24-dind
stage: build
except:
refs:
- /^release-.*/
before_script:
- echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL
script:
......@@ -86,6 +110,9 @@ build-helm:
test-metadata-service:
image: maven:3-openjdk-17
stage: test
except:
refs:
- /^release-.*/
needs:
- build-metadata-service
script:
......@@ -105,6 +132,9 @@ test-metadata-service:
test-data-service:
image: maven:3-openjdk-17
stage: test
except:
refs:
- /^release-.*/
needs:
- build-data-service
script:
......@@ -124,6 +154,9 @@ test-data-service:
test-analyse-service:
image: python:3.9-slim
stage: test
except:
refs:
- /^release-.*/
variables:
PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile"
needs:
......@@ -146,6 +179,9 @@ test-analyse-service:
test-frontend:
image: node:14-alpine
stage: test
except:
refs:
- /^release-.*/
needs:
- build-frontend
script:
......@@ -170,7 +206,6 @@ scan-analyse-service:
only:
refs:
- master
- release-v1.4
allow_failure: true
script:
- trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-analyse-service-report.json docker.io/dbrepo/analyse-service:latest
......@@ -191,7 +226,6 @@ scan-authentication-service:
only:
refs:
- master
- release-v1.4
allow_failure: true
script:
- trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-authentication-service-report.json docker.io/dbrepo/authentication-service:latest
......@@ -212,7 +246,6 @@ scan-broker-service:
only:
refs:
- master
- release-v1.4
allow_failure: true
script:
- trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-broker-service-report.json docker.io/dbrepo/broker-service:latest
......@@ -233,7 +266,6 @@ scan-gateway-service:
only:
refs:
- master
- release-v1.4
allow_failure: true
script:
- trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-gateway-service-report.json docker.io/nginx:1.25.0-alpine-slim
......@@ -254,7 +286,6 @@ scan-metadata-service:
only:
refs:
- master
- release-v1.4
allow_failure: true
script:
- trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-metadata-service-report.json docker.io/dbrepo/metadata-service:latest
......@@ -275,7 +306,6 @@ scan-data-service:
only:
refs:
- master
- release-v1.4
allow_failure: true
script:
- trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-data-service-report.json docker.io/dbrepo/data-service:latest
......@@ -296,7 +326,6 @@ scan-search-db:
only:
refs:
- master
- release-v1.4
allow_failure: true
script:
- trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-search-db-report.json docker.io/dbrepo/search-db:latest
......@@ -317,7 +346,6 @@ scan-search-dashboard:
only:
refs:
- master
- release-v1.4
allow_failure: true
script:
- trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-search-dashboard-report.json docker.io/opensearchproject/opensearch-dashboards:2.10.0
......@@ -338,7 +366,6 @@ scan-search-db-init:
only:
refs:
- master
- release-v1.4
allow_failure: true
script:
- trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-search-db-init-report.json docker.io/dbrepo/search-db-init:latest
......@@ -359,7 +386,6 @@ scan-data-db:
only:
refs:
- master
- release-v1.4
allow_failure: true
script:
- trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-data-db-report.json docker.io/bitnami/mariadb:11.2.2-debian-11-r0
......@@ -380,7 +406,6 @@ scan-metadata-db:
only:
refs:
- master
- release-v1.4
allow_failure: true
script:
- trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-metadata-db-report.json docker.io/dbrepo/metadata-db:latest
......@@ -401,7 +426,6 @@ scan-ui:
only:
refs:
- master
- release-v1.4
allow_failure: true
script:
- trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-ui-report.json docker.io/dbrepo/ui:latest
......@@ -422,7 +446,6 @@ scan-storage-service:
only:
refs:
- master
- release-v1.4
allow_failure: true
script:
- trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-storage-service-report.json docker.io/chrislusf/seaweedfs:3.59
......@@ -443,7 +466,6 @@ scan-storage-service-init:
only:
refs:
- master
- release-v1.4
allow_failure: true
script:
- trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-storage-service-init-report.json docker.io/dbrepo/storage-service-init:latest
......@@ -458,55 +480,27 @@ scan-storage-service-init:
reports:
container_scanning: ./.trivy/trivy-storage-service-init-report.json
release-latest:
stage: release
image: docker:24-dind
needs:
- test-metadata-service
- test-data-service
- test-analyse-service
- test-frontend
only:
refs:
- master
before_script:
- echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL
- echo "$CI_REGISTRY2_PASSWORD" | docker login --username "$CI_REGISTRY2_USER" --password-stdin $CI_REGISTRY2_URL
script:
- "ifconfig eth0 mtu 1450 up"
- "apk add make bash"
- TAG=latest make release
release-1.3:
stage: release
image: docker:24-dind
needs:
- test-metadata-service
- test-data-service
- test-analyse-service
- test-frontend
docs-registry:
stage: docs
image: docker.io/python:3.11-slim
only:
refs:
- release-v1.3
before_script:
- echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL
- echo "$CI_REGISTRY2_PASSWORD" | docker login --username "$CI_REGISTRY2_USER" --password-stdin $CI_REGISTRY2_URL
- /^release-.*/
script:
- "ifconfig eth0 mtu 1450 up"
- "apk add make bash"
- "TAG=1.3.0 make release"
- pip install -r ./requirements.txt
- python3 .docs/docker/release.py
release-1.4:
release-images:
stage: release
image: docker:24-dind
needs:
dependencies:
- test-metadata-service
- test-data-service
- test-analyse-service
- test-frontend
only:
refs:
- release-v1.4
- /^release-.*/
before_script:
- echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL
- echo "$CI_REGISTRY2_PASSWORD" | docker login --username "$CI_REGISTRY2_USER" --password-stdin $CI_REGISTRY2_URL
......@@ -515,12 +509,12 @@ release-1.4:
- "apk add make bash"
- "TAG=${APP_VERSION} make release"
release-helm-1.4:
release-chart:
stage: release
image: docker:24-dind
only:
refs:
- release-v1.4
- /^release-.*/
before_script:
- echo "$CI_REGISTRY2_PASSWORD" | docker login --username "$CI_REGISTRY2_USER" --password-stdin $CI_REGISTRY2_URL
script:
......@@ -531,205 +525,19 @@ release-helm-1.4:
- helm package ./helm-charts/dbrepo --destination ./build
- helm push "./build/dbrepo-${CHART_VERSION}.tgz" "oci://${CI_REGISTRY2_URL}/helm"
build-api-latest:
stage: build
image: docker.io/alpine:3.18
only:
refs:
- master
- release-v1.3
- release-v1.4
script:
- apk add bash git
- git fetch && git checkout master
- bash .docs/.swagger/swagger-site.sh
- find ./site -type f -exec sed -i -e "s/__APPVERSION__/latest/g" {} \;
- mkdir -p ./swagger/latest
- cp -r ./site/* ./swagger/latest/
cache:
paths:
- ./swagger/latest
artifacts:
when: always
paths:
- ./swagger/latest
expire_in: 1 days
build-api-1.3:
stage: build
image: docker.io/alpine:3.18
only:
refs:
- master
- release-v1.3
- release-v1.4
script:
- apk add bash git maven
- git fetch && git checkout release-v1.3
- bash .docs/.swagger/generate.sh
- mkdir -p ./swagger/1.3.0
- cp -r ./site/* ./swagger/1.3.0/
cache:
paths:
- ./swagger/1.3.0
artifacts:
when: always
paths:
- ./swagger/1.3.0
expire_in: 1 days
build-api-1.4:
stage: build
image: docker.io/alpine:3.18
only:
refs:
- master
- release-v1.3
- release-v1.4
script:
- apk add bash git maven
- git fetch && git checkout release-v1.4
- bash .docs/.swagger/swagger-site.sh
- find ./site -type f -exec sed -i -e "s/__APPVERSION__/${APP_VERSION}/g" {} \;
- mkdir -p ./swagger/${APP_VERSION}
- cp -r ./site/* ./swagger/${APP_VERSION}/
cache:
paths:
- ./swagger/${APP_VERSION}
artifacts:
when: always
paths:
- ./swagger/${APP_VERSION}
expire_in: 1 days
docs-registry:
stage: docs
image: docker.io/python:3.11-slim
only:
refs:
- master
- release-v1.3
- release-v1.4
script:
- pip install -r ./requirements.txt
- python3 .docs/docker/release.py
cache:
paths:
- ./final
artifacts:
when: always
paths:
- ./final
expire_in: 1 days
docs-latest:
stage: docs
image: docker.io/python:3.9-slim
needs:
- build-api-latest
- build-api-1.3
- build-api-1.4
only:
refs:
- master
- release-v1.3
- release-v1.4
script:
- apt-get update && apt-get install -y git make sed
- git fetch && git checkout master
- pip install -r ./requirements.txt
- mkdir -p ./final
- sed -i -e "s/__APPVERSION__/${APP_VERSION}/g" .docs/redirect.html
- cp ./.docs/redirect.html ./final/index.html
- find .docs/ -type f -exec sed -i -e "s/__APPVERSION__/latest/g" {} \;
- find .docs/ -type f -exec sed -i -e "s/__CHARTVERSION__/${CHART_VERSION}/g" {} \;
- mkdocs build && cp -r ./site ./final/latest
- cp -r ./swagger/latest ./final/latest/swagger
cache:
paths:
- ./final
artifacts:
when: always
paths:
- ./final
expire_in: 1 days
docs-1.3:
stage: docs
image: docker.io/python:3.9-slim
needs:
- build-api-latest
- build-api-1.3
only:
refs:
- master
- release-v1.3
- release-v1.4
script:
- apt-get update && apt-get install -y git make wget
- git fetch && git checkout release-v1.3
- pip install -r ./requirements.txt
- wget https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/.docs/overrides/main.html -O .docs/overrides/main.html -q
- mkdir -p ./final
- mkdocs build && cp -r ./site ./final/1.3.0
- cp -r ./swagger/1.3.0 ./final/1.3.0/swagger
cache:
paths:
- ./final
artifacts:
when: always
paths:
- ./final
expire_in: 1 days
docs-1.4:
stage: docs
image: docker.io/python:3.11-slim
needs:
- build-api-latest
- build-api-1.3
- build-api-1.4
only:
refs:
- master
- release-v1.3
- release-v1.4
script:
- apt-get update && apt-get install -y git make sed wget
- git fetch && git checkout release-v1.4
- pip install -r ./requirements.txt
- wget https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/.docs/overrides/main.html -O .docs/overrides/main.html -q
- mkdir -p ./final
- find .docs/ -type f -exec sed -i -e "s/__APPVERSION__/${APP_VERSION}/g" {} \;
- find .docs/ -type f -exec sed -i -e "s/__CHARTVERSION__/${CHART_VERSION}/g" {} \;
- mkdocs build && cp -r ./site ./final/${APP_VERSION}
- cp -r ./swagger/${APP_VERSION} ./final/${APP_VERSION}/swagger
cache:
paths:
- ./final
artifacts:
when: always
paths:
- ./final
expire_in: 1 days
release-docs:
stage: release
image: docker.io/finalgene/openssh:9.1
needs:
- docs-latest
- docs-1.3
- docs-1.4
image: docker.io/python:3.11-slim
only:
refs:
- master
- release-v1.3
- release-v1.4
- /^release-.*/
script:
- apt-get update && apt-get install -y git make sed wget ssh
- make docs
- eval $(ssh-agent -s)
- echo "$CI_KEY_PRIVATE" > /root/.ssh/id_rsa && chmod 0600 /root/.ssh/id_rsa
- echo "$CI_KEY_PUBLIC" > /root/.ssh/id_rsa.pub
- echo "$CI_DOC_ID" > ~/.ssh/known_hosts
- tar czfv final.tar.gz ./final
- tar czfv ./final.tar.gz ./final
- "scp -oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedAlgorithms=+ssh-rsa final.tar.gz $CI_DOC_USER@$CI_DOC_IP:final.tar.gz"
- "ssh -oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedAlgorithms=+ssh-rsa $CI_DOC_USER@$CI_DOC_IP 'rm -rf /system/user/ifs/infrastructures/public_html/dbrepo/*; tar xzfv ./final.tar.gz; rm -f ./final.tar.gz; cp -r ./final/* /system/user/ifs/infrastructures/public_html/dbrepo; rm -rf ./final'"
Makefile
+ 4
− 1
View file @ eef50499
......@@ -222,5 +222,8 @@ test: test-backend test-frontend
teardown:
./bin/teardown.sh
build-api: build-docker
build-api:
bash .docs/.swagger/swagger-generate.sh
docs:
bash .docs/build-website.sh
\ No newline at end of file
mkdocs.yml
+ 2
− 0
View file @ eef50499
......@@ -43,6 +43,8 @@ nav:
- contact.md
extra_css:
- stylesheets/extra.css
extra_javascript:
- scripts/extra.js
theme:
favicon: images/signet_white.png
custom_dir: .docs/overrides
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment