Skip to content
Snippets Groups Projects

Wrong model

1 file
+ 2
0
Compare changes
  • Side-by-side
  • Inline
+ 97
54
variables:
BUILD_VERSION: ""
HOSTALIASES: "./hosts"
DOCKER_HOST: "unix:///var/run/dind/docker.sock"
DOCKER_VERSION: "27"
@@ -9,9 +10,9 @@ variables:
NODE_VERSION: "18"
SONARQUBE_VERSION: "10.0"
BUN_VERSION: "1.1.40"
DOC_VERSION: "1.6"
APP_VERSION: "1.6.5"
CHART_VERSION: "1.6.5"
DOC_VERSION: "1.7"
APP_VERSION: "1.7.0"
CHART_VERSION: "1.7.0"
CACHE_FALLBACK_KEY: "${CI_DEFAULT_BRANCH}"
# This will supress any download for dependencies and plugins or upload messages which would clutter the console log.
# `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work.
@@ -30,13 +31,20 @@ cache:
paths:
- .m2/repository
workflow:
rules:
- if: $CI_COMMIT_REF_NAME == "dev"
variables:
BUILD_VERSION: "+git.${CI_COMMIT_SHORT_SHA}"
- when: always
stages:
- lint
- build
- deploy
- test
- docs
- release
- deploy
- verify
- scan
@@ -80,9 +88,9 @@ lint-helm-chart:
- apk --no-cache add helm git bash
- cp ./helm/dbrepo/values.schema.json ./CI_values.schema.json
- helm plugin install https://github.com/losisin/helm-values-schema-json.git
- helm dependency update ./helm/seaweedfs
- helm package ./helm/seaweedfs --destination ./build
- helm dependency update ./helm/dbrepo
- helm registry login --username "${CI_REGISTRY_USER}" --password "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY_URL}"
- helm registry login --username "${CI_REGISTRY2_USER}" --password "${CI_REGISTRY2_PASSWORD}" "${CI_REGISTRY2_URL}"
script:
- bash .scripts/check-helm.sh
- helm lint ./helm/dbrepo
@@ -137,6 +145,8 @@ lint-open-api-version:
build-metadata-service:
image: maven:3-openjdk-${JAVA_VERSION}
stage: build
only:
- merge_requests
script:
- "mvn -f ./dbrepo-metadata-service/pom.xml clean install $MAVEN_OPTS -DskipTests"
# Compiled classes are needed for SonarQube in later stages
@@ -155,6 +165,8 @@ build-metadata-service:
build-analyse-service:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: build
only:
- merge_requests
variables:
PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile"
script:
@@ -164,6 +176,8 @@ build-analyse-service:
build-lib:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: build
only:
- merge_requests
variables:
PIPENV_PIPFILE: "./lib/python/Pipfile"
script:
@@ -173,12 +187,15 @@ build-lib:
build-data-service:
image: maven:3-openjdk-${JAVA_VERSION}
stage: build
only:
- merge_requests
needs:
- build-metadata-service
dependencies:
- build-metadata-service
script:
before_script:
- "mvn -f ./dbrepo-metadata-service/pom.xml clean install $MAVEN_OPTS -DskipTests"
script:
- "mvn -f ./dbrepo-data-service/pom.xml clean package $MAVEN_OPTS -DskipTests"
# Compiled classes are needed for SonarQube in later stages
artifacts:
@@ -192,19 +209,26 @@ build-data-service:
build-ui:
image: oven/bun:${BUN_VERSION}-alpine
stage: build
only:
- merge_requests
script:
- "cd ./dbrepo-ui && bun install && bun run build"
build-search-service:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: build
script:
only:
- merge_requests
before_script:
- "pip install pipenv"
script:
- "cd dbrepo-search-service && pipenv install --system --deploy"
build-images:
image: docker.io/docker:${DOCKER_VERSION}-dind
stage: build
only:
- merge_requests
before_script:
- "apk add --no-cache make"
- echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL
@@ -214,6 +238,8 @@ build-images:
build-helm:
image: docker.io/docker:${DOCKER_VERSION}-dind
stage: build
only:
- merge_requests
before_script:
- apk add --no-cache helm make
- echo "$CI_GPG_KEYRING" | base64 -d > ./secring.gpg
@@ -221,30 +247,11 @@ build-helm:
script:
- make build-helm
deploy-staging:
image: docker.io/alpine:${ALPINE_VERSION}
stage: deploy
variables:
KUBECONFIG: ./kubecfg
environment:
name: staging/datalab
url: ${CI_ENV_STAGING_URL}
before_script:
- apk add --no-cache helm kubectl make
- echo ${CI_K8S_CONFIG} | base64 -d > ./kubecfg
- make build-helm
- helm -n ${CI_ENV_STAGING_NAMESPACE} uninstall ${CI_ENV_STAGING_RELEASE_NAME} --ignore-not-found
- kubectl -n ${CI_ENV_STAGING_NAMESPACE} delete pvc --all
script:
- helm -n ${CI_ENV_STAGING_NAMESPACE} upgrade --install ${CI_ENV_STAGING_RELEASE_NAME} ./build/dbrepo-${CHART_VERSION}.tgz --create-namespace -f ./.gitlab/agents/dev/values.yaml
only:
refs:
- dev
- master
test-metadata-service:
image: maven:3-openjdk-${JAVA_VERSION}
stage: test
only:
- merge_requests
needs:
- build-metadata-service
dependencies:
@@ -265,6 +272,8 @@ test-metadata-service:
test-data-service:
image: maven:3-openjdk-${JAVA_VERSION}
stage: test
only:
- merge_requests
needs:
- build-data-service
dependencies:
@@ -283,16 +292,11 @@ test-data-service:
junit: ./dbrepo-data-service/rest-service/target/surefire-reports/TEST-*.xml
coverage: '/Total.*?([0-9]{1,3})%/'
test-upload-service:
image: maven:3-openjdk-${JAVA_VERSION}
stage: test
script:
- "mvn -f ./dbrepo-metadata-service/pom.xml clean install $MAVEN_OPTS -DskipTests"
- "mvn -f ./dbrepo-upload-service/pom.xml clean test $MAVEN_OPTS"
test-analyse-service:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: test
only:
- merge_requests
variables:
PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile"
needs:
@@ -302,21 +306,23 @@ test-analyse-service:
script:
- "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy"
- cd ./dbrepo-analyse-service/ && coverage run -m pytest tests/test_determine_dt.py tests/test_determine_pk.py tests/test_s3_client.py --junitxml=report.xml && coverage html --omit="tests/*" && coverage report --omit="tests/*" > ./coverage.txt
- cd ./dbrepo-analyse-service/ && coverage run --rcfile=.coveragerc -m pytest tests/test_determine_dt.py tests/test_determine_pk.py tests/test_s3_client.py && coverage html && coverage xml && coverage report > ./coverage.txt
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts:
when: always
paths:
- ./dbrepo-analyse-service/report.xml
- ./dbrepo-analyse-service/coverage.xml
- ./dbrepo-analyse-service/coverage.txt
expire_in: 1 days
reports:
junit: ./dbrepo-analyse-service/report.xml
junit: ./dbrepo-analyse-service/coverage.xml
coverage: '/TOTAL.*?([0-9]{1,3})%/'
test-auth-service-init:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: test
only:
- merge_requests
variables:
PIPENV_PIPFILE: "./dbrepo-auth-service/init/Pipfile"
before_script:
@@ -324,21 +330,23 @@ test-auth-service-init:
- "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy"
script:
- cd ./dbrepo-auth-service/init/ && coverage run -m pytest tests/test_unit_app.py --junitxml=report.xml && coverage html --omit="tests/*" && coverage report --omit="tests/*" > ./coverage.txt
- cd ./dbrepo-auth-service/init/ && coverage run --rcfile=.coveragerc -m pytest tests/test_unit_app.py && coverage html && coverage xml && coverage report > ./coverage.txt
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts:
when: always
paths:
- ./dbrepo-auth-service/init/report.xml
- ./dbrepo-auth-service/init/coverage.xml
- ./dbrepo-auth-service/init/coverage.txt
expire_in: 1 days
reports:
junit: ./dbrepo-auth-service/init/report.xml
junit: ./dbrepo-auth-service/init/coverage.xml
coverage: '/TOTAL.*?([0-9]{1,3})%/'
test-search-service:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: test
only:
- merge_requests
variables:
PIPENV_PIPFILE: "./dbrepo-search-service/Pipfile"
needs:
@@ -355,21 +363,23 @@ test-search-service:
script:
- "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy"
- cd ./dbrepo-search-service/ && coverage run -m pytest tests/test_app.py tests/test_jwt.py tests/test_opensearch_client.py tests/test_keycloak_client.py --junitxml=report.xml && coverage html && coverage report > ./coverage.txt
- cd ./dbrepo-search-service/ && coverage run --rcfile=.coveragerc -m pytest tests/test_app.py tests/test_jwt.py tests/test_opensearch_client.py tests/test_keycloak_client.py && coverage html && coverage xml && coverage report > ./coverage.txt
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts:
when: always
paths:
- ./dbrepo-search-service/report.xml
- ./dbrepo-search-service/coverage.xml
- ./dbrepo-search-service/coverage.txt
expire_in: 1 days
reports:
junit: ./dbrepo-search-service/report.xml
junit: ./dbrepo-search-service/coverage.xml
coverage: '/TOTAL.*?([0-9]{1,3})%/'
test-search-service-init:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: test
only:
- merge_requests
variables:
PIPENV_PIPFILE: "./dbrepo-search-service/init/Pipfile"
needs:
@@ -379,21 +389,23 @@ test-search-service-init:
script:
- "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy"
- cd ./dbrepo-search-service/init/ && coverage run -m pytest tests/test_app.py tests/test_keycloak_client.py tests/test_opensearch_client.py --junitxml=report.xml && coverage html && coverage report > ./coverage.txt
- cd ./dbrepo-search-service/init/ && coverage run --rcfile=.coveragerc -m pytest tests/test_app.py tests/test_keycloak_client.py tests/test_opensearch_client.py && coverage html && coverage xml && coverage report > ./coverage.txt
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts:
when: always
paths:
- ./dbrepo-search-service/report.xml
- ./dbrepo-search-service/coverage.xml
- ./dbrepo-search-service/coverage.txt
expire_in: 1 days
reports:
junit: ./dbrepo-search-service/report.xml
junit: ./dbrepo-search-service/coverage.xml
coverage: '/TOTAL.*?([0-9]{1,3})%/'
test-lib:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: test
only:
- merge_requests
variables:
PIPENV_PIPFILE: "./lib/python/Pipfile"
needs:
@@ -403,21 +415,23 @@ test-lib:
script:
- "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy"
- cd ./lib/python/ && coverage run -m pytest tests/test_unit_analyse.py tests/test_unit_container.py tests/test_unit_database.py tests/test_unit_identifier.py tests/test_unit_license.py tests/test_unit_query.py tests/test_unit_rest_client.py tests/test_unit_table.py tests/test_unit_user.py tests/test_unit_view.py tests/test_unit_rest_client.py --junitxml=report.xml && coverage html --omit="tests/*" && coverage report --omit="tests/*" > ./coverage.txt
- cd ./lib/python/ && coverage run --rcfile=.coveragerc -m pytest tests/test_unit_container.py tests/test_unit_messages.py tests/test_unit_image.py tests/test_unit_concept.py tests/test_unit_database.py tests/test_unit_identifier.py tests/test_unit_license.py tests/test_unit_query.py tests/test_unit_rest_client.py tests/test_unit_table.py tests/test_unit_user.py tests/test_unit_view.py tests/test_unit_unit.py && coverage html && coverage xml && coverage report > ./coverage.txt
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts:
when: always
paths:
- ./lib/python/report.xml
- ./lib/python/coverage.xml
- ./lib/python/coverage.txt
expire_in: 1 days
reports:
junit: ./lib/python/report.xml
junit: ./lib/python/coverage.xml
coverage: '/TOTAL.*?([0-9]{1,3})%/'
test-ui:
stage: test
image: docker.io/docker:${DOCKER_VERSION}-dind
stage: test
only:
- merge_requests
needs:
- build-ui
dependencies:
@@ -434,11 +448,17 @@ release-images:
stage: release
image: docker:${DOCKER_VERSION}-dind
dependencies:
- test-metadata-service
- test-data-service
- test-analyse-service
- test-auth-service-init
- test-data-service
- test-lib
- test-metadata-service
- test-search-service
- test-search-service-init
- test-ui
only:
refs:
- dev
- /^release-.*/
before_script:
- "docker logout ${CI_REGISTRY_URL}"
@@ -470,6 +490,8 @@ release-helm:
- "mkdir -p ~/.gnupg"
- echo "$CI_GPG_KEYRING" | base64 -d > ~/.gnupg/secring.gpg
- echo "$CI_GPG_KEYRING2" | base64 -d > ~/.gnupg/pubring.gpg
- helm registry login --username "${CI_REGISTRY_USER}" --password "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY_URL}"
- helm registry login --username "${CI_REGISTRY2_USER}" --password "${CI_REGISTRY2_PASSWORD}" "${CI_REGISTRY2_URL}"
- make build-helm
- "helm package ./helm/dbrepo --sign --key 'Martin Weise' --keyring ~/.gnupg/secring.gpg --destination ./build"
- "helm plugin install https://github.com/sigstore/helm-sigstore"
@@ -552,6 +574,27 @@ verify-dist:
script:
- "curl -v --output /dev/null --fail https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/${APP_VERSION}/dist.tar.gz"
deploy-staging:
image: docker.io/alpine:${ALPINE_VERSION}
stage: deploy
only:
refs:
- dev
variables:
KUBECONFIG: ./kubecfg
environment:
name: staging/datalab
url: ${CI_ENV_STAGING_URL}
before_script:
- apk add --no-cache helm kubectl make
- echo ${CI_K8S_CONFIG} | base64 -d > ./kubecfg
- make build-helm
- helm -n ${CI_ENV_STAGING_NAMESPACE} uninstall ${CI_ENV_STAGING_RELEASE_NAME} --ignore-not-found
- kubectl -n ${CI_ENV_STAGING_NAMESPACE} delete pvc --all
- echo "${CI_K8S_VALUES}" | base64 -d > ./values.yaml
script:
- helm -n ${CI_ENV_STAGING_NAMESPACE} upgrade --install ${CI_ENV_STAGING_RELEASE_NAME} ./build/dbrepo-${CHART_VERSION}.tgz --create-namespace -f ./values.yaml
scan-sonarqube:
image: sonarsource/sonar-scanner-cli:${SONARQUBE_VERSION}
stage: scan
Loading