diff --git a/.docs/contributing.md b/.docs/contributing.md
index d77c6b49480d5a92adc5ef013757e06a68a4218a..2faa03fcdab0cb1f1a9a6a5182d53b0b81cec2ef 100644
--- a/.docs/contributing.md
+++ b/.docs/contributing.md
@@ -39,7 +39,7 @@ Python-based services have the coverage reports generated by `coverage` in the `
and `coverage.txt` log file respectively.
We run SonarQube quality checks on the `master` branch regularly to ensure security, maintainability and remove code
-smells. The internal instance can be found at [http://57.153.70.97:9000](http://57.153.70.97:9000)
+smells. The internal instance can be found at [https://s34.datalab.tuwien.ac.at/](https://s34.datalab.tuwien.ac.at/)
(requires internal authentication).
## Code Versioning
@@ -61,7 +61,8 @@ to run our pipeline:
<figcaption>Figure 2: Gitlab runner configuration in the cluster.</figcaption>
</figure>
-Minikube cluster with 6vCPU and 28GB RAM. The CI pipeline is configured as follows in the `config.toml`:
+Kubernetes cluster with PVC-enabled provisioner with 8 vCPU and 32GB RAM. The CI pipeline is configured as follows in
+the `config.toml`:
```toml
concurrent = 10
@@ -70,13 +71,19 @@ concurrent = 10
environment = [
"FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY=false"
]
+ [runners.cache]
+ Type = "s3"
+ Shared = true
+ [runners.cache.s3]
+ ServerAddress = "storageservice-s3:9000"
+ ...
+ Insecure = true
[runners.kubernetes]
namespace = "{{.Release.Namespace}}"
privileged = true
allowed_services = ["docker:24-dind"]
[[runners.kubernetes.services]]
name = "docker:24-dind"
- command = [ "--registry-mirror=http://docker-io-mirror:80", "--insecure-registry=docker-io-mirror:80", "--registry-mirror=http://gcr-io-mirror:80", "--insecure-registry=gcr-io-mirror:80", "--registry-mirror=http://ghcr-io-mirror:80", "--insecure-registry=ghcr-io-mirror:80", "--registry-mirror=http://k8s-io-mirror:80", "--insecure-registry=k8s-io-mirror:80" ]
alias = "docker"
[[runners.kubernetes.volumes.empty_dir]]
name = "rundind"
@@ -100,10 +107,6 @@ dind-sidecar container `svc-0` is started that exposes the Docker socket at `/va
container you can freely configure how you want. We are aware this is not optimal as it exposes *root* privileges in the
cluster.
-The full CI/CD pipeline Helm chart is documented in
-the [`fda-deployment`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-deployment/-/tree/master/charts/dbrepo-devops)
-repository.
-
## Documentation
For consistency reasons across the documentation, the resolution needs to be 1280x800 (16:10 ratio)
diff --git a/.docs/images/architecture.drawio b/.docs/images/architecture.drawio
index 57c8ee946752a574d8cc7c049ee71f6acbb1a7ce..711215b7ca8b4441dd02b9af6b0a6534576e48ab 100644
--- a/.docs/images/architecture.drawio
+++ b/.docs/images/architecture.drawio
@@ -1,4 +1,4 @@
-<mxfile host="Electron" modified="2024-02-23T12:34:29.996Z" agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/23.1.5 Chrome/120.0.6099.109 Electron/28.1.0 Safari/537.36" etag="wHh4Le3WT9S2OAsQk6sP" version="23.1.5" type="device" pages="8">
+<mxfile host="Electron" modified="2024-07-03T03:24:26.258Z" agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/24.5.3 Chrome/124.0.6367.207 Electron/30.0.6 Safari/537.36" etag="RiZoZviH4PcE-qe7O5P3" version="24.5.3" type="device" pages="8">
<diagram id="mvBsv1rP8O80Qe3yGnn_" name="docker-compose">
<mxGraphModel dx="1434" dy="822" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1169" pageHeight="827" math="0" shadow="0">
<root>
@@ -780,18 +780,18 @@
</mxGraphModel>
</diagram>
<diagram id="0gRvLy_AUZ0Xau8SBKI8" name="Gitlab Runner">
- <mxGraphModel dx="1434" dy="822" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1169" pageHeight="827" math="0" shadow="0">
+ <mxGraphModel dx="925" dy="530" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1169" pageHeight="827" math="0" shadow="0">
<root>
<mxCell id="0" />
<mxCell id="1" parent="0" />
<mxCell id="r9iJ3a_E54lvpWGN0BaG-1" value="" style="rounded=1;whiteSpace=wrap;html=1;arcSize=2;" parent="1" vertex="1">
- <mxGeometry x="280" y="40" width="730" height="260" as="geometry" />
+ <mxGeometry x="250" y="40" width="760" height="260" as="geometry" />
</mxCell>
- <mxCell id="r9iJ3a_E54lvpWGN0BaG-3" value="minikube" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=16;fontStyle=1" parent="1" vertex="1">
- <mxGeometry x="322" y="50" width="110" height="10" as="geometry" />
+ <mxCell id="r9iJ3a_E54lvpWGN0BaG-3" value="Kubernetes" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=16;fontStyle=1" parent="1" vertex="1">
+ <mxGeometry x="292" y="50" width="110" height="10" as="geometry" />
</mxCell>
- <mxCell id="r9iJ3a_E54lvpWGN0BaG-4" value="<font style="font-weight: normal; font-size: 10px;">128.130.202.116</font>" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=16;fontStyle=1" parent="1" vertex="1">
- <mxGeometry x="322" y="65" width="110" height="10" as="geometry" />
+ <mxCell id="r9iJ3a_E54lvpWGN0BaG-4" value="<font style="font-weight: normal; font-size: 10px;">azimuth.datalab.tuwien.ac.at</font>" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=16;fontStyle=1" parent="1" vertex="1">
+ <mxGeometry x="292" y="65" width="110" height="10" as="geometry" />
</mxCell>
<mxCell id="r9iJ3a_E54lvpWGN0BaG-30" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;" parent="1" source="r9iJ3a_E54lvpWGN0BaG-5" target="r9iJ3a_E54lvpWGN0BaG-6" edge="1">
<mxGeometry relative="1" as="geometry" />
@@ -813,14 +813,14 @@
<mxCell id="r9iJ3a_E54lvpWGN0BaG-36" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" parent="1" source="r9iJ3a_E54lvpWGN0BaG-5" target="r9iJ3a_E54lvpWGN0BaG-29" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
- <mxCell id="r9iJ3a_E54lvpWGN0BaG-5" value="<b>agentpool</b><br>6 CPU, 24GB RAM" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+ <mxCell id="r9iJ3a_E54lvpWGN0BaG-5" value="<b>agentpool</b><br>8 vCPU, 32GB RAM" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="567.5" y="75" width="115" height="40" as="geometry" />
</mxCell>
<mxCell id="r9iJ3a_E54lvpWGN0BaG-6" value="<b>gitlab-runner</b><br>kubernetes" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;" parent="1" vertex="1">
- <mxGeometry x="300" y="140" width="100" height="40" as="geometry" />
+ <mxGeometry x="270" y="140" width="100" height="40" as="geometry" />
</mxCell>
- <mxCell id="r9iJ3a_E54lvpWGN0BaG-7" value="<b>gitlab-runner</b><br>kubernetes" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;" parent="1" vertex="1">
- <mxGeometry x="420" y="140" width="100" height="40" as="geometry" />
+ <mxCell id="r9iJ3a_E54lvpWGN0BaG-7" value="<b>seaweedfs-s3</b><br>kubernetes" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;" parent="1" vertex="1">
+ <mxGeometry x="390" y="140" width="100" height="40" as="geometry" />
</mxCell>
<mxCell id="r9iJ3a_E54lvpWGN0BaG-20" value="" style="group" parent="1" vertex="1" connectable="0">
<mxGeometry x="540" y="140" width="170" height="140" as="geometry" />
@@ -881,8 +881,8 @@
<mxCell id="r9iJ3a_E54lvpWGN0BaG-29" value="•&nbsp;&nbsp;•&nbsp;&nbsp;•&nbsp; •&nbsp;&nbsp;•" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
<mxGeometry x="734" y="193" width="60" height="30" as="geometry" />
</mxCell>
- <mxCell id="vhbaQO3R_VC2zLA3inCI-4" value="" style="shape=image;verticalLabelPosition=bottom;labelBackgroundColor=default;verticalAlign=top;aspect=fixed;imageAspect=0;image=https://miro.medium.com/v2/resize:fit:400/0*KzqL3xqmXzV5PPjX.png;" parent="1" vertex="1">
- <mxGeometry x="290" y="50" width="25.78" height="25" as="geometry" />
+ <mxCell id="EFzMD_oVcaiTEb37ajv_-6" value="" style="shape=image;verticalLabelPosition=bottom;labelBackgroundColor=default;verticalAlign=top;aspect=fixed;imageAspect=0;image=https://cdn4.iconfinder.com/data/icons/logos-and-brands/512/144_Gitlab_logo_logos-512.png;" vertex="1" parent="1">
+ <mxGeometry x="259" y="47" width="30" height="30" as="geometry" />
</mxCell>
</root>
</mxGraphModel>
diff --git a/.docs/images/gitlab-runner.png b/.docs/images/gitlab-runner.png
index 8c530df1e3088aece21467dd58a7909374e9aeaa..e35da45655428a31e516d9bf133265239f334642 100644
Binary files a/.docs/images/gitlab-runner.png and b/.docs/images/gitlab-runner.png differ
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9362ff1808cf91436282b2048d3e6a48797738b8..35ff4a9f745a39a9b08a71b29028d2d5c4f533df 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,21 +1,27 @@
variables:
HOSTALIASES: ./hosts
- TRIVY_NO_PROGRESS: "true"
- TRIVY_CACHE_DIR: ".trivycache/"
DOCKER_HOST: "unix:///var/run/dind/docker.sock"
TESTCONTAINERS_RYUK_DISABLED: "false"
PYTHON_VERSION: "3.11"
APP_VERSION: "1.4.5"
CHART_VERSION: "1.4.5"
+ CACHE_FALLBACK_KEY: ${CI_DEFAULT_BRANCH}
+ # This will supress any download for dependencies and plugins or upload messages which would clutter the console log.
+ # `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work.
+ MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true -Dstyle.color=always"
+ # As of Maven 3.3.0 instead of this you may define these options in `.mvn/maven.config` so the same config is used
+ # when running from the command line.
+ # `installAtEnd` and `deployAtEnd` are only effective with recent version of the corresponding plugins.
+ MAVEN_CLI_OPTS: "--batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true"
image: debian:12-slim
+# Cache downloaded dependencies and plugins between builds.
+# To keep cache across branches add 'key: "$CI_JOB_NAME"'
cache:
- policy: pull
- key: ${CI_BUILD_REF_NAME}
+ key: ${CI_COMMIT_REF_SLUG}
paths:
- - final/
- - .m2/
+ - .m2/repository
stages:
- build
@@ -33,7 +39,7 @@ build-metadata-service:
refs:
- /^release-.*/
script:
- - "mvn -f ./dbrepo-metadata-service/pom.xml clean install -Dstyle.color=always -DskipTests"
+ - "mvn -f ./dbrepo-metadata-service/pom.xml clean install $MAVEN_OPTS -DskipTests"
build-analyse-service:
image: docker.io/python:3.11-alpine
@@ -80,8 +86,7 @@ build-data-service:
needs:
- build-metadata-service
script:
- - "mvn -f ./dbrepo-metadata-service/pom.xml clean install -Dstyle.color=always -DskipTests"
- - "mvn -f ./dbrepo-data-service/pom.xml clean package -Dstyle.color=always -DskipTests"
+ - "mvn -f ./dbrepo-data-service/pom.xml clean package $MAVEN_OPTS -DskipTests"
build-ui:
image: oven/bun:1.0.26-alpine
@@ -163,8 +168,7 @@ test-metadata-service:
needs:
- build-metadata-service
script:
- - "mvn -f ./dbrepo-metadata-service/pom.xml clean install -Dstyle.color=always -DskipTests"
- - "mvn -f ./dbrepo-metadata-service/pom.xml clean test -Dstyle.color=always verify"
+ - "mvn -f ./dbrepo-metadata-service/pom.xml clean test $MAVEN_OPTS verify"
- "cat ./dbrepo-metadata-service/report/target/site/jacoco-aggregate/index.html | grep -o 'Total[^%]*%' | sed 's/<.*>/ /; s/Total/Jacoco Coverage Total:/'"
artifacts:
when: always
@@ -185,8 +189,7 @@ test-data-service:
needs:
- build-data-service
script:
- - "mvn -f ./dbrepo-metadata-service/pom.xml clean install -Dstyle.color=always -DskipTests"
- - "mvn -f ./dbrepo-data-service/pom.xml clean test verify -Dstyle.color=always"
+ - "mvn -f ./dbrepo-data-service/pom.xml clean test verify $MAVEN_OPTS"
- "cat ./dbrepo-data-service/report/target/site/jacoco-aggregate/index.html | grep -o 'Total[^%]*%' | sed 's/<.*>/ /; s/Total/Jacoco Coverage Total:/'"
artifacts:
when: always
@@ -274,8 +277,11 @@ test-lib:
coverage: '/TOTAL.*?([0-9]{1,3})%/'
scan-sonarqube:
- stage: scan
image: sonarsource/sonar-scanner-cli:10.0
+ stage: scan
+ only:
+ refs:
+ - master
needs:
- test-data-service
- test-metadata-service
diff --git a/sonar-project.properties b/sonar-project.properties
index 3f91e17ad3107ecdcc4cbf381b96ce8782c97e7a..05bed6ec2f5c08b529e8d310470d96a779854d1a 100644
--- a/sonar-project.properties
+++ b/sonar-project.properties
@@ -1,6 +1,6 @@
# sonarqube
-sonar.projectKey=fair-data-austria-db-repository_fda-services_d2c9835a-dab9-4600-9255-ec7d50358811
-sonar.host.url=http://57.153.70.97:9000
+sonar.projectKey=fair-data-austria-db-repository_fda-services_a57fa043-ab99-4cdd-a721-162d9a916d77
+sonar.host.url=https://s34.datalab.tuwien.ac.at
# project
sonar.projectVersion=1.4.4
# general