Dev

981de84f · Martin Weise · ebf4b87d · 981de84f · 981de84f · ebf4b87d
Commit 981de84f authored 1 year ago by Martin Weise
--- a/.docs/contributing.md
+++ b/.docs/contributing.md
@@ -39,7 +39,7 @@ Python-based services have the coverage reports generated by `coverage` in the `
 and `coverage.txt` log file respectively.
 We run SonarQube quality checks on the `master` branch regularly to ensure security, maintainability and remove code
-smells. The internal instance can be found at [http://57.153.70.97:9000](http://57.153.70.97:9000)
+smells. The internal instance can be found at [https://s34.datalab.tuwien.ac.at/](https://s34.datalab.tuwien.ac.at/)
 (requires internal authentication).
 ## Code Versioning
@@ -61,7 +61,8 @@ to run our pipeline:
 <figcaption>Figure 2: Gitlab runner configuration in the cluster.</figcaption>
 </figure>
-Minikube cluster with 6vCPU and 28GB RAM. The CI pipeline is configured as follows in the `config.toml`:
+Kubernetes cluster with PVC-enabled provisioner with 8 vCPU and 32GB RAM. The CI pipeline is configured as follows in
+the `config.toml`:
 ```toml
 concurrent = 10
@@ -70,13 +71,19 @@ concurrent = 10
  environment = [
    "FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY=false"
  ]
+  [runners.cache]
+    Type = "s3"
+    Shared = true
+    [runners.cache.s3]
+      ServerAddress = "storageservice-s3:9000"
+      ...
+      Insecure = true
  [runners.kubernetes]
    namespace = "{{.Release.Namespace}}"
    privileged = true
    allowed_services = ["docker:24-dind"]
    [[runners.kubernetes.services]]
      name = "docker:24-dind"
-      command = [ "--registry-mirror=http://docker-io-mirror:80", "--insecure-registry=docker-io-mirror:80", "--registry-mirror=http://gcr-io-mirror:80", "--insecure-registry=gcr-io-mirror:80", "--registry-mirror=http://ghcr-io-mirror:80", "--insecure-registry=ghcr-io-mirror:80", "--registry-mirror=http://k8s-io-mirror:80", "--insecure-registry=k8s-io-mirror:80" ]
      alias = "docker"
    [[runners.kubernetes.volumes.empty_dir]]
      name = "rundind"
@@ -100,10 +107,6 @@ dind-sidecar container `svc-0` is started that exposes the Docker socket at `/va
 container you can freely configure how you want. We are aware this is not optimal as it exposes *root* privileges in the
 cluster.
-The full CI/CD pipeline Helm chart is documented in
-the [`fda-deployment`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-deployment/-/tree/master/charts/dbrepo-devops)
-repository.
 ## Documentation
 For consistency reasons across the documentation, the resolution needs to be 1280x800 (16:10 ratio)

--- a/.docs/images/architecture.drawio
+++ b/.docs/images/architecture.drawio
-<mxfile host="Electron" modified="2024-02-23T12:34:29.996Z" agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/23.1.5 Chrome/120.0.6099.109 Electron/28.1.0 Safari/537.36" etag="wHh4Le3WT9S2OAsQk6sP" version="23.1.5" type="device" pages="8">
+<mxfile host="Electron" modified="2024-07-03T03:24:26.258Z" agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/24.5.3 Chrome/124.0.6367.207 Electron/30.0.6 Safari/537.36" etag="RiZoZviH4PcE-qe7O5P3" version="24.5.3" type="device" pages="8">
  <diagram id="mvBsv1rP8O80Qe3yGnn_" name="docker-compose">
    <mxGraphModel dx="1434" dy="822" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1169" pageHeight="827" math="0" shadow="0">
      <root>
@@ -780,18 +780,18 @@
    </mxGraphModel>
  </diagram>
  <diagram id="0gRvLy_AUZ0Xau8SBKI8" name="Gitlab Runner">
-    <mxGraphModel dx="1434" dy="822" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1169" pageHeight="827" math="0" shadow="0">
+    <mxGraphModel dx="925" dy="530" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1169" pageHeight="827" math="0" shadow="0">
      <root>
        <mxCell id="0" />
        <mxCell id="1" parent="0" />
        <mxCell id="r9iJ3a_E54lvpWGN0BaG-1" value="" style="rounded=1;whiteSpace=wrap;html=1;arcSize=2;" parent="1" vertex="1">
-          <mxGeometry x="280" y="40" width="730" height="260" as="geometry" />
+          <mxGeometry x="250" y="40" width="760" height="260" as="geometry" />
        </mxCell>
-        <mxCell id="r9iJ3a_E54lvpWGN0BaG-3" value="minikube" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=16;fontStyle=1" parent="1" vertex="1">
+        <mxCell id="r9iJ3a_E54lvpWGN0BaG-3" value="Kubernetes" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=16;fontStyle=1" parent="1" vertex="1">
-          <mxGeometry x="322" y="50" width="110" height="10" as="geometry" />
+          <mxGeometry x="292" y="50" width="110" height="10" as="geometry" />
        </mxCell>
-        <mxCell id="r9iJ3a_E54lvpWGN0BaG-4" value="&lt;font style=&quot;font-weight: normal; font-size: 10px;&quot;&gt;128.130.202.116&lt;/font&gt;" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=16;fontStyle=1" parent="1" vertex="1">
+        <mxCell id="r9iJ3a_E54lvpWGN0BaG-4" value="&lt;font style=&quot;font-weight: normal; font-size: 10px;&quot;&gt;azimuth.datalab.tuwien.ac.at&lt;/font&gt;" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=16;fontStyle=1" parent="1" vertex="1">
-          <mxGeometry x="322" y="65" width="110" height="10" as="geometry" />
+          <mxGeometry x="292" y="65" width="110" height="10" as="geometry" />
        </mxCell>
        <mxCell id="r9iJ3a_E54lvpWGN0BaG-30" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;" parent="1" source="r9iJ3a_E54lvpWGN0BaG-5" target="r9iJ3a_E54lvpWGN0BaG-6" edge="1">
          <mxGeometry relative="1" as="geometry" />
@@ -813,14 +813,14 @@
        <mxCell id="r9iJ3a_E54lvpWGN0BaG-36" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" parent="1" source="r9iJ3a_E54lvpWGN0BaG-5" target="r9iJ3a_E54lvpWGN0BaG-29" edge="1">
          <mxGeometry relative="1" as="geometry" />
        </mxCell>
-        <mxCell id="r9iJ3a_E54lvpWGN0BaG-5" value="&lt;b&gt;agentpool&lt;/b&gt;&lt;br&gt;6 CPU, 24GB RAM" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+        <mxCell id="r9iJ3a_E54lvpWGN0BaG-5" value="&lt;b&gt;agentpool&lt;/b&gt;&lt;br&gt;8 vCPU, 32GB RAM" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
          <mxGeometry x="567.5" y="75" width="115" height="40" as="geometry" />
        </mxCell>
        <mxCell id="r9iJ3a_E54lvpWGN0BaG-6" value="&lt;b&gt;gitlab-runner&lt;/b&gt;&lt;br&gt;kubernetes" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;" parent="1" vertex="1">
-          <mxGeometry x="300" y="140" width="100" height="40" as="geometry" />
+          <mxGeometry x="270" y="140" width="100" height="40" as="geometry" />
        </mxCell>
-        <mxCell id="r9iJ3a_E54lvpWGN0BaG-7" value="&lt;b&gt;gitlab-runner&lt;/b&gt;&lt;br&gt;kubernetes" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;" parent="1" vertex="1">
+        <mxCell id="r9iJ3a_E54lvpWGN0BaG-7" value="&lt;b&gt;seaweedfs-s3&lt;/b&gt;&lt;br&gt;kubernetes" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;" parent="1" vertex="1">
-          <mxGeometry x="420" y="140" width="100" height="40" as="geometry" />
+          <mxGeometry x="390" y="140" width="100" height="40" as="geometry" />
        </mxCell>
        <mxCell id="r9iJ3a_E54lvpWGN0BaG-20" value="" style="group" parent="1" vertex="1" connectable="0">
          <mxGeometry x="540" y="140" width="170" height="140" as="geometry" />
@@ -881,8 +881,8 @@
        <mxCell id="r9iJ3a_E54lvpWGN0BaG-29" value="•&amp;nbsp;&amp;nbsp;•&amp;nbsp;&amp;nbsp;•&amp;nbsp; •&amp;nbsp;&amp;nbsp;•" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
          <mxGeometry x="734" y="193" width="60" height="30" as="geometry" />
        </mxCell>
-        <mxCell id="vhbaQO3R_VC2zLA3inCI-4" value="" style="shape=image;verticalLabelPosition=bottom;labelBackgroundColor=default;verticalAlign=top;aspect=fixed;imageAspect=0;image=https://miro.medium.com/v2/resize:fit:400/0*KzqL3xqmXzV5PPjX.png;" parent="1" vertex="1">
+        <mxCell id="EFzMD_oVcaiTEb37ajv_-6" value="" style="shape=image;verticalLabelPosition=bottom;labelBackgroundColor=default;verticalAlign=top;aspect=fixed;imageAspect=0;image=https://cdn4.iconfinder.com/data/icons/logos-and-brands/512/144_Gitlab_logo_logos-512.png;" vertex="1" parent="1">
-          <mxGeometry x="290" y="50" width="25.78" height="25" as="geometry" />
+          <mxGeometry x="259" y="47" width="30" height="30" as="geometry" />
        </mxCell>
      </root>
    </mxGraphModel>

--- a/.docs/images/gitlab-runner.png
+++ b/.docs/images/gitlab-runner.png
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
 variables:
  HOSTALIASES: ./hosts
-  TRIVY_NO_PROGRESS: "true"
-  TRIVY_CACHE_DIR: ".trivycache/"
  DOCKER_HOST: "unix:///var/run/dind/docker.sock"
  TESTCONTAINERS_RYUK_DISABLED: "false"
  PYTHON_VERSION: "3.11"
  APP_VERSION: "1.4.5"
  CHART_VERSION: "1.4.5"
+  CACHE_FALLBACK_KEY: ${CI_DEFAULT_BRANCH}
+  # This will supress any download for dependencies and plugins or upload messages which would clutter the console log.
+  # `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work.
+  MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true -Dstyle.color=always"
+  # As of Maven 3.3.0 instead of this you may define these options in `.mvn/maven.config` so the same config is used
+  # when running from the command line.
+  # `installAtEnd` and `deployAtEnd` are only effective with recent version of the corresponding plugins.
+  MAVEN_CLI_OPTS: "--batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true"
 image: debian:12-slim
+# Cache downloaded dependencies and plugins between builds.
+# To keep cache across branches add 'key: "$CI_JOB_NAME"'
 cache:
-  policy: pull
+  key: ${CI_COMMIT_REF_SLUG}
-  key: ${CI_BUILD_REF_NAME}
  paths:
-    - final/
+    - .m2/repository
-    - .m2/
 stages:
  - build
@@ -33,7 +39,7 @@ build-metadata-service:
    refs:
      - /^release-.*/
  script:
-    - "mvn -f ./dbrepo-metadata-service/pom.xml clean install -Dstyle.color=always -DskipTests"
+    - "mvn -f ./dbrepo-metadata-service/pom.xml clean install $MAVEN_OPTS -DskipTests"
 build-analyse-service:
  image: docker.io/python:3.11-alpine
@@ -80,8 +86,7 @@ build-data-service:
  needs:
    - build-metadata-service
  script:
-    - "mvn -f ./dbrepo-metadata-service/pom.xml clean install -Dstyle.color=always -DskipTests"
+    - "mvn -f ./dbrepo-data-service/pom.xml clean package $MAVEN_OPTS -DskipTests"
-    - "mvn -f ./dbrepo-data-service/pom.xml clean package -Dstyle.color=always -DskipTests"
 build-ui:
  image: oven/bun:1.0.26-alpine
@@ -163,8 +168,7 @@ test-metadata-service:
  needs:
    - build-metadata-service
  script:
-    - "mvn -f ./dbrepo-metadata-service/pom.xml clean install -Dstyle.color=always -DskipTests"
+    - "mvn -f ./dbrepo-metadata-service/pom.xml clean test $MAVEN_OPTS verify"
-    - "mvn -f ./dbrepo-metadata-service/pom.xml clean test -Dstyle.color=always verify"
    - "cat ./dbrepo-metadata-service/report/target/site/jacoco-aggregate/index.html | grep -o 'Total[^%]*%' | sed 's/<.*>/ /; s/Total/Jacoco Coverage Total:/'"
  artifacts:
    when: always
@@ -185,8 +189,7 @@ test-data-service:
  needs:
    - build-data-service
  script:
-    - "mvn -f ./dbrepo-metadata-service/pom.xml clean install -Dstyle.color=always -DskipTests"
+    - "mvn -f ./dbrepo-data-service/pom.xml clean test verify $MAVEN_OPTS"
-    - "mvn -f ./dbrepo-data-service/pom.xml clean test verify -Dstyle.color=always"
    - "cat ./dbrepo-data-service/report/target/site/jacoco-aggregate/index.html | grep -o 'Total[^%]*%' | sed 's/<.*>/ /; s/Total/Jacoco Coverage Total:/'"
  artifacts:
    when: always
@@ -274,8 +277,11 @@ test-lib:
  coverage: '/TOTAL.*?([0-9]{1,3})%/'
 scan-sonarqube:
-  stage: scan
  image: sonarsource/sonar-scanner-cli:10.0
+  stage: scan
+  only:
+    refs:
+      - master
  needs:
    - test-data-service
    - test-metadata-service

--- a/sonar-project.properties
+++ b/sonar-project.properties
 # sonarqube
-sonar.projectKey=fair-data-austria-db-repository_fda-services_d2c9835a-dab9-4600-9255-ec7d50358811
+sonar.projectKey=fair-data-austria-db-repository_fda-services_a57fa043-ab99-4cdd-a721-162d9a916d77
-sonar.host.url=http://57.153.70.97:9000
+sonar.host.url=https://s34.datalab.tuwien.ac.at
 # project
 sonar.projectVersion=1.4.4
 # general