Merge branch '309-ci-cd-vulnerability-scanner' into 'dev'

Resolve "CI/CD vulnerability scanner"

See merge request !158

.gitignore

@@ -32,6 +32,9 @@ fda-ui/
 # Environment
 .env
 
+# scanning
+.trivy/trivy-*.json
+
 # Debug
 debug.txt
 

.gitlab-ci.yml

@@ -9,7 +9,9 @@ before_script:
 variables:
   HOSTALIASES: ./hosts
   DOCKER_TLS_CERTDIR: /certs
-  TAG: ${TAG:-latest}
+  TAG: "${TAG:-latest}"
+  TRIVY_NO_PROGRESS: "true"
+  TRIVY_CACHE_DIR: ".trivycache/"
 
 cache:
   paths:
@@ -24,6 +26,7 @@ stages:
   - test-backend
   - test-frontend
   - build-docker
+  - scan-docker
   - release
 
 build-metadata-db:
@@ -334,10 +337,332 @@ build-frontend:
   script:
     - make build-frontend
 
+scan-analyse-service:
+  stage: scan-docker
+  needs:
+    - build-docker
+  allow_failure: true
+  before_script:
+    - docker logout ghcr.io
+  script:
+    - make scan-analyse-service
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-analyse-service-report.json
+
+scan-authentication-service:
+  stage: scan-docker
+  needs:
+    - build-docker
+  allow_failure: true
+  before_script:
+    - docker logout ghcr.io
+  script:
+    - make scan-authentication-service
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-authentication-service-report.json
+
+scan-broker-service:
+  stage: scan-docker
+  needs:
+    - build-docker
+  allow_failure: true
+  before_script:
+    - docker logout ghcr.io
+  script:
+    - make scan-broker-service
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-broker-service-report.json
+
+scan-container-service:
+  stage: scan-docker
+  needs:
+    - build-docker
+  allow_failure: true
+  before_script:
+    - docker logout ghcr.io
+  script:
+    - make scan-container-service
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-container-service-report.json
+
+scan-database-service:
+  stage: scan-docker
+  needs:
+    - build-docker
+  allow_failure: true
+  before_script:
+    - docker logout ghcr.io
+  script:
+    - make scan-database-service
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-database-service-report.json
+
+scan-discovery-service:
+  stage: scan-docker
+  needs:
+    - build-docker
+  allow_failure: true
+  before_script:
+    - docker logout ghcr.io
+  script:
+    - make scan-discovery-service
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-discovery-service-report.json
+
+scan-gateway-service:
+  stage: scan-docker
+  needs:
+    - build-docker
+  allow_failure: true
+  before_script:
+    - docker logout ghcr.io
+  script:
+    - make scan-gateway-service
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-gateway-service-report.json
+
+scan-identifier-service:
+  stage: scan-docker
+  needs:
+    - build-docker
+  allow_failure: true
+  before_script:
+    - docker logout ghcr.io
+  script:
+    - make scan-identifier-service
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-identifier-service-report.json
+
+scan-metadata-db:
+  stage: scan-docker
+  needs:
+    - build-docker
+  allow_failure: true
+  before_script:
+    - docker logout ghcr.io
+  script:
+    - make scan-metadata-db
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-metadata-db-report.json
+
+scan-metadata-service:
+  stage: scan-docker
+  needs:
+    - build-docker
+  allow_failure: true
+  before_script:
+    - docker logout ghcr.io
+  script:
+    - make scan-metadata-service
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-metadata-service-report.json
+
+scan-proxy:
+  stage: scan-docker
+  needs:
+    - build-docker
+  allow_failure: true
+  before_script:
+    - docker logout ghcr.io
+  script:
+    - make scan-proxy
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-proxy-report.json
+
+scan-query-service:
+  stage: scan-docker
+  needs:
+    - build-docker
+  allow_failure: true
+  before_script:
+    - docker logout ghcr.io
+  script:
+    - make scan-query-service
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-query-service-report.json
+
+scan-search-service:
+  stage: scan-docker
+  needs:
+    - build-docker
+  allow_failure: true
+  before_script:
+    - docker logout ghcr.io
+  script:
+    - make scan-search-service
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-search-service-report.json
+
+scan-semantics-service:
+  stage: scan-docker
+  needs:
+    - build-docker
+  allow_failure: true
+  before_script:
+    - docker logout ghcr.io
+  script:
+    - make scan-semantics-service
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-semantics-service-report.json
+
+scan-table-service:
+  stage: scan-docker
+  needs:
+    - build-docker
+  allow_failure: true
+  before_script:
+    - docker logout ghcr.io
+  script:
+    - make scan-table-service
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-table-service-report.json
+
+scan-ui:
+  stage: scan-docker
+  needs:
+    - build-docker
+  allow_failure: true
+  before_script:
+    - docker logout ghcr.io
+  script:
+    - make scan-ui
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-ui-report.json
+
+scan-user-service:
+  stage: scan-docker
+  needs:
+    - build-docker
+  allow_failure: true
+  before_script:
+    - docker logout ghcr.io
+  script:
+    - make scan-user-service
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-user-service-report.json
+
 release-latest:
   stage: release
   needs:
-    - build-docker
+    - scan-analyse-service
+    - scan-authentication-service
+    - scan-broker-service
+    - scan-container-service
+    - scan-database-service
+    - scan-discovery-service
+    - scan-gateway-service
+    - scan-identifier-service
+    - scan-metadata-db
+    - scan-metadata-service
+    - scan-proxy
+    - scan-query-service
+    - scan-search-service
+    - scan-semantics-service
+    - scan-table-service
+    - scan-ui
+    - scan-user-service
   only:
     refs:
       - dev
@@ -349,7 +674,23 @@ release-latest:
 release-version:
   stage: release
   needs:
-    - build-docker
+    - scan-analyse-service
+    - scan-authentication-service
+    - scan-broker-service
+    - scan-container-service
+    - scan-database-service
+    - scan-discovery-service
+    - scan-gateway-service
+    - scan-identifier-service
+    - scan-metadata-db
+    - scan-metadata-service
+    - scan-proxy
+    - scan-query-service
+    - scan-search-service
+    - scan-semantics-service
+    - scan-table-service
+    - scan-ui
+    - scan-user-service
   only:
     refs:
       - master

.trivy/gitlab.tpl

+{{- /* Template based on https://docs.gitlab.com/ee/user/application_security/container_scanning/#reports-json-format */ -}}
+{
+  "version": "14.0.6",
+  "vulnerabilities": [
+  {{- $t_first := true }}
+  {{- range . }}
+  {{- $target := .Target }}
+    {{- $image := $target | regexFind "[^\\s]+" }}
+    {{- range .Vulnerabilities -}}
+    {{- if $t_first -}}
+      {{- $t_first = false -}}
+    {{ else -}}
+      ,
+    {{- end }}
+    {
+      "id": "{{ .VulnerabilityID }}",
+      "category": "container_scanning",
+      "message": {{ .Title | printf "%q" }},
+      "description": {{ .Description | printf "%q" }},
+      {{- /* cve is a deprecated key, use id instead */}}
+      "cve": "{{ .VulnerabilityID }}",
+      "severity": {{ if eq .Severity "UNKNOWN" -}}
+                    "Unknown"
+                  {{- else if eq .Severity "LOW" -}}
+                    "Low"
+                  {{- else if eq .Severity "MEDIUM" -}}
+                    "Medium"
+                  {{- else if eq .Severity "HIGH" -}}
+                    "High"
+                  {{- else if eq .Severity "CRITICAL" -}}
+                    "Critical"
+                  {{-  else -}}
+                    "{{ .Severity }}"
+                  {{- end }},
+      "solution": {{ if .FixedVersion -}}
+                    "Upgrade {{ .PkgName }} to {{ .FixedVersion }}"
+                  {{- else -}}
+                    "No solution provided"
+                  {{- end }},
+      "scanner": {
+        "id": "trivy",
+        "name": "trivy"
+      },
+      "location": {
+        "dependency": {
+          "package": {
+            "name": "{{ .PkgName }}"
+          },
+          "version": "{{ .InstalledVersion }}"
+        },
+        {{- /* TODO: No mapping available - https://github.com/aquasecurity/trivy/issues/332 */}}
+        "operating_system": "Unknown",
+        "image": "{{ $image }}"
+      },
+      "identifiers": [
+        {
+	  {{- /* TODO: Type not extractable - https://github.com/aquasecurity/trivy-db/pull/24 */}}
+          "type": "cve",
+          "name": "{{ .VulnerabilityID }}",
+          "value": "{{ .VulnerabilityID }}",
+          "url": "{{ .PrimaryURL }}"
+        }
+      ],
+      "links": [
+        {{- $l_first := true -}}
+        {{- range .References -}}
+        {{- if $l_first -}}
+          {{- $l_first = false }}
+        {{- else -}}
+          ,
+        {{- end -}}
+        {
+          "url": "{{ regexFind "[^ ]+" . }}"
+        }
+        {{- end }}
+      ]
+    }
+    {{- end -}}
+  {{- end }}
+  ],
+  "remediations": []
+}

Makefile

 .PHONY: clean all
 
 TAG ?= latest
+TRIVY_VERSION ?= v0.41.0
 
 all:
 
@@ -189,6 +190,93 @@ test-semantics-service: build-semantics-service
 test-analyse-service: build-analyse-service
 	bash ./dbrepo-analyse-service/test.sh
 
+scan: scan-analyse-service scan-authentication-service scan-broker-service scan-container-service scan-database-service scan-discovery-service scan-gateway-service scan-identifier-service scan-metadata-db scan-metadata-service scan-proxy scan-query-service scan-search-service scan-semantics-service scan-table-service scan-ui scan-user-service
+
+scan-analyse-service:
+	trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-analyse-service-report.json dbrepo-analyse-service:latest
+	trivy image --insecure --exit-code 0 dbrepo-analyse-service:latest
+	trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-analyse-service:latest
+
+scan-authentication-service:
+	trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-authentication-service-report.json dbrepo-authentication-service:latest
+	trivy image --insecure --exit-code 0 dbrepo-authentication-service:latest
+	trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-authentication-service:latest
+
+scan-broker-service:
+	trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-broker-service-report.json dbrepo-broker-service:latest
+	trivy image --insecure --exit-code 0 dbrepo-broker-service:latest
+	trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-broker-service:latest
+
+scan-container-service:
+	trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-container-service-report.json dbrepo-container-service:latest
+	trivy image --insecure --exit-code 0 dbrepo-container-service:latest
+	trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-container-service:latest
+
+scan-database-service:
+	trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-database-service-report.json dbrepo-database-service:latest
+	trivy image --insecure --exit-code 0 dbrepo-database-service:latest
+	trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-database-service:latest
+
+scan-discovery-service:
+	trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-discovery-service-report.json dbrepo-discovery-service:latest
+	trivy image --insecure --exit-code 0 dbrepo-discovery-service:latest
+	trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-discovery-service:latest
+
+scan-gateway-service:
+	trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-gateway-service-report.json dbrepo-gateway-service:latest
+	trivy image --insecure --exit-code 0 dbrepo-gateway-service:latest
+	trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-gateway-service:latest
+
+scan-identifier-service:
+	trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-identifier-service-report.json dbrepo-identifier-service:latest
+	trivy image --insecure --exit-code 0 dbrepo-identifier-service:latest
+	trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-identifier-service:latest
+
+scan-metadata-db:
+	trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-metadata-db-report.json dbrepo-metadata-db:latest
+	trivy image --insecure --exit-code 0 dbrepo-metadata-db:latest
+	trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-metadata-db:latest
+
+scan-metadata-service:
+	trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-metadata-service-report.json dbrepo-metadata-service:latest
+	trivy image --insecure --exit-code 0 dbrepo-metadata-service:latest
+	trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-metadata-service:latest
+
+scan-proxy:
+	trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-proxy-report.json dbrepo-proxy:latest
+	trivy image --insecure --exit-code 0 dbrepo-proxy:latest
+	trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-proxy:latest
+
+scan-query-service:
+	trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-query-service-report.json dbrepo-query-service:latest
+	trivy image --insecure --exit-code 0 dbrepo-query-service:latest
+	trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-query-service:latest
+
+scan-search-service:
+	trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-search-service-report.json dbrepo-search-service:latest
+	trivy image --insecure --exit-code 0 dbrepo-search-service:latest
+	trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-search-service:latest
+
+scan-semantics-service:
+	trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-semantics-service-report.json dbrepo-semantics-service:latest
+	trivy image --insecure --exit-code 0 dbrepo-semantics-service:latest
+	trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-semantics-service:latest
+
+scan-table-service:
+	trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-table-service-report.json dbrepo-table-service:latest
+	trivy image --insecure --exit-code 0 dbrepo-table-service:latest
+	trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-table-service:latest
+
+scan-ui:
+	trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-ui-report.json dbrepo-ui:latest
+	trivy image --insecure --exit-code 0 dbrepo-ui:latest
+	trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-ui:latest
+
+scan-user-service:
+	trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-user-service-report.json dbrepo-user-service:latest
+	trivy image --insecure --exit-code 0 dbrepo-user-service:latest
+	trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-user-service:latest
+
 coverage-frontend: build-frontend
 	yarn --cwd ./dbrepo-ui run coverage || true