From 2574fc8aff0100d927ebda85cf03a6717ca6e807 Mon Sep 17 00:00:00 2001
From: Martin Weise <martin.weise@tuwien.ac.at>
Date: Tue, 5 Dec 2023 10:09:37 +0100
Subject: [PATCH] Simplified the bucket creation via own image

---
 .gitlab-ci.yml                                | 42 +++++++++++++++++++
 Makefile                                      |  8 ++++
 dbrepo-storage-service/Dockerfile             | 10 -----
 dbrepo-storage-service/create-buckets.sh      | 18 --------
 dbrepo-storage-service/docker-entrypoint.sh   |  3 --
 dbrepo-storage-service/init/Dockerfile        |  7 ++++
 dbrepo-storage-service/init/create-buckets.sh | 10 +++++
 docker-compose.prod.yml                       | 20 +++++++--
 docker-compose.yml                            | 20 +++++++--
 9 files changed, 101 insertions(+), 37 deletions(-)
 delete mode 100644 dbrepo-storage-service/Dockerfile
 delete mode 100644 dbrepo-storage-service/create-buckets.sh
 delete mode 100644 dbrepo-storage-service/docker-entrypoint.sh
 create mode 100644 dbrepo-storage-service/init/Dockerfile
 create mode 100644 dbrepo-storage-service/init/create-buckets.sh

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 13d9a94dd1..461eeaaea8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -424,6 +424,48 @@ scan-log-service:
     reports:
       container_scanning: ./.trivy/trivy-log-service-report.json
 
+scan-storage-service:
+  image: bitnami/trivy:latest
+  stage: scan
+  only:
+    refs:
+      - dev
+      - master
+  allow_failure: true
+  script:
+    - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-storage-service-report.json docker.io/chrislusf/seaweedfs:3.59
+    - trivy image --insecure --exit-code 0 docker.io/chrislusf/seaweedfs:3.59
+    - trivy image --insecure --exit-code 1 --severity CRITICAL docker.io/chrislusf/seaweedfs:3.59
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-storage-service-report.json
+
+scan-storage-service-init:
+  image: bitnami/trivy:latest
+  stage: scan
+  only:
+    refs:
+      - dev
+      - master
+  allow_failure: true
+  script:
+    - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-storage-service-init-report.json docker.io/dbrepo/storage-service-init:latest
+    - trivy image --insecure --exit-code 0 docker.io/dbrepo/storage-service-init:latest
+    - trivy image --insecure --exit-code 1 --severity CRITICAL docker.io/dbrepo/storage-service-init:latest
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-storage-service-init-report.json
+
 release-latest:
   stage: release
   image: docker:24-dind
diff --git a/Makefile b/Makefile
index 0a9d7b3c93..56d500ab7e 100644
--- a/Makefile
+++ b/Makefile
@@ -94,6 +94,10 @@ tag-search-service:
 	docker tag dbrepo-search-service:latest "${REPOSITORY_URL}/search-service:${TAG}"
 	docker tag dbrepo-search-service:latest "${REPOSITORY2_URL}/search-service:${TAG}"
 
+tag-storage-service-init:
+	docker tag dbrepo-storage-service-init:latest "${REPOSITORY_URL}/storage-service-init:${TAG}"
+	docker tag dbrepo-storage-service-init:latest "${REPOSITORY2_URL}/storage-service-init:${TAG}"
+
 release: build-docker tag release-analyse-service release-authentication-service release-metadata-db release-ui release-metadata-service release-data-service release-log-service release-search-db release-mirror-service release-search-db-init release-search-service release-data-db-sidecar
 
 release-analyse-service: tag-analyse-service
@@ -144,6 +148,10 @@ release-search-service: tag-search-service
 	docker push "${REPOSITORY_URL}/search-service:${TAG}"
 	docker push "${REPOSITORY2_URL}/search-service:${TAG}"
 
+release-storage-service-init: tag-storage-service-init
+	docker push "${REPOSITORY_URL}/storage-service-init:${TAG}"
+	docker push "${REPOSITORY2_URL}/storage-service-init:${TAG}"
+
 test-backend: test-metadata-service test-analyse-service test-data-service test-mirror-service
 
 test-data-service: build-data-service
diff --git a/dbrepo-storage-service/Dockerfile b/dbrepo-storage-service/Dockerfile
deleted file mode 100644
index 39eaae4603..0000000000
--- a/dbrepo-storage-service/Dockerfile
+++ /dev/null
@@ -1,10 +0,0 @@
-FROM chrislusf/seaweedfs:3.59 as runtime
-
-RUN apk add curl
-
-WORKDIR /app
-
-COPY ./create-buckets.sh ./create-buckets.sh
-COPY ./docker-entrypoint.sh ./docker-entrypoint.sh
-
-ENTRYPOINT [ "/bin/sh", "./docker-entrypoint.sh" ]
\ No newline at end of file
diff --git a/dbrepo-storage-service/create-buckets.sh b/dbrepo-storage-service/create-buckets.sh
deleted file mode 100644
index bc57fdf5cc..0000000000
--- a/dbrepo-storage-service/create-buckets.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-function log {
-  echo "$(date '+%Y-%m-%d %H:%M:%S') $1"
-}
-
-log "Sleep 15s to start S3 API"
-sleep 15
-log "Start polling"
-until curl -sSL 127.0.0.1:9000
-do
-    log "S3 API not ready on port 9000, wait 5s ..."
-    sleep 5
-done
-log "Ready"
-echo "s3.bucket.create -name dbrepo-upload" | weed shell
-log "Created bucket dbrepo-upload"
-echo "s3.bucket.create -name dbrepo-download" | weed shell
-log "Created bucket dbrepo-download"
\ No newline at end of file
diff --git a/dbrepo-storage-service/docker-entrypoint.sh b/dbrepo-storage-service/docker-entrypoint.sh
deleted file mode 100644
index a1121f5443..0000000000
--- a/dbrepo-storage-service/docker-entrypoint.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-/bin/sh ./create-buckets.sh &
-/entrypoint.sh server -dir=/data -s3 -s3.port=9000 -s3.config=/app/s3_config.json -metricsPort=9091
\ No newline at end of file
diff --git a/dbrepo-storage-service/init/Dockerfile b/dbrepo-storage-service/init/Dockerfile
new file mode 100644
index 0000000000..72b080d6ce
--- /dev/null
+++ b/dbrepo-storage-service/init/Dockerfile
@@ -0,0 +1,7 @@
+FROM chrislusf/seaweedfs:3.59 as runtime
+
+WORKDIR /app
+
+COPY ./create-buckets.sh ./create-buckets.sh
+
+ENTRYPOINT [ "/bin/sh", "/app/create-buckets.sh" ]
\ No newline at end of file
diff --git a/dbrepo-storage-service/init/create-buckets.sh b/dbrepo-storage-service/init/create-buckets.sh
new file mode 100644
index 0000000000..d0e3223f66
--- /dev/null
+++ b/dbrepo-storage-service/init/create-buckets.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+function log {
+  echo "$(date '+%Y-%m-%d %H:%M:%S') $1"
+}
+
+log "Starting to create buckets dbrepo-upload, dbrepo-download"
+echo "s3.bucket.create -name dbrepo-upload" | weed shell -master="${SEAWEEDFS_ENDPOINT}"
+log "Created bucket dbrepo-upload"
+echo "s3.bucket.create -name dbrepo-download" | weed shell -master="${SEAWEEDFS_ENDPOINT}"
+log "Created bucket dbrepo-download"
\ No newline at end of file
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 463fa97f2d..7e858d9d40 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -343,21 +343,35 @@ services:
     restart: "no"
     container_name: dbrepo-storage-service
     hostname: storage-service
-    image: docker.io/dbrepo/storage-service:latest
-    build: ./dbrepo-storage-service
+    image: docker.io/chrislusf/seaweedfs:3.59
+    command: [ "server", "-dir=/data", "-s3", "-s3.port=9000", "-s3.config=/app/s3_config.json", "-metricsPort=9091" ]
     ports:
       - 9000:9000
     volumes:
       - ./dist/s3_config.json:/app/s3_config.json
       - storage-service-data:/data
     healthcheck:
-      test: curl -sSL 127.0.0.1:9000 || exit 1
+      test: echo "cluster.check" | weed shell | grep "checking master.*ok" || exit 1
       interval: 10s
       timeout: 5s
       retries: 12
     logging:
       driver: json-file
 
+  dbrepo-storage-service-init:
+    restart: "no"
+    container_name: dbrepo-storage-service-init
+    hostname: storage-service-init
+    image: dbrepo-storage-service-init:latest
+    build: ./dbrepo-storage-service/init
+    environment:
+      SEAWEEDFS_ENDPOINT: "${STORAGE_SEAWEEDFS_ENDPOINT:-storage-service:9333}"
+    depends_on:
+      dbrepo-storage-service:
+        condition: service_healthy
+    logging:
+      driver: json-file
+
   dbrepo-upload-service:
     restart: "no"
     container_name: dbrepo-upload-service
diff --git a/docker-compose.yml b/docker-compose.yml
index 711bea696e..a2ad475e50 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -352,21 +352,35 @@ services:
     restart: "no"
     container_name: dbrepo-storage-service
     hostname: storage-service
-    image: dbrepo-storage-service:latest
-    build: ./dbrepo-storage-service
+    image: docker.io/chrislusf/seaweedfs:3.59
+    command: [ "server", "-dir=/data", "-s3", "-s3.port=9000", "-s3.config=/app/s3_config.json", "-metricsPort=9091" ]
     ports:
       - 9000:9000
     volumes:
       - ./dbrepo-storage-service/s3_config.json:/app/s3_config.json
       - storage-service-data:/data
     healthcheck:
-      test: curl -sSL 127.0.0.1:9000 || exit 1
+      test: echo "cluster.check" | weed shell | grep "checking master.*ok" || exit 1
       interval: 10s
       timeout: 5s
       retries: 12
     logging:
       driver: json-file
 
+  dbrepo-storage-service-init:
+    restart: "no"
+    container_name: dbrepo-storage-service-init
+    hostname: storage-service-init
+    image: dbrepo-storage-service-init:latest
+    build: ./dbrepo-storage-service/init
+    environment:
+      SEAWEEDFS_ENDPOINT: "${STORAGE_SEAWEEDFS_ENDPOINT:-storage-service:9333}"
+    depends_on:
+      dbrepo-storage-service:
+        condition: service_healthy
+    logging:
+      driver: json-file
+
   dbrepo-upload-service:
     restart: "no"
     container_name: dbrepo-upload-service
-- 
GitLab