Merge branch '398-replace-minio-with-seaweedfs' into 'dev'

Simplified the bucket creation via own image

See merge request !219

.gitlab-ci.yml

@@ -424,6 +424,48 @@ scan-log-service:
     reports:
       container_scanning: ./.trivy/trivy-log-service-report.json
 
+scan-storage-service:
+  image: bitnami/trivy:latest
+  stage: scan
+  only:
+    refs:
+      - dev
+      - master
+  allow_failure: true
+  script:
+    - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-storage-service-report.json docker.io/chrislusf/seaweedfs:3.59
+    - trivy image --insecure --exit-code 0 docker.io/chrislusf/seaweedfs:3.59
+    - trivy image --insecure --exit-code 1 --severity CRITICAL docker.io/chrislusf/seaweedfs:3.59
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-storage-service-report.json
+
+scan-storage-service-init:
+  image: bitnami/trivy:latest
+  stage: scan
+  only:
+    refs:
+      - dev
+      - master
+  allow_failure: true
+  script:
+    - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-storage-service-init-report.json docker.io/dbrepo/storage-service-init:latest
+    - trivy image --insecure --exit-code 0 docker.io/dbrepo/storage-service-init:latest
+    - trivy image --insecure --exit-code 1 --severity CRITICAL docker.io/dbrepo/storage-service-init:latest
+  cache:
+    paths:
+      - .trivycache/
+  artifacts:
+    when: always
+    expire_in: 1 days
+    reports:
+      container_scanning: ./.trivy/trivy-storage-service-init-report.json
+
 release-latest:
   stage: release
   image: docker:24-dind

Makefile

@@ -94,6 +94,10 @@ tag-search-service:
 	docker tag dbrepo-search-service:latest "${REPOSITORY_URL}/search-service:${TAG}"
 	docker tag dbrepo-search-service:latest "${REPOSITORY2_URL}/search-service:${TAG}"
 
+tag-storage-service-init:
+	docker tag dbrepo-storage-service-init:latest "${REPOSITORY_URL}/storage-service-init:${TAG}"
+	docker tag dbrepo-storage-service-init:latest "${REPOSITORY2_URL}/storage-service-init:${TAG}"
+
 release: build-docker tag release-analyse-service release-authentication-service release-metadata-db release-ui release-metadata-service release-data-service release-log-service release-search-db release-mirror-service release-search-db-init release-search-service release-data-db-sidecar
 
 release-analyse-service: tag-analyse-service
@@ -144,6 +148,10 @@ release-search-service: tag-search-service
 	docker push "${REPOSITORY_URL}/search-service:${TAG}"
 	docker push "${REPOSITORY2_URL}/search-service:${TAG}"
 
+release-storage-service-init: tag-storage-service-init
+	docker push "${REPOSITORY_URL}/storage-service-init:${TAG}"
+	docker push "${REPOSITORY2_URL}/storage-service-init:${TAG}"
+
 test-backend: test-metadata-service test-analyse-service test-data-service test-mirror-service
 
 test-data-service: build-data-service

dbrepo-storage-service/docker-entrypoint.sh

-#!/bin/sh
-/bin/sh ./create-buckets.sh &
-/entrypoint.sh server -dir=/data -s3 -s3.port=9000 -s3.config=/app/s3_config.json -metricsPort=9091
\ No newline at end of file

dbrepo-storage-service/Dockerfile → dbrepo-storage-service/init/Dockerfile

 FROM chrislusf/seaweedfs:3.59 as runtime
 
-RUN apk add curl
-
 WORKDIR /app
 
 COPY ./create-buckets.sh ./create-buckets.sh
-COPY ./docker-entrypoint.sh ./docker-entrypoint.sh
 
-ENTRYPOINT [ "/bin/sh", "./docker-entrypoint.sh" ]
+ENTRYPOINT [ "/bin/sh", "/app/create-buckets.sh" ]
\ No newline at end of file

dbrepo-storage-service/create-buckets.sh → dbrepo-storage-service/init/create-buckets.sh

@@ -3,16 +3,8 @@ function log {
   echo "$(date '+%Y-%m-%d %H:%M:%S') $1"
 }
 
-log "Sleep 15s to start S3 API"
-sleep 15
-log "Start polling"
-until curl -sSL 127.0.0.1:9000
-do
-    log "S3 API not ready on port 9000, wait 5s ..."
-    sleep 5
-done
-log "Ready"
-echo "s3.bucket.create -name dbrepo-upload" | weed shell
+log "Starting to create buckets dbrepo-upload, dbrepo-download"
+echo "s3.bucket.create -name dbrepo-upload" | weed shell -master="${SEAWEEDFS_ENDPOINT}"
 log "Created bucket dbrepo-upload"
-echo "s3.bucket.create -name dbrepo-download" | weed shell
+echo "s3.bucket.create -name dbrepo-download" | weed shell -master="${SEAWEEDFS_ENDPOINT}"
 log "Created bucket dbrepo-download"
\ No newline at end of file

docker-compose.prod.yml

@@ -343,21 +343,35 @@ services:
     restart: "no"
     container_name: dbrepo-storage-service
     hostname: storage-service
-    image: docker.io/dbrepo/storage-service:latest
-    build: ./dbrepo-storage-service
+    image: docker.io/chrislusf/seaweedfs:3.59
+    command: [ "server", "-dir=/data", "-s3", "-s3.port=9000", "-s3.config=/app/s3_config.json", "-metricsPort=9091" ]
     ports:
       - 9000:9000
     volumes:
       - ./dist/s3_config.json:/app/s3_config.json
       - storage-service-data:/data
     healthcheck:
-      test: curl -sSL 127.0.0.1:9000 || exit 1
+      test: echo "cluster.check" | weed shell | grep "checking master.*ok" || exit 1
       interval: 10s
       timeout: 5s
       retries: 12
     logging:
       driver: json-file
 
+  dbrepo-storage-service-init:
+    restart: "no"
+    container_name: dbrepo-storage-service-init
+    hostname: storage-service-init
+    image: dbrepo-storage-service-init:latest
+    build: ./dbrepo-storage-service/init
+    environment:
+      SEAWEEDFS_ENDPOINT: "${STORAGE_SEAWEEDFS_ENDPOINT:-storage-service:9333}"
+    depends_on:
+      dbrepo-storage-service:
+        condition: service_healthy
+    logging:
+      driver: json-file
+
   dbrepo-upload-service:
     restart: "no"
     container_name: dbrepo-upload-service

docker-compose.yml

@@ -352,21 +352,35 @@ services:
     restart: "no"
     container_name: dbrepo-storage-service
     hostname: storage-service
-    image: dbrepo-storage-service:latest
-    build: ./dbrepo-storage-service
+    image: docker.io/chrislusf/seaweedfs:3.59
+    command: [ "server", "-dir=/data", "-s3", "-s3.port=9000", "-s3.config=/app/s3_config.json", "-metricsPort=9091" ]
     ports:
       - 9000:9000
     volumes:
       - ./dbrepo-storage-service/s3_config.json:/app/s3_config.json
       - storage-service-data:/data
     healthcheck:
-      test: curl -sSL 127.0.0.1:9000 || exit 1
+      test: echo "cluster.check" | weed shell | grep "checking master.*ok" || exit 1
       interval: 10s
       timeout: 5s
       retries: 12
     logging:
       driver: json-file
 
+  dbrepo-storage-service-init:
+    restart: "no"
+    container_name: dbrepo-storage-service-init
+    hostname: storage-service-init
+    image: dbrepo-storage-service-init:latest
+    build: ./dbrepo-storage-service/init
+    environment:
+      SEAWEEDFS_ENDPOINT: "${STORAGE_SEAWEEDFS_ENDPOINT:-storage-service:9333}"
+    depends_on:
+      dbrepo-storage-service:
+        condition: service_healthy
+    logging:
+      driver: json-file
+
   dbrepo-upload-service:
     restart: "no"
     container_name: dbrepo-upload-service