diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 13d9a94dd11fad3a7a5229f80494eeb072608d1e..461eeaaea80674f4f36936b3ebb8045d2abd91f3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -424,6 +424,48 @@ scan-log-service: reports: container_scanning: ./.trivy/trivy-log-service-report.json +scan-storage-service: + image: bitnami/trivy:latest + stage: scan + only: + refs: + - dev + - master + allow_failure: true + script: + - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-storage-service-report.json docker.io/chrislusf/seaweedfs:3.59 + - trivy image --insecure --exit-code 0 docker.io/chrislusf/seaweedfs:3.59 + - trivy image --insecure --exit-code 1 --severity CRITICAL docker.io/chrislusf/seaweedfs:3.59 + cache: + paths: + - .trivycache/ + artifacts: + when: always + expire_in: 1 days + reports: + container_scanning: ./.trivy/trivy-storage-service-report.json + +scan-storage-service-init: + image: bitnami/trivy:latest + stage: scan + only: + refs: + - dev + - master + allow_failure: true + script: + - trivy image --insecure --exit-code 0 --format template --template "@.trivy/gitlab.tpl" -o ./.trivy/trivy-storage-service-init-report.json docker.io/dbrepo/storage-service-init:latest + - trivy image --insecure --exit-code 0 docker.io/dbrepo/storage-service-init:latest + - trivy image --insecure --exit-code 1 --severity CRITICAL docker.io/dbrepo/storage-service-init:latest + cache: + paths: + - .trivycache/ + artifacts: + when: always + expire_in: 1 days + reports: + container_scanning: ./.trivy/trivy-storage-service-init-report.json + release-latest: stage: release image: docker:24-dind diff --git a/Makefile b/Makefile index 0a9d7b3c934935a7ab23e2679b48f8c2806816c5..56d500ab7e33b2f9154f285e3bd4aa85f9f4bffc 100644 --- a/Makefile +++ b/Makefile @@ -94,6 +94,10 @@ tag-search-service: docker tag dbrepo-search-service:latest "${REPOSITORY_URL}/search-service:${TAG}" docker tag dbrepo-search-service:latest "${REPOSITORY2_URL}/search-service:${TAG}" +tag-storage-service-init: + docker tag dbrepo-storage-service-init:latest "${REPOSITORY_URL}/storage-service-init:${TAG}" + docker tag dbrepo-storage-service-init:latest "${REPOSITORY2_URL}/storage-service-init:${TAG}" + release: build-docker tag release-analyse-service release-authentication-service release-metadata-db release-ui release-metadata-service release-data-service release-log-service release-search-db release-mirror-service release-search-db-init release-search-service release-data-db-sidecar release-analyse-service: tag-analyse-service @@ -144,6 +148,10 @@ release-search-service: tag-search-service docker push "${REPOSITORY_URL}/search-service:${TAG}" docker push "${REPOSITORY2_URL}/search-service:${TAG}" +release-storage-service-init: tag-storage-service-init + docker push "${REPOSITORY_URL}/storage-service-init:${TAG}" + docker push "${REPOSITORY2_URL}/storage-service-init:${TAG}" + test-backend: test-metadata-service test-analyse-service test-data-service test-mirror-service test-data-service: build-data-service diff --git a/dbrepo-storage-service/Dockerfile b/dbrepo-storage-service/Dockerfile deleted file mode 100644 index 39eaae4603993fe24f9ff0b8d113c0d7149301bc..0000000000000000000000000000000000000000 --- a/dbrepo-storage-service/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM chrislusf/seaweedfs:3.59 as runtime - -RUN apk add curl - -WORKDIR /app - -COPY ./create-buckets.sh ./create-buckets.sh -COPY ./docker-entrypoint.sh ./docker-entrypoint.sh - -ENTRYPOINT [ "/bin/sh", "./docker-entrypoint.sh" ] \ No newline at end of file diff --git a/dbrepo-storage-service/create-buckets.sh b/dbrepo-storage-service/create-buckets.sh deleted file mode 100644 index bc57fdf5cc90edef167bc9b849bb48d8e3a29ebd..0000000000000000000000000000000000000000 --- a/dbrepo-storage-service/create-buckets.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -function log { - echo "$(date '+%Y-%m-%d %H:%M:%S') $1" -} - -log "Sleep 15s to start S3 API" -sleep 15 -log "Start polling" -until curl -sSL 127.0.0.1:9000 -do - log "S3 API not ready on port 9000, wait 5s ..." - sleep 5 -done -log "Ready" -echo "s3.bucket.create -name dbrepo-upload" | weed shell -log "Created bucket dbrepo-upload" -echo "s3.bucket.create -name dbrepo-download" | weed shell -log "Created bucket dbrepo-download" \ No newline at end of file diff --git a/dbrepo-storage-service/docker-entrypoint.sh b/dbrepo-storage-service/docker-entrypoint.sh deleted file mode 100644 index a1121f5443b2ba26b3e56b4f50da2b6b0e4f7b8d..0000000000000000000000000000000000000000 --- a/dbrepo-storage-service/docker-entrypoint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -/bin/sh ./create-buckets.sh & -/entrypoint.sh server -dir=/data -s3 -s3.port=9000 -s3.config=/app/s3_config.json -metricsPort=9091 \ No newline at end of file diff --git a/dbrepo-storage-service/init/Dockerfile b/dbrepo-storage-service/init/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..72b080d6ce059260eb16f3a703ec45dd0d77d8c6 --- /dev/null +++ b/dbrepo-storage-service/init/Dockerfile @@ -0,0 +1,7 @@ +FROM chrislusf/seaweedfs:3.59 as runtime + +WORKDIR /app + +COPY ./create-buckets.sh ./create-buckets.sh + +ENTRYPOINT [ "/bin/sh", "/app/create-buckets.sh" ] \ No newline at end of file diff --git a/dbrepo-storage-service/init/create-buckets.sh b/dbrepo-storage-service/init/create-buckets.sh new file mode 100644 index 0000000000000000000000000000000000000000..d0e3223f669147a3fb87da50bf5cde02b7c2362a --- /dev/null +++ b/dbrepo-storage-service/init/create-buckets.sh @@ -0,0 +1,10 @@ +#!/bin/bash +function log { + echo "$(date '+%Y-%m-%d %H:%M:%S') $1" +} + +log "Starting to create buckets dbrepo-upload, dbrepo-download" +echo "s3.bucket.create -name dbrepo-upload" | weed shell -master="${SEAWEEDFS_ENDPOINT}" +log "Created bucket dbrepo-upload" +echo "s3.bucket.create -name dbrepo-download" | weed shell -master="${SEAWEEDFS_ENDPOINT}" +log "Created bucket dbrepo-download" \ No newline at end of file diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 463fa97f2dc8615ae3226879a597752e530b2200..7e858d9d40a87a777366a4154cdf9eac83aeee0c 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -343,21 +343,35 @@ services: restart: "no" container_name: dbrepo-storage-service hostname: storage-service - image: docker.io/dbrepo/storage-service:latest - build: ./dbrepo-storage-service + image: docker.io/chrislusf/seaweedfs:3.59 + command: [ "server", "-dir=/data", "-s3", "-s3.port=9000", "-s3.config=/app/s3_config.json", "-metricsPort=9091" ] ports: - 9000:9000 volumes: - ./dist/s3_config.json:/app/s3_config.json - storage-service-data:/data healthcheck: - test: curl -sSL 127.0.0.1:9000 || exit 1 + test: echo "cluster.check" | weed shell | grep "checking master.*ok" || exit 1 interval: 10s timeout: 5s retries: 12 logging: driver: json-file + dbrepo-storage-service-init: + restart: "no" + container_name: dbrepo-storage-service-init + hostname: storage-service-init + image: dbrepo-storage-service-init:latest + build: ./dbrepo-storage-service/init + environment: + SEAWEEDFS_ENDPOINT: "${STORAGE_SEAWEEDFS_ENDPOINT:-storage-service:9333}" + depends_on: + dbrepo-storage-service: + condition: service_healthy + logging: + driver: json-file + dbrepo-upload-service: restart: "no" container_name: dbrepo-upload-service diff --git a/docker-compose.yml b/docker-compose.yml index 711bea696e40dc3b073eea11a31c52b47bc54d66..a2ad475e50a60c5654713b0f7b966998bedc335f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -352,21 +352,35 @@ services: restart: "no" container_name: dbrepo-storage-service hostname: storage-service - image: dbrepo-storage-service:latest - build: ./dbrepo-storage-service + image: docker.io/chrislusf/seaweedfs:3.59 + command: [ "server", "-dir=/data", "-s3", "-s3.port=9000", "-s3.config=/app/s3_config.json", "-metricsPort=9091" ] ports: - 9000:9000 volumes: - ./dbrepo-storage-service/s3_config.json:/app/s3_config.json - storage-service-data:/data healthcheck: - test: curl -sSL 127.0.0.1:9000 || exit 1 + test: echo "cluster.check" | weed shell | grep "checking master.*ok" || exit 1 interval: 10s timeout: 5s retries: 12 logging: driver: json-file + dbrepo-storage-service-init: + restart: "no" + container_name: dbrepo-storage-service-init + hostname: storage-service-init + image: dbrepo-storage-service-init:latest + build: ./dbrepo-storage-service/init + environment: + SEAWEEDFS_ENDPOINT: "${STORAGE_SEAWEEDFS_ENDPOINT:-storage-service:9333}" + depends_on: + dbrepo-storage-service: + condition: service_healthy + logging: + driver: json-file + dbrepo-upload-service: restart: "no" container_name: dbrepo-upload-service