From e2d816c56f612ecf583b4aaf5c0cc5e824b910a8 Mon Sep 17 00:00:00 2001 From: Martin Weise <martin.weise@tuwien.ac.at> Date: Sun, 3 Dec 2023 17:04:18 +0100 Subject: [PATCH] Remove remaining traces of minIO --- dbrepo-analyse-service/Dockerfile | 4 +- .../clients/{minio_client.py => s3_client.py} | 6 +-- dbrepo-analyse-service/determine_dt.py | 8 ++-- dbrepo-analyse-service/determine_pk.py | 8 ++-- ...test_minio_client.py => test_s3_client.py} | 26 ++++++------- dbrepo-metadata-service/Dockerfile | 4 +- docker-compose.prod.yml | 37 +++++++++---------- install.sh | 1 + 8 files changed, 46 insertions(+), 48 deletions(-) rename dbrepo-analyse-service/clients/{minio_client.py => s3_client.py} (97%) rename dbrepo-analyse-service/test/{test_minio_client.py => test_s3_client.py} (68%) diff --git a/dbrepo-analyse-service/Dockerfile b/dbrepo-analyse-service/Dockerfile index 52912bdceb..bc0f52276a 100644 --- a/dbrepo-analyse-service/Dockerfile +++ b/dbrepo-analyse-service/Dockerfile @@ -15,8 +15,8 @@ ENV FLASK_ENV=production ENV HOSTNAME=analyse-service ENV LOG_LEVEL=INFO ENV S3_STORAGE_ENDPOINT="http://storage-service:9000" -ENV S3_ACCESS_KEY_ID="minioadmin" -ENV S3_SECRET_ACCESS_KEY="minioadmin" +ENV S3_ACCESS_KEY_ID="seaweedfsadmin" +ENV S3_SECRET_ACCESS_KEY="seaweedfsadmin" COPY ./as-yml ./as-yml COPY ./clients ./clients diff --git a/dbrepo-analyse-service/clients/minio_client.py b/dbrepo-analyse-service/clients/s3_client.py similarity index 97% rename from dbrepo-analyse-service/clients/minio_client.py rename to dbrepo-analyse-service/clients/s3_client.py index d88da1b983..c0df983fcc 100644 --- a/dbrepo-analyse-service/clients/minio_client.py +++ b/dbrepo-analyse-service/clients/s3_client.py @@ -5,12 +5,12 @@ import logging from botocore.exceptions import ClientError -class MinioClient: +class S3Client: def __init__(self): endpoint_url = os.getenv('S3_STORAGE_ENDPOINT', 'http://localhost:9000') - aws_access_key_id = os.getenv('S3_ACCESS_KEY_ID', 'minioadmin') - aws_secret_access_key = os.getenv('S3_SECRET_ACCESS_KEY', 'minioadmin') + aws_access_key_id = os.getenv('S3_ACCESS_KEY_ID', 'seaweedfsadmin') + aws_secret_access_key = os.getenv('S3_SECRET_ACCESS_KEY', 'seaweedfsadmin') logging.info("retrieve file from S3, endpoint_url=%s, aws_access_key_id=%s, aws_secret_access_key=(hidden)", endpoint_url, aws_access_key_id) self.client = boto3.client(service_name='s3', endpoint_url=endpoint_url, aws_access_key_id=aws_access_key_id, diff --git a/dbrepo-analyse-service/determine_dt.py b/dbrepo-analyse-service/determine_dt.py index 9584bd0351..8ac23898db 100644 --- a/dbrepo-analyse-service/determine_dt.py +++ b/dbrepo-analyse-service/determine_dt.py @@ -6,7 +6,7 @@ import json import csv import logging import io -from clients.minio_client import MinioClient +from clients.s3_client import S3Client import messytables, pandas as pd from messytables import CSVTableSet, type_guess, \ @@ -17,9 +17,9 @@ def determine_datatypes(filename, enum=False, enum_tol=0.0001, separator=None) - # Use option enum=True for searching Postgres ENUM Types in CSV file. Remark # Enum is not SQL standard, hence, it might not be supported by all db-engines. # However, it can be used in Postgres and MySQL. - minio_client = MinioClient() - minio_client.file_exists('dbrepo-upload', filename) - response = minio_client.get_file('dbrepo-upload', filename) + s3_client = S3Client() + s3_client.file_exists('dbrepo-upload', filename) + response = s3_client.get_file('dbrepo-upload', filename) stream = response['Body'] if response['ContentLength'] == 0: logging.warning(f'Failed to determine data types: file {filename} has empty body') diff --git a/dbrepo-analyse-service/determine_pk.py b/dbrepo-analyse-service/determine_pk.py index 8431e96d30..cc18205289 100644 --- a/dbrepo-analyse-service/determine_pk.py +++ b/dbrepo-analyse-service/determine_pk.py @@ -5,7 +5,7 @@ import random import numpy as np import math from determine_dt import determine_datatypes -from clients.minio_client import MinioClient +from clients.s3_client import S3Client def determine_pk(filename, separator=','): @@ -15,9 +15,9 @@ def determine_pk(filename, separator=','): colnames = dt.keys() colindex = list(range(0, len(colnames))) - minio_client = MinioClient() - minio_client.file_exists('dbrepo-upload', filename) - response = minio_client.get_file('dbrepo-upload', filename) + s3_client = S3Client() + s3_client.file_exists('dbrepo-upload', filename) + response = s3_client.get_file('dbrepo-upload', filename) stream = response['Body'] if response['ContentLength'] == 0: logging.warning(f'Failed to determine primary key: file {filename} has empty body') diff --git a/dbrepo-analyse-service/test/test_minio_client.py b/dbrepo-analyse-service/test/test_s3_client.py similarity index 68% rename from dbrepo-analyse-service/test/test_minio_client.py rename to dbrepo-analyse-service/test/test_s3_client.py index 9c125a3522..8fc6affbaf 100644 --- a/dbrepo-analyse-service/test/test_minio_client.py +++ b/dbrepo-analyse-service/test/test_s3_client.py @@ -9,16 +9,16 @@ import unittest from botocore.exceptions import ClientError -from clients.minio_client import MinioClient +from clients.s3_client import S3Client -class MinioClientTest(unittest.TestCase): +class S3ClientTest(unittest.TestCase): # @Test def test_upload_file_succeeds(self): # test - response = MinioClient().upload_file(filename="testdt01.csv", path="./data/") + response = S3Client().upload_file(filename="testdt01.csv", path="./data/") self.assertTrue(response) # @Test @@ -26,7 +26,7 @@ class MinioClientTest(unittest.TestCase): # test try: - MinioClient().upload_file(filename="testdt06.csv", path="./data/") + S3Client().upload_file(filename="testdt06.csv", path="./data/") except FileNotFoundError: pass except Exception: @@ -38,10 +38,10 @@ class MinioClientTest(unittest.TestCase): def test_download_file_succeeds(self): # mock - MinioClient().upload_file(filename="testdt01.csv", path="./data/", bucket="dbrepo-upload") + S3Client().upload_file(filename="testdt01.csv", path="./data/", bucket="dbrepo-upload") # test - response = MinioClient().download_file(filename="testdt01.csv") + response = S3Client().download_file(filename="testdt01.csv") self.assertTrue(response) # @Test @@ -49,7 +49,7 @@ class MinioClientTest(unittest.TestCase): # test try: - MinioClient().download_file(filename="testdt01.csv") + S3Client().download_file(filename="testdt01.csv") except ClientError: pass except Exception: @@ -61,10 +61,10 @@ class MinioClientTest(unittest.TestCase): def test_get_file_succeeds(self): # mock - MinioClient().upload_file(filename="testdt01.csv", path="./data/", bucket="dbrepo-upload") + S3Client().upload_file(filename="testdt01.csv", path="./data/", bucket="dbrepo-upload") # test - response = MinioClient().get_file(bucket="dbrepo-upload", filename="testdt01.csv") + response = S3Client().get_file(bucket="dbrepo-upload", filename="testdt01.csv") self.assertIsNotNone(response) # @Test @@ -72,7 +72,7 @@ class MinioClientTest(unittest.TestCase): # test try: - MinioClient().get_file(bucket="dbrepo-upload", filename="idonotexist.csv") + S3Client().get_file(bucket="dbrepo-upload", filename="idonotexist.csv") except ClientError: pass except Exception: @@ -84,7 +84,7 @@ class MinioClientTest(unittest.TestCase): def test_bucket_exists_succeeds(self): # test - response = MinioClient().bucket_exists_or_exit("dbrepo-upload") + response = S3Client().bucket_exists_or_exit("dbrepo-upload") self.assertIsNotNone(response) # @Test @@ -92,7 +92,7 @@ class MinioClientTest(unittest.TestCase): # test try: - MinioClient().bucket_exists_or_exit("idnonotexist") + S3Client().bucket_exists_or_exit("idnonotexist") except FileNotFoundError: pass except Exception: @@ -105,7 +105,7 @@ class MinioClientTest(unittest.TestCase): # test try: - MinioClient().bucket_exists_or_exit("idnonotexist") + S3Client().bucket_exists_or_exit("idnonotexist") except FileNotFoundError: pass except Exception: diff --git a/dbrepo-metadata-service/Dockerfile b/dbrepo-metadata-service/Dockerfile index f3cebd4f4e..4a7bd0a5ba 100644 --- a/dbrepo-metadata-service/Dockerfile +++ b/dbrepo-metadata-service/Dockerfile @@ -75,8 +75,8 @@ ENV DATACITE_PREFIX="" ENV DATACITE_USERNAME="" ENV DATACITE_PASSWORD="" ENV S3_STORAGE_ENDPOINT="http://storage-service:9000" -ENV S3_ACCESS_KEY_ID="minioadmin" -ENV S3_SECRET_ACCESS_KEY="minioadmin" +ENV S3_ACCESS_KEY_ID="seaweedfsadmin" +ENV S3_SECRET_ACCESS_KEY="seaweedfsadmin" WORKDIR /app diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index f27ba6337f..463fa97f2d 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -39,7 +39,7 @@ services: restart: "no" container_name: dbrepo-data-db hostname: data-db - image: docker.io/bitnami/mariadb:11.1.3 + image: docker.io/bitnami/mariadb-galera:11.1.3-debian-11-r0 volumes: - data-db-data:/bitnami/mariadb - "${SHARED_FILESYSTEM:-/tmp}:/tmp" @@ -47,6 +47,7 @@ services: - "3307:3306" environment: MARIADB_ROOT_PASSWORD: "${USER_DB_PASSWORD:-dbrepo}" + MARIADB_GALERA_MARIABACKUP_PASSWORD: "${USER_DB_BACKUP_PASSWORD:-dbrepo}" healthcheck: test: mysqladmin ping --user="${USER_DB_USERNAME:-root}" --password="${USER_DB_PASSWORD:-dbrepo}" --silent interval: 10s @@ -178,8 +179,8 @@ services: - "5000:5000" environment: S3_STORAGE_ENDPOINT: "${STORAGE_ENDPOINT:-http://storage-service:9000}" - S3_ACCESS_KEY_ID: "${STORAGE_USERNAME:-minioadmin}" - S3_SECRET_ACCESS_KEY: "${STORAGE_PASSWORD:-minioadmin}" + S3_ACCESS_KEY_ID: "${STORAGE_USERNAME:-seaweedfsadmin}" + S3_SECRET_ACCESS_KEY: "${STORAGE_PASSWORD:-seaweedfsadmin}" volumes: - "${SHARED_FILESYSTEM:-/tmp}:/tmp" healthcheck: @@ -258,8 +259,8 @@ services: environment: FLASK_DEBUG: ${SEARCH_DEBUG_MODE:-true} S3_STORAGE_ENDPOINT: "${STORAGE_ENDPOINT:-http://storage-service:9000}" - S3_ACCESS_KEY_ID: "${STORAGE_USERNAME:-minioadmin}" - S3_SECRET_ACCESS_KEY: ${STORAGE_PASSWORD:-minioadmin} + S3_ACCESS_KEY_ID: "${STORAGE_USERNAME:-seaweedfsadmin}" + S3_SECRET_ACCESS_KEY: ${STORAGE_PASSWORD:-seaweedfsadmin} volumes: - "${SHARED_FILESYSTEM:-/tmp}:/tmp" healthcheck: @@ -342,22 +343,18 @@ services: restart: "no" container_name: dbrepo-storage-service hostname: storage-service - image: docker.io/bitnami/minio:2023-debian-11 - environment: - MINIO_ROOT_USER: "${STORAGE_USERNAME:-minioadmin}" - MINIO_ROOT_PASSWORD: "${STORAGE_PASSWORD:-minioadmin}" - MINIO_DEFAULT_BUCKETS: "${STORAGE_DBREPO_BUCKET:-dbrepo-upload:upload,dbrepo-download:download}" - MINIO_REGION_NAME: "${STORAGE_REGION_NAME:-eu-west-1}" - MINIO_BROWSER_REDIRECT_URL: "${STORAGE_BASE_URL:-http://localhost/admin/storage/}" + image: docker.io/dbrepo/storage-service:latest + build: ./dbrepo-storage-service ports: - 9000:9000 + volumes: + - ./dist/s3_config.json:/app/s3_config.json + - storage-service-data:/data healthcheck: - test: [ "CMD", "mc", "ready", "local" ] - interval: 5s + test: curl -sSL 127.0.0.1:9000 || exit 1 + interval: 10s timeout: 5s - retries: 5 - volumes: - - storage-service-data:/bitnami/minio/data + retries: 12 logging: driver: json-file @@ -373,9 +370,9 @@ services: - "-s3-endpoint=${STORAGE_ENDPOINT:-http://storage-service:9000}" - "-s3-bucket=dbrepo-upload" environment: - AWS_ACCESS_KEY_ID: "${STORAGE_USERNAME:-minioadmin}" - AWS_SECRET_ACCESS_KEY: "${STORAGE_PASSWORD:-minioadmin}" - AWS_REGION: "${STORAGE_REGION_NAME:-eu-west-1}" + AWS_ACCESS_KEY_ID: "${STORAGE_USERNAME:-seaweedfsadmin}" + AWS_SECRET_ACCESS_KEY: "${STORAGE_PASSWORD:-seaweedfsadmin}" + AWS_REGION: "${STORAGE_REGION_NAME:-default}" depends_on: dbrepo-storage-service: condition: service_healthy diff --git a/install.sh b/install.sh index 0a2941913f..da07dd9482 100644 --- a/install.sh +++ b/install.sh @@ -23,6 +23,7 @@ curl -sSL -o ./dist/definitions.json https://gitlab.phaidra.org/fair-data-austri curl -sSL -o ./dist/dbrepo.conf https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/dbrepo-gateway-service/dbrepo.conf curl -sSL -o ./dist/opensearch_dashboards.yml https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/dbrepo-search-db/opensearch_dashboards.yml curl -sSL -o ./dist/dbrepo.config.json https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/dbrepo-ui/dbrepo.config.json +curl -sSL -o ./dist/s3_config.json https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/dbrepo-storage-service/s3_config.json echo "[📦] Pulling images ..." docker compose pull -- GitLab