Remove remaining traces of minIO

dbrepo-analyse-service/Dockerfile

@@ -15,8 +15,8 @@ ENV FLASK_ENV=production
 ENV HOSTNAME=analyse-service
 ENV LOG_LEVEL=INFO
 ENV S3_STORAGE_ENDPOINT="http://storage-service:9000"
-ENV S3_ACCESS_KEY_ID="minioadmin"
-ENV S3_SECRET_ACCESS_KEY="minioadmin"
+ENV S3_ACCESS_KEY_ID="seaweedfsadmin"
+ENV S3_SECRET_ACCESS_KEY="seaweedfsadmin"
 
 COPY ./as-yml ./as-yml
 COPY ./clients ./clients

dbrepo-analyse-service/clients/minio_client.py → dbrepo-analyse-service/clients/s3_client.py

@@ -5,12 +5,12 @@ import logging
 from botocore.exceptions import ClientError
 
 
-class MinioClient:
+class S3Client:
 
     def __init__(self):
         endpoint_url = os.getenv('S3_STORAGE_ENDPOINT', 'http://localhost:9000')
-        aws_access_key_id = os.getenv('S3_ACCESS_KEY_ID', 'minioadmin')
-        aws_secret_access_key = os.getenv('S3_SECRET_ACCESS_KEY', 'minioadmin')
+        aws_access_key_id = os.getenv('S3_ACCESS_KEY_ID', 'seaweedfsadmin')
+        aws_secret_access_key = os.getenv('S3_SECRET_ACCESS_KEY', 'seaweedfsadmin')
         logging.info("retrieve file from S3, endpoint_url=%s, aws_access_key_id=%s, aws_secret_access_key=(hidden)",
                      endpoint_url, aws_access_key_id)
         self.client = boto3.client(service_name='s3', endpoint_url=endpoint_url, aws_access_key_id=aws_access_key_id,

dbrepo-analyse-service/determine_dt.py

@@ -6,7 +6,7 @@ import json
 import csv
 import logging
 import io
-from clients.minio_client import MinioClient
+from clients.s3_client import S3Client
 
 import messytables, pandas as pd
 from messytables import CSVTableSet, type_guess, \
@@ -17,9 +17,9 @@ def determine_datatypes(filename, enum=False, enum_tol=0.0001, separator=None) -
     # Use option enum=True for searching Postgres ENUM Types in CSV file. Remark
     # Enum is not SQL standard, hence, it might not be supported by all db-engines.
     # However, it can be used in Postgres and MySQL.
-    minio_client = MinioClient()
-    minio_client.file_exists('dbrepo-upload', filename)
-    response = minio_client.get_file('dbrepo-upload', filename)
+    s3_client = S3Client()
+    s3_client.file_exists('dbrepo-upload', filename)
+    response = s3_client.get_file('dbrepo-upload', filename)
     stream = response['Body']
     if response['ContentLength'] == 0:
         logging.warning(f'Failed to determine data types: file {filename} has empty body')

dbrepo-analyse-service/determine_pk.py

@@ -5,7 +5,7 @@ import random
 import numpy as np
 import math
 from determine_dt import determine_datatypes
-from clients.minio_client import MinioClient
+from clients.s3_client import S3Client
 
 
 def determine_pk(filename, separator=','):
@@ -15,9 +15,9 @@ def determine_pk(filename, separator=','):
     colnames = dt.keys()
     colindex = list(range(0, len(colnames)))
 
-    minio_client = MinioClient()
-    minio_client.file_exists('dbrepo-upload', filename)
-    response = minio_client.get_file('dbrepo-upload', filename)
+    s3_client = S3Client()
+    s3_client.file_exists('dbrepo-upload', filename)
+    response = s3_client.get_file('dbrepo-upload', filename)
     stream = response['Body']
     if response['ContentLength'] == 0:
         logging.warning(f'Failed to determine primary key: file {filename} has empty body')

dbrepo-analyse-service/test/test_minio_client.py → dbrepo-analyse-service/test/test_s3_client.py

@@ -9,16 +9,16 @@ import unittest
 
 from botocore.exceptions import ClientError
 
-from clients.minio_client import MinioClient
+from clients.s3_client import S3Client
 
 
-class MinioClientTest(unittest.TestCase):
+class S3ClientTest(unittest.TestCase):
 
     # @Test
     def test_upload_file_succeeds(self):
 
         # test
-        response = MinioClient().upload_file(filename="testdt01.csv", path="./data/")
+        response = S3Client().upload_file(filename="testdt01.csv", path="./data/")
         self.assertTrue(response)
 
     # @Test
@@ -26,7 +26,7 @@ class MinioClientTest(unittest.TestCase):
 
         # test
         try:
-            MinioClient().upload_file(filename="testdt06.csv", path="./data/")
+            S3Client().upload_file(filename="testdt06.csv", path="./data/")
         except FileNotFoundError:
             pass
         except Exception:
@@ -38,10 +38,10 @@ class MinioClientTest(unittest.TestCase):
     def test_download_file_succeeds(self):
 
         # mock
-        MinioClient().upload_file(filename="testdt01.csv", path="./data/", bucket="dbrepo-upload")
+        S3Client().upload_file(filename="testdt01.csv", path="./data/", bucket="dbrepo-upload")
 
         # test
-        response = MinioClient().download_file(filename="testdt01.csv")
+        response = S3Client().download_file(filename="testdt01.csv")
         self.assertTrue(response)
 
     # @Test
@@ -49,7 +49,7 @@ class MinioClientTest(unittest.TestCase):
 
         # test
         try:
-            MinioClient().download_file(filename="testdt01.csv")
+            S3Client().download_file(filename="testdt01.csv")
         except ClientError:
             pass
         except Exception:
@@ -61,10 +61,10 @@ class MinioClientTest(unittest.TestCase):
     def test_get_file_succeeds(self):
 
         # mock
-        MinioClient().upload_file(filename="testdt01.csv", path="./data/", bucket="dbrepo-upload")
+        S3Client().upload_file(filename="testdt01.csv", path="./data/", bucket="dbrepo-upload")
 
         # test
-        response = MinioClient().get_file(bucket="dbrepo-upload", filename="testdt01.csv")
+        response = S3Client().get_file(bucket="dbrepo-upload", filename="testdt01.csv")
         self.assertIsNotNone(response)
 
     # @Test
@@ -72,7 +72,7 @@ class MinioClientTest(unittest.TestCase):
 
         # test
         try:
-            MinioClient().get_file(bucket="dbrepo-upload", filename="idonotexist.csv")
+            S3Client().get_file(bucket="dbrepo-upload", filename="idonotexist.csv")
         except ClientError:
             pass
         except Exception:
@@ -84,7 +84,7 @@ class MinioClientTest(unittest.TestCase):
     def test_bucket_exists_succeeds(self):
 
         # test
-        response = MinioClient().bucket_exists_or_exit("dbrepo-upload")
+        response = S3Client().bucket_exists_or_exit("dbrepo-upload")
         self.assertIsNotNone(response)
 
     # @Test
@@ -92,7 +92,7 @@ class MinioClientTest(unittest.TestCase):
 
         # test
         try:
-            MinioClient().bucket_exists_or_exit("idnonotexist")
+            S3Client().bucket_exists_or_exit("idnonotexist")
         except FileNotFoundError:
             pass
         except Exception:
@@ -105,7 +105,7 @@ class MinioClientTest(unittest.TestCase):
 
         # test
         try:
-            MinioClient().bucket_exists_or_exit("idnonotexist")
+            S3Client().bucket_exists_or_exit("idnonotexist")
         except FileNotFoundError:
             pass
         except Exception:

dbrepo-metadata-service/Dockerfile

@@ -75,8 +75,8 @@ ENV DATACITE_PREFIX=""
 ENV DATACITE_USERNAME=""
 ENV DATACITE_PASSWORD=""
 ENV S3_STORAGE_ENDPOINT="http://storage-service:9000"
-ENV S3_ACCESS_KEY_ID="minioadmin"
-ENV S3_SECRET_ACCESS_KEY="minioadmin"
+ENV S3_ACCESS_KEY_ID="seaweedfsadmin"
+ENV S3_SECRET_ACCESS_KEY="seaweedfsadmin"
 
 WORKDIR /app
 

docker-compose.prod.yml

@@ -39,7 +39,7 @@ services:
     restart: "no"
     container_name: dbrepo-data-db
     hostname: data-db
-    image: docker.io/bitnami/mariadb:11.1.3
+    image: docker.io/bitnami/mariadb-galera:11.1.3-debian-11-r0
     volumes:
       - data-db-data:/bitnami/mariadb
       - "${SHARED_FILESYSTEM:-/tmp}:/tmp"
@@ -47,6 +47,7 @@ services:
       - "3307:3306"
     environment:
       MARIADB_ROOT_PASSWORD: "${USER_DB_PASSWORD:-dbrepo}"
+      MARIADB_GALERA_MARIABACKUP_PASSWORD: "${USER_DB_BACKUP_PASSWORD:-dbrepo}"
     healthcheck:
       test: mysqladmin ping --user="${USER_DB_USERNAME:-root}" --password="${USER_DB_PASSWORD:-dbrepo}" --silent
       interval: 10s
@@ -178,8 +179,8 @@ services:
       - "5000:5000"
     environment:
       S3_STORAGE_ENDPOINT: "${STORAGE_ENDPOINT:-http://storage-service:9000}"
-      S3_ACCESS_KEY_ID: "${STORAGE_USERNAME:-minioadmin}"
-      S3_SECRET_ACCESS_KEY: "${STORAGE_PASSWORD:-minioadmin}"
+      S3_ACCESS_KEY_ID: "${STORAGE_USERNAME:-seaweedfsadmin}"
+      S3_SECRET_ACCESS_KEY: "${STORAGE_PASSWORD:-seaweedfsadmin}"
     volumes:
       - "${SHARED_FILESYSTEM:-/tmp}:/tmp"
     healthcheck:
@@ -258,8 +259,8 @@ services:
     environment:
       FLASK_DEBUG: ${SEARCH_DEBUG_MODE:-true}
       S3_STORAGE_ENDPOINT: "${STORAGE_ENDPOINT:-http://storage-service:9000}"
-      S3_ACCESS_KEY_ID: "${STORAGE_USERNAME:-minioadmin}"
-      S3_SECRET_ACCESS_KEY: ${STORAGE_PASSWORD:-minioadmin}
+      S3_ACCESS_KEY_ID: "${STORAGE_USERNAME:-seaweedfsadmin}"
+      S3_SECRET_ACCESS_KEY: ${STORAGE_PASSWORD:-seaweedfsadmin}
     volumes:
       - "${SHARED_FILESYSTEM:-/tmp}:/tmp"
     healthcheck:
@@ -342,22 +343,18 @@ services:
     restart: "no"
     container_name: dbrepo-storage-service
     hostname: storage-service
-    image: docker.io/bitnami/minio:2023-debian-11
-    environment:
-      MINIO_ROOT_USER: "${STORAGE_USERNAME:-minioadmin}"
-      MINIO_ROOT_PASSWORD: "${STORAGE_PASSWORD:-minioadmin}"
-      MINIO_DEFAULT_BUCKETS: "${STORAGE_DBREPO_BUCKET:-dbrepo-upload:upload,dbrepo-download:download}"
-      MINIO_REGION_NAME: "${STORAGE_REGION_NAME:-eu-west-1}"
-      MINIO_BROWSER_REDIRECT_URL: "${STORAGE_BASE_URL:-http://localhost/admin/storage/}"
+    image: docker.io/dbrepo/storage-service:latest
+    build: ./dbrepo-storage-service
     ports:
       - 9000:9000
+    volumes:
+      - ./dist/s3_config.json:/app/s3_config.json
+      - storage-service-data:/data
     healthcheck:
-      test: [ "CMD", "mc", "ready", "local" ]
-      interval: 5s
+      test: curl -sSL 127.0.0.1:9000 || exit 1
+      interval: 10s
       timeout: 5s
-      retries: 5
-    volumes:
-      - storage-service-data:/bitnami/minio/data
+      retries: 12
     logging:
       driver: json-file
 
@@ -373,9 +370,9 @@ services:
       - "-s3-endpoint=${STORAGE_ENDPOINT:-http://storage-service:9000}"
       - "-s3-bucket=dbrepo-upload"
     environment:
-      AWS_ACCESS_KEY_ID: "${STORAGE_USERNAME:-minioadmin}"
-      AWS_SECRET_ACCESS_KEY: "${STORAGE_PASSWORD:-minioadmin}"
-      AWS_REGION: "${STORAGE_REGION_NAME:-eu-west-1}"
+      AWS_ACCESS_KEY_ID: "${STORAGE_USERNAME:-seaweedfsadmin}"
+      AWS_SECRET_ACCESS_KEY: "${STORAGE_PASSWORD:-seaweedfsadmin}"
+      AWS_REGION: "${STORAGE_REGION_NAME:-default}"
     depends_on:
       dbrepo-storage-service:
         condition: service_healthy

install.sh

@@ -23,6 +23,7 @@ curl -sSL -o ./dist/definitions.json https://gitlab.phaidra.org/fair-data-austri
 curl -sSL -o ./dist/dbrepo.conf https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/dbrepo-gateway-service/dbrepo.conf
 curl -sSL -o ./dist/opensearch_dashboards.yml https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/dbrepo-search-db/opensearch_dashboards.yml
 curl -sSL -o ./dist/dbrepo.config.json https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/dbrepo-ui/dbrepo.config.json
+curl -sSL -o ./dist/s3_config.json https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/dbrepo-storage-service/s3_config.json
 
 echo "[📦] Pulling images ..."
 docker compose pull