Skip to content
Snippets Groups Projects
Unverified Commit e2d816c5 authored by Martin Weise's avatar Martin Weise
Browse files

Remove remaining traces of minIO

parent 072239ba
No related branches found
No related tags found
4 merge requests!231CI: Remove build for log-service,!228Better error message handling in the frontend,!223Release of version 1.4.0,!218Replacing minIO with SeaweedFS
......@@ -15,8 +15,8 @@ ENV FLASK_ENV=production
ENV HOSTNAME=analyse-service
ENV LOG_LEVEL=INFO
ENV S3_STORAGE_ENDPOINT="http://storage-service:9000"
ENV S3_ACCESS_KEY_ID="minioadmin"
ENV S3_SECRET_ACCESS_KEY="minioadmin"
ENV S3_ACCESS_KEY_ID="seaweedfsadmin"
ENV S3_SECRET_ACCESS_KEY="seaweedfsadmin"
COPY ./as-yml ./as-yml
COPY ./clients ./clients
......
......@@ -5,12 +5,12 @@ import logging
from botocore.exceptions import ClientError
class MinioClient:
class S3Client:
def __init__(self):
endpoint_url = os.getenv('S3_STORAGE_ENDPOINT', 'http://localhost:9000')
aws_access_key_id = os.getenv('S3_ACCESS_KEY_ID', 'minioadmin')
aws_secret_access_key = os.getenv('S3_SECRET_ACCESS_KEY', 'minioadmin')
aws_access_key_id = os.getenv('S3_ACCESS_KEY_ID', 'seaweedfsadmin')
aws_secret_access_key = os.getenv('S3_SECRET_ACCESS_KEY', 'seaweedfsadmin')
logging.info("retrieve file from S3, endpoint_url=%s, aws_access_key_id=%s, aws_secret_access_key=(hidden)",
endpoint_url, aws_access_key_id)
self.client = boto3.client(service_name='s3', endpoint_url=endpoint_url, aws_access_key_id=aws_access_key_id,
......
......@@ -6,7 +6,7 @@ import json
import csv
import logging
import io
from clients.minio_client import MinioClient
from clients.s3_client import S3Client
import messytables, pandas as pd
from messytables import CSVTableSet, type_guess, \
......@@ -17,9 +17,9 @@ def determine_datatypes(filename, enum=False, enum_tol=0.0001, separator=None) -
# Use option enum=True for searching Postgres ENUM Types in CSV file. Remark
# Enum is not SQL standard, hence, it might not be supported by all db-engines.
# However, it can be used in Postgres and MySQL.
minio_client = MinioClient()
minio_client.file_exists('dbrepo-upload', filename)
response = minio_client.get_file('dbrepo-upload', filename)
s3_client = S3Client()
s3_client.file_exists('dbrepo-upload', filename)
response = s3_client.get_file('dbrepo-upload', filename)
stream = response['Body']
if response['ContentLength'] == 0:
logging.warning(f'Failed to determine data types: file {filename} has empty body')
......
......@@ -5,7 +5,7 @@ import random
import numpy as np
import math
from determine_dt import determine_datatypes
from clients.minio_client import MinioClient
from clients.s3_client import S3Client
def determine_pk(filename, separator=','):
......@@ -15,9 +15,9 @@ def determine_pk(filename, separator=','):
colnames = dt.keys()
colindex = list(range(0, len(colnames)))
minio_client = MinioClient()
minio_client.file_exists('dbrepo-upload', filename)
response = minio_client.get_file('dbrepo-upload', filename)
s3_client = S3Client()
s3_client.file_exists('dbrepo-upload', filename)
response = s3_client.get_file('dbrepo-upload', filename)
stream = response['Body']
if response['ContentLength'] == 0:
logging.warning(f'Failed to determine primary key: file {filename} has empty body')
......
......@@ -9,16 +9,16 @@ import unittest
from botocore.exceptions import ClientError
from clients.minio_client import MinioClient
from clients.s3_client import S3Client
class MinioClientTest(unittest.TestCase):
class S3ClientTest(unittest.TestCase):
# @Test
def test_upload_file_succeeds(self):
# test
response = MinioClient().upload_file(filename="testdt01.csv", path="./data/")
response = S3Client().upload_file(filename="testdt01.csv", path="./data/")
self.assertTrue(response)
# @Test
......@@ -26,7 +26,7 @@ class MinioClientTest(unittest.TestCase):
# test
try:
MinioClient().upload_file(filename="testdt06.csv", path="./data/")
S3Client().upload_file(filename="testdt06.csv", path="./data/")
except FileNotFoundError:
pass
except Exception:
......@@ -38,10 +38,10 @@ class MinioClientTest(unittest.TestCase):
def test_download_file_succeeds(self):
# mock
MinioClient().upload_file(filename="testdt01.csv", path="./data/", bucket="dbrepo-upload")
S3Client().upload_file(filename="testdt01.csv", path="./data/", bucket="dbrepo-upload")
# test
response = MinioClient().download_file(filename="testdt01.csv")
response = S3Client().download_file(filename="testdt01.csv")
self.assertTrue(response)
# @Test
......@@ -49,7 +49,7 @@ class MinioClientTest(unittest.TestCase):
# test
try:
MinioClient().download_file(filename="testdt01.csv")
S3Client().download_file(filename="testdt01.csv")
except ClientError:
pass
except Exception:
......@@ -61,10 +61,10 @@ class MinioClientTest(unittest.TestCase):
def test_get_file_succeeds(self):
# mock
MinioClient().upload_file(filename="testdt01.csv", path="./data/", bucket="dbrepo-upload")
S3Client().upload_file(filename="testdt01.csv", path="./data/", bucket="dbrepo-upload")
# test
response = MinioClient().get_file(bucket="dbrepo-upload", filename="testdt01.csv")
response = S3Client().get_file(bucket="dbrepo-upload", filename="testdt01.csv")
self.assertIsNotNone(response)
# @Test
......@@ -72,7 +72,7 @@ class MinioClientTest(unittest.TestCase):
# test
try:
MinioClient().get_file(bucket="dbrepo-upload", filename="idonotexist.csv")
S3Client().get_file(bucket="dbrepo-upload", filename="idonotexist.csv")
except ClientError:
pass
except Exception:
......@@ -84,7 +84,7 @@ class MinioClientTest(unittest.TestCase):
def test_bucket_exists_succeeds(self):
# test
response = MinioClient().bucket_exists_or_exit("dbrepo-upload")
response = S3Client().bucket_exists_or_exit("dbrepo-upload")
self.assertIsNotNone(response)
# @Test
......@@ -92,7 +92,7 @@ class MinioClientTest(unittest.TestCase):
# test
try:
MinioClient().bucket_exists_or_exit("idnonotexist")
S3Client().bucket_exists_or_exit("idnonotexist")
except FileNotFoundError:
pass
except Exception:
......@@ -105,7 +105,7 @@ class MinioClientTest(unittest.TestCase):
# test
try:
MinioClient().bucket_exists_or_exit("idnonotexist")
S3Client().bucket_exists_or_exit("idnonotexist")
except FileNotFoundError:
pass
except Exception:
......
......@@ -75,8 +75,8 @@ ENV DATACITE_PREFIX=""
ENV DATACITE_USERNAME=""
ENV DATACITE_PASSWORD=""
ENV S3_STORAGE_ENDPOINT="http://storage-service:9000"
ENV S3_ACCESS_KEY_ID="minioadmin"
ENV S3_SECRET_ACCESS_KEY="minioadmin"
ENV S3_ACCESS_KEY_ID="seaweedfsadmin"
ENV S3_SECRET_ACCESS_KEY="seaweedfsadmin"
WORKDIR /app
......
......@@ -39,7 +39,7 @@ services:
restart: "no"
container_name: dbrepo-data-db
hostname: data-db
image: docker.io/bitnami/mariadb:11.1.3
image: docker.io/bitnami/mariadb-galera:11.1.3-debian-11-r0
volumes:
- data-db-data:/bitnami/mariadb
- "${SHARED_FILESYSTEM:-/tmp}:/tmp"
......@@ -47,6 +47,7 @@ services:
- "3307:3306"
environment:
MARIADB_ROOT_PASSWORD: "${USER_DB_PASSWORD:-dbrepo}"
MARIADB_GALERA_MARIABACKUP_PASSWORD: "${USER_DB_BACKUP_PASSWORD:-dbrepo}"
healthcheck:
test: mysqladmin ping --user="${USER_DB_USERNAME:-root}" --password="${USER_DB_PASSWORD:-dbrepo}" --silent
interval: 10s
......@@ -178,8 +179,8 @@ services:
- "5000:5000"
environment:
S3_STORAGE_ENDPOINT: "${STORAGE_ENDPOINT:-http://storage-service:9000}"
S3_ACCESS_KEY_ID: "${STORAGE_USERNAME:-minioadmin}"
S3_SECRET_ACCESS_KEY: "${STORAGE_PASSWORD:-minioadmin}"
S3_ACCESS_KEY_ID: "${STORAGE_USERNAME:-seaweedfsadmin}"
S3_SECRET_ACCESS_KEY: "${STORAGE_PASSWORD:-seaweedfsadmin}"
volumes:
- "${SHARED_FILESYSTEM:-/tmp}:/tmp"
healthcheck:
......@@ -258,8 +259,8 @@ services:
environment:
FLASK_DEBUG: ${SEARCH_DEBUG_MODE:-true}
S3_STORAGE_ENDPOINT: "${STORAGE_ENDPOINT:-http://storage-service:9000}"
S3_ACCESS_KEY_ID: "${STORAGE_USERNAME:-minioadmin}"
S3_SECRET_ACCESS_KEY: ${STORAGE_PASSWORD:-minioadmin}
S3_ACCESS_KEY_ID: "${STORAGE_USERNAME:-seaweedfsadmin}"
S3_SECRET_ACCESS_KEY: ${STORAGE_PASSWORD:-seaweedfsadmin}
volumes:
- "${SHARED_FILESYSTEM:-/tmp}:/tmp"
healthcheck:
......@@ -342,22 +343,18 @@ services:
restart: "no"
container_name: dbrepo-storage-service
hostname: storage-service
image: docker.io/bitnami/minio:2023-debian-11
environment:
MINIO_ROOT_USER: "${STORAGE_USERNAME:-minioadmin}"
MINIO_ROOT_PASSWORD: "${STORAGE_PASSWORD:-minioadmin}"
MINIO_DEFAULT_BUCKETS: "${STORAGE_DBREPO_BUCKET:-dbrepo-upload:upload,dbrepo-download:download}"
MINIO_REGION_NAME: "${STORAGE_REGION_NAME:-eu-west-1}"
MINIO_BROWSER_REDIRECT_URL: "${STORAGE_BASE_URL:-http://localhost/admin/storage/}"
image: docker.io/dbrepo/storage-service:latest
build: ./dbrepo-storage-service
ports:
- 9000:9000
volumes:
- ./dist/s3_config.json:/app/s3_config.json
- storage-service-data:/data
healthcheck:
test: [ "CMD", "mc", "ready", "local" ]
interval: 5s
test: curl -sSL 127.0.0.1:9000 || exit 1
interval: 10s
timeout: 5s
retries: 5
volumes:
- storage-service-data:/bitnami/minio/data
retries: 12
logging:
driver: json-file
......@@ -373,9 +370,9 @@ services:
- "-s3-endpoint=${STORAGE_ENDPOINT:-http://storage-service:9000}"
- "-s3-bucket=dbrepo-upload"
environment:
AWS_ACCESS_KEY_ID: "${STORAGE_USERNAME:-minioadmin}"
AWS_SECRET_ACCESS_KEY: "${STORAGE_PASSWORD:-minioadmin}"
AWS_REGION: "${STORAGE_REGION_NAME:-eu-west-1}"
AWS_ACCESS_KEY_ID: "${STORAGE_USERNAME:-seaweedfsadmin}"
AWS_SECRET_ACCESS_KEY: "${STORAGE_PASSWORD:-seaweedfsadmin}"
AWS_REGION: "${STORAGE_REGION_NAME:-default}"
depends_on:
dbrepo-storage-service:
condition: service_healthy
......
......@@ -23,6 +23,7 @@ curl -sSL -o ./dist/definitions.json https://gitlab.phaidra.org/fair-data-austri
curl -sSL -o ./dist/dbrepo.conf https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/dbrepo-gateway-service/dbrepo.conf
curl -sSL -o ./dist/opensearch_dashboards.yml https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/dbrepo-search-db/opensearch_dashboards.yml
curl -sSL -o ./dist/dbrepo.config.json https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/dbrepo-ui/dbrepo.config.json
curl -sSL -o ./dist/s3_config.json https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/dev/dbrepo-storage-service/s3_config.json
echo "[📦] Pulling images ..."
docker compose pull
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment