diff --git a/dbrepo-data-db/sidecar/Dockerfile b/dbrepo-data-db/sidecar/Dockerfile index 755ee168eba7f9d1005319385f24b11055121f0a..de0f2d76f2ee5d4788083fba56e22f719a93636a 100644 --- a/dbrepo-data-db/sidecar/Dockerfile +++ b/dbrepo-data-db/sidecar/Dockerfile @@ -19,8 +19,8 @@ COPY --chown=1001 ./ds-yml ./ds-yml COPY --chown=1001 ./app.py ./app.py ENV S3_STORAGE_ENDPOINT="http://storage-service:9000" -ENV S3_ACCESS_KEY_ID="minioadmin" -ENV S3_SECRET_ACCESS_KEY="minioadmin" +ENV S3_ACCESS_KEY_ID="seaweedfsadmin" +ENV S3_SECRET_ACCESS_KEY="seaweedfsadmin" EXPOSE 3305 diff --git a/dbrepo-data-db/sidecar/app.py b/dbrepo-data-db/sidecar/app.py index 105f3f03f2a7d0c0796ce975b51fd088c6999391..1bcf3a4a0f041d70d796d2c89319b40201583e13 100644 --- a/dbrepo-data-db/sidecar/app.py +++ b/dbrepo-data-db/sidecar/app.py @@ -5,7 +5,7 @@ import logging from flasgger import LazyJSONEncoder, Swagger from flask import Flask, request, Response from flasgger.utils import swag_from -from clients.minio_client import MinioClient +from clients.s3_client import S3Client from prometheus_flask_exporter import PrometheusMetrics logging.basicConfig(level=logging.DEBUG) @@ -106,8 +106,8 @@ def health(): @swag_from("ds-yml/import.yml") def import_csv(filename): logging.debug('endpoint import csv, filename=%s, body=%s', filename, request) - minio_client = MinioClient() - response = minio_client.download_file(filename) + s3_client = S3Client() + response = s3_client.download_file(filename) if response is False: return Response(), 400 return Response(json.dumps(response)), 202 @@ -117,8 +117,8 @@ def import_csv(filename): @swag_from("ds-yml/export.yml") def import_csv(filename): logging.debug('endpoint export csv, filename=%s, body=%s', filename, request) - minio_client = MinioClient() - response = minio_client.upload_file(filename) + s3_client = S3Client() + response = s3_client.upload_file(filename) if response is False: return Response(), 400 return Response(), 202 diff --git a/dbrepo-data-db/sidecar/clients/minio_client.py b/dbrepo-data-db/sidecar/clients/s3_client.py similarity index 72% rename from dbrepo-data-db/sidecar/clients/minio_client.py rename to dbrepo-data-db/sidecar/clients/s3_client.py index 9f38a83b497d5ebd4faadcd859a44dd244f4705e..65766cc02f248f93ad5fe74070552ee93dd9d176 100644 --- a/dbrepo-data-db/sidecar/clients/minio_client.py +++ b/dbrepo-data-db/sidecar/clients/s3_client.py @@ -6,14 +6,14 @@ import sys from botocore.exceptions import ClientError -class MinioClient: +class S3Client: def __init__(self): endpoint_url = os.getenv('S3_STORAGE_ENDPOINT', 'http://localhost:9000') - aws_access_key_id = os.getenv('S3_ACCESS_KEY_ID', 'minioadmin') - aws_secret_access_key = os.getenv('S3_SECRET_ACCESS_KEY', 'minioadmin') - logging.info("retrieve file from S3, endpoint_url=%s, aws_access_key_id=%s, aws_secret_access_key=(hidden)", - endpoint_url, aws_access_key_id) + aws_access_key_id = os.getenv('S3_ACCESS_KEY_ID', 'seaweedfsadmin') + aws_secret_access_key = os.getenv('S3_SECRET_ACCESS_KEY', 'seaweedfsadmin') + logging.info( + f"retrieve file from S3, endpoint_url={endpoint_url}, aws_access_key_id={aws_access_key_id}, aws_secret_access_key=(hidden)") self.client = boto3.client(service_name='s3', endpoint_url=endpoint_url, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) self.bucket_exists_or_exit("dbrepo-upload") @@ -29,7 +29,7 @@ class MinioClient: filepath = os.path.join("/tmp/", filename) try: self.client.upload_file(filepath, "dbrepo-download", filename) - logging.info("Uploaded .csv %s with key %s", filepath, filename) + logging.info(f"Uploaded .csv {filepath} with key {filename} into bucket dbrepo-download") return True except ClientError as e: logging.error(e) @@ -46,7 +46,7 @@ class MinioClient: filepath = os.path.join("/tmp/", filename) try: self.client.download_file("dbrepo-upload", filename, filepath) - logging.info("Downloaded .csv with key %s into %s", filename, filepath) + logging.info(f"Downloaded .csv with key {filename} into {filepath} from bucket dbrepo-upload") return True except ClientError as e: logging.error(e) @@ -58,10 +58,10 @@ class MinioClient: logging.debug(f"file with name {filename} exists in bucket with name {bucket}") except ClientError as e: if e.response["Error"]["Code"] == "404": - logging.error("Failed to find key %s in bucket %s", filename, bucket) + logging.error(f"Failed to find key {filename} in bucket {bucket}") else: - logging.error("Unexpected error when finding key %s in bucket %s: %s", filename, bucket, - e.response["Error"]["Code"]) + logging.error( + f"Unexpected error when finding key {filename} in bucket {bucket}: {e.response['Error']['Code']}") raise e def bucket_exists_or_exit(self, bucket): @@ -70,8 +70,7 @@ class MinioClient: logging.debug(f"bucket {bucket} exists.") except ClientError as e: if e.response["Error"]["Code"] == "404": - logging.error("Failed to find bucket %s", bucket) + logging.error(f"Failed to find bucket {bucket}") else: - logging.error("Unexpected error when finding bucket %s: %s", bucket, - e.response["Error"]["Code"]) + logging.error(f"Unexpected error when finding bucket {bucket}: {e.response['Error']['Code']}") sys.exit(1)