Import works so far

dbrepo-data-db/sidecar/Dockerfile

@@ -22,6 +22,6 @@ ENV S3_SECRET_ACCESS_KEY="minioadmin"
 
 RUN ls -la ./clients
 
-EXPOSE 5000
+EXPOSE 3305
 
-ENTRYPOINT [ "gunicorn", "-w", "4", "-b", ":5000", "app:app" ]
+ENTRYPOINT [ "gunicorn", "-w", "4", "-b", ":3305", "app:app" ]

dbrepo-data-db/sidecar/README.md

@@ -4,6 +4,6 @@ Sidecar that downloads the .csv from the Upload Service to deposit on the same p
 
 ## Endpoints
 
-* Prometheus metrics [`/metrics`](http://localhost:8080/metrics)
-* Health check [`/health`](http://localhost:8080/health)
-* Swagger API [`/swagger-ui/`](http://localhost:8080/swagger-ui/)
\ No newline at end of file
+* Prometheus metrics [`/metrics`](http://localhost:3305/metrics)
+* Health check [`/health`](http://localhost:3305/health)
+* Swagger API [`/swagger-ui/`](http://localhost:3305/swagger-ui/)
\ No newline at end of file

dbrepo-metadata-db/setup-schema.sql

@@ -52,6 +52,8 @@ CREATE TABLE IF NOT EXISTS `mdb_containers`
     name                character varying(255) NOT NULL,
     host                character varying(255) NOT NULL,
     port                integer                NOT NULL,
+    sidecar_host        character varying(255) NOT NULL,
+    sidecar_port        integer                NOT NULL,
     image_id            bigint                 NOT NULL,
     created             timestamp              NOT NULL DEFAULT NOW(),
     last_modified       timestamp,

dbrepo-metadata-db/setup-schema_local.sql

 BEGIN;
 
-INSERT INTO `mdb_containers` (name, internal_name, image_id, host, port, privileged_username, privileged_password)
-VALUES ('MariaDB 10.5', 'mariadb_10_5', 1, 'data-db', 3306, 'root', 'dbrepo');
+INSERT INTO `mdb_containers` (name, internal_name, image_id, host, port, sidecar_host, sidecar_port,
+                              privileged_username, privileged_password)
+VALUES ('MariaDB 10.5', 'mariadb_10_5', 1, 'data-db', 3306, 'data-db-sidecar', 3305, 'root', 'dbrepo');
 
 COMMIT;

dbrepo-metadata-service/Dockerfile

@@ -58,8 +58,6 @@ ENV PID_BASE="http://localhost/pid/"
 ENV REPOSITORY_NAME="Example Repository"
 ENV SEARCH_USERNAME=admin
 ENV SEARCH_PASSWORD=admin
-ENV SHARED_FILESYSTEM=/tmp
-ENV DELETE_AFTER_IMPORT=true
 ENV USER_NETWORK=userdb
 ENV WEBSITE="http://localhost"
 ENV KEYCLOAK_HOST="http://authentication-service:8080"
@@ -76,6 +74,9 @@ ENV DATACITE_URL="https://api.test.datacite.org"
 ENV DATACITE_PREFIX=""
 ENV DATACITE_USERNAME=""
 ENV DATACITE_PASSWORD=""
+ENV S3_STORAGE_ENDPOINT="http://storage-service:9000"
+ENV S3_ACCESS_KEY_ID="minioadmin"
+ENV S3_SECRET_ACCESS_KEY="minioadmin"
 
 WORKDIR /app
 

dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/ContainerDto.java

@@ -40,6 +40,14 @@ public class ContainerDto {
 
     private Integer port;
 
+    @NotBlank
+    @JsonProperty("sidecar_host")
+    private String sidecarHost;
+
+    @NotNull
+    @JsonProperty("sidecar_port")
+    private Integer sidecarPort;
+
     private ImageBriefDto image;
 
     @NotNull

dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/query/ImportDto.java

@@ -20,7 +20,7 @@ import lombok.extern.jackson.Jacksonized;
 public class ImportDto {
 
     @NotBlank(message = "location is required")
-    @Schema(example = "/tmp/file.csv")
+    @Schema(example = "file.csv")
     private String location;
 
     @Min(value = 0L)

dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/Container.java

@@ -45,6 +45,12 @@ public class Container {
     @Column
     private Integer port;
 
+    @Column(nullable = false)
+    private String sidecarHost;
+
+    @Column(nullable = false)
+    private Integer sidecarPort;
+
     @ToString.Exclude
     @org.springframework.data.annotation.Transient
     @OneToMany(fetch = FetchType.LAZY, cascade = CascadeType.ALL)

dbrepo-metadata-service/pom.xml

@@ -69,6 +69,7 @@
         <opensearch-client.version>1.1.0</opensearch-client.version>
         <opensearch-rest-client.version>2.8.0</opensearch-rest-client.version>
         <jackson.version>2.15.2</jackson.version>
+        <minio.version>8.5.6</minio.version>
     </properties>
 
     <dependencies>
@@ -239,6 +240,12 @@
             <artifactId>springdoc-openapi-starter-webmvc-api</artifactId>
             <version>${springdoc-openapi.version}</version>
         </dependency>
+        <!-- blob storage -->
+        <dependency>
+            <groupId>io.minio</groupId>
+            <artifactId>minio</artifactId>
+            <version>${minio.version}</version>
+        </dependency>
         <!-- Testing -->
         <dependency>
             <groupId>org.springframework.boot</groupId>

dbrepo-metadata-service/repositories/src/main/java/at/tuwien/exception/DataDbSidecarException.java

+package at.tuwien.exception;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+import java.io.IOException;
+
+@ResponseStatus(code = HttpStatus.UNPROCESSABLE_ENTITY)
+public class DataDbSidecarException extends IOException {
+
+    public DataDbSidecarException(String msg) {
+        super(msg);
+    }
+
+    public DataDbSidecarException(String msg, Throwable thr) {
+        super(msg, thr);
+    }
+
+    public DataDbSidecarException(Throwable thr) {
+        super(thr);
+    }
+
+}

dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/QueryMapper.java

@@ -157,7 +157,7 @@ public interface QueryMapper {
     }
 
     default PreparedStatement pathToRawInsertQuery(Connection connection, Table table, ImportDto data) throws QueryMalformedException {
-        final StringBuilder statement = new StringBuilder("LOAD DATA LOCAL INFILE '")
+        final StringBuilder statement = new StringBuilder("LOAD DATA LOCAL INFILE '/tmp/")
                 .append(data.getLocation())
                 .append("' INTO TABLE `")
                 .append(table.getDatabase().getInternalName())

dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/ExportEndpoint.java

 package at.tuwien.endpoints;
 
 import at.tuwien.ExportResource;
+import at.tuwien.api.error.ApiErrorDto;
+import at.tuwien.api.identifier.IdentifierDto;
 import at.tuwien.entities.database.Database;
 import at.tuwien.exception.*;
 import at.tuwien.service.DatabaseService;
@@ -9,6 +11,10 @@ import at.tuwien.utils.PrincipalUtil;
 import at.tuwien.utils.UserUtil;
 import io.micrometer.core.annotation.Timed;
 import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
 import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import jakarta.validation.constraints.NotNull;
 import lombok.extern.log4j.Log4j2;
@@ -41,13 +47,50 @@ public class ExportEndpoint {
     @Transactional(readOnly = true)
     @Timed(value = "table.export", description = "Time needed to export table data")
     @Operation(summary = "Export table", security = @SecurityRequirement(name = "bearerAuth"))
+    @ApiResponses(value = {
+            @ApiResponse(responseCode = "201",
+                    description = "Created identifier",
+                    content = {@Content(
+                            mediaType = "application/json",
+                            schema = @Schema(implementation = IdentifierDto.class))}),
+            @ApiResponse(responseCode = "400",
+                    description = "Images is not supported or table/query is malformed",
+                    content = {@Content(
+                            mediaType = "application/json",
+                            schema = @Schema(implementation = ApiErrorDto.class))}),
+            @ApiResponse(responseCode = "403",
+                    description = "Operation is not allowed",
+                    content = {@Content(
+                            mediaType = "application/json",
+                            schema = @Schema(implementation = ApiErrorDto.class))}),
+            @ApiResponse(responseCode = "404",
+                    description = "Table, database or user was not found",
+                    content = {@Content(
+                            mediaType = "application/json",
+                            schema = @Schema(implementation = ApiErrorDto.class))}),
+            @ApiResponse(responseCode = "410",
+                    description = "Blob storage operation could not be completed",
+                    content = {@Content(
+                            mediaType = "application/json",
+                            schema = @Schema(implementation = ApiErrorDto.class))}),
+            @ApiResponse(responseCode = "422",
+                    description = "Sidecar operation could not be completed",
+                    content = {@Content(
+                            mediaType = "application/json",
+                            schema = @Schema(implementation = ApiErrorDto.class))}),
+            @ApiResponse(responseCode = "503",
+                    description = "Database connection could not be established",
+                    content = {@Content(
+                            mediaType = "application/json",
+                            schema = @Schema(implementation = ApiErrorDto.class))}),
+    })
     public ResponseEntity<InputStreamResource> export(@NotNull @PathVariable("id") Long databaseId,
                                                       @NotNull @PathVariable("tableId") Long tableId,
                                                       @RequestParam(required = false) Instant timestamp,
                                                       Principal principal)
             throws TableNotFoundException, DatabaseConnectionException, TableMalformedException,
             DatabaseNotFoundException, ImageNotSupportedException, PaginationException, FileStorageException,
-            QueryMalformedException, UserNotFoundException, NotAllowedException {
+            QueryMalformedException, UserNotFoundException, NotAllowedException, DataDbSidecarException {
         log.debug("endpoint export table, id={}, tableId={}, timestamp={}, {}", databaseId, tableId, timestamp, PrincipalUtil.formatForDebug(principal));
         final Database database = databaseService.find(databaseId);
         if (!database.getIsPublic()) {

dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableDataEndpoint.java

@@ -121,7 +121,7 @@ public class TableDataEndpoint {
                                           @NotNull Principal principal)
             throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException,
             ImageNotSupportedException, DatabaseConnectionException, QueryMalformedException, UserNotFoundException,
-            NotAllowedException, AccessDeniedException {
+            NotAllowedException, AccessDeniedException, DataDbSidecarException {
         log.debug("endpoint insert data from csv, databaseId={}, tableId={}, data={}, {}", databaseId, tableId, data, PrincipalUtil.formatForDebug(principal));
         /* check */
         endpointValidator.validateOnlyWriteOwnOrWriteAllAccess(databaseId, tableId, principal);

dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml

@@ -53,6 +53,10 @@ fda:
     base: https://example.com/pid/
   broker:
     endpoint: http://localhost:15672
+  minio:
+    endpoint: http://localhost:9000
+    accessKeyId: minioadmin
+    secretAccessKey: minioadmin
   jwt:
     issuer: http://localhost/realms/dbrepo
     public_key: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB
@@ -69,8 +73,6 @@ fda:
   exchangeName: "dbrepo"
   routingKey: "dbrepo.#"
   connectionTimeout: 60000
-  sharedFilesystem: /tmp
-  deleteAfterImport: true
 dbrepo:
   repository-name: TU Wien Database Repository
   base-url: https://dbrepo1.ec.tuwien.at/api/oai

dbrepo-metadata-service/rest-service/src/main/resources/application.yml

@@ -66,6 +66,10 @@ fda:
     base: "${PID_BASE}"
   broker:
     endpoint: "${BROKER_ENDPOINT}"
+  minio:
+    endpoint: "${S3_STORAGE_ENDPOINT}"
+    accessKeyId: "${S3_ACCESS_KEY_ID}"
+    secretAccessKey: "${S3_SECRET_ACCESS_KEY}"
   jwt:
     issuer: "${JWT_ISSUER}"
     public_key: "${JWT_PUBKEY}"
@@ -82,8 +86,6 @@ fda:
   exchangeName: "${EXCHANGE_NAME}"
   routingKey: "${ROUTING_KEY}"
   connectionTimeout: ${CONNECTION_TIMEOUT}
-  sharedFilesystem: ${SHARED_FILESYSTEM}
-  deleteAfterImport: ${DELETE_AFTER_IMPORT}
 dbrepo:
   repository-name: "${REPOSITORY_NAME}"
   base-url: "${BASE_URL}"

dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableDataEndpointUnitTest.java

@@ -138,7 +138,7 @@ public class TableDataEndpointUnitTest extends BaseUnitTest {
     @WithMockUser(username = USER_1_USERNAME, authorities = {"insert-table-data"})
     public void import_publicWriteAll_succeeds() throws UserNotFoundException, TableNotFoundException, NotAllowedException,
             TableMalformedException, DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException,
-            ImageNotSupportedException, AccessDeniedException {
+            ImageNotSupportedException, AccessDeniedException, DataDbSidecarException {
 
         /* test */
         generic_import(DATABASE_3_ID, DATABASE_3, TABLE_8_ID, TABLE_8, USER_1_ID,
@@ -149,7 +149,7 @@ public class TableDataEndpointUnitTest extends BaseUnitTest {
     @WithMockUser(username = USER_1_USERNAME, authorities = {"insert-table-data"})
     public void import_privateWriteAll_succeeds() throws UserNotFoundException, TableNotFoundException, NotAllowedException,
             TableMalformedException, DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException,
-            ImageNotSupportedException, AccessDeniedException {
+            ImageNotSupportedException, AccessDeniedException, DataDbSidecarException {
 
         /* test */
         generic_import(DATABASE_1_ID, DATABASE_1, TABLE_1_ID, TABLE_1, USER_1_ID,
@@ -426,7 +426,7 @@ public class TableDataEndpointUnitTest extends BaseUnitTest {
                                DatabaseAccess access, Principal principal) throws DatabaseNotFoundException,
             TableNotFoundException, NotAllowedException, UserNotFoundException, TableMalformedException,
             DatabaseConnectionException, QueryMalformedException, ImageNotSupportedException,
-            AccessDeniedException {
+            AccessDeniedException, DataDbSidecarException {
         final ImportDto request = ImportDto.builder().location("test:csv/csv_01.csv").build();
 
         /* mock */

dbrepo-metadata-service/services/src/main/java/at/tuwien/config/GatewayConfig.java

@@ -36,4 +36,12 @@ public class GatewayConfig {
         return restTemplate;
     }
 
+    @Bean("sidecarRestTemplate")
+    public RestTemplate sidecarRestTemplate() {
+        final RestTemplate restTemplate = new RestTemplate();
+        restTemplate.getInterceptors()
+                .add(new BasicAuthenticationInterceptor(brokerUsername, brokerPassword));
+        return restTemplate;
+    }
+
 }

dbrepo-metadata-service/services/src/main/java/at/tuwien/config/MinioConfig.java

+package at.tuwien.config;
+
+import io.minio.MinioClient;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Slf4j
+@Configuration
+public class MinioConfig {
+
+    @Value("${fda.minio.endpoint}")
+    private String minioEndpoint;
+
+    @Value("${fda.minio.accessKeyId}")
+    private String minioAccessKeyId;
+
+    @Value("${fda.minio.secretAccessKey}")
+    private String minioSecretAccessKey;
+
+    @Bean
+    public MinioClient minioClient() {
+        return MinioClient.builder()
+                .endpoint(minioEndpoint)
+                .credentials(minioAccessKeyId, minioSecretAccessKey)
+                .build();
+    }
+
+}

dbrepo-metadata-service/services/src/main/java/at/tuwien/config/QueryConfig.java

@@ -14,10 +14,4 @@ public class QueryConfig {
     @Value("${fda.unsupported}")
     private String[] notSupportedKeywords;
 
-    @Value("${fda.sharedFilesystem}")
-    private String sharedFilesystem;
-
-    @Value("${fda.deleteAfterImport}")
-    private Boolean deleteAfterImport;
-
 }

dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/DataDbSidecarGateway.java

+package at.tuwien.gateway;
+
+import at.tuwien.exception.DataDbSidecarException;
+
+public interface DataDbSidecarGateway {
+    void importFile(String hostname, Integer port, String filename) throws DataDbSidecarException;
+
+    void exportFile(String hostname, Integer port, String filename) throws DataDbSidecarException;
+}