diff --git a/dbrepo-data-db/sidecar/Dockerfile b/dbrepo-data-db/sidecar/Dockerfile
index eb4a1541fdebd96aacae56e436329d11f51e200e..0d307ae675c01aa543d1da0dfd6164cdb771a7f3 100644
--- a/dbrepo-data-db/sidecar/Dockerfile
+++ b/dbrepo-data-db/sidecar/Dockerfile
@@ -22,6 +22,6 @@ ENV S3_SECRET_ACCESS_KEY="minioadmin"
RUN ls -la ./clients
-EXPOSE 5000
+EXPOSE 3305
-ENTRYPOINT [ "gunicorn", "-w", "4", "-b", ":5000", "app:app" ]
+ENTRYPOINT [ "gunicorn", "-w", "4", "-b", ":3305", "app:app" ]
diff --git a/dbrepo-data-db/sidecar/README.md b/dbrepo-data-db/sidecar/README.md
index 83815a632f93ff66b7e50464ecd09b5135610746..9f6a6e2073414711e681621cf9dabc92a129454b 100644
--- a/dbrepo-data-db/sidecar/README.md
+++ b/dbrepo-data-db/sidecar/README.md
@@ -4,6 +4,6 @@ Sidecar that downloads the .csv from the Upload Service to deposit on the same p
## Endpoints
-* Prometheus metrics [`/metrics`](http://localhost:8080/metrics)
-* Health check [`/health`](http://localhost:8080/health)
-* Swagger API [`/swagger-ui/`](http://localhost:8080/swagger-ui/)
\ No newline at end of file
+* Prometheus metrics [`/metrics`](http://localhost:3305/metrics)
+* Health check [`/health`](http://localhost:3305/health)
+* Swagger API [`/swagger-ui/`](http://localhost:3305/swagger-ui/)
\ No newline at end of file
diff --git a/dbrepo-metadata-db/setup-schema.sql b/dbrepo-metadata-db/setup-schema.sql
index 0c281ba9a05a807e812a4278775ec3869a770f8b..21081998abdc10c0ea8c72282133792c4f350765 100644
--- a/dbrepo-metadata-db/setup-schema.sql
+++ b/dbrepo-metadata-db/setup-schema.sql
@@ -52,6 +52,8 @@ CREATE TABLE IF NOT EXISTS `mdb_containers`
name character varying(255) NOT NULL,
host character varying(255) NOT NULL,
port integer NOT NULL,
+ sidecar_host character varying(255) NOT NULL,
+ sidecar_port integer NOT NULL,
image_id bigint NOT NULL,
created timestamp NOT NULL DEFAULT NOW(),
last_modified timestamp,
@@ -274,22 +276,22 @@ CREATE TABLE IF NOT EXISTS `mdb_constraints_checks`
CREATE TABLE IF NOT EXISTS `mdb_concepts`
(
- id bigint NOT NULL AUTO_INCREMENT,
- uri text not null,
- name VARCHAR(255) null,
- description TEXT null,
- created timestamp NOT NULL DEFAULT NOW(),
+ id bigint NOT NULL AUTO_INCREMENT,
+ uri text not null,
+ name VARCHAR(255) null,
+ description TEXT null,
+ created timestamp NOT NULL DEFAULT NOW(),
PRIMARY KEY (id),
UNIQUE (uri(200))
) WITH SYSTEM VERSIONING;
CREATE TABLE IF NOT EXISTS `mdb_units`
(
- id bigint NOT NULL AUTO_INCREMENT,
- uri text not null,
- name VARCHAR(255) null,
- description TEXT null,
- created timestamp NOT NULL DEFAULT NOW(),
+ id bigint NOT NULL AUTO_INCREMENT,
+ uri text not null,
+ name VARCHAR(255) null,
+ description TEXT null,
+ created timestamp NOT NULL DEFAULT NOW(),
PRIMARY KEY (id),
UNIQUE (uri(200))
) WITH SYSTEM VERSIONING;
diff --git a/dbrepo-metadata-db/setup-schema_local.sql b/dbrepo-metadata-db/setup-schema_local.sql
index 2d5f2cae2e85d55cd594b8de9163b1c10db267b4..c7d132bc0c0da980f3776e3b7e82041b72620f24 100644
--- a/dbrepo-metadata-db/setup-schema_local.sql
+++ b/dbrepo-metadata-db/setup-schema_local.sql
@@ -1,6 +1,7 @@
BEGIN;
-INSERT INTO `mdb_containers` (name, internal_name, image_id, host, port, privileged_username, privileged_password)
-VALUES ('MariaDB 10.5', 'mariadb_10_5', 1, 'data-db', 3306, 'root', 'dbrepo');
+INSERT INTO `mdb_containers` (name, internal_name, image_id, host, port, sidecar_host, sidecar_port,
+ privileged_username, privileged_password)
+VALUES ('MariaDB 10.5', 'mariadb_10_5', 1, 'data-db', 3306, 'data-db-sidecar', 3305, 'root', 'dbrepo');
COMMIT;
diff --git a/dbrepo-metadata-service/Dockerfile b/dbrepo-metadata-service/Dockerfile
index c73815ab18a5a02f2187b035a08325ab291e3975..f3cebd4f4e6caa3e271d2ba4b2fa5967fb3d0fed 100644
--- a/dbrepo-metadata-service/Dockerfile
+++ b/dbrepo-metadata-service/Dockerfile
@@ -58,8 +58,6 @@ ENV PID_BASE="http://localhost/pid/"
ENV REPOSITORY_NAME="Example Repository"
ENV SEARCH_USERNAME=admin
ENV SEARCH_PASSWORD=admin
-ENV SHARED_FILESYSTEM=/tmp
-ENV DELETE_AFTER_IMPORT=true
ENV USER_NETWORK=userdb
ENV WEBSITE="http://localhost"
ENV KEYCLOAK_HOST="http://authentication-service:8080"
@@ -76,6 +74,9 @@ ENV DATACITE_URL="https://api.test.datacite.org"
ENV DATACITE_PREFIX=""
ENV DATACITE_USERNAME=""
ENV DATACITE_PASSWORD=""
+ENV S3_STORAGE_ENDPOINT="http://storage-service:9000"
+ENV S3_ACCESS_KEY_ID="minioadmin"
+ENV S3_SECRET_ACCESS_KEY="minioadmin"
WORKDIR /app
diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/ContainerDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/ContainerDto.java
index 11ca31ae8afcdc4e49d563170c835c34e73b8e60..bedba0f11369ed3243e36d7c55671a08b82cd788 100644
--- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/ContainerDto.java
+++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/ContainerDto.java
@@ -40,6 +40,14 @@ public class ContainerDto {
private Integer port;
+ @NotBlank
+ @JsonProperty("sidecar_host")
+ private String sidecarHost;
+
+ @NotNull
+ @JsonProperty("sidecar_port")
+ private Integer sidecarPort;
+
private ImageBriefDto image;
@NotNull
diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/query/ImportDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/query/ImportDto.java
index bb2425edf01a315666dcf97b8a481125a5041ca0..d94ed58767610d7300c593a1150b75aed9b175f1 100644
--- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/query/ImportDto.java
+++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/query/ImportDto.java
@@ -20,7 +20,7 @@ import lombok.extern.jackson.Jacksonized;
public class ImportDto {
@NotBlank(message = "location is required")
- @Schema(example = "/tmp/file.csv")
+ @Schema(example = "file.csv")
private String location;
@Min(value = 0L)
diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/Container.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/Container.java
index c95945caa91cb562eade5e941ae80515f6693434..e44c9313841bd556b8b7cb6671fff75a53bddc93 100644
--- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/Container.java
+++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/Container.java
@@ -45,6 +45,12 @@ public class Container {
@Column
private Integer port;
+ @Column(nullable = false)
+ private String sidecarHost;
+
+ @Column(nullable = false)
+ private Integer sidecarPort;
+
@ToString.Exclude
@org.springframework.data.annotation.Transient
@OneToMany(fetch = FetchType.LAZY, cascade = CascadeType.ALL)
diff --git a/dbrepo-metadata-service/pom.xml b/dbrepo-metadata-service/pom.xml
index 882c2f4379b2d4df0afb0646a7728fee71debfb8..0dd04aee226b89eb6ea5f2467a1f3102901e8252 100644
--- a/dbrepo-metadata-service/pom.xml
+++ b/dbrepo-metadata-service/pom.xml
@@ -69,6 +69,7 @@
<opensearch-client.version>1.1.0</opensearch-client.version>
<opensearch-rest-client.version>2.8.0</opensearch-rest-client.version>
<jackson.version>2.15.2</jackson.version>
+ <minio.version>8.5.6</minio.version>
</properties>
<dependencies>
@@ -239,6 +240,12 @@
<artifactId>springdoc-openapi-starter-webmvc-api</artifactId>
<version>${springdoc-openapi.version}</version>
</dependency>
+ <!-- blob storage -->
+ <dependency>
+ <groupId>io.minio</groupId>
+ <artifactId>minio</artifactId>
+ <version>${minio.version}</version>
+ </dependency>
<!-- Testing -->
<dependency>
<groupId>org.springframework.boot</groupId>
diff --git a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/exception/DataDbSidecarException.java b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/exception/DataDbSidecarException.java
new file mode 100644
index 0000000000000000000000000000000000000000..036b23f6314fd6e230ca98b1ef8c08138a3f49e0
--- /dev/null
+++ b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/exception/DataDbSidecarException.java
@@ -0,0 +1,23 @@
+package at.tuwien.exception;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+import java.io.IOException;
+
+@ResponseStatus(code = HttpStatus.UNPROCESSABLE_ENTITY)
+public class DataDbSidecarException extends IOException {
+
+ public DataDbSidecarException(String msg) {
+ super(msg);
+ }
+
+ public DataDbSidecarException(String msg, Throwable thr) {
+ super(msg, thr);
+ }
+
+ public DataDbSidecarException(Throwable thr) {
+ super(thr);
+ }
+
+}
diff --git a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/QueryMapper.java b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/QueryMapper.java
index fcd0097cb26a69fa8dd8eec616bee9d7429759fa..f02e38456f5f284b3afb9a96034b2570339deed5 100644
--- a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/QueryMapper.java
+++ b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/QueryMapper.java
@@ -157,7 +157,7 @@ public interface QueryMapper {
}
default PreparedStatement pathToRawInsertQuery(Connection connection, Table table, ImportDto data) throws QueryMalformedException {
- final StringBuilder statement = new StringBuilder("LOAD DATA LOCAL INFILE '")
+ final StringBuilder statement = new StringBuilder("LOAD DATA LOCAL INFILE '/tmp/")
.append(data.getLocation())
.append("' INTO TABLE `")
.append(table.getDatabase().getInternalName())
diff --git a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/ExportEndpoint.java b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/ExportEndpoint.java
index e7af2c81df1668d56e38f6ff298f8d2678265a9c..844667233a8ebe738e0313edacdd1978049c5b93 100644
--- a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/ExportEndpoint.java
+++ b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/ExportEndpoint.java
@@ -1,6 +1,8 @@
package at.tuwien.endpoints;
import at.tuwien.ExportResource;
+import at.tuwien.api.error.ApiErrorDto;
+import at.tuwien.api.identifier.IdentifierDto;
import at.tuwien.entities.database.Database;
import at.tuwien.exception.*;
import at.tuwien.service.DatabaseService;
@@ -9,6 +11,10 @@ import at.tuwien.utils.PrincipalUtil;
import at.tuwien.utils.UserUtil;
import io.micrometer.core.annotation.Timed;
import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.security.SecurityRequirement;
import jakarta.validation.constraints.NotNull;
import lombok.extern.log4j.Log4j2;
@@ -41,13 +47,50 @@ public class ExportEndpoint {
@Transactional(readOnly = true)
@Timed(value = "table.export", description = "Time needed to export table data")
@Operation(summary = "Export table", security = @SecurityRequirement(name = "bearerAuth"))
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "201",
+ description = "Created identifier",
+ content = {@Content(
+ mediaType = "application/json",
+ schema = @Schema(implementation = IdentifierDto.class))}),
+ @ApiResponse(responseCode = "400",
+ description = "Images is not supported or table/query is malformed",
+ content = {@Content(
+ mediaType = "application/json",
+ schema = @Schema(implementation = ApiErrorDto.class))}),
+ @ApiResponse(responseCode = "403",
+ description = "Operation is not allowed",
+ content = {@Content(
+ mediaType = "application/json",
+ schema = @Schema(implementation = ApiErrorDto.class))}),
+ @ApiResponse(responseCode = "404",
+ description = "Table, database or user was not found",
+ content = {@Content(
+ mediaType = "application/json",
+ schema = @Schema(implementation = ApiErrorDto.class))}),
+ @ApiResponse(responseCode = "410",
+ description = "Blob storage operation could not be completed",
+ content = {@Content(
+ mediaType = "application/json",
+ schema = @Schema(implementation = ApiErrorDto.class))}),
+ @ApiResponse(responseCode = "422",
+ description = "Sidecar operation could not be completed",
+ content = {@Content(
+ mediaType = "application/json",
+ schema = @Schema(implementation = ApiErrorDto.class))}),
+ @ApiResponse(responseCode = "503",
+ description = "Database connection could not be established",
+ content = {@Content(
+ mediaType = "application/json",
+ schema = @Schema(implementation = ApiErrorDto.class))}),
+ })
public ResponseEntity<InputStreamResource> export(@NotNull @PathVariable("id") Long databaseId,
@NotNull @PathVariable("tableId") Long tableId,
@RequestParam(required = false) Instant timestamp,
Principal principal)
throws TableNotFoundException, DatabaseConnectionException, TableMalformedException,
DatabaseNotFoundException, ImageNotSupportedException, PaginationException, FileStorageException,
- QueryMalformedException, UserNotFoundException, NotAllowedException {
+ QueryMalformedException, UserNotFoundException, NotAllowedException, DataDbSidecarException {
log.debug("endpoint export table, id={}, tableId={}, timestamp={}, {}", databaseId, tableId, timestamp, PrincipalUtil.formatForDebug(principal));
final Database database = databaseService.find(databaseId);
if (!database.getIsPublic()) {
diff --git a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableDataEndpoint.java b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableDataEndpoint.java
index ac1ba10e22fe959504625ea024ad4d4ba1ec08e3..7a75c659ee41316537fb7bdff292c42e0bf817b8 100644
--- a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableDataEndpoint.java
+++ b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableDataEndpoint.java
@@ -121,7 +121,7 @@ public class TableDataEndpoint {
@NotNull Principal principal)
throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException,
ImageNotSupportedException, DatabaseConnectionException, QueryMalformedException, UserNotFoundException,
- NotAllowedException, AccessDeniedException {
+ NotAllowedException, AccessDeniedException, DataDbSidecarException {
log.debug("endpoint insert data from csv, databaseId={}, tableId={}, data={}, {}", databaseId, tableId, data, PrincipalUtil.formatForDebug(principal));
/* check */
endpointValidator.validateOnlyWriteOwnOrWriteAllAccess(databaseId, tableId, principal);
diff --git a/dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml b/dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml
index 63e3625dbc4dfb972185d0fb974243f4562973cc..b5623606f4be911fed1f612c43a96b8392878438 100644
--- a/dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml
+++ b/dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml
@@ -53,6 +53,10 @@ fda:
base: https://example.com/pid/
broker:
endpoint: http://localhost:15672
+ minio:
+ endpoint: http://localhost:9000
+ accessKeyId: minioadmin
+ secretAccessKey: minioadmin
jwt:
issuer: http://localhost/realms/dbrepo
public_key: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB
@@ -69,8 +73,6 @@ fda:
exchangeName: "dbrepo"
routingKey: "dbrepo.#"
connectionTimeout: 60000
- sharedFilesystem: /tmp
- deleteAfterImport: true
dbrepo:
repository-name: TU Wien Database Repository
base-url: https://dbrepo1.ec.tuwien.at/api/oai
diff --git a/dbrepo-metadata-service/rest-service/src/main/resources/application.yml b/dbrepo-metadata-service/rest-service/src/main/resources/application.yml
index 7b2e68fa3b8fcf615e1390c5cd4ce8c3dca87df3..b5b7291016f2db218706f62b0940d498b0d37a1b 100644
--- a/dbrepo-metadata-service/rest-service/src/main/resources/application.yml
+++ b/dbrepo-metadata-service/rest-service/src/main/resources/application.yml
@@ -66,6 +66,10 @@ fda:
base: "${PID_BASE}"
broker:
endpoint: "${BROKER_ENDPOINT}"
+ minio:
+ endpoint: "${S3_STORAGE_ENDPOINT}"
+ accessKeyId: "${S3_ACCESS_KEY_ID}"
+ secretAccessKey: "${S3_SECRET_ACCESS_KEY}"
jwt:
issuer: "${JWT_ISSUER}"
public_key: "${JWT_PUBKEY}"
@@ -82,8 +86,6 @@ fda:
exchangeName: "${EXCHANGE_NAME}"
routingKey: "${ROUTING_KEY}"
connectionTimeout: ${CONNECTION_TIMEOUT}
- sharedFilesystem: ${SHARED_FILESYSTEM}
- deleteAfterImport: ${DELETE_AFTER_IMPORT}
dbrepo:
repository-name: "${REPOSITORY_NAME}"
base-url: "${BASE_URL}"
diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableDataEndpointUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableDataEndpointUnitTest.java
index 801aec38422b9e10d082a799736e742b0d760894..2a61c8b883d7ad3c1e629cfd5a55f3bfe5d12281 100644
--- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableDataEndpointUnitTest.java
+++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableDataEndpointUnitTest.java
@@ -138,7 +138,7 @@ public class TableDataEndpointUnitTest extends BaseUnitTest {
@WithMockUser(username = USER_1_USERNAME, authorities = {"insert-table-data"})
public void import_publicWriteAll_succeeds() throws UserNotFoundException, TableNotFoundException, NotAllowedException,
TableMalformedException, DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException,
- ImageNotSupportedException, AccessDeniedException {
+ ImageNotSupportedException, AccessDeniedException, DataDbSidecarException {
/* test */
generic_import(DATABASE_3_ID, DATABASE_3, TABLE_8_ID, TABLE_8, USER_1_ID,
@@ -149,7 +149,7 @@ public class TableDataEndpointUnitTest extends BaseUnitTest {
@WithMockUser(username = USER_1_USERNAME, authorities = {"insert-table-data"})
public void import_privateWriteAll_succeeds() throws UserNotFoundException, TableNotFoundException, NotAllowedException,
TableMalformedException, DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException,
- ImageNotSupportedException, AccessDeniedException {
+ ImageNotSupportedException, AccessDeniedException, DataDbSidecarException {
/* test */
generic_import(DATABASE_1_ID, DATABASE_1, TABLE_1_ID, TABLE_1, USER_1_ID,
@@ -426,7 +426,7 @@ public class TableDataEndpointUnitTest extends BaseUnitTest {
DatabaseAccess access, Principal principal) throws DatabaseNotFoundException,
TableNotFoundException, NotAllowedException, UserNotFoundException, TableMalformedException,
DatabaseConnectionException, QueryMalformedException, ImageNotSupportedException,
- AccessDeniedException {
+ AccessDeniedException, DataDbSidecarException {
final ImportDto request = ImportDto.builder().location("test:csv/csv_01.csv").build();
/* mock */
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/config/GatewayConfig.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/config/GatewayConfig.java
index 5c1dd846aa4c09dbfe22f13d823f824d3f457dc2..7fb10fe679d4db37d8b89eda15ccc4cb6a95908d 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/config/GatewayConfig.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/config/GatewayConfig.java
@@ -36,4 +36,12 @@ public class GatewayConfig {
return restTemplate;
}
+ @Bean("sidecarRestTemplate")
+ public RestTemplate sidecarRestTemplate() {
+ final RestTemplate restTemplate = new RestTemplate();
+ restTemplate.getInterceptors()
+ .add(new BasicAuthenticationInterceptor(brokerUsername, brokerPassword));
+ return restTemplate;
+ }
+
}
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/config/MinioConfig.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/config/MinioConfig.java
new file mode 100644
index 0000000000000000000000000000000000000000..84be372b043c6a6d9af77bf656b9cd6f2c274408
--- /dev/null
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/config/MinioConfig.java
@@ -0,0 +1,30 @@
+package at.tuwien.config;
+
+import io.minio.MinioClient;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Slf4j
+@Configuration
+public class MinioConfig {
+
+ @Value("${fda.minio.endpoint}")
+ private String minioEndpoint;
+
+ @Value("${fda.minio.accessKeyId}")
+ private String minioAccessKeyId;
+
+ @Value("${fda.minio.secretAccessKey}")
+ private String minioSecretAccessKey;
+
+ @Bean
+ public MinioClient minioClient() {
+ return MinioClient.builder()
+ .endpoint(minioEndpoint)
+ .credentials(minioAccessKeyId, minioSecretAccessKey)
+ .build();
+ }
+
+}
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/config/QueryConfig.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/config/QueryConfig.java
index a8b7b56f9ee32bc82186bc4f1bc614a97089772a..e3bcf500207d61f8dfacff2f5be61bf4e2e75975 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/config/QueryConfig.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/config/QueryConfig.java
@@ -14,10 +14,4 @@ public class QueryConfig {
@Value("${fda.unsupported}")
private String[] notSupportedKeywords;
- @Value("${fda.sharedFilesystem}")
- private String sharedFilesystem;
-
- @Value("${fda.deleteAfterImport}")
- private Boolean deleteAfterImport;
-
}
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/DataDbSidecarGateway.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/DataDbSidecarGateway.java
new file mode 100644
index 0000000000000000000000000000000000000000..c7dff9c9e3e8d340fbfb490398e772f51881ef67
--- /dev/null
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/DataDbSidecarGateway.java
@@ -0,0 +1,9 @@
+package at.tuwien.gateway;
+
+import at.tuwien.exception.DataDbSidecarException;
+
+public interface DataDbSidecarGateway {
+ void importFile(String hostname, Integer port, String filename) throws DataDbSidecarException;
+
+ void exportFile(String hostname, Integer port, String filename) throws DataDbSidecarException;
+}
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/DataDbSidecarGatewayImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/DataDbSidecarGatewayImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..6e4b2b56be874c1387d408f872a39158d732a086
--- /dev/null
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/DataDbSidecarGatewayImpl.java
@@ -0,0 +1,48 @@
+package at.tuwien.gateway.impl;
+
+import at.tuwien.exception.DataDbSidecarException;
+import at.tuwien.gateway.DataDbSidecarGateway;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.stereotype.Service;
+import org.springframework.web.client.HttpServerErrorException;
+import org.springframework.web.client.ResourceAccessException;
+import org.springframework.web.client.RestTemplate;
+
+@Slf4j
+@Service
+public class DataDbSidecarGatewayImpl implements DataDbSidecarGateway {
+
+ private final RestTemplate restTemplate;
+
+ public DataDbSidecarGatewayImpl(@Qualifier("sidecarRestTemplate") RestTemplate restTemplate) {
+ this.restTemplate = restTemplate;
+ }
+
+ @Override
+ public void importFile(String hostname, Integer port, String filename) throws DataDbSidecarException {
+ final HttpHeaders headers = new HttpHeaders();
+ headers.set("Accept", "application/json");
+ try {
+ restTemplate.exchange("http://" + hostname + ":" + port + "/sidecar/import/" + filename, HttpMethod.POST, new HttpEntity<>(null, headers), Void.class);
+ } catch (ResourceAccessException | HttpServerErrorException.ServiceUnavailable e) {
+ log.error("Failed to import .csv in data-db sidecar: {}", e.getMessage());
+ throw new DataDbSidecarException("Failed to import .csv in data-db sidecar: " + e.getMessage());
+ }
+ }
+
+ @Override
+ public void exportFile(String hostname, Integer port, String filename) throws DataDbSidecarException {
+ final HttpHeaders headers = new HttpHeaders();
+ headers.set("Accept", "application/json");
+ try {
+ restTemplate.exchange("http://" + hostname + ":" + port + "/sidecar/export/" + filename, HttpMethod.POST, new HttpEntity<>(null, headers), Void.class);
+ } catch (ResourceAccessException | HttpServerErrorException.ServiceUnavailable e) {
+ log.error("Failed to export .csv in data-db sidecar: {}", e.getMessage());
+ throw new DataDbSidecarException("Failed to export .csv in data-db sidecar: " + e.getMessage());
+ }
+ }
+}
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/QueryService.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/QueryService.java
index 534b30c69b1007dbed44cb14813b4cc34d84e572..e5d7f33d6d6d93579a0a08cc153996dfcb5900a1 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/QueryService.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/QueryService.java
@@ -133,7 +133,7 @@ public interface QueryService {
ExportResource tableFindAll(Long databaseId, Long tableId, Instant timestamp, Principal principal)
throws TableNotFoundException, DatabaseNotFoundException, ImageNotSupportedException,
DatabaseConnectionException, TableMalformedException, PaginationException,
- FileStorageException, QueryMalformedException, UserNotFoundException;
+ FileStorageException, QueryMalformedException, UserNotFoundException, DataDbSidecarException;
/**
* Select all data known in the view id tuple and return a page of specific size.
@@ -270,7 +270,7 @@ public interface QueryService {
* @throws QueryMalformedException The query is malformed.
*/
void insert(Long databaseId, Long tableId, ImportDto data, Principal principal) throws ImageNotSupportedException,
- TableMalformedException, DatabaseNotFoundException, TableNotFoundException, DatabaseConnectionException, QueryMalformedException, UserNotFoundException;
+ TableMalformedException, DatabaseNotFoundException, TableNotFoundException, DatabaseConnectionException, QueryMalformedException, UserNotFoundException, DataDbSidecarException;
/**
* Parses the stored columns from a given query.
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java
index ad6794956ada69a5909251df413d9ad24f78c6ef..a148e7fc83bbc99dc85116fe45aed253716d15e9 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java
@@ -14,6 +14,7 @@ import at.tuwien.entities.database.View;
import at.tuwien.entities.database.table.Table;
import at.tuwien.entities.database.table.columns.TableColumn;
import at.tuwien.exception.*;
+import at.tuwien.gateway.DataDbSidecarGateway;
import at.tuwien.mapper.QueryMapper;
import at.tuwien.querystore.Query;
import at.tuwien.repository.mdb.TableColumnRepository;
@@ -22,6 +23,9 @@ import at.tuwien.service.QueryService;
import at.tuwien.service.StoreService;
import at.tuwien.service.TableService;
import com.mchange.v2.c3p0.ComboPooledDataSource;
+import io.minio.GetObjectArgs;
+import io.minio.MinioClient;
+import io.minio.errors.*;
import lombok.extern.log4j.Log4j2;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.parser.CCJSqlParserManager;
@@ -37,7 +41,10 @@ import org.springframework.transaction.annotation.Transactional;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
import java.io.StringReader;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
import java.security.Principal;
import java.sql.Connection;
import java.sql.PreparedStatement;
@@ -54,21 +61,27 @@ import java.util.stream.Collectors;
@Service
public class QueryServiceImpl extends HibernateConnector implements QueryService {
- private final QueryConfig queryConfig;
+ private final MinioClient minioClient;
private final QueryMapper queryMapper;
private final StoreService storeService;
private final TableService tableService;
private final DatabaseService databaseService;
+ private final DataDbSidecarGateway dataDbSidecarGateway;
private final TableColumnRepository tableColumnRepository;
+ private static final String BUCKET_NAME_DOWNLOAD = "dbrepo-download";
+ private static final String BUCKET_NAME_UPLOAD = "dbrepo-upload";
+
@Autowired
- public QueryServiceImpl(QueryConfig queryConfig, QueryMapper queryMapper, TableService tableService, DatabaseService databaseService,
- StoreService storeService, TableColumnRepository tableColumnRepository) {
- this.queryConfig = queryConfig;
+ public QueryServiceImpl(MinioClient minioClient, QueryMapper queryMapper, TableService tableService,
+ DatabaseService databaseService, StoreService storeService,
+ DataDbSidecarGateway dataDbSidecarGateway, TableColumnRepository tableColumnRepository) {
+ this.minioClient = minioClient;
this.queryMapper = queryMapper;
this.tableService = tableService;
this.storeService = storeService;
this.databaseService = databaseService;
+ this.dataDbSidecarGateway = dataDbSidecarGateway;
this.tableColumnRepository = tableColumnRepository;
}
@@ -227,7 +240,8 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
@Override
@Transactional(readOnly = true)
public ExportResource tableFindAll(Long databaseId, Long tableId, Instant timestamp, Principal principal)
- throws TableNotFoundException, DatabaseNotFoundException, FileStorageException, QueryMalformedException {
+ throws TableNotFoundException, DatabaseNotFoundException, FileStorageException, QueryMalformedException,
+ DataDbSidecarException {
final String filename = RandomStringUtils.randomAlphabetic(40) + ".csv";
/* find */
final Database database = databaseService.find(databaseId);
@@ -235,31 +249,31 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
/* run query */
final ComboPooledDataSource dataSource = getPrivilegedDataSource(database.getContainer().getImage(),
database.getContainer(), database);
- /* read file */
- final InputStreamResource resource;
try {
final Connection connection = dataSource.getConnection();
final PreparedStatement preparedStatement = queryMapper.tableToRawExportQuery(connection, table, timestamp, filename);
preparedStatement.executeUpdate();
- final String location = queryConfig.getSharedFilesystem() + File.separator + filename;
- final File file = new File(location);
- resource = new InputStreamResource(FileUtils.openInputStream(file));
- if (queryConfig.getDeleteAfterImport()) {
- log.debug("attempt to delete file: {}", location);
- FileUtils.forceDelete(file);
- } else {
- log.trace("skipping deletion of file as per configuration");
- }
- } catch (IOException | SQLException e) {
+ } catch (SQLException e) {
log.error("Failed to execute query and/or export file: {}", e.getMessage());
throw new FileStorageException("Failed to execute query and/or export file: " + e.getMessage(), e);
} finally {
dataSource.close();
}
- return ExportResource.builder()
- .resource(resource)
- .filename(filename)
- .build();
+ /* upload from sidecar into blob storage */
+ dataDbSidecarGateway.exportFile(database.getContainer().getSidecarHost(), database.getContainer().getSidecarPort(), filename);
+ /* export file from blob storage */
+ try (InputStream stream = minioClient.getObject(GetObjectArgs.builder().bucket(BUCKET_NAME_DOWNLOAD).object(filename).build())) {
+ log.debug("found object with key {} in bucket {}", filename, BUCKET_NAME_DOWNLOAD);
+ return ExportResource.builder()
+ .resource(new InputStreamResource(stream))
+ .filename(filename)
+ .build();
+ } catch (ServerException | InsufficientDataException | ErrorResponseException | IOException |
+ NoSuchAlgorithmException | InvalidKeyException | InvalidResponseException | XmlParserException |
+ InternalException e) {
+ log.error("Failed to find object {} in bucket {}", filename, BUCKET_NAME_DOWNLOAD);
+ throw new FileStorageException("Failed to find object " + filename + " in bucket " + BUCKET_NAME_DOWNLOAD);
+ }
}
@Override
@@ -382,7 +396,8 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
@Override
@Transactional
public void insert(Long databaseId, Long tableId, ImportDto data, Principal principal)
- throws TableMalformedException, DatabaseNotFoundException, TableNotFoundException, QueryMalformedException {
+ throws TableMalformedException, DatabaseNotFoundException, TableNotFoundException, QueryMalformedException,
+ DataDbSidecarException {
/* find */
final Database database = databaseService.find(databaseId);
final Table table = tableService.find(databaseId, tableId);
@@ -396,7 +411,6 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
.executeUpdate();
} catch (SQLException e) {
log.error("Failed to drop temporary table: {}", e.getMessage());
- log.trace("failed to drop temporary table {}", table);
throw new TableMalformedException("Failed to drop temporary table", e);
}
try {
@@ -408,21 +422,16 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
dataSource.close();
throw new TableMalformedException("Failed to create temporary table", e);
}
- data.setLocation(queryConfig.getSharedFilesystem() + File.separator + data.getLocation());
+ /* import .csv from blob storage to sidecar */
+ dataDbSidecarGateway.importFile(database.getContainer().getSidecarHost(), database.getContainer().getSidecarPort(), data.getLocation());
+ /* import .csv from sidecar to database */
try {
final Connection connection = dataSource.getConnection();
queryMapper.pathToRawInsertQuery(connection, table, data)
.executeUpdate();
- if (queryConfig.getDeleteAfterImport()) {
- log.debug("attempt to delete file: {}", data.getLocation());
- final File file = new File(data.getLocation());
- FileUtils.forceDelete(file);
- } else {
- log.trace("skipping deletion of file as per configuration");
- }
queryMapper.generateInsertFromTemporaryTableSQL(connection, table)
.executeUpdate();
- } catch (SQLException | IOException e) {
+ } catch (SQLException e) {
log.error("Failed to insert temporary table: {}", e.getMessage());
dataSource.close();
throw new TableMalformedException("Failed to insert temporary table", e);
diff --git a/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java b/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java
index 615d13d2493cdd9a51521b7549f147e09e6ef012..227d89cd090fa8669237ba79991b5137791dbe1f 100644
--- a/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java
+++ b/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java
@@ -826,7 +826,10 @@ public abstract class BaseTest {
public final static String CONTAINER_1_INTERNALNAME = "dbrepo-userdb-u01";
public final static String CONTAINER_1_IP = "127.0.0.1";
public final static Boolean CONTAINER_1_RUNNING = true;
+ public final static String CONTAINER_1_HOST = "localhost";
public final static Integer CONTAINER_1_PORT = 3308;
+ public final static String CONTAINER_1_SIDECAR_HOST = "localhost";
+ public final static Integer CONTAINER_1_SIDECAR_PORT = 33081;
public final static String CONTAINER_1_PRIVILEGED_USERNAME = "root";
public final static String CONTAINER_1_PRIVILEGED_PASSWORD = "dbrepo";
public final static Instant CONTAINER_1_CREATED = Instant.ofEpochSecond(1677399629) /* 2023-02-26 08:20:29 (UTC) */;
@@ -838,8 +841,10 @@ public abstract class BaseTest {
.imageId(IMAGE_1_ID)
.image(CONTAINER_1_IMAGE)
.created(CONTAINER_1_CREATED)
- .host(CONTAINER_1_IP)
+ .host(CONTAINER_1_HOST)
.port(CONTAINER_1_PORT)
+ .sidecarHost(CONTAINER_1_SIDECAR_HOST)
+ .sidecarPort(CONTAINER_1_SIDECAR_PORT)
.privilegedUsername(CONTAINER_1_PRIVILEGED_USERNAME)
.privilegedPassword(CONTAINER_1_PRIVILEGED_PASSWORD)
.build();
@@ -851,8 +856,10 @@ public abstract class BaseTest {
.imageId(IMAGE_1_ID)
.image(null /* for jpa */)
.created(CONTAINER_1_CREATED)
- .host(CONTAINER_1_IP)
+ .host(CONTAINER_1_HOST)
.port(CONTAINER_1_PORT)
+ .sidecarHost(CONTAINER_1_SIDECAR_HOST)
+ .sidecarPort(CONTAINER_1_SIDECAR_PORT)
.privilegedUsername(CONTAINER_1_PRIVILEGED_USERNAME)
.privilegedPassword(CONTAINER_1_PRIVILEGED_PASSWORD)
.build();
@@ -863,7 +870,10 @@ public abstract class BaseTest {
.internalName(CONTAINER_1_INTERNALNAME)
.image(CONTAINER_1_IMAGE_BRIEF_DTO)
.created(CONTAINER_1_CREATED)
- .host(CONTAINER_1_IP)
+ .host(CONTAINER_1_HOST)
+ .port(CONTAINER_1_PORT)
+ .sidecarHost(CONTAINER_1_SIDECAR_HOST)
+ .sidecarPort(CONTAINER_1_SIDECAR_PORT)
.build();
public final static ContainerBriefDto CONTAINER_1_DTO_BRIEF = ContainerBriefDto.builder()
diff --git a/docker-compose.yml b/docker-compose.yml
index 3a51aa40cbeb9b5dc594b6ba72c8ea8280847cf3..7bfcf542651f4d11f22882ff1ba0dcdb399ade46 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -261,7 +261,7 @@ services:
build: ./dbrepo-data-db/sidecar
image: dbrepo-data-db-sidecar:latest
ports:
- - "3600:5000"
+ - "3305:3305"
environment:
FLASK_DEBUG: ${SEARCH_DEBUG_MODE:-true}
S3_STORAGE_ENDPOINT: "${STORAGE_ENDPOINT:-http://storage-service:9000}"