From 3657b1a96828b3449ac0fdf859397613a84e0297 Mon Sep 17 00:00:00 2001
From: Martin Weise <martin.weise@tuwien.ac.at>
Date: Sat, 11 Nov 2023 10:07:04 +0100
Subject: [PATCH] Fixed the stream
---
.../tuwien/endpoints/PersistenceEndpoint.java | 2 +-
.../at/tuwien/endpoints/QueryEndpoint.java | 2 +-
.../src/main/resources/application-local.yml | 2 +-
.../endpoints/IdentifierEndpointUnitTest.java | 6 ++--
.../service/QueryServiceIntegrationTest.java | 2 +-
.../at/tuwien/service/IdentifierService.java | 2 +-
.../java/at/tuwien/service/QueryService.java | 2 +-
.../impl/DataCiteIdentifierServiceImpl.java | 3 +-
.../service/impl/IdentifierServiceImpl.java | 6 ++--
.../tuwien/service/impl/QueryServiceImpl.java | 28 ++++++++-----------
dbrepo-ui/api/table.service.js | 2 +-
.../_database_id/table/_table_id/import.vue | 6 ++--
12 files changed, 29 insertions(+), 34 deletions(-)
diff --git a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/PersistenceEndpoint.java b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/PersistenceEndpoint.java
index 90ae5f6e80..3284977e77 100644
--- a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/PersistenceEndpoint.java
+++ b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/PersistenceEndpoint.java
@@ -92,7 +92,7 @@ public class PersistenceEndpoint {
@NotNull Principal principal) throws IdentifierNotFoundException,
QueryNotFoundException, RemoteUnavailableException, IdentifierRequestException, UserNotFoundException,
QueryStoreException, TableMalformedException, DatabaseConnectionException, QueryMalformedException,
- DatabaseNotFoundException, ImageNotSupportedException, FileStorageException {
+ DatabaseNotFoundException, ImageNotSupportedException, FileStorageException, DataDbSidecarException {
log.debug("endpoint find identifier, pid={}, accept={}", pid, accept);
final Identifier identifier = identifierService.find(pid);
log.info("Found persistent identifier with id {}", identifier.getId());
diff --git a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/QueryEndpoint.java b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/QueryEndpoint.java
index 7c92ab18f5..e312d1110b 100644
--- a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/QueryEndpoint.java
+++ b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/QueryEndpoint.java
@@ -140,7 +140,7 @@ public class QueryEndpoint {
Principal principal)
throws QueryStoreException, QueryNotFoundException, DatabaseNotFoundException, ImageNotSupportedException,
TableMalformedException, FileStorageException, QueryMalformedException, DatabaseConnectionException,
- UserNotFoundException, NotAllowedException {
+ UserNotFoundException, NotAllowedException, DataDbSidecarException {
log.debug("endpoint export query, databaseId={}, queryId={}, accept={}, {}", databaseId, queryId, accept, PrincipalUtil.formatForDebug(principal));
final Database database = databaseService.find(databaseId);
if (!database.getIsPublic()) {
diff --git a/dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml b/dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml
index b5623606f4..001d0734e7 100644
--- a/dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml
+++ b/dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml
@@ -58,7 +58,7 @@ fda:
accessKeyId: minioadmin
secretAccessKey: minioadmin
jwt:
- issuer: http://localhost/realms/dbrepo
+ issuer: http://localhost/api/auth/realms/dbrepo
public_key: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB
keycloak:
endpoint: "http://authentication-service:8080"
diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/IdentifierEndpointUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/IdentifierEndpointUnitTest.java
index 5870812253..c496b3fa03 100644
--- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/IdentifierEndpointUnitTest.java
+++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/IdentifierEndpointUnitTest.java
@@ -86,7 +86,7 @@ public class IdentifierEndpointUnitTest extends BaseUnitTest {
public void find_json_succeeds() throws IdentifierNotFoundException, QueryNotFoundException,
RemoteUnavailableException, IdentifierRequestException, UserNotFoundException, QueryStoreException,
TableMalformedException, DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException,
- ImageNotSupportedException, FileStorageException {
+ ImageNotSupportedException, FileStorageException, DataDbSidecarException {
final String accept = "application/json";
/* mock */
@@ -151,7 +151,7 @@ public class IdentifierEndpointUnitTest extends BaseUnitTest {
public void find_httpRedirect_succeeds() throws IdentifierNotFoundException, QueryNotFoundException,
RemoteUnavailableException, IdentifierRequestException, UserNotFoundException, QueryStoreException,
TableMalformedException, DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException,
- ImageNotSupportedException, FileStorageException {
+ ImageNotSupportedException, FileStorageException, DataDbSidecarException {
/* test */
final ResponseEntity<?> response = generic_find(null, null, null);
@@ -320,7 +320,7 @@ public class IdentifierEndpointUnitTest extends BaseUnitTest {
throws IdentifierNotFoundException, QueryNotFoundException, RemoteUnavailableException,
IdentifierRequestException, UserNotFoundException, QueryStoreException, TableMalformedException,
DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException,
- FileStorageException {
+ FileStorageException, DataDbSidecarException {
/* mock */
when(identifierService.find(IDENTIFIER_1_ID))
diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java
index 32f6119398..09ca1a1d4d 100644
--- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java
+++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java
@@ -501,7 +501,7 @@ public class QueryServiceIntegrationTest extends BaseUnitTest {
@Test
public void count_emptySet_succeeds() throws DatabaseConnectionException, TableMalformedException,
DatabaseNotFoundException, ImageNotSupportedException, QueryMalformedException, UserNotFoundException,
- QueryStoreException, QueryNotFoundException, FileStorageException, SQLException {
+ QueryStoreException, QueryNotFoundException, FileStorageException, SQLException, DataDbSidecarException {
final Query query = Query.builder()
.id(QUERY_1_ID)
.query("SELECT `location`, `lat`, `lng` FROM `weather_location` WHERE `location` = \"Vienna\"")
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/IdentifierService.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/IdentifierService.java
index f8bc21adff..6a2ccee838 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/IdentifierService.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/IdentifierService.java
@@ -120,7 +120,7 @@ public interface IdentifierService {
InputStreamResource exportResource(Long identifierId, Principal principal)
throws IdentifierNotFoundException, QueryNotFoundException, FileStorageException,
IdentifierRequestException, UserNotFoundException, QueryStoreException, TableMalformedException,
- DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException;
+ DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, DataDbSidecarException;
/**
* Updated the metadata (only) on the identifier for a given id in the metadata database.
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/QueryService.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/QueryService.java
index e5d7f33d6d..1bc8fa77c8 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/QueryService.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/QueryService.java
@@ -173,7 +173,7 @@ public interface QueryService {
*/
ExportResource findOne(Long databaseId, Long queryId, Principal principal) throws DatabaseNotFoundException,
ImageNotSupportedException, TableMalformedException, FileStorageException, QueryStoreException,
- QueryNotFoundException, QueryMalformedException, DatabaseConnectionException, UserNotFoundException;
+ QueryNotFoundException, QueryMalformedException, DatabaseConnectionException, UserNotFoundException, DataDbSidecarException;
/**
* Count the total tuples for a given table id within a database id at a given time.
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/DataCiteIdentifierServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/DataCiteIdentifierServiceImpl.java
index e22cc03de4..9d10b155a1 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/DataCiteIdentifierServiceImpl.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/DataCiteIdentifierServiceImpl.java
@@ -191,7 +191,8 @@ public class DataCiteIdentifierServiceImpl implements IdentifierService {
public InputStreamResource exportResource(Long identifierId, Principal principal)
throws IdentifierNotFoundException, QueryNotFoundException, FileStorageException,
IdentifierRequestException, UserNotFoundException, QueryStoreException, TableMalformedException,
- DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException {
+ DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException,
+ DataDbSidecarException {
return identifierService.exportResource(identifierId, principal);
}
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/IdentifierServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/IdentifierServiceImpl.java
index 21a8a392ed..2cfc3b74ca 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/IdentifierServiceImpl.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/IdentifierServiceImpl.java
@@ -240,9 +240,9 @@ public class IdentifierServiceImpl implements IdentifierService {
@Override
@Transactional(readOnly = true)
public InputStreamResource exportResource(Long identifierId, Principal principal) throws IdentifierNotFoundException,
- QueryNotFoundException, IdentifierRequestException, UserNotFoundException,
- QueryStoreException, TableMalformedException, DatabaseConnectionException, QueryMalformedException,
- DatabaseNotFoundException, ImageNotSupportedException, FileStorageException {
+ QueryNotFoundException, IdentifierRequestException, UserNotFoundException, QueryStoreException,
+ TableMalformedException, DatabaseConnectionException, QueryMalformedException,
+ DatabaseNotFoundException, ImageNotSupportedException, FileStorageException, DataDbSidecarException {
/* check */
final Identifier identifier = find(identifierId);
if (identifier.getType().equals(IdentifierType.DATABASE)) {
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java
index a148e7fc83..957161e468 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java
@@ -9,6 +9,7 @@ import at.tuwien.api.database.table.TableCsvDeleteDto;
import at.tuwien.api.database.table.TableCsvDto;
import at.tuwien.api.database.table.TableCsvUpdateDto;
import at.tuwien.config.QueryConfig;
+import at.tuwien.entities.container.Container;
import at.tuwien.entities.database.Database;
import at.tuwien.entities.database.View;
import at.tuwien.entities.database.table.Table;
@@ -33,6 +34,7 @@ import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.statement.Statement;
import net.sf.jsqlparser.statement.select.*;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.InputStreamResource;
@@ -259,10 +261,15 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
} finally {
dataSource.close();
}
+ return retrieveBlobAsResource(database.getContainer(), filename);
+ }
+
+ private ExportResource retrieveBlobAsResource(Container container, String filename) throws DataDbSidecarException, FileStorageException {
/* upload from sidecar into blob storage */
- dataDbSidecarGateway.exportFile(database.getContainer().getSidecarHost(), database.getContainer().getSidecarPort(), filename);
+ dataDbSidecarGateway.exportFile(container.getSidecarHost(), container.getSidecarPort(), filename);
/* export file from blob storage */
- try (InputStream stream = minioClient.getObject(GetObjectArgs.builder().bucket(BUCKET_NAME_DOWNLOAD).object(filename).build())) {
+ try {
+ final InputStream stream = minioClient.getObject(GetObjectArgs.builder().bucket(BUCKET_NAME_DOWNLOAD).object(filename).build());
log.debug("found object with key {} in bucket {}", filename, BUCKET_NAME_DOWNLOAD);
return ExportResource.builder()
.resource(new InputStreamResource(stream))
@@ -280,7 +287,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
@Transactional(readOnly = true)
public ExportResource findOne(Long databaseId, Long queryId, Principal principal)
throws DatabaseNotFoundException, ImageNotSupportedException, FileStorageException, QueryStoreException,
- QueryNotFoundException, QueryMalformedException, DatabaseConnectionException, UserNotFoundException {
+ QueryNotFoundException, QueryMalformedException, DatabaseConnectionException, UserNotFoundException, DataDbSidecarException {
final String filename = RandomStringUtils.randomAlphabetic(40) + ".csv";
/* find */
final Database database = databaseService.find(databaseId);
@@ -288,30 +295,17 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
/* run query */
final ComboPooledDataSource dataSource = getPrivilegedDataSource(database.getContainer().getImage(),
database.getContainer(), database);
- /* read file */
- final InputStreamResource resource;
try {
final Connection connection = dataSource.getConnection();
final PreparedStatement preparedStatement = queryMapper.queryToRawExportQuery(connection, query, filename);
preparedStatement.executeUpdate();
- final File file = new File("/tmp/" + filename);
- resource = new InputStreamResource(FileUtils.openInputStream(file));
- if (!FileUtils.deleteQuietly(file)) {
- log.warn("Failed to delete exported file");
- }
} catch (SQLException e) {
log.error("Failed to execute query: {}", e.getMessage());
throw new QueryStoreException("Failed to execute query: " + e.getMessage(), e);
- } catch (IOException e) {
- log.error("Failed to export query: {}", e.getMessage());
- throw new FileStorageException("Failed to export query: " + e.getMessage(), e);
} finally {
dataSource.close();
}
- return ExportResource.builder()
- .resource(resource)
- .filename(filename)
- .build();
+ return retrieveBlobAsResource(database.getContainer(), filename);
}
@Override
diff --git a/dbrepo-ui/api/table.service.js b/dbrepo-ui/api/table.service.js
index 62e8cf8385..6d9f929539 100644
--- a/dbrepo-ui/api/table.service.js
+++ b/dbrepo-ui/api/table.service.js
@@ -128,7 +128,7 @@ class TableService {
exportDataTimestamp (databaseId, tableId, timestamp) {
return new Promise((resolve, reject) => {
- api.get(`/api/database/${databaseId}/table/${tableId}/export?timestamp=${timestamp}`, { responseType: 'text' })
+ api.get(`/api/database/${databaseId}/table/${tableId}/export${timestamp ? ('?timestamp=' + timestamp) : ''}`, { responseType: 'text' })
.then((response) => {
const data = response.data
console.debug('response data', data)
diff --git a/dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue b/dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue
index 804c81f93b..e9e8bf9993 100644
--- a/dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue
+++ b/dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue
@@ -188,9 +188,9 @@ export default {
uploadAndImport () {
this.loading = true
MiddlewareService.upload(this.fileModel)
- .then((file) => {
- console.debug('uploaded file', file)
- this.tableImport.location = file.path
+ .then((metadata) => {
+ console.debug('uploaded file', metadata)
+ this.tableImport.location = metadata.originalname
QueryService.importCsv(this.$route.params.database_id, this.$route.params.table_id, this.tableImport)
.then((metadata) => {
console.debug('successfully imported data', metadata)
--
GitLab