diff --git a/.docs/api/data-service.md b/.docs/api/data-service.md
index 64f03976132352cea195c6c231e3cc16ebc3d54b..afaa77215d6117afab48b4f19ebb9b1fb4068460 100644
--- a/.docs/api/data-service.md
+++ b/.docs/api/data-service.md
@@ -28,8 +28,17 @@ The Data Service is responsible for inserting AMQP tuples from the Broker Servic
via [Spring AMQP](https://docs.spring.io/spring-amqp/reference/html/). To increase the number of consumers, scale the
Data Service up.
+## Data Processing
+
+The Data Service uses [Apache Spark](https://spark.apache.org/), a data engine to load data from/into
+the [Data Database](../data-db) with a wide range of open-source connectors. The default deployment uses a local mode of
+embedded processing directly in the service until there exists
+a [Bitnami Chart](https://artifacthub.io/packages/helm/bitnami/spark) for Spark 4.
+
## Limitations
+* Currently only local processing (slow) embedded in the service using a two-thread `local[2]` configuration.
+
!!! question "Do you miss functionality? Do these limitations affect you?"
We strongly encourage you to help us implement it as we are welcoming contributors to open-source software and get
diff --git a/.docs/api/ui.md b/.docs/api/ui.md
index 08e21b1c26e8ad6a25ca548de475c76ae82e5dc8..30b32c0a0ccde771c8bde22f21ca630d6354a9b3 100644
--- a/.docs/api/ui.md
+++ b/.docs/api/ui.md
@@ -101,7 +101,8 @@ See the [API Overview](..) page for detailed examples.
## Limitations
-(none)
+* When developing locally, the `axios` module does not parse custom headers (such as `X-Count`, `X-Headers`) and/or
+ blocks CORS requests wrongfully.
!!! question "Do you miss functionality? Do these limitations affect you?"
diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java
index a3df64a1337e838badb234ce3d52a11550afbe27..55f6013479353b5c96f80cec1c8d2623a23f18e9 100644
--- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java
+++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java
@@ -1,5 +1,8 @@
package at.tuwien.endpoints;
+import at.tuwien.ExportResourceDto;
+import at.tuwien.api.database.ViewColumnDto;
+import at.tuwien.api.database.ViewDto;
import at.tuwien.api.database.internal.PrivilegedDatabaseDto;
import at.tuwien.api.database.query.ExecuteStatementDto;
import at.tuwien.api.database.query.QueryDto;
@@ -7,11 +10,10 @@ import at.tuwien.api.database.query.QueryPersistDto;
import at.tuwien.api.error.ApiErrorDto;
import at.tuwien.exception.*;
import at.tuwien.gateway.MetadataServiceGateway;
-import at.tuwien.mapper.MariaDbMapper;
+import at.tuwien.mapper.MetadataMapper;
+import at.tuwien.service.SchemaService;
import at.tuwien.service.StorageService;
import at.tuwien.service.SubsetService;
-import at.tuwien.service.TableService;
-import at.tuwien.service.ViewService;
import at.tuwien.utils.UserUtil;
import at.tuwien.validation.EndpointValidator;
import io.micrometer.observation.annotation.Observed;
@@ -27,6 +29,8 @@ import jakarta.servlet.http.HttpServletRequest;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotNull;
import lombok.extern.log4j.Log4j2;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
@@ -47,22 +51,20 @@ import java.util.UUID;
@RequestMapping(path = "/api/database/{databaseId}/subset")
public class SubsetEndpoint extends AbstractEndpoint {
- private final ViewService viewService;
- private final TableService tableService;
- private final MariaDbMapper mariaDbMapper;
+ private final SchemaService schemaService;
private final SubsetService subsetService;
+ private final MetadataMapper metadataMapper;
private final StorageService storageService;
private final EndpointValidator endpointValidator;
private final MetadataServiceGateway metadataServiceGateway;
@Autowired
- public SubsetEndpoint(ViewService viewService, TableService tableService, MariaDbMapper mariaDbMapper,
- SubsetService queryService, StorageService storageService, EndpointValidator endpointValidator,
+ public SubsetEndpoint(SchemaService schemaService, SubsetService subsetService, MetadataMapper metadataMapper,
+ StorageService storageService, EndpointValidator endpointValidator,
MetadataServiceGateway metadataServiceGateway) {
- this.viewService = viewService;
- this.tableService = tableService;
- this.mariaDbMapper = mariaDbMapper;
- this.subsetService = queryService;
+ this.schemaService = schemaService;
+ this.subsetService = subsetService;
+ this.metadataMapper = metadataMapper;
this.storageService = storageService;
this.endpointValidator = endpointValidator;
this.metadataServiceGateway = metadataServiceGateway;
@@ -156,14 +158,15 @@ public class SubsetEndpoint extends AbstractEndpoint {
@RequestParam(required = false) Instant timestamp)
throws DatabaseUnavailableException, DatabaseNotFoundException, RemoteUnavailableException,
QueryNotFoundException, FormatNotAvailableException, StorageUnavailableException, UserNotFoundException,
- MetadataServiceException {
+ MetadataServiceException, TableNotFoundException, ViewMalformedException, SQLException,
+ QueryMalformedException {
String accept = httpServletRequest.getHeader("Accept");
log.debug("endpoint find subset in database, databaseId={}, subsetId={}, accept={}, timestamp={}", databaseId,
subsetId, accept, timestamp);
final PrivilegedDatabaseDto database = metadataServiceGateway.getDatabaseById(databaseId);
- final QueryDto query;
+ final QueryDto subset;
try {
- query = subsetService.findById(database, subsetId);
+ subset = subsetService.findById(database, subsetId);
} catch (SQLException e) {
log.error("Failed to establish connection to database: {}", e.getMessage());
throw new DatabaseUnavailableException("Failed to establish connection to database: " + e.getMessage(), e);
@@ -180,17 +183,16 @@ public class SubsetEndpoint extends AbstractEndpoint {
switch (accept) {
case MediaType.APPLICATION_JSON_VALUE:
log.trace("accept header matches json");
- return ResponseEntity.ok(query);
+ return ResponseEntity.ok(subset);
case "text/csv":
log.trace("accept header matches csv");
-// final ExportResourceDto resource = storageService.transformDataset(subsetService.reExecute(database, query, null, null));
+ final ExportResourceDto resource = storageService.transformDataset(subsetService.getData(database, subset, null, null));
final HttpHeaders headers = new HttpHeaders();
-// headers.add("Content-Disposition", "attachment; filename=\"" + resource.getFilename() + "\"");
-// log.trace("export table resulted in resource {}", resource);
+ headers.add("Content-Disposition", "attachment; filename=\"" + resource.getFilename() + "\"");
+ log.trace("export table resulted in resource {}", resource);
return ResponseEntity.ok()
.headers(headers)
- .build();
-// .body(resource.getResource());
+ .body(resource.getResource());
}
throw new FormatNotAvailableException("Must provide either application/json or text/csv headers");
}
@@ -241,13 +243,14 @@ public class SubsetEndpoint extends AbstractEndpoint {
@Valid @RequestBody ExecuteStatementDto data,
Principal principal,
@NotNull HttpServletRequest request,
+ @RequestParam(required = false) Instant timestamp,
@RequestParam(required = false) Long page,
- @RequestParam(required = false) Long size,
- @RequestParam(required = false) Instant timestamp)
+ @RequestParam(required = false) Long size)
throws DatabaseUnavailableException, DatabaseNotFoundException, RemoteUnavailableException,
QueryNotFoundException, StorageUnavailableException, QueryMalformedException, StorageNotFoundException,
QueryStoreInsertException, TableMalformedException, PaginationException, QueryNotSupportedException,
- NotAllowedException, UserNotFoundException, MetadataServiceException, TableNotFoundException, ViewMalformedException {
+ NotAllowedException, UserNotFoundException, MetadataServiceException, TableNotFoundException,
+ ViewMalformedException, ViewNotFoundException {
log.debug("endpoint create subset in database, databaseId={}, data.statement={}, page={}, size={}, " +
"timestamp={}", databaseId, data.getStatement(), page, size,
timestamp);
@@ -287,8 +290,10 @@ public class SubsetEndpoint extends AbstractEndpoint {
@ApiResponses(value = {
@ApiResponse(responseCode = "200",
description = "Retrieved subset data",
- headers = {@Header(name = "X-Count", description = "Number of rows", schema = @Schema(implementation = Long.class), required = true),
- @Header(name = "Access-Control-Expose-Headers", description = "Expose `X-Count` custom header", schema = @Schema(implementation = String.class), required = true)},
+ headers = {@Header(name = "X-Count", description = "Number of rows", schema = @Schema(implementation = Long.class)),
+ @Header(name = "X-Headers", description = "The list of headers separated by comma", schema = @Schema(implementation = String.class)),
+ @Header(name = "X-Id", description = "The subset id", schema = @Schema(implementation = Long.class), required = true),
+ @Header(name = "Access-Control-Expose-Headers", description = "Reverse proxy exposing of custom headers", schema = @Schema(implementation = String.class), required = true)},
content = {@Content(
mediaType = "application/json",
schema = @Schema(implementation = List.class))}),
@@ -321,8 +326,8 @@ public class SubsetEndpoint extends AbstractEndpoint {
@RequestParam(required = false) Long size)
throws PaginationException, DatabaseNotFoundException, RemoteUnavailableException, NotAllowedException,
QueryNotFoundException, DatabaseUnavailableException, TableMalformedException, QueryMalformedException,
- UserNotFoundException, MetadataServiceException {
- log.debug("endpoint re-execute query, databaseId={}, subsetId={}, principal.name={} page={}, size={}",
+ UserNotFoundException, MetadataServiceException, TableNotFoundException, ViewMalformedException, ViewNotFoundException {
+ log.debug("endpoint get subset data, databaseId={}, subsetId={}, principal.name={} page={}, size={}",
databaseId, subsetId, principal != null ? principal.getName() : null, page, size);
endpointValidator.validateDataParams(page, size);
final PrivilegedDatabaseDto database = metadataServiceGateway.getDatabaseById(databaseId);
@@ -343,18 +348,24 @@ public class SubsetEndpoint extends AbstractEndpoint {
log.debug("size not set: default to {}", size);
}
try {
- final QueryDto query = subsetService.findById(database, subsetId);
+ final HttpHeaders headers = new HttpHeaders();
+ headers.set("X-Id", "" + subsetId);
+ final QueryDto subset = subsetService.findById(database, subsetId);
if (request.getMethod().equals("HEAD")) {
- final HttpHeaders headers = new HttpHeaders();
- headers.set("Access-Control-Expose-Headers", "X-Count");
- final Long count = subsetService.reExecuteCount(database, query);
+ headers.set("Access-Control-Expose-Headers", "X-Count X-Id");
+ final Long count = subsetService.reExecuteCount(database, subset);
headers.set("X-Count", "" + count);
return ResponseEntity.ok()
.headers(headers)
.build();
}
+ final Dataset<Row> dataset = subsetService.getData(database, subset, page, size);
+ final ViewDto view = schemaService.inspectView(database, metadataMapper.queryDtoToViewName(subset));
+ headers.set("Access-Control-Expose-Headers", "X-Id X-Headers");
+ headers.set("X-Headers", String.join(",", view.getColumns().stream().map(ViewColumnDto::getInternalName).toList()));
return ResponseEntity.ok()
- .body(transform(subsetService.reExecute(database, query, page, size, null, null)));
+ .headers(headers)
+ .body(transform(dataset));
} catch (SQLException e) {
log.error("Failed to establish connection to database: {}", e.getMessage());
throw new DatabaseUnavailableException("Failed to establish connection to database: " + e.getMessage(), e);
diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
index 8e1c848a91f123c0580ce97262849a9ce22094ca..6745db81b607be8a0e4b5c1fc47b443b7ffd1914 100644
--- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
+++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
@@ -6,15 +6,14 @@ import at.tuwien.api.database.DatabaseDto;
import at.tuwien.api.database.internal.PrivilegedDatabaseDto;
import at.tuwien.api.database.query.ImportDto;
import at.tuwien.api.database.table.*;
+import at.tuwien.api.database.table.columns.ColumnDto;
import at.tuwien.api.database.table.internal.PrivilegedTableDto;
import at.tuwien.api.database.table.internal.TableCreateDto;
import at.tuwien.api.error.ApiErrorDto;
import at.tuwien.exception.*;
import at.tuwien.gateway.MetadataServiceGateway;
-import at.tuwien.mapper.MariaDbMapper;
import at.tuwien.service.SchemaService;
import at.tuwien.service.StorageService;
-import at.tuwien.service.SubsetService;
import at.tuwien.service.TableService;
import at.tuwien.utils.UserUtil;
import at.tuwien.validation.EndpointValidator;
@@ -54,21 +53,16 @@ import java.util.Map;
public class TableEndpoint extends AbstractEndpoint {
private final TableService tableService;
- private final MariaDbMapper mariaDbMapper;
private final SchemaService schemaService;
- private final SubsetService subsetService;
private final StorageService storageService;
private final EndpointValidator endpointValidator;
private final MetadataServiceGateway metadataServiceGateway;
@Autowired
- public TableEndpoint(TableService tableService, MariaDbMapper mariaDbMapper, SchemaService schemaService,
- SubsetService subsetService, StorageService storageService,
+ public TableEndpoint(TableService tableService, SchemaService schemaService, StorageService storageService,
EndpointValidator endpointValidator, MetadataServiceGateway metadataServiceGateway) {
this.tableService = tableService;
- this.mariaDbMapper = mariaDbMapper;
this.schemaService = schemaService;
- this.subsetService = subsetService;
this.storageService = storageService;
this.endpointValidator = endpointValidator;
this.metadataServiceGateway = metadataServiceGateway;
@@ -213,9 +207,8 @@ public class TableEndpoint extends AbstractEndpoint {
@NotNull HttpServletRequest request,
Principal principal)
throws DatabaseUnavailableException, RemoteUnavailableException, TableNotFoundException,
- TableMalformedException, PaginationException, QueryMalformedException, MetadataServiceException,
- NotAllowedException {
- log.debug("endpoint find table data, databaseId={}, tableId={}, timestamp={}, page={}, size={}", databaseId,
+ PaginationException, QueryMalformedException, MetadataServiceException, NotAllowedException {
+ log.debug("endpoint get table data, databaseId={}, tableId={}, timestamp={}, page={}, size={}", databaseId,
tableId, timestamp, page, size);
endpointValidator.validateDataParams(page, size);
/* parameters */
@@ -240,18 +233,20 @@ public class TableEndpoint extends AbstractEndpoint {
metadataServiceGateway.getAccess(databaseId, UserUtil.getId(principal));
}
try {
+ final HttpHeaders headers = new HttpHeaders();
if (request.getMethod().equals("HEAD")) {
- final HttpHeaders headers = new HttpHeaders();
headers.set("Access-Control-Expose-Headers", "X-Count");
headers.set("X-Count", "" + tableService.getCount(table, timestamp));
return ResponseEntity.ok()
.headers(headers)
.build();
}
- final Dataset<Row> dataset = tableService.getData(table.getDatabase(), table.getInternalName(), timestamp,
- null, null, null, null);
+ headers.set("Access-Control-Expose-Headers", "X-Headers");
+ headers.set("X-Headers", String.join(",", table.getColumns().stream().map(ColumnDto::getInternalName).toList()));
return ResponseEntity.ok()
- .body(transform(dataset));
+ .headers(headers)
+ .body(transform(tableService.getData(table.getDatabase(), table.getInternalName(), timestamp,
+ null, null, null, null)));
} catch (SQLException e) {
log.error("Failed to establish connection to database: {}", e.getMessage());
throw new DatabaseUnavailableException("Failed to establish connection to database: " + e.getMessage(), e);
diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java
index 07249338774e592e7799b6fee1d70c40895e9929..001350246e68e7e75bbfc781695b284138c01ace 100644
--- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java
+++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java
@@ -9,9 +9,7 @@ import at.tuwien.api.database.internal.PrivilegedViewDto;
import at.tuwien.api.error.ApiErrorDto;
import at.tuwien.exception.*;
import at.tuwien.gateway.MetadataServiceGateway;
-import at.tuwien.mapper.MariaDbMapper;
import at.tuwien.service.StorageService;
-import at.tuwien.service.SubsetService;
import at.tuwien.service.TableService;
import at.tuwien.service.ViewService;
import at.tuwien.utils.UserUtil;
@@ -50,20 +48,15 @@ public class ViewEndpoint extends AbstractEndpoint {
private final ViewService viewService;
private final TableService tableService;
- private final MariaDbMapper mariaDbMapper;
- private final SubsetService subsetService;
private final StorageService storageService;
private final EndpointValidator endpointValidator;
private final MetadataServiceGateway metadataServiceGateway;
@Autowired
- public ViewEndpoint(ViewService viewService, TableService tableService, MariaDbMapper mariaDbMapper,
- SubsetService subsetService, StorageService storageService, EndpointValidator endpointValidator,
- MetadataServiceGateway metadataServiceGateway) {
+ public ViewEndpoint(ViewService viewService, TableService tableService, StorageService storageService,
+ EndpointValidator endpointValidator, MetadataServiceGateway metadataServiceGateway) {
this.viewService = viewService;
this.tableService = tableService;
- this.mariaDbMapper = mariaDbMapper;
- this.subsetService = subsetService;
this.storageService = storageService;
this.endpointValidator = endpointValidator;
this.metadataServiceGateway = metadataServiceGateway;
@@ -280,19 +273,18 @@ public class ViewEndpoint extends AbstractEndpoint {
metadataServiceGateway.getAccess(databaseId, UserUtil.getId(principal));
}
try {
+ final HttpHeaders headers = new HttpHeaders();
if (request.getMethod().equals("HEAD")) {
- final HttpHeaders headers = new HttpHeaders();
headers.set("Access-Control-Expose-Headers", "X-Count");
headers.set("X-Count", "" + viewService.count(view, timestamp));
return ResponseEntity.ok()
.headers(headers)
.build();
}
- final List<String> columns = view.getColumns()
- .stream()
- .map(ViewColumnDto::getInternalName)
- .toList();
+ headers.set("Access-Control-Expose-Headers", "X-Headers");
+ headers.set("X-Headers", String.join(",", view.getColumns().stream().map(ViewColumnDto::getInternalName).toList()));
return ResponseEntity.ok()
+ .headers(headers)
.body(transform(tableService.getData(view.getDatabase(), view.getInternalName(), timestamp, page,
size, null, null)));
} catch (SQLException e) {
diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java
index 0d3fee5723ac101ca36a8db3b1c34f42bb034493..e7aeaa48d7cde9173650af9f84a4980540ba11c7 100644
--- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java
+++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java
@@ -399,9 +399,10 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
// }
@Test
- public void getData_head_succeeds() throws DatabaseNotFoundException, RemoteUnavailableException, UserNotFoundException,
- NotAllowedException, SQLException, QueryNotFoundException, TableMalformedException, QueryMalformedException,
- DatabaseUnavailableException, PaginationException, MetadataServiceException, TableNotFoundException, ViewMalformedException {
+ public void getData_head_succeeds() throws DatabaseNotFoundException, RemoteUnavailableException,
+ UserNotFoundException, NotAllowedException, SQLException, QueryNotFoundException, TableMalformedException,
+ QueryMalformedException, DatabaseUnavailableException, PaginationException, MetadataServiceException,
+ TableNotFoundException, ViewMalformedException, ViewNotFoundException {
/* mock */
when(metadataServiceGateway.getDatabaseById(DATABASE_3_ID))
@@ -483,7 +484,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
public void getData_privateHead_succeeds() throws DatabaseNotFoundException, RemoteUnavailableException,
UserNotFoundException, DatabaseUnavailableException, NotAllowedException, TableMalformedException,
QueryMalformedException, QueryNotFoundException, PaginationException, SQLException,
- MetadataServiceException, TableNotFoundException, ViewMalformedException {
+ MetadataServiceException, TableNotFoundException, ViewMalformedException, ViewNotFoundException {
/* mock */
when(metadataServiceGateway.getDatabaseById(DATABASE_1_ID))
diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java
index c11f85be80cdb0981a81251b63ea601ba8ffdab1..01bce16b08f84e7ad3901beb143ef0737d9ad571 100644
--- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java
+++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java
@@ -136,7 +136,7 @@ public class PrometheusEndpointMvcTest extends AbstractUnitTest {
/* ignore */
}
try {
- subsetEndpoint.create(DATABASE_1_ID, ExecuteStatementDto.builder().statement(QUERY_5_STATEMENT).build(), USER_1_PRINCIPAL, httpServletRequest, 0L, 10L, null);
+ subsetEndpoint.create(DATABASE_1_ID, ExecuteStatementDto.builder().statement(QUERY_5_STATEMENT).build(), USER_1_PRINCIPAL, httpServletRequest, null, 0L, 10L);
} catch (Exception e) {
/* ignore */
}
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java
index eaa8a2d6d4338df3fc896262ee4dc4ef46a6800c..99480719fac1157771d81ae09e3c5160f501af59 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java
@@ -421,6 +421,12 @@ public interface MariaDbMapper {
return statement.toString();
}
+ default String selectExistsTableOrViewRawQuery() {
+ final String statement = "SELECT IF((SELECT 1 FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?), 1, 0) AS `exists`";
+ log.trace("mapped select exists table or view statement: {}", statement);
+ return statement;
+ }
+
default Long resultSetToNumber(ResultSet data) throws QueryMalformedException, SQLException {
if (!data.next()) {
throw new QueryMalformedException("Failed to map number");
@@ -428,6 +434,13 @@ public interface MariaDbMapper {
return data.getLong(1);
}
+ default Boolean resultSetToBoolean(ResultSet data) throws QueryMalformedException, SQLException {
+ if (!data.next()) {
+ throw new QueryMalformedException("Failed to map boolean");
+ }
+ return data.getBoolean(1);
+ }
+
default List<Map<String, Object>> resultSetToList(ResultSet data, List<String> columns) throws SQLException {
final List<Map<String, Object>> list = new LinkedList<>();
while (data.next()) {
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MetadataMapper.java b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MetadataMapper.java
index a585f2e98ad8d39b4c0c290ae9c48fc8840f7581..fb59360e9740914c0b1309b2546c3f55b9909c4d 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MetadataMapper.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MetadataMapper.java
@@ -9,6 +9,7 @@ import at.tuwien.api.database.ViewColumnDto;
import at.tuwien.api.database.ViewDto;
import at.tuwien.api.database.internal.PrivilegedDatabaseDto;
import at.tuwien.api.database.internal.PrivilegedViewDto;
+import at.tuwien.api.database.query.QueryDto;
import at.tuwien.api.database.table.TableBriefDto;
import at.tuwien.api.database.table.TableDto;
import at.tuwien.api.database.table.columns.ColumnDto;
@@ -25,6 +26,10 @@ public interface MetadataMapper {
org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(MetadataMapper.class);
+ default String queryDtoToViewName(QueryDto subset) {
+ return subset.getQueryHash();
+ }
+
PrivilegedContainerDto containerDtoToPrivilegedContainerDto(ContainerDto data);
DatabaseDto privilegedDatabaseDtoToDatabaseDto(PrivilegedDatabaseDto data);
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/SubsetService.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/SubsetService.java
index 6fbeca72364601214312b4b8379d0a99c43176ae..4a3455fbc4e10f9632148fbe12e92f4298973f9d 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/SubsetService.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/SubsetService.java
@@ -1,6 +1,5 @@
package at.tuwien.service;
-import at.tuwien.api.SortTypeDto;
import at.tuwien.api.container.internal.PrivilegedContainerDto;
import at.tuwien.api.database.internal.PrivilegedDatabaseDto;
import at.tuwien.api.database.query.QueryDto;
@@ -26,6 +25,22 @@ public interface SubsetService {
void createQueryStore(PrivilegedContainerDto container, String databaseName) throws SQLException,
QueryStoreCreateException;
+ /**
+ * Retrieve data from a subset in a database and optionally paginate with number of page and size of results.
+ *
+ * @param database The database.
+ * @param subset The subset.
+ * @param page The page number.
+ * @param size Te result size.
+ * @return The data.
+ * @throws ViewMalformedException The view is malformed.
+ * @throws SQLException The connection to the database could not be established.
+ * @throws QueryMalformedException The mapped query produced a database error.
+ * @throws TableNotFoundException The database table is malformed.
+ */
+ Dataset<Row> getData(PrivilegedDatabaseDto database, QueryDto subset, Long page, Long size)
+ throws ViewMalformedException, SQLException, QueryMalformedException, TableNotFoundException;
+
/**
* Creates a subset from the given statement at given time in the given database.
*
@@ -40,10 +55,6 @@ public interface SubsetService {
Long create(PrivilegedDatabaseDto database, String statement, Instant timestamp, UUID userId)
throws QueryStoreInsertException, SQLException;
- Dataset<Row> reExecute(PrivilegedDatabaseDto database, QueryDto query, Long page, Long size,
- SortTypeDto sortDirection, String sortColumn) throws TableMalformedException,
- SQLException;
-
/**
* Counts the subset row count of a query of a given subset in the given database.
*
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/TableService.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/TableService.java
index 287399c981be412afbe14c9c686a942877065f2a..dc0b2041f04dad9fb6fb5246b5ac6c098f2a1edf 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/TableService.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/TableService.java
@@ -152,7 +152,7 @@ public interface TableService {
void updateTuple(PrivilegedTableDto table, TupleUpdateDto data) throws SQLException,
QueryMalformedException, TableMalformedException;
- Dataset<Row> getData(PrivilegedDatabaseDto database, String query, Instant timestamp,
+ Dataset<Row> getData(PrivilegedDatabaseDto database, String tableOrView, Instant timestamp,
Long page, Long size, SortTypeDto sortDirection, String sortColumn)
throws QueryMalformedException, TableNotFoundException;
}
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/ViewService.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/ViewService.java
index 3d4174f7e56d2d6cd615c263e88d1f549272d64f..ec7a723261372d1414f0590e74ef915bb62264f4 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/ViewService.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/ViewService.java
@@ -4,6 +4,7 @@ import at.tuwien.api.database.ViewCreateDto;
import at.tuwien.api.database.ViewDto;
import at.tuwien.api.database.internal.PrivilegedDatabaseDto;
import at.tuwien.api.database.internal.PrivilegedViewDto;
+import at.tuwien.api.database.query.QueryDto;
import at.tuwien.exception.DatabaseMalformedException;
import at.tuwien.exception.QueryMalformedException;
import at.tuwien.exception.ViewMalformedException;
@@ -15,6 +16,9 @@ import java.util.List;
public interface ViewService {
+ Boolean existsByName(PrivilegedDatabaseDto database, String name) throws SQLException,
+ QueryMalformedException;
+
/**
* Gets the metadata schema for a given database.
*
@@ -27,6 +31,16 @@ public interface ViewService {
List<ViewDto> getSchemas(PrivilegedDatabaseDto database) throws SQLException, DatabaseMalformedException,
ViewNotFoundException;
+ /**
+ * Creates a view if not already exists.
+ * @param database
+ * @param subset
+ * @return
+ * @throws ViewMalformedException
+ * @throws SQLException
+ */
+ ViewDto create(PrivilegedDatabaseDto database, QueryDto subset) throws ViewMalformedException, SQLException;
+
/**
* Creates a view in the given data database.
*
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/AccessServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/AccessServiceMariaDbImpl.java
index 7ed5150095a1320c29cfd92ff9687eeab8ef57ee..1493f579fdb6cfc9e5429a27ea6e44314a5d8760 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/AccessServiceMariaDbImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/AccessServiceMariaDbImpl.java
@@ -43,22 +43,22 @@ public class AccessServiceMariaDbImpl extends HibernateConnector implements Acce
long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.databaseCreateUserQuery(user.getUsername(), user.getPassword()))
.execute();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
/* grant access */
final String grants = access != AccessTypeDto.READ ? grantDefaultWrite : grantDefaultRead;
start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.databaseGrantPrivilegesQuery(user.getUsername(), grants))
.execute();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
/* grant query store */
start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.databaseGrantProcedureQuery(user.getUsername(), "store_query"))
.execute();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
/* apply access rights */
start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.databaseFlushPrivilegesQuery());
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
@@ -82,7 +82,7 @@ public class AccessServiceMariaDbImpl extends HibernateConnector implements Acce
final long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.databaseGrantPrivilegesQuery(user.getUsername(), grants))
.execute();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
/* apply access rights */
connection.prepareStatement(mariaDbMapper.databaseFlushPrivilegesQuery());
connection.commit();
@@ -106,12 +106,12 @@ public class AccessServiceMariaDbImpl extends HibernateConnector implements Acce
long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.databaseRevokePrivilegesQuery(user.getUsername()))
.execute();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
/* apply access rights */
start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.databaseFlushPrivilegesQuery())
.execute();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceMariaDbImpl.java
index b2a9b0c8408953e0a694a1ae6d3e36c8ebfe485d..db446de2817b6f5499105856d5748202b1469b4b 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceMariaDbImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceMariaDbImpl.java
@@ -40,7 +40,7 @@ public class DatabaseServiceMariaDbImpl extends HibernateConnector implements Da
final long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.databaseCreateDatabaseQuery(data.getInternalName()))
.execute();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
@@ -76,7 +76,7 @@ public class DatabaseServiceMariaDbImpl extends HibernateConnector implements Da
final long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.databaseSetPasswordQuery(data.getUsername(), data.getPassword()))
.execute();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java
index 797de6567445ea6acdc45c656d8910f68084ac8b..aff85137870749a600d1e4c76ef317a978c6c54e 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java
@@ -52,7 +52,7 @@ public class QueueServiceRabbitMqImpl extends HibernateConnector implements Queu
}
final long start = System.currentTimeMillis();
preparedStatement.executeUpdate();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
log.trace("successfully inserted tuple");
amqpDataAccessCounter.increment();
} finally {
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SchemaServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SchemaServiceMariaDbImpl.java
index 1aba3c6b9942386f171c1a4fe029658667a633a1..370d86e2881fbea3abd8254693e80b605d32662e 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SchemaServiceMariaDbImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SchemaServiceMariaDbImpl.java
@@ -55,7 +55,7 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche
statement1.setString(2, tableName);
log.trace("1={}, 2={}", database.getInternalName(), tableName);
TableDto table = dataMapper.schemaResultSetToTable(metadataMapper.privilegedDatabaseDtoToDatabaseDto(database), statement1.executeQuery());
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
/* obtain columns metadata */
start = System.currentTimeMillis();
final PreparedStatement statement2 = connection.prepareStatement(mariaDbMapper.databaseTableColumnsSelectRawQuery());
@@ -63,7 +63,7 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche
statement2.setString(2, tableName);
log.trace("1={}, 2={}", database.getInternalName(), tableName);
final ResultSet resultSet2 = statement2.executeQuery();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
while (resultSet2.next()) {
table = dataMapper.resultSetToTable(resultSet2, table, queryConfig);
}
@@ -74,7 +74,7 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche
statement3.setString(2, tableName);
log.trace("1={}, 2={}", database.getInternalName(), tableName);
final ResultSet resultSet3 = statement3.executeQuery();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
while (resultSet3.next()) {
final String clause = resultSet3.getString(1);
table.getConstraints()
@@ -89,7 +89,7 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche
statement4.setString(2, tableName);
log.trace("1={}, 2={}", database.getInternalName(), tableName);
final ResultSet resultSet4 = statement4.executeQuery();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
while (resultSet4.next()) {
table = dataMapper.resultSetToConstraint(resultSet4, table);
for (UniqueDto uk : table.getConstraints().getUniques()) {
@@ -136,7 +136,7 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche
statement1.setString(2, viewName);
log.trace("1={}, 2={}", database.getInternalName(), viewName);
final ResultSet resultSet1 = statement1.executeQuery();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
if (!resultSet1.next()) {
throw new ViewNotFoundException("Failed to find view in the information schema");
}
@@ -152,7 +152,7 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche
statement2.setString(2, viewName);
log.trace("1={}, 2={}", database.getInternalName(), viewName);
final ResultSet resultSet2 = statement2.executeQuery();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
TableDto tmp = TableDto.builder()
.columns(new LinkedList<>())
.build();
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java
index 3eb0a7e244a4eaad8cbf6fa1cfc255a38204251a..1e1e78603ba219e0f929a65e40efad6557f6f1ae 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java
@@ -1,23 +1,22 @@
package at.tuwien.service.impl;
-import at.tuwien.api.SortTypeDto;
import at.tuwien.api.container.internal.PrivilegedContainerDto;
import at.tuwien.api.database.internal.PrivilegedDatabaseDto;
import at.tuwien.api.database.query.QueryDto;
-import at.tuwien.api.database.table.columns.ColumnDto;
import at.tuwien.api.identifier.IdentifierDto;
import at.tuwien.api.identifier.IdentifierTypeDto;
import at.tuwien.exception.*;
import at.tuwien.gateway.MetadataServiceGateway;
import at.tuwien.mapper.DataMapper;
import at.tuwien.mapper.MariaDbMapper;
+import at.tuwien.mapper.MetadataMapper;
import at.tuwien.service.SubsetService;
+import at.tuwien.service.TableService;
+import at.tuwien.service.ViewService;
import com.mchange.v2.c3p0.ComboPooledDataSource;
import lombok.extern.log4j.Log4j2;
-import net.sf.jsqlparser.JSQLParserException;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SparkSession;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -32,16 +31,21 @@ import java.util.UUID;
public class SubsetServiceMariaDbImpl extends HibernateConnector implements SubsetService {
private final DataMapper dataMapper;
- private final SparkSession sparkSession;
+ private final ViewService viewService;
+ private final TableService tableService;
private final MariaDbMapper mariaDbMapper;
+ private final MetadataMapper metadataMapper;
private final MetadataServiceGateway metadataServiceGateway;
@Autowired
- public SubsetServiceMariaDbImpl(DataMapper dataMapper, SparkSession sparkSession, MariaDbMapper mariaDbMapper,
+ public SubsetServiceMariaDbImpl(DataMapper dataMapper, ViewService viewService, TableService tableService,
+ MariaDbMapper mariaDbMapper, MetadataMapper metadataMapper,
MetadataServiceGateway metadataServiceGateway) {
this.dataMapper = dataMapper;
- this.sparkSession = sparkSession;
+ this.viewService = viewService;
+ this.tableService = tableService;
this.mariaDbMapper = mariaDbMapper;
+ this.metadataMapper = metadataMapper;
this.metadataServiceGateway = metadataServiceGateway;
}
@@ -55,23 +59,23 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.queryStoreCreateSequenceRawQuery())
.execute();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.queryStoreCreateTableRawQuery())
.execute();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.queryStoreCreateHashTableProcedureRawQuery())
.execute();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.queryStoreCreateStoreQueryProcedureRawQuery())
.execute();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.queryStoreCreateInternalStoreQueryProcedureRawQuery())
.execute();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
@@ -84,33 +88,21 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
}
@Override
- public Long create(PrivilegedDatabaseDto database, String statement, Instant timestamp, UUID userId)
- throws QueryStoreInsertException, SQLException {
- return storeQuery(database, statement, timestamp, userId);
+ public Dataset<Row> getData(PrivilegedDatabaseDto database, QueryDto subset, Long page, Long size)
+ throws ViewMalformedException, SQLException, QueryMalformedException, TableNotFoundException {
+ if (!viewService.existsByName(database, metadataMapper.queryDtoToViewName(subset))) {
+ log.warn("Missing internal view {} for subset with id {}: create it from subset query", metadataMapper.queryDtoToViewName(subset), subset.getId());
+ viewService.create(database, subset);
+ } else {
+ log.debug("internal view {} for subset with id {} exists", metadataMapper.queryDtoToViewName(subset), subset.getId());
+ }
+ return tableService.getData(database, metadataMapper.queryDtoToViewName(subset), subset.getExecution(), page, size, null, null);
}
@Override
- public Dataset<Row> reExecute(PrivilegedDatabaseDto database, QueryDto query, Long page, Long size,
- SortTypeDto sortDirection, String sortColumn)
- throws TableMalformedException, SQLException {
- final ComboPooledDataSource dataSource = getPrivilegedDataSource(database);
- final Connection connection = dataSource.getConnection();
- try {
- final long start = System.currentTimeMillis();
- final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.selectRawSelectQuery(query.getQuery(), query.getExecution(), page, size))
- .executeQuery();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
- final List<String> columns = dataMapper.parseColumns(database.getId(), database.getTables(), query.getQuery())
- .stream()
- .map(ColumnDto::getInternalName)
- .toList();
- return sparkSession.createDataFrame(mariaDbMapper.resultSetToList(resultSet, columns), Row.class);
- } catch (SQLException | JSQLParserException e) {
- log.error("Failed to map object: {}", e.getMessage());
- throw new TableMalformedException("Failed to map object: " + e.getMessage(), e);
- } finally {
- dataSource.close();
- }
+ public Long create(PrivilegedDatabaseDto database, String statement, Instant timestamp, UUID userId)
+ throws QueryStoreInsertException, SQLException {
+ return storeQuery(database, statement, timestamp, userId);
}
@Override
@@ -133,7 +125,7 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
log.trace("filter persisted only {}", filterPersisted);
}
final ResultSet resultSet = statement.executeQuery();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
final List<QueryDto> queries = new LinkedList<>();
while (resultSet.next()) {
final QueryDto query = dataMapper.resultSetToQueryDto(resultSet);
@@ -162,7 +154,7 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
final long start = System.currentTimeMillis();
final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.countRawSelectQuery(statement, timestamp))
.executeQuery();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
return mariaDbMapper.resultSetToNumber(resultSet);
} catch (SQLException e) {
log.error("Failed to map object: {}", e.getMessage());
@@ -182,7 +174,7 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
final PreparedStatement preparedStatement = connection.prepareStatement(mariaDbMapper.queryStoreFindQueryRawQuery());
preparedStatement.setLong(1, queryId);
final ResultSet resultSet = preparedStatement.executeQuery();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
if (!resultSet.next()) {
throw new QueryNotFoundException("Failed to find query");
}
@@ -219,7 +211,7 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
callableStatement.setTimestamp(3, Timestamp.from(timestamp));
callableStatement.registerOutParameter(4, Types.BIGINT);
callableStatement.executeUpdate();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
queryId = callableStatement.getLong(4);
callableStatement.close();
log.info("Stored query with id {} in database with name {}", queryId, database.getInternalName());
@@ -246,7 +238,7 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
preparedStatement.setBoolean(1, persist);
preparedStatement.setLong(2, queryId);
preparedStatement.executeUpdate();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
} catch (SQLException e) {
log.error("Failed to (un-)persist query: {}", e.getMessage());
throw new QueryStorePersistException("Failed to (un-)persist query", e);
@@ -264,7 +256,7 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
final long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.queryStoreDeleteStaleQueriesRawQuery())
.executeUpdate();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
} catch (SQLException e) {
log.error("Failed to delete stale queries: {}", e.getMessage());
throw new QueryStoreGCException("Failed to delete stale queries: " + e.getMessage(), e);
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java
index 957c6cdbd5a8d27bb1e923fbb7e1c2f18065cbe9..4e7aa7dcfafb614e05928749f08a699d0ba34942 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java
@@ -62,7 +62,7 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
final PreparedStatement statement = connection.prepareStatement(mariaDbMapper.databaseTablesSelectRawQuery());
statement.setString(1, database.getInternalName());
final ResultSet resultSet1 = statement.executeQuery();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
while (resultSet1.next()) {
final String tableName = resultSet1.getString(1);
if (database.getTables().stream().anyMatch(t -> t.getInternalName().equals(tableName))) {
@@ -100,7 +100,7 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
} else {
final ResultSet resultSet = connection.prepareStatement(query)
.executeQuery();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
statistic = dataMapper.resultSetToTableStatistic(resultSet);
final TableDto tmpTable = schemaService.inspectTable(table.getDatabase(), table.getInternalName());
statistic.setAvgRowLength(tmpTable.getAvgRowLength());
@@ -141,7 +141,7 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
final long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.tableCreateDtoToCreateTableRawQuery(data))
.execute();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
@@ -170,7 +170,7 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
final long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.dropTableRawQuery(tableName))
.execute();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
@@ -194,7 +194,7 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.selectHistoryRawQuery(
table.getDatabase().getInternalName(), table.getInternalName(), size))
.executeQuery();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
history = dataMapper.resultSetToTableHistory(resultSet);
connection.commit();
} catch (SQLException e) {
@@ -220,7 +220,7 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.selectCountRawQuery(
table.getDatabase().getInternalName(), table.getInternalName(), timestamp))
.executeQuery();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
queryResult = mariaDbMapper.resultSetToNumber(resultSet);
connection.commit();
} catch (SQLException e) {
@@ -301,7 +301,7 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
}
final long start = System.currentTimeMillis();
statement.executeUpdate();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
@@ -345,7 +345,7 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
}
final long start = System.currentTimeMillis();
statement.executeUpdate();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
@@ -381,7 +381,7 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
}
final long start = System.currentTimeMillis();
statement.executeUpdate();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/ViewServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/ViewServiceMariaDbImpl.java
index c49ad5255d5cfc8c01970759045f94342aa0db7c..b5f61e6c79744e8181f7ab7ab69a162d9d549bde 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/ViewServiceMariaDbImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/ViewServiceMariaDbImpl.java
@@ -4,6 +4,7 @@ import at.tuwien.api.database.ViewCreateDto;
import at.tuwien.api.database.ViewDto;
import at.tuwien.api.database.internal.PrivilegedDatabaseDto;
import at.tuwien.api.database.internal.PrivilegedViewDto;
+import at.tuwien.api.database.query.QueryDto;
import at.tuwien.config.QueryConfig;
import at.tuwien.exception.DatabaseMalformedException;
import at.tuwien.exception.QueryMalformedException;
@@ -12,7 +13,6 @@ import at.tuwien.exception.ViewNotFoundException;
import at.tuwien.mapper.DataMapper;
import at.tuwien.mapper.MariaDbMapper;
import at.tuwien.mapper.MetadataMapper;
-import at.tuwien.service.SchemaService;
import at.tuwien.service.ViewService;
import com.google.common.hash.Hashing;
import com.mchange.v2.c3p0.ComboPooledDataSource;
@@ -36,19 +36,41 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe
private final DataMapper dataMapper;
private final QueryConfig queryConfig;
private final MariaDbMapper mariaDbMapper;
- private final SchemaService schemaService;
private final MetadataMapper metadataMapper;
@Autowired
public ViewServiceMariaDbImpl(DataMapper dataMapper, QueryConfig queryConfig, MariaDbMapper mariaDbMapper,
- SchemaService schemaService, MetadataMapper metadataMapper) {
+ MetadataMapper metadataMapper) {
this.dataMapper = dataMapper;
this.queryConfig = queryConfig;
this.mariaDbMapper = mariaDbMapper;
- this.schemaService = schemaService;
this.metadataMapper = metadataMapper;
}
+ @Override
+ public Boolean existsByName(PrivilegedDatabaseDto database, String name) throws SQLException,
+ QueryMalformedException {
+ final ComboPooledDataSource dataSource = getPrivilegedDataSource(database);
+ final Connection connection = dataSource.getConnection();
+ final Boolean queryResult;
+ try {
+ /* find view data */
+ final long start = System.currentTimeMillis();
+ final PreparedStatement statement = connection.prepareStatement(mariaDbMapper.selectExistsTableOrViewRawQuery());
+ statement.setString(1, database.getInternalName());
+ statement.setString(2, name);
+ final ResultSet resultSet = statement.executeQuery();
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
+ queryResult = mariaDbMapper.resultSetToBoolean(resultSet);
+ } catch (SQLException e) {
+ log.error("Failed to prepare statement {}", e.getMessage());
+ throw new QueryMalformedException("Failed to prepare statement: " + e.getMessage(), e);
+ } finally {
+ dataSource.close();
+ }
+ return queryResult;
+ }
+
@Override
public List<ViewDto> getSchemas(PrivilegedDatabaseDto database) throws SQLException, DatabaseMalformedException,
ViewNotFoundException {
@@ -61,18 +83,14 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe
statement.setString(1, database.getInternalName());
final long start = System.currentTimeMillis();
final ResultSet resultSet1 = statement.executeQuery();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
while (resultSet1.next()) {
final String viewName = resultSet1.getString(1);
if (database.getViews().stream().anyMatch(v -> v.getInternalName().equals(viewName))) {
log.trace("view {}.{} already known to metadata database, skip.", database.getInternalName(), viewName);
continue;
}
- final ViewDto view;
- view = schemaService.inspectView(database, viewName);
- if (database.getTables().stream().noneMatch(t -> t.getInternalName().equals(view.getInternalName()))) {
- views.add(view);
- }
+
}
} catch (SQLException e) {
log.error("Failed to get view schemas: {}", e.getMessage());
@@ -84,6 +102,17 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe
return views;
}
+ @Override
+ public ViewDto create(PrivilegedDatabaseDto database, QueryDto subset) throws ViewMalformedException,
+ SQLException {
+ final ViewCreateDto data = ViewCreateDto.builder()
+ .name(metadataMapper.queryDtoToViewName(subset))
+ .query(subset.getQuery())
+ .isPublic(false)
+ .build();
+ return create(database, data);
+ }
+
@Override
public ViewDto create(PrivilegedDatabaseDto database, ViewCreateDto data) throws SQLException,
ViewMalformedException {
@@ -110,7 +139,7 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe
final long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.viewCreateRawQuery(view.getInternalName(), data.getQuery()))
.execute();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
/* select view columns */
final PreparedStatement statement2 = connection.prepareStatement(mariaDbMapper.databaseTableColumnsSelectRawQuery());
statement2.setString(1, database.getInternalName());
@@ -140,7 +169,7 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe
final long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.dropViewRawQuery(viewName))
.execute();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
@@ -165,7 +194,7 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe
final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.selectCountRawQuery(
view.getDatabase().getInternalName(), view.getInternalName(), timestamp))
.executeQuery();
- log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
queryResult = mariaDbMapper.resultSetToNumber(resultSet);
connection.commit();
} catch (SQLException e) {
diff --git a/dbrepo-ui/components/subset/Results.vue b/dbrepo-ui/components/subset/Results.vue
index 4ba414309c5009bdf88914150b6c1afe11e9a65e..ba063dcf5d575071d851da73bfcd0b9da80b5fed 100644
--- a/dbrepo-ui/components/subset/Results.vue
+++ b/dbrepo-ui/components/subset/Results.vue
@@ -171,10 +171,10 @@ export default {
}
},
mapResults (data) {
- this.result.headers = data.headers.map((h) => {
+ this.result.headers = data.headers.map((header) => {
return {
- title: Object.keys(h)[0],
- value: Object.keys(h)[0],
+ title: header,
+ value: header,
sortable: false
}
})
diff --git a/dbrepo-ui/composables/query-service.ts b/dbrepo-ui/composables/query-service.ts
index b3c21c605344db722259df375d7eba98404c65de..e7bb860f658ab017a7b9f0e38a94b4bf17167631 100644
--- a/dbrepo-ui/composables/query-service.ts
+++ b/dbrepo-ui/composables/query-service.ts
@@ -84,7 +84,13 @@ export const useQueryService = (): any => {
axios.post<QueryResultDto>(`/api/database/${databaseId}/subset`, data, {params: mapFilter(timestamp, page, size), timeout: 600_000})
.then((response) => {
console.info('Executed query with id', response.data.id, ' in database with id', databaseId)
- resolve(response.data)
+ console.debug('=======>', response)
+ const result: QueryResultDto = {
+ id: 1,
+ headers: [],
+ result: response.data
+ }
+ resolve(result)
})
.catch((error) => {
console.error('Failed to execute query', error)
@@ -100,7 +106,12 @@ export const useQueryService = (): any => {
axios.get<QueryResultDto>(`/api/database/${databaseId}/subset/${queryId}/data`, { params: mapFilter(null, page, size) })
.then((response) => {
console.info('Re-executed query in database with id', databaseId)
- resolve(response.data)
+ const result: QueryResultDto = {
+ id: Number(response.headers['x-id']),
+ headers: response.headers['x-headers'] ? response.headers['x-headers'].split(',') : [],
+ result: response.data
+ }
+ resolve(result)
})
.catch((error) => {
console.error('Failed to re-execute query', error)
diff --git a/dbrepo-ui/composables/table-service.ts b/dbrepo-ui/composables/table-service.ts
index 3d87e68d4febed8b825a56b1aada946bd6b0f4d5..102d5dfe58d24d937165b6524070f8b3eef1f3f2 100644
--- a/dbrepo-ui/composables/table-service.ts
+++ b/dbrepo-ui/composables/table-service.ts
@@ -74,7 +74,12 @@ export const useTableService = (): any => {
axios.get<QueryResultDto>(`/api/database/${databaseId}/table/${tableId}/data`, { params: mapFilter(timestamp, page, size) })
.then((response) => {
console.info('Got data for table with id', tableId, 'in database with id', databaseId)
- resolve(response.data)
+ const result: QueryResultDto = {
+ id: tableId,
+ headers: response.headers['x-headers'] ? response.headers['x-headers'].split(',') : [],
+ result: response.data
+ }
+ resolve(result)
})
.catch((error) => {
console.error('Failed to get data', error)
diff --git a/dbrepo-ui/composables/view-service.ts b/dbrepo-ui/composables/view-service.ts
index 5b3a25a149813ddf30f622fcb3d51fccb31f6730..0c17c353ef4b8faed5b28a5ee367806911de7db4 100644
--- a/dbrepo-ui/composables/view-service.ts
+++ b/dbrepo-ui/composables/view-service.ts
@@ -41,7 +41,12 @@ export const useViewService = (): any => {
axios.get<QueryResultDto>(`/api/database/${databaseId}/view/${viewId}/data`, { params: {page, size} })
.then((response) => {
console.info('Re-executed view with id', viewId, 'in database with id', databaseId)
- resolve(response.data)
+ const result: QueryResultDto = {
+ id: viewId,
+ headers: response.headers['x-headers'] ? response.headers['x-headers'].split(',') : [],
+ result: response.data
+ }
+ resolve(result)
})
.catch((error) => {
console.error('Failed to re-execute view', error)
diff --git a/dbrepo-ui/dto/index.ts b/dbrepo-ui/dto/index.ts
index ba4c41304209a08200b99de17077eed49d63e357..c068862f67ed7a64062fd5d8bd7e958b24dc12aa 100644
--- a/dbrepo-ui/dto/index.ts
+++ b/dbrepo-ui/dto/index.ts
@@ -537,7 +537,7 @@ interface ImportCsv {
interface QueryResultDto {
id: number | null;
result: any;
- headers: any;
+ headers: string[];
}
interface TableHistoryDto {
diff --git a/dbrepo-ui/locales/en-US.json b/dbrepo-ui/locales/en-US.json
index 8b6807bdbf24edd94b08087ef538d499e47784eb..3317a3ff2a02027d30da4efe416deaed97183cfc 100644
--- a/dbrepo-ui/locales/en-US.json
+++ b/dbrepo-ui/locales/en-US.json
@@ -312,8 +312,8 @@
"size": {
"title": "Size"
},
- "result-rows": {
- "title": "Rows"
+ "rows": {
+ "title": "Result Rows"
},
"owner": {
"title": "Owner"
diff --git a/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/data.vue b/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/data.vue
index d4d707fab871efd1773884d8eb5b23f720fba410..3ac8f40d645b44449f5607b9f3b031d06fa463d6 100644
--- a/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/data.vue
+++ b/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/data.vue
@@ -148,7 +148,7 @@ export default {
const url = URL.createObjectURL(data)
const link = document.createElement('a')
link.href = url
- link.download = 'table.csv'
+ link.download = 'subset.csv'
document.body.appendChild(link)
link.click()
})
diff --git a/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/info.vue b/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/info.vue
index 0d59b1ed25e405db59c50e3f492d9122f9fbf5ae..b9ed74a04287e3f8ecb76b88309399b31d234cd4 100644
--- a/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/info.vue
+++ b/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/info.vue
@@ -66,7 +66,7 @@
<pre>{{ $t('pages.subset.hash.prefix') }}:{{ result_hash }}</pre>
</v-list-item>
<v-list-item
- :title="$t('pages.subset.result-rows.title')"
+ :title="$t('pages.subset.rows.title')"
density="compact">
{{ subset.result_number }}
</v-list-item>
diff --git a/dbrepo-ui/pages/database/[database_id]/table/[table_id]/info.vue b/dbrepo-ui/pages/database/[database_id]/table/[table_id]/info.vue
index e109a7db3e2bb35926c5a8f87d700185286d4d53..1fb012598afa50820c8cf7aa79b4abee6bf408df 100644
--- a/dbrepo-ui/pages/database/[database_id]/table/[table_id]/info.vue
+++ b/dbrepo-ui/pages/database/[database_id]/table/[table_id]/info.vue
@@ -41,7 +41,7 @@
{{ sizeToHumanLabel(table.data_length) }}
</v-list-item>
<v-list-item
- :title="$t('pages.table.result-rows.title')">
+ :title="$t('pages.table.rows.title')">
{{ table.num_rows }}
</v-list-item>
<v-list-item