diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java
index ed867715e1f10051089a4687b3d8568ec6e5f3cf..878502f1bc31776f208fe4af1a085d2af8e2705c 100644
--- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java
+++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java
@@ -9,6 +9,7 @@ import at.tuwien.api.database.query.QueryDto;
import at.tuwien.api.database.query.QueryPersistDto;
import at.tuwien.api.error.ApiErrorDto;
import at.tuwien.exception.*;
+import at.tuwien.mapper.MariaDbMapper;
import at.tuwien.mapper.MetadataMapper;
import at.tuwien.service.*;
import at.tuwien.validation.EndpointValidator;
@@ -48,6 +49,7 @@ import java.util.UUID;
@RequestMapping(path = "/api/database/{databaseId}/subset")
public class SubsetEndpoint extends RestEndpoint {
+ private final MariaDbMapper mariaDbMapper;
private final SubsetService subsetService;
private final MetadataMapper metadataMapper;
private final MetricsService metricsService;
@@ -57,9 +59,10 @@ public class SubsetEndpoint extends RestEndpoint {
private final EndpointValidator endpointValidator;
@Autowired
- public SubsetEndpoint(SubsetService subsetService, MetadataMapper metadataMapper, MetricsService metricsService,
- StorageService storageService, DatabaseService databaseService,
+ public SubsetEndpoint(MariaDbMapper mariaDbMapper, SubsetService subsetService, MetadataMapper metadataMapper,
+ MetricsService metricsService, StorageService storageService, DatabaseService databaseService,
CredentialService credentialService, EndpointValidator endpointValidator) {
+ this.mariaDbMapper = mariaDbMapper;
this.subsetService = subsetService;
this.metadataMapper = metadataMapper;
this.metricsService = metricsService;
@@ -188,20 +191,16 @@ public class SubsetEndpoint extends RestEndpoint {
return ResponseEntity.ok(subset);
case "text/csv":
log.trace("accept header matches csv");
- try {
- final Dataset<Row> dataset = subsetService.getData(database, subset, null, null);
- metricsService.countSubsetGetData(databaseId, subsetId);
- final ExportResourceDto resource = storageService.transformDataset(dataset);
- final HttpHeaders headers = new HttpHeaders();
- headers.add("Content-Disposition", "attachment; filename=\"" + resource.getFilename() + "\"");
- log.trace("export table resulted in resource {}", resource);
- return ResponseEntity.ok()
- .headers(headers)
- .body(resource.getResource());
- } catch (SQLException e) {
- log.error("Failed to find data: {}", e.getMessage());
- throw new DatabaseUnavailableException("Failed to find data: " + e.getMessage(), e);
- }
+ final String query = mariaDbMapper.rawSelectQuery(subset.getQuery(), timestamp, null, null);
+ final Dataset<Row> dataset = subsetService.getData(database, query, timestamp, null, null, null, null);
+ metricsService.countSubsetGetData(databaseId, subsetId);
+ final ExportResourceDto resource = storageService.transformDataset(dataset);
+ final HttpHeaders headers = new HttpHeaders();
+ headers.add("Content-Disposition", "attachment; filename=\"" + resource.getFilename() + "\"");
+ log.trace("export table resulted in resource {}", resource);
+ return ResponseEntity.ok()
+ .headers(headers)
+ .body(resource.getResource());
}
throw new FormatNotAvailableException("Must provide either application/json or text/csv value for header 'Accept': provided " + accept + " instead");
}
@@ -290,7 +289,7 @@ public class SubsetEndpoint extends RestEndpoint {
endpointValidator.validateOnlyPrivateSchemaAccess(database, principal);
try {
final Long subsetId = subsetService.create(database, data.getStatement(), timestamp, userId);
- return getData(databaseId, subsetId, principal, request, page, size);
+ return getData(databaseId, subsetId, principal, request, timestamp, page, size);
} catch (SQLException e) {
log.error("Failed to establish connection to database: {}", e.getMessage());
throw new DatabaseUnavailableException("Failed to establish connection to database: " + e.getMessage(), e);
@@ -337,11 +336,12 @@ public class SubsetEndpoint extends RestEndpoint {
@NotNull @PathVariable("subsetId") Long subsetId,
Principal principal,
@NotNull HttpServletRequest request,
+ @RequestParam(required = false) Instant timestamp,
@RequestParam(required = false) Long page,
@RequestParam(required = false) Long size)
throws PaginationException, DatabaseNotFoundException, RemoteUnavailableException, NotAllowedException,
QueryNotFoundException, DatabaseUnavailableException, TableMalformedException, QueryMalformedException,
- UserNotFoundException, MetadataServiceException, TableNotFoundException, ViewMalformedException, ViewNotFoundException {
+ UserNotFoundException, MetadataServiceException, TableNotFoundException, ViewNotFoundException {
log.debug("endpoint get subset data, databaseId={}, subsetId={}, principal.name={} page={}, size={}",
databaseId, subsetId, principal != null ? principal.getName() : null, page, size);
endpointValidator.validateDataParams(page, size);
@@ -363,6 +363,10 @@ public class SubsetEndpoint extends RestEndpoint {
size = 10L;
log.debug("size not set: default to {}", size);
}
+ if (timestamp == null) {
+ timestamp = Instant.now();
+ log.debug("timestamp not set: default to {}", timestamp);
+ }
try {
final HttpHeaders headers = new HttpHeaders();
headers.set("X-Id", "" + subsetId);
@@ -375,7 +379,8 @@ public class SubsetEndpoint extends RestEndpoint {
.headers(headers)
.build();
}
- final Dataset<Row> dataset = subsetService.getData(database, subset, page, size);
+ final String query = mariaDbMapper.rawSelectQuery(subset.getQuery(), timestamp, page, size);
+ final Dataset<Row> dataset = subsetService.getData(database, query, timestamp, page, size, null, null);
metricsService.countSubsetGetData(databaseId, subsetId);
final String viewName = metadataMapper.queryDtoToViewName(subset);
final ViewDto view = databaseService.inspectView(database, viewName);
diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
index 92ba523f367dc170bd0e60e7113334e34cc697de..82ed0a96f7596e025301f11e6ebceed2cd393acc 100644
--- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
+++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
@@ -10,6 +10,7 @@ import at.tuwien.api.database.table.internal.TableCreateDto;
import at.tuwien.api.error.ApiErrorDto;
import at.tuwien.exception.*;
import at.tuwien.gateway.MetadataServiceGateway;
+import at.tuwien.mapper.MariaDbMapper;
import at.tuwien.service.*;
import at.tuwien.validation.EndpointValidator;
import io.micrometer.observation.annotation.Observed;
@@ -48,6 +49,8 @@ import java.util.Map;
public class TableEndpoint extends RestEndpoint {
private final TableService tableService;
+ private final MariaDbMapper mariaDbMapper;
+ private final SubsetService subsetService;
private final MetricsService metricsService;
private final StorageService storageService;
private final DatabaseService databaseService;
@@ -56,10 +59,13 @@ public class TableEndpoint extends RestEndpoint {
private final MetadataServiceGateway metadataServiceGateway;
@Autowired
- public TableEndpoint(TableService tableService, MetricsService metricsService, StorageService storageService,
- DatabaseService databaseService, CredentialService credentialService,
- EndpointValidator endpointValidator, MetadataServiceGateway metadataServiceGateway) {
+ public TableEndpoint(TableService tableService, MariaDbMapper mariaDbMapper, SubsetService subsetService,
+ MetricsService metricsService, StorageService storageService, DatabaseService databaseService,
+ CredentialService credentialService, EndpointValidator endpointValidator,
+ MetadataServiceGateway metadataServiceGateway) {
this.tableService = tableService;
+ this.mariaDbMapper = mariaDbMapper;
+ this.subsetService = subsetService;
this.metricsService = metricsService;
this.storageService = storageService;
this.databaseService = databaseService;
@@ -287,8 +293,10 @@ public class TableEndpoint extends RestEndpoint {
}
headers.set("Access-Control-Expose-Headers", "X-Headers");
headers.set("X-Headers", String.join(",", table.getColumns().stream().map(ColumnDto::getInternalName).toList()));
- final Dataset<Row> dataset = tableService.getData(credentialService.getDatabase(table.getTdbid()),
- table.getInternalName(), timestamp, page, size, null, null);
+ final String query = mariaDbMapper.defaultRawSelectQuery(table.getDatabase().getInternalName(),
+ table.getInternalName(), timestamp, page, size);
+ final Dataset<Row> dataset = subsetService.getData(credentialService.getDatabase(table.getTdbid()),
+ query, timestamp, page, size, null, null);
metricsService.countTableGetData(databaseId, tableId);
return ResponseEntity.ok()
.headers(headers)
@@ -624,8 +632,10 @@ public class TableEndpoint extends RestEndpoint {
}
credentialService.getAccess(databaseId, getId(principal));
}
- final Dataset<Row> dataset = tableService.getData(credentialService.getDatabase(table.getTdbid()),
- table.getInternalName(), timestamp, null, null, null, null);
+ final String query = mariaDbMapper.defaultRawSelectQuery(table.getDatabase().getInternalName(),
+ table.getInternalName(), timestamp, null, null);
+ final Dataset<Row> dataset = subsetService.getData(credentialService.getDatabase(table.getTdbid()),
+ query, timestamp, null, null, null, null);
metricsService.countTableGetData(databaseId, tableId);
final ExportResourceDto resource = storageService.transformDataset(dataset);
final HttpHeaders headers = new HttpHeaders();
diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java
index c662bf49c1a10ada160f41b45020613448d0c396..d4eccd0772eba12e007c6e99ae42a60b1304fca5 100644
--- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java
+++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java
@@ -1,12 +1,13 @@
package at.tuwien.endpoints;
import at.tuwien.ExportResourceDto;
+import at.tuwien.api.database.CreateViewDto;
import at.tuwien.api.database.DatabaseDto;
import at.tuwien.api.database.ViewColumnDto;
-import at.tuwien.api.database.ViewCreateDto;
import at.tuwien.api.database.ViewDto;
import at.tuwien.api.error.ApiErrorDto;
import at.tuwien.exception.*;
+import at.tuwien.mapper.MariaDbMapper;
import at.tuwien.service.*;
import at.tuwien.validation.EndpointValidator;
import io.micrometer.observation.annotation.Observed;
@@ -45,6 +46,8 @@ public class ViewEndpoint extends RestEndpoint {
private final ViewService viewService;
private final TableService tableService;
+ private final MariaDbMapper mariaDbMapper;
+ private final SubsetService subsetService;
private final MetricsService metricsService;
private final StorageService storageService;
private final DatabaseService databaseService;
@@ -52,11 +55,14 @@ public class ViewEndpoint extends RestEndpoint {
private final EndpointValidator endpointValidator;
@Autowired
- public ViewEndpoint(ViewService viewService, TableService tableService, MetricsService metricsService,
- StorageService storageService, DatabaseService databaseService,
- CredentialService credentialService, EndpointValidator endpointValidator) {
+ public ViewEndpoint(ViewService viewService, TableService tableService, MariaDbMapper mariaDbMapper,
+ SubsetService subsetService, MetricsService metricsService, StorageService storageService,
+ DatabaseService databaseService, CredentialService credentialService,
+ EndpointValidator endpointValidator) {
this.viewService = viewService;
this.tableService = tableService;
+ this.mariaDbMapper = mariaDbMapper;
+ this.subsetService = subsetService;
this.metricsService = metricsService;
this.storageService = storageService;
this.databaseService = databaseService;
@@ -147,7 +153,7 @@ public class ViewEndpoint extends RestEndpoint {
schema = @Schema(implementation = ApiErrorDto.class))}),
})
public ResponseEntity<ViewDto> create(@NotNull @PathVariable("databaseId") Long databaseId,
- @Valid @RequestBody ViewCreateDto data) throws DatabaseUnavailableException,
+ @Valid @RequestBody CreateViewDto data) throws DatabaseUnavailableException,
DatabaseNotFoundException, RemoteUnavailableException, ViewMalformedException, MetadataServiceException {
log.debug("endpoint create view, databaseId={}, data.name={}", databaseId, data.getName());
final DatabaseDto database = credentialService.getDatabase(databaseId);
@@ -288,8 +294,10 @@ public class ViewEndpoint extends RestEndpoint {
}
headers.set("Access-Control-Expose-Headers", "X-Headers");
headers.set("X-Headers", String.join(",", view.getColumns().stream().map(ViewColumnDto::getInternalName).toList()));
- final Dataset<Row> dataset = tableService.getData(credentialService.getDatabase(databaseId),
- view.getInternalName(), timestamp, page, size, null, null);
+ final String query = mariaDbMapper.defaultRawSelectQuery(view.getDatabase().getInternalName(),
+ view.getInternalName(), timestamp, page, size);
+ final Dataset<Row> dataset = subsetService.getData(credentialService.getDatabase(databaseId),
+ query, timestamp, page, size, null, null);
metricsService.countViewGetData(databaseId, viewId);
return ResponseEntity.ok()
.headers(headers)
@@ -353,8 +361,10 @@ public class ViewEndpoint extends RestEndpoint {
}
credentialService.getAccess(databaseId, getId(principal));
}
- final Dataset<Row> dataset = tableService.getData(credentialService.getDatabase(databaseId),
- view.getInternalName(), timestamp, null, null, null, null);
+ final String query = mariaDbMapper.defaultRawSelectQuery(view.getDatabase().getInternalName(),
+ view.getInternalName(), timestamp, null, null);
+ final Dataset<Row> dataset = subsetService.getData(credentialService.getDatabase(databaseId),
+ query, timestamp, null, null, null, null);
metricsService.countViewGetData(databaseId, viewId);
final ExportResourceDto resource = storageService.transformDataset(dataset);
final HttpHeaders headers = new HttpHeaders();
diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java
index 002374fe9d5f6d3cd2e3b94f0df49b5a3365c6d7..53393be4073d4804b65c4dde13b248612b59d143 100644
--- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java
+++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java
@@ -1,5 +1,6 @@
package at.tuwien.endpoint;
+import at.tuwien.api.SortTypeDto;
import at.tuwien.api.database.DatabaseDto;
import at.tuwien.api.database.query.ExecuteStatementDto;
import at.tuwien.api.database.query.QueryDto;
@@ -206,7 +207,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
.thenReturn(QUERY_5_DTO);
when(storageService.transformDataset(any(Dataset.class)))
.thenReturn(EXPORT_RESOURCE_DTO);
- when(subsetService.getData(any(DatabaseDto.class), any(QueryDto.class), eq(0L), eq(10L)))
+ when(subsetService.getData(any(DatabaseDto.class), anyString(), any(Instant.class), eq(0L), eq(10L), any(SortTypeDto.class), anyString()))
.thenReturn(mock);
/* test */
@@ -232,7 +233,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
@WithAnonymousUser
public void findById_publicDataPublicSchemaAnonymous_fails() throws DatabaseNotFoundException, SQLException,
RemoteUnavailableException, UserNotFoundException, QueryMalformedException, StorageUnavailableException,
- QueryNotFoundException, MetadataServiceException, TableNotFoundException, ViewMalformedException {
+ QueryNotFoundException, MetadataServiceException, TableNotFoundException {
final Dataset<Row> mock = sparkSession.emptyDataFrame();
/* mock */
@@ -240,7 +241,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
.thenReturn(DATABASE_3_DTO);
when(subsetService.findById(DATABASE_4_DTO, QUERY_5_ID))
.thenReturn(QUERY_5_DTO);
- when(subsetService.getData(any(DatabaseDto.class), any(QueryDto.class), eq(0L), eq(10L)))
+ when(subsetService.getData(any(DatabaseDto.class), anyString(), any(Instant.class), eq(0L), eq(10L), any(SortTypeDto.class), anyString()))
.thenReturn(mock);
when(storageService.transformDataset(any(Dataset.class)))
.thenReturn(EXPORT_RESOURCE_DTO);
@@ -289,8 +290,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
@WithMockUser(username = USER_1_USERNAME)
public void findById_publicDataPrivateSchemaUnavailableExport_fails() throws DatabaseNotFoundException,
RemoteUnavailableException, MetadataServiceException, SQLException, QueryMalformedException,
- UserNotFoundException, QueryNotFoundException, TableNotFoundException, ViewMalformedException,
- StorageUnavailableException {
+ UserNotFoundException, QueryNotFoundException, TableNotFoundException, StorageUnavailableException {
/* mock */
when(credentialService.getDatabase(DATABASE_3_ID))
@@ -301,7 +301,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
.thenReturn(EXPORT_RESOURCE_DTO);
doThrow(SQLException.class)
.when(subsetService)
- .getData(eq(DATABASE_3_DTO), eq(QUERY_5_DTO), eq(0L), eq(10L));
+ .getData(any(DatabaseDto.class), anyString(), any(Instant.class), eq(0L), eq(10L), any(SortTypeDto.class), anyString());
/* test */
assertThrows(DatabaseUnavailableException.class, () -> {
@@ -324,7 +324,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
/* mock */
when(credentialService.getDatabase(DATABASE_3_ID))
.thenReturn(DATABASE_3_DTO);
- when(subsetService.getData(eq(DATABASE_3_DTO), any(QueryDto.class), eq(0L), eq(10L)))
+ when(subsetService.getData(any(DatabaseDto.class), anyString(), any(Instant.class), eq(0L), eq(10L), any(SortTypeDto.class), anyString()))
.thenReturn(mock);
when(subsetService.findById(eq(DATABASE_3_DTO), anyLong()))
.thenReturn(QUERY_5_DTO);
@@ -370,7 +370,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
.thenReturn(DATABASE_3_DTO);
when(subsetService.findById(eq(DATABASE_3_DTO), anyLong()))
.thenReturn(QUERY_5_DTO);
- when(subsetService.getData(eq(DATABASE_3_DTO), any(QueryDto.class), eq(0L), eq(10L)))
+ when(subsetService.getData(any(DatabaseDto.class), anyString(), any(Instant.class), eq(0L), eq(10L), any(SortTypeDto.class), anyString()))
.thenReturn(mock);
when(httpServletRequest.getMethod())
.thenReturn("POST");
@@ -394,7 +394,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
.thenReturn(DATABASE_3_DTO);
doThrow(SQLException.class)
.when(subsetService)
- .getData(eq(DATABASE_3_DTO), any(QueryDto.class), eq(0L), eq(10L));
+ .getData(any(DatabaseDto.class), anyString(), any(Instant.class), eq(0L), eq(10L), any(SortTypeDto.class), anyString());
when(subsetService.findById(eq(DATABASE_3_DTO), anyLong()))
.thenReturn(QUERY_5_DTO);
when(metadataServiceGateway.getAccess(DATABASE_3_ID, USER_1_ID))
@@ -447,7 +447,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
/* mock */
when(credentialService.getDatabase(DATABASE_3_ID))
.thenReturn(DATABASE_3_DTO);
- when(subsetService.getData(eq(DATABASE_3_DTO), any(QueryDto.class), eq(0L), eq(10L)))
+ when(subsetService.getData(any(DatabaseDto.class), anyString(), any(Instant.class), eq(0L), eq(10L), any(SortTypeDto.class), anyString()))
.thenReturn(mock);
when(subsetService.findById(eq(DATABASE_3_DTO), anyLong()))
.thenReturn(QUERY_5_DTO);
@@ -475,7 +475,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
.thenReturn(DATABASE_4_DTO);
when(subsetService.findById(eq(DATABASE_4_DTO), anyLong()))
.thenReturn(QUERY_5_DTO);
- when(subsetService.getData(eq(DATABASE_4_DTO), any(QueryDto.class), eq(0L), eq(10L)))
+ when(subsetService.getData(any(DatabaseDto.class), anyString(), any(Instant.class), eq(0L), eq(10L), any(SortTypeDto.class), anyString()))
.thenReturn(mock);
when(httpServletRequest.getMethod())
.thenReturn("POST");
@@ -501,7 +501,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
.thenReturn(DATABASE_1_DTO);
when(subsetService.findById(eq(DATABASE_1_DTO), anyLong()))
.thenReturn(QUERY_1_DTO);
- when(subsetService.getData(eq(DATABASE_1_DTO), any(QueryDto.class), eq(0L), eq(10L)))
+ when(subsetService.getData(any(DatabaseDto.class), anyString(), any(Instant.class), eq(0L), eq(10L), any(SortTypeDto.class), anyString()))
.thenReturn(mock);
when(httpServletRequest.getMethod())
.thenReturn("POST");
@@ -514,7 +514,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
@WithAnonymousUser
public void create_privateDataPublicSchemaAnonymous_fails() throws DatabaseNotFoundException, SQLException,
MetadataServiceException, UserNotFoundException, QueryNotFoundException, QueryMalformedException,
- TableNotFoundException, ViewMalformedException, ViewNotFoundException, RemoteUnavailableException {
+ TableNotFoundException, RemoteUnavailableException {
final Dataset<Row> mock = sparkSession.emptyDataFrame();
final ExecuteStatementDto request = ExecuteStatementDto.builder()
.statement(QUERY_5_STATEMENT)
@@ -525,7 +525,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
.thenReturn(DATABASE_2_DTO);
when(subsetService.findById(eq(DATABASE_2_DTO), anyLong()))
.thenReturn(QUERY_2_DTO);
- when(subsetService.getData(eq(DATABASE_2_DTO), any(QueryDto.class), eq(0L), eq(10L)))
+ when(subsetService.getData(any(DatabaseDto.class), anyString(), any(Instant.class), eq(0L), eq(10L), any(SortTypeDto.class), anyString()))
.thenReturn(mock);
when(httpServletRequest.getMethod())
.thenReturn("POST");
@@ -540,7 +540,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
public void getData_succeeds() throws DatabaseNotFoundException, RemoteUnavailableException, UserNotFoundException,
NotAllowedException, SQLException, QueryNotFoundException, TableMalformedException, QueryMalformedException,
DatabaseUnavailableException, PaginationException, MetadataServiceException, TableNotFoundException,
- ViewMalformedException, ViewNotFoundException {
+ ViewNotFoundException {
final Dataset<Row> mock = sparkSession.emptyDataFrame();
/* mock */
@@ -550,13 +550,13 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
.thenReturn(QUERY_5_DTO);
when(subsetService.reExecuteCount(DATABASE_3_DTO, QUERY_5_DTO))
.thenReturn(QUERY_5_RESULT_NUMBER);
- when(subsetService.getData(eq(DATABASE_3_DTO), any(QueryDto.class), eq(0L), eq(10L)))
+ when(subsetService.getData(any(DatabaseDto.class), anyString(), any(Instant.class), eq(0L), eq(10L), any(SortTypeDto.class), anyString()))
.thenReturn(mock);
when(httpServletRequest.getMethod())
.thenReturn("GET");
/* test */
- final ResponseEntity<List<Map<String, Object>>> response = subsetEndpoint.getData(DATABASE_3_ID, QUERY_5_ID, null, httpServletRequest, null, null);
+ final ResponseEntity<List<Map<String, Object>>> response = subsetEndpoint.getData(DATABASE_3_ID, QUERY_5_ID, null, httpServletRequest, null, null, null);
assertEquals(HttpStatus.OK, response.getStatusCode());
assertNotNull(response.getBody());
}
@@ -565,7 +565,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
public void getData_head_succeeds() throws DatabaseNotFoundException, RemoteUnavailableException,
UserNotFoundException, NotAllowedException, SQLException, QueryNotFoundException, TableMalformedException,
QueryMalformedException, DatabaseUnavailableException, PaginationException, MetadataServiceException,
- TableNotFoundException, ViewMalformedException, ViewNotFoundException {
+ TableNotFoundException, ViewNotFoundException {
/* mock */
when(credentialService.getDatabase(DATABASE_3_ID))
@@ -578,7 +578,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
.thenReturn("HEAD");
/* test */
- final ResponseEntity<List<Map<String, Object>>> response = subsetEndpoint.getData(DATABASE_3_ID, QUERY_5_ID, null, httpServletRequest, null, null);
+ final ResponseEntity<List<Map<String, Object>>> response = subsetEndpoint.getData(DATABASE_3_ID, QUERY_5_ID, null, httpServletRequest, null, null, null);
assertEquals(HttpStatus.OK, response.getStatusCode());
assertNotNull(response.getHeaders().get("X-Count"));
assertEquals(1, response.getHeaders().get("X-Count").size());
@@ -590,7 +590,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
public void getData_private_succeeds() throws DatabaseNotFoundException, RemoteUnavailableException,
UserNotFoundException, DatabaseUnavailableException, NotAllowedException, TableMalformedException,
QueryMalformedException, QueryNotFoundException, PaginationException, SQLException,
- MetadataServiceException, TableNotFoundException, ViewMalformedException, ViewNotFoundException {
+ MetadataServiceException, TableNotFoundException, ViewNotFoundException {
final Dataset<Row> mock = sparkSession.emptyDataFrame();
/* mock */
@@ -600,13 +600,13 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
.thenReturn(QUERY_1_DTO);
when(subsetService.reExecuteCount(DATABASE_1_DTO, QUERY_1_DTO))
.thenReturn(QUERY_1_RESULT_NUMBER);
- when(subsetService.getData(DATABASE_1_DTO, QUERY_1_DTO, 0L, 10L))
+ when(subsetService.getData(DATABASE_1_DTO, QUERY_1_STATEMENT, Instant.now(), 0L, 10L, null, null))
.thenReturn(mock);
when(httpServletRequest.getMethod())
.thenReturn("GET");
/* test */
- final ResponseEntity<List<Map<String, Object>>> response = subsetEndpoint.getData(DATABASE_1_ID, QUERY_1_ID, USER_1_PRINCIPAL, httpServletRequest, null, null);
+ final ResponseEntity<List<Map<String, Object>>> response = subsetEndpoint.getData(DATABASE_1_ID, QUERY_1_ID, USER_1_PRINCIPAL, httpServletRequest, null, null, null);
assertEquals(HttpStatus.OK, response.getStatusCode());
assertNotNull(response.getBody());
}
@@ -622,7 +622,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
/* test */
assertThrows(NotAllowedException.class, () -> {
- subsetEndpoint.getData(DATABASE_1_ID, QUERY_1_ID, null, httpServletRequest, null, null);
+ subsetEndpoint.getData(DATABASE_1_ID, QUERY_1_ID, null, httpServletRequest, null, null, null);
});
}
@@ -640,7 +640,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
/* test */
assertThrows(NotAllowedException.class, () -> {
- subsetEndpoint.getData(DATABASE_1_ID, QUERY_1_ID, USER_1_PRINCIPAL, httpServletRequest, null, null);
+ subsetEndpoint.getData(DATABASE_1_ID, QUERY_1_ID, USER_1_PRINCIPAL, httpServletRequest, null, null, null);
});
}
@@ -649,7 +649,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
public void getData_privateHead_succeeds() throws DatabaseNotFoundException, RemoteUnavailableException,
UserNotFoundException, DatabaseUnavailableException, NotAllowedException, TableMalformedException,
QueryMalformedException, QueryNotFoundException, PaginationException, SQLException,
- MetadataServiceException, TableNotFoundException, ViewMalformedException, ViewNotFoundException {
+ MetadataServiceException, TableNotFoundException, ViewNotFoundException {
/* mock */
when(credentialService.getDatabase(DATABASE_1_ID))
@@ -662,7 +662,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
.thenReturn("HEAD");
/* test */
- final ResponseEntity<List<Map<String, Object>>> response = subsetEndpoint.getData(DATABASE_1_ID, QUERY_1_ID, USER_1_PRINCIPAL, httpServletRequest, null, null);
+ final ResponseEntity<List<Map<String, Object>>> response = subsetEndpoint.getData(DATABASE_1_ID, QUERY_1_ID, USER_1_PRINCIPAL, httpServletRequest, null, null, null);
assertEquals(HttpStatus.OK, response.getStatusCode());
assertNotNull(response.getHeaders().get("X-Count"));
assertEquals(1, response.getHeaders().get("X-Count").size());
@@ -673,7 +673,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
@WithMockUser(username = USER_1_USERNAME)
public void getData_unavailable_fails() throws DatabaseNotFoundException, RemoteUnavailableException, SQLException,
UserNotFoundException, QueryNotFoundException, MetadataServiceException, QueryMalformedException,
- TableNotFoundException, ViewMalformedException {
+ TableNotFoundException {
/* mock */
when(credentialService.getDatabase(DATABASE_1_ID))
@@ -684,11 +684,11 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
.thenReturn("GET");
doThrow(SQLException.class)
.when(subsetService)
- .getData(DATABASE_1_DTO, QUERY_1_DTO, 0L, 10L);
+ .getData(DATABASE_1_DTO, QUERY_1_STATEMENT, Instant.now(), 0L, 10L, null, null);
/* test */
assertThrows(DatabaseUnavailableException.class, () -> {
- subsetEndpoint.getData(DATABASE_1_ID, QUERY_1_ID, USER_1_PRINCIPAL, httpServletRequest, null, null);
+ subsetEndpoint.getData(DATABASE_1_ID, QUERY_1_ID, USER_1_PRINCIPAL, httpServletRequest, null, null, null);
});
}
diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java
index c7a74de17fc6ecc15e410aefb1366a0ab0e81549..ecd745e3eeee7a85c2f54b2525a6f620606a077b 100644
--- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java
+++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java
@@ -8,6 +8,7 @@ import at.tuwien.exception.*;
import at.tuwien.gateway.MetadataServiceGateway;
import at.tuwien.service.CredentialService;
import at.tuwien.service.DatabaseService;
+import at.tuwien.service.SubsetService;
import at.tuwien.service.TableService;
import at.tuwien.test.AbstractUnitTest;
import jakarta.servlet.http.HttpServletRequest;
@@ -59,6 +60,9 @@ public class TableEndpointUnitTest extends AbstractUnitTest {
@MockBean
private TableService tableService;
+ @MockBean
+ private SubsetService subsetService;
+
@MockBean
private DatabaseService databaseService;
@@ -283,7 +287,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest {
/* mock */
when(credentialService.getTable(DATABASE_3_ID, TABLE_8_ID))
.thenReturn(TABLE_8_DTO);
- when(tableService.getData(eq(DATABASE_3_DTO), eq(TABLE_8_INTERNAL_NAME), any(Instant.class), eq(0L), eq(10L), eq(null), eq(null)))
+ when(subsetService.getData(eq(DATABASE_3_DTO), eq(TABLE_8_INTERNAL_NAME), any(Instant.class), eq(0L), eq(10L), eq(null), eq(null)))
.thenReturn(mock);
when(httpServletRequest.getMethod())
.thenReturn("GET");
@@ -306,7 +310,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest {
.thenReturn(TABLE_8_DTO);
when(tableService.getCount(eq(TABLE_8_DTO), any(Instant.class)))
.thenReturn(3L);
- when(tableService.getData(eq(DATABASE_3_DTO), eq(TABLE_8_INTERNAL_NAME), any(Instant.class), eq(0L), eq(10L), eq(null), eq(null)))
+ when(subsetService.getData(eq(DATABASE_3_DTO), eq(TABLE_8_INTERNAL_NAME), any(Instant.class), eq(0L), eq(10L), eq(null), eq(null)))
.thenReturn(mock);
when(httpServletRequest.getMethod())
.thenReturn("HEAD");
@@ -363,7 +367,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest {
when(credentialService.getTable(DATABASE_3_ID, TABLE_8_ID))
.thenReturn(TABLE_8_DTO);
doThrow(QueryMalformedException.class)
- .when(tableService)
+ .when(subsetService)
.getData(eq(DATABASE_3_DTO), eq(TABLE_8_INTERNAL_NAME), any(Instant.class), eq(0L), eq(10L), eq(null), eq(null));
when(httpServletRequest.getMethod())
.thenReturn("GET");
@@ -405,7 +409,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest {
.thenReturn(TABLE_1_DTO);
when(credentialService.getAccess(DATABASE_1_ID, USER_2_ID))
.thenReturn(access);
- when(tableService.getData(eq(DATABASE_1_DTO), eq(TABLE_1_INTERNAL_NAME), any(Instant.class), eq(0L), eq(10L), eq(null), eq(null)))
+ when(subsetService.getData(eq(DATABASE_1_DTO), eq(TABLE_1_INTERNAL_NAME), any(Instant.class), eq(0L), eq(10L), eq(null), eq(null)))
.thenReturn(mock);
when(httpServletRequest.getMethod())
.thenReturn("GET");
@@ -1146,7 +1150,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest {
/* mock */
when(credentialService.getTable(DATABASE_3_ID, TABLE_8_ID))
.thenReturn(TABLE_8_DTO);
- when(tableService.getData(eq(DATABASE_3_DTO), eq(TABLE_8_INTERNAL_NAME), any(Instant.class), eq(0L), eq(10L), eq(null), eq(null)))
+ when(subsetService.getData(eq(DATABASE_3_DTO), eq(TABLE_8_INTERNAL_NAME), any(Instant.class), eq(0L), eq(10L), eq(null), eq(null)))
.thenReturn(mock);
/* test */
@@ -1167,7 +1171,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest {
.thenReturn(TABLE_1_DTO);
when(credentialService.getAccess(DATABASE_1_ID, USER_2_ID))
.thenReturn(access);
- when(tableService.getData(eq(DATABASE_1_DTO), eq(TABLE_1_INTERNAL_NAME), any(Instant.class), eq(0L), eq(10L), eq(null), eq(null)))
+ when(subsetService.getData(eq(DATABASE_1_DTO), eq(TABLE_1_INTERNAL_NAME), any(Instant.class), eq(0L), eq(10L), eq(null), eq(null)))
.thenReturn(mock);
/* test */
diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/ViewEndpointUnitTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/ViewEndpointUnitTest.java
index 8586f8f92da772235df50a56ee0f423db9238f10..c69c8d338e854d89892db62de126f465430d1fa3 100644
--- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/ViewEndpointUnitTest.java
+++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/ViewEndpointUnitTest.java
@@ -5,7 +5,7 @@ import at.tuwien.endpoints.ViewEndpoint;
import at.tuwien.exception.*;
import at.tuwien.service.CredentialService;
import at.tuwien.service.DatabaseService;
-import at.tuwien.service.TableService;
+import at.tuwien.service.SubsetService;
import at.tuwien.service.ViewService;
import at.tuwien.test.AbstractUnitTest;
import jakarta.servlet.http.HttpServletRequest;
@@ -51,7 +51,7 @@ public class ViewEndpointUnitTest extends AbstractUnitTest {
private HttpServletRequest httpServletRequest;
@MockBean
- private TableService tableService;
+ private SubsetService subsetService;
@Autowired
private ViewEndpoint viewEndpoint;
@@ -282,7 +282,7 @@ public class ViewEndpointUnitTest extends AbstractUnitTest {
.thenReturn(VIEW_1_DTO);
when(credentialService.getAccess(DATABASE_1_ID, USER_1_ID))
.thenReturn(DATABASE_1_USER_1_READ_ACCESS_DTO);
- when(tableService.getData(eq(DATABASE_1_DTO), eq(VIEW_1_INTERNAL_NAME), any(Instant.class), eq(0L), eq(10L), eq(null), eq(null)))
+ when(subsetService.getData(eq(DATABASE_1_DTO), eq(VIEW_1_INTERNAL_NAME), any(Instant.class), eq(0L), eq(10L), eq(null), eq(null)))
.thenReturn(mock);
when(httpServletRequest.getMethod())
.thenReturn("GET");
diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java
index e48f0c048da2dd6c83934566e6e5d9956838ba62..120fe49d8f98f61e36840b69d8c33fa00bc27053 100644
--- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java
+++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java
@@ -140,7 +140,7 @@ public class PrometheusEndpointMvcTest extends AbstractUnitTest {
/* ignore */
}
try {
- subsetEndpoint.getData(DATABASE_1_ID, QUERY_1_ID, USER_1_PRINCIPAL, httpServletRequest, 0L, 10L);
+ subsetEndpoint.getData(DATABASE_1_ID, QUERY_1_ID, USER_1_PRINCIPAL, httpServletRequest, null, 0L, 10L);
} catch (Exception e) {
/* ignore */
}
diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceIntegrationTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceIntegrationTest.java
index 2b7476e50fdaf2fc3b00eaee4552be18959b48f0..550589df794b5839e37b359f4a1b97992d410309 100644
--- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceIntegrationTest.java
+++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceIntegrationTest.java
@@ -4,12 +4,12 @@ import at.tuwien.api.database.ViewColumnDto;
import at.tuwien.api.database.ViewDto;
import at.tuwien.api.database.table.TableBriefDto;
import at.tuwien.api.database.table.TableDto;
-import at.tuwien.api.database.table.columns.ColumnCreateDto;
import at.tuwien.api.database.table.columns.ColumnDto;
import at.tuwien.api.database.table.columns.ColumnTypeDto;
-import at.tuwien.api.database.table.constraints.ConstraintsCreateDto;
+import at.tuwien.api.database.table.columns.CreateTableColumnDto;
import at.tuwien.api.database.table.constraints.ConstraintsDto;
-import at.tuwien.api.database.table.constraints.foreign.ForeignKeyCreateDto;
+import at.tuwien.api.database.table.constraints.CreateTableConstraintsDto;
+import at.tuwien.api.database.table.constraints.foreign.CreateForeignKeyDto;
import at.tuwien.api.database.table.constraints.foreign.ForeignKeyDto;
import at.tuwien.api.database.table.constraints.foreign.ForeignKeyReferenceDto;
import at.tuwien.api.database.table.constraints.foreign.ReferenceTypeDto;
@@ -619,8 +619,8 @@ public class DatabaseServiceIntegrationTest extends AbstractUnitTest {
final at.tuwien.api.database.table.internal.TableCreateDto request = TableCreateDto.builder()
.name("missing_foreign_key")
.columns(List.of())
- .constraints(ConstraintsCreateDto.builder()
- .foreignKeys(List.of(ForeignKeyCreateDto.builder()
+ .constraints(CreateTableConstraintsDto.builder()
+ .foreignKeys(List.of(CreateForeignKeyDto.builder()
.columns(List.of("i_do_not_exist"))
.referencedTable("neither_do_i")
.referencedColumns(List.of("behold"))
@@ -639,27 +639,27 @@ public class DatabaseServiceIntegrationTest extends AbstractUnitTest {
TableExistsException {
final at.tuwien.api.database.table.internal.TableCreateDto request = TableCreateDto.builder()
.name("composite_primary_key")
- .columns(List.of(ColumnCreateDto.builder()
+ .columns(List.of(CreateTableColumnDto.builder()
.name("name")
.type(ColumnTypeDto.VARCHAR)
.size(255L)
.nullAllowed(false)
.build(),
- ColumnCreateDto.builder()
+ CreateTableColumnDto.builder()
.name("lat")
.type(ColumnTypeDto.DECIMAL)
.size(10L)
.d(10L)
.nullAllowed(false)
.build(),
- ColumnCreateDto.builder()
+ CreateTableColumnDto.builder()
.name("lng")
.type(ColumnTypeDto.DECIMAL)
.size(10L)
.d(10L)
.nullAllowed(false)
.build()))
- .constraints(ConstraintsCreateDto.builder()
+ .constraints(CreateTableConstraintsDto.builder()
.primaryKey(Set.of("lat", "lng"))
.foreignKeys(List.of())
.checks(Set.of())
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java
index d3af54816436699c413db11ff59cf213656b46a7..0e89f79cb116682b78bac224981255c0c5196ad2 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java
@@ -4,9 +4,9 @@ import at.tuwien.api.database.table.TableDto;
import at.tuwien.api.database.table.TupleDeleteDto;
import at.tuwien.api.database.table.TupleDto;
import at.tuwien.api.database.table.TupleUpdateDto;
-import at.tuwien.api.database.table.columns.ColumnCreateDto;
import at.tuwien.api.database.table.columns.ColumnDto;
import at.tuwien.api.database.table.columns.ColumnTypeDto;
+import at.tuwien.api.database.table.columns.CreateTableColumnDto;
import at.tuwien.exception.QueryMalformedException;
import at.tuwien.exception.TableMalformedException;
import at.tuwien.utils.MariaDbUtil;
@@ -237,7 +237,7 @@ public interface MariaDbMapper {
* @param data The column definition.
* @return The MySQL string.
*/
- default String columnTypeDtoToDataType(ColumnCreateDto data) {
+ default String columnTypeDtoToDataType(CreateTableColumnDto data) {
return switch (data.getType()) {
case CHAR -> "CHAR(" + Objects.requireNonNullElse(data.getSize(), "1") + ")";
case VARCHAR -> "VARCHAR(" + Objects.requireNonNullElse(data.getSize(), "255") + ")";
@@ -260,7 +260,7 @@ public interface MariaDbMapper {
};
}
- default String columnCreateDtoToPrimaryKeyLengthSpecification(ColumnCreateDto data) {
+ default String columnCreateDtoToPrimaryKeyLengthSpecification(CreateTableColumnDto data) {
if (EnumSet.of(ColumnTypeDto.BLOB, ColumnTypeDto.TEXT).contains(data.getType())) {
return "(" + Objects.requireNonNullElse(data.getIndexLength(), 255) + ")";
}
@@ -309,7 +309,7 @@ public interface MariaDbMapper {
.append("` (");
log.trace("primary key column(s) exist: {}", data.getConstraints().getPrimaryKey());
final int[] idx = {0};
- for (ColumnCreateDto column : data.getColumns()) {
+ for (CreateTableColumnDto column : data.getColumns()) {
stringBuilder.append(idx[0]++ > 0 ? ", " : "")
.append("`")
.append(nameToInternalName(column.getName()))
@@ -336,11 +336,11 @@ public interface MariaDbMapper {
.getPrimaryKey()
.stream()
.map(c -> {
- final Optional<ColumnCreateDto> optional = data.getColumns()
+ final Optional<CreateTableColumnDto> optional = data.getColumns()
.stream()
.filter(cc -> cc.getName().equals(c))
.findFirst();
- log.trace("lookup {} in columns: {}", c, data.getColumns().stream().map(ColumnCreateDto::getName).toList());
+ log.trace("lookup {} in columns: {}", c, data.getColumns().stream().map(CreateTableColumnDto::getName).toList());
return "`" + nameToInternalName(c) + "`" + columnCreateDtoToPrimaryKeyLengthSpecification(optional.get());
})
.toArray(String[]::new)))
@@ -725,6 +725,30 @@ public interface MariaDbMapper {
}
}
+ default String rawSelectQuery(String query, Instant timestamp, Long page, Long size) {
+ /* query check (this is enforced by the db also) */
+ final StringBuilder statement = new StringBuilder("SELECT * FROM (")
+ .append(query);
+ statement.append(")");
+ if (timestamp != null) {
+ statement.append(" FOR SYSTEM_TIME AS OF TIMESTAMP '")
+ .append(mariaDbFormatter.format(timestamp))
+ .append("'");
+ }
+ statement.append(" as tbl");
+ /* pagination */
+ if (size != null && page != null) {
+ log.trace("pagination size/limit of {}", size);
+ statement.append(" LIMIT ")
+ .append(size);
+ log.trace("pagination page/offset of {}", page);
+ statement.append(" OFFSET ")
+ .append(page * size);
+ }
+ log.trace("mapped select query: {}", statement);
+ return statement.toString();
+ }
+
default String defaultRawSelectQuery(String databaseName, String tableOrViewName, Instant timestamp, Long page,
Long size) {
/* query check (this is enforced by the db also) */
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/DatabaseService.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/DatabaseService.java
index 5a120f44d288f5a1fadb5d0c75d478a097558725..314148663b1230cab6a2d3afc903cd301abbd2e0 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/DatabaseService.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/DatabaseService.java
@@ -1,7 +1,7 @@
package at.tuwien.service;
+import at.tuwien.api.database.CreateViewDto;
import at.tuwien.api.database.DatabaseDto;
-import at.tuwien.api.database.ViewCreateDto;
import at.tuwien.api.database.ViewDto;
import at.tuwien.api.database.table.TableDto;
import at.tuwien.api.database.table.internal.TableCreateDto;
@@ -48,7 +48,7 @@ public interface DatabaseService {
* @throws SQLException
* @throws ViewMalformedException
*/
- ViewDto createView(DatabaseDto database, ViewCreateDto data) throws SQLException,
+ ViewDto createView(DatabaseDto database, CreateViewDto data) throws SQLException,
ViewMalformedException;
/**
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/SubsetService.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/SubsetService.java
index b2de5cecca0d5ec607563e658ee210186997a91e..e2fc56002799d26e75fc05ca496cb76d330cdb07 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/SubsetService.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/SubsetService.java
@@ -1,5 +1,6 @@
package at.tuwien.service;
+import at.tuwien.api.SortTypeDto;
import at.tuwien.api.database.DatabaseDto;
import at.tuwien.api.database.query.QueryDto;
import at.tuwien.exception.*;
@@ -13,22 +14,6 @@ import java.util.UUID;
public interface SubsetService {
- /**
- * Retrieve data from a subset in a database and optionally paginate with number of page and size of results.
- *
- * @param database The database.
- * @param subset The subset.
- * @param page The page number.
- * @param size Te result size.
- * @return The data.
- * @throws ViewMalformedException The view is malformed.
- * @throws SQLException The connection to the database could not be established.
- * @throws QueryMalformedException The mapped query produced a database error.
- * @throws TableNotFoundException The database table is malformed.
- */
- Dataset<Row> getData(DatabaseDto database, QueryDto subset, Long page, Long size)
- throws ViewMalformedException, SQLException, QueryMalformedException, TableNotFoundException;
-
/**
* Creates a subset from the given statement at given time in the given database.
*
@@ -55,6 +40,21 @@ public interface SubsetService {
Long reExecuteCount(DatabaseDto database, QueryDto query) throws TableMalformedException,
SQLException, QueryMalformedException;
+ /**
+ * Retrieve data from a subset in a database and optionally paginate with number of page and size of results.
+ *
+ * @param database The database.
+ * @param query The query statements.
+ * @param page The page number.
+ * @param size Te result size.
+ * @return The data.
+ * @throws QueryMalformedException The mapped query produced a database error.
+ * @throws TableNotFoundException The database table is malformed.
+ */
+ Dataset<Row> getData(DatabaseDto database, String query, Instant timestamp, Long page, Long size,
+ SortTypeDto sortDirection, String sortColumn) throws QueryMalformedException,
+ TableNotFoundException;
+
/**
* Finds all queries in the query store of the given database id and query id.
*
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/TableService.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/TableService.java
index 636e12ceb687cb3c6ce9d3988931caea676ef7ad..f664a82cf32050512f5eeb7d9e2b206289153568 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/TableService.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/TableService.java
@@ -1,12 +1,8 @@
package at.tuwien.service;
-import at.tuwien.api.SortTypeDto;
-import at.tuwien.api.database.DatabaseDto;
import at.tuwien.api.database.query.ImportDto;
import at.tuwien.api.database.table.*;
import at.tuwien.exception.*;
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.Row;
import java.sql.SQLException;
import java.time.Instant;
@@ -122,8 +118,4 @@ public interface TableService {
*/
void updateTuple(TableDto table, TupleUpdateDto data) throws SQLException,
QueryMalformedException, TableMalformedException;
-
- Dataset<Row> getData(DatabaseDto database, String tableOrView, Instant timestamp,
- Long page, Long size, SortTypeDto sortDirection, String sortColumn)
- throws QueryMalformedException, TableNotFoundException;
}
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DataConnector.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DataConnector.java
index c21d37721b051bf4109b6062a1ffad493c503850..1786ca5cb44ccef1c52d56af1411b4caf436fa1e 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DataConnector.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DataConnector.java
@@ -49,15 +49,11 @@ public abstract class DataConnector {
}
public String getSparkUrl(TableDto table) {
- return getSparkUrl(table.getDatabase().getContainer(), null);
+ return getSparkUrl(table.getDatabase().getContainer(), table.getDatabase().getInternalName());
}
public String getSparkUrl(DatabaseDto databaseDto) {
- return getSparkUrl(databaseDto.getContainer(), null);
- }
-
- public String getSparkUrl(ContainerDto container) {
- return getSparkUrl(container, null);
+ return getSparkUrl(databaseDto.getContainer(), databaseDto.getInternalName());
}
public String getJdbcUrl(ContainerDto container, String databaseName) {
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceMariaDbImpl.java
index c4afe44287abbb5c184fa6ec0f8873d17cf9edcb..d733800a3662adc0d57d2d81624914a8753c1b98 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceMariaDbImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceMariaDbImpl.java
@@ -1,7 +1,7 @@
package at.tuwien.service.impl;
+import at.tuwien.api.database.CreateViewDto;
import at.tuwien.api.database.DatabaseDto;
-import at.tuwien.api.database.ViewCreateDto;
import at.tuwien.api.database.ViewDto;
import at.tuwien.api.database.table.TableDto;
import at.tuwien.api.database.table.constraints.unique.UniqueDto;
@@ -146,7 +146,7 @@ public class DatabaseServiceMariaDbImpl extends DataConnector implements Databas
}
@Override
- public ViewDto createView(DatabaseDto database, ViewCreateDto data) throws SQLException,
+ public ViewDto createView(DatabaseDto database, CreateViewDto data) throws SQLException,
ViewMalformedException {
final ComboPooledDataSource dataSource = getDataSource(database);
final Connection connection = dataSource.getConnection();
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java
index 0a208b2890e8702228fb07ca31654625a17b0dd2..bdfdb14838a608ce87b40a94faebe52a27546a63 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java
@@ -1,7 +1,7 @@
package at.tuwien.service.impl;
+import at.tuwien.api.SortTypeDto;
import at.tuwien.api.database.DatabaseDto;
-import at.tuwien.api.database.ViewCreateDto;
import at.tuwien.api.database.query.QueryDto;
import at.tuwien.api.identifier.IdentifierBriefDto;
import at.tuwien.api.identifier.IdentifierTypeDto;
@@ -9,14 +9,13 @@ import at.tuwien.exception.*;
import at.tuwien.gateway.MetadataServiceGateway;
import at.tuwien.mapper.DataMapper;
import at.tuwien.mapper.MariaDbMapper;
-import at.tuwien.mapper.MetadataMapper;
-import at.tuwien.service.DatabaseService;
import at.tuwien.service.SubsetService;
-import at.tuwien.service.TableService;
import com.mchange.v2.c3p0.ComboPooledDataSource;
import lombok.extern.log4j.Log4j2;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.catalyst.ExtendedAnalysisException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -31,40 +30,41 @@ import java.util.UUID;
public class SubsetServiceMariaDbImpl extends DataConnector implements SubsetService {
private final DataMapper dataMapper;
- private final TableService tableService;
+ private final SparkSession sparkSession;
private final MariaDbMapper mariaDbMapper;
- private final MetadataMapper metadataMapper;
- private final DatabaseService databaseService;
private final MetadataServiceGateway metadataServiceGateway;
@Autowired
- public SubsetServiceMariaDbImpl(DataMapper dataMapper, TableService tableService, MariaDbMapper mariaDbMapper,
- MetadataMapper metadataMapper, DatabaseService databaseService,
+ public SubsetServiceMariaDbImpl(DataMapper dataMapper, MariaDbMapper mariaDbMapper, SparkSession sparkSession,
MetadataServiceGateway metadataServiceGateway) {
this.dataMapper = dataMapper;
- this.tableService = tableService;
+ this.sparkSession = sparkSession;
this.mariaDbMapper = mariaDbMapper;
- this.metadataMapper = metadataMapper;
- this.databaseService = databaseService;
this.metadataServiceGateway = metadataServiceGateway;
}
@Override
- public Dataset<Row> getData(DatabaseDto database, QueryDto subset, Long page, Long size)
- throws ViewMalformedException, SQLException, QueryMalformedException, TableNotFoundException {
- final String viewName = metadataMapper.queryDtoToViewName(subset);
- if (!databaseService.existsView(database, viewName)) {
- log.warn("Missing internal view {} for subset with id {}: create it from subset query", viewName, subset.getId());
- databaseService.createView(database, ViewCreateDto.builder()
- .isPublic(false)
- .isSchemaPublic(false)
- .name(viewName)
- .query(subset.getQuery())
- .build());
- } else {
- log.debug("internal view {}.{} for subset with id {} exists", database.getInternalName(), viewName, subset.getId());
+ public Dataset<Row> getData(DatabaseDto database, String query, Instant timestamp, Long page, Long size,
+ SortTypeDto sortDirection, String sortColumn)
+ throws QueryMalformedException, TableNotFoundException {
+ try {
+ return sparkSession.read()
+ .format("jdbc")
+ .option("user", database.getContainer().getUsername())
+ .option("password", database.getContainer().getPassword())
+ .option("url", getSparkUrl(database))
+ .option("query", query)
+ .load();
+ } catch (Exception e) {
+ if (e instanceof ExtendedAnalysisException exception) {
+ if (exception.getSimpleMessage().contains("TABLE_OR_VIEW_NOT_FOUND")) {
+ log.error("Failed to find named reference: {}", exception.getSimpleMessage());
+ throw new TableNotFoundException("Failed to find named reference: " + exception.getSimpleMessage()) /* remove throwable on purpose, clutters the output */;
+ }
+ }
+ log.error("Malformed query: {}", e.getMessage());
+ throw new QueryMalformedException("Malformed query: " + e.getMessage(), e);
}
- return tableService.getData(database, viewName, subset.getExecution(), page, size, null, null);
}
@Override
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java
index 4d4121f06049251cadebe498ebb32ef9f6a4d409..466f7539fd250666962d08a75ffffd5aa61480fa 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java
@@ -1,7 +1,5 @@
package at.tuwien.service.impl;
-import at.tuwien.api.SortTypeDto;
-import at.tuwien.api.database.DatabaseDto;
import at.tuwien.api.database.query.ImportDto;
import at.tuwien.api.database.table.*;
import at.tuwien.api.database.table.columns.ColumnDto;
@@ -16,8 +14,10 @@ import at.tuwien.service.TableService;
import at.tuwien.utils.MariaDbUtil;
import com.mchange.v2.c3p0.ComboPooledDataSource;
import lombok.extern.log4j.Log4j2;
-import org.apache.spark.sql.*;
-import org.apache.spark.sql.catalyst.ExtendedAnalysisException;
+import org.apache.spark.sql.AnalysisException;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SaveMode;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -36,16 +36,14 @@ import java.util.Properties;
public class TableServiceMariaDbImpl extends DataConnector implements TableService {
private final DataMapper dataMapper;
- private final SparkSession sparkSession;
private final MariaDbMapper mariaDbMapper;
private final StorageService storageService;
private final DatabaseService databaseService;
@Autowired
- public TableServiceMariaDbImpl(DataMapper dataMapper, SparkSession sparkSession, MariaDbMapper mariaDbMapper,
- StorageService storageService, DatabaseService databaseService) {
+ public TableServiceMariaDbImpl(DataMapper dataMapper, MariaDbMapper mariaDbMapper, StorageService storageService,
+ DatabaseService databaseService) {
this.dataMapper = dataMapper;
- this.sparkSession = sparkSession;
this.mariaDbMapper = mariaDbMapper;
this.storageService = storageService;
this.databaseService = databaseService;
@@ -368,30 +366,4 @@ public class TableServiceMariaDbImpl extends DataConnector implements TableServi
.getColumnType();
}
- @Override
- public Dataset<Row> getData(DatabaseDto database, String tableOrView, Instant timestamp,
- Long page, Long size, SortTypeDto sortDirection, String sortColumn)
- throws QueryMalformedException, TableNotFoundException {
- try {
- return sparkSession.read()
- .format("jdbc")
- .option("user", database.getContainer().getUsername())
- .option("password", database.getContainer().getPassword())
- .option("url", getSparkUrl(database))
- .option("query", mariaDbMapper.defaultRawSelectQuery(database.getInternalName(), tableOrView,
- timestamp, page, size))
- .load();
-
- } catch (Exception e) {
- if (e instanceof ExtendedAnalysisException exception) {
- if (exception.getSimpleMessage().contains("TABLE_OR_VIEW_NOT_FOUND")) {
- log.error("Failed to find named reference: {}", exception.getSimpleMessage());
- throw new TableNotFoundException("Failed to find named reference: " + exception.getSimpleMessage()) /* remove throwable on purpose, clutters the output */;
- }
- }
- log.error("Malformed query: {}", e.getMessage());
- throw new QueryMalformedException("Malformed query: " + e.getMessage(), e);
- }
- }
-
}