From df33e92b98e2b8793fc177a6368204ef084e901f Mon Sep 17 00:00:00 2001
From: Moritz Staudinger <moritz.staudinger@tuwien.ac.at>
Date: Thu, 3 Mar 2022 04:05:07 +0100
Subject: [PATCH] Added pagination
Former-commit-id: b66860f974d57d4f382f76a9b93c17db13b7f2e2
---
.../api/database/query/QueryResultDto.java | 4 ++
.../at/tuwien/endpoint/QueryEndpoint.java | 4 +-
.../QueryEndpointIntegrationTest.java | 2 +-
.../endpoint/QueryEndpointUnitTest.java | 6 +--
.../service/QueryServiceIntegrationTest.java | 4 +-
.../java/at/tuwien/mapper/QueryMapper.java | 16 +++++--
.../main/java/at/tuwien/querystore/Query.java | 1 +
.../java/at/tuwien/service/QueryService.java | 4 +-
.../tuwien/service/impl/QueryServiceImpl.java | 45 ++++++++++++++++---
9 files changed, 65 insertions(+), 21 deletions(-)
diff --git a/fda-metadata-db/api/src/main/java/at/tuwien/api/database/query/QueryResultDto.java b/fda-metadata-db/api/src/main/java/at/tuwien/api/database/query/QueryResultDto.java
index 31622806f0..b7205a08d1 100644
--- a/fda-metadata-db/api/src/main/java/at/tuwien/api/database/query/QueryResultDto.java
+++ b/fda-metadata-db/api/src/main/java/at/tuwien/api/database/query/QueryResultDto.java
@@ -4,6 +4,7 @@ import io.swagger.annotations.ApiModelProperty;
import lombok.*;
import javax.validation.constraints.NotNull;
+import java.math.BigInteger;
import java.util.List;
import java.util.Map;
@@ -24,4 +25,7 @@ public class QueryResultDto {
@ApiModelProperty(notes = "query id")
private Long id;
+ @ApiModelProperty(notes = "result number")
+ private Long resultNumber;
+
}
diff --git a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java
index 2b0266168f..c26201c7f7 100644
--- a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java
+++ b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java
@@ -55,7 +55,7 @@ public class QueryEndpoint {
@Valid @RequestBody ExecuteStatementDto data,
@RequestParam(value = "page", required = false ) Long page, @RequestParam(value = "size", required = false) Long size)
throws DatabaseNotFoundException, ImageNotSupportedException, QueryStoreException, QueryMalformedException,
- TableNotFoundException, ContainerNotFoundException, SQLException, JSQLParserException {
+ TableNotFoundException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException {
/* validation */
if (data.getStatement() == null || data.getStatement().isBlank()) {
log.error("Query is empty");
@@ -105,7 +105,7 @@ public class QueryEndpoint {
@NotNull @PathVariable("queryId") Long queryId,
@RequestParam(value = "page", required = false) Long page, @RequestParam(value = "size", required = false) Long size)
throws QueryStoreException, QueryNotFoundException, DatabaseNotFoundException, ImageNotSupportedException,
- TableNotFoundException, QueryMalformedException, ContainerNotFoundException, SQLException, JSQLParserException {
+ TableNotFoundException, QueryMalformedException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException {
final Query query = storeService.findOne(id, databaseId, queryId);
log.debug(query.toString());
final QueryResultDto result = queryService.reExecute(id, databaseId, query, page, size);
diff --git a/fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointIntegrationTest.java b/fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointIntegrationTest.java
index d5fef7bf9d..760222876c 100644
--- a/fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointIntegrationTest.java
+++ b/fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointIntegrationTest.java
@@ -132,7 +132,7 @@ public class QueryEndpointIntegrationTest extends BaseUnitTest {
@Test
public void reExecute_succeeds() throws TableNotFoundException, QueryStoreException, QueryMalformedException,
DatabaseNotFoundException, ImageNotSupportedException, QueryNotFoundException, InterruptedException,
- SQLException, ContainerNotFoundException, JSQLParserException {
+ SQLException, ContainerNotFoundException, JSQLParserException, TableMalformedException {
final QueryResultDto result = QueryResultDto.builder()
.id(QUERY_1_ID)
.result(List.of(Map.of("MinTemp", 13.4, "Rainfall", 0.6, "id", 1)))
diff --git a/fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointUnitTest.java b/fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointUnitTest.java
index 6aaedeb4ca..da894a812e 100644
--- a/fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointUnitTest.java
+++ b/fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointUnitTest.java
@@ -47,7 +47,7 @@ public class QueryEndpointUnitTest extends BaseUnitTest {
@Test
public void execute_succeeds() throws TableNotFoundException, QueryStoreException, QueryMalformedException,
- DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException {
+ DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException {
final ExecuteStatementDto request = ExecuteStatementDto.builder()
.statement(QUERY_1_STATEMENT)
.build();
@@ -73,7 +73,7 @@ public class QueryEndpointUnitTest extends BaseUnitTest {
@Test
public void execute_emptyResult_succeeds() throws TableNotFoundException, QueryStoreException,
- QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException {
+ QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException {
final ExecuteStatementDto request = ExecuteStatementDto.builder()
.statement(QUERY_1_STATEMENT)
.build();
@@ -99,7 +99,7 @@ public class QueryEndpointUnitTest extends BaseUnitTest {
@Test
public void execute_tableNotFound_fails() throws TableNotFoundException, QueryMalformedException,
- DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, QueryStoreException, SQLException, JSQLParserException {
+ DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, QueryStoreException, SQLException, JSQLParserException, TableMalformedException {
final ExecuteStatementDto request = ExecuteStatementDto.builder()
.statement(QUERY_1_STATEMENT)
.build();
diff --git a/fda-query-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java b/fda-query-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java
index 13d99fdd74..aa1052b621 100644
--- a/fda-query-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java
+++ b/fda-query-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java
@@ -174,7 +174,7 @@ public class QueryServiceIntegrationTest extends BaseUnitTest {
@Test
public void execute_succeeds() throws DatabaseNotFoundException, ImageNotSupportedException, InterruptedException,
- QueryMalformedException, TableNotFoundException, QueryStoreException, ContainerNotFoundException, SQLException, JSQLParserException {
+ QueryMalformedException, TableNotFoundException, QueryStoreException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException {
final ExecuteStatementDto request = ExecuteStatementDto.builder()
.statement(QUERY_1_STATEMENT)
.build();
@@ -208,7 +208,7 @@ public class QueryServiceIntegrationTest extends BaseUnitTest {
@Disabled
public void execute_modifyData_fails() throws DatabaseNotFoundException, ImageNotSupportedException,
InterruptedException, QueryMalformedException, TableNotFoundException, QueryStoreException,
- ContainerNotFoundException, SQLException, JSQLParserException {
+ ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException {
final ExecuteStatementDto request = ExecuteStatementDto.builder()
.statement("DELETE FROM `weather_aus`;")
.build();
diff --git a/fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java b/fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java
index 65b8491531..96df51c79f 100644
--- a/fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java
+++ b/fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java
@@ -18,6 +18,7 @@ import org.mariadb.jdbc.MariaDbBlob;
import org.springframework.transaction.annotation.Transactional;
import java.math.BigInteger;
+import java.nio.charset.MalformedInputException;
import java.text.Normalizer;
import java.time.*;
import java.time.format.DateTimeFormatter;
@@ -171,7 +172,7 @@ public interface QueryMapper {
throw new ImageNotSupportedException("Currently only MariaDB is supported");
}
if (timestamp == null) {
- timestamp = Instant.now();
+ throw new IllegalArgumentException("Timestamp must be provided");
}
return "SELECT COUNT(*) FROM " + query.toLowerCase(Locale.ROOT).split("from ")[1] +
" FOR SYSTEM_TIME AS OF TIMESTAMP'" +
@@ -179,7 +180,7 @@ public interface QueryMapper {
"';";
}
- default String queryToRawTimestampedQuery(String query, Database database, Instant timestamp) throws ImageNotSupportedException {
+ default String queryToRawTimestampedQuery(String query, Database database, Instant timestamp, Long page, Long size) throws ImageNotSupportedException {
/* param check */
if (!database.getContainer().getImage().getRepository().equals("mariadb")) {
throw new ImageNotSupportedException("Currently only MariaDB is supported");
@@ -187,10 +188,17 @@ public interface QueryMapper {
if (timestamp == null) {
timestamp = Instant.now();
}
+ if(size != null && page != null && size > 0 && page >=0) {
+ return query +
+ " FOR SYSTEM_TIME AS OF TIMESTAMP'" +
+ LocalDateTime.ofInstant(timestamp, ZoneId.of("Europe/Vienna")) +
+ " LIMIT " + size + " OFFSET " + (page*size) +
+ "';";
+ }
return query +
" FOR SYSTEM_TIME AS OF TIMESTAMP'" +
- LocalDateTime.ofInstant(timestamp, ZoneId.of("Europe/Vienna")) +
- "';";
+ LocalDateTime.ofInstant(timestamp, ZoneId.of("Europe/Vienna")) + "';";
+
}
diff --git a/fda-query-service/services/src/main/java/at/tuwien/querystore/Query.java b/fda-query-service/services/src/main/java/at/tuwien/querystore/Query.java
index bbdeb2b783..de1d12f70f 100644
--- a/fda-query-service/services/src/main/java/at/tuwien/querystore/Query.java
+++ b/fda-query-service/services/src/main/java/at/tuwien/querystore/Query.java
@@ -8,6 +8,7 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener;
import javax.persistence.*;
import java.io.Serializable;
+import java.math.BigInteger;
import java.time.Instant;
import java.util.List;
diff --git a/fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java b/fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java
index 774748022e..3dfbfdf2e3 100644
--- a/fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java
+++ b/fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java
@@ -32,7 +32,7 @@ public interface QueryService {
* @throws ImageNotSupportedException
*/
QueryResultDto execute(Long containerId, Long databaseId, ExecuteStatementDto query, Long page, Long size) throws TableNotFoundException,
- QueryStoreException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException;
+ QueryStoreException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException;
/**
* Re-Executes an arbitrary query on the database container. We allow the user to only view the data, therefore the
@@ -50,7 +50,7 @@ public interface QueryService {
* @throws ImageNotSupportedException
*/
QueryResultDto reExecute(Long containerId, Long databaseId, Query query, Long page, Long size) throws TableNotFoundException,
- QueryStoreException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException;
+ QueryStoreException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException;
/**
diff --git a/fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java b/fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java
index ccbf4f5ff6..7daf02a5cd 100644
--- a/fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java
+++ b/fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java
@@ -61,17 +61,16 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
@Override
@Transactional
public QueryResultDto execute(Long containerId, Long databaseId, ExecuteStatementDto statement, Long page, Long size)
- throws DatabaseNotFoundException, ImageNotSupportedException, QueryMalformedException, QueryStoreException, ContainerNotFoundException, TableNotFoundException, SQLException, JSQLParserException {
+ throws DatabaseNotFoundException, ImageNotSupportedException, QueryMalformedException, QueryStoreException, ContainerNotFoundException, TableNotFoundException, SQLException, JSQLParserException, TableMalformedException {
Instant i = Instant.now();
Query q = storeService.insert(containerId, databaseId, null, statement, i);
final QueryResultDto result = this.reExecute(containerId,databaseId,q,page,size);
- Long resultNumber = 0L;
- q = storeService.update(containerId,databaseId,result, resultNumber,q);
+ q = storeService.update(containerId,databaseId,result, result.getResultNumber(),q);
return result;
}
@Override
- public QueryResultDto reExecute(Long containerId, Long databaseId, Query query, Long page, Long size) throws TableNotFoundException, QueryStoreException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException {
+ public QueryResultDto reExecute(Long containerId, Long databaseId, Query query, Long page, Long size) throws TableNotFoundException, QueryStoreException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException {
/* find */
final Database database = databaseService.find(databaseId);
if (!database.getContainer().getImage().getRepository().equals("mariadb")) {
@@ -85,7 +84,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
session.beginTransaction();
/* prepare the statement */
Instant i = Instant.now();
- final NativeQuery<?> nativeQuery = session.createSQLQuery(queryMapper.queryToRawTimestampedQuery(query.getQuery(), database, query.getExecution()));
+ final NativeQuery<?> nativeQuery = session.createSQLQuery(queryMapper.queryToRawTimestampedQuery(query.getQuery(), database, query.getExecution(),page, size));
final int affectedTuples;
try {
log.debug("execute raw view-only query {}", query);
@@ -100,8 +99,9 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
}
/* map the result to the tables (with respective columns) from the statement metadata */
final List<TableColumn> columns = parseColumns(query, database);
- final QueryResultDto result = queryMapper.resultListToQueryResultDto(columns, nativeQuery.getResultList());
- /* Save query in the querystore */
+ QueryResultDto result = queryMapper.resultListToQueryResultDto(columns, nativeQuery.getResultList());
+ result.setResultNumber(query.getResultNumber()!=null ? query.getResultNumber() : countQueryResults(containerId,databaseId,query));
+ result.setId(query.getId());
session.close();
factory.close();
return result;
@@ -350,5 +350,36 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
}
+ @Transactional
+ Long countQueryResults(Long containerId, Long databaseId, Query query)
+ throws DatabaseNotFoundException, TableNotFoundException,
+ TableMalformedException, ImageNotSupportedException {
+ /* find */
+ final Database database = databaseService.find(databaseId);
+ /* run query */
+ final long startSession = System.currentTimeMillis();
+ final SessionFactory factory = getSessionFactory(database, false);
+ final Session session = factory.openSession();
+ log.debug("opened hibernate session in {} ms", System.currentTimeMillis() - startSession);
+ session.beginTransaction();
+ final NativeQuery<BigInteger> nativeQuery = session.createSQLQuery(queryMapper.queryToRawTimestampedCountQuery(query.getQuery(), database, query.getExecution()));
+ final int affectedTuples;
+ try {
+ affectedTuples = nativeQuery.executeUpdate();
+ log.info("Counted {} tuples from query {}", affectedTuples, query.getId());
+ } catch (PersistenceException e) {
+ log.error("Failed to count tuples");
+ session.close();
+ factory.close();
+ throw new TableMalformedException("Data not found", e);
+ }
+ session.getTransaction()
+ .commit();
+ final Long count = nativeQuery.getSingleResult().longValue();
+ session.close();
+ factory.close();
+ return count;
+ }
+
}
--
GitLab