Added pagination

Former-commit-id: b66860f9

fda-metadata-db/api/src/main/java/at/tuwien/api/database/query/QueryResultDto.java

@@ -4,6 +4,7 @@ import io.swagger.annotations.ApiModelProperty;
 import lombok.*;
 
 import javax.validation.constraints.NotNull;
+import java.math.BigInteger;
 import java.util.List;
 import java.util.Map;
 
@@ -24,4 +25,7 @@ public class QueryResultDto {
     @ApiModelProperty(notes = "query id")
     private Long id;
 
+    @ApiModelProperty(notes = "result number")
+    private Long resultNumber;
+
 }

fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java

@@ -55,7 +55,7 @@ public class QueryEndpoint {
                                                   @Valid @RequestBody ExecuteStatementDto data,
                                                   @RequestParam(value = "page", required = false ) Long page, @RequestParam(value = "size", required = false) Long size)
             throws DatabaseNotFoundException, ImageNotSupportedException, QueryStoreException, QueryMalformedException,
-            TableNotFoundException, ContainerNotFoundException, SQLException, JSQLParserException {
+            TableNotFoundException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException {
         /* validation */
         if (data.getStatement() == null || data.getStatement().isBlank()) {
             log.error("Query is empty");
@@ -105,7 +105,7 @@ public class QueryEndpoint {
                                                     @NotNull @PathVariable("queryId") Long queryId,
                                                     @RequestParam(value = "page", required = false) Long page, @RequestParam(value = "size", required = false) Long size)
             throws QueryStoreException, QueryNotFoundException, DatabaseNotFoundException, ImageNotSupportedException,
-            TableNotFoundException, QueryMalformedException, ContainerNotFoundException, SQLException, JSQLParserException {
+            TableNotFoundException, QueryMalformedException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException {
         final Query query = storeService.findOne(id, databaseId, queryId);
         log.debug(query.toString());
         final QueryResultDto result = queryService.reExecute(id, databaseId, query, page, size);

fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointIntegrationTest.java

@@ -132,7 +132,7 @@ public class QueryEndpointIntegrationTest extends BaseUnitTest {
     @Test
     public void reExecute_succeeds() throws TableNotFoundException, QueryStoreException, QueryMalformedException,
             DatabaseNotFoundException, ImageNotSupportedException, QueryNotFoundException, InterruptedException,
-            SQLException, ContainerNotFoundException, JSQLParserException {
+            SQLException, ContainerNotFoundException, JSQLParserException, TableMalformedException {
         final QueryResultDto result = QueryResultDto.builder()
                 .id(QUERY_1_ID)
                 .result(List.of(Map.of("MinTemp", 13.4, "Rainfall", 0.6, "id", 1)))

fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointUnitTest.java

@@ -47,7 +47,7 @@ public class QueryEndpointUnitTest extends BaseUnitTest {
 
     @Test
     public void execute_succeeds() throws TableNotFoundException, QueryStoreException, QueryMalformedException,
-            DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException {
+            DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException {
         final ExecuteStatementDto request = ExecuteStatementDto.builder()
                 .statement(QUERY_1_STATEMENT)
                 .build();
@@ -73,7 +73,7 @@ public class QueryEndpointUnitTest extends BaseUnitTest {
 
     @Test
     public void execute_emptyResult_succeeds() throws TableNotFoundException, QueryStoreException,
-            QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException {
+            QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException {
         final ExecuteStatementDto request = ExecuteStatementDto.builder()
                 .statement(QUERY_1_STATEMENT)
                 .build();
@@ -99,7 +99,7 @@ public class QueryEndpointUnitTest extends BaseUnitTest {
 
     @Test
     public void execute_tableNotFound_fails() throws TableNotFoundException, QueryMalformedException,
-            DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, QueryStoreException, SQLException, JSQLParserException {
+            DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, QueryStoreException, SQLException, JSQLParserException, TableMalformedException {
         final ExecuteStatementDto request = ExecuteStatementDto.builder()
                 .statement(QUERY_1_STATEMENT)
                 .build();

fda-query-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java

@@ -174,7 +174,7 @@ public class QueryServiceIntegrationTest extends BaseUnitTest {
 
     @Test
     public void execute_succeeds() throws DatabaseNotFoundException, ImageNotSupportedException, InterruptedException,
-            QueryMalformedException, TableNotFoundException, QueryStoreException, ContainerNotFoundException, SQLException, JSQLParserException {
+            QueryMalformedException, TableNotFoundException, QueryStoreException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException {
         final ExecuteStatementDto request = ExecuteStatementDto.builder()
                 .statement(QUERY_1_STATEMENT)
                 .build();
@@ -208,7 +208,7 @@ public class QueryServiceIntegrationTest extends BaseUnitTest {
     @Disabled
     public void execute_modifyData_fails() throws DatabaseNotFoundException, ImageNotSupportedException,
             InterruptedException, QueryMalformedException, TableNotFoundException, QueryStoreException,
-            ContainerNotFoundException, SQLException, JSQLParserException {
+            ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException {
         final ExecuteStatementDto request = ExecuteStatementDto.builder()
                 .statement("DELETE FROM `weather_aus`;")
                 .build();

fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java

@@ -18,6 +18,7 @@ import org.mariadb.jdbc.MariaDbBlob;
 import org.springframework.transaction.annotation.Transactional;
 
 import java.math.BigInteger;
+import java.nio.charset.MalformedInputException;
 import java.text.Normalizer;
 import java.time.*;
 import java.time.format.DateTimeFormatter;
@@ -171,7 +172,7 @@ public interface QueryMapper {
             throw new ImageNotSupportedException("Currently only MariaDB is supported");
         }
         if (timestamp == null) {
-            timestamp = Instant.now();
+            throw new IllegalArgumentException("Timestamp must be provided");
         }
         return "SELECT COUNT(*) FROM " +  query.toLowerCase(Locale.ROOT).split("from ")[1] +
                 " FOR SYSTEM_TIME AS OF TIMESTAMP'" +
@@ -179,7 +180,7 @@ public interface QueryMapper {
                 "';";
     }
 
-    default String queryToRawTimestampedQuery(String query, Database database, Instant timestamp) throws ImageNotSupportedException {
+    default String queryToRawTimestampedQuery(String query, Database database, Instant timestamp, Long page, Long size) throws ImageNotSupportedException {
         /* param check */
         if (!database.getContainer().getImage().getRepository().equals("mariadb")) {
             throw new ImageNotSupportedException("Currently only MariaDB is supported");
@@ -187,11 +188,18 @@ public interface QueryMapper {
         if (timestamp == null) {
             timestamp = Instant.now();
         }
+        if(size != null && page != null && size > 0 && page >=0) {
             return query +
                     " FOR SYSTEM_TIME AS OF TIMESTAMP'" +
                     LocalDateTime.ofInstant(timestamp, ZoneId.of("Europe/Vienna")) +
+                    " LIMIT " + size + " OFFSET " + (page*size) +
                     "';";
         }
+        return query +
+                " FOR SYSTEM_TIME AS OF TIMESTAMP'" +
+                LocalDateTime.ofInstant(timestamp, ZoneId.of("Europe/Vienna")) + "';";
+
+    }
 
 
 

fda-query-service/services/src/main/java/at/tuwien/querystore/Query.java

@@ -8,6 +8,7 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener;
 
 import javax.persistence.*;
 import java.io.Serializable;
+import java.math.BigInteger;
 import java.time.Instant;
 import java.util.List;
 

fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java

@@ -32,7 +32,7 @@ public interface QueryService {
      * @throws ImageNotSupportedException
      */
     QueryResultDto execute(Long containerId, Long databaseId, ExecuteStatementDto query, Long page, Long size) throws TableNotFoundException,
-            QueryStoreException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException;
+            QueryStoreException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException;
 
     /**
      * Re-Executes an arbitrary query on the database container. We allow the user to only view the data, therefore the
@@ -50,7 +50,7 @@ public interface QueryService {
      * @throws ImageNotSupportedException
      */
     QueryResultDto reExecute(Long containerId, Long databaseId, Query query, Long page, Long size) throws TableNotFoundException,
-            QueryStoreException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException;
+            QueryStoreException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException;
 
 
     /**

fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java

@@ -61,17 +61,16 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
     @Override
     @Transactional
     public QueryResultDto execute(Long containerId, Long databaseId, ExecuteStatementDto statement, Long page, Long size)
-            throws DatabaseNotFoundException, ImageNotSupportedException, QueryMalformedException, QueryStoreException, ContainerNotFoundException, TableNotFoundException, SQLException, JSQLParserException {
+            throws DatabaseNotFoundException, ImageNotSupportedException, QueryMalformedException, QueryStoreException, ContainerNotFoundException, TableNotFoundException, SQLException, JSQLParserException, TableMalformedException {
         Instant i = Instant.now();
         Query q = storeService.insert(containerId, databaseId, null, statement, i);
         final QueryResultDto result = this.reExecute(containerId,databaseId,q,page,size);
-        Long resultNumber = 0L;
-        q = storeService.update(containerId,databaseId,result, resultNumber,q);
+        q = storeService.update(containerId,databaseId,result, result.getResultNumber(),q);
         return result;
     }
 
     @Override
-    public QueryResultDto reExecute(Long containerId, Long databaseId, Query query, Long page, Long size) throws TableNotFoundException, QueryStoreException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException {
+    public QueryResultDto reExecute(Long containerId, Long databaseId, Query query, Long page, Long size) throws TableNotFoundException, QueryStoreException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException {
         /* find */
         final Database database = databaseService.find(databaseId);
         if (!database.getContainer().getImage().getRepository().equals("mariadb")) {
@@ -85,7 +84,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
         session.beginTransaction();
         /* prepare the statement */
         Instant i = Instant.now();
-        final NativeQuery<?> nativeQuery = session.createSQLQuery(queryMapper.queryToRawTimestampedQuery(query.getQuery(), database, query.getExecution()));
+        final NativeQuery<?> nativeQuery = session.createSQLQuery(queryMapper.queryToRawTimestampedQuery(query.getQuery(), database, query.getExecution(),page, size));
         final int affectedTuples;
         try {
             log.debug("execute raw view-only query {}", query);
@@ -100,8 +99,9 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
         }
         /* map the result to the tables (with respective columns) from the statement metadata */
         final List<TableColumn> columns = parseColumns(query, database);
-        final QueryResultDto result = queryMapper.resultListToQueryResultDto(columns, nativeQuery.getResultList());
-        /* Save query in the querystore */
+        QueryResultDto result = queryMapper.resultListToQueryResultDto(columns, nativeQuery.getResultList());
+        result.setResultNumber(query.getResultNumber()!=null ? query.getResultNumber() : countQueryResults(containerId,databaseId,query));
+        result.setId(query.getId());
         session.close();
         factory.close();
         return result;
@@ -350,5 +350,36 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
 
     }
 
+    @Transactional
+    Long countQueryResults(Long containerId, Long databaseId, Query query)
+            throws DatabaseNotFoundException, TableNotFoundException,
+            TableMalformedException, ImageNotSupportedException {
+        /* find */
+        final Database database = databaseService.find(databaseId);
+        /* run query */
+        final long startSession = System.currentTimeMillis();
+        final SessionFactory factory = getSessionFactory(database, false);
+        final Session session = factory.openSession();
+        log.debug("opened hibernate session in {} ms", System.currentTimeMillis() - startSession);
+        session.beginTransaction();
+        final NativeQuery<BigInteger> nativeQuery = session.createSQLQuery(queryMapper.queryToRawTimestampedCountQuery(query.getQuery(), database, query.getExecution()));
+        final int affectedTuples;
+        try {
+            affectedTuples = nativeQuery.executeUpdate();
+            log.info("Counted {} tuples from query {}", affectedTuples, query.getId());
+        } catch (PersistenceException e) {
+            log.error("Failed to count tuples");
+            session.close();
+            factory.close();
+            throw new TableMalformedException("Data not found", e);
+        }
+        session.getTransaction()
+                .commit();
+        final Long count = nativeQuery.getSingleResult().longValue();
+        session.close();
+        factory.close();
+        return count;
+    }
+
 
 }