diff --git a/fda-metadata-db/api/src/main/java/at/tuwien/api/database/query/ExecuteStatementDto.java b/fda-metadata-db/api/src/main/java/at/tuwien/api/database/query/ExecuteStatementDto.java index 15a8edcc4e5aa717f573ab7aa2d54cd1c18fb427..d19b9ab92b2d8577e8dce66f6284670d203632eb 100644 --- a/fda-metadata-db/api/src/main/java/at/tuwien/api/database/query/ExecuteStatementDto.java +++ b/fda-metadata-db/api/src/main/java/at/tuwien/api/database/query/ExecuteStatementDto.java @@ -14,6 +14,7 @@ import java.util.List; @Builder @AllArgsConstructor @NoArgsConstructor +@ToString public class ExecuteStatementDto { @NotBlank(message = "statement is required") diff --git a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java index a04384712e1209ae8fe4b0173a88255a55a29537..b9149086af0dbbb5db484e676f00a93b432ae868 100644 --- a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java +++ b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java @@ -37,6 +37,7 @@ public class QueryEndpoint { this.storeService = storeService; } + @Deprecated @PutMapping @Transactional @PreAuthorize("hasRole('ROLE_RESEARCHER')") @@ -61,10 +62,8 @@ public class QueryEndpoint { log.error("Table list is empty"); throw new QueryMalformedException("Invalid table"); } + log.debug("Data for execution: {}", data); final QueryResultDto result = queryService.execute(id, databaseId, data); - final QueryDto query = queryMapper.queryToQueryDto(storeService.insert(id, databaseId, result, data, - Instant.now())); - result.setId(query.getId()); return ResponseEntity.status(HttpStatus.ACCEPTED) .body(result); } @@ -104,8 +103,11 @@ public class QueryEndpoint { throws QueryStoreException, QueryNotFoundException, DatabaseNotFoundException, ImageNotSupportedException, TableNotFoundException, QueryMalformedException, ContainerNotFoundException { final Query query = storeService.findOne(id, databaseId, queryId); + log.debug(query.toString()); final QueryDto queryDto = queryMapper.queryToQueryDto(query); + log.debug(queryDto.toString()); final ExecuteStatementDto statement = queryMapper.queryDtoToExecuteStatementDto(queryDto); + log.debug(statement.toString()); final QueryResultDto result = queryService.execute(id, databaseId, statement); result.setId(queryId); return ResponseEntity.status(HttpStatus.ACCEPTED) diff --git a/fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointUnitTest.java b/fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointUnitTest.java index 8cd08fa6fa22318b5854e28ebaa8d295a908b2e7..ff3b0fc2809a365472f20db7056557f9a05f3858 100644 --- a/fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointUnitTest.java +++ b/fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointUnitTest.java @@ -93,7 +93,7 @@ public class QueryEndpointUnitTest extends BaseUnitTest { @Test public void execute_tableNotFound_fails() throws TableNotFoundException, QueryMalformedException, - DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException { + DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, QueryStoreException { final ExecuteStatementDto request = ExecuteStatementDto.builder() .statement(QUERY_1_STATEMENT) .build(); diff --git a/fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java b/fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java index afff49c34841c2d43e3dcb5e77e7a7f24ac8bc88..3ca553167e1041d215b1d139af0527d18f6d579d 100644 --- a/fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java +++ b/fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java @@ -2,7 +2,9 @@ package at.tuwien.mapper; import at.tuwien.InsertTableRawQuery; import at.tuwien.api.database.query.*; +import at.tuwien.api.database.table.TableBriefDto; import at.tuwien.api.database.table.TableCsvDto; +import at.tuwien.entities.database.Database; import at.tuwien.exception.TableMalformedException; import at.tuwien.querystore.Query; import at.tuwien.entities.database.table.Table; @@ -163,6 +165,38 @@ public interface QueryMapper { "';"; } + default String queryToRawTimestampedCountQuery(String query, Database database, Instant timestamp) throws ImageNotSupportedException { + /* param check */ + if (!database.getContainer().getImage().getRepository().equals("mariadb")) { + throw new ImageNotSupportedException("Currently only MariaDB is supported"); + } + if (timestamp == null) { + timestamp = Instant.now(); + } + return "SELECT COUNT(*) FROM " + query.toLowerCase(Locale.ROOT).split("from ")[1] + + " FOR SYSTEM_TIME AS OF TIMESTAMP'" + + LocalDateTime.ofInstant(timestamp, ZoneId.of("Europe/Vienna")) + + "';"; + } + + default String queryToRawTimestampedQuery(String query, Database database, Instant timestamp) throws ImageNotSupportedException { + /* param check */ + if (!database.getContainer().getImage().getRepository().equals("mariadb")) { + throw new ImageNotSupportedException("Currently only MariaDB is supported"); + } + if (timestamp == null) { + timestamp = Instant.now(); + } + return query + + " FOR SYSTEM_TIME AS OF TIMESTAMP'" + + LocalDateTime.ofInstant(timestamp, ZoneId.of("Europe/Vienna")) + + "';"; + } + + + + + default String tableToRawFindAllQuery(Table table, Instant timestamp, Long size, Long page) throws ImageNotSupportedException { /* param check */ diff --git a/fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java b/fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java index 09100ba5f690dc972c3dfdbc43ad7cc1e2091415..2cc4a3a9b1eb0addcee17235ebf290aaede2974d 100644 --- a/fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java +++ b/fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java @@ -3,6 +3,7 @@ package at.tuwien.service.impl; import at.tuwien.InsertTableRawQuery; import at.tuwien.api.database.query.ExecuteStatementDto; import at.tuwien.api.database.query.ImportDto; +import at.tuwien.api.database.query.QueryDto; import at.tuwien.api.database.query.QueryResultDto; import at.tuwien.api.database.table.TableCsvDto; import at.tuwien.entities.database.Database; @@ -10,9 +11,15 @@ import at.tuwien.entities.database.table.Table; import at.tuwien.entities.database.table.columns.TableColumn; import at.tuwien.exception.*; import at.tuwien.mapper.QueryMapper; +import at.tuwien.querystore.Query; import at.tuwien.repository.jpa.TableColumnRepository; import at.tuwien.service.*; import lombok.extern.log4j.Log4j2; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.parser.CCJSqlParserManager; +import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.statement.select.*; import org.hibernate.Session; import org.hibernate.SessionFactory; import org.hibernate.exception.SQLGrammarException; @@ -22,9 +29,12 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import javax.persistence.PersistenceException; +import java.io.StringReader; import java.math.BigInteger; +import java.sql.SQLException; import java.time.DateTimeException; import java.time.Instant; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Optional; @@ -38,20 +48,22 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService private final TableService tableService; private final DatabaseService databaseService; private final TableColumnRepository tableColumnRepository; + private final StoreService storeService; @Autowired public QueryServiceImpl(QueryMapper queryMapper, TableService tableService, DatabaseService databaseService, - TableColumnRepository tableColumnRepository) { + TableColumnRepository tableColumnRepository, StoreService storeService) { this.queryMapper = queryMapper; this.tableService = tableService; this.databaseService = databaseService; this.tableColumnRepository = tableColumnRepository; + this.storeService = storeService; } @Override @Transactional public QueryResultDto execute(Long containerId, Long databaseId, ExecuteStatementDto statement) - throws DatabaseNotFoundException, ImageNotSupportedException, QueryMalformedException { + throws DatabaseNotFoundException, ImageNotSupportedException, QueryMalformedException, QueryStoreException, ContainerNotFoundException { /* find */ final Database database = databaseService.find(databaseId); if (!database.getContainer().getImage().getRepository().equals("mariadb")) { @@ -64,7 +76,8 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService log.debug("opened hibernate session in {} ms", System.currentTimeMillis() - startSession); session.beginTransaction(); /* prepare the statement */ - final NativeQuery<?> query = session.createSQLQuery(statement.getStatement()); + Instant i = Instant.now(); + final NativeQuery<?> query = session.createSQLQuery(queryMapper.queryToRawTimestampedQuery(statement.getStatement(), database, i)); final int affectedTuples; try { log.debug("execute raw view-only query {}", statement); @@ -80,6 +93,9 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService /* map the result to the tables (with respective columns) from the statement metadata */ final List<TableColumn> columns = parseColumns(databaseId, statement); final QueryResultDto result = queryMapper.resultListToQueryResultDto(columns, query.getResultList()); + /* Ssave query in the querystore */ + Query q = storeService.insert(containerId, databaseId, result, statement, i); + result.setId(q.getId()); session.close(); factory.close(); log.debug("query id {}", result.getId()); @@ -238,6 +254,8 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService private List<TableColumn> parseColumns(Long databaseId, ExecuteStatementDto statement) { final List<TableColumn> columns = new LinkedList<>(); final int[] idx = new int[]{0}; + log.debug("Database id: {}", databaseId); + log.debug("ExecuteStatement: {}", statement.toString()); statement.getTables() .forEach(table -> { columns.addAll(statement.getColumns() @@ -252,4 +270,83 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService return columns; } + private Query parse(Query query, Database database) throws SQLException, ImageNotSupportedException, JSQLParserException { + Instant ts = Instant.now(); + query.setExecution(ts); + CCJSqlParserManager parserRealSql = new CCJSqlParserManager(); + Statement statement = parserRealSql.parse(new StringReader(query.getQuery())); + log.debug("Given query {}", query.getQuery()); + + if(statement instanceof Select) { + Select selectStatement = (Select) statement; + PlainSelect ps = (PlainSelect)selectStatement.getSelectBody(); + List<SelectItem> selectItems = ps.getSelectItems(); + + //Parse all tables + List<FromItem> fromItems = new ArrayList<>(); + fromItems.add(ps.getFromItem()); + if(ps.getJoins() != null && ps.getJoins().size() > 0) { + for (Join j : ps.getJoins()) { + if (j.getRightItem() != null) { + fromItems.add(j.getRightItem()); + } + } + } + //Checking if all tables exist + List<TableColumn> allColumns = new ArrayList<>(); + for(FromItem f : fromItems) { + boolean i = false; + log.debug("from item iterated through: {}", f); + for(Table t : database.getTables()) { + if(f.toString().equals(t.getInternalName()) || f.toString().equals(t.getName())) { + allColumns.addAll(t.getColumns()); + i=false; + break; + } + i = true; + } + if(i) { + throw new JSQLParserException("Table "+f.toString() + " does not exist"); + } + } + + //Checking if all columns exist + for(SelectItem s : selectItems) { + String select = s.toString(); + if(select.trim().equals("*")) { + log.debug("Please do not use * to query data"); + continue; + } + // ignore prefixes + if(select.contains(".")) { + log.debug(select); + select = select.split("\\.")[1]; + } + boolean i = false; + for(TableColumn tc : allColumns ) { + log.debug("{},{},{}", tc.getInternalName(), tc.getName(), s); + if(select.equals(tc.getInternalName()) || select.toString().equals(tc.getName())) { + i=false; + break; + } + i = true; + } + if(i) { + throw new JSQLParserException("Column "+s.toString() + " does not exist"); + } + } + //TODO Future work + if(ps.getWhere() != null) { + Expression where = ps.getWhere(); + log.debug("Where clause: {}", where); + } + return query; + } + else { + throw new JSQLParserException("SQL Query is not a SELECT statement - please only use SELECT statements"); + } + + } + + }