diff --git a/fda-metadata-db/api/src/main/java/at/tuwien/api/database/query/ExecuteStatementDto.java b/fda-metadata-db/api/src/main/java/at/tuwien/api/database/query/ExecuteStatementDto.java index 15a8edcc4e5aa717f573ab7aa2d54cd1c18fb427..d19b9ab92b2d8577e8dce66f6284670d203632eb 100644 --- a/fda-metadata-db/api/src/main/java/at/tuwien/api/database/query/ExecuteStatementDto.java +++ b/fda-metadata-db/api/src/main/java/at/tuwien/api/database/query/ExecuteStatementDto.java @@ -14,6 +14,7 @@ import java.util.List; @Builder @AllArgsConstructor @NoArgsConstructor +@ToString public class ExecuteStatementDto { @NotBlank(message = "statement is required") diff --git a/fda-metadata-db/api/src/main/java/at/tuwien/api/database/query/QueryResultDto.java b/fda-metadata-db/api/src/main/java/at/tuwien/api/database/query/QueryResultDto.java index 31622806f01b599e86e594b2ad848c6260379f7b..b7205a08d11ea4993a79e2deadc8697d3345ab07 100644 --- a/fda-metadata-db/api/src/main/java/at/tuwien/api/database/query/QueryResultDto.java +++ b/fda-metadata-db/api/src/main/java/at/tuwien/api/database/query/QueryResultDto.java @@ -4,6 +4,7 @@ import io.swagger.annotations.ApiModelProperty; import lombok.*; import javax.validation.constraints.NotNull; +import java.math.BigInteger; import java.util.List; import java.util.Map; @@ -24,4 +25,7 @@ public class QueryResultDto { @ApiModelProperty(notes = "query id") private Long id; + @ApiModelProperty(notes = "result number") + private Long resultNumber; + } diff --git a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java index a04384712e1209ae8fe4b0173a88255a55a29537..0a321c31066e7d261563c5e4585d126374cfaeed 100644 --- a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java +++ b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java @@ -10,6 +10,7 @@ import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; import lombok.extern.log4j.Log4j2; +import net.sf.jsqlparser.JSQLParserException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -19,6 +20,7 @@ import org.springframework.web.bind.annotation.*; import javax.validation.Valid; import javax.validation.constraints.NotNull; +import java.sql.SQLException; import java.time.Instant; @Log4j2 @@ -49,46 +51,21 @@ public class QueryEndpoint { @ApiResponse(code = 409, message = "The container image is not supported."),}) public ResponseEntity<QueryResultDto> execute(@NotNull @PathVariable("id") Long id, @NotNull @PathVariable("databaseId") Long databaseId, - @Valid @RequestBody ExecuteStatementDto data) + @Valid @RequestBody ExecuteStatementDto data, + @RequestParam(value = "page", required = false ) Long page, @RequestParam(value = "size", required = false) Long size) throws DatabaseNotFoundException, ImageNotSupportedException, QueryStoreException, QueryMalformedException, - TableNotFoundException, ContainerNotFoundException { + TableNotFoundException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException { /* validation */ if (data.getStatement() == null || data.getStatement().isBlank()) { log.error("Query is empty"); throw new QueryMalformedException("Invalid query"); } - if (data.getTables().size() == 0) { - log.error("Table list is empty"); - throw new QueryMalformedException("Invalid table"); - } - final QueryResultDto result = queryService.execute(id, databaseId, data); - final QueryDto query = queryMapper.queryToQueryDto(storeService.insert(id, databaseId, result, data, - Instant.now())); - result.setId(query.getId()); + log.debug("Data for execution: {}", data); + final QueryResultDto result = queryService.execute(id, databaseId, data, page, size); return ResponseEntity.status(HttpStatus.ACCEPTED) .body(result); } - @PostMapping - @Transactional - @PreAuthorize("hasRole('ROLE_RESEARCHER')") - @ApiOperation(value = "saves a query without execution") - @ApiResponses(value = { - @ApiResponse(code = 200, message = "Executed the query, Saved it and return the results"), - @ApiResponse(code = 404, message = "The database does not exist."), - @ApiResponse(code = 405, message = "The container is not running."), - @ApiResponse(code = 409, message = "The container image is not supported."),}) - public ResponseEntity<QueryDto> save(@NotNull @PathVariable("id") Long id, - @NotNull @PathVariable("databaseId") Long databaseId, - @Valid @RequestBody SaveStatementDto data) - throws DatabaseNotFoundException, ImageNotSupportedException, QueryStoreException, - ContainerNotFoundException { - final Query query = storeService.insert(id, databaseId, null, data); - final QueryDto queryDto = queryMapper.queryToQueryDto(query); - return ResponseEntity.status(HttpStatus.ACCEPTED) - .body(queryDto); - } - @PutMapping("/{queryId}") @Transactional @PreAuthorize("hasRole('ROLE_RESEARCHER')") @@ -100,13 +77,13 @@ public class QueryEndpoint { @ApiResponse(code = 409, message = "The container image is not supported."),}) public ResponseEntity<QueryResultDto> reExecute(@NotNull @PathVariable("id") Long id, @NotNull @PathVariable("databaseId") Long databaseId, - @NotNull @PathVariable("queryId") Long queryId) + @NotNull @PathVariable("queryId") Long queryId, + @RequestParam(value = "page", required = false) Long page, @RequestParam(value = "size", required = false) Long size) throws QueryStoreException, QueryNotFoundException, DatabaseNotFoundException, ImageNotSupportedException, - TableNotFoundException, QueryMalformedException, ContainerNotFoundException { + TableNotFoundException, QueryMalformedException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException { final Query query = storeService.findOne(id, databaseId, queryId); - final QueryDto queryDto = queryMapper.queryToQueryDto(query); - final ExecuteStatementDto statement = queryMapper.queryDtoToExecuteStatementDto(queryDto); - final QueryResultDto result = queryService.execute(id, databaseId, statement); + log.debug(query.toString()); + final QueryResultDto result = queryService.reExecute(id, databaseId, query, page, size); result.setId(queryId); return ResponseEntity.status(HttpStatus.ACCEPTED) .body(result); diff --git a/fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointIntegrationTest.java b/fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointIntegrationTest.java index c199e8459fc06a38b41cbb40953d95b353c574e2..1188d4e14a263df2cd8c0c05387f18a20533f827 100644 --- a/fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointIntegrationTest.java +++ b/fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointIntegrationTest.java @@ -18,6 +18,7 @@ import com.github.dockerjava.api.exception.NotModifiedException; import com.github.dockerjava.api.model.Bind; import com.github.dockerjava.api.model.Network; import lombok.extern.log4j.Log4j2; +import net.sf.jsqlparser.JSQLParserException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -131,7 +132,7 @@ public class QueryEndpointIntegrationTest extends BaseUnitTest { @Test public void reExecute_succeeds() throws TableNotFoundException, QueryStoreException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, QueryNotFoundException, InterruptedException, - SQLException, ContainerNotFoundException { + SQLException, ContainerNotFoundException, JSQLParserException, TableMalformedException { final QueryResultDto result = QueryResultDto.builder() .id(QUERY_1_ID) .result(List.of(Map.of("MinTemp", 13.4, "Rainfall", 0.6, "id", 1))) @@ -147,36 +148,13 @@ public class QueryEndpointIntegrationTest extends BaseUnitTest { storeService.insert(CONTAINER_1_ID, DATABASE_1_ID, result, statement, execution); /* test */ + //FIXME final ResponseEntity<QueryResultDto> response = queryEndpoint.reExecute(CONTAINER_1_ID, DATABASE_1_ID, - QUERY_1_ID); + QUERY_1_ID,0L,0L); assertEquals(HttpStatus.ACCEPTED, response.getStatusCode()); assertNotNull(response.getBody()); assertEquals(QUERY_1_ID, response.getBody().getId()); } - @Test - public void save_succeeds() throws QueryStoreException, DatabaseNotFoundException, ImageNotSupportedException, - InterruptedException, SQLException, ContainerNotFoundException { - final SaveStatementDto statement = SaveStatementDto.builder() - .statement(QUERY_1_STATEMENT) - .build(); - - /* mock */ - DockerConfig.startContainer(CONTAINER_1); - MariaDbConfig.clearQueryStore(TABLE_1); - - /* test */ - final ResponseEntity<QueryDto> response = queryEndpoint.save(CONTAINER_1_ID, DATABASE_1_ID, statement); - assertEquals(HttpStatus.ACCEPTED, response.getStatusCode()); - assertNotNull(response.getBody()); - assertEquals(QUERY_1_ID, response.getBody().getId()); - assertEquals(CONTAINER_1_ID, response.getBody().getCid()); - assertEquals(DATABASE_1_ID, response.getBody().getDbid()); - assertEquals(QUERY_1_STATEMENT, response.getBody().getQuery()); - assertEquals(QUERY_1_STATEMENT, response.getBody().getQueryNormalized()); - assertNull(response.getBody().getResultNumber()); - assertNull(response.getBody().getResultHash()); - } - } diff --git a/fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointUnitTest.java b/fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointUnitTest.java index 8cd08fa6fa22318b5854e28ebaa8d295a908b2e7..3ac55d09531818841de72176114edbec4690e33a 100644 --- a/fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointUnitTest.java +++ b/fda-query-service/rest-service/src/test/java/at/tuwien/endpoint/QueryEndpointUnitTest.java @@ -10,6 +10,7 @@ import at.tuwien.exception.*; import at.tuwien.service.StoreService; import at.tuwien.service.impl.QueryServiceImpl; import lombok.extern.log4j.Log4j2; +import net.sf.jsqlparser.JSQLParserException; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -19,6 +20,7 @@ import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.test.context.junit.jupiter.SpringExtension; +import java.sql.SQLException; import java.time.Instant; import java.util.List; import java.util.Map; @@ -45,7 +47,7 @@ public class QueryEndpointUnitTest extends BaseUnitTest { @Test public void execute_succeeds() throws TableNotFoundException, QueryStoreException, QueryMalformedException, - DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException { + DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException { final ExecuteStatementDto request = ExecuteStatementDto.builder() .statement(QUERY_1_STATEMENT) .build(); @@ -56,20 +58,22 @@ public class QueryEndpointUnitTest extends BaseUnitTest { final Instant execution = Instant.now(); /* mock */ - when(queryService.execute(CONTAINER_1_ID, DATABASE_1_ID, request)) + //FIXME + when(queryService.execute(CONTAINER_1_ID, DATABASE_1_ID, request, 0L, 0L)) .thenReturn(result); when(storeService.insert(CONTAINER_1_ID, DATABASE_1_ID, result, request, execution)) .thenReturn(QUERY_1); /* test */ - final ResponseEntity<QueryResultDto> response = queryEndpoint.execute(CONTAINER_1_ID, DATABASE_1_ID, request); + //FIXME + final ResponseEntity<QueryResultDto> response = queryEndpoint.execute(CONTAINER_1_ID, DATABASE_1_ID, request,0L,0L); assertEquals(HttpStatus.ACCEPTED, response.getStatusCode()); assertEquals(result, response.getBody()); } @Test public void execute_emptyResult_succeeds() throws TableNotFoundException, QueryStoreException, - QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException { + QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException { final ExecuteStatementDto request = ExecuteStatementDto.builder() .statement(QUERY_1_STATEMENT) .build(); @@ -80,65 +84,35 @@ public class QueryEndpointUnitTest extends BaseUnitTest { final Instant execution = Instant.now(); /* mock */ - when(queryService.execute(CONTAINER_1_ID, DATABASE_1_ID, request)) + //FIXME + when(queryService.execute(CONTAINER_1_ID, DATABASE_1_ID, request, 0L, 0L)) .thenReturn(result); when(storeService.insert(CONTAINER_1_ID, DATABASE_1_ID, result, request, execution)) .thenReturn(QUERY_1); /* test */ - final ResponseEntity<QueryResultDto> response = queryEndpoint.execute(CONTAINER_1_ID, DATABASE_1_ID, request); + //FIXME + final ResponseEntity<QueryResultDto> response = queryEndpoint.execute(CONTAINER_1_ID, DATABASE_1_ID, request,0L,0L); assertEquals(HttpStatus.ACCEPTED, response.getStatusCode()); assertEquals(result, response.getBody()); } @Test public void execute_tableNotFound_fails() throws TableNotFoundException, QueryMalformedException, - DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException { + DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, QueryStoreException, SQLException, JSQLParserException, TableMalformedException { final ExecuteStatementDto request = ExecuteStatementDto.builder() .statement(QUERY_1_STATEMENT) .build(); /* mock */ - when(queryService.execute(CONTAINER_1_ID, DATABASE_1_ID, request)) + //FIXME + when(queryService.execute(CONTAINER_1_ID, DATABASE_1_ID, request, 0L, 0L)) .thenThrow(TableNotFoundException.class); /* test */ assertThrows(TableNotFoundException.class, () -> { - queryEndpoint.execute(CONTAINER_1_ID, DATABASE_1_ID, request); - }); - } - - @Test - public void save_succeeds() throws QueryStoreException, DatabaseNotFoundException, ImageNotSupportedException, - ContainerNotFoundException { - final SaveStatementDto request = SaveStatementDto.builder() - .statement(QUERY_1_STATEMENT) - .build(); - - /* mock */ - when(storeService.insert(CONTAINER_1_ID, DATABASE_1_ID, null, request)) - .thenReturn(QUERY_1); - - /* test */ - final ResponseEntity<QueryDto> response = queryEndpoint.save(CONTAINER_1_ID, DATABASE_1_ID, request); - assertEquals(HttpStatus.ACCEPTED, response.getStatusCode()); - assertEquals(QUERY_1_DTO, response.getBody()); - } - - @Test - public void save_dbNotFound_fails() throws QueryStoreException, DatabaseNotFoundException, - ImageNotSupportedException, ContainerNotFoundException { - final SaveStatementDto request = SaveStatementDto.builder() - .statement(QUERY_1_STATEMENT) - .build(); - - /* mock */ - when(storeService.insert(CONTAINER_1_ID, DATABASE_1_ID, null, request)) - .thenThrow(DatabaseNotFoundException.class); - - /* test */ - assertThrows(DatabaseNotFoundException.class, () -> { - queryEndpoint.save(CONTAINER_1_ID, DATABASE_1_ID, request); + //FIXME + queryEndpoint.execute(CONTAINER_1_ID, DATABASE_1_ID, request,0L,0L); }); } diff --git a/fda-query-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java b/fda-query-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java index fa5cb52c411f7978867e0da1ba7bfd4cb4d62ca7..aa1052b62170be32eedeb6414870410f5b549598 100644 --- a/fda-query-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java +++ b/fda-query-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java @@ -17,6 +17,7 @@ import com.github.dockerjava.api.model.Bind; import com.github.dockerjava.api.model.Network; import lombok.SneakyThrows; import lombok.extern.log4j.Log4j2; +import net.sf.jsqlparser.JSQLParserException; import org.junit.Rule; import org.junit.rules.Timeout; import org.junit.jupiter.api.*; @@ -173,7 +174,7 @@ public class QueryServiceIntegrationTest extends BaseUnitTest { @Test public void execute_succeeds() throws DatabaseNotFoundException, ImageNotSupportedException, InterruptedException, - QueryMalformedException, TableNotFoundException, QueryStoreException, ContainerNotFoundException { + QueryMalformedException, TableNotFoundException, QueryStoreException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException { final ExecuteStatementDto request = ExecuteStatementDto.builder() .statement(QUERY_1_STATEMENT) .build(); @@ -182,7 +183,8 @@ public class QueryServiceIntegrationTest extends BaseUnitTest { DockerConfig.startContainer(CONTAINER_1); /* test */ - final QueryResultDto response = queryService.execute(CONTAINER_1_ID, DATABASE_1_ID, request); + //FIXME + final QueryResultDto response = queryService.execute(CONTAINER_1_ID, DATABASE_1_ID, request, 0L, 0L); assertEquals(3, response.getResult().size()); assertEquals(BigInteger.valueOf(1L), response.getResult().get(0).get(COLUMN_1_1_NAME)); assertEquals(toInstant("2008-12-01"), response.getResult().get(0).get(COLUMN_1_2_NAME)); @@ -206,7 +208,7 @@ public class QueryServiceIntegrationTest extends BaseUnitTest { @Disabled public void execute_modifyData_fails() throws DatabaseNotFoundException, ImageNotSupportedException, InterruptedException, QueryMalformedException, TableNotFoundException, QueryStoreException, - ContainerNotFoundException { + ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException { final ExecuteStatementDto request = ExecuteStatementDto.builder() .statement("DELETE FROM `weather_aus`;") .build(); @@ -215,7 +217,8 @@ public class QueryServiceIntegrationTest extends BaseUnitTest { DockerConfig.startContainer(CONTAINER_1); /* test */ - final QueryResultDto response = queryService.execute(CONTAINER_1_ID, DATABASE_1_ID, request); + //FIXME + final QueryResultDto response = queryService.execute(CONTAINER_1_ID, DATABASE_1_ID, request, 0L, 0L); assertNotNull(response.getResult()); assertEquals(3, response.getResult().size()); } @@ -231,7 +234,8 @@ public class QueryServiceIntegrationTest extends BaseUnitTest { /* test */ assertThrows(DatabaseNotFoundException.class, () -> { - queryService.execute(CONTAINER_1_ID, 9999L, request); + //FIXME + queryService.execute(CONTAINER_1_ID, 9999L, request, 0L, 0L); }); } @@ -247,7 +251,8 @@ public class QueryServiceIntegrationTest extends BaseUnitTest { /* test */ assertThrows(PersistenceException.class, () -> { - queryService.execute(CONTAINER_1_ID, DATABASE_1_ID, request); + //FIXME + queryService.execute(CONTAINER_1_ID, DATABASE_1_ID, request, 0L, 0L); }); } @@ -262,7 +267,8 @@ public class QueryServiceIntegrationTest extends BaseUnitTest { /* test */ assertThrows(PersistenceException.class, () -> { - queryService.execute(CONTAINER_1_ID, DATABASE_1_ID, request); + //FIXME + queryService.execute(CONTAINER_1_ID, DATABASE_1_ID, request, 0L, 0L); }); } @@ -277,7 +283,8 @@ public class QueryServiceIntegrationTest extends BaseUnitTest { /* test */ assertThrows(QueryMalformedException.class, () -> { - queryService.execute(CONTAINER_1_ID, DATABASE_1_ID, request); + //FIXME + queryService.execute(CONTAINER_1_ID, DATABASE_1_ID, request, 0L, 0L); }); } diff --git a/fda-query-service/services/src/main/java/at/tuwien/config/WebSecurityConfig.java b/fda-query-service/services/src/main/java/at/tuwien/config/WebSecurityConfig.java index 32142db9b262d0ead390e30e5b9db12beb1fb029..20cb1215a930de2d386633cef30c07ed11ef5602 100644 --- a/fda-query-service/services/src/main/java/at/tuwien/config/WebSecurityConfig.java +++ b/fda-query-service/services/src/main/java/at/tuwien/config/WebSecurityConfig.java @@ -62,6 +62,16 @@ public class WebSecurityConfig extends WebSecurityConfigurerAdapter { .antMatchers(HttpMethod.GET, "/api/container/**/database/**/table/**/export/**").permitAll() .antMatchers(HttpMethod.GET, "/api/container/**/database/query/**").permitAll() .antMatchers(HttpMethod.GET, "/api/container/**/database/**/query/**").permitAll() + .antMatchers("/v2/api-docs", + "/configuration/ui", + "/swagger-resources", + "/configuration/security", + "/swagger-ui.html", + "/webjars/**", + "/swagger-resources/configuration/ui", + "/swagger-ui.html", + "/v3/api-docs/**", + "/swagger-ui/**").permitAll() /* insert endpoint */ .antMatchers(HttpMethod.POST, "/api/container/**/database/**/table/**/data").permitAll() /* our private endpoints */ diff --git a/fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java b/fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java index afff49c34841c2d43e3dcb5e77e7a7f24ac8bc88..f56c15c42a309f326ece44b3e6081a11d3d7b259 100644 --- a/fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java +++ b/fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java @@ -2,7 +2,9 @@ package at.tuwien.mapper; import at.tuwien.InsertTableRawQuery; import at.tuwien.api.database.query.*; +import at.tuwien.api.database.table.TableBriefDto; import at.tuwien.api.database.table.TableCsvDto; +import at.tuwien.entities.database.Database; import at.tuwien.exception.TableMalformedException; import at.tuwien.querystore.Query; import at.tuwien.entities.database.table.Table; @@ -16,6 +18,7 @@ import org.mariadb.jdbc.MariaDbBlob; import org.springframework.transaction.annotation.Transactional; import java.math.BigInteger; +import java.nio.charset.MalformedInputException; import java.text.Normalizer; import java.time.*; import java.time.format.DateTimeFormatter; @@ -158,11 +161,74 @@ public interface QueryMapper { timestamp = Instant.now(); } return "SELECT COUNT(*) FROM `" + nameToInternalName(table.getName()) + - "` FOR SYSTEM_TIME AS OF TIMESTAMP'" + + "` FOR SYSTEM_TIME AS OF TIMESTAMP '" + LocalDateTime.ofInstant(timestamp, ZoneId.of("Europe/Vienna")) + "';"; } + default String queryToRawTimestampedCountQuery(String query, Database database, Instant timestamp) throws ImageNotSupportedException { + /* param check */ + if (!database.getContainer().getImage().getRepository().equals("mariadb")) { + throw new ImageNotSupportedException("Currently only MariaDB is supported"); + } + if (timestamp == null) { + throw new IllegalArgumentException("Timestamp must be provided"); + } + StringBuilder sb = new StringBuilder(); + sb.append("SELECT COUNT(*) FROM"); + if(query.contains("where")) { + sb.append(query.toLowerCase(Locale.ROOT).split("from ")[1].split("where")[0]); + } else { + sb.append(query.toLowerCase(Locale.ROOT).split("from ")[1]); + } + sb.append("FOR SYSTEM_TIME AS OF TIMESTAMP '"); + sb.append(LocalDateTime.ofInstant(timestamp, ZoneId.of("Europe/Vienna"))); + sb.append("' "); + if(query.contains("where")) { + sb.append("where "); + sb.append(query.toLowerCase(Locale.ROOT).split("from ")[1].split("where")[1]); + } + sb.append(";"); + log.debug(sb.toString()); + return sb.toString(); + } + + default String queryToRawTimestampedQuery(String query, Database database, Instant timestamp, Long page, Long size) throws ImageNotSupportedException { + /* param check */ + if (!database.getContainer().getImage().getRepository().equals("mariadb")) { + throw new ImageNotSupportedException("Currently only MariaDB is supported"); + } + if (timestamp == null) { + throw new IllegalArgumentException("Please provide a timestamp before"); + } + StringBuilder sb = new StringBuilder(); + if(query.contains("where")) { + sb.append(query.toLowerCase(Locale.ROOT).split("where")[0]); + } else { + sb.append(query.toLowerCase(Locale.ROOT)); + } + sb.append("FOR SYSTEM_TIME AS OF TIMESTAMP '"); + sb.append(LocalDateTime.ofInstant(timestamp, ZoneId.of("Europe/Vienna"))); + sb.append("' "); + if(query.contains("where")) { + sb.append("where"); + sb.append(query.toLowerCase(Locale.ROOT).split("from ")[1].split("where")[1]); + } + if(size != null && page != null && size > 0 && page >=0) { + sb.append(" LIMIT " + size + " OFFSET " + (page*size)); + } + sb.append(";"); + + log.debug(sb.toString()); + + return sb.toString(); + + } + + + + + default String tableToRawFindAllQuery(Table table, Instant timestamp, Long size, Long page) throws ImageNotSupportedException { /* param check */ @@ -184,7 +250,7 @@ public interface QueryMapper { .append("`")); query.append(" FROM `") .append(nameToInternalName(table.getName())) - .append("` FOR SYSTEM_TIME AS OF TIMESTAMP'") + .append("` FOR SYSTEM_TIME AS OF TIMESTAMP '") .append(LocalDateTime.ofInstant(timestamp, ZoneId.of("Europe/Vienna"))) .append("'"); if (size != null && page != null) { @@ -261,4 +327,14 @@ public interface QueryMapper { throw new IllegalArgumentException("Column type not known"); } } + + @Named("EscapedString") + default String stringToEscapedString(String name) throws ImageNotSupportedException { + log.debug("StringToEscapedString: {}",name); + if(name!=null && !name.startsWith("`") && !name.endsWith("`")) { + return "`"+name+"`"; + } + return name; + } + } diff --git a/fda-query-service/services/src/main/java/at/tuwien/querystore/Query.java b/fda-query-service/services/src/main/java/at/tuwien/querystore/Query.java index bbdeb2b78386e69b0b94e3d29d056f4f915220b3..de1d12f70f0b695599a69f973ef39e30531491ed 100644 --- a/fda-query-service/services/src/main/java/at/tuwien/querystore/Query.java +++ b/fda-query-service/services/src/main/java/at/tuwien/querystore/Query.java @@ -8,6 +8,7 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener; import javax.persistence.*; import java.io.Serializable; +import java.math.BigInteger; import java.time.Instant; import java.util.List; diff --git a/fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java b/fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java index 268ca3c68b6b0f77bb3019e2b90c4c212485418e..3dfbfdf2e3f2809fd383b569b4a50a494b9c2dd6 100644 --- a/fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java +++ b/fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java @@ -5,9 +5,12 @@ import at.tuwien.api.database.query.ImportDto; import at.tuwien.api.database.query.QueryResultDto; import at.tuwien.api.database.table.TableCsvDto; import at.tuwien.exception.*; +import at.tuwien.querystore.Query; +import net.sf.jsqlparser.JSQLParserException; import org.springframework.stereotype.Service; import java.math.BigInteger; +import java.sql.SQLException; import java.time.Instant; @Service @@ -19,6 +22,8 @@ public interface QueryService { * * @param databaseId The database id. * @param query The query. + * @param page + * @param size * @return The result. * @throws TableNotFoundException * @throws QueryStoreException @@ -26,8 +31,27 @@ public interface QueryService { * @throws DatabaseNotFoundException * @throws ImageNotSupportedException */ - QueryResultDto execute(Long containerId, Long databaseId, ExecuteStatementDto query) throws TableNotFoundException, - QueryStoreException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException; + QueryResultDto execute(Long containerId, Long databaseId, ExecuteStatementDto query, Long page, Long size) throws TableNotFoundException, + QueryStoreException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException; + + /** + * Re-Executes an arbitrary query on the database container. We allow the user to only view the data, therefore the + * default "mariadb" user is allowed read-only access "SELECT". + * + * @param databaseId The database id. + * @param query The query. + * @param page + * @param size + * @return The result. + * @throws TableNotFoundException + * @throws QueryStoreException + * @throws QueryMalformedException + * @throws DatabaseNotFoundException + * @throws ImageNotSupportedException + */ + QueryResultDto reExecute(Long containerId, Long databaseId, Query query, Long page, Long size) throws TableNotFoundException, + QueryStoreException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException; + /** * Select all data known in the database-table id tuple at a given time and return a page of specific size, using diff --git a/fda-query-service/services/src/main/java/at/tuwien/service/StoreService.java b/fda-query-service/services/src/main/java/at/tuwien/service/StoreService.java index ed49db862f2cb9469737dbe9477798dbc84535e5..12368f14b025ad2955b9ce521d1140a9dba1507d 100644 --- a/fda-query-service/services/src/main/java/at/tuwien/service/StoreService.java +++ b/fda-query-service/services/src/main/java/at/tuwien/service/StoreService.java @@ -6,6 +6,7 @@ import at.tuwien.api.database.query.SaveStatementDto; import at.tuwien.exception.*; import at.tuwien.querystore.Query; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import java.time.Instant; import java.util.List; @@ -69,4 +70,8 @@ public interface StoreService { Instant execution) throws QueryStoreException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException; + @Transactional(readOnly = true) + Query update(Long containerId, Long databaseId, QueryResultDto result, Long resultNumber, Query metadata) + throws QueryStoreException, DatabaseNotFoundException, ImageNotSupportedException, + ContainerNotFoundException; } diff --git a/fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java b/fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java index 09100ba5f690dc972c3dfdbc43ad7cc1e2091415..7daf02a5cd98b6b30eb6b256fb496a1bc9c37de9 100644 --- a/fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java +++ b/fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java @@ -10,9 +10,14 @@ import at.tuwien.entities.database.table.Table; import at.tuwien.entities.database.table.columns.TableColumn; import at.tuwien.exception.*; import at.tuwien.mapper.QueryMapper; +import at.tuwien.querystore.Query; import at.tuwien.repository.jpa.TableColumnRepository; import at.tuwien.service.*; import lombok.extern.log4j.Log4j2; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.parser.CCJSqlParserManager; +import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.statement.select.*; import org.hibernate.Session; import org.hibernate.SessionFactory; import org.hibernate.exception.SQLGrammarException; @@ -22,9 +27,12 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import javax.persistence.PersistenceException; +import java.io.StringReader; import java.math.BigInteger; +import java.sql.SQLException; import java.time.DateTimeException; import java.time.Instant; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Optional; @@ -38,20 +46,31 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService private final TableService tableService; private final DatabaseService databaseService; private final TableColumnRepository tableColumnRepository; + private final StoreService storeService; @Autowired public QueryServiceImpl(QueryMapper queryMapper, TableService tableService, DatabaseService databaseService, - TableColumnRepository tableColumnRepository) { + TableColumnRepository tableColumnRepository, StoreService storeService) { this.queryMapper = queryMapper; this.tableService = tableService; this.databaseService = databaseService; this.tableColumnRepository = tableColumnRepository; + this.storeService = storeService; } @Override @Transactional - public QueryResultDto execute(Long containerId, Long databaseId, ExecuteStatementDto statement) - throws DatabaseNotFoundException, ImageNotSupportedException, QueryMalformedException { + public QueryResultDto execute(Long containerId, Long databaseId, ExecuteStatementDto statement, Long page, Long size) + throws DatabaseNotFoundException, ImageNotSupportedException, QueryMalformedException, QueryStoreException, ContainerNotFoundException, TableNotFoundException, SQLException, JSQLParserException, TableMalformedException { + Instant i = Instant.now(); + Query q = storeService.insert(containerId, databaseId, null, statement, i); + final QueryResultDto result = this.reExecute(containerId,databaseId,q,page,size); + q = storeService.update(containerId,databaseId,result, result.getResultNumber(),q); + return result; + } + + @Override + public QueryResultDto reExecute(Long containerId, Long databaseId, Query query, Long page, Long size) throws TableNotFoundException, QueryStoreException, QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, ContainerNotFoundException, SQLException, JSQLParserException, TableMalformedException { /* find */ final Database database = databaseService.find(databaseId); if (!database.getContainer().getImage().getRepository().equals("mariadb")) { @@ -64,11 +83,12 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService log.debug("opened hibernate session in {} ms", System.currentTimeMillis() - startSession); session.beginTransaction(); /* prepare the statement */ - final NativeQuery<?> query = session.createSQLQuery(statement.getStatement()); + Instant i = Instant.now(); + final NativeQuery<?> nativeQuery = session.createSQLQuery(queryMapper.queryToRawTimestampedQuery(query.getQuery(), database, query.getExecution(),page, size)); final int affectedTuples; try { - log.debug("execute raw view-only query {}", statement); - affectedTuples = query.executeUpdate(); + log.debug("execute raw view-only query {}", query); + affectedTuples = nativeQuery.executeUpdate(); log.info("Execution on database id {} affected {} rows", databaseId, affectedTuples); session.getTransaction() .commit(); @@ -78,11 +98,12 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService throw new QueryMalformedException("Query not valid for this database", e); } /* map the result to the tables (with respective columns) from the statement metadata */ - final List<TableColumn> columns = parseColumns(databaseId, statement); - final QueryResultDto result = queryMapper.resultListToQueryResultDto(columns, query.getResultList()); + final List<TableColumn> columns = parseColumns(query, database); + QueryResultDto result = queryMapper.resultListToQueryResultDto(columns, nativeQuery.getResultList()); + result.setResultNumber(query.getResultNumber()!=null ? query.getResultNumber() : countQueryResults(containerId,databaseId,query)); + result.setId(query.getId()); session.close(); factory.close(); - log.debug("query id {}", result.getId()); return result; } @@ -238,6 +259,8 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService private List<TableColumn> parseColumns(Long databaseId, ExecuteStatementDto statement) { final List<TableColumn> columns = new LinkedList<>(); final int[] idx = new int[]{0}; + log.debug("Database id: {}", databaseId); + log.debug("ExecuteStatement: {}", statement.toString()); statement.getTables() .forEach(table -> { columns.addAll(statement.getColumns() @@ -252,4 +275,111 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService return columns; } + private List<TableColumn> parseColumns(Query query, Database database) throws SQLException, ImageNotSupportedException, JSQLParserException { + final List<TableColumn> columns = new ArrayList<>(); + + CCJSqlParserManager parserRealSql = new CCJSqlParserManager(); + Statement statement = parserRealSql.parse(new StringReader(query.getQuery())); + log.debug("Given query {}", query.getQuery()); + + if(statement instanceof Select) { + Select selectStatement = (Select) statement; + PlainSelect ps = (PlainSelect)selectStatement.getSelectBody(); + List<SelectItem> selectItems = ps.getSelectItems(); + + //Parse all tables + List<FromItem> fromItems = new ArrayList<>(); + fromItems.add(ps.getFromItem()); + if(ps.getJoins() != null && ps.getJoins().size() > 0) { + for (Join j : ps.getJoins()) { + if (j.getRightItem() != null) { + fromItems.add(j.getRightItem()); + } + } + } + //Checking if all tables exist + List<TableColumn> allColumns = new ArrayList<>(); + for(FromItem f : fromItems) { + boolean i = false; + log.debug("from item iterated through: {}", f); + for(Table t : database.getTables()) { + if(queryMapper.stringToEscapedString(f.toString()).equals(queryMapper.stringToEscapedString(t.getInternalName()))) { + allColumns.addAll(t.getColumns()); + i=false; + break; + } + i = true; + } + if(i) { + throw new JSQLParserException("Table "+queryMapper.stringToEscapedString(f.toString())+ " does not exist"); + } + } + + //Checking if all columns exist + for(SelectItem s : selectItems) { + String select = queryMapper.stringToEscapedString(s.toString()); + log.debug(select); + if(select.trim().equals("*")) { + log.debug("Please do not use * to query data"); + continue; + } + // ignore prefixes + if(select.contains(".")) { + log.debug(select); + select = select.split("\\.")[1]; + } + boolean i = false; + for(TableColumn tc : allColumns ) { + log.debug("{},{},{}", tc.getInternalName(), tc.getName(), s); + if(select.equals(queryMapper.stringToEscapedString(tc.getInternalName()))) { + i=false; + columns.add(tc); + break; + } + i = true; + } + if(i) { + throw new JSQLParserException("Column "+s.toString() + " does not exist"); + } + } + return columns; + } + else { + throw new JSQLParserException("SQL Query is not a SELECT statement - please only use SELECT statements"); + } + + } + + @Transactional + Long countQueryResults(Long containerId, Long databaseId, Query query) + throws DatabaseNotFoundException, TableNotFoundException, + TableMalformedException, ImageNotSupportedException { + /* find */ + final Database database = databaseService.find(databaseId); + /* run query */ + final long startSession = System.currentTimeMillis(); + final SessionFactory factory = getSessionFactory(database, false); + final Session session = factory.openSession(); + log.debug("opened hibernate session in {} ms", System.currentTimeMillis() - startSession); + session.beginTransaction(); + final NativeQuery<BigInteger> nativeQuery = session.createSQLQuery(queryMapper.queryToRawTimestampedCountQuery(query.getQuery(), database, query.getExecution())); + final int affectedTuples; + try { + affectedTuples = nativeQuery.executeUpdate(); + log.info("Counted {} tuples from query {}", affectedTuples, query.getId()); + } catch (PersistenceException e) { + log.error("Failed to count tuples"); + session.close(); + factory.close(); + throw new TableMalformedException("Data not found", e); + } + session.getTransaction() + .commit(); + final Long count = nativeQuery.getSingleResult().longValue(); + session.close(); + factory.close(); + return count; + } + + } diff --git a/fda-query-service/services/src/main/java/at/tuwien/service/impl/StoreServiceImpl.java b/fda-query-service/services/src/main/java/at/tuwien/service/impl/StoreServiceImpl.java index 7fbda87023bc08a3391360c1f49c91d7bc973c87..33c4177d447152a92e261ac27fb13365d1750f7b 100644 --- a/fda-query-service/services/src/main/java/at/tuwien/service/impl/StoreServiceImpl.java +++ b/fda-query-service/services/src/main/java/at/tuwien/service/impl/StoreServiceImpl.java @@ -146,4 +146,36 @@ public class StoreServiceImpl extends HibernateConnector implements StoreService return query; } + @Override + @Transactional(readOnly = true) + public Query update(Long containerId, Long databaseId, QueryResultDto result, Long resultNumber, Query query) + throws QueryStoreException, DatabaseNotFoundException, ImageNotSupportedException, + ContainerNotFoundException { + /* find */ + final Container container = containerService.find(containerId); + final Database database = databaseService.find(databaseId); + if (!database.getContainer().getImage().getRepository().equals("mariadb")) { + throw new ImageNotSupportedException("Currently only MariaDB is supported"); + } + + log.debug("Update database id {}, metadata {}", databaseId, query); + /* save */ + final SessionFactory factory = getSessionFactory(database, true); + final Session session = factory.openSession(); + final Transaction transaction = session.beginTransaction(); + query.setQueryHash(DigestUtils.sha256Hex(query.getQuery())); + query.setResultNumber(resultNumber); + query.setResultHash(storeMapper.queryResultDtoToString(result)); + session.update(query); + transaction.commit(); + /* store the result in the query store */ + log.info("Update query with id {}", query.getId()); + log.debug("saved query {}", query); + session.close(); + factory.close(); + return query; + } + + + } diff --git a/fda-ui/Dockerfile b/fda-ui/Dockerfile index 4a814387b7317802f07da4a45f0095e20e6b1f8b..ade44caf4feef43de89d3f425b37a1399293ce4b 100644 --- a/fda-ui/Dockerfile +++ b/fda-ui/Dockerfile @@ -25,6 +25,7 @@ COPY ./plugins ./plugins COPY ./server-middleware ./server-middleware COPY ./static ./static COPY ./store ./store +COPY ./utils ./utils RUN yarn build > /dev/null diff --git a/fda-ui/components/DBToolbar.vue b/fda-ui/components/DBToolbar.vue index fb55105b4b6ab0c091c71f9521a5899e0f9d4b4f..c630502d738e9c73487d83e70432246a8165f6f9 100644 --- a/fda-ui/components/DBToolbar.vue +++ b/fda-ui/components/DBToolbar.vue @@ -29,9 +29,9 @@ <v-tab :to="`/container/${$route.params.container_id}/database/${databaseId}/query`"> Queries </v-tab> -<!-- <v-tab :to="`/container/${$route.params.container_id}/database/${databaseId}/admin`">--> -<!-- Admin--> -<!-- </v-tab>--> + <!-- <v-tab :to="`/container/${$route.params.container_id}/database/${databaseId}/admin`">--> + <!-- Admin--> + <!-- </v-tab>--> </v-tabs> </template> </v-toolbar> diff --git a/fda-ui/components/QueryList.vue b/fda-ui/components/QueryList.vue index e67154c3bab8d6e824a5256bb61650770f564f00..182730cdacc50556e1b09657a26ff37b8a51acd5 100644 --- a/fda-ui/components/QueryList.vue +++ b/fda-ui/components/QueryList.vue @@ -10,7 +10,7 @@ <v-expansion-panels v-if="!loading && queries.length > 0" accordion> <v-expansion-panel v-for="(item, i) in queries" :key="i" @click="details(item)"> <v-expansion-panel-header> - <span v-bind:class="{'font-weight-black': item.identifier !== undefined}">{{ title(item) }}</span> + <span :class="{'font-weight-black': item.identifier !== undefined}">{{ title(item) }}</span> </v-expansion-panel-header> <v-expansion-panel-content> <v-row dense> diff --git a/fda-ui/components/TableList.vue b/fda-ui/components/TableList.vue index d5b93adc538564065356c9e98cc8fe96e27f06d0..67972b85ae886948ab629ad0e7addf67f097b1f1 100644 --- a/fda-ui/components/TableList.vue +++ b/fda-ui/components/TableList.vue @@ -85,8 +85,9 @@ </v-row> <v-row dense> <v-col> - <v-btn color="blue-grey" class="white--text" :to="`/container/${$route.params.container_id}/database/${$route.params.database_id}/table/${item.id}`"> - More + <v-btn outlined :to="`/container/${$route.params.container_id}/database/${$route.params.database_id}/table/${item.id}`"> + <v-icon>mdi-table</v-icon> + View </v-btn> </v-col> <v-col class="align-right"> @@ -176,7 +177,8 @@ export default { }, mounted () { this.$root.$on('table-create', this.refresh) - this.refresh() + const table = this.$store.state.table + this.refresh(table ? table.id : null) }, methods: { async details (tableId, clicked = false) { @@ -186,8 +188,8 @@ export default { } try { const res = await this.$axios.get(`/api/container/${this.$route.params.container_id}/database/${this.$route.params.database_id}/table/${tableId}`) - console.debug('table', res.data) this.tableDetails = res.data + this.$store.commit('SET_TABLE', this.tableDetails) } catch (err) { this.tableDetails = undefined this.$toast.error('Could not get table details.') @@ -205,8 +207,9 @@ export default { this.loading = false if (tableId) { this.openPanelByTableId(tableId) } } catch (err) { - this.$toast.error('Could not list table.') + this.$toast.error('Could not load tables.') } + this.$store.commit('SET_TABLE', null) }, async deleteTable () { try { diff --git a/fda-ui/components/dialogs/CreateDB.vue b/fda-ui/components/dialogs/CreateDB.vue index ee59c65cbf58c124e671ad63d563bc2f9282ce12..ee5bcf87c8feafba16f91b8e3f027559a1c1bad8 100644 --- a/fda-ui/components/dialogs/CreateDB.vue +++ b/fda-ui/components/dialogs/CreateDB.vue @@ -68,6 +68,8 @@ </template> <script> +const { notEmpty } = require('@/utils') + export default { data () { return { @@ -120,9 +122,7 @@ export default { setTimeout(resolve, ms) }) }, - notEmpty (str) { - return typeof str === 'string' && str.trim().length > 0 - }, + notEmpty, async createDB () { let res // create a container diff --git a/fda-ui/components/dialogs/PersistQuery.vue b/fda-ui/components/dialogs/PersistQuery.vue index 69004470652f0f86f1241f5422b3534b5390f175..fbab09a285348b95985d9ac0c118d0d09b3d0d14 100644 --- a/fda-ui/components/dialogs/PersistQuery.vue +++ b/fda-ui/components/dialogs/PersistQuery.vue @@ -130,6 +130,7 @@ export default { } this.$toast.success('Query persisted.') this.$emit('close') + this.loading = false }, async loadUser () { this.loading = true @@ -143,6 +144,7 @@ export default { this.$toast.error('Failed load user data') console.error('load user data failed', err) } + this.loading = false } } } diff --git a/fda-ui/components/query/Builder.vue b/fda-ui/components/query/Builder.vue index 2574d77c569f2c85d616676a8292c1f718474bb1..f49960605b76d553d333a0d76ce1cce84c240792 100644 --- a/fda-ui/components/query/Builder.vue +++ b/fda-ui/components/query/Builder.vue @@ -1,14 +1,11 @@ <template> <div> - <v-form - ref="form" - v-model="valid" - lazy-validation> + <v-form ref="form"> <v-toolbar flat> <v-toolbar-title>Create Query</v-toolbar-title> <v-spacer /> <v-toolbar-title> - <v-btn :disabled="!valid || !token" color="blue-grey white--text" @click="save"> + <v-btn v-if="false" :disabled="!valid || !token" color="blue-grey white--text" @click="save"> Save without execution </v-btn> <v-btn :disabled="!valid || !token" color="primary" @click="execute"> @@ -23,7 +20,6 @@ <v-col cols="6"> <v-select v-model="table" - :rules="[rules.required]" :items="tables" item-text="name" return-object @@ -33,7 +29,6 @@ <v-col cols="6"> <v-select v-model="select" - :rules="[rules.required]" item-text="name" :disabled="!table" :items="selectItems" @@ -55,12 +50,7 @@ <v-row> <v-col> <p>Results</p> - <v-data-table - :headers="result.headers" - :items="result.rows" - :loading="loading" - :items-per-page="30" - class="elevation-1" /> + <QueryResults ref="queryResults" v-model="queryId" /> </v-col> </v-row> <v-row> @@ -77,12 +67,9 @@ </template> <script> -import _ from 'lodash' - export default { data () { return { - valid: false, table: null, tables: [], tableDetails: null, @@ -91,15 +78,7 @@ export default { sql: '' }, select: [], - clauses: [], - result: { - headers: [], - rows: [] - }, - rules: { - required: value => !!value || 'Required' - }, - loading: false + clauses: [] } }, computed: { @@ -108,7 +87,7 @@ export default { return columns || [] }, columnNames () { - return this.selectItems && this.selectItems.map(s => s.internalName) + return this.selectItems && this.selectItems.map(s => s.internal_name) }, databaseId () { return this.$route.params.database_id @@ -124,6 +103,10 @@ export default { return null } return { Authorization: `Bearer ${this.token}` } + }, + valid () { + // we need to have at least one column selected + return this.select.length } }, watch: { @@ -131,7 +114,14 @@ export default { deep: true, handler () { this.buildQuery() + this.queryId = null } + }, + table () { + this.queryId = null + }, + select () { + this.queryId = null } }, beforeMount () { @@ -149,53 +139,8 @@ export default { this.$toast.error('Could not list table.') } }, - async execute () { - this.$refs.form.validate() - this.loading = true - try { - const data = { - statement: this.query.sql, - tables: [_.pick(this.table, ['id', 'name', 'internal_name'])], - columns: [this.select.map(function (column) { - return _.pick(column, ['id', 'name', 'internal_name']) - })] - } - console.debug('send data', data) - const res = await this.$axios.put(`/api/container/${this.$route.params.container_id}/database/${this.databaseId}/query`, data, { - headers: this.headers - }) - console.debug('query result', res) - this.$toast.success('Successfully executed query') - this.loading = false - this.queryId = res.data.id - this.result.headers = this.select.map((s) => { - return { text: s.name, value: s.name, sortable: false } - }) - this.result.rows = res.data.result - } catch (err) { - console.error('query execute', err) - this.$toast.error('Could not execute query') - this.loading = false - } - }, - async save () { - this.$refs.form.validate() - const query = this.query.sql.replaceAll('`', '') - this.loading = true - try { - const res = await this.$axios.post(`/api/container/${this.$route.params.container_id}/database/${this.databaseId}/query`, { statement: query }, { - headers: this.headers - }) - console.debug('query result', res) - this.$toast.success('Successfully saved query') - this.loading = false - this.queryId = res.data.id - this.$router.push(`/container/${this.$route.params.container_id}/database/${this.$route.params.database_id}/query/${this.queryId}`) - } catch (err) { - console.error('query save', err) - this.$toast.error('Could not save query') - this.loading = false - } + execute () { + this.$refs.queryResults.executeFirstTime(this) }, async buildQuery () { if (!this.table) { @@ -204,7 +149,7 @@ export default { const url = '/server-middleware/query/build' const data = { table: this.table.internal_name, - select: this.select.map(s => s.name), + select: this.select.map(s => s.internal_name), clauses: this.clauses } try { diff --git a/fda-ui/components/query/Results.vue b/fda-ui/components/query/Results.vue new file mode 100644 index 0000000000000000000000000000000000000000..89e399d6f22b0a7c005fcff7ba056d9ab810330f --- /dev/null +++ b/fda-ui/components/query/Results.vue @@ -0,0 +1,136 @@ +<template> + <v-data-table + :headers="result.headers" + :items="result.rows" + :loading="loading" + :options.sync="options" + :server-items-length="total" + class="elevation-1" /> +</template> + +<script> +import _ from 'lodash' + +export default { + props: { + value: { type: Number, default: () => 0 } + }, + data () { + return { + parent: null, + loading: false, + result: { + headers: [], + rows: [] + }, + options: { + page: 1, + itemsPerPage: 10 + }, + total: 0 + } + }, + computed: { + token () { + return this.$store.state.token + }, + headers () { + if (this.token === null) { + return null + } + return { Authorization: `Bearer ${this.token}` } + } + }, + watch: { + value () { + if (this.value) { + this.execute() + } + }, + options (newVal, oldVal) { + if (typeof oldVal.groupBy === 'undefined') { + // initially, options do not have the groupBy field. + // don't run the execute method twice, when a new query is created + return + } + if (!this.value) { + this.$toast.error('Cannot paginate invalidated Query: press Execute') + return + } + this.execute() + } + }, + mounted () { + }, + methods: { + async executeFirstTime (parent) { + this.parent = parent + this.loading = true + try { + const data = { + statement: this.parent.query.sql, + tables: [_.pick(this.parent.table, ['id', 'name', 'internal_name'])], + columns: [this.parent.select.map(function (column) { + return _.pick(column, ['id', 'name', 'internal_name']) + })] + } + console.debug('send data', data) + const page = 0 + const urlParams = `page=${page}&size=${this.options.itemsPerPage}` + const res = await this.$axios.put(`/api/container/ +${this.$route.params.container_id}/database/${this.$route.params.database_id}/query +${this.parent.queryId ? `/${this.parent.queryId}` : ''} +?${urlParams}`, data, { + headers: this.headers + }) + console.debug('query result', res) + this.$toast.success('Successfully executed query') + this.loading = false + this.parent.queryId = res.data.id + this.result.headers = this.parent.select.map((s) => { + return { text: s.name, value: s.name, sortable: false } + }) + this.result.rows = res.data.result + this.total = res.data.resultNumber + } catch (err) { + console.error('query execute', err) + this.$toast.error('Could not execute query') + this.loading = false + } + }, + buildHeaders (firstLine) { + return Object.keys(firstLine).map(k => ({ + text: k, + value: k, + sortable: false + })) + }, + async execute () { + this.loading = true + try { + const page = this.options.page - 1 + const urlParams = `page=${page}&size=${this.options.itemsPerPage}` + const res = await this.$axios.put(`/api/container/ +${this.$route.params.container_id}/database/${this.$route.params.database_id}/query +/${this.value} +?${urlParams}`, {}, { + headers: this.headers + }) + this.loading = false + if (res.data.result.length) { + this.result.headers = this.buildHeaders(res.data.result[0]) + } + this.result.rows = res.data.result + this.total = res.data.resultNumber + } catch (err) { + console.error('query execute', err) + this.$toast.error('Could not execute query') + this.loading = false + } + } + } +} +</script> + +<style scoped> +</style> diff --git a/fda-ui/layouts/default.vue b/fda-ui/layouts/default.vue index 24a2e9e63070d3b3ec69aa9b213b869db62926fd..47279b468d95ee2d5389bda6cf9b28113e7eae58 100644 --- a/fda-ui/layouts/default.vue +++ b/fda-ui/layouts/default.vue @@ -64,6 +64,7 @@ {{ nextTheme }} Theme </v-list-item> <v-list-item + v-if="token" @click="logout"> Logout </v-list-item> diff --git a/fda-ui/nuxt.config.js b/fda-ui/nuxt.config.js index 52b78f9b5b25eb6144def208cdbc5ad5dc60a1c0..a0fe0da0a5d1f5d1850ef4599b7d64b941185bc9 100644 --- a/fda-ui/nuxt.config.js +++ b/fda-ui/nuxt.config.js @@ -30,6 +30,7 @@ if (!process.env.KEY || !process.env.CERT) { export default { target: 'server', + ssr: false, telemetry: false, diff --git a/fda-ui/pages/container/_container_id/database/_database_id/query/_query_id/index.vue b/fda-ui/pages/container/_container_id/database/_database_id/query/_query_id/index.vue index d40ffb92e2d6bc4901e569d346f46d7a2d601415..16768d2a1b6748baf92fc97dbf96bd2b0c635e8b 100644 --- a/fda-ui/pages/container/_container_id/database/_database_id/query/_query_id/index.vue +++ b/fda-ui/pages/container/_container_id/database/_database_id/query/_query_id/index.vue @@ -4,15 +4,15 @@ <v-toolbar-title>{{ identifier.title }}</v-toolbar-title> <v-spacer /> <v-toolbar-title> - <v-btn color="blue-grey white--text" class="mr-2" :disabled="!query.execution || identifier.id || !token" @click.stop="persistQueryDialog = true"> + <v-btn color="blue-grey white--text" class="mr-2" :disabled="!query.execution || !!identifier.id || !token" @click.stop="persistQueryDialog = true"> <v-icon left>mdi-fingerprint</v-icon> Persist </v-btn> - <v-btn color="primary" :disabled="!token" @click.stop="reExecute"> + <v-btn v-if="false" color="primary" :disabled="!token" @click.stop="reExecute"> <v-icon left>mdi-run</v-icon> Re-Execute </v-btn> </v-toolbar-title> </v-toolbar> - <v-card v-if="!loading" flat> + <v-card v-if="!loading" class="pb-2" flat> <v-card-title> Query Information </v-card-title> @@ -70,6 +70,7 @@ Username: <code v-if="query.username">{{ query.username }}</code><span v-if="!query.username">(empty)</span> </p> </v-card-text> + <QueryResults ref="queryResults" v-model="query.id" class="ml-2 mr-2 mt-0" /> </v-card> <v-breadcrumbs :items="items" class="pa-0 mt-2" /> <v-dialog @@ -164,6 +165,11 @@ export default { this.loading = false } this.loading = false + + // refresh QueryResults table + setTimeout(() => { + this.$refs.queryResults.execute() + }, 200) }, async reExecute () { try { diff --git a/fda-ui/pages/container/_container_id/database/_database_id/table/_table_id/import.vue b/fda-ui/pages/container/_container_id/database/_database_id/table/_table_id/import.vue index 08b02651cd9c905080752d6626dc8af6b91b4e49..7fd8e3bb7156e2e293aed5417c5d293d7e2a13ad 100644 --- a/fda-ui/pages/container/_container_id/database/_database_id/table/_table_id/import.vue +++ b/fda-ui/pages/container/_container_id/database/_database_id/table/_table_id/import.vue @@ -73,11 +73,14 @@ </v-row> </v-card-text> <v-card-actions> - <v-col> - <v-btn :disabled="!file" :loading="loading" color="primary" @click="upload">Next</v-btn> - </v-col> + <v-btn :disabled="!file" :loading="loading" color="primary" @click="upload">Upload</v-btn> + <v-btn :to="`/container/${$route.params.container_id}/database/${$route.params.database_id}/table/${$route.params.table_id}`" outlined> + <v-icon>mdi-table</v-icon> + View + </v-btn> </v-card-actions> </v-card> + <v-breadcrumbs :items="items" class="pa-0 mt-2" /> </div> </template> <script> @@ -107,7 +110,15 @@ export default { false_element: null }, file: null, - fileLocation: null + fileLocation: null, + items: [ + { text: 'Databases', to: '/container', activeClass: '' }, + { + text: `${this.$route.params.database_id}`, + to: `/container/${this.$route.params.container_id}/database/${this.$route.params.database_id}/info`, + activeClass: '' + } + ] } }, computed: { diff --git a/fda-ui/pages/container/_container_id/database/_database_id/table/_table_id/index.vue b/fda-ui/pages/container/_container_id/database/_database_id/table/_table_id/index.vue index e8606594643ac0e979dfda72c12ff2842448412d..c001970115f061c3dcf80742ea60af3827a785f6 100644 --- a/fda-ui/pages/container/_container_id/database/_database_id/table/_table_id/index.vue +++ b/fda-ui/pages/container/_container_id/database/_database_id/table/_table_id/index.vue @@ -7,10 +7,13 @@ </v-toolbar-title> <v-spacer /> <v-toolbar-title> + <v-btn class="mr-2" :to="`/container/${$route.params.container_id}/database/${$route.params.database_id}/table`"> + <v-icon left>mdi-arrow-left</v-icon> Back + </v-btn> <v-btn class="mr-2" :disabled="!token" :to="`/container/${$route.params.container_id}/database/${$route.params.database_id}/table/${$route.params.table_id}/import`"> <v-icon left>mdi-cloud-upload</v-icon> Import csv </v-btn> - <v-btn color="primary" :disabled="!token" :href="`/api/container/${$route.params.container_id}/database/${$route.params.database_id}/table/${$route.params.table_id}/data/export`" target="_blank"> + <v-btn v-if="false" color="primary" :disabled="!token" :href="`/api/container/${$route.params.container_id}/database/${$route.params.database_id}/table/${$route.params.table_id}/data/export`" target="_blank"> <v-icon left>mdi-download</v-icon> Download </v-btn> </v-toolbar-title> @@ -121,6 +124,9 @@ export default { version (newVersion, oldVersion) { console.info('selected new version', newVersion) this.loadData() + }, + options () { + this.loadData() } }, mounted () { @@ -135,8 +141,14 @@ export default { console.debug('headers', res.data.columns) this.headers = res.data.columns.map((c) => { return { - value: c.internal_name, - text: this.columnAddition(c) + c.name + value: c.name, + text: this.columnAddition(c) + c.name, + + // sorting is disabled for now + // backed has sorting functionality in 8cf84d4d3502202c5947eefb49bc6f48cebff234, + // branch 53-task-provide-property-information-for-metadata-db-frontend + // currenlty unmergable to dev + sortable: false } }) } catch (err) { @@ -154,7 +166,7 @@ export default { } const res = await this.$axios.get(url) console.debug('version', this.datetime, 'table data', res.data) - this.total = res.headers['fda-count'] + this.total = parseInt(res.headers['fda-count']) this.rows = res.data.result } catch (err) { console.error('failed to load data', err) diff --git a/fda-ui/pages/container/_container_id/database/_database_id/table/import.vue b/fda-ui/pages/container/_container_id/database/_database_id/table/import.vue index f9b0e65e5123d1a5dc15c8c052c2cc04223129b7..14b7d43d7604c9fc225f4984a85f05594a320d4d 100644 --- a/fda-ui/pages/container/_container_id/database/_database_id/table/import.vue +++ b/fda-ui/pages/container/_container_id/database/_database_id/table/import.vue @@ -14,7 +14,7 @@ <v-col cols="8"> <v-text-field v-model="tableCreate.name" - required + :rules="[v => notEmpty(v) || $t('Required')]" autocomplete="off" label="Name *" /> </v-col> @@ -23,14 +23,14 @@ <v-col cols="8"> <v-text-field v-model="tableCreate.description" - required + :rules="[v => notEmpty(v) || $t('Required')]" autocomplete="off" label="Description *" /> </v-col> </v-row> <v-row dense> <v-col cols="8"> - <v-btn :disabled="!step1Valid" color="primary" type="submit" @click="step = 2"> + <v-btn :disabled="!validStep1" color="primary" type="submit" @click="step = 2"> Continue </v-btn> </v-col> @@ -48,7 +48,7 @@ <v-col cols="8"> <v-select v-model="tableCreate.separator" - :rules="[rules.required]" + :rules="[v => notEmpty(v) || $t('Required')]" :items="separators" required hint="Character separating the values" @@ -59,10 +59,12 @@ <v-col cols="8"> <v-text-field v-model="tableCreate.skip_lines" - :rules="[rules.required, rules.positive]" + :rules="[ + v => notEmpty(v) || $t('Required'), + v => isNonNegativeInteger(v) || $t('Number of lines to skip')]" type="number" required - hint="Skip n lines from the top" + hint="Skip n lines from the top. These may include comments or the header of column names." label="Skip Lines *" placeholder="e.g. 0" /> </v-col> @@ -96,7 +98,7 @@ </v-row> <v-row dense> <v-col cols="6"> - <v-btn :disabled="!tableCreate.separator || !tableCreate.skip_lines" :loading="loading" color="primary" type="submit" @click="step = 3">Next</v-btn> + <v-btn :disabled="!validStep2 || !tableCreate.separator || !tableCreate.skip_lines" :loading="loading" color="primary" type="submit" @click="step = 3">Next</v-btn> </v-col> </v-row> </v-form> @@ -216,6 +218,8 @@ </div> </template> <script> +const { notEmpty, isNonNegativeInteger } = require('@/utils') + export default { name: 'TableFromCSV', components: { @@ -242,11 +246,11 @@ export default { text: `${this.$route.params.database_id}`, to: `/container/${this.$route.params.container_id}/database/${this.$route.params.database_id}/info`, activeClass: '' - } + }, + { text: 'Tables', to: `/container/${this.$route.params.container_id}/database/${this.$route.params.database_id}/table`, activeClass: '' } ], rules: { - required: value => !!value || 'Required', - positive: value => value >= 0 || 'Positive number' + required: value => !!value || 'Required' }, dateFormats: [], tableCreate: { @@ -257,7 +261,7 @@ export default { true_element: null, null_element: null, separator: ',', - skip_lines: 0 + skip_lines: '1' }, loading: false, file: null, @@ -265,11 +269,12 @@ export default { fileLocation: null, columns: [], columnTypes: [ - { value: 'ENUM', text: 'Enumeration' }, + // { value: 'ENUM', text: 'Enumeration' }, // Disabled for now, not implemented, #145 { value: 'BOOLEAN', text: 'Boolean' }, { value: 'NUMBER', text: 'Number' }, { value: 'BLOB', text: 'Binary Large Object' }, { value: 'DATE', text: 'Date' }, + { value: 'DECIMAL', text: 'Decimal' }, { value: 'STRING', text: 'Character Varying' }, { value: 'TEXT', text: 'Text' } ], @@ -277,9 +282,6 @@ export default { } }, computed: { - step1Valid () { - return this.tableCreate.name !== null && this.tableCreate.name.length > 0 && this.tableCreate.description !== null && this.tableCreate.description.length > 0 - }, token () { return this.$store.state.token } @@ -288,6 +290,8 @@ export default { this.loadDateFormats() }, methods: { + notEmpty, + isNonNegativeInteger, submit () { this.$refs.form.validate() }, @@ -343,19 +347,24 @@ export default { }, async createTable () { /* make enum values to array */ - this.tableCreate.columns.forEach((column) => { + const validColumns = this.tableCreate.columns.map((column) => { // validate `id` column: must be a PK if (column.name === 'id' && (!column.primary_key)) { this.$toast.error('Column `id` has to be a Primary Key') - return + return false } - if (column.enum_values == null) { - return + if (column.enum_values === null) { + return false } if (column.enum_values.length > 0) { column.enum_values = column.enum_values.split(',') } + return true }) + + // bail out if there is a problem with one of the columns + if (!validColumns.every(Boolean)) { return } + const createUrl = `/api/container/${this.$route.params.container_id}/database/${this.$route.params.database_id}/table` let createResult try { diff --git a/fda-ui/store/index.js b/fda-ui/store/index.js index b5fd011b1721d292a52751655953995d0aa03604..f823bb0e8566a14e4f9781503f0aecf33f558710 100644 --- a/fda-ui/store/index.js +++ b/fda-ui/store/index.js @@ -1,7 +1,8 @@ export const state = () => ({ - db: null, token: null, - user: null + user: null, + db: null, + table: null }) export const mutations = { @@ -13,5 +14,13 @@ export const mutations = { }, SET_USER (state, user) { state.user = user + }, + + /** + Workaround. Helps to go 'back' from table data view and + have the accordion open on the same table + */ + SET_TABLE (state, table) { + state.table = table } } diff --git a/fda-ui/test/e2e/database.js b/fda-ui/test/e2e/database.js index c0aa2ea11586a5e0b77fbe06d2b89ef77eca8b5f..1820f89fbf91fa90839baa5c082e46e1b2b41798 100644 --- a/fda-ui/test/e2e/database.js +++ b/fda-ui/test/e2e/database.js @@ -1,6 +1,6 @@ const test = require('ava') +const axios = require('axios') const { pageMacro, before, after } = require('./_utils') -const axios = require("axios"); test.before(before) test.after(after) diff --git a/fda-ui/utils/index.js b/fda-ui/utils/index.js new file mode 100644 index 0000000000000000000000000000000000000000..58c28e61d32030a4529b0cb68f3f6e052119cf11 --- /dev/null +++ b/fda-ui/utils/index.js @@ -0,0 +1,32 @@ +function notEmpty (str) { + return typeof str === 'string' && str.trim().length > 0 +} + +/** + * From https://stackoverflow.com/questions/10834796/validate-that-a-string-is-a-positive-integer + + Tests: + + "0" : true + "23" : true + "-10" : false + "10.30" : false + "-40.1" : false + "string" : false + "1234567890" : true + "129000098131766699.1" : false + "-1e7" : false + "1e7" : true + "1e10" : false + "1edf" : false + " " : false + "" : false + */ +function isNonNegativeInteger (str) { + return str >>> 0 === parseFloat(str) +} + +module.exports = { + notEmpty, + isNonNegativeInteger +}