diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 20d39e1074ee57b7e1870e59d05053ee6063bc7d..ae242539e888253ee3646cc24c149d6b26f2c325 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -54,8 +54,6 @@ services: image: dbrepo/discovery-service:latest networks: core: - ports: - - "9090:9090" env_file: - .env logging: @@ -86,8 +84,6 @@ services: networks: userdb: core: - ports: - - "9092:9092" env_file: - .env volumes: @@ -109,8 +105,6 @@ services: image: dbrepo/container-service:latest networks: core: - ports: - - "9091:9091" env_file: - .env volumes: @@ -128,8 +122,6 @@ services: image: dbrepo/authentication-service:latest networks: core: - ports: - - "9097:9097" env_file: - .env depends_on: @@ -150,8 +142,6 @@ services: networks: core: userdb: - ports: - - "9093:9093" env_file: - .env volumes: @@ -172,8 +162,6 @@ services: networks: core: userdb: - ports: - - "9094:9094" env_file: - .env volumes: @@ -196,8 +184,6 @@ services: image: dbrepo/identifier-service:latest networks: core: - ports: - - "9096:9096" env_file: - .env volumes: @@ -219,8 +205,6 @@ services: core: env_file: - .env - ports: - - "9099:9099" depends_on: metadata-db: condition: service_started @@ -236,8 +220,6 @@ services: core: userdb: command: sh -c "/wait && flask run" # docker-compose should not test the implementation - ports: - - "5000:5000" volumes: - ${SHARED_FILESYSTEM}:/tmp - /var/run/docker.sock:/var/run/docker.sock @@ -254,8 +236,6 @@ services: image: dbrepo/units-service:latest networks: core: - ports: - - "5010:5010" volumes: - /tmp:/tmp - /var/run/docker.sock:/var/run/docker.sock @@ -273,8 +253,6 @@ services: networks: core: ports: - - "5672:5672" - - "9098:9098" - "15672:15672" env_file: - .env @@ -301,9 +279,6 @@ services: depends_on: discovery-service: condition: service_healthy - ports: - - 9200:9200 - - 9600:9600 volumes: - search-service-data:/usr/share/elasticsearch/data logging: diff --git a/fda-query-service/Dockerfile b/fda-query-service/Dockerfile index b2dac0fa1813fdca86d8cf4ee3e9bcf86008fb80..8b071ee3368a8b89af8655f6e827987991afa454 100644 --- a/fda-query-service/Dockerfile +++ b/fda-query-service/Dockerfile @@ -31,6 +31,7 @@ ENV GATEWAY_ENDPOINT=http://gateway-service:9095 ENV SHARED_FILESYSTEM=/tmp ENV BROKER_CONSUMERS=2 ENV LOG_LEVEL=debug +ENV NOT_SUPPORTED_KEYWORDS=\\*,AVG,BIT_AND,BIT_OR,BIT_XOR,COUNT,COUNTDISTINCT,GROUP_CONCAT,JSON_ARRAYAGG,JSON_OBJECTAGG,MAX,MIN,STD,STDDEV,STDDEV_POP,STDDEV_SAMP,SUM,VARIANCE,VAR_POP,VAR_SAMP,-- COPY ./service_ready /usr/bin RUN chmod +x /usr/bin/service_ready diff --git a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/AbstractEndpoint.java b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/AbstractEndpoint.java index af0232f142d04469d981aedf100e7aa8b1e4ebde..674a1f9c4cf0fd36f31879d32bafd624b6788154 100644 --- a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/AbstractEndpoint.java +++ b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/AbstractEndpoint.java @@ -2,21 +2,20 @@ package at.tuwien.endpoint; import at.tuwien.SortType; import at.tuwien.api.database.query.ExecuteStatementDto; +import at.tuwien.config.QueryConfig; import at.tuwien.entities.database.Database; -import at.tuwien.entities.database.table.Table; import at.tuwien.entities.identifier.Identifier; import at.tuwien.exception.*; import at.tuwien.service.DatabaseService; import at.tuwien.service.IdentifierService; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.core.Authentication; -import java.io.File; import java.io.IOException; -import java.nio.charset.Charset; import java.security.Principal; +import java.util.Arrays; +import java.util.LinkedList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -26,11 +25,14 @@ import static at.tuwien.entities.identifier.VisibilityType.EVERYONE; @Slf4j public abstract class AbstractEndpoint { + private final QueryConfig queryConfig; private final DatabaseService databaseService; private final IdentifierService identifierService; @Autowired - protected AbstractEndpoint(DatabaseService databaseService, IdentifierService identifierService) { + protected AbstractEndpoint(QueryConfig queryConfig, DatabaseService databaseService, + IdentifierService identifierService) { + this.queryConfig = queryConfig; this.databaseService = databaseService; this.identifierService = identifierService; } @@ -103,23 +105,27 @@ public abstract class AbstractEndpoint { } } - protected void validateForbiddenStatements(ExecuteStatementDto data) throws QueryMalformedException, - QueryStoreException { - final StringBuilder regex = new StringBuilder("["); - try { - FileUtils.readLines(new File("src/main/resources/forbidden.txt"), Charset.defaultCharset()) - .forEach(regex::append); - } catch (IOException e) { - log.error("Failed to load forbidden keywords list, reason {}", e.getMessage()); - throw new QueryStoreException("Failed to load forbidden keywords list", e); - } - final Pattern pattern = Pattern.compile(regex + "]"); - final Matcher matcher = pattern.matcher(data.getStatement()); - final boolean found = matcher.find(); - if (found) { - log.error("Query contains blacklisted character"); - throw new QueryMalformedException("Query contains blacklisted character"); - } + /** + * Do not allow aggregate functions and comments + * https://mariadb.com/kb/en/aggregate-functions/ + */ + protected void validateForbiddenStatements(ExecuteStatementDto data) throws QueryMalformedException { + final List<String> words = new LinkedList<>(); + Arrays.stream(queryConfig.getNotSupportedKeywords()) + .forEach(keyword -> { + final Pattern pattern = Pattern.compile(keyword); + final Matcher matcher = pattern.matcher(data.getStatement()); + final boolean found = matcher.find(); + if (found) { + words.add(keyword); + } + }); + if (words.size() == 0) { + return; + } + log.error("Query contains forbidden keyword(s): {}", words); + log.debug("forbidden keywords: {}", words); + throw new QueryMalformedException("Query contains forbidden keyword(s): " + Arrays.toString(words.toArray())); } protected Boolean hasQueuePermission(Long containerId, Long databaseId, Long tableId, String permissionCode, diff --git a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/ExportEndpoint.java b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/ExportEndpoint.java index edac44b71762e5257711614a715f21f7033dff21..d7522723ee572de210760755329f69395f2883f2 100644 --- a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/ExportEndpoint.java +++ b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/ExportEndpoint.java @@ -1,6 +1,7 @@ package at.tuwien.endpoint; import at.tuwien.ExportResource; +import at.tuwien.config.QueryConfig; import at.tuwien.exception.*; import at.tuwien.service.DatabaseService; import at.tuwien.service.IdentifierService; @@ -29,9 +30,9 @@ public class ExportEndpoint extends AbstractEndpoint { private final QueryService queryService; @Autowired - public ExportEndpoint(QueryService queryService, DatabaseService databaseService, + public ExportEndpoint(QueryConfig queryConfig, QueryService queryService, DatabaseService databaseService, IdentifierService identifierService) { - super(databaseService, identifierService); + super(queryConfig, databaseService, identifierService); this.queryService = queryService; } diff --git a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java index 3abdccb05c4079688a4baa84a60dd778f1156d0d..b43e00194963309d06383a3a201630492895f757 100644 --- a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java +++ b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java @@ -3,6 +3,7 @@ package at.tuwien.endpoint; import at.tuwien.ExportResource; import at.tuwien.SortType; import at.tuwien.api.database.query.*; +import at.tuwien.config.QueryConfig; import at.tuwien.querystore.Query; import at.tuwien.exception.*; import at.tuwien.service.*; @@ -20,8 +21,6 @@ import org.springframework.web.bind.annotation.*; import javax.validation.Valid; import javax.validation.constraints.NotNull; import java.security.Principal; -import java.util.regex.Matcher; -import java.util.regex.Pattern; @Log4j2 @RestController @@ -32,9 +31,9 @@ public class QueryEndpoint extends AbstractEndpoint { private final StoreService storeService; @Autowired - public QueryEndpoint(QueryService queryService, StoreService storeService, DatabaseService databaseService, + public QueryEndpoint(QueryConfig queryConfig, QueryService queryService, StoreService storeService, DatabaseService databaseService, IdentifierService identifierService) { - super(databaseService, identifierService); + super(queryConfig, databaseService, identifierService); this.queryService = queryService; this.storeService = storeService; } diff --git a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/StoreEndpoint.java b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/StoreEndpoint.java index 7582107b8d77199caff47977d6e85ece3ae4bae1..1de0af75051fe38a4a14a1f72fa69939270a40c4 100644 --- a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/StoreEndpoint.java +++ b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/StoreEndpoint.java @@ -2,7 +2,7 @@ package at.tuwien.endpoint; import at.tuwien.api.database.query.QueryBriefDto; import at.tuwien.api.database.query.QueryDto; -import at.tuwien.api.database.query.QueryResultDto; +import at.tuwien.config.QueryConfig; import at.tuwien.entities.user.User; import at.tuwien.mapper.UserMapper; import at.tuwien.querystore.Query; @@ -36,10 +36,10 @@ public class StoreEndpoint extends AbstractEndpoint { private final StoreService storeService; @Autowired - public StoreEndpoint(UserMapper userMapper, QueryMapper queryMapper, + public StoreEndpoint(QueryConfig queryConfig, UserMapper userMapper, QueryMapper queryMapper, UserService userService, StoreService storeService, DatabaseService databaseService, IdentifierService identifierService) { - super(databaseService, identifierService); + super(queryConfig, databaseService, identifierService); this.userMapper = userMapper; this.queryMapper = queryMapper; this.userService = userService; diff --git a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableDataEndpoint.java b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableDataEndpoint.java index 85120ecf9077b22e217b65b8080bad004d9c2eeb..e7215bac27d632b7f62b5149501f2c5de0499031 100644 --- a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableDataEndpoint.java +++ b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableDataEndpoint.java @@ -6,6 +6,7 @@ import at.tuwien.api.database.query.QueryResultDto; import at.tuwien.api.database.table.TableCsvDeleteDto; import at.tuwien.api.database.table.TableCsvDto; import at.tuwien.api.database.table.TableCsvUpdateDto; +import at.tuwien.config.QueryConfig; import at.tuwien.exception.*; import at.tuwien.service.*; import io.micrometer.core.annotation.Timed; @@ -32,9 +33,9 @@ public class TableDataEndpoint extends AbstractEndpoint { private final QueryService queryService; @Autowired - public TableDataEndpoint(QueryService queryService, DatabaseService databaseService, + public TableDataEndpoint(QueryConfig queryConfig, QueryService queryService, DatabaseService databaseService, IdentifierService identifierService) { - super(databaseService, identifierService); + super(queryConfig, databaseService, identifierService); this.queryService = queryService; } diff --git a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableHistoryEndpoint.java b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableHistoryEndpoint.java index 2845209af46359fdefa2b497b61fd0c9361e6d19..73fa9fe8473b354d817ce23d9d7619f27590d1ac 100644 --- a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableHistoryEndpoint.java +++ b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableHistoryEndpoint.java @@ -1,6 +1,7 @@ package at.tuwien.endpoint; import at.tuwien.api.database.table.TableHistoryDto; +import at.tuwien.config.QueryConfig; import at.tuwien.exception.*; import at.tuwien.service.*; import io.micrometer.core.annotation.Timed; @@ -25,9 +26,9 @@ public class TableHistoryEndpoint extends AbstractEndpoint { private final TableService tableService; @Autowired - public TableHistoryEndpoint(TableService tableService, DatabaseService databaseService, + public TableHistoryEndpoint(QueryConfig queryConfig, TableService tableService, DatabaseService databaseService, IdentifierService identifierService) { - super(databaseService, identifierService); + super(queryConfig, databaseService, identifierService); this.tableService = tableService; } diff --git a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/ViewEndpoint.java b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/ViewEndpoint.java index 70bbf4461a43a2c8183d9e1bb1311f3c3e491708..69b86dbea8d2cb41c40b613aec10f7126a3fb52a 100644 --- a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/ViewEndpoint.java +++ b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/ViewEndpoint.java @@ -6,6 +6,7 @@ import at.tuwien.api.database.ViewDto; import at.tuwien.api.database.query.ExecuteStatementDto; import at.tuwien.api.database.query.QueryResultDto; import at.tuwien.api.database.query.QueryTypeDto; +import at.tuwien.config.QueryConfig; import at.tuwien.entities.database.Database; import at.tuwien.entities.database.View; import at.tuwien.exception.*; @@ -38,9 +39,9 @@ public class ViewEndpoint extends AbstractEndpoint { private final DatabaseService databaseService; @Autowired - public ViewEndpoint(ViewService viewService, DatabaseService databaseService, IdentifierService identifierService, + public ViewEndpoint(QueryConfig queryConfig, ViewService viewService, DatabaseService databaseService, IdentifierService identifierService, ViewMapper viewMapper, QueryService queryService) { - super(databaseService, identifierService); + super(queryConfig, databaseService, identifierService); this.viewService = viewService; this.databaseService = databaseService; this.viewMapper = viewMapper; diff --git a/fda-query-service/rest-service/src/main/resources/application-docker.yml b/fda-query-service/rest-service/src/main/resources/application-docker.yml index 933074053eb3c0244e53045264c5d54f01e3077a..de1b4101c1018baa6777a6644e62d65fe6b151a9 100644 --- a/fda-query-service/rest-service/src/main/resources/application-docker.yml +++ b/fda-query-service/rest-service/src/main/resources/application-docker.yml @@ -39,3 +39,4 @@ fda: gateway.endpoint: "${GATEWAY_ENDPOINT}" ready.path: /ready consumers: 2 + unsupported: "${NOT_SUPPORTED_KEYWORDS}" diff --git a/fda-query-service/rest-service/src/main/resources/application-local.yml b/fda-query-service/rest-service/src/main/resources/application-local.yml index 6708486f5e07fa88d89e5eba2c058f2247673bc0..d9fbe51756f37e7b89a2445ccecb1baa74836da2 100644 --- a/fda-query-service/rest-service/src/main/resources/application-local.yml +++ b/fda-query-service/rest-service/src/main/resources/application-local.yml @@ -39,3 +39,4 @@ fda: gateway.endpoint: http://localhost:9095 ready.path: ./ready consumers: 2 + unsupported: \\*,AVG,BIT_AND,BIT_OR,BIT_XOR,COUNT,COUNTDISTINCT,GROUP_CONCAT,JSON_ARRAYAGG,JSON_OBJECTAGG,MAX,MIN,STD,STDDEV,STDDEV_POP,STDDEV_SAMP,SUM,VARIANCE,VAR_POP,VAR_SAMP,-- diff --git a/fda-query-service/rest-service/src/main/resources/application.yml b/fda-query-service/rest-service/src/main/resources/application.yml index 4139a27347632709ba0a630e4c3151e0505e2af9..a309b0a6bad388704931705c4bc3020d8c25a49a 100644 --- a/fda-query-service/rest-service/src/main/resources/application.yml +++ b/fda-query-service/rest-service/src/main/resources/application.yml @@ -39,3 +39,4 @@ fda: gateway.endpoint: "${GATEWAY_ENDPOINT}" ready.path: /ready consumers: "${BROKER_CONSUMERS}" + unsupported: "${NOT_SUPPORTED_KEYWORDS}" diff --git a/fda-query-service/rest-service/src/main/resources/forbidden.txt b/fda-query-service/rest-service/src/main/resources/forbidden.txt deleted file mode 100644 index 89bdcae71069ee0e80035dbc2a0143d570ab253d..0000000000000000000000000000000000000000 --- a/fda-query-service/rest-service/src/main/resources/forbidden.txt +++ /dev/null @@ -1 +0,0 @@ - \* \ No newline at end of file diff --git a/fda-query-service/services/src/main/java/at/tuwien/config/QueryConfig.java b/fda-query-service/services/src/main/java/at/tuwien/config/QueryConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..3b9be44361f26dcba516db7629fee55ad7046d0e --- /dev/null +++ b/fda-query-service/services/src/main/java/at/tuwien/config/QueryConfig.java @@ -0,0 +1,14 @@ +package at.tuwien.config; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +@Getter +@Configuration +public class QueryConfig { + + @Value("${fda.unsupported}") + private String[] notSupportedKeywords; + +} diff --git a/fda-ui/components/query/Builder.vue b/fda-ui/components/query/Builder.vue index 81888b813f9c6b9b69dfb433740147d1ddc73f21..aa4be844754c3c426ca55b1468087acec6246c55 100644 --- a/fda-ui/components/query/Builder.vue +++ b/fda-ui/components/query/Builder.vue @@ -102,7 +102,11 @@ <v-alert border="left" color="info"> - Currently, comments in the query (e.g. <code>-- Comment</code>) are not supported! + Currently, comments and <a href="https://mariadb.com/kb/en/aggregate-functions/" target="_blank">aggregation functions</a> + <sup> + <v-icon dense x-small>mdi-open-in-new</v-icon> + </sup> + are not supported! </v-alert> </v-col> </v-row> @@ -146,9 +150,34 @@ export default { table: {}, tables: [], views: [], + foundForbiddenKeywords: [], + forbiddenKeywords: [ + '\\*', + 'AVG', + 'BIT_AND', + 'BIT_OR', + 'BIT_XOR', + 'COUNT', + 'COUNT', 'DISTINCT', + 'GROUP_CONCAT', + 'JSON_ARRAYAGG', + 'JSON_OBJECTAGG', + 'MAX', + 'MIN', + 'STD', + 'STDDEV', + 'STDDEV_POP', + 'STDDEV_SAMP', + 'SUM', + 'VARIANCE', + 'VAR_POP', + 'VAR_SAMP', + '--' + ], tableDetails: null, resultId: null, valid: false, + errorKeyword: null, query: { sql: '' }, @@ -195,11 +224,9 @@ export default { if (this.tabs === 0) { return this.query.sql } else if (this.tabs === 1) { - const sql = this.rawSQL.replaceAll('\n', ' ') /* remove newline */ + return this.rawSQL.replaceAll('\n', ' ') /* remove newline */ .replaceAll(/\s+/g, ' ') /* remove whitespace */ .trim() - console.debug('raw sql', sql) - return sql } return null }, diff --git a/fda-ui/components/query/Results.vue b/fda-ui/components/query/Results.vue index 235e8a44b1665989a2fcdb75cbe7cff1888aefce..d5c13f81121e5dcb48cca496bd359967c120aa33 100644 --- a/fda-ui/components/query/Results.vue +++ b/fda-ui/components/query/Results.vue @@ -69,8 +69,8 @@ export default { this.loading = false parent.resultId = res.data.id } catch (err) { - console.error('failed to execute query', err) - this.$toast.error('Failed to execute query: ' + err.response.data.message) + console.error('Failed to execute query', err.response.data) + this.$toast.error(err.response.data.message) this.loading = false } },