From bc209c123d785c33250fabb439ed25867b1d139e Mon Sep 17 00:00:00 2001 From: Martin Weise <martin.weise@tuwien.ac.at> Date: Fri, 21 Feb 2025 08:51:55 +0100 Subject: [PATCH] Fix more tests Signed-off-by: Martin Weise <martin.weise@tuwien.ac.at> --- .gitlab-ci.yml | 10 +++--- .../at/tuwien/endpoints/SubsetEndpoint.java | 2 +- .../java/at/tuwien/config/MariaDbConfig.java | 11 ++++-- .../endpoint/SubsetEndpointUnitTest.java | 6 ++++ .../MetadataServiceGatewayUnitTest.java | 4 +++ .../listener/DefaultListenerUnitTest.java | 4 +-- .../service/SubsetServiceIntegrationTest.java | 34 ++++++++++--------- .../src/test/resources/init/querystore.sql | 6 ++-- .../impl/MetadataServiceGatewayImpl.java | 4 +-- .../main/java/at/tuwien/test/BaseTest.java | 18 +++++----- 10 files changed, 59 insertions(+), 40 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 502cd67b02..54050ee103 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -304,7 +304,7 @@ test-analyse-service: script: - "pip install pipenv" - "pipenv install gunicorn && pipenv install --dev --system --deploy" - - cd ./dbrepo-analyse-service/ && coverage run --rcfile=.coveragerc -m pytest tests/test_determine_dt.py tests/test_determine_pk.py tests/test_s3_client.py && coverage html && coverage report > ./coverage.txt + - cd ./dbrepo-analyse-service/ && coverage run --rcfile=.coveragerc -m pytest tests/test_determine_dt.py tests/test_determine_pk.py tests/test_s3_client.py && coverage html && coverage xml && coverage report > ./coverage.txt - "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'" artifacts: when: always @@ -326,7 +326,7 @@ test-auth-service-init: - "pip install pipenv" - "pipenv install gunicorn && pipenv install --dev --system --deploy" script: - - cd ./dbrepo-auth-service/init/ && coverage run --rcfile=.coveragerc -m pytest tests/test_unit_app.py && coverage html && coverage report > ./coverage.txt + - cd ./dbrepo-auth-service/init/ && coverage run --rcfile=.coveragerc -m pytest tests/test_unit_app.py && coverage html && coverage xml && coverage report > ./coverage.txt - "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'" artifacts: when: always @@ -357,7 +357,7 @@ test-search-service: script: - "pip install pipenv" - "pipenv install gunicorn && pipenv install --dev --system --deploy" - - cd ./dbrepo-search-service/ && coverage run --rcfile=.coveragerc -m pytest tests/test_app.py tests/test_jwt.py tests/test_opensearch_client.py tests/test_keycloak_client.py && coverage html && coverage report > ./coverage.txt + - cd ./dbrepo-search-service/ && coverage run --rcfile=.coveragerc -m pytest tests/test_app.py tests/test_jwt.py tests/test_opensearch_client.py tests/test_keycloak_client.py && coverage html && coverage xml && coverage report > ./coverage.txt - "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'" artifacts: when: always @@ -381,7 +381,7 @@ test-search-service-init: script: - "pip install pipenv" - "pipenv install gunicorn && pipenv install --dev --system --deploy" - - cd ./dbrepo-search-service/init/ && coverage run --rcfile=.coveragerc -m pytest tests/test_app.py tests/test_keycloak_client.py tests/test_opensearch_client.py && coverage html && coverage report > ./coverage.txt + - cd ./dbrepo-search-service/init/ && coverage run --rcfile=.coveragerc -m pytest tests/test_app.py tests/test_keycloak_client.py tests/test_opensearch_client.py && coverage html && coverage xml && coverage report > ./coverage.txt - "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'" artifacts: when: always @@ -405,7 +405,7 @@ test-lib: script: - "pip install pipenv" - "pipenv install gunicorn && pipenv install --dev --system --deploy" - - cd ./lib/python/ && coverage run --rcfile=.coveragerc -m pytest tests/test_unit_container.py tests/test_unit_concept.py tests/test_unit_database.py tests/test_unit_identifier.py tests/test_unit_license.py tests/test_unit_query.py tests/test_unit_rest_client.py tests/test_unit_table.py tests/test_unit_user.py tests/test_unit_view.py tests/test_unit_unit.py && coverage html && coverage report > ./coverage.txt + - cd ./lib/python/ && coverage run --rcfile=.coveragerc -m pytest tests/test_unit_container.py tests/test_unit_concept.py tests/test_unit_database.py tests/test_unit_identifier.py tests/test_unit_license.py tests/test_unit_query.py tests/test_unit_rest_client.py tests/test_unit_table.py tests/test_unit_user.py tests/test_unit_view.py tests/test_unit_unit.py && coverage html && coverage xml && coverage report > ./coverage.txt - "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'" artifacts: when: always diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java index 0ca1ee86b8..245774726f 100644 --- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java +++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java @@ -181,7 +181,7 @@ public class SubsetEndpoint extends RestEndpoint { timestamp = Instant.now(); log.debug("timestamp not set: default to {}", timestamp); } - if (accept == null || accept.isEmpty() || accept.isBlank()) { + if (accept == null || accept.isBlank()) { accept = MediaType.APPLICATION_JSON_VALUE; log.debug("accept header not set: default to {}", accept); } diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/config/MariaDbConfig.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/config/MariaDbConfig.java index d4daa90741..709e389386 100644 --- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/config/MariaDbConfig.java +++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/config/MariaDbConfig.java @@ -141,11 +141,11 @@ public class MariaDbConfig { } } - public static void insertQueryStore(DatabaseDto database, QueryDto query, UUID userId) throws SQLException { + public static UUID insertQueryStore(DatabaseDto database, QueryDto query, UUID userId) throws SQLException { final String jdbc = "jdbc:mariadb://" + database.getContainer().getHost() + ":" + database.getContainer().getPort() + "/" + database.getInternalName(); log.trace("connect to database: {}", jdbc); try (Connection connection = DriverManager.getConnection(jdbc, database.getContainer().getUsername(), database.getContainer().getPassword())) { - final PreparedStatement prepareStatement = connection.prepareStatement( + PreparedStatement prepareStatement = connection.prepareStatement( "INSERT INTO qs_queries (created_by, query, query_normalized, is_persisted, query_hash, result_hash, result_number, created, executed) VALUES (?,?,?,?,?,?,?,?,?)"); prepareStatement.setString(1, String.valueOf(userId)); prepareStatement.setString(2, query.getQuery()); @@ -158,6 +158,13 @@ public class MariaDbConfig { prepareStatement.setTimestamp(9, Timestamp.from(query.getExecution())); log.trace("prepared statement: {}", prepareStatement); prepareStatement.executeUpdate(); + /* select */ + prepareStatement = connection.prepareStatement("SELECT id FROM qs_queries WHERE query_hash = ? LIMIT 1"); + prepareStatement.setString(1, query.getQueryHash()); + final ResultSet result = prepareStatement.executeQuery(); + UUID queryId; + result.next(); + return UUID.fromString(result.getString(1)); } } diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java index 122811ecec..f292264060 100644 --- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java +++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java @@ -309,6 +309,8 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest { .thenReturn(mock); when(subsetService.findById(eq(DATABASE_3_PRIVILEGED_DTO), any(UUID.class))) .thenReturn(QUERY_5_DTO); + when(subsetService.create(eq(DATABASE_3_PRIVILEGED_DTO), eq(QUERY_5_STATEMENT), any(Instant.class), any(UUID.class))) + .thenReturn(QUERY_5_ID); when(databaseService.inspectView(any(DatabaseDto.class), anyString())) .thenReturn(QUERY_5_VIEW_DTO); when(httpServletRequest.getMethod()) @@ -406,6 +408,8 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest { .thenReturn(DATABASE_4_PRIVILEGED_DTO); when(subsetService.findById(eq(DATABASE_4_PRIVILEGED_DTO), any(UUID.class))) .thenReturn(QUERY_5_DTO); + when(subsetService.create(eq(DATABASE_4_PRIVILEGED_DTO), eq(QUERY_5_STATEMENT), any(Instant.class), eq(null))) + .thenReturn(QUERY_5_ID); when(subsetService.getData(any(DatabaseDto.class), anyString())) .thenReturn(mock); when(databaseService.inspectView(any(DatabaseDto.class), anyString())) @@ -434,6 +438,8 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest { .thenReturn(DATABASE_1_PRIVILEGED_DTO); when(subsetService.findById(eq(DATABASE_1_PRIVILEGED_DTO), any(UUID.class))) .thenReturn(QUERY_1_DTO); + when(subsetService.create(eq(DATABASE_1_PRIVILEGED_DTO), eq(QUERY_1_STATEMENT), any(Instant.class), any(UUID.class))) + .thenReturn(QUERY_1_ID); when(subsetService.getData(any(DatabaseDto.class), anyString())) .thenReturn(mock); when(databaseService.inspectView(any(DatabaseDto.class), anyString())) diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/gateway/MetadataServiceGatewayUnitTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/gateway/MetadataServiceGatewayUnitTest.java index affb562080..664e504248 100644 --- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/gateway/MetadataServiceGatewayUnitTest.java +++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/gateway/MetadataServiceGatewayUnitTest.java @@ -54,6 +54,7 @@ public class MetadataServiceGatewayUnitTest extends AbstractUnitTest { final HttpHeaders headers = new HttpHeaders(); headers.set("X-Username", CONTAINER_1_PRIVILEGED_USERNAME); headers.set("X-Password", CONTAINER_1_PRIVILEGED_PASSWORD); + headers.set("X-Jdbc-Method", IMAGE_1_JDBC); /* mock */ when(internalRestTemplate.exchange(anyString(), eq(HttpMethod.GET), eq(HttpEntity.EMPTY), eq(TableDto.class))) @@ -158,6 +159,7 @@ public class MetadataServiceGatewayUnitTest extends AbstractUnitTest { final HttpHeaders headers = new HttpHeaders(); headers.set("X-Username", CONTAINER_1_PRIVILEGED_USERNAME); headers.set("X-Password", CONTAINER_1_PRIVILEGED_PASSWORD); + headers.set("X-Jdbc-Method", IMAGE_1_JDBC); /* mock */ when(internalRestTemplate.exchange(anyString(), eq(HttpMethod.GET), eq(HttpEntity.EMPTY), eq(DatabaseDto.class))) @@ -256,6 +258,7 @@ public class MetadataServiceGatewayUnitTest extends AbstractUnitTest { final HttpHeaders headers = new HttpHeaders(); headers.set("X-Username", CONTAINER_1_PRIVILEGED_USERNAME); headers.set("X-Password", CONTAINER_1_PRIVILEGED_PASSWORD); + headers.set("X-Jdbc-Method", IMAGE_1_JDBC); /* mock */ when(internalRestTemplate.exchange(anyString(), eq(HttpMethod.GET), eq(HttpEntity.EMPTY), eq(ContainerDto.class))) @@ -354,6 +357,7 @@ public class MetadataServiceGatewayUnitTest extends AbstractUnitTest { final HttpHeaders headers = new HttpHeaders(); headers.set("X-Username", CONTAINER_1_PRIVILEGED_USERNAME); headers.set("X-Password", CONTAINER_1_PRIVILEGED_PASSWORD); + headers.set("X-Jdbc-Method", IMAGE_1_JDBC); /* mock */ when(internalRestTemplate.exchange(anyString(), eq(HttpMethod.GET), any(HttpEntity.class), eq(ViewDto.class))) diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/listener/DefaultListenerUnitTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/listener/DefaultListenerUnitTest.java index 74042aa578..120f593680 100644 --- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/listener/DefaultListenerUnitTest.java +++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/listener/DefaultListenerUnitTest.java @@ -78,7 +78,7 @@ public class DefaultListenerUnitTest extends AbstractUnitTest { @Test public void onMessage_messageMalformed_fails(CapturedOutput output) throws TableNotFoundException, RemoteUnavailableException, MetadataServiceException { - final Message request = buildMessage("dbrepo.1.1", "{,}", new HashMap<>()); + final Message request = buildMessage(TABLE_1_ROUTING_KEY, "{,}", new HashMap<>()); /* mock */ when(credentialService.getTable(DATABASE_1_ID, TABLE_1_ID)) @@ -92,7 +92,7 @@ public class DefaultListenerUnitTest extends AbstractUnitTest { @Test public void onMessage_tableNotFound_fails(CapturedOutput output) throws TableNotFoundException, RemoteUnavailableException, MetadataServiceException { - final Message request = buildMessage("dbrepo.1.1", "{\"id\":1}", new HashMap<>()); + final Message request = buildMessage(TABLE_1_ROUTING_KEY, "{\"id\": 1}", new HashMap<>()); /* mock */ doThrow(TableNotFoundException.class) diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/SubsetServiceIntegrationTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/SubsetServiceIntegrationTest.java index a3f9f4e0d9..cd92d5579c 100644 --- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/SubsetServiceIntegrationTest.java +++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/SubsetServiceIntegrationTest.java @@ -56,8 +56,8 @@ public class SubsetServiceIntegrationTest extends AbstractUnitTest { /* test */ final List<QueryDto> response = findAll_generic(null); assertEquals(2, response.size()); - assertEquals(QUERY_1_ID, response.get(0).getId()); - assertEquals(QUERY_2_ID, response.get(1).getId()); + assertNotNull(response.get(0).getId()); + assertNotNull(response.get(1).getId()); } @Test @@ -67,7 +67,7 @@ public class SubsetServiceIntegrationTest extends AbstractUnitTest { /* test */ final List<QueryDto> response = findAll_generic(true); assertEquals(1, response.size()); - assertEquals(QUERY_1_ID, response.get(0).getId()); + assertNotNull(response.get(0).getId()); } @Test @@ -77,15 +77,18 @@ public class SubsetServiceIntegrationTest extends AbstractUnitTest { /* test */ final List<QueryDto> response = findAll_generic(false); assertEquals(1, response.size()); - assertEquals(QUERY_2_ID, response.get(0).getId()); + assertNotNull(response.get(0).getId()); } @Test public void findById_succeeds() throws SQLException, QueryNotFoundException, UserNotFoundException, RemoteUnavailableException, MetadataServiceException, DatabaseNotFoundException, InterruptedException { + /* mock */ + final UUID queryId = MariaDbConfig.insertQueryStore(DATABASE_1_PRIVILEGED_DTO, QUERY_1_DTO, USER_1_ID); + /* test */ - findById_generic(QUERY_1_ID); + findById_generic(queryId); } @Test @@ -103,13 +106,14 @@ public class SubsetServiceIntegrationTest extends AbstractUnitTest { InterruptedException { /* mock */ + final UUID queryId2 = MariaDbConfig.insertQueryStore(DATABASE_1_PRIVILEGED_DTO, QUERY_2_DTO, USER_1_ID); when(metadataServiceGateway.getUserById(USER_1_ID)) .thenReturn(USER_1_DTO); /* test */ - persist_generic(QUERY_2_ID, List.of(IDENTIFIER_5_BRIEF_DTO), true); - final QueryDto response = queryService.findById(DATABASE_1_PRIVILEGED_DTO, QUERY_2_ID); - assertEquals(QUERY_2_ID, response.getId()); + persist_generic(queryId2, List.of(IDENTIFIER_5_BRIEF_DTO), true); + final QueryDto response = queryService.findById(DATABASE_1_PRIVILEGED_DTO, queryId2); + assertEquals(queryId2, response.getId()); assertTrue(response.getIsPersisted()); } @@ -119,13 +123,14 @@ public class SubsetServiceIntegrationTest extends AbstractUnitTest { InterruptedException { /* mock */ + final UUID queryId1 = MariaDbConfig.insertQueryStore(DATABASE_1_PRIVILEGED_DTO, QUERY_1_DTO, USER_1_ID); when(metadataServiceGateway.getUserById(USER_1_ID)) .thenReturn(USER_1_DTO); /* test */ - persist_generic(QUERY_1_ID, List.of(IDENTIFIER_2_BRIEF_DTO), false); - final QueryDto response = queryService.findById(DATABASE_1_PRIVILEGED_DTO, QUERY_1_ID); - assertEquals(QUERY_1_ID, response.getId()); + persist_generic(queryId1, List.of(IDENTIFIER_2_BRIEF_DTO), false); + final QueryDto response = queryService.findById(DATABASE_1_PRIVILEGED_DTO, queryId1); + assertEquals(queryId1, response.getId()); assertFalse(response.getIsPersisted()); } @@ -137,15 +142,14 @@ public class SubsetServiceIntegrationTest extends AbstractUnitTest { Thread.sleep(1000) /* wait for test container some more */; /* mock */ - when(metadataServiceGateway.getIdentifiers(DATABASE_1_ID, QUERY_1_ID)) + when(metadataServiceGateway.getIdentifiers(DATABASE_1_ID, queryId)) .thenReturn(List.of(IDENTIFIER_2_BRIEF_DTO)); when(metadataServiceGateway.getUserById(USER_1_ID)) .thenReturn(USER_1_DTO); - MariaDbConfig.insertQueryStore(DATABASE_1_PRIVILEGED_DTO, QUERY_1_DTO, USER_1_ID); /* test */ final QueryDto response = queryService.findById(DATABASE_1_PRIVILEGED_DTO, queryId); - assertEquals(QUERY_1_ID, response.getId()); + assertEquals(queryId, response.getId()); } protected List<QueryDto> findAll_generic(Boolean filterPersisted) throws SQLException, QueryNotFoundException, @@ -175,8 +179,6 @@ public class SubsetServiceIntegrationTest extends AbstractUnitTest { /* mock */ when(metadataServiceGateway.getIdentifiers(DATABASE_1_ID, queryId)) .thenReturn(identifiers); - MariaDbConfig.insertQueryStore(DATABASE_1_PRIVILEGED_DTO, QUERY_1_DTO, USER_1_ID); - MariaDbConfig.insertQueryStore(DATABASE_1_PRIVILEGED_DTO, QUERY_2_DTO, USER_1_ID); /* test */ queryService.persist(DATABASE_1_PRIVILEGED_DTO, queryId, persist); diff --git a/dbrepo-data-service/rest-service/src/test/resources/init/querystore.sql b/dbrepo-data-service/rest-service/src/test/resources/init/querystore.sql index 3e7471df3e..c5cb8d31dc 100644 --- a/dbrepo-data-service/rest-service/src/test/resources/init/querystore.sql +++ b/dbrepo-data-service/rest-service/src/test/resources/init/querystore.sql @@ -1,5 +1,5 @@ CREATE SEQUENCE `qs_queries_seq` NOCACHE; -CREATE TABLE `qs_queries` ( `id` bigint not null primary key default nextval(`qs_queries_seq`), `created` datetime not null default now(), `executed` datetime not null default now(), `created_by` varchar(36), `query` text not null, `query_normalized` text not null, `is_persisted` boolean not null, `query_hash` varchar(255) not null, `result_hash` varchar(255), `result_number` bigint ) WITH SYSTEM VERSIONING; +CREATE TABLE `qs_queries` ( `id` varchar(36) not null primary key default uuid(), `created` datetime not null default now(), `executed` datetime not null default now(), `created_by` varchar(36), `query` text not null, `query_normalized` text not null, `is_persisted` boolean not null, `query_hash` varchar(255) not null, `result_hash` varchar(255), `result_number` bigint ) WITH SYSTEM VERSIONING; CREATE PROCEDURE hash_table(IN name VARCHAR(255), OUT hash VARCHAR(255), OUT count BIGINT) BEGIN DECLARE _sql TEXT; SELECT CONCAT('SELECT SHA2(GROUP_CONCAT(CONCAT_WS(\'\',', GROUP_CONCAT(CONCAT('`', column_name, '`') ORDER BY column_name), ') SEPARATOR \',\'), 256) AS hash, COUNT(*) AS count FROM `', name, '` INTO @hash, @count;') FROM `information_schema`.`columns` WHERE `table_schema` = DATABASE() AND `table_name` = name INTO _sql; PREPARE stmt FROM _sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; SET hash = @hash; SET count = @count; END; -CREATE PROCEDURE store_query(IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _username varchar(255) DEFAULT REGEXP_REPLACE(current_user(), '@.*', ''); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END; -CREATE DEFINER = 'root' PROCEDURE _store_query(IN _username VARCHAR(255), IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END; \ No newline at end of file +CREATE PROCEDURE store_query(IN query TEXT, IN executed DATETIME, OUT queryId varchar(36)) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _username varchar(255) DEFAULT REGEXP_REPLACE(current_user(), '@.*', ''); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END; +CREATE DEFINER = 'root' PROCEDURE _store_query(IN _username VARCHAR(255), IN query TEXT, IN executed DATETIME, OUT queryId varchar(36)) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END; \ No newline at end of file diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java index 7d970cf3e8..d90250f028 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java @@ -48,7 +48,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { ContainerNotFoundException, MetadataServiceException { final ResponseEntity<ContainerDto> response; final String url = "/api/container/" + containerId; - log.debug("get container info from metadata service: {}", url); + log.debug("get container info from metadata service: {}", url); try { response = internalRestTemplate.exchange(url, HttpMethod.GET, HttpEntity.EMPTY, ContainerDto.class); @@ -125,7 +125,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { RemoteUnavailableException, MetadataServiceException { final ResponseEntity<TableDto> response; final String url = "/api/database/" + databaseId + "/table/" + id; - log.debug("get table info from metadata service: {}", url); + log.debug("get table info from metadata service: {}", url); try { response = internalRestTemplate.exchange(url, HttpMethod.GET, HttpEntity.EMPTY, TableDto.class); } catch (ResourceAccessException | HttpServerErrorException e) { diff --git a/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java b/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java index 659efd090e..f90db02119 100644 --- a/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java +++ b/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java @@ -1486,7 +1486,7 @@ public abstract class BaseTest { public static final Boolean TABLE_1_PROCESSED_CONSTRAINTS = true; public static final String TABLE_1_DESCRIPTION = "Weather in Australia"; public static final String TABLE_1_QUEUE_NAME = TABLE_1_INTERNAL_NAME; - public static final String TABLE_1_ROUTING_KEY = "dbrepo\\." + DATABASE_1_ID + "\\." + TABLE_1_ID; + public static final String TABLE_1_ROUTING_KEY = "dbrepo." + DATABASE_1_ID + "." + TABLE_1_ID; public static final Long TABLE_1_AVG_ROW_LENGTH = 3L; public static final Long TABLE_1_NUM_ROWS = 3L; public static final Long TABLE_1_DATA_LENGTH = 2000L; @@ -1687,7 +1687,7 @@ public abstract class BaseTest { public static final Boolean TABLE_2_PROCESSED_CONSTRAINTS = true; public static final String TABLE_2_DESCRIPTION = "Weather location"; public static final String TABLE_2_QUEUE_NAME = TABLE_2_INTERNALNAME; - public static final String TABLE_2_ROUTING_KEY = "dbrepo\\." + DATABASE_1_ID + "\\." + TABLE_2_ID; + public static final String TABLE_2_ROUTING_KEY = "dbrepo." + DATABASE_1_ID + "." + TABLE_2_ID; public static final Instant TABLE_2_CREATED = Instant.ofEpochSecond(1677400007L) /* 2023-02-26 08:26:47 (UTC) */; public static final Instant TABLE_2_LAST_MODIFIED = Instant.ofEpochSecond(1677400007L) /* 2023-02-26 08:26:47 (UTC) */; public static final Long TABLE_2_AVG_ROW_LENGTH = 3L; @@ -1782,7 +1782,7 @@ public abstract class BaseTest { public static final Boolean TABLE_3_PROCESSED_CONSTRAINTS = true; public static final String TABLE_3_DESCRIPTION = "Some sensor data"; public static final String TABLE_3_QUEUE_NAME = TABLE_3_INTERNALNAME; - public static final String TABLE_3_ROUTING_KEY = "dbrepo\\." + DATABASE_1_ID + "\\." + TABLE_3_ID; + public static final String TABLE_3_ROUTING_KEY = "dbrepo." + DATABASE_1_ID + "." + TABLE_3_ID; public static final Instant TABLE_3_CREATED = Instant.ofEpochSecond(1677400031L) /* 2023-02-26 08:27:11 (UTC) */; public static final Instant TABLE_3_LAST_MODIFIED = Instant.ofEpochSecond(1677400031L) /* 2023-02-26 08:27:11 (UTC) */; public static final Long TABLE_3_AVG_ROW_LENGTH = 6L; @@ -1886,7 +1886,7 @@ public abstract class BaseTest { public static final Boolean TABLE_5_PROCESSED_CONSTRAINTS = true; public static final String TABLE_5_DESCRIPTION = "Some Kaggle dataset"; public static final String TABLE_5_QUEUE_NAME = TABLE_5_INTERNALNAME; - public static final String TABLE_5_ROUTING_KEY = "dbrepo\\." + DATABASE_2_ID + "\\." + TABLE_5_ID; + public static final String TABLE_5_ROUTING_KEY = "dbrepo." + DATABASE_2_ID + "." + TABLE_5_ID; public static final Instant TABLE_5_CREATED = Instant.ofEpochSecond(1677400067L) /* 2023-02-26 08:27:47 (UTC) */; public static final Instant TABLE_5_LAST_MODIFIED = Instant.ofEpochSecond(1677400067L) /* 2023-02-26 08:27:47 (UTC) */; public static final Long TABLE_5_AVG_ROW_LENGTH = 1080L; @@ -1972,7 +1972,7 @@ public abstract class BaseTest { public static final Boolean TABLE_6_PROCESSED_CONSTRAINTS = true; public static final String TABLE_6_DESCRIPTION = "Some names dataset"; public static final String TABLE_6_QUEUE_NAME = TABLE_6_INTERNALNAME; - public static final String TABLE_6_ROUTING_KEY = "dbrepo\\." + DATABASE_2_ID + "\\." + TABLE_6_ID; + public static final String TABLE_6_ROUTING_KEY = "dbrepo." + DATABASE_2_ID + "." + TABLE_6_ID; public static final Instant TABLE_6_CREATED = Instant.ofEpochSecond(1677400117L) /* 2023-02-26 08:28:37 (UTC) */; public static final Instant TABLE_6_LAST_MODIFIED = Instant.ofEpochSecond(1677400117L) /* 2023-02-26 08:28:37 (UTC) */; @@ -2032,7 +2032,7 @@ public abstract class BaseTest { public static final Boolean TABLE_7_PROCESSED_CONSTRAINTS = true; public static final String TABLE_7_DESCRIPTION = "Some likes dataset"; public static final String TABLE_7_QUEUE_NAME = TABLE_7_INTERNAL_NAME; - public static final String TABLE_7_ROUTING_KEY = "dbrepo\\." + DATABASE_2_ID + "\\." + TABLE_7_ID; + public static final String TABLE_7_ROUTING_KEY = "dbrepo." + DATABASE_2_ID + "." + TABLE_7_ID; public static final Instant TABLE_7_CREATED = Instant.ofEpochSecond(1677400147L) /* 2023-02-26 08:29:07 (UTC) */; public static final Instant TABLE_7_LAST_MODIFIED = Instant.ofEpochSecond(1677400147L) /* 2023-02-26 08:29:07 (UTC) */; @@ -2091,7 +2091,7 @@ public abstract class BaseTest { public static final Boolean TABLE_4_SCHEMA_PUBLIC = false; public static final String TABLE_4_DESCRIPTION = "Hello sensor"; public static final String TABLE_4_QUEUE_NAME = TABLE_4_INTERNALNAME; - public static final String TABLE_4_ROUTING_KEY = "dbrepo\\." + DATABASE_1_ID + "\\." + TABLE_4_ID; + public static final String TABLE_4_ROUTING_KEY = "dbrepo." + DATABASE_1_ID + "." + TABLE_4_ID; public static final Instant TABLE_4_CREATED = Instant.ofEpochSecond(1677400175L) /* 2023-02-26 08:29:35 (UTC) */; public static final Instant TABLE_4_LAST_MODIFIED = Instant.ofEpochSecond(1677400175L) /* 2023-02-26 08:29:35 (UTC) */; public static final Long TABLE_4_AVG_ROW_LENGTH = 0L; @@ -2267,7 +2267,7 @@ public abstract class BaseTest { public static final Boolean TABLE_8_SCHEMA_PUBLIC = false; public static final String TABLE_8_DESCRIPTION = "Hello mfcc"; public static final String TABLE_8_QUEUE_NAME = TABLE_8_INTERNAL_NAME; - public static final String TABLE_8_ROUTING_KEY = "dbrepo\\." + DATABASE_3_ID + "\\." + TABLE_8_ID; + public static final String TABLE_8_ROUTING_KEY = "dbrepo." + DATABASE_3_ID + "." + TABLE_8_ID; public static final Instant TABLE_8_CREATED = Instant.ofEpochSecond(1688400185L) /* 2023-02-26 08:29:35 (UTC) */; public static final Instant TABLE_8_LAST_MODIFIED = Instant.ofEpochSecond(1688400185L) /* 2023-02-26 08:29:35 (UTC) */; @@ -2348,7 +2348,7 @@ public abstract class BaseTest { public static final Boolean TABLE_9_PROCESSED_CONSTRAINTS = true; public static final String TABLE_9_DESCRIPTION = "Hello mfcc"; public static final String TABLE_9_QUEUE_NAME = TABLE_9_INTERNAL_NAME; - public static final String TABLE_9_ROUTING_KEY = "dbrepo\\." + DATABASE_3_ID + "\\." + TABLE_9_ID; + public static final String TABLE_9_ROUTING_KEY = "dbrepo." + DATABASE_3_ID + "." + TABLE_9_ID; public static final Instant TABLE_9_CREATED = Instant.ofEpochSecond(1688400185L) /* 2023-02-26 08:29:35 (UTC) */; public static final Instant TABLE_9_LAST_MODIFIED = Instant.ofEpochSecond(1688400185L) /* 2023-02-26 08:29:35 (UTC) */; -- GitLab