From 686d5224ad5cc0e0d3e31dae638b5cc6df7c3b57 Mon Sep 17 00:00:00 2001 From: Martin Weise <martin.weise@tuwien.ac.at> Date: Wed, 7 Aug 2024 10:07:03 +0000 Subject: [PATCH] Hotfix/error messages --- .docs/api/analyse-service.md | 7 +- dbrepo-analyse-service/app.py | 1 + dbrepo-analyse-service/determine_dt.py | 10 ++- .../gateway/impl/KeycloakGatewayImpl.java | 6 +- .../impl/MetadataServiceGatewayImpl.java | 13 ++-- .../java/at/tuwien/mapper/MariaDbMapper.java | 78 +++++++++++++++++++ .../java/at/tuwien/service/ViewService.java | 4 +- .../impl/AccessServiceMariaDbImpl.java | 25 ++++-- .../impl/DatabaseServiceMariaDbImpl.java | 4 + .../service/impl/HibernateConnector.java | 3 +- .../impl/QueueServiceRabbitMqImpl.java | 2 + .../impl/SchemaServiceMariaDbImpl.java | 12 +++ .../impl/SubsetServiceMariaDbImpl.java | 48 +++++++++--- .../service/impl/TableServiceMariaDbImpl.java | 27 ++++++- .../service/impl/ViewServiceMariaDbImpl.java | 17 +++- .../impl/BrokerServiceGatewayImpl.java | 8 +- .../gateway/impl/CrossrefGatewayImpl.java | 2 +- helm/dbrepo/values.yaml | 4 +- 18 files changed, 227 insertions(+), 44 deletions(-) diff --git a/.docs/api/analyse-service.md b/.docs/api/analyse-service.md index 3f5921968c..3529f0f3df 100644 --- a/.docs/api/analyse-service.md +++ b/.docs/api/analyse-service.md @@ -31,8 +31,11 @@ the [Storage Service](../storage-service), analysis for data types and primary k with [`csv.Sniff().sniff(...)`](https://docs.python.org/3/library/csv.html#csv.Sniffer). This step is optional when the separator was provided via HTTP-payload: `{"separator": ";", ...}` 3. With the separator known (either from step 2 or via HTTP-payload), the [`Pandas`](https://pypi.org/project/pandas/) - guesses the headers and column types and enums, if the HTTP-payload contains `{"enum": true, ...}`. The data type - is guessed by a combination of Pandas and heuristics. + guesses the headers and column types and enums by analysing the first 10.000 rows, if the HTTP-payload contains + `{"enum": true, ...}`. The data type is guessed by a combination of Pandas and heuristics. + +If your datasets are larger than 10.000 rows, increase the number of lines analysed by setting the `ANALYSE_NROWS` +variable to the desired integer. ### Examples diff --git a/dbrepo-analyse-service/app.py b/dbrepo-analyse-service/app.py index 2dc9161746..def401c0e2 100644 --- a/dbrepo-analyse-service/app.py +++ b/dbrepo-analyse-service/app.py @@ -225,6 +225,7 @@ app.config["S3_ACCESS_KEY_ID"] = os.getenv('S3_ACCESS_KEY_ID', 'seaweedfsadmin') app.config["S3_BUCKET"] = os.getenv('S3_BUCKET', 'dbrepo') app.config["S3_ENDPOINT"] = os.getenv('S3_ENDPOINT', 'http://localhost:9000') app.config["S3_SECRET_ACCESS_KEY"] = os.getenv('S3_SECRET_ACCESS_KEY', 'seaweedfsadmin') +app.config["ANALYSE_NROWS"] = int(os.getenv('ANALYSE_NROWS', '10000')) app.json_encoder = LazyJSONEncoder diff --git a/dbrepo-analyse-service/determine_dt.py b/dbrepo-analyse-service/determine_dt.py index a0890c2b7a..462f8b652b 100644 --- a/dbrepo-analyse-service/determine_dt.py +++ b/dbrepo-analyse-service/determine_dt.py @@ -10,7 +10,7 @@ import pandas from numpy import dtype, max, min from flask import current_app from pandas import DataFrame -from pandas.errors import EmptyDataError +from pandas.errors import EmptyDataError, ParserError from api.dto import ColumnAnalysisDto, DataTypeDto, AnalysisDto from clients.s3_client import S3Client @@ -45,12 +45,14 @@ def determine_datatypes(filename, enum=False, enum_tol=0.0001, separator=',') -> for encoding in ['utf-8', 'cp1252', 'latin1', 'iso-8859-1']: try: logging.debug(f"attempt parsing .csv using encoding {encoding}") - df = pandas.read_csv(fh, delimiter=separator, nrows=100, lineterminator=line_terminator, - index_col=False, encoding=encoding) + df = pandas.read_csv(fh, delimiter=separator, nrows=current_app.config['ANALYSE_NROWS'], + lineterminator=line_terminator, index_col=False, encoding=encoding) logging.debug(f"parsing .csv using encoding {encoding} was successful") break + except ParserError as error: + raise IOError(f"Failed to parse .csv using separator {separator}: {error}") except (UnicodeDecodeError, EmptyDataError) as error: - logging.warning(f"Failed to parse .csv using encoding {encoding}: {error}") + raise IOError(f"Failed to parse .csv using encoding {encoding}: {error}") if df is None: raise IOError( f"Failed to parse .csv: no supported encoding found (one of: utf-8, cp1252, latin1, iso-8859-1)") diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java index 6e233395d9..1d73f6219a 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java @@ -58,20 +58,20 @@ public class KeycloakGatewayImpl implements KeycloakGateway { .exchange(url, HttpMethod.POST, new HttpEntity<>(payload, headers), TokenDto.class); } catch (HttpServerErrorException e) { log.error("Failed to obtain user token: {}", e.getMessage()); - throw new AuthServiceConnectionException("Service unavailable", e); + throw new AuthServiceConnectionException("Failed to obtain user token: " + e.getMessage(), e); } catch (HttpClientErrorException.BadRequest e) { if (e.getResponseBodyAsByteArray() != null && e.getResponseBodyAsByteArray().length > 0) { final KeycloakErrorDto error = e.getResponseBodyAs(KeycloakErrorDto.class); if (error != null && error.getError().equals("invalid_grant")) { log.error("Failed to obtain user token: {}", error.getErrorDescription()); - throw new AccountNotSetupException(error.getErrorDescription()); + throw new AccountNotSetupException("Failed to obtain user token: " + error.getErrorDescription(), e); } } log.error("Failed to obtain user token: bad request"); throw new CredentialsInvalidException("Bad request", e); } catch (HttpClientErrorException.Unauthorized e) { log.error("Failed to obtain user token: invalid credentials"); - throw new CredentialsInvalidException("Invalid credentials", e); + throw new CredentialsInvalidException("Invalid credentials: " + e.getMessage(), e); } return response.getBody(); } diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java index 06641b738b..b4cb2ff504 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java @@ -7,7 +7,6 @@ import at.tuwien.api.database.ViewDto; import at.tuwien.api.database.internal.PrivilegedDatabaseDto; import at.tuwien.api.database.internal.PrivilegedViewDto; import at.tuwien.api.database.table.TableDto; -import at.tuwien.api.database.table.TableStatisticDto; import at.tuwien.api.database.table.internal.PrivilegedTableDto; import at.tuwien.api.identifier.IdentifierDto; import at.tuwien.api.user.PrivilegedUserDto; @@ -56,7 +55,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { throw new RemoteUnavailableException("Failed to find container: " + e.getMessage(), e); } catch (HttpClientErrorException.NotFound e) { log.error("Failed to find container with id {}: {}", containerId, e.getMessage()); - throw new ContainerNotFoundException("Failed to find container: " + e.getMessage()); + throw new ContainerNotFoundException("Failed to find container: " + e.getMessage(), e); } if (response.getStatusCode() != HttpStatus.OK) { log.error("Failed to find container with id {}: service responded unsuccessful: {}", containerId, response.getStatusCode()); @@ -88,7 +87,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { throw new RemoteUnavailableException("Failed to find database: " + e.getMessage(), e); } catch (HttpClientErrorException.NotFound e) { log.error("Failed to find database with id {}: body is null", id); - throw new DatabaseNotFoundException("Failed to find database: body is null", e); + throw new DatabaseNotFoundException("Failed to find database: body is null: " + e.getMessage(), e); } if (response.getStatusCode() != HttpStatus.OK) { log.error("Failed to find database with id {}: service responded unsuccessful: {}", id, response.getStatusCode()); @@ -141,7 +140,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { throw new RemoteUnavailableException("Failed to find table: " + e.getMessage(), e); } catch (HttpClientErrorException.NotFound e) { log.error("Failed to find table with id {}: not found: {}", id, e.getMessage()); - throw new TableNotFoundException("Failed to find table: " + e.getMessage()); + throw new TableNotFoundException("Failed to find table: " + e.getMessage(), e); } if (!response.getStatusCode().equals(HttpStatus.OK)) { log.error("Failed to find table with id {}: service responded unsuccessful: {}", id, response.getStatusCode()); @@ -179,7 +178,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { throw new RemoteUnavailableException("Failed to find view: " + e.getMessage(), e); } catch (HttpClientErrorException.NotFound e) { log.error("Failed to find view with id {}: not found: {}", id, e.getMessage()); - throw new ViewNotFoundException("Failed to find view: " + e.getMessage()); + throw new ViewNotFoundException("Failed to find view: " + e.getMessage(), e); } if (!response.getStatusCode().equals(HttpStatus.OK)) { log.error("Failed to find view with id {}: service responded unsuccessful: {}", id, response.getStatusCode()); @@ -214,7 +213,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { throw new RemoteUnavailableException("Failed to find user: " + e.getMessage(), e); } catch (HttpClientErrorException.NotFound e) { log.error("Failed to find user with id {}: not found: {}", userId, e.getMessage()); - throw new UserNotFoundException("Failed to find user: " + e.getMessage()); + throw new UserNotFoundException("Failed to find user: " + e.getMessage(), e); } if (!response.getStatusCode().equals(HttpStatus.OK)) { log.error("Failed to find user with id {}: service responded unsuccessful: {}", userId, response.getStatusCode()); @@ -238,7 +237,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { throw new RemoteUnavailableException("Failed to find user: " + e.getMessage(), e); } catch (HttpClientErrorException.NotFound e) { log.error("Failed to find user with id {}: not found: {}", userId, e.getMessage()); - throw new UserNotFoundException("Failed to find user: " + e.getMessage()); + throw new UserNotFoundException("Failed to find user: " + e.getMessage(), e); } if (!response.getStatusCode().equals(HttpStatus.OK)) { log.error("Failed to find user with id {}: service responded unsuccessful: {}", userId, response.getStatusCode()); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java index 74e9ffe66d..a88cdb5078 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java @@ -73,6 +73,30 @@ public interface MariaDbMapper { return statement.toString(); } + default String databaseRevokePrivilegesQuery(String username) { + final StringBuilder statement = new StringBuilder("REVOKE ALL PRIVILEGES ON *.* FROM `") + .append(username) + .append("`@`%`;"); + log.trace("mapped revoke privileges statement: {}", statement); + return statement.toString(); + } + + default String databaseGrantProcedureQuery(String username, String procedure) { + final StringBuilder statement = new StringBuilder("GRANT EXECUTE ON PROCEDURE `") + .append(procedure) + .append("` TO `") + .append(username) + .append("`@`%`;"); + log.trace("mapped revoke privileges statement: {}", statement); + return statement.toString(); + } + + default String databaseFlushPrivilegesQuery() { + final String statement = "FLUSH PRIVILEGES;"; + log.trace("mapped flush privileges statement: {}", statement); + return statement; + } + @Named("createDatabase") default String databaseCreateDatabaseQuery(String database) { final StringBuilder statement = new StringBuilder("CREATE DATABASE `") @@ -82,6 +106,60 @@ public interface MariaDbMapper { return statement.toString(); } + default String queryStoreCreateSequenceRawQuery() { + final String statement = "CREATE SEQUENCE `qs_queries_seq` NOCACHE;"; + log.trace("mapped create query store sequence statement: {}", statement); + return statement; + } + + default String queryStoreCreateTableRawQuery() { + final String statement = "CREATE TABLE `qs_queries` ( `id` bigint not null primary key default nextval(`qs_queries_seq`), `created` datetime not null default now(), `executed` datetime not null default now(), `created_by` varchar(36) not null, `query` text not null, `query_normalized` text not null, `is_persisted` boolean not null, `query_hash` varchar(255) not null, `result_hash` varchar(255), `result_number` bigint);"; + log.trace("mapped create query store table statement: {}", statement); + return statement; + } + + default String queryStoreCreateHashTableProcedureRawQuery() { + final String statement = "CREATE PROCEDURE hash_table(IN name VARCHAR(255), OUT hash VARCHAR(255), OUT count BIGINT) BEGIN DECLARE _sql TEXT; SELECT CONCAT('SELECT SHA2(GROUP_CONCAT(CONCAT_WS(\\'\\',', GROUP_CONCAT(CONCAT('`', column_name, '`') ORDER BY column_name), ') SEPARATOR \\',\\'), 256) AS hash, COUNT(*) AS count FROM `', name, '` INTO @hash, @count;') FROM `information_schema`.`columns` WHERE `table_schema` = DATABASE() AND `table_name` = name INTO _sql; PREPARE stmt FROM _sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; SET hash = @hash; SET count = @count; END;"; + log.trace("mapped create query store hash_table procedure statement: {}", statement); + return statement; + } + + default String queryStoreCreateStoreQueryProcedureRawQuery() { + final String statement = "CREATE PROCEDURE store_query(IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _username varchar(255) DEFAULT REGEXP_REPLACE(current_user(), '@.*', ''); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END;"; + log.trace("mapped create query store store_query procedure statement: {}", statement); + return statement; + } + + default String queryStoreCreateInternalStoreQueryProcedureRawQuery() { + final String statement = "CREATE DEFINER = 'root' PROCEDURE _store_query(IN _username VARCHAR(255), IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END;"; + log.trace("mapped create query store _store_query procedure statement: {}", statement); + return statement; + } + + default String queryStoreStoreQueryRawQuery() { + final String statement = "{call _store_query(?, ?, ?, ?)}"; + log.trace("mapped store query statement: {}", statement); + return statement; + } + + default String queryStoreUpdateQueryRawQuery() { + final String statement = "UPDATE `qs_queries` SET `is_persisted` = ? WHERE `id` = ?"; + log.trace("mapped update query statement: {}", statement); + return statement; + } + + default String queryStoreDeleteStaleQueriesRawQuery() { + final String statement = "DELETE FROM `qs_queries` WHERE `is_persisted` = false AND ABS(DATEDIFF(`created`, NOW())) >= 1"; + log.trace("mapped delete stale queries statement: {}", statement); + return statement; + } + + default String queryStoreFindQueryRawQuery() { + final String statement = "SELECT `id`, `created`, `created_by`, `query`, `query_hash`, `result_hash`, `result_number`, `is_persisted`, `executed` FROM `qs_queries` q WHERE q.`id` = ?"; + log.trace("mapped find query statement: {}", statement); + return statement; + } + default String databaseTablesSelectRawQuery() { final String statement = "SELECT DISTINCT t.`TABLE_NAME` FROM information_schema.TABLES t WHERE t.`TABLE_SCHEMA` = ? AND t.`TABLE_TYPE` = 'SYSTEM VERSIONED' AND t.`TABLE_NAME` != 'qs_queries' ORDER BY t.`TABLE_NAME` ASC"; log.trace("mapped select tables statement: {}", statement); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/ViewService.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/ViewService.java index 5acca0018d..f4bef4f067 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/ViewService.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/ViewService.java @@ -20,12 +20,10 @@ public interface ViewService { * @return The list of view metadata. * @throws SQLException * @throws DatabaseMalformedException - * @throws ViewMalformedException * @throws ViewNotFoundException - * @throws ViewSchemaException */ List<ViewDto> getSchemas(PrivilegedDatabaseDto database) throws SQLException, DatabaseMalformedException, - ViewMalformedException, ViewNotFoundException, ViewSchemaException; + ViewNotFoundException; /** * Creates a view in the given data database. diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/AccessServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/AccessServiceMariaDbImpl.java index 8c52e02010..4fdb8c32a6 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/AccessServiceMariaDbImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/AccessServiceMariaDbImpl.java @@ -40,17 +40,25 @@ public class AccessServiceMariaDbImpl extends HibernateConnector implements Acce final Connection connection = dataSource.getConnection(); try { /* create user if not exists */ + long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.databaseCreateUserQuery(user.getUsername(), user.getPassword())) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); /* grant access */ final String grants = access != AccessTypeDto.READ ? grantDefaultWrite : grantDefaultRead; + start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.databaseGrantPrivilegesQuery(user.getUsername(), grants)) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); /* grant query store */ - connection.prepareStatement("GRANT EXECUTE ON PROCEDURE `store_query` TO `" + user.getUsername() + "`@`%`;") + start = System.currentTimeMillis(); + connection.prepareStatement(mariaDbMapper.databaseGrantProcedureQuery(user.getUsername(), "store_query")) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); /* apply access rights */ - connection.prepareStatement("FLUSH PRIVILEGES;"); + start = System.currentTimeMillis(); + connection.prepareStatement(mariaDbMapper.databaseFlushPrivilegesQuery()); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -71,10 +79,12 @@ public class AccessServiceMariaDbImpl extends HibernateConnector implements Acce try { /* grant access */ final String grants = access != AccessTypeDto.READ ? grantDefaultWrite : grantDefaultRead; + final long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.databaseGrantPrivilegesQuery(user.getUsername(), grants)) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); /* apply access rights */ - connection.prepareStatement("FLUSH PRIVILEGES;"); + connection.prepareStatement(mariaDbMapper.databaseFlushPrivilegesQuery()); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -93,10 +103,15 @@ public class AccessServiceMariaDbImpl extends HibernateConnector implements Acce final Connection connection = dataSource.getConnection(); try { /* revoke access */ - connection.prepareStatement("REVOKE ALL PRIVILEGES ON *.* FROM `" + user.getUsername() + "`@`%`;") + long start = System.currentTimeMillis(); + connection.prepareStatement(mariaDbMapper.databaseRevokePrivilegesQuery(user.getUsername())) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); /* apply access rights */ - connection.prepareStatement("FLUSH PRIVILEGES;"); + start = System.currentTimeMillis(); + connection.prepareStatement(mariaDbMapper.databaseFlushPrivilegesQuery()) + .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceMariaDbImpl.java index 4ee483ad5e..2d58744a21 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceMariaDbImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceMariaDbImpl.java @@ -43,8 +43,10 @@ public class DatabaseServiceMariaDbImpl extends HibernateConnector implements Da final Connection connection = dataSource.getConnection(); try { /* create database if not exists */ + final long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.databaseCreateDatabaseQuery(data.getInternalName())) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -77,8 +79,10 @@ public class DatabaseServiceMariaDbImpl extends HibernateConnector implements Da final Connection connection = dataSource.getConnection(); try { /* update user password */ + final long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.databaseSetPasswordQuery(data.getUsername(), data.getPassword())) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java index 6a583b274a..22bb599c60 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java @@ -12,6 +12,7 @@ import org.springframework.stereotype.Service; public abstract class HibernateConnector { public static ComboPooledDataSource getPrivilegedDataSource(PrivilegedContainerDto container, String databaseName) { + final long start = System.currentTimeMillis(); final ComboPooledDataSource dataSource = new ComboPooledDataSource(); dataSource.setJdbcUrl(url(container, databaseName)); dataSource.setUser(container.getUsername()); @@ -21,7 +22,7 @@ public abstract class HibernateConnector { dataSource.setAcquireIncrement(5); dataSource.setMaxPoolSize(20); dataSource.setMaxStatements(100); - log.trace("created pooled data source {} (user={}, password=(hidden))", url(container, databaseName), container.getUsername()); + log.trace("created pooled data source {} in {} ms (user={}, password=(hidden))", url(container, databaseName), System.currentTimeMillis() - start, container.getUsername()); return dataSource; } diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java index f2675d4e5b..0b1dc7caa1 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java @@ -46,7 +46,9 @@ public class QueueServiceRabbitMqImpl extends HibernateConnector implements Queu dataMapper.prepareStatementWithColumnTypeObject(preparedStatement, optional.get().getColumnType(), idx[0]++, entry.getValue()); } + final long start = System.currentTimeMillis(); preparedStatement.executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); log.trace("successfully inserted tuple"); } finally { dataSource.close(); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SchemaServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SchemaServiceMariaDbImpl.java index cc5840080b..a0fbd1434c 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SchemaServiceMariaDbImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SchemaServiceMariaDbImpl.java @@ -49,26 +49,32 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche final Connection connection = dataSource.getConnection(); try { /* obtain only table metadata */ + long start = System.currentTimeMillis(); final PreparedStatement statement1 = connection.prepareStatement(mariaDbMapper.databaseTableSelectRawQuery()); statement1.setString(1, database.getInternalName()); statement1.setString(2, tableName); log.trace("1={}, 2={}", database.getInternalName(), tableName); TableDto table = dataMapper.schemaResultSetToTable(metadataMapper.privilegedDatabaseDtoToDatabaseDto(database), statement1.executeQuery()); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); /* obtain columns metadata */ + start = System.currentTimeMillis(); final PreparedStatement statement2 = connection.prepareStatement(mariaDbMapper.databaseTableColumnsSelectRawQuery()); statement2.setString(1, database.getInternalName()); statement2.setString(2, tableName); log.trace("1={}, 2={}", database.getInternalName(), tableName); final ResultSet resultSet2 = statement2.executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); while (resultSet2.next()) { table = dataMapper.resultSetToTable(resultSet2, table, queryConfig); } /* obtain check constraints metadata */ + start = System.currentTimeMillis(); final PreparedStatement statement3 = connection.prepareStatement(mariaDbMapper.columnsCheckConstraintSelectRawQuery()); statement3.setString(1, database.getInternalName()); statement3.setString(2, tableName); log.trace("1={}, 2={}", database.getInternalName(), tableName); final ResultSet resultSet3 = statement3.executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); while (resultSet3.next()) { final String clause = resultSet3.getString(1); table.getConstraints() @@ -77,11 +83,13 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche log.trace("found check clause: {}", clause); } /* obtain column constraints metadata */ + start = System.currentTimeMillis(); final PreparedStatement statement4 = connection.prepareStatement(mariaDbMapper.databaseTableConstraintsSelectRawQuery()); statement4.setString(1, database.getInternalName()); statement4.setString(2, tableName); log.trace("1={}, 2={}", database.getInternalName(), tableName); final ResultSet resultSet4 = statement4.executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); while (resultSet4.next()) { table = dataMapper.resultSetToConstraint(resultSet4, table); for (UniqueDto uk : table.getConstraints().getUniques()) { @@ -122,11 +130,13 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche final DatabaseDto database = metadataMapper.privilegedDatabaseDtoToDatabaseDto(privilegedDatabase); try { /* obtain only view metadata */ + long start = System.currentTimeMillis(); final PreparedStatement statement1 = connection.prepareStatement(mariaDbMapper.databaseViewSelectRawQuery()); statement1.setString(1, database.getInternalName()); statement1.setString(2, viewName); log.trace("1={}, 2={}", database.getInternalName(), viewName); final ResultSet resultSet1 = statement1.executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); if (!resultSet1.next()) { throw new ViewNotFoundException("Failed to find view in the information schema"); } @@ -136,11 +146,13 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche view.setCreator(database.getCreator()); view.setCreatedBy(database.getCreator().getId()); /* obtain view columns */ + start = System.currentTimeMillis(); final PreparedStatement statement2 = connection.prepareStatement(mariaDbMapper.databaseTableColumnsSelectRawQuery()); statement2.setString(1, database.getInternalName()); statement2.setString(2, viewName); log.trace("1={}, 2={}", database.getInternalName(), viewName); final ResultSet resultSet2 = statement2.executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); TableDto tmp = TableDto.builder() .columns(new LinkedList<>()) .build(); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java index 3d19276196..53d93d35df 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java @@ -68,16 +68,26 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs final Connection connection = dataSource.getConnection(); try { /* create query store */ - connection.prepareStatement("CREATE SEQUENCE `qs_queries_seq` NOCACHE;") + long start = System.currentTimeMillis(); + connection.prepareStatement(mariaDbMapper.queryStoreCreateSequenceRawQuery()) .execute(); - connection.prepareStatement("CREATE TABLE `qs_queries` ( `id` bigint not null primary key default nextval(`qs_queries_seq`), `created` datetime not null default now(), `executed` datetime not null default now(), `created_by` varchar(36) not null, `query` text not null, `query_normalized` text not null, `is_persisted` boolean not null, `query_hash` varchar(255) not null, `result_hash` varchar(255), `result_number` bigint );") + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); + start = System.currentTimeMillis(); + connection.prepareStatement(mariaDbMapper.queryStoreCreateTableRawQuery()) .execute(); - connection.prepareStatement("CREATE PROCEDURE hash_table(IN name VARCHAR(255), OUT hash VARCHAR(255), OUT count BIGINT) BEGIN DECLARE _sql TEXT; SELECT CONCAT('SELECT SHA2(GROUP_CONCAT(CONCAT_WS(\\'\\',', GROUP_CONCAT(CONCAT('`', column_name, '`') ORDER BY column_name), ') SEPARATOR \\',\\'), 256) AS hash, COUNT(*) AS count FROM `', name, '` INTO @hash, @count;') FROM `information_schema`.`columns` WHERE `table_schema` = DATABASE() AND `table_name` = name INTO _sql; PREPARE stmt FROM _sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; SET hash = @hash; SET count = @count; END;") + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); + start = System.currentTimeMillis(); + connection.prepareStatement(mariaDbMapper.queryStoreCreateHashTableProcedureRawQuery()) .execute(); - connection.prepareStatement("CREATE PROCEDURE store_query(IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _username varchar(255) DEFAULT REGEXP_REPLACE(current_user(), '@.*', ''); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END;") + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); + start = System.currentTimeMillis(); + connection.prepareStatement(mariaDbMapper.queryStoreCreateStoreQueryProcedureRawQuery()) .execute(); - connection.prepareStatement("CREATE DEFINER = 'root' PROCEDURE _store_query(IN _username VARCHAR(255), IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END;") + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); + start = System.currentTimeMillis(); + connection.prepareStatement(mariaDbMapper.queryStoreCreateInternalStoreQueryProcedureRawQuery()) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -130,12 +140,14 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs final ComboPooledDataSource dataSource = getPrivilegedDataSource(database); final Connection connection = dataSource.getConnection(); try { + final long start = System.currentTimeMillis(); final PreparedStatement statement = connection.prepareStatement(mariaDbMapper.filterToGetQueriesRawQuery(filterPersisted)); if (filterPersisted != null) { statement.setBoolean(1, filterPersisted); log.trace("filter persisted only {}", filterPersisted); } final ResultSet resultSet = statement.executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); final List<QueryDto> queries = new LinkedList<>(); while (resultSet.next()) { final QueryDto query = dataMapper.resultSetToQueryDto(resultSet); @@ -168,12 +180,18 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs final Connection connection = dataSource.getConnection(); try { /* export to data database sidecar */ + long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.subsetToRawTemporaryViewQuery(viewName, query.getQuery())) .executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); + start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.subsetToRawExportQuery(viewName, timestamp, filePath)) .executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); + start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.dropViewRawQuery(viewName)) .executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -191,8 +209,10 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs final ComboPooledDataSource dataSource = getPrivilegedDataSource(database); final Connection connection = dataSource.getConnection(); try { + final long start = System.currentTimeMillis(); final PreparedStatement preparedStatement = connection.prepareStatement(statement); final ResultSet resultSet = preparedStatement.executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); return dataMapper.resultListToQueryResultDto(columns, resultSet); } catch (SQLException e) { log.error("Failed to execute and map time-versioned query: {}", e.getMessage()); @@ -208,8 +228,10 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs final ComboPooledDataSource dataSource = getPrivilegedDataSource(database); final Connection connection = dataSource.getConnection(); try { + final long start = System.currentTimeMillis(); final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.countRawSelectQuery(statement, timestamp)) .executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); return mariaDbMapper.resultSetToNumber(resultSet); } catch (SQLException e) { log.error("Failed to map object: {}", e.getMessage()); @@ -225,9 +247,11 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs final ComboPooledDataSource dataSource = getPrivilegedDataSource(database); final Connection connection = dataSource.getConnection(); try { - final PreparedStatement preparedStatement = connection.prepareStatement("SELECT `id`, `created`, `created_by`, `query`, `query_hash`, `result_hash`, `result_number`, `is_persisted`, `executed` FROM `qs_queries` q WHERE q.`id` = ?"); + final long start = System.currentTimeMillis(); + final PreparedStatement preparedStatement = connection.prepareStatement(mariaDbMapper.queryStoreFindQueryRawQuery()); preparedStatement.setLong(1, queryId); final ResultSet resultSet = preparedStatement.executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); if (!resultSet.next()) { throw new QueryNotFoundException("Failed to find query"); } @@ -255,12 +279,14 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs final Connection connection = dataSource.getConnection(); try { /* insert query into query store */ - final CallableStatement callableStatement = connection.prepareCall("{call _store_query(?, ?, ?, ?)}"); + final long start = System.currentTimeMillis(); + final CallableStatement callableStatement = connection.prepareCall(mariaDbMapper.queryStoreStoreQueryRawQuery()); callableStatement.setString(1, String.valueOf(userId)); callableStatement.setString(2, query); callableStatement.setTimestamp(3, Timestamp.from(timestamp)); callableStatement.registerOutParameter(4, Types.BIGINT); callableStatement.executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); queryId = callableStatement.getLong(4); callableStatement.close(); log.info("Stored query with id {} in database with name {}", queryId, database.getInternalName()); @@ -282,10 +308,12 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs final Connection connection = dataSource.getConnection(); try { /* update query */ - final PreparedStatement preparedStatement = connection.prepareStatement("UPDATE `qs_queries` SET `is_persisted` = ? WHERE `id` = ?"); + final long start = System.currentTimeMillis(); + final PreparedStatement preparedStatement = connection.prepareStatement(mariaDbMapper.queryStoreUpdateQueryRawQuery()); preparedStatement.setBoolean(1, persist); preparedStatement.setLong(2, queryId); preparedStatement.executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); } catch (SQLException e) { log.error("Failed to (un-)persist query: {}", e.getMessage()); throw new QueryStorePersistException("Failed to (un-)persist query", e); @@ -300,8 +328,10 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs final ComboPooledDataSource dataSource = getPrivilegedDataSource(database); final Connection connection = dataSource.getConnection(); try { - connection.prepareStatement("DELETE FROM `qs_queries` WHERE `is_persisted` = false AND ABS(DATEDIFF(`created`, NOW())) >= 1") + final long start = System.currentTimeMillis(); + connection.prepareStatement(mariaDbMapper.queryStoreDeleteStaleQueriesRawQuery()) .executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); } catch (SQLException e) { log.error("Failed to delete stale queries: {}", e.getMessage()); throw new QueryStoreGCException("Failed to delete stale queries: " + e.getMessage(), e); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java index 4dacc1e094..265c361cd7 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java @@ -25,6 +25,7 @@ import org.apache.commons.lang3.RandomStringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.io.File; import java.sql.*; import java.time.Instant; import java.util.*; @@ -60,9 +61,11 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table final List<TableDto> tables = new LinkedList<>(); try { /* inspect tables before views */ + final long start = System.currentTimeMillis(); final PreparedStatement statement = connection.prepareStatement(mariaDbMapper.databaseTablesSelectRawQuery()); statement.setString(1, database.getInternalName()); final ResultSet resultSet1 = statement.executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); while (resultSet1.next()) { final String tableName = resultSet1.getString(1); if (database.getTables().stream().anyMatch(t -> t.getInternalName().equals(tableName))) { @@ -92,8 +95,10 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table final TableStatisticDto statistic; try { /* obtain statistic */ + final long start = System.currentTimeMillis(); final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.tableColumnStatisticsSelectRawQuery(table.getColumns(), table.getInternalName())) .executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); statistic = dataMapper.resultSetToTableStatistic(resultSet); final TableDto tmpTable = schemaService.inspectTable(table.getDatabase(), table.getInternalName()); statistic.setAvgRowLength(tmpTable.getAvgRowLength()); @@ -129,8 +134,10 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table final Connection connection = dataSource.getConnection(); try { /* create table if not exists */ + final long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.tableCreateDtoToCreateTableRawQuery(data)) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -156,8 +163,10 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table final Connection connection = dataSource.getConnection(); try { /* create table if not exists */ + final long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.dropTableRawQuery(tableName)) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -177,10 +186,12 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table final QueryResultDto queryResult; try { /* find table data */ + final long start = System.currentTimeMillis(); final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.selectDatasetRawQuery( table.getDatabase().getInternalName(), table.getInternalName(), table.getColumns(), timestamp, size, page)) .executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); queryResult = dataMapper.resultListToQueryResultDto(table.getColumns(), resultSet); } catch (SQLException e) { @@ -203,9 +214,11 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table final List<TableHistoryDto> history; try { /* find table data */ + final long start = System.currentTimeMillis(); final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.selectHistoryRawQuery( table.getDatabase().getInternalName(), table.getInternalName(), size)) .executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); history = dataMapper.resultSetToTableHistory(resultSet); connection.commit(); } catch (SQLException e) { @@ -227,9 +240,11 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table final Long queryResult; try { /* find table data */ + final long start = System.currentTimeMillis(); final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.selectCountRawQuery( table.getDatabase().getInternalName(), table.getInternalName(), timestamp)) .executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); queryResult = mariaDbMapper.resultSetToNumber(resultSet); connection.commit(); } catch (SQLException e) { @@ -253,9 +268,11 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table final Connection connection = dataSource.getConnection(); try { /* import tuple */ - data.setLocation(s3Config.getS3FilePath() + "/" + data.getLocation()); + data.setLocation(s3Config.getS3FilePath() + File.separator + data.getLocation()); + final long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.datasetToRawInsertQuery(table.getDatabase().getInternalName(), table, data)) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -283,7 +300,9 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table getColumnType(table.getColumns(), column), idx[0], column, data.getKeys().get(column)); idx[0]++; } + final long start = System.currentTimeMillis(); statement.executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -325,7 +344,9 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table getColumnType(table.getColumns(), entry.getKey()), idx[0], entry.getKey(), entry.getValue()); idx[0]++; } + final long start = System.currentTimeMillis(); statement.executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -359,7 +380,9 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table getColumnType(table.getColumns(), entry.getKey()), idx[0], entry.getKey(), entry.getValue()); idx[0]++; } + final long start = System.currentTimeMillis(); statement.executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -392,9 +415,11 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table final Connection connection = dataSource.getConnection(); try { /* export to data database sidecar */ + final long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.tableOrViewToRawExportQuery(table.getDatabase().getInternalName(), table.getInternalName(), table.getColumns(), timestamp, filePath)) .executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/ViewServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/ViewServiceMariaDbImpl.java index 3cdba35f08..06cf42ae6e 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/ViewServiceMariaDbImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/ViewServiceMariaDbImpl.java @@ -24,6 +24,7 @@ import org.apache.commons.lang3.RandomStringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.io.File; import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.PreparedStatement; @@ -63,7 +64,7 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe @Override public List<ViewDto> getSchemas(PrivilegedDatabaseDto database) throws SQLException, DatabaseMalformedException, - ViewMalformedException, ViewNotFoundException, ViewSchemaException { + ViewNotFoundException { final ComboPooledDataSource dataSource = getPrivilegedDataSource(database); final Connection connection = dataSource.getConnection(); final List<ViewDto> views = new LinkedList<>(); @@ -71,7 +72,9 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe /* inspect tables before views */ final PreparedStatement statement = connection.prepareStatement(mariaDbMapper.databaseViewsSelectRawQuery()); statement.setString(1, database.getInternalName()); + final long start = System.currentTimeMillis(); final ResultSet resultSet1 = statement.executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); while (resultSet1.next()) { final String viewName = resultSet1.getString(1); if (database.getViews().stream().anyMatch(v -> v.getInternalName().equals(viewName))) { @@ -117,8 +120,10 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe .build(); try { /* create view if not exists */ + final long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.viewCreateRawQuery(view.getInternalName(), data.getQuery())) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); /* select view columns */ final PreparedStatement statement2 = connection.prepareStatement(mariaDbMapper.databaseTableColumnsSelectRawQuery()); statement2.setString(1, database.getInternalName()); @@ -151,10 +156,12 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe .stream() .map(metadataMapper::viewColumnDtoToColumnDto) .toList(); + final long start = System.currentTimeMillis(); final ResultSet resultSet = connection.prepareStatement( mariaDbMapper.selectDatasetRawQuery(view.getDatabase().getInternalName(), view.getInternalName(), mappedColumns, timestamp, size, page)) .executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); queryResult = dataMapper.resultListToQueryResultDto(mappedColumns, resultSet); queryResult.setId(view.getId()); connection.commit(); @@ -174,8 +181,10 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe final Connection connection = dataSource.getConnection(); try { /* drop view if exists */ + final long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.dropViewRawQuery(view.getInternalName())) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -196,9 +205,11 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe final Long queryResult; try { /* find view data */ + final long start = System.currentTimeMillis(); final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.selectCountRawQuery( view.getDatabase().getInternalName(), view.getInternalName(), timestamp)) .executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); queryResult = mariaDbMapper.resultSetToNumber(resultSet); connection.commit(); } catch (SQLException e) { @@ -217,7 +228,7 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe throws SQLException, QueryMalformedException, StorageNotFoundException, StorageUnavailableException, RemoteUnavailableException, SidecarExportException { final String fileName = RandomStringUtils.randomAlphabetic(40) + ".csv"; - final String filePath = s3Config.getS3FilePath() + "/" + fileName; + final String filePath = s3Config.getS3FilePath() + File.separator + fileName; final ComboPooledDataSource dataSource = getPrivilegedDataSource(database); final Connection connection = dataSource.getConnection(); try { @@ -226,9 +237,11 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe .stream() .map(metadataMapper::viewColumnDtoToColumnDto) .toList(); + final long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.tableOrViewToRawExportQuery(database.getInternalName(), view.getInternalName(), columns, timestamp, filePath)) .executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/BrokerServiceGatewayImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/BrokerServiceGatewayImpl.java index 307c166feb..da1c289185 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/BrokerServiceGatewayImpl.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/BrokerServiceGatewayImpl.java @@ -38,9 +38,9 @@ public class BrokerServiceGatewayImpl implements BrokerServiceGateway { response = restTemplate.exchange(path, HttpMethod.PUT, new HttpEntity<>(data), Void.class); } catch (HttpServerErrorException e) { log.error("Failed to grant topic permissions: {}", e.getMessage()); - throw new BrokerServiceConnectionException("Failed to grant topic permissions: " + e.getMessage()); + throw new BrokerServiceConnectionException("Failed to grant topic permissions: " + e.getMessage(), e); } catch (Exception e) { - log.error("Failed to grant topic permissions: unexpected response: {}", e.getMessage()); + log.error("Failed to grant topic permissions: unexpected response: {}", e.getMessage(), e); throw new BrokerServiceException("Failed to grant topic permissions: unexpected response: " + e.getMessage(), e); } if (!response.getStatusCode().equals(HttpStatus.CREATED) && !response.getStatusCode().equals(HttpStatus.NO_CONTENT)) { @@ -59,7 +59,7 @@ public class BrokerServiceGatewayImpl implements BrokerServiceGateway { response = restTemplate.exchange(path, HttpMethod.PUT, new HttpEntity<>(data), Void.class); } catch (HttpServerErrorException e) { log.error("Failed to grant virtual host permissions: {}", e.getMessage()); - throw new BrokerServiceConnectionException("Failed to grant virtual host permissions: " + e.getMessage()); + throw new BrokerServiceConnectionException("Failed to grant virtual host permissions: " + e.getMessage(), e); } catch (Exception e) { log.error("Failed to grant virtual host permissions: unexpected response: {}", e.getMessage()); throw new BrokerServiceException("Failed to grant virtual host permissions: unexpected response: " + e.getMessage(), e); @@ -80,7 +80,7 @@ public class BrokerServiceGatewayImpl implements BrokerServiceGateway { response = restTemplate.exchange(path, HttpMethod.PUT, new HttpEntity<>(data), Void.class); } catch (HttpServerErrorException e) { log.error("Failed to grant exchange permissions: {}", e.getMessage()); - throw new BrokerServiceConnectionException("Failed to grant exchange permissions: " + e.getMessage()); + throw new BrokerServiceConnectionException("Failed to grant exchange permissions: " + e.getMessage(), e); } catch (Exception e) { log.error("Failed to grant exchange permissions: unexpected response: {}", e.getMessage()); throw new BrokerServiceException("Failed to grant exchange permissions: unexpected response: " + e.getMessage(), e); diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/CrossrefGatewayImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/CrossrefGatewayImpl.java index 542d9c981d..8fd957f330 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/CrossrefGatewayImpl.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/CrossrefGatewayImpl.java @@ -37,7 +37,7 @@ public class CrossrefGatewayImpl implements CrossrefGateway { response = restTemplate.exchange(gatewayConfig.getCrossRefEndpoint() + path, HttpMethod.GET, HttpEntity.EMPTY, CrossrefDto.class); } catch (HttpServerErrorException e) { log.error("Failed to retrieve crossref metadata: {}", e.getMessage()); - throw new DoiNotFoundException("Failed to retrieve crossref metadata: " + e.getMessage()); + throw new DoiNotFoundException("Failed to retrieve crossref metadata: " + e.getMessage(), e); } return response.getBody(); } diff --git a/helm/dbrepo/values.yaml b/helm/dbrepo/values.yaml index 46a6e8130d..af810436af 100644 --- a/helm/dbrepo/values.yaml +++ b/helm/dbrepo/values.yaml @@ -56,8 +56,8 @@ metadatadb: enabled: false ## @skip metadatadb.initdbScriptsConfigMap The initial database scripts. initdbScriptsConfigMap: metadata-db-setup - ## @param metadatadb.initdbScripts Additional init.db scripts that are executed on the first start. - initdbScripts: { } + ## @param metadatadb.extraInitDbScripts Additional init.db scripts that are executed on the first start. + extraInitDbScripts: { } # 03-additional-data.sql: | # BEGIN; # INSERT INTO `mdb_containers` (name, internal_name, image_id, host, port, sidecar_host, sidecar_port, privileged_username, privileged_password) -- GitLab