From 2f7a67ad1a7c20b61627e3eb7b5cef8eac9c33d4 Mon Sep 17 00:00:00 2001
From: Martin Weise <martin.weise@tuwien.ac.at>
Date: Sun, 24 Jul 2022 12:51:02 +0800
Subject: [PATCH] Added the basic mapping
---
.../at/tuwien/endpoint/QueryEndpoint.java | 7 +-
.../at/tuwien/endpoint/TableDataEndpoint.java | 39 +--
.../java/at/tuwien/mapper/QueryMapper.java | 43 +--
.../java/at/tuwien/service/QueryService.java | 38 +--
.../service/impl/HibernateConnector.java | 84 +++--
.../tuwien/service/impl/QueryServiceImpl.java | 290 ++++--------------
6 files changed, 156 insertions(+), 345 deletions(-)
diff --git a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java
index 02b340f324..906f268a8a 100644
--- a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java
+++ b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java
@@ -48,7 +48,7 @@ public class QueryEndpoint extends AbstractEndpoint {
@NotNull Principal principal)
throws DatabaseNotFoundException, ImageNotSupportedException, QueryStoreException, QueryMalformedException,
ContainerNotFoundException, ColumnParseException, UserNotFoundException, TableMalformedException,
- NotAllowedException {
+ NotAllowedException, DatabaseConnectionException {
if (!hasDatabasePermission(containerId, databaseId, "QUERY_EXECUTE", principal)) {
log.error("Missing execute query permission");
throw new NotAllowedException("Missing execute query permission");
@@ -74,7 +74,7 @@ public class QueryEndpoint extends AbstractEndpoint {
@NotNull Principal principal)
throws QueryStoreException, QueryNotFoundException, DatabaseNotFoundException, ImageNotSupportedException,
TableNotFoundException, QueryMalformedException, ContainerNotFoundException, TableMalformedException,
- ColumnParseException, NotAllowedException {
+ ColumnParseException, NotAllowedException, DatabaseConnectionException {
if (!hasQueryPermission(containerId, databaseId, queryId, "QUERY_RE_EXECUTE", principal)) {
log.error("Missing re-execute query permission");
throw new NotAllowedException("Missing re-execute query permission");
@@ -94,7 +94,8 @@ public class QueryEndpoint extends AbstractEndpoint {
@NotNull @PathVariable("queryId") Long queryId,
@NotNull Principal principal)
throws QueryStoreException, QueryNotFoundException, DatabaseNotFoundException, ImageNotSupportedException,
- ContainerNotFoundException, TableMalformedException, FileStorageException, NotAllowedException, QueryMalformedException {
+ ContainerNotFoundException, TableMalformedException, FileStorageException, NotAllowedException,
+ QueryMalformedException, DatabaseConnectionException {
if (!hasQueryPermission(containerId, databaseId, queryId, "QUERY_EXPORT", principal)) {
log.error("Missing export query permission");
throw new NotAllowedException("Missing export query permission");
diff --git a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableDataEndpoint.java b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableDataEndpoint.java
index 04833a1abf..bb604fb5c4 100644
--- a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableDataEndpoint.java
+++ b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableDataEndpoint.java
@@ -4,7 +4,6 @@ import at.tuwien.api.database.query.ImportDto;
import at.tuwien.api.database.query.QueryResultDto;
import at.tuwien.api.database.table.TableCsvDeleteDto;
import at.tuwien.api.database.table.TableCsvDto;
-import at.tuwien.api.database.table.TableCsvUpdateDto;
import at.tuwien.exception.*;
import at.tuwien.service.*;
import io.swagger.v3.oas.annotations.Operation;
@@ -18,7 +17,6 @@ import org.springframework.web.bind.annotation.*;
import javax.validation.Valid;
import javax.validation.constraints.NotNull;
-import java.math.BigInteger;
import java.security.Principal;
import java.time.Instant;
@@ -40,37 +38,20 @@ public class TableDataEndpoint extends AbstractEndpoint {
@PostMapping
@Transactional
@Operation(summary = "Insert data", security = @SecurityRequirement(name = "bearerAuth"))
- public ResponseEntity<Integer> insert(@NotNull @PathVariable("id") Long containerId,
+ public ResponseEntity<Void> insert(@NotNull @PathVariable("id") Long containerId,
@NotNull @PathVariable("databaseId") Long databaseId,
@NotNull @PathVariable("tableId") Long tableId,
@NotNull @Valid @RequestBody TableCsvDto data,
@NotNull Principal principal)
throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException,
- ImageNotSupportedException, ContainerNotFoundException, NotAllowedException {
+ ImageNotSupportedException, ContainerNotFoundException, NotAllowedException, DatabaseConnectionException {
if (!hasDatabasePermission(containerId, databaseId, "DATA_INSERT", principal)) {
log.error("Missing data insert permission");
throw new NotAllowedException("Missing data insert permission");
}
+ queryService.insert(containerId, databaseId, tableId, data);
return ResponseEntity.accepted()
- .body(queryService.insert(containerId, databaseId, tableId, data));
- }
-
- @PutMapping
- @Transactional
- @Operation(summary = "Update data", security = @SecurityRequirement(name = "bearerAuth"))
- public ResponseEntity<Integer> update(@NotNull @PathVariable("id") Long containerId,
- @NotNull @PathVariable("databaseId") Long databaseId,
- @NotNull @PathVariable("tableId") Long tableId,
- @NotNull @Valid @RequestBody TableCsvUpdateDto data,
- @NotNull Principal principal)
- throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException,
- ImageNotSupportedException, NotAllowedException, ContainerNotFoundException {
- if (!hasDatabasePermission(containerId, databaseId, "DATA_UPDATE", principal)) {
- log.error("Missing data update permission");
- throw new NotAllowedException("Missing data update permission");
- }
- return ResponseEntity.accepted()
- .body(queryService.update(containerId, databaseId, tableId, data));
+ .build();
}
@DeleteMapping
@@ -82,7 +63,8 @@ public class TableDataEndpoint extends AbstractEndpoint {
@NotNull @Valid @RequestBody TableCsvDeleteDto data,
@NotNull Principal principal)
throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException,
- ImageNotSupportedException, TupleDeleteException, NotAllowedException, ContainerNotFoundException {
+ ImageNotSupportedException, TupleDeleteException, NotAllowedException, ContainerNotFoundException,
+ DatabaseConnectionException {
if (!hasDatabasePermission(containerId, databaseId, "DATA_DELETE", principal)) {
log.error("Missing data delete permission");
throw new NotAllowedException("Missing data delete permission");
@@ -95,20 +77,21 @@ public class TableDataEndpoint extends AbstractEndpoint {
@PostMapping("/import")
@Transactional
@Operation(summary = "Insert data from csv", security = @SecurityRequirement(name = "bearerAuth"))
- public ResponseEntity<Integer> importCsv(@NotNull @PathVariable("id") Long containerId,
+ public ResponseEntity<Void> importCsv(@NotNull @PathVariable("id") Long containerId,
@NotNull @PathVariable("databaseId") Long databaseId,
@NotNull @PathVariable("tableId") Long tableId,
@NotNull @Valid @RequestBody ImportDto data,
@NotNull Principal principal)
throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException,
- ImageNotSupportedException, ContainerNotFoundException, NotAllowedException {
+ ImageNotSupportedException, ContainerNotFoundException, NotAllowedException, DatabaseConnectionException {
if (!hasDatabasePermission(containerId, databaseId, "DATA_INSERT", principal)) {
log.error("Missing data insert permission");
throw new NotAllowedException("Missing data insert permission");
}
log.info("Insert data from location {} into database id {}", data, databaseId);
+ queryService.insert(containerId, databaseId, tableId, data);
return ResponseEntity.accepted()
- .body(queryService.insert(containerId, databaseId, tableId, data));
+ .build();
}
@RequestMapping(method = {RequestMethod.GET, RequestMethod.HEAD})
@@ -140,7 +123,7 @@ public class TableDataEndpoint extends AbstractEndpoint {
if (size != null && size <= 0) {
throw new PaginationException("Page number cannot be lower or equal to 0");
}
- final BigInteger count = queryService.count(containerId, databaseId, tableId, timestamp);
+ final Long count = queryService.count(containerId, databaseId, tableId, timestamp);
final HttpHeaders headers = new HttpHeaders();
headers.set("FDA-COUNT", count.toString());
final QueryResultDto response = queryService.findAll(containerId, databaseId, tableId, timestamp, page, size);
diff --git a/fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java b/fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java
index accebc2dee..ab12123959 100644
--- a/fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java
+++ b/fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java
@@ -23,6 +23,8 @@ import org.mariadb.jdbc.MariaDbBlob;
import org.springframework.transaction.annotation.Transactional;
import java.math.BigInteger;
+import java.sql.ResultSet;
+import java.sql.SQLException;
import java.sql.Timestamp;
import java.text.Normalizer;
import java.time.*;
@@ -70,23 +72,16 @@ public interface QueryMapper {
return slug.toLowerCase(Locale.ENGLISH);
}
- default QueryResultDto resultListToQueryResultDto(List<TableColumn> columns, List<?> result) {
- final Iterator<?> iterator = result.iterator();
+ default QueryResultDto resultListToQueryResultDto(List<TableColumn> columns, ResultSet result) throws SQLException {
final List<Map<String, Object>> resultList = new LinkedList<>();
- log.trace("result has {} columns and {} rows", columns.size(), result.size());
- while (iterator.hasNext()) {
+ log.trace("result has {} columns", columns.size());
+ while (result.next()) {
/* map the result set to the columns through the stored metadata in the metadata database */
int[] idx = new int[]{0};
- final Object[] data;
- if (columns.size() == 1) {
- data = new Object[]{iterator.next()};
- } else {
- data = (Object[]) iterator.next();
- }
final Map<String, Object> map = new HashMap<>();
- columns
- .forEach(column -> map.put(column.getName(),
- dataColumnToObject(data[idx[0]++], column)));
+ for (int i = 0; i < columns.size(); i++) {
+ map.put(columns.get(i).getInternalName(), dataColumnToObject(result.getObject(idx[0]++), columns.get(i)));
+ }
resultList.add(map);
}
return QueryResultDto.builder()
@@ -576,6 +571,17 @@ public interface QueryMapper {
return statement;
}
+ default Long resultSetToLong(ResultSet data) {
+ try {
+ if (data.next()) {
+ return data.getLong(1);
+ }
+ } catch (SQLException e) {
+ return null;
+ }
+ return null;
+ }
+
default String tableToRawFindAllQuery(Table table, Instant timestamp, Long size, Long page)
throws ImageNotSupportedException {
/* param check */
@@ -614,16 +620,15 @@ public interface QueryMapper {
return query.toString();
}
- default QueryResultDto queryTableToQueryResultDto(List<?> result, Table table) throws DateTimeException {
- final Iterator<?> iterator = result.iterator();
+ default QueryResultDto queryTableToQueryResultDto(ResultSet result, Table table) throws DateTimeException, SQLException {
final List<Map<String, Object>> queryResult = new LinkedList<>();
- while (iterator.hasNext()) {
+ while (result.next()) {
/* map the result set to the columns through the stored metadata in the metadata database */
int[] idx = new int[]{0};
- final Object[] data = (Object[]) iterator.next();
final Map<String, Object> map = new HashMap<>();
- table.getColumns()
- .forEach(column -> map.put(column.getInternalName(), dataColumnToObject(data[idx[0]++], column)));
+ for (int i = 0; i < table.getColumns().size(); i++) {
+ map.put(table.getColumns().get(i).getInternalName(), dataColumnToObject(result.getObject(idx[0]++), table.getColumns().get(i)));
+ }
queryResult.add(map);
}
log.info("Selected {} records from table id {}", queryResult.size(), table.getId());
diff --git a/fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java b/fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java
index a65bece0d5..ae969cf385 100644
--- a/fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java
+++ b/fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java
@@ -10,7 +10,6 @@ import at.tuwien.api.database.table.TableCsvUpdateDto;
import at.tuwien.exception.*;
import at.tuwien.querystore.Query;
import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Transactional;
import java.math.BigInteger;
import java.security.Principal;
@@ -39,7 +38,7 @@ public interface QueryService {
QueryResultDto execute(Long containerId, Long databaseId, ExecuteStatementDto statement,
Principal principal, Long page, Long size)
throws DatabaseNotFoundException, ImageNotSupportedException, QueryMalformedException, QueryStoreException,
- ContainerNotFoundException, ColumnParseException, UserNotFoundException, TableMalformedException;
+ ContainerNotFoundException, ColumnParseException, UserNotFoundException, TableMalformedException, DatabaseConnectionException;
/**
* Re-Executes an arbitrary query on the database container. We allow the user to only view the data, therefore the
@@ -62,7 +61,7 @@ public interface QueryService {
*/
QueryResultDto reExecute(Long containerId, Long databaseId, Query query, Long page, Long size)
throws TableNotFoundException, QueryStoreException, QueryMalformedException, DatabaseNotFoundException,
- ImageNotSupportedException, ContainerNotFoundException, TableMalformedException, ColumnParseException;
+ ImageNotSupportedException, ContainerNotFoundException, TableMalformedException, ColumnParseException, DatabaseConnectionException;
/**
@@ -127,7 +126,7 @@ public interface QueryService {
*/
ExportResource findOne(Long containerId, Long databaseId, Long queryId)
throws DatabaseNotFoundException, ImageNotSupportedException, TableMalformedException,
- ContainerNotFoundException, FileStorageException, QueryStoreException, QueryNotFoundException, QueryMalformedException;
+ ContainerNotFoundException, FileStorageException, QueryStoreException, QueryNotFoundException, QueryMalformedException, DatabaseConnectionException;
/**
* Count the total tuples for a given table id within a container-database id tuple at a given time.
@@ -143,9 +142,9 @@ public interface QueryService {
* @throws TableMalformedException The table columns are messed up what we got from the metadata database.
* @throws ImageNotSupportedException The image is not supported.
*/
- BigInteger count(Long containerId, Long databaseId, Long tableId, Instant timestamp)
+ Long count(Long containerId, Long databaseId, Long tableId, Instant timestamp)
throws ContainerNotFoundException, DatabaseNotFoundException, TableNotFoundException,
- TableMalformedException, ImageNotSupportedException;
+ TableMalformedException, ImageNotSupportedException, DatabaseConnectionException;
/**
* Insert data from AMQP client into a table of a table-database id tuple, we need the "root" role for this as the
@@ -155,30 +154,14 @@ public interface QueryService {
* @param databaseId The database id.
* @param tableId The table id.
* @param data The data.
- * @return The number of tuples affected.
* @throws ImageNotSupportedException The image is not supported.
* @throws TableMalformedException The table does not exist in the metadata database.
* @throws DatabaseNotFoundException The database is not found in the metadata database.
* @throws TableNotFoundException The table is not found in the metadata database.
* @throws ContainerNotFoundException The container was not found in the metadata database.
*/
- Integer insert(Long containerId, Long databaseId, Long tableId, TableCsvDto data) throws ImageNotSupportedException,
- TableMalformedException, DatabaseNotFoundException, TableNotFoundException, ContainerNotFoundException;
-
- /**
- * @param containerId The container id.
- * @param databaseId The database id.
- * @param tableId The table id.
- * @param data The updated tuple with the list of primary key columns.
- * @return The number of records updated.
- * @throws ImageNotSupportedException The image is not supported.
- * @throws TableMalformedException The table does not exist in the metadata database.
- * @throws DatabaseNotFoundException The database is not found in the metadata database.
- * @throws TableNotFoundException The table is not found in the metadata database.
- */
- Integer update(Long containerId, Long databaseId, Long tableId, TableCsvUpdateDto data)
- throws ImageNotSupportedException, TableMalformedException, DatabaseNotFoundException,
- TableNotFoundException, ContainerNotFoundException;
+ void insert(Long containerId, Long databaseId, Long tableId, TableCsvDto data) throws ImageNotSupportedException,
+ TableMalformedException, DatabaseNotFoundException, TableNotFoundException, ContainerNotFoundException, DatabaseConnectionException;
/**
* Deletes a tuple by given constraint set
@@ -195,7 +178,7 @@ public interface QueryService {
*/
void delete(Long containerId, Long databaseId, Long tableId, TableCsvDeleteDto data)
throws ImageNotSupportedException, TableMalformedException, DatabaseNotFoundException,
- TableNotFoundException, TupleDeleteException, ContainerNotFoundException;
+ TableNotFoundException, TupleDeleteException, ContainerNotFoundException, DatabaseConnectionException;
/**
* Insert data from a csv into a table of a table-database id tuple, we need the "root" role for this as the
@@ -204,13 +187,12 @@ public interface QueryService {
* @param databaseId The database id.
* @param tableId The table id.
* @param data The data path.
- * @return The number of tuples affected.
* @throws ImageNotSupportedException The image is not supported.
* @throws TableMalformedException The table does not exist in the metadata database.
* @throws DatabaseNotFoundException The database is not found in the metadata database.
* @throws TableNotFoundException The table is not found in the metadata database.
* @throws ContainerNotFoundException The container was not found in the metadata database.
*/
- Integer insert(Long containerId, Long databaseId, Long tableId, ImportDto data) throws ImageNotSupportedException,
- TableMalformedException, DatabaseNotFoundException, TableNotFoundException, ContainerNotFoundException;
+ void insert(Long containerId, Long databaseId, Long tableId, ImportDto data) throws ImageNotSupportedException,
+ TableMalformedException, DatabaseNotFoundException, TableNotFoundException, ContainerNotFoundException, DatabaseConnectionException;
}
diff --git a/fda-query-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java b/fda-query-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java
index 6538e8cfb6..0bcd9658bf 100644
--- a/fda-query-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java
+++ b/fda-query-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java
@@ -5,66 +5,84 @@ import at.tuwien.entities.container.image.ContainerImage;
import at.tuwien.entities.container.image.ContainerImageEnvironmentItem;
import at.tuwien.entities.container.image.ContainerImageEnvironmentItemType;
import at.tuwien.entities.database.Database;
+import at.tuwien.exception.DatabaseConnectionException;
+import com.mchange.v2.c3p0.ComboPooledDataSource;
import lombok.extern.log4j.Log4j2;
-import org.hibernate.Session;
-import org.hibernate.SessionFactory;
-import org.hibernate.Transaction;
-import org.hibernate.cfg.Configuration;
-import org.hibernate.query.NativeQuery;
import org.springframework.stereotype.Service;
-import javax.persistence.PersistenceException;
-import java.util.List;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Collection;
import java.util.stream.Collectors;
@Log4j2
@Service
public abstract class HibernateConnector {
- protected static Session getCurrentSession(ContainerImage image, Container container, Database database) {
- final String url = "jdbc:" + image.getJdbcMethod() + "://" + container.getInternalName() + "/" + database.getInternalName();
+ protected static Connection getConnection(ContainerImage image, Container container, Database database) throws DatabaseConnectionException {
+ final ComboPooledDataSource dataSource = new ComboPooledDataSource();
+ final String url = "jdbc:" + image.getJdbcMethod() + "://" + container.getInternalName() + "/" + (database != null ? database.getInternalName() : "");
+ dataSource.setJdbcUrl(url);
final String username = image.getEnvironment()
.stream()
.filter(e -> e.getType().equals(ContainerImageEnvironmentItemType.PRIVILEGED_USERNAME))
.map(ContainerImageEnvironmentItem::getValue)
.collect(Collectors.toList())
.get(0);
+ dataSource.setUser(username);
final String password = image.getEnvironment()
.stream()
.filter(e -> e.getType().equals(ContainerImageEnvironmentItemType.PRIVILEGED_PASSWORD))
.map(ContainerImageEnvironmentItem::getValue)
.collect(Collectors.toList())
.get(0);
+ dataSource.setPassword(password);
+ dataSource.setInitialPoolSize(5);
+ dataSource.setMinPoolSize(5);
+ dataSource.setAcquireIncrement(5);
+ dataSource.setMaxPoolSize(20);
+ dataSource.setMaxStatements(100);
+ final Connection connection;
+ try {
+ connection = dataSource.getConnection();
+ } catch (SQLException e) {
+ log.error("Failed to connect to the database");
+ log.debug("failed to connect to the database {}", database);
+ throw new DatabaseConnectionException("Failed to connect to the database");
+ }
+ return connection;
+ }
- final Configuration config = new Configuration();
- config.configure("mariadb_hibernate.cfg.xml");
- config.setProperty("hibernate.connection.url", url);
- config.setProperty("hibernate.connection.username", username);
- config.setProperty("hibernate.connection.password", password);
- config.setProperty("hibernate.connection.driver_class", image.getDriverClass());
- config.setProperty("hibernate.dialect", image.getDialect());
- final SessionFactory sessionFactory = config.buildSessionFactory();
- Session session = sessionFactory.getCurrentSession();
- if (!session.isOpen()) {
- log.debug("Session is closed, opening...");
- session = sessionFactory.openSession();
+ protected static Long activeConnection(Connection connection) throws DatabaseConnectionException {
+ final ResultSet resultSet = execute(connection, "SHOW STATUS LIKE 'threads_connected'");
+ try {
+ if (resultSet.next()) {
+ return resultSet.getLong(2);
+ }
+ } catch (SQLException e) {
+ log.error("Failed to determine active connections");
+ throw new DatabaseConnectionException("Failed to determine active connections", e);
}
- return session;
+ log.error("Failed to determine active connections");
+ throw new DatabaseConnectionException("Failed to determine active connections");
+ }
+
+ protected static ResultSet execute(Connection connection, String statement) throws DatabaseConnectionException {
+ return execute(connection, statement, null);
}
- protected static Long activeConnection(Session session) {
- final NativeQuery<?> nativeQuery = session.createSQLQuery("SHOW STATUS LIKE 'threads_connected'");
- final List<?> result;
+ protected static ResultSet execute(Connection connection, String statement, Collection<Object> data) throws DatabaseConnectionException {
+ final PreparedStatement preparedStatement;
try {
- result = nativeQuery.getResultList();
- } catch (PersistenceException e) {
- log.error("Failed to collect number of used connections");
- /* ignore */
- return null;
+ preparedStatement = connection.prepareStatement(statement, data);
+ return preparedStatement.executeQuery();
+ } catch (SQLException e) {
+ log.error("Failed to execute statement");
+ log.debug("failed to execute statement {}", statement);
+ throw new DatabaseConnectionException("Failed to execute statement", e);
}
- final Object[] row = (Object[]) result.get(0);
- log.debug("current number of connections: {}", Long.parseLong(String.valueOf(row[1])));
- return Long.parseLong(String.valueOf(row[1]));
}
}
diff --git a/fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java b/fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java
index 1241ef11ad..26a347d35f 100644
--- a/fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java
+++ b/fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java
@@ -7,7 +7,6 @@ import at.tuwien.api.database.query.ImportDto;
import at.tuwien.api.database.query.QueryResultDto;
import at.tuwien.api.database.table.TableCsvDeleteDto;
import at.tuwien.api.database.table.TableCsvDto;
-import at.tuwien.api.database.table.TableCsvUpdateDto;
import at.tuwien.entities.database.Database;
import at.tuwien.entities.database.table.Table;
import at.tuwien.entities.database.table.columns.TableColumn;
@@ -22,21 +21,19 @@ import net.sf.jsqlparser.statement.Statement;
import net.sf.jsqlparser.statement.select.*;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.RandomStringUtils;
-import org.hibernate.Session;
-import org.hibernate.Transaction;
-import org.hibernate.query.NativeQuery;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.InputStreamResource;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
-import javax.persistence.PersistenceException;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
-import java.math.BigInteger;
import java.security.Principal;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
import java.time.DateTimeException;
import java.time.Instant;
import java.time.format.DateTimeParseException;
@@ -66,7 +63,8 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
public QueryResultDto execute(Long containerId, Long databaseId, ExecuteStatementDto statement,
Principal principal, Long page, Long size)
throws DatabaseNotFoundException, ImageNotSupportedException, QueryMalformedException, QueryStoreException,
- ContainerNotFoundException, ColumnParseException, UserNotFoundException, TableMalformedException {
+ ContainerNotFoundException, ColumnParseException, UserNotFoundException, DatabaseConnectionException,
+ TableMalformedException {
Query q = storeService.insert(containerId, databaseId, null, statement, principal, Instant.now());
final QueryResultDto result = this.reExecute(containerId, databaseId, q, page, size);
q = storeService.update(containerId, databaseId, result, result.getResultNumber(), q);
@@ -77,33 +75,16 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
@Transactional(readOnly = true)
public QueryResultDto reExecute(Long containerId, Long databaseId, Query query, Long page, Long size)
throws QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException,
- ColumnParseException, TableMalformedException {
+ ColumnParseException, DatabaseConnectionException, TableMalformedException {
/* find */
final Database database = databaseService.find(containerId, databaseId);
if (!database.getContainer().getImage().getRepository().equals("mariadb")) {
throw new ImageNotSupportedException("Currently only MariaDB is supported");
}
/* run query */
- final Session session = getCurrentSession(database.getContainer().getImage(), database.getContainer(), database);
- final Transaction transaction = session.beginTransaction();
- /* prepare the statement */
- final NativeQuery<?> nativeQuery = session.createSQLQuery(
- queryMapper.queryToRawTimestampedQuery(query.getQuery(), database, query.getExecution(), page, size));
- final List<?> result;
- try {
- log.debug("affected {} rows", nativeQuery.executeUpdate());
- result = nativeQuery.getResultList();
- activeConnection(session);
- transaction.commit();
- } catch (PersistenceException e) {
- log.error("Query not valid for this database");
- session.close();
- throw new QueryMalformedException("Query not valid for this database", e);
- } finally {
- if (session.isOpen()) {
- session.close();
- }
- }
+ final Connection connection = getConnection(database.getContainer().getImage(), database.getContainer(), database);
+ final ResultSet resultSet = execute(connection, queryMapper.queryToRawTimestampedQuery(query.getQuery(), database, query.getExecution(), page, size));
+ log.debug("number of active connections {}", activeConnection(connection));
/* map the result to the tables (with respective columns) from the statement metadata */
final List<TableColumn> columns;
try {
@@ -112,7 +93,13 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
log.error("Failed to map/parse columns.");
throw new ColumnParseException("Failed to map/parse columns", e);
}
- final QueryResultDto dto = queryMapper.resultListToQueryResultDto(columns, result);
+ final QueryResultDto dto;
+ try {
+ dto = queryMapper.resultListToQueryResultDto(columns, resultSet);
+ } catch (SQLException e) {
+ log.error("Failed to map object");
+ throw new TableMalformedException("Failed to map object", e);
+ }
dto.setId(query.getId());
dto.setResultNumber(countQueryResults(containerId, databaseId, query));
return dto;
@@ -128,31 +115,17 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
final Database database = databaseService.find(containerId, databaseId);
final Table table = tableService.find(containerId, databaseId, tableId);
/* run query */
- final Session session = getCurrentSession(database.getContainer().getImage(), database.getContainer(), database);
- final Transaction transaction = session.beginTransaction();
- final NativeQuery<?> query = session.createSQLQuery(
- queryMapper.tableToRawFindAllQuery(table, timestamp, size, page));
- final List<?> resultList;
- try {
- log.debug("affected {} tuples in database id {}", query.executeUpdate(), databaseId);
- resultList = query.getResultList();
- activeConnection(session);
- transaction.commit();
- } catch (PersistenceException e) {
- log.error("Failed to find data");
- session.close();
- throw new TableMalformedException("\"Failed to find data", e);
- } finally {
- if (session.isOpen()) {
- session.close();
- }
- }
+ final Connection connection = getConnection(database.getContainer().getImage(), database.getContainer(), database);
+ final ResultSet resultSet = execute(connection, queryMapper.tableToRawFindAllQuery(table, timestamp, size, page));
final QueryResultDto result;
try {
- result = queryMapper.queryTableToQueryResultDto(resultList, table);
+ result = queryMapper.queryTableToQueryResultDto(resultSet, table);
} catch (DateTimeException e) {
log.error("Failed to parse date from the one stored in the metadata database");
throw new TableMalformedException("Could not parse date from format", e);
+ } catch (SQLException e) {
+ log.error("Failed to map object");
+ throw new TableMalformedException("Failed to map object", e);
}
return result;
}
@@ -168,23 +141,8 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
final Database database = databaseService.find(containerId, databaseId);
final Table table = tableService.find(containerId, databaseId, tableId);
/* run query */
- final Session session = getCurrentSession(database.getContainer().getImage(), database.getContainer(), database);
- final Transaction transaction = session.beginTransaction();
- final NativeQuery<?> query = session.createSQLQuery(
- queryMapper.tableToRawExportQuery(table, timestamp, filename));
- try {
- log.debug("affected tuples {}", query.executeUpdate());
- activeConnection(session);
- transaction.commit();
- } catch (PersistenceException e) {
- log.error("Failed to export table");
- session.close();
- throw new TableMalformedException("Data not found", e);
- } finally {
- if (session.isOpen()) {
- session.close();
- }
- }
+ final Connection connection = getConnection(database.getContainer().getImage(), database.getContainer(), database);
+ execute(connection, queryMapper.tableToRawExportQuery(table, timestamp, filename));
/* read file */
final InputStream inputStream;
try {
@@ -201,30 +159,16 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
@Override
@Transactional(readOnly = true)
public ExportResource findOne(Long containerId, Long databaseId, Long queryId)
- throws DatabaseNotFoundException, ImageNotSupportedException, TableMalformedException,
+ throws DatabaseNotFoundException, ImageNotSupportedException,
ContainerNotFoundException, FileStorageException, QueryStoreException, QueryNotFoundException,
- QueryMalformedException {
+ QueryMalformedException, DatabaseConnectionException {
final String filename = RandomStringUtils.randomAlphabetic(40) + ".csv";
/* find */
final Database database = databaseService.find(containerId, databaseId);
final Query query = storeService.findOne(containerId, databaseId, queryId);
/* run query */
- final Session session = getCurrentSession(database.getContainer().getImage(), database.getContainer(), database);
- final Transaction transaction = session.beginTransaction();
- final NativeQuery<?> query2 = session.createSQLQuery(queryMapper.queryToRawExportQuery(query, filename));
- try {
- log.debug("affected tuples {}", query2.executeUpdate());
- activeConnection(session);
- transaction.commit();
- } catch (PersistenceException e) {
- log.error("Failed to export query");
- session.close();
- throw new TableMalformedException("Failed to export query", e);
- } finally {
- if (session.isOpen()) {
- session.close();
- }
- }
+ final Connection connection = getConnection(database.getContainer().getImage(), database.getContainer(), database);
+ execute(connection, queryMapper.queryToRawExportQuery(query, filename));
/* read file */
final InputStream inputStream;
try {
@@ -242,182 +186,78 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
@Override
@Transactional
- public BigInteger count(Long containerId, Long databaseId, Long tableId, Instant timestamp)
+ public Long count(Long containerId, Long databaseId, Long tableId, Instant timestamp)
throws DatabaseNotFoundException, TableNotFoundException,
- TableMalformedException, ImageNotSupportedException {
+ ImageNotSupportedException, DatabaseConnectionException {
/* find */
final Database database = databaseService.find(containerId, databaseId);
final Table table = tableService.find(containerId, databaseId, tableId);
/* run query */
- final Session session = getCurrentSession(database.getContainer().getImage(), database.getContainer(), database);
- final Transaction transaction = session.beginTransaction();
- final NativeQuery<BigInteger> query = session.createSQLQuery(
- queryMapper.tableToRawCountAllQuery(table, timestamp));
- final BigInteger count;
- try {
- log.info("counted {} tuples in table id {}", query.executeUpdate(), tableId);
- count = query.getSingleResult();
- activeConnection(session);
- transaction.commit();
- } catch (PersistenceException e) {
- log.error("Failed to count tuples");
- session.close();
- throw new TableMalformedException("Failed to count tuples", e);
- } finally {
- if (session.isOpen()) {
- session.close();
- }
- }
- return count;
+ final Connection connection = getConnection(database.getContainer().getImage(), database.getContainer(), database);
+ final ResultSet resultSet = execute(connection, queryMapper.tableToRawCountAllQuery(table, timestamp));
+ return queryMapper.resultSetToLong(resultSet);
}
@Override
@Transactional
- public Integer insert(Long containerId, Long databaseId, Long tableId, TableCsvDto data)
+ public void insert(Long containerId, Long databaseId, Long tableId, TableCsvDto data)
throws ImageNotSupportedException, TableMalformedException, DatabaseNotFoundException,
- TableNotFoundException, ContainerNotFoundException {
+ TableNotFoundException, ContainerNotFoundException, DatabaseConnectionException {
/* find */
final Database database = databaseService.find(containerId, databaseId);
final Table table = tableService.find(containerId, databaseId, tableId);
/* run query */
- if (data.getData().size() == 0) return null;
- final Session session = getCurrentSession(database.getContainer().getImage(), database.getContainer(), database);
- final Transaction transaction = session.beginTransaction();
- activeConnection(session);
+ if (data.getData().size() == 0) return;
+ final Connection connection = getConnection(database.getContainer().getImage(), database.getContainer(), database);
/* prepare the statement */
final InsertTableRawQuery raw;
try {
raw = queryMapper.tableCsvDtoToRawInsertQuery(table, data);
} catch (DateTimeParseException e) {
log.error("Failed to parse date: {}", e.getMessage());
- session.close();
- return 0;
+ return;
} catch (NumberFormatException e) {
log.error("Failed to parse number: {}", e.getMessage());
- session.close();
- return 0;
+ return;
} catch (Exception e) {
log.error("Failed for unknown reason: {}", e.getMessage());
- session.close();
- return 0;
+ return;
}
- final NativeQuery<?> query = session.createSQLQuery(raw.getQuery());
- log.trace("query with parameters {}", query.setParameterList(1, raw.getData()));
- return execute(query, session);
- }
-
- @Override
- @Transactional
- public Integer update(Long containerId, Long databaseId, Long tableId, TableCsvUpdateDto data)
- throws ImageNotSupportedException, TableMalformedException, DatabaseNotFoundException,
- TableNotFoundException {
- /* find */
- final Database database = databaseService.find(containerId, databaseId);
- final Table table = tableService.find(containerId, databaseId, tableId);
- /* run query */
- if (data.getData().size() == 0 || data.getKeys().size() == 0) return null;
- final Session session = getCurrentSession(database.getContainer().getImage(), database.getContainer(), database);
- /* prepare the statement */
- final InsertTableRawQuery raw = queryMapper.tableCsvDtoToRawUpdateQuery(table, data);
- final NativeQuery<?> query = session.createSQLQuery(raw.getQuery());
- final int[] idx = new int[]{0};
- data.getData()
- .forEach((key, value) -> query.setParameter(idx[0]++, value));
- log.trace("query with parameters {}", query);
- return execute(query, session);
+ execute(connection, raw.getQuery(), raw.getData());
}
@Override
@Transactional
public void delete(Long containerId, Long databaseId, Long tableId, TableCsvDeleteDto data)
throws ImageNotSupportedException, TableMalformedException, DatabaseNotFoundException,
- TableNotFoundException, TupleDeleteException {
+ TableNotFoundException, DatabaseConnectionException {
/* find */
final Database database = databaseService.find(containerId, databaseId);
final Table table = tableService.find(containerId, databaseId, tableId);
/* run query */
if (data.getKeys().size() == 0) return;
- final Session session = getCurrentSession(database.getContainer().getImage(), database.getContainer(), database);
- final Transaction transaction = session.beginTransaction();
+ final Connection connection = getConnection(database.getContainer().getImage(), database.getContainer(), database);
/* prepare the statement */
- final NativeQuery<?> query = session.createSQLQuery(queryMapper.tableCsvDtoToRawDeleteQuery(table, data));
- final int[] idx = new int[]{0};
- data.getKeys()
- .forEach((key, value) -> query.setParameter(idx[0]++, value));
- final int affectedTuples;
- try {
- affectedTuples = query.executeUpdate();
- activeConnection(session);
- transaction.commit();
- } catch (PersistenceException e) {
- log.error("Failed to delete data");
- log.debug("failed to delete data: {}", e.getMessage());
- session.close();
- throw new TableMalformedException("Could not delete data", e);
- } finally {
- if (session.isOpen()) {
- session.close();
- }
- }
- if (affectedTuples == 0) {
- log.error("No tuples were deleted");
- throw new TupleDeleteException("No tuples deleted");
- }
- log.info("Deleted {} tuple(s)", affectedTuples);
+ execute(connection, queryMapper.tableCsvDtoToRawDeleteQuery(table, data), data.getKeys().values());
+ log.info("Deleted tuple(s)");
log.debug("Deleted tuple(s) {}", data);
}
@Override
@Transactional
- public Integer insert(Long containerId, Long databaseId, Long tableId, ImportDto data)
+ public void insert(Long containerId, Long databaseId, Long tableId, ImportDto data)
throws ImageNotSupportedException, TableMalformedException, DatabaseNotFoundException,
- TableNotFoundException, ContainerNotFoundException {
+ TableNotFoundException, ContainerNotFoundException, DatabaseConnectionException {
/* find */
final Database database = databaseService.find(containerId, databaseId);
final Table table = tableService.find(containerId, databaseId, tableId);
/* preparing the statements */
- final Session session1 = getCurrentSession(database.getContainer().getImage(), database.getContainer(), database);
- session1.beginTransaction();
- final Session session2 = getCurrentSession(database.getContainer().getImage(), database.getContainer(), database);
- session2.beginTransaction();
- final Session session3 = getCurrentSession(database.getContainer().getImage(), database.getContainer(), database);
- session3.beginTransaction();
- final Session session4 = getCurrentSession(database.getContainer().getImage(), database.getContainer(), database);
- session4.beginTransaction();
-
+ final Connection connection = getConnection(database.getContainer().getImage(), database.getContainer(), database);
/* Create a temporary table, insert there, transfer with update on duplicate key and lastly drops the temporary table */
- execute(session1.createSQLQuery(queryMapper.generateTemporaryTableSQL(table)), session1);
- execute(session2.createSQLQuery(queryMapper.pathToRawInsertQuery(table, data).getQuery()), session2);
- final Integer affectedTuples = execute(session3.createSQLQuery(queryMapper.generateInsertFromTemporaryTableSQL(table)), session3);
- execute(session4.createSQLQuery(queryMapper.dropTemporaryTableSQL(table)), session4);
- return affectedTuples;
- }
-
- /**
- * Executes a generic native query for a given session and factory
- *
- * @param query The query.
- * @param session The session.
- * @return The number of affected tuples.
- * @throws TableMalformedException The table where the query was applied to is malformed.
- */
- private Integer execute(NativeQuery<?> query, Session session)
- throws TableMalformedException {
- final int affectedTuples;
- try {
- affectedTuples = query.executeUpdate();
- session.getTransaction()
- .commit();
- } catch (PersistenceException e) {
- log.error("Could not insert data: {}", e.getMessage());
- session.close();
- throw new TableMalformedException("Could not insert data", e);
- } finally {
- if (session.isOpen()) {
- session.close();
- }
- }
- return affectedTuples;
+ execute(connection, queryMapper.generateTemporaryTableSQL(table));
+ execute(connection, queryMapper.pathToRawInsertQuery(table, data).getQuery());
+ execute(connection, queryMapper.generateInsertFromTemporaryTableSQL(table));
+ execute(connection, queryMapper.dropTemporaryTableSQL(table));
}
/**
@@ -513,35 +353,17 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
* @param query The query object.
* @return The number of tuples this query returns.
* @throws DatabaseNotFoundException The user database was not found in the container.
- * @throws TableMalformedException The table is malformed in the database (in the container).
* @throws ImageNotSupportedException The database image is not supported.
*/
@Transactional(readOnly = true)
protected Long countQueryResults(Long containerId, Long databaseId, Query query)
- throws DatabaseNotFoundException, TableMalformedException, ImageNotSupportedException {
+ throws DatabaseNotFoundException, ImageNotSupportedException, DatabaseConnectionException {
/* find */
final Database database = databaseService.find(containerId, databaseId);
/* run query */
- final Session session = getCurrentSession(database.getContainer().getImage(), database.getContainer(), database);
- final Transaction transaction = session.beginTransaction();
- final NativeQuery<BigInteger> nativeQuery = session.createSQLQuery(
- queryMapper.queryToRawTimestampedCountQuery(query.getQuery(), database, query.getExecution()));
- final BigInteger result;
- try {
- log.debug("counted {} tuples from query {}", nativeQuery.executeUpdate(), query.getId());
- result = nativeQuery.getSingleResult();
- activeConnection(session);
- transaction.commit();
- } catch (PersistenceException e) {
- log.error("Failed to count tuples");
- session.close();
- throw new TableMalformedException("Failed to count tuples", e);
- } finally {
- if (session.isOpen()) {
- session.close();
- }
- }
- return result.longValue();
+ final Connection connection = getConnection(database.getContainer().getImage(), database.getContainer(), database);
+ final ResultSet resultSet = execute(connection, queryMapper.queryToRawTimestampedCountQuery(query.getQuery(), database, query.getExecution()));
+ return queryMapper.resultSetToLong(resultSet);
}
--
GitLab