Skip to content
Snippets Groups Projects
Commit 2f7a67ad authored by Martin Weise's avatar Martin Weise
Browse files

Added the basic mapping

parent 10a2c221
Branches
Tags
2 merge requests!81New stable release,!80Multiple features connected with user management and ownership of databases
Expand all Show whitespace changes
Inline Side-by-side
Showing
with 156 additions and 345 deletions
fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java
+ 4
− 3
View file @ 2f7a67ad
......@@ -48,7 +48,7 @@ public class QueryEndpoint extends AbstractEndpoint {
@NotNull Principal principal)
throws DatabaseNotFoundException, ImageNotSupportedException, QueryStoreException, QueryMalformedException,
ContainerNotFoundException, ColumnParseException, UserNotFoundException, TableMalformedException,
NotAllowedException {
NotAllowedException, DatabaseConnectionException {
if (!hasDatabasePermission(containerId, databaseId, "QUERY_EXECUTE", principal)) {
log.error("Missing execute query permission");
throw new NotAllowedException("Missing execute query permission");
......@@ -74,7 +74,7 @@ public class QueryEndpoint extends AbstractEndpoint {
@NotNull Principal principal)
throws QueryStoreException, QueryNotFoundException, DatabaseNotFoundException, ImageNotSupportedException,
TableNotFoundException, QueryMalformedException, ContainerNotFoundException, TableMalformedException,
ColumnParseException, NotAllowedException {
ColumnParseException, NotAllowedException, DatabaseConnectionException {
if (!hasQueryPermission(containerId, databaseId, queryId, "QUERY_RE_EXECUTE", principal)) {
log.error("Missing re-execute query permission");
throw new NotAllowedException("Missing re-execute query permission");
......@@ -94,7 +94,8 @@ public class QueryEndpoint extends AbstractEndpoint {
@NotNull @PathVariable("queryId") Long queryId,
@NotNull Principal principal)
throws QueryStoreException, QueryNotFoundException, DatabaseNotFoundException, ImageNotSupportedException,
ContainerNotFoundException, TableMalformedException, FileStorageException, NotAllowedException, QueryMalformedException {
ContainerNotFoundException, TableMalformedException, FileStorageException, NotAllowedException,
QueryMalformedException, DatabaseConnectionException {
if (!hasQueryPermission(containerId, databaseId, queryId, "QUERY_EXPORT", principal)) {
log.error("Missing export query permission");
throw new NotAllowedException("Missing export query permission");
......
fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableDataEndpoint.java
+ 11
− 28
View file @ 2f7a67ad
......@@ -4,7 +4,6 @@ import at.tuwien.api.database.query.ImportDto;
import at.tuwien.api.database.query.QueryResultDto;
import at.tuwien.api.database.table.TableCsvDeleteDto;
import at.tuwien.api.database.table.TableCsvDto;
import at.tuwien.api.database.table.TableCsvUpdateDto;
import at.tuwien.exception.*;
import at.tuwien.service.*;
import io.swagger.v3.oas.annotations.Operation;
......@@ -18,7 +17,6 @@ import org.springframework.web.bind.annotation.*;
import javax.validation.Valid;
import javax.validation.constraints.NotNull;
import java.math.BigInteger;
import java.security.Principal;
import java.time.Instant;
......@@ -40,37 +38,20 @@ public class TableDataEndpoint extends AbstractEndpoint {
@PostMapping
@Transactional
@Operation(summary = "Insert data", security = @SecurityRequirement(name = "bearerAuth"))
public ResponseEntity<Integer> insert(@NotNull @PathVariable("id") Long containerId,
public ResponseEntity<Void> insert(@NotNull @PathVariable("id") Long containerId,
@NotNull @PathVariable("databaseId") Long databaseId,
@NotNull @PathVariable("tableId") Long tableId,
@NotNull @Valid @RequestBody TableCsvDto data,
@NotNull Principal principal)
throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException,
ImageNotSupportedException, ContainerNotFoundException, NotAllowedException {
ImageNotSupportedException, ContainerNotFoundException, NotAllowedException, DatabaseConnectionException {
if (!hasDatabasePermission(containerId, databaseId, "DATA_INSERT", principal)) {
log.error("Missing data insert permission");
throw new NotAllowedException("Missing data insert permission");
}
queryService.insert(containerId, databaseId, tableId, data);
return ResponseEntity.accepted()
.body(queryService.insert(containerId, databaseId, tableId, data));
}
@PutMapping
@Transactional
@Operation(summary = "Update data", security = @SecurityRequirement(name = "bearerAuth"))
public ResponseEntity<Integer> update(@NotNull @PathVariable("id") Long containerId,
@NotNull @PathVariable("databaseId") Long databaseId,
@NotNull @PathVariable("tableId") Long tableId,
@NotNull @Valid @RequestBody TableCsvUpdateDto data,
@NotNull Principal principal)
throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException,
ImageNotSupportedException, NotAllowedException, ContainerNotFoundException {
if (!hasDatabasePermission(containerId, databaseId, "DATA_UPDATE", principal)) {
log.error("Missing data update permission");
throw new NotAllowedException("Missing data update permission");
}
return ResponseEntity.accepted()
.body(queryService.update(containerId, databaseId, tableId, data));
.build();
}
@DeleteMapping
......@@ -82,7 +63,8 @@ public class TableDataEndpoint extends AbstractEndpoint {
@NotNull @Valid @RequestBody TableCsvDeleteDto data,
@NotNull Principal principal)
throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException,
ImageNotSupportedException, TupleDeleteException, NotAllowedException, ContainerNotFoundException {
ImageNotSupportedException, TupleDeleteException, NotAllowedException, ContainerNotFoundException,
DatabaseConnectionException {
if (!hasDatabasePermission(containerId, databaseId, "DATA_DELETE", principal)) {
log.error("Missing data delete permission");
throw new NotAllowedException("Missing data delete permission");
......@@ -95,20 +77,21 @@ public class TableDataEndpoint extends AbstractEndpoint {
@PostMapping("/import")
@Transactional
@Operation(summary = "Insert data from csv", security = @SecurityRequirement(name = "bearerAuth"))
public ResponseEntity<Integer> importCsv(@NotNull @PathVariable("id") Long containerId,
public ResponseEntity<Void> importCsv(@NotNull @PathVariable("id") Long containerId,
@NotNull @PathVariable("databaseId") Long databaseId,
@NotNull @PathVariable("tableId") Long tableId,
@NotNull @Valid @RequestBody ImportDto data,
@NotNull Principal principal)
throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException,
ImageNotSupportedException, ContainerNotFoundException, NotAllowedException {
ImageNotSupportedException, ContainerNotFoundException, NotAllowedException, DatabaseConnectionException {
if (!hasDatabasePermission(containerId, databaseId, "DATA_INSERT", principal)) {
log.error("Missing data insert permission");
throw new NotAllowedException("Missing data insert permission");
}
log.info("Insert data from location {} into database id {}", data, databaseId);
queryService.insert(containerId, databaseId, tableId, data);
return ResponseEntity.accepted()
.body(queryService.insert(containerId, databaseId, tableId, data));
.build();
}
@RequestMapping(method = {RequestMethod.GET, RequestMethod.HEAD})
......@@ -140,7 +123,7 @@ public class TableDataEndpoint extends AbstractEndpoint {
if (size != null && size <= 0) {
throw new PaginationException("Page number cannot be lower or equal to 0");
}
final BigInteger count = queryService.count(containerId, databaseId, tableId, timestamp);
final Long count = queryService.count(containerId, databaseId, tableId, timestamp);
final HttpHeaders headers = new HttpHeaders();
headers.set("FDA-COUNT", count.toString());
final QueryResultDto response = queryService.findAll(containerId, databaseId, tableId, timestamp, page, size);
......
fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java
+ 24
− 19
View file @ 2f7a67ad
......@@ -23,6 +23,8 @@ import org.mariadb.jdbc.MariaDbBlob;
import org.springframework.transaction.annotation.Transactional;
import java.math.BigInteger;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.text.Normalizer;
import java.time.*;
......@@ -70,23 +72,16 @@ public interface QueryMapper {
return slug.toLowerCase(Locale.ENGLISH);
}
default QueryResultDto resultListToQueryResultDto(List<TableColumn> columns, List<?> result) {
final Iterator<?> iterator = result.iterator();
default QueryResultDto resultListToQueryResultDto(List<TableColumn> columns, ResultSet result) throws SQLException {
final List<Map<String, Object>> resultList = new LinkedList<>();
log.trace("result has {} columns and {} rows", columns.size(), result.size());
while (iterator.hasNext()) {
log.trace("result has {} columns", columns.size());
while (result.next()) {
/* map the result set to the columns through the stored metadata in the metadata database */
int[] idx = new int[]{0};
final Object[] data;
if (columns.size() == 1) {
data = new Object[]{iterator.next()};
} else {
data = (Object[]) iterator.next();
}
final Map<String, Object> map = new HashMap<>();
columns
.forEach(column -> map.put(column.getName(),
dataColumnToObject(data[idx[0]++], column)));
for (int i = 0; i < columns.size(); i++) {
map.put(columns.get(i).getInternalName(), dataColumnToObject(result.getObject(idx[0]++), columns.get(i)));
}
resultList.add(map);
}
return QueryResultDto.builder()
......@@ -576,6 +571,17 @@ public interface QueryMapper {
return statement;
}
default Long resultSetToLong(ResultSet data) {
try {
if (data.next()) {
return data.getLong(1);
}
} catch (SQLException e) {
return null;
}
return null;
}
default String tableToRawFindAllQuery(Table table, Instant timestamp, Long size, Long page)
throws ImageNotSupportedException {
/* param check */
......@@ -614,16 +620,15 @@ public interface QueryMapper {
return query.toString();
}
default QueryResultDto queryTableToQueryResultDto(List<?> result, Table table) throws DateTimeException {
final Iterator<?> iterator = result.iterator();
default QueryResultDto queryTableToQueryResultDto(ResultSet result, Table table) throws DateTimeException, SQLException {
final List<Map<String, Object>> queryResult = new LinkedList<>();
while (iterator.hasNext()) {
while (result.next()) {
/* map the result set to the columns through the stored metadata in the metadata database */
int[] idx = new int[]{0};
final Object[] data = (Object[]) iterator.next();
final Map<String, Object> map = new HashMap<>();
table.getColumns()
.forEach(column -> map.put(column.getInternalName(), dataColumnToObject(data[idx[0]++], column)));
for (int i = 0; i < table.getColumns().size(); i++) {
map.put(table.getColumns().get(i).getInternalName(), dataColumnToObject(result.getObject(idx[0]++), table.getColumns().get(i)));
}
queryResult.add(map);
}
log.info("Selected {} records from table id {}", queryResult.size(), table.getId());
......
fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java
+ 10
− 28
View file @ 2f7a67ad
......@@ -10,7 +10,6 @@ import at.tuwien.api.database.table.TableCsvUpdateDto;
import at.tuwien.exception.*;
import at.tuwien.querystore.Query;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.math.BigInteger;
import java.security.Principal;
......@@ -39,7 +38,7 @@ public interface QueryService {
QueryResultDto execute(Long containerId, Long databaseId, ExecuteStatementDto statement,
Principal principal, Long page, Long size)
throws DatabaseNotFoundException, ImageNotSupportedException, QueryMalformedException, QueryStoreException,
ContainerNotFoundException, ColumnParseException, UserNotFoundException, TableMalformedException;
ContainerNotFoundException, ColumnParseException, UserNotFoundException, TableMalformedException, DatabaseConnectionException;
/**
* Re-Executes an arbitrary query on the database container. We allow the user to only view the data, therefore the
......@@ -62,7 +61,7 @@ public interface QueryService {
*/
QueryResultDto reExecute(Long containerId, Long databaseId, Query query, Long page, Long size)
throws TableNotFoundException, QueryStoreException, QueryMalformedException, DatabaseNotFoundException,
ImageNotSupportedException, ContainerNotFoundException, TableMalformedException, ColumnParseException;
ImageNotSupportedException, ContainerNotFoundException, TableMalformedException, ColumnParseException, DatabaseConnectionException;
/**
......@@ -127,7 +126,7 @@ public interface QueryService {
*/
ExportResource findOne(Long containerId, Long databaseId, Long queryId)
throws DatabaseNotFoundException, ImageNotSupportedException, TableMalformedException,
ContainerNotFoundException, FileStorageException, QueryStoreException, QueryNotFoundException, QueryMalformedException;
ContainerNotFoundException, FileStorageException, QueryStoreException, QueryNotFoundException, QueryMalformedException, DatabaseConnectionException;
/**
* Count the total tuples for a given table id within a container-database id tuple at a given time.
......@@ -143,9 +142,9 @@ public interface QueryService {
* @throws TableMalformedException The table columns are messed up what we got from the metadata database.
* @throws ImageNotSupportedException The image is not supported.
*/
BigInteger count(Long containerId, Long databaseId, Long tableId, Instant timestamp)
Long count(Long containerId, Long databaseId, Long tableId, Instant timestamp)
throws ContainerNotFoundException, DatabaseNotFoundException, TableNotFoundException,
TableMalformedException, ImageNotSupportedException;
TableMalformedException, ImageNotSupportedException, DatabaseConnectionException;
/**
* Insert data from AMQP client into a table of a table-database id tuple, we need the "root" role for this as the
......@@ -155,30 +154,14 @@ public interface QueryService {
* @param databaseId The database id.
* @param tableId The table id.
* @param data The data.
* @return The number of tuples affected.
* @throws ImageNotSupportedException The image is not supported.
* @throws TableMalformedException The table does not exist in the metadata database.
* @throws DatabaseNotFoundException The database is not found in the metadata database.
* @throws TableNotFoundException The table is not found in the metadata database.
* @throws ContainerNotFoundException The container was not found in the metadata database.
*/
Integer insert(Long containerId, Long databaseId, Long tableId, TableCsvDto data) throws ImageNotSupportedException,
TableMalformedException, DatabaseNotFoundException, TableNotFoundException, ContainerNotFoundException;
/**
* @param containerId The container id.
* @param databaseId The database id.
* @param tableId The table id.
* @param data The updated tuple with the list of primary key columns.
* @return The number of records updated.
* @throws ImageNotSupportedException The image is not supported.
* @throws TableMalformedException The table does not exist in the metadata database.
* @throws DatabaseNotFoundException The database is not found in the metadata database.
* @throws TableNotFoundException The table is not found in the metadata database.
*/
Integer update(Long containerId, Long databaseId, Long tableId, TableCsvUpdateDto data)
throws ImageNotSupportedException, TableMalformedException, DatabaseNotFoundException,
TableNotFoundException, ContainerNotFoundException;
void insert(Long containerId, Long databaseId, Long tableId, TableCsvDto data) throws ImageNotSupportedException,
TableMalformedException, DatabaseNotFoundException, TableNotFoundException, ContainerNotFoundException, DatabaseConnectionException;
/**
* Deletes a tuple by given constraint set
......@@ -195,7 +178,7 @@ public interface QueryService {
*/
void delete(Long containerId, Long databaseId, Long tableId, TableCsvDeleteDto data)
throws ImageNotSupportedException, TableMalformedException, DatabaseNotFoundException,
TableNotFoundException, TupleDeleteException, ContainerNotFoundException;
TableNotFoundException, TupleDeleteException, ContainerNotFoundException, DatabaseConnectionException;
/**
* Insert data from a csv into a table of a table-database id tuple, we need the "root" role for this as the
......@@ -204,13 +187,12 @@ public interface QueryService {
* @param databaseId The database id.
* @param tableId The table id.
* @param data The data path.
* @return The number of tuples affected.
* @throws ImageNotSupportedException The image is not supported.
* @throws TableMalformedException The table does not exist in the metadata database.
* @throws DatabaseNotFoundException The database is not found in the metadata database.
* @throws TableNotFoundException The table is not found in the metadata database.
* @throws ContainerNotFoundException The container was not found in the metadata database.
*/
Integer insert(Long containerId, Long databaseId, Long tableId, ImportDto data) throws ImageNotSupportedException,
TableMalformedException, DatabaseNotFoundException, TableNotFoundException, ContainerNotFoundException;
void insert(Long containerId, Long databaseId, Long tableId, ImportDto data) throws ImageNotSupportedException,
TableMalformedException, DatabaseNotFoundException, TableNotFoundException, ContainerNotFoundException, DatabaseConnectionException;
}
fda-query-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java
+ 51
− 33
View file @ 2f7a67ad
......@@ -5,66 +5,84 @@ import at.tuwien.entities.container.image.ContainerImage;
import at.tuwien.entities.container.image.ContainerImageEnvironmentItem;
import at.tuwien.entities.container.image.ContainerImageEnvironmentItemType;
import at.tuwien.entities.database.Database;
import at.tuwien.exception.DatabaseConnectionException;
import com.mchange.v2.c3p0.ComboPooledDataSource;
import lombok.extern.log4j.Log4j2;
import org.hibernate.Session;
import org.hibernate.SessionFactory;
import org.hibernate.Transaction;
import org.hibernate.cfg.Configuration;
import org.hibernate.query.NativeQuery;
import org.springframework.stereotype.Service;
import javax.persistence.PersistenceException;
import java.util.List;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Collection;
import java.util.stream.Collectors;
@Log4j2
@Service
public abstract class HibernateConnector {
protected static Session getCurrentSession(ContainerImage image, Container container, Database database) {
final String url = "jdbc:" + image.getJdbcMethod() + "://" + container.getInternalName() + "/" + database.getInternalName();
protected static Connection getConnection(ContainerImage image, Container container, Database database) throws DatabaseConnectionException {
final ComboPooledDataSource dataSource = new ComboPooledDataSource();
final String url = "jdbc:" + image.getJdbcMethod() + "://" + container.getInternalName() + "/" + (database != null ? database.getInternalName() : "");
dataSource.setJdbcUrl(url);
final String username = image.getEnvironment()
.stream()
.filter(e -> e.getType().equals(ContainerImageEnvironmentItemType.PRIVILEGED_USERNAME))
.map(ContainerImageEnvironmentItem::getValue)
.collect(Collectors.toList())
.get(0);
dataSource.setUser(username);
final String password = image.getEnvironment()
.stream()
.filter(e -> e.getType().equals(ContainerImageEnvironmentItemType.PRIVILEGED_PASSWORD))
.map(ContainerImageEnvironmentItem::getValue)
.collect(Collectors.toList())
.get(0);
dataSource.setPassword(password);
dataSource.setInitialPoolSize(5);
dataSource.setMinPoolSize(5);
dataSource.setAcquireIncrement(5);
dataSource.setMaxPoolSize(20);
dataSource.setMaxStatements(100);
final Connection connection;
try {
connection = dataSource.getConnection();
} catch (SQLException e) {
log.error("Failed to connect to the database");
log.debug("failed to connect to the database {}", database);
throw new DatabaseConnectionException("Failed to connect to the database");
}
return connection;
}
final Configuration config = new Configuration();
config.configure("mariadb_hibernate.cfg.xml");
config.setProperty("hibernate.connection.url", url);
config.setProperty("hibernate.connection.username", username);
config.setProperty("hibernate.connection.password", password);
config.setProperty("hibernate.connection.driver_class", image.getDriverClass());
config.setProperty("hibernate.dialect", image.getDialect());
final SessionFactory sessionFactory = config.buildSessionFactory();
Session session = sessionFactory.getCurrentSession();
if (!session.isOpen()) {
log.debug("Session is closed, opening...");
session = sessionFactory.openSession();
protected static Long activeConnection(Connection connection) throws DatabaseConnectionException {
final ResultSet resultSet = execute(connection, "SHOW STATUS LIKE 'threads_connected'");
try {
if (resultSet.next()) {
return resultSet.getLong(2);
}
return session;
} catch (SQLException e) {
log.error("Failed to determine active connections");
throw new DatabaseConnectionException("Failed to determine active connections", e);
}
log.error("Failed to determine active connections");
throw new DatabaseConnectionException("Failed to determine active connections");
}
protected static ResultSet execute(Connection connection, String statement) throws DatabaseConnectionException {
return execute(connection, statement, null);
}
protected static Long activeConnection(Session session) {
final NativeQuery<?> nativeQuery = session.createSQLQuery("SHOW STATUS LIKE 'threads_connected'");
final List<?> result;
protected static ResultSet execute(Connection connection, String statement, Collection<Object> data) throws DatabaseConnectionException {
final PreparedStatement preparedStatement;
try {
result = nativeQuery.getResultList();
} catch (PersistenceException e) {
log.error("Failed to collect number of used connections");
/* ignore */
return null;
preparedStatement = connection.prepareStatement(statement, data);
return preparedStatement.executeQuery();
} catch (SQLException e) {
log.error("Failed to execute statement");
log.debug("failed to execute statement {}", statement);
throw new DatabaseConnectionException("Failed to execute statement", e);
}
final Object[] row = (Object[]) result.get(0);
log.debug("current number of connections: {}", Long.parseLong(String.valueOf(row[1])));
return Long.parseLong(String.valueOf(row[1]));
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment