Skip to content
Snippets Groups Projects
Commit 2f7a67ad authored by Martin Weise's avatar Martin Weise
Browse files

Added the basic mapping

parent 10a2c221
No related branches found
No related tags found
2 merge requests!81New stable release,!80Multiple features connected with user management and ownership of databases
This commit is part of merge request !81. Comments created here will be created in the context of that merge request.
Expand all Show whitespace changes
Inline Side-by-side
Showing
with 156 additions and 345 deletions
fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java
+ 4
− 3
Edit View file @ 2f7a67ad
......@@ -48,7 +48,7 @@ public class QueryEndpoint extends AbstractEndpoint {
@NotNull Principal principal)
throws DatabaseNotFoundException, ImageNotSupportedException, QueryStoreException, QueryMalformedException,
ContainerNotFoundException, ColumnParseException, UserNotFoundException, TableMalformedException,
NotAllowedException {
NotAllowedException, DatabaseConnectionException {
if (!hasDatabasePermission(containerId, databaseId, "QUERY_EXECUTE", principal)) {
log.error("Missing execute query permission");
throw new NotAllowedException("Missing execute query permission");
......@@ -74,7 +74,7 @@ public class QueryEndpoint extends AbstractEndpoint {
@NotNull Principal principal)
throws QueryStoreException, QueryNotFoundException, DatabaseNotFoundException, ImageNotSupportedException,
TableNotFoundException, QueryMalformedException, ContainerNotFoundException, TableMalformedException,
ColumnParseException, NotAllowedException {
ColumnParseException, NotAllowedException, DatabaseConnectionException {
if (!hasQueryPermission(containerId, databaseId, queryId, "QUERY_RE_EXECUTE", principal)) {
log.error("Missing re-execute query permission");
throw new NotAllowedException("Missing re-execute query permission");
......@@ -94,7 +94,8 @@ public class QueryEndpoint extends AbstractEndpoint {
@NotNull @PathVariable("queryId") Long queryId,
@NotNull Principal principal)
throws QueryStoreException, QueryNotFoundException, DatabaseNotFoundException, ImageNotSupportedException,
ContainerNotFoundException, TableMalformedException, FileStorageException, NotAllowedException, QueryMalformedException {
ContainerNotFoundException, TableMalformedException, FileStorageException, NotAllowedException,
QueryMalformedException, DatabaseConnectionException {
if (!hasQueryPermission(containerId, databaseId, queryId, "QUERY_EXPORT", principal)) {
log.error("Missing export query permission");
throw new NotAllowedException("Missing export query permission");
......
fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableDataEndpoint.java
+ 11
− 28
Edit View file @ 2f7a67ad
......@@ -4,7 +4,6 @@ import at.tuwien.api.database.query.ImportDto;
import at.tuwien.api.database.query.QueryResultDto;
import at.tuwien.api.database.table.TableCsvDeleteDto;
import at.tuwien.api.database.table.TableCsvDto;
import at.tuwien.api.database.table.TableCsvUpdateDto;
import at.tuwien.exception.*;
import at.tuwien.service.*;
import io.swagger.v3.oas.annotations.Operation;
......@@ -18,7 +17,6 @@ import org.springframework.web.bind.annotation.*;
import javax.validation.Valid;
import javax.validation.constraints.NotNull;
import java.math.BigInteger;
import java.security.Principal;
import java.time.Instant;
......@@ -40,37 +38,20 @@ public class TableDataEndpoint extends AbstractEndpoint {
@PostMapping
@Transactional
@Operation(summary = "Insert data", security = @SecurityRequirement(name = "bearerAuth"))
public ResponseEntity<Integer> insert(@NotNull @PathVariable("id") Long containerId,
public ResponseEntity<Void> insert(@NotNull @PathVariable("id") Long containerId,
@NotNull @PathVariable("databaseId") Long databaseId,
@NotNull @PathVariable("tableId") Long tableId,
@NotNull @Valid @RequestBody TableCsvDto data,
@NotNull Principal principal)
throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException,
ImageNotSupportedException, ContainerNotFoundException, NotAllowedException {
ImageNotSupportedException, ContainerNotFoundException, NotAllowedException, DatabaseConnectionException {
if (!hasDatabasePermission(containerId, databaseId, "DATA_INSERT", principal)) {
log.error("Missing data insert permission");
throw new NotAllowedException("Missing data insert permission");
}
queryService.insert(containerId, databaseId, tableId, data);
return ResponseEntity.accepted()
.body(queryService.insert(containerId, databaseId, tableId, data));
}
@PutMapping
@Transactional
@Operation(summary = "Update data", security = @SecurityRequirement(name = "bearerAuth"))
public ResponseEntity<Integer> update(@NotNull @PathVariable("id") Long containerId,
@NotNull @PathVariable("databaseId") Long databaseId,
@NotNull @PathVariable("tableId") Long tableId,
@NotNull @Valid @RequestBody TableCsvUpdateDto data,
@NotNull Principal principal)
throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException,
ImageNotSupportedException, NotAllowedException, ContainerNotFoundException {
if (!hasDatabasePermission(containerId, databaseId, "DATA_UPDATE", principal)) {
log.error("Missing data update permission");
throw new NotAllowedException("Missing data update permission");
}
return ResponseEntity.accepted()
.body(queryService.update(containerId, databaseId, tableId, data));
.build();
}
@DeleteMapping
......@@ -82,7 +63,8 @@ public class TableDataEndpoint extends AbstractEndpoint {
@NotNull @Valid @RequestBody TableCsvDeleteDto data,
@NotNull Principal principal)
throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException,
ImageNotSupportedException, TupleDeleteException, NotAllowedException, ContainerNotFoundException {
ImageNotSupportedException, TupleDeleteException, NotAllowedException, ContainerNotFoundException,
DatabaseConnectionException {
if (!hasDatabasePermission(containerId, databaseId, "DATA_DELETE", principal)) {
log.error("Missing data delete permission");
throw new NotAllowedException("Missing data delete permission");
......@@ -95,20 +77,21 @@ public class TableDataEndpoint extends AbstractEndpoint {
@PostMapping("/import")
@Transactional
@Operation(summary = "Insert data from csv", security = @SecurityRequirement(name = "bearerAuth"))
public ResponseEntity<Integer> importCsv(@NotNull @PathVariable("id") Long containerId,
public ResponseEntity<Void> importCsv(@NotNull @PathVariable("id") Long containerId,
@NotNull @PathVariable("databaseId") Long databaseId,
@NotNull @PathVariable("tableId") Long tableId,
@NotNull @Valid @RequestBody ImportDto data,
@NotNull Principal principal)
throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException,
ImageNotSupportedException, ContainerNotFoundException, NotAllowedException {
ImageNotSupportedException, ContainerNotFoundException, NotAllowedException, DatabaseConnectionException {
if (!hasDatabasePermission(containerId, databaseId, "DATA_INSERT", principal)) {
log.error("Missing data insert permission");
throw new NotAllowedException("Missing data insert permission");
}
log.info("Insert data from location {} into database id {}", data, databaseId);
queryService.insert(containerId, databaseId, tableId, data);
return ResponseEntity.accepted()
.body(queryService.insert(containerId, databaseId, tableId, data));
.build();
}
@RequestMapping(method = {RequestMethod.GET, RequestMethod.HEAD})
......@@ -140,7 +123,7 @@ public class TableDataEndpoint extends AbstractEndpoint {
if (size != null && size <= 0) {
throw new PaginationException("Page number cannot be lower or equal to 0");
}
final BigInteger count = queryService.count(containerId, databaseId, tableId, timestamp);
final Long count = queryService.count(containerId, databaseId, tableId, timestamp);
final HttpHeaders headers = new HttpHeaders();
headers.set("FDA-COUNT", count.toString());
final QueryResultDto response = queryService.findAll(containerId, databaseId, tableId, timestamp, page, size);
......
fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java
+ 24
− 19
Edit View file @ 2f7a67ad
......@@ -23,6 +23,8 @@ import org.mariadb.jdbc.MariaDbBlob;
import org.springframework.transaction.annotation.Transactional;
import java.math.BigInteger;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.text.Normalizer;
import java.time.*;
......@@ -70,23 +72,16 @@ public interface QueryMapper {
return slug.toLowerCase(Locale.ENGLISH);
}
default QueryResultDto resultListToQueryResultDto(List<TableColumn> columns, List<?> result) {
final Iterator<?> iterator = result.iterator();
default QueryResultDto resultListToQueryResultDto(List<TableColumn> columns, ResultSet result) throws SQLException {
final List<Map<String, Object>> resultList = new LinkedList<>();
log.trace("result has {} columns and {} rows", columns.size(), result.size());
while (iterator.hasNext()) {
log.trace("result has {} columns", columns.size());
while (result.next()) {
/* map the result set to the columns through the stored metadata in the metadata database */
int[] idx = new int[]{0};
final Object[] data;
if (columns.size() == 1) {
data = new Object[]{iterator.next()};
} else {
data = (Object[]) iterator.next();
}
final Map<String, Object> map = new HashMap<>();
columns
.forEach(column -> map.put(column.getName(),
dataColumnToObject(data[idx[0]++], column)));
for (int i = 0; i < columns.size(); i++) {
map.put(columns.get(i).getInternalName(), dataColumnToObject(result.getObject(idx[0]++), columns.get(i)));
}
resultList.add(map);
}
return QueryResultDto.builder()
......@@ -576,6 +571,17 @@ public interface QueryMapper {
return statement;
}
default Long resultSetToLong(ResultSet data) {
try {
if (data.next()) {
return data.getLong(1);
}
} catch (SQLException e) {
return null;
}
return null;
}
default String tableToRawFindAllQuery(Table table, Instant timestamp, Long size, Long page)
throws ImageNotSupportedException {
/* param check */
......@@ -614,16 +620,15 @@ public interface QueryMapper {
return query.toString();
}
default QueryResultDto queryTableToQueryResultDto(List<?> result, Table table) throws DateTimeException {
final Iterator<?> iterator = result.iterator();
default QueryResultDto queryTableToQueryResultDto(ResultSet result, Table table) throws DateTimeException, SQLException {
final List<Map<String, Object>> queryResult = new LinkedList<>();
while (iterator.hasNext()) {
while (result.next()) {
/* map the result set to the columns through the stored metadata in the metadata database */
int[] idx = new int[]{0};
final Object[] data = (Object[]) iterator.next();
final Map<String, Object> map = new HashMap<>();
table.getColumns()
.forEach(column -> map.put(column.getInternalName(), dataColumnToObject(data[idx[0]++], column)));
for (int i = 0; i < table.getColumns().size(); i++) {
map.put(table.getColumns().get(i).getInternalName(), dataColumnToObject(result.getObject(idx[0]++), table.getColumns().get(i)));
}
queryResult.add(map);
}
log.info("Selected {} records from table id {}", queryResult.size(), table.getId());
......
fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java
+ 10
− 28
Edit View file @ 2f7a67ad
......@@ -10,7 +10,6 @@ import at.tuwien.api.database.table.TableCsvUpdateDto;
import at.tuwien.exception.*;
import at.tuwien.querystore.Query;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.math.BigInteger;
import java.security.Principal;
......@@ -39,7 +38,7 @@ public interface QueryService {
QueryResultDto execute(Long containerId, Long databaseId, ExecuteStatementDto statement,
Principal principal, Long page, Long size)
throws DatabaseNotFoundException, ImageNotSupportedException, QueryMalformedException, QueryStoreException,
ContainerNotFoundException, ColumnParseException, UserNotFoundException, TableMalformedException;
ContainerNotFoundException, ColumnParseException, UserNotFoundException, TableMalformedException, DatabaseConnectionException;
/**
* Re-Executes an arbitrary query on the database container. We allow the user to only view the data, therefore the
......@@ -62,7 +61,7 @@ public interface QueryService {
*/
QueryResultDto reExecute(Long containerId, Long databaseId, Query query, Long page, Long size)
throws TableNotFoundException, QueryStoreException, QueryMalformedException, DatabaseNotFoundException,
ImageNotSupportedException, ContainerNotFoundException, TableMalformedException, ColumnParseException;
ImageNotSupportedException, ContainerNotFoundException, TableMalformedException, ColumnParseException, DatabaseConnectionException;
/**
......@@ -127,7 +126,7 @@ public interface QueryService {
*/
ExportResource findOne(Long containerId, Long databaseId, Long queryId)
throws DatabaseNotFoundException, ImageNotSupportedException, TableMalformedException,
ContainerNotFoundException, FileStorageException, QueryStoreException, QueryNotFoundException, QueryMalformedException;
ContainerNotFoundException, FileStorageException, QueryStoreException, QueryNotFoundException, QueryMalformedException, DatabaseConnectionException;
/**
* Count the total tuples for a given table id within a container-database id tuple at a given time.
......@@ -143,9 +142,9 @@ public interface QueryService {
* @throws TableMalformedException The table columns are messed up what we got from the metadata database.
* @throws ImageNotSupportedException The image is not supported.
*/
BigInteger count(Long containerId, Long databaseId, Long tableId, Instant timestamp)
Long count(Long containerId, Long databaseId, Long tableId, Instant timestamp)
throws ContainerNotFoundException, DatabaseNotFoundException, TableNotFoundException,
TableMalformedException, ImageNotSupportedException;
TableMalformedException, ImageNotSupportedException, DatabaseConnectionException;
/**
* Insert data from AMQP client into a table of a table-database id tuple, we need the "root" role for this as the
......@@ -155,30 +154,14 @@ public interface QueryService {
* @param databaseId The database id.
* @param tableId The table id.
* @param data The data.
* @return The number of tuples affected.
* @throws ImageNotSupportedException The image is not supported.
* @throws TableMalformedException The table does not exist in the metadata database.
* @throws DatabaseNotFoundException The database is not found in the metadata database.
* @throws TableNotFoundException The table is not found in the metadata database.
* @throws ContainerNotFoundException The container was not found in the metadata database.
*/
Integer insert(Long containerId, Long databaseId, Long tableId, TableCsvDto data) throws ImageNotSupportedException,
TableMalformedException, DatabaseNotFoundException, TableNotFoundException, ContainerNotFoundException;
/**
* @param containerId The container id.
* @param databaseId The database id.
* @param tableId The table id.
* @param data The updated tuple with the list of primary key columns.
* @return The number of records updated.
* @throws ImageNotSupportedException The image is not supported.
* @throws TableMalformedException The table does not exist in the metadata database.
* @throws DatabaseNotFoundException The database is not found in the metadata database.
* @throws TableNotFoundException The table is not found in the metadata database.
*/
Integer update(Long containerId, Long databaseId, Long tableId, TableCsvUpdateDto data)
throws ImageNotSupportedException, TableMalformedException, DatabaseNotFoundException,
TableNotFoundException, ContainerNotFoundException;
void insert(Long containerId, Long databaseId, Long tableId, TableCsvDto data) throws ImageNotSupportedException,
TableMalformedException, DatabaseNotFoundException, TableNotFoundException, ContainerNotFoundException, DatabaseConnectionException;
/**
* Deletes a tuple by given constraint set
......@@ -195,7 +178,7 @@ public interface QueryService {
*/
void delete(Long containerId, Long databaseId, Long tableId, TableCsvDeleteDto data)
throws ImageNotSupportedException, TableMalformedException, DatabaseNotFoundException,
TableNotFoundException, TupleDeleteException, ContainerNotFoundException;
TableNotFoundException, TupleDeleteException, ContainerNotFoundException, DatabaseConnectionException;
/**
* Insert data from a csv into a table of a table-database id tuple, we need the "root" role for this as the
......@@ -204,13 +187,12 @@ public interface QueryService {
* @param databaseId The database id.
* @param tableId The table id.
* @param data The data path.
* @return The number of tuples affected.
* @throws ImageNotSupportedException The image is not supported.
* @throws TableMalformedException The table does not exist in the metadata database.
* @throws DatabaseNotFoundException The database is not found in the metadata database.
* @throws TableNotFoundException The table is not found in the metadata database.
* @throws ContainerNotFoundException The container was not found in the metadata database.
*/
Integer insert(Long containerId, Long databaseId, Long tableId, ImportDto data) throws ImageNotSupportedException,
TableMalformedException, DatabaseNotFoundException, TableNotFoundException, ContainerNotFoundException;
void insert(Long containerId, Long databaseId, Long tableId, ImportDto data) throws ImageNotSupportedException,
TableMalformedException, DatabaseNotFoundException, TableNotFoundException, ContainerNotFoundException, DatabaseConnectionException;
}
fda-query-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java
+ 51
− 33
Edit View file @ 2f7a67ad
......@@ -5,66 +5,84 @@ import at.tuwien.entities.container.image.ContainerImage;
import at.tuwien.entities.container.image.ContainerImageEnvironmentItem;
import at.tuwien.entities.container.image.ContainerImageEnvironmentItemType;
import at.tuwien.entities.database.Database;
import at.tuwien.exception.DatabaseConnectionException;
import com.mchange.v2.c3p0.ComboPooledDataSource;
import lombok.extern.log4j.Log4j2;
import org.hibernate.Session;
import org.hibernate.SessionFactory;
import org.hibernate.Transaction;
import org.hibernate.cfg.Configuration;
import org.hibernate.query.NativeQuery;
import org.springframework.stereotype.Service;
import javax.persistence.PersistenceException;
import java.util.List;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Collection;
import java.util.stream.Collectors;
@Log4j2
@Service
public abstract class HibernateConnector {
protected static Session getCurrentSession(ContainerImage image, Container container, Database database) {
final String url = "jdbc:" + image.getJdbcMethod() + "://" + container.getInternalName() + "/" + database.getInternalName();
protected static Connection getConnection(ContainerImage image, Container container, Database database) throws DatabaseConnectionException {
final ComboPooledDataSource dataSource = new ComboPooledDataSource();
final String url = "jdbc:" + image.getJdbcMethod() + "://" + container.getInternalName() + "/" + (database != null ? database.getInternalName() : "");
dataSource.setJdbcUrl(url);
final String username = image.getEnvironment()
.stream()
.filter(e -> e.getType().equals(ContainerImageEnvironmentItemType.PRIVILEGED_USERNAME))
.map(ContainerImageEnvironmentItem::getValue)
.collect(Collectors.toList())
.get(0);
dataSource.setUser(username);
final String password = image.getEnvironment()
.stream()
.filter(e -> e.getType().equals(ContainerImageEnvironmentItemType.PRIVILEGED_PASSWORD))
.map(ContainerImageEnvironmentItem::getValue)
.collect(Collectors.toList())
.get(0);
dataSource.setPassword(password);
dataSource.setInitialPoolSize(5);
dataSource.setMinPoolSize(5);
dataSource.setAcquireIncrement(5);
dataSource.setMaxPoolSize(20);
dataSource.setMaxStatements(100);
final Connection connection;
try {
connection = dataSource.getConnection();
} catch (SQLException e) {
log.error("Failed to connect to the database");
log.debug("failed to connect to the database {}", database);
throw new DatabaseConnectionException("Failed to connect to the database");
}
return connection;
}
final Configuration config = new Configuration();
config.configure("mariadb_hibernate.cfg.xml");
config.setProperty("hibernate.connection.url", url);
config.setProperty("hibernate.connection.username", username);
config.setProperty("hibernate.connection.password", password);
config.setProperty("hibernate.connection.driver_class", image.getDriverClass());
config.setProperty("hibernate.dialect", image.getDialect());
final SessionFactory sessionFactory = config.buildSessionFactory();
Session session = sessionFactory.getCurrentSession();
if (!session.isOpen()) {
log.debug("Session is closed, opening...");
session = sessionFactory.openSession();
protected static Long activeConnection(Connection connection) throws DatabaseConnectionException {
final ResultSet resultSet = execute(connection, "SHOW STATUS LIKE 'threads_connected'");
try {
if (resultSet.next()) {
return resultSet.getLong(2);
}
return session;
} catch (SQLException e) {
log.error("Failed to determine active connections");
throw new DatabaseConnectionException("Failed to determine active connections", e);
}
log.error("Failed to determine active connections");
throw new DatabaseConnectionException("Failed to determine active connections");
}
protected static ResultSet execute(Connection connection, String statement) throws DatabaseConnectionException {
return execute(connection, statement, null);
}
protected static Long activeConnection(Session session) {
final NativeQuery<?> nativeQuery = session.createSQLQuery("SHOW STATUS LIKE 'threads_connected'");
final List<?> result;
protected static ResultSet execute(Connection connection, String statement, Collection<Object> data) throws DatabaseConnectionException {
final PreparedStatement preparedStatement;
try {
result = nativeQuery.getResultList();
} catch (PersistenceException e) {
log.error("Failed to collect number of used connections");
/* ignore */
return null;
preparedStatement = connection.prepareStatement(statement, data);
return preparedStatement.executeQuery();
} catch (SQLException e) {
log.error("Failed to execute statement");
log.debug("failed to execute statement {}", statement);
throw new DatabaseConnectionException("Failed to execute statement", e);
}
final Object[] row = (Object[]) result.get(0);
log.debug("current number of connections: {}", Long.parseLong(String.valueOf(row[1])));
return Long.parseLong(String.valueOf(row[1]));
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment