Skip to content
Snippets Groups Projects
Commit 8be84589 authored by Martin Weise's avatar Martin Weise
Browse files

Merge branch 'bugfix/connect-frontend-backend' into 89-zenodo-sandbox-integration-for-pid-e-g-doi

parents 96ee52b8 0992cb74
Branches
Tags
4 merge requests!81New stable release,!43Merge dev to master,!27Draft: Resolve "Zenodo Sandbox integration for PID (e.g. DOI)",!26Draft: Resolve "Task: Prepare tests for Sprint 2 Milestone"
Expand all Show whitespace changes
Inline Side-by-side
Showing
with 56043 additions and 316 deletions
fda-query-service/services/src/main/java/at/tuwien/service/PostgresService.java deleted 100644 → 0
+ 0
− 117
View file @ 96ee52b8
package at.tuwien.service;
import at.tuwien.entities.database.Database;
import at.tuwien.entities.database.query.Query;
import at.tuwien.exception.DatabaseConnectionException;
import at.tuwien.exception.QueryMalformedException;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.*;
@Log4j2
@Service
public class PostgresService extends JdbcConnector {
private final Properties postgresProperties;
@Autowired
protected PostgresService(Properties postgresProperties) {
this.postgresProperties = postgresProperties;
}
private Connection getConnection(Database database) throws DatabaseConnectionException {
Connection connection;
final String URL = "jdbc:postgresql://" + database.getContainer().getInternalName() + ":"
+ database.getContainer().getImage().getDefaultPort() + "/" + database.getInternalName();
try {
connection = open(URL, postgresProperties);
} catch (SQLException e) {
log.error("Could not connect to the database container, is it running from Docker container? IT DOES NOT WORK FROM IDE! URL: {} Params: {}", URL, postgresProperties);
throw new DatabaseConnectionException("Could not connect to the database container, is it running?", e);
}
return connection;
}
public void createQuerystore(Database database) throws DatabaseConnectionException {
try {
final PreparedStatement statement = getCreateQueryStoreStatement(getConnection(database));
statement.execute();
} catch (SQLException e) {
log.error("The SQL statement seems to contain invalid syntax");
}
}
@Override
List<Query> getQueries(Database database) throws DatabaseConnectionException, QueryMalformedException {
log.debug("Get Queries from Querystore");
final String getQueries = "SELECT id,query, query_normalized, execution_timestamp FROM querystore ORDER BY execution_timestamp desc;";
List<Query> results = new ArrayList<>();
Connection connection = getConnection(database);
try {
PreparedStatement statement = connection.prepareStatement(getQueries);
ResultSet result = statement.executeQuery();
while (result.next()) {
results.add(Query.builder()
.id(result.getLong("id"))
.query(result.getString("query"))
.queryNormalized(result.getString("query_normalized"))
.executionTimestamp(result.getTimestamp("execution_timestamp"))
.build());
}
} catch (SQLException e) {
log.error("sql failed: {}", e.getMessage());
throw new QueryMalformedException("sql malformed", e);
}
return results;
}
@Override
public Boolean saveQuery(Database database, Query query) {
log.debug("Save Query {}", query.toString());
final String saveQuery = "INSERT INTO querystore(query,query_normalized, query_hash, execution_timestamp, result_hash, result_number) VALUES(?,?,?,?,?,?);";
try {
Connection connection = getConnection(database);
PreparedStatement statement = connection.prepareStatement(saveQuery);
System.out.println(query.toString());
statement.setString(1, "'" + query.getQuery() + "'");
statement.setString(2, "'" + query.getQueryNormalized() + "'");
statement.setString(3, query.getQueryHash());
statement.setTimestamp(4, query.getExecutionTimestamp());
statement.setString(5, query.getResultHash());
statement.setInt(6, query.getResultNumber());
log.debug(statement.toString());
return statement.execute();
} catch (SQLException e) {
log.error("The SQL statement seems to contain invalid syntax");
} catch (DatabaseConnectionException e) {
log.error("Problem with connecting to the database while inserting into Querystore");
}
return false;
}
@Override
PreparedStatement getCreateQueryStoreStatement(Connection connection) throws SQLException {
log.debug("create querystore");
final String createQuery = "CREATE TABLE IF NOT EXISTS querystore (" +
" id serial PRIMARY KEY," +
" query text," +
" query_normalized text," +
" query_hash text," +
" execution_timestamp timestamp," +
" result_hash text," +
" result_number integer" +
" );";
log.debug("compiled query as \"{}\"", createQuery);
return connection.prepareStatement(createQuery);
}
}
fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java
+ 205
− 68
View file @ 8be84589
...@@ -3,115 +3,252 @@ package at.tuwien.service; ...@@ -3,115 +3,252 @@ package at.tuwien.service;
import at.tuwien.api.database.query.QueryResultDto; import at.tuwien.api.database.query.QueryResultDto;
import at.tuwien.entities.database.Database; import at.tuwien.entities.database.Database;
import at.tuwien.entities.database.query.Query; import at.tuwien.entities.database.query.Query;
import at.tuwien.exception.DatabaseConnectionException; import at.tuwien.entities.database.table.Table;
import at.tuwien.exception.DatabaseNotFoundException; import at.tuwien.entities.database.table.columns.TableColumn;
import at.tuwien.exception.ImageNotSupportedException; import at.tuwien.exception.*;
import at.tuwien.exception.QueryMalformedException; import at.tuwien.mapper.ImageMapper;
import at.tuwien.repository.DatabaseRepository; import at.tuwien.mapper.QueryMapper;
import at.tuwien.repository.jpa.DatabaseRepository;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.parser.CCJSqlParserManager; import net.sf.jsqlparser.parser.CCJSqlParserManager;
import net.sf.jsqlparser.statement.Statement; import net.sf.jsqlparser.statement.Statement;
import net.sf.jsqlparser.statement.select.PlainSelect; import net.sf.jsqlparser.statement.select.*;
import net.sf.jsqlparser.statement.select.Select; import org.jooq.DSLContext;
import net.sf.jsqlparser.statement.select.SelectItem; import org.jooq.Record;
import org.jooq.Result;
import org.jooq.ResultQuery;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import javax.persistence.EntityNotFoundException; import java.awt.print.Pageable;
import java.io.StringReader; import java.io.StringReader;
import java.sql.SQLFeatureNotSupportedException; import java.sql.SQLException;
import java.sql.Timestamp; import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Locale;
import java.util.Optional; import java.util.Optional;
import static org.jooq.impl.DSL.constraint;
@Log4j2 @Log4j2
@Service @Service
public class QueryService { public class QueryService extends JdbcConnector {
private final DatabaseRepository databaseRepository; private final DatabaseRepository databaseRepository;
private final PostgresService postgresService; private final QueryStoreService queryStoreService;
private final QueryMapper queryMapper;
@Autowired @Autowired
public QueryService(DatabaseRepository databaseRepository, PostgresService postgresService) { public QueryService(ImageMapper imageMapper, QueryMapper queryMapper, DatabaseRepository databaseRepository, QueryStoreService queryStoreService) {
super(imageMapper, queryMapper);
this.databaseRepository = databaseRepository; this.databaseRepository = databaseRepository;
this.postgresService = postgresService; this.queryStoreService = queryStoreService;
this.queryMapper = queryMapper;
} }
@Transactional @Transactional
public List<Query> findAll(Long id) throws ImageNotSupportedException, DatabaseNotFoundException, DatabaseConnectionException, QueryMalformedException { public QueryResultDto execute(Long id, Query query) throws ImageNotSupportedException, DatabaseNotFoundException, JSQLParserException, SQLException, QueryMalformedException, QueryStoreException {
return postgresService.getQueries(findDatabase(id)); Database database = findDatabase(id);
if(database.getContainer().getImage().getDialect().equals("MARIADB")){
if(!queryStoreService.exists(database)) {
queryStoreService.create(id);
}
}
DSLContext context = open(database);
//TODO Fix that import
StringBuilder parsedQuery = new StringBuilder();
String q = parse(query, database).getQuery();
if(q.charAt(q.length()-1) == ';') {
parsedQuery.append(q.substring(0, q.length()-2));
} else {
parsedQuery.append(q);
}
parsedQuery.append(";");
ResultQuery<Record> resultQuery = context.resultQuery(parsedQuery.toString());
Result<Record> result = resultQuery.fetch();
QueryResultDto queryResultDto = queryMapper.recordListToQueryResultDto(result);
log.debug("Result of the query is: \n {}", result.toString());
// Save the query in the store
boolean b = queryStoreService.saveQuery(database, query, queryResultDto);
log.debug("Save query returned code {}", b);
return queryResultDto;
} }
public QueryResultDto executeStatement(Long id, Query query) throws ImageNotSupportedException, DatabaseNotFoundException, JSQLParserException, SQLFeatureNotSupportedException { private Query parse(Query query, Database database) throws SQLException, ImageNotSupportedException, JSQLParserException {
Timestamp ts = new Timestamp(System.currentTimeMillis());
query.setExecutionTimestamp(ts);
CCJSqlParserManager parserRealSql = new CCJSqlParserManager(); CCJSqlParserManager parserRealSql = new CCJSqlParserManager();
Statement statement = parserRealSql.parse(new StringReader(query.getQuery()));
log.debug("Given query {}", query.getQuery());
Statement stmt = parserRealSql.parse(new StringReader(query.getQuery())); if(statement instanceof Select) {
Database database = findDatabase(id); Select selectStatement = (Select) statement;
if (stmt instanceof Select) {
Select selectStatement = (Select) stmt;
PlainSelect ps = (PlainSelect)selectStatement.getSelectBody(); PlainSelect ps = (PlainSelect)selectStatement.getSelectBody();
List<SelectItem> selectItems = ps.getSelectItems();
List<SelectItem> selectitems = ps.getSelectItems();
System.out.println(ps.getFromItem().toString()); //Parse all tables
selectitems.stream().forEach(selectItem -> System.out.println(selectItem.toString())); List<FromItem> fromItems = new ArrayList<>();
} else { fromItems.add(ps.getFromItem());
throw new SQLFeatureNotSupportedException("SQL Query is not a SELECT statement - please only use SELECT statements"); if(ps.getJoins() != null && ps.getJoins().size() > 0) {
for (Join j : ps.getJoins()) {
if (j.getRightItem() != null) {
fromItems.add(j.getRightItem());
} }
saveQuery(database, query, null);
return null;
} }
}
public void create(Long id) throws DatabaseConnectionException, ImageNotSupportedException, DatabaseNotFoundException { //Checking if all tables exist
postgresService.createQuerystore(findDatabase(id)); List<TableColumn> allColumns = new ArrayList<>();
for(FromItem f : fromItems) {
boolean i = false;
log.debug("from item iterated through: {}", f);
for(Table t : database.getTables()) {
if(f.toString().equals(t.getInternalName()) || f.toString().equals(t.getName())) {
allColumns.addAll(t.getColumns());
i=false;
break;
}
i = true;
}
if(i) {
throw new JSQLParserException("Table "+f.toString() + " does not exist");
}
} }
/* helper functions */ //Checking if all columns exist
for(SelectItem s : selectItems) {
String select = s.toString();
if(select.trim().equals("*")) {
log.debug("Please do not use * to query data");
continue;
}
// ignore prefixes
if(select.contains(".")) {
log.debug(select);
select = select.split("\\.")[1];
}
boolean i = false;
for(TableColumn tc : allColumns ) {
log.debug("{},{},{}", tc.getInternalName(), tc.getName(), s);
if(select.equals(tc.getInternalName()) || select.toString().equals(tc.getName())) {
i=false;
break;
}
i = true;
}
if(i) {
throw new JSQLParserException("Column "+s.toString() + " does not exist");
}
}
//TODO Future work
if(ps.getWhere() != null) {
Expression where = ps.getWhere();
log.debug("Where clause: {}", where);
}
return query;
}
else {
throw new JSQLParserException("SQL Query is not a SELECT statement - please only use SELECT statements");
}
private Database findDatabase(Long id) throws DatabaseNotFoundException, ImageNotSupportedException {
final Optional<Database> database;
try {
database = databaseRepository.findById(id);
} catch (EntityNotFoundException e) {
log.error("database not found in metadata database");
throw new DatabaseNotFoundException("database not found in metadata database", e);
} }
@Transactional
public Database findDatabase(Long id) throws DatabaseNotFoundException {
final Optional<Database> database = databaseRepository.findById(id);
if (database.isEmpty()) { if (database.isEmpty()) {
log.error("no database with this id found in metadata database"); log.error("no database with this id found in metadata database");
throw new DatabaseNotFoundException("database not found in metadata database"); throw new DatabaseNotFoundException("database not found in metadata database");
} }
log.debug("retrieved db {}", database);
if (!database.get().getContainer().getImage().getRepository().equals("postgres")) {
log.error("Right now only PostgreSQL is supported!");
throw new ImageNotSupportedException("Currently only PostgreSQL is supported");
}
return database.get(); return database.get();
} }
private Query saveQuery(Database database, Query query, QueryResultDto queryResult) {
//TODO in next sprint public QueryResultDto reexecute(Long databaseId, Long queryId, Integer page, Integer size) throws DatabaseNotFoundException, SQLException, ImageNotSupportedException {
String q = query.getQuery(); log.info("re-execute query with the id {}", queryId);
query.setExecutionTimestamp(new Timestamp(System.currentTimeMillis())); DSLContext context = open(findDatabase(databaseId));
query.setQueryNormalized(normalizeQuery(query.getQuery())); QueryResultDto savedQuery = queryStoreService.findOne(databaseId, queryId);
query.setQueryHash(query.getQueryNormalized().hashCode() + ""); StringBuilder query = new StringBuilder();
query.setResultHash(query.getQueryHash()); query.append("SELECT * FROM (");
query.setResultNumber(0); String q = (String)savedQuery.getResult().get(0).get("query");
postgresService.saveQuery(database, query); if(q.toLowerCase(Locale.ROOT).contains("where")) {
return null; String[] split = q.toLowerCase(Locale.ROOT).split("where");
if(split.length > 2) {
//TODO FIX SUBQUERIES WITH MULTIPLE Wheres
throw new SQLException("Query Contains Subqueries, this will be supported in a future version");
} else {
query.append(split[0]);
query.append(" FOR SYSTEM_TIME AS OF TIMESTAMP'");
Timestamp t = (Timestamp)savedQuery.getResult().get(0).get("execution_timestamp");
query.append(t.toLocalDateTime().toString());
query.append("' WHERE");
query.append(split[1]);
query.append(") as tab");
} }
} else {
query.append(q);
query.append(" FOR SYSTEM_TIME AS OF TIMESTAMP'");
Timestamp t = (Timestamp)savedQuery.getResult().get(0).get("execution_timestamp");
private String normalizeQuery(String query) { query.append(t.toLocalDateTime().toString());
return query; query.append("') as tab");
} }
private boolean checkValidity(String query) {
String queryparts[] = query.toLowerCase().split("from"); if(page != null && size != null) {
if (queryparts[0].contains("select")) { page = Math.abs(page);
//TODO add more checks size = Math.abs(size);
return true; query.append(" LIMIT ");
query.append(size);
query.append(" OFFSET ");
query.append(page * size);
}
query.append(";");
log.debug(query.toString());
ResultQuery<Record> resultQuery = context.resultQuery(query.toString());
Result<Record> result = resultQuery.fetch();
QueryResultDto queryResultDto = queryMapper.recordListToQueryResultDto(result);
log.debug(result.toString());
return queryResultDto;
}
@Transactional
public QueryResultDto save(Long id, Query query) throws SQLException, ImageNotSupportedException, DatabaseNotFoundException, QueryStoreException, JSQLParserException {
Database database = findDatabase(id);
if(database.getContainer().getImage().getDialect().equals("MARIADB")){
if(!queryStoreService.exists(database)) {
queryStoreService.create(id);
}
}
DSLContext context = open(database);
StringBuilder parsedQuery = new StringBuilder();
String q = parse(query, database).getQuery();
if(q.charAt(q.length()-1) == ';') {
parsedQuery.append(q.substring(0, q.length()-2));
} else {
parsedQuery.append(q);
} }
return false; parsedQuery.append(";");
ResultQuery<Record> resultQuery = context.resultQuery(parsedQuery.toString());
Result<Record> result = resultQuery.fetch();
QueryResultDto queryResultDto = queryMapper.recordListToQueryResultDto(result);
log.debug("Result of the query is: \n {}", result.toString());
// Save the query in the store
boolean b = queryStoreService.saveQuery(database, query, queryResultDto);
log.debug("Save query returned code {}", b);
QueryResultDto savedQuery = queryStoreService.findLast(database.getId());
return savedQuery;
} }
} }
fda-query-service/services/src/main/java/at/tuwien/service/QueryStoreService.java 0 → 100644
+ 182
− 0
View file @ 8be84589
package at.tuwien.service;
import at.tuwien.api.database.query.QueryResultDto;
import at.tuwien.entities.database.Database;
import at.tuwien.entities.database.query.Query;
import at.tuwien.exception.*;
import at.tuwien.mapper.ImageMapper;
import at.tuwien.mapper.QueryMapper;
import at.tuwien.repository.jpa.DatabaseRepository;
import lombok.extern.log4j.Log4j2;
import org.jooq.*;
import org.jooq.impl.DSL;
import org.junit.jupiter.params.shadow.com.univocity.parsers.common.DataProcessingException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
import java.util.Optional;
import static org.jooq.impl.DSL.*;
import static org.jooq.impl.SQLDataType.*;
@Log4j2
@Service
public class QueryStoreService extends JdbcConnector {
private final DatabaseRepository databaseRepository;
private final String QUERYSTORENAME = "mdb_querystore";
private final QueryMapper queryMapper;
@Autowired
public QueryStoreService(ImageMapper imageMapper, QueryMapper queryMapper, DatabaseRepository databaseRepository) {
super(imageMapper, queryMapper);
this.databaseRepository = databaseRepository;
this.queryMapper = queryMapper;
}
@Transactional
public List<Query> findAll(Long id) throws ImageNotSupportedException, DatabaseNotFoundException, DatabaseConnectionException, QueryMalformedException, SQLException, QueryStoreException {
Database database = findDatabase(id);
if(!exists(database)){
create(id);
}
DSLContext context = open(database);
ResultQuery<org.jooq.Record> resultQuery = context.selectQuery();
Result<org.jooq.Record> result = resultQuery.fetch();
log.debug(result.toString());
return queryMapper.recordListToQueryList(context
.selectFrom(QUERYSTORENAME) //TODO Order after timestamps
.fetch());
}
public QueryResultDto findOne(Long databaseId, Long queryId) throws DatabaseNotFoundException, SQLException, ImageNotSupportedException {
Database database = findDatabase(databaseId);
DSLContext context = open(database);
ResultQuery<org.jooq.Record> resultQuery = context.selectQuery();
Result<org.jooq.Record> result = resultQuery.fetch();
log.debug(result.toString());
return queryMapper.recordListToQueryResultDto(context
.selectFrom(QUERYSTORENAME).where("id = "+ queryId)
.fetch());
}
public QueryResultDto findLast(Long databaseId) throws DatabaseNotFoundException, SQLException, ImageNotSupportedException {
Database database = findDatabase(databaseId);
DSLContext context = open(database);
StringBuilder sb = new StringBuilder();
sb.append("SELECT * FROM ");
sb.append(QUERYSTORENAME);
sb.append(" ORDER BY id desc LIMIT 1;");
return queryMapper.recordListToQueryResultDto(context
.fetch(sb.toString()));
}
/**
* Creates the querystore for a given database
* @param databaseId
* @throws ImageNotSupportedException
* @throws DatabaseNotFoundException
*/
@Transactional
public void create(Long databaseId) throws ImageNotSupportedException, DatabaseNotFoundException, QueryStoreException, SQLException {
log.info("Create QueryStore");
final Database database = findDatabase(databaseId);
if(exists(database)) {
log.info("Querystore already exists");
throw new QueryStoreException("Querystore already exists");
}
/* create database in container */
try {
final DSLContext context = open(database);
context.createTable(QUERYSTORENAME)
.column("id", INTEGER)
.column("doi", VARCHAR(255).nullable(false))
.column("query", VARCHAR(255).nullable(false))
.column("query_hash", VARCHAR(255))
.column("execution_timestamp", TIMESTAMP)
.column("result_hash", VARCHAR(255))
.column("result_number", INTEGER)
.constraints(
constraint("pk").primaryKey("id"),
constraint("uk").unique("doi")
)
.execute();
} catch (SQLException e) {
throw new DataProcessingException("could not create table", e);
}
}
public boolean saveQuery(Database database, Query query, QueryResultDto queryResult) throws SQLException, ImageNotSupportedException {
log.debug("Save Query");
String q = query.getQuery();
query.setExecutionTimestamp(new Timestamp(System.currentTimeMillis()));
query.setQueryNormalized(normalizeQuery(query.getQuery()));
query.setQueryHash(query.getQueryNormalized().toLowerCase(Locale.ROOT).hashCode() + "");
query.setResultHash(query.getQueryHash());
query.setResultNumber(0);
query.setId(Long.valueOf(maxId(database))+1);
DSLContext context = open(database);
int success = context.insertInto(table(QUERYSTORENAME))
.columns(field("id"),
field("doi"),
field("query"),
field("query_hash"),
field("execution_timestamp"),
field("result_hash"),
field("result_number"))
.values(query.getId(), "doi/"+query.getId(), query.getQuery(), query.getQueryHash(), query.getExecutionTimestamp(), ""+queryResult.hashCode(), queryResult.getResult().size()).execute();
return success == 1 ? true : false;
}
public Integer maxId(Database database) throws SQLException, ImageNotSupportedException {
DSLContext context = open(database);
QueryResultDto queryResultDto = queryMapper.recordListToQueryResultDto(context
.selectFrom(QUERYSTORENAME).orderBy(field("id").desc()).limit(1)
.fetch());
if(queryResultDto.getResult() == null || queryResultDto.getResult().size() == 0) {
return 0;
}
return (Integer) queryResultDto.getResult().get(0).get("id");
}
private String normalizeQuery(String query) {
return query;
}
private boolean checkValidity(String query) {
String queryparts[] = query.toLowerCase().split("from");
if (queryparts[0].contains("select")) {
//TODO add more checks
return true;
}
return false;
}
@Transactional
public Database findDatabase(Long id) throws DatabaseNotFoundException {
final Optional<Database> database = databaseRepository.findById(id);
if (database.isEmpty()) {
log.error("no database with this id found in metadata database");
throw new DatabaseNotFoundException("database not found in metadata database");
}
return database.get();
}
public boolean exists(Database database) throws SQLException, ImageNotSupportedException {
final DSLContext context = open(database);
return context.select(count())
.from("information_schema.tables")
.where("table_name like '" + QUERYSTORENAME + "'")
.fetchOne(0, int.class) == 1;
}
}
fda-table-service/.gitignore
+ 1
− 0
View file @ 8be84589
...@@ -8,6 +8,7 @@ target/ ...@@ -8,6 +8,7 @@ target/
.env .env
### Generated ### ### Generated ###
ready
mapping.xml mapping.xml
rest-service/src/main/java/at/tuwien/userdb/Table.java rest-service/src/main/java/at/tuwien/userdb/Table.java
......
fda-table-service/pom.xml
+ 5
− 6
View file @ 8be84589
...@@ -74,6 +74,10 @@ ...@@ -74,6 +74,10 @@
<groupId>org.postgresql</groupId> <groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId> <artifactId>postgresql</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-elasticsearch</artifactId>
</dependency>
<dependency> <dependency>
<groupId>org.mariadb.jdbc</groupId> <groupId>org.mariadb.jdbc</groupId>
<artifactId>mariadb-java-client</artifactId> <artifactId>mariadb-java-client</artifactId>
...@@ -183,12 +187,6 @@ ...@@ -183,12 +187,6 @@
<artifactId>swagger-codegen-maven-plugin</artifactId> <artifactId>swagger-codegen-maven-plugin</artifactId>
<version>2.4.21</version> <version>2.4.21</version>
</dependency> </dependency>
<!-- Gateway -->
<dependency>
<groupId>org.projectreactor</groupId>
<artifactId>reactor-spring</artifactId>
<version>1.0.1.RELEASE</version>
</dependency>
</dependencies> </dependencies>
<build> <build>
...@@ -203,6 +201,7 @@ ...@@ -203,6 +201,7 @@
<exclude>at/tuwien/exception/**/*</exclude> <exclude>at/tuwien/exception/**/*</exclude>
<exclude>at/tuwien/utils/**/*</exclude> <exclude>at/tuwien/utils/**/*</exclude>
<exclude>at/tuwien/config/**/*</exclude> <exclude>at/tuwien/config/**/*</exclude>
<exclude>at/tuwien/handlers/**/*</exclude>
<exclude>**/FdaTableServiceApplication.class</exclude> <exclude>**/FdaTableServiceApplication.class</exclude>
<exclude>**/JdbcConnector.class</exclude> <exclude>**/JdbcConnector.class</exclude>
</excludes> </excludes>
......
fda-table-service/rest-service/src/main/java/at/tuwien/FdaTableServiceApplication.java
+ 3
− 1
View file @ 8be84589
...@@ -3,6 +3,7 @@ package at.tuwien; ...@@ -3,6 +3,7 @@ package at.tuwien;
import org.springframework.boot.SpringApplication; import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.autoconfigure.domain.EntityScan;
import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories;
import org.springframework.data.jpa.repository.config.EnableJpaAuditing; import org.springframework.data.jpa.repository.config.EnableJpaAuditing;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.transaction.annotation.EnableTransactionManagement; import org.springframework.transaction.annotation.EnableTransactionManagement;
...@@ -13,7 +14,8 @@ import springfox.documentation.oas.annotations.EnableOpenApi; ...@@ -13,7 +14,8 @@ import springfox.documentation.oas.annotations.EnableOpenApi;
@EnableJpaAuditing @EnableJpaAuditing
@EnableOpenApi @EnableOpenApi
@EnableTransactionManagement @EnableTransactionManagement
@EnableJpaRepositories(basePackages = {"at.tuwien.repository"}) @EnableElasticsearchRepositories(basePackages = {"at.tuwien.repository.elastic"})
@EnableJpaRepositories(basePackages = {"at.tuwien.repository.jpa"})
@EntityScan(basePackages = {"at.tuwien.entities"}) @EntityScan(basePackages = {"at.tuwien.entities"})
public class FdaTableServiceApplication { public class FdaTableServiceApplication {
......
fda-table-service/rest-service/src/main/java/at/tuwien/endpoints/DataEndpoint.java
+ 6
− 9
View file @ 8be84589
...@@ -18,6 +18,7 @@ import org.springframework.transaction.annotation.Transactional; ...@@ -18,6 +18,7 @@ import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
import javax.validation.Valid; import javax.validation.Valid;
import java.sql.Timestamp;
@Log4j2 @Log4j2
@CrossOrigin(origins = "*") @CrossOrigin(origins = "*")
...@@ -66,13 +67,8 @@ public class DataEndpoint { ...@@ -66,13 +67,8 @@ public class DataEndpoint {
public ResponseEntity<?> insertFromTuple(@PathVariable("id") Long databaseId, public ResponseEntity<?> insertFromTuple(@PathVariable("id") Long databaseId,
@PathVariable("tableId") Long tableId, @PathVariable("tableId") Long tableId,
@Valid @RequestBody TableCsvDto data) throws ImageNotSupportedException, @Valid @RequestBody TableCsvDto data) throws ImageNotSupportedException,
TableMalformedException { TableMalformedException, TableNotFoundException, DatabaseNotFoundException {
final Table table = Table.builder() final Table table = dataService.findById(databaseId, tableId);
.id(tableId)
.database(Database.builder()
.id(databaseId)
.build())
.build();
dataService.insert(table, data); dataService.insert(table, data);
return ResponseEntity.accepted() return ResponseEntity.accepted()
.build(); .build();
...@@ -88,10 +84,11 @@ public class DataEndpoint { ...@@ -88,10 +84,11 @@ public class DataEndpoint {
@ApiResponse(code = 405, message = "The connection to the database was unsuccessful."), @ApiResponse(code = 405, message = "The connection to the database was unsuccessful."),
}) })
public ResponseEntity<QueryResultDto> getAll(@PathVariable("id") Long databaseId, public ResponseEntity<QueryResultDto> getAll(@PathVariable("id") Long databaseId,
@PathVariable("tableId") Long tableId) throws TableNotFoundException, @PathVariable("tableId") Long tableId, @RequestParam(required = false) Timestamp timestamp, @RequestParam(name="page", required= false) Integer page, @RequestParam(name = "size", required = false) Integer size) throws TableNotFoundException,
DatabaseNotFoundException, DatabaseConnectionException, ImageNotSupportedException { DatabaseNotFoundException, DatabaseConnectionException, ImageNotSupportedException {
final QueryResultDto data = dataService.selectAll(databaseId, tableId); final QueryResultDto data = dataService.selectAll(databaseId, tableId, timestamp, page, size);
return ResponseEntity.ok(data); return ResponseEntity.ok(data);
} }
} }
fda-table-service/rest-service/src/test/java/at/tuwien/BaseUnitTest.java
+ 68
− 47
View file @ 8be84589
package at.tuwien; package at.tuwien;
import at.tuwien.api.database.table.TableCreateDto; import at.tuwien.api.database.table.TableCreateDto;
import at.tuwien.api.database.table.TableDto;
import at.tuwien.api.database.table.columns.ColumnCreateDto; import at.tuwien.api.database.table.columns.ColumnCreateDto;
import at.tuwien.api.database.table.columns.ColumnDto;
import at.tuwien.api.database.table.columns.ColumnTypeDto; import at.tuwien.api.database.table.columns.ColumnTypeDto;
import at.tuwien.entities.container.Container; import at.tuwien.entities.container.Container;
import at.tuwien.entities.container.image.ContainerImage; import at.tuwien.entities.container.image.ContainerImage;
...@@ -21,7 +19,7 @@ import java.util.List; ...@@ -21,7 +19,7 @@ import java.util.List;
import static java.time.temporal.ChronoUnit.HOURS; import static java.time.temporal.ChronoUnit.HOURS;
@TestPropertySource(locations = "classpath:application.properties") @TestPropertySource(locations = "classpath:application.properties")
public abstract class BaseUnitTest { public abstract class BaseUnitTest extends CsvUnitTest {
public final static Long DATABASE_1_ID = 1L; public final static Long DATABASE_1_ID = 1L;
public final static String DATABASE_1_NAME = "Weather"; public final static String DATABASE_1_NAME = "Weather";
...@@ -33,6 +31,11 @@ public abstract class BaseUnitTest { ...@@ -33,6 +31,11 @@ public abstract class BaseUnitTest {
public final static String DATABASE_2_INTERNALNAME = "river"; public final static String DATABASE_2_INTERNALNAME = "river";
public final static String DATABASE_2_EXCHANGE = "fda." + DATABASE_2_INTERNALNAME; public final static String DATABASE_2_EXCHANGE = "fda." + DATABASE_2_INTERNALNAME;
public final static Long DATABASE_3_ID = 3L;
public final static String DATABASE_3_NAME = "Csv Table";
public final static String DATABASE_3_INTERNALNAME = "csv_table";
public final static String DATABASE_3_EXCHANGE = "fda." + DATABASE_3_INTERNALNAME;
public final static Long TABLE_1_ID = 1L; public final static Long TABLE_1_ID = 1L;
public final static String TABLE_1_NAME = "Weather AUS"; public final static String TABLE_1_NAME = "Weather AUS";
public final static String TABLE_1_INTERNALNAME = "weather_aus"; public final static String TABLE_1_INTERNALNAME = "weather_aus";
...@@ -45,6 +48,12 @@ public abstract class BaseUnitTest { ...@@ -45,6 +48,12 @@ public abstract class BaseUnitTest {
public final static String TABLE_2_DESCRIPTION = "River Data"; public final static String TABLE_2_DESCRIPTION = "River Data";
public final static String TABLE_2_TOPIC = DATABASE_2_EXCHANGE + "." + TABLE_2_INTERNALNAME; public final static String TABLE_2_TOPIC = DATABASE_2_EXCHANGE + "." + TABLE_2_INTERNALNAME;
public final static Long TABLE_3_ID = 3L;
public final static String TABLE_3_NAME = "Csv Table";
public final static String TABLE_3_INTERNALNAME = "csv_table";
public final static String TABLE_3_DESCRIPTION = "CSV Is Nice";
public final static String TABLE_3_TOPIC = DATABASE_3_EXCHANGE + "." + TABLE_3_INTERNALNAME;
public final static Long COLUMN_1_ID = 1L; public final static Long COLUMN_1_ID = 1L;
public final static Integer COLUMN_1_ORDINALPOS = 0; public final static Integer COLUMN_1_ORDINALPOS = 0;
public final static Boolean COLUMN_1_PRIMARY = true; public final static Boolean COLUMN_1_PRIMARY = true;
...@@ -75,7 +84,7 @@ public abstract class BaseUnitTest { ...@@ -75,7 +84,7 @@ public abstract class BaseUnitTest {
public final static Integer COLUMN_3_ORDINALPOS = 2; public final static Integer COLUMN_3_ORDINALPOS = 2;
public final static Boolean COLUMN_3_PRIMARY = false; public final static Boolean COLUMN_3_PRIMARY = false;
public final static String COLUMN_3_NAME = "MinTemp"; public final static String COLUMN_3_NAME = "MinTemp";
public final static String COLUMN_3_INTERNAL_NAME = "mdb_min_temp"; public final static String COLUMN_3_INTERNAL_NAME = "mdb_mintemp";
public final static TableColumnType COLUMN_3_TYPE = TableColumnType.NUMBER; public final static TableColumnType COLUMN_3_TYPE = TableColumnType.NUMBER;
public final static ColumnTypeDto COLUMN_3_TYPE_DTO = ColumnTypeDto.NUMBER; public final static ColumnTypeDto COLUMN_3_TYPE_DTO = ColumnTypeDto.NUMBER;
public final static Boolean COLUMN_3_NULL = true; public final static Boolean COLUMN_3_NULL = true;
...@@ -141,6 +150,9 @@ public abstract class BaseUnitTest { ...@@ -141,6 +150,9 @@ public abstract class BaseUnitTest {
public final static List<String> IMAGE_1_ENVIRONMENT = List.of("POSTGRES_USER=postgres", public final static List<String> IMAGE_1_ENVIRONMENT = List.of("POSTGRES_USER=postgres",
"POSTGRES_PASSWORD=postgres", "POSTGRES_DB=" + DATABASE_1_INTERNALNAME); "POSTGRES_PASSWORD=postgres", "POSTGRES_DB=" + DATABASE_1_INTERNALNAME);
public final static List<String> IMAGE_3_ENVIRONMENT = List.of("POSTGRES_USER=postgres",
"POSTGRES_PASSWORD=postgres", "POSTGRES_DB=" + DATABASE_3_INTERNALNAME);
public final static ContainerImage IMAGE_1 = ContainerImage.builder() public final static ContainerImage IMAGE_1 = ContainerImage.builder()
.id(IMAGE_1_ID) .id(IMAGE_1_ID)
.repository(IMAGE_1_REPOSITORY) .repository(IMAGE_1_REPOSITORY)
...@@ -170,6 +182,13 @@ public abstract class BaseUnitTest { ...@@ -170,6 +182,13 @@ public abstract class BaseUnitTest {
public final static String CONTAINER_2_INTERNALNAME = "not3x1st1ng"; public final static String CONTAINER_2_INTERNALNAME = "not3x1st1ng";
public final static Instant CONTAINER_2_CREATED = Instant.now().minus(1, HOURS); public final static Instant CONTAINER_2_CREATED = Instant.now().minus(1, HOURS);
public final static Long CONTAINER_3_ID = 3L;
public final static String CONTAINER_3_HASH = "deadbeef";
public final static String CONTAINER_3_NAME = "u02";
public final static String CONTAINER_3_IP = "172.28.0.6";
public final static String CONTAINER_3_INTERNALNAME = "fda-userdb-u02";
public final static Instant CONTAINER_3_CREATED = Instant.now().minus(1, HOURS);
public final static Container CONTAINER_1 = Container.builder() public final static Container CONTAINER_1 = Container.builder()
.id(CONTAINER_1_ID) .id(CONTAINER_1_ID)
.name(CONTAINER_1_NAME) .name(CONTAINER_1_NAME)
...@@ -188,6 +207,15 @@ public abstract class BaseUnitTest { ...@@ -188,6 +207,15 @@ public abstract class BaseUnitTest {
.containerCreated(CONTAINER_2_CREATED) .containerCreated(CONTAINER_2_CREATED)
.build(); .build();
public final static Container CONTAINER_3 = Container.builder()
.id(CONTAINER_3_ID)
.name(CONTAINER_3_NAME)
.internalName(CONTAINER_3_INTERNALNAME)
.image(CONTAINER_1_IMAGE)
.hash(CONTAINER_3_HASH)
.containerCreated(CONTAINER_3_CREATED)
.build();
public final static List<TableColumn> TABLE_1_COLUMNS = List.of(TableColumn.builder() public final static List<TableColumn> TABLE_1_COLUMNS = List.of(TableColumn.builder()
.id(COLUMN_1_ID) .id(COLUMN_1_ID)
.ordinalPosition(COLUMN_1_ORDINALPOS) .ordinalPosition(COLUMN_1_ORDINALPOS)
...@@ -197,6 +225,7 @@ public abstract class BaseUnitTest { ...@@ -197,6 +225,7 @@ public abstract class BaseUnitTest {
.internalName(COLUMN_1_INTERNAL_NAME) .internalName(COLUMN_1_INTERNAL_NAME)
.columnType(COLUMN_1_TYPE) .columnType(COLUMN_1_TYPE)
.isNullAllowed(COLUMN_1_NULL) .isNullAllowed(COLUMN_1_NULL)
.isUnique(COLUMN_1_UNIQUE)
.isPrimaryKey(COLUMN_1_PRIMARY) .isPrimaryKey(COLUMN_1_PRIMARY)
.enumValues(COLUMN_1_ENUM_VALUES) .enumValues(COLUMN_1_ENUM_VALUES)
.build(), .build(),
...@@ -209,6 +238,7 @@ public abstract class BaseUnitTest { ...@@ -209,6 +238,7 @@ public abstract class BaseUnitTest {
.internalName(COLUMN_2_INTERNAL_NAME) .internalName(COLUMN_2_INTERNAL_NAME)
.columnType(COLUMN_2_TYPE) .columnType(COLUMN_2_TYPE)
.isNullAllowed(COLUMN_2_NULL) .isNullAllowed(COLUMN_2_NULL)
.isUnique(COLUMN_2_UNIQUE)
.isPrimaryKey(COLUMN_2_PRIMARY) .isPrimaryKey(COLUMN_2_PRIMARY)
.enumValues(COLUMN_2_ENUM_VALUES) .enumValues(COLUMN_2_ENUM_VALUES)
.build(), .build(),
...@@ -221,6 +251,7 @@ public abstract class BaseUnitTest { ...@@ -221,6 +251,7 @@ public abstract class BaseUnitTest {
.internalName(COLUMN_3_INTERNAL_NAME) .internalName(COLUMN_3_INTERNAL_NAME)
.columnType(COLUMN_3_TYPE) .columnType(COLUMN_3_TYPE)
.isNullAllowed(COLUMN_3_NULL) .isNullAllowed(COLUMN_3_NULL)
.isUnique(COLUMN_3_UNIQUE)
.isPrimaryKey(COLUMN_3_PRIMARY) .isPrimaryKey(COLUMN_3_PRIMARY)
.enumValues(COLUMN_3_ENUM_VALUES) .enumValues(COLUMN_3_ENUM_VALUES)
.build(), .build(),
...@@ -233,6 +264,7 @@ public abstract class BaseUnitTest { ...@@ -233,6 +264,7 @@ public abstract class BaseUnitTest {
.internalName(COLUMN_4_INTERNAL_NAME) .internalName(COLUMN_4_INTERNAL_NAME)
.columnType(COLUMN_4_TYPE) .columnType(COLUMN_4_TYPE)
.isNullAllowed(COLUMN_4_NULL) .isNullAllowed(COLUMN_4_NULL)
.isUnique(COLUMN_4_UNIQUE)
.isPrimaryKey(COLUMN_4_PRIMARY) .isPrimaryKey(COLUMN_4_PRIMARY)
.enumValues(COLUMN_4_ENUM_VALUES) .enumValues(COLUMN_4_ENUM_VALUES)
.build(), .build(),
...@@ -245,6 +277,7 @@ public abstract class BaseUnitTest { ...@@ -245,6 +277,7 @@ public abstract class BaseUnitTest {
.internalName(COLUMN_5_INTERNAL_NAME) .internalName(COLUMN_5_INTERNAL_NAME)
.columnType(COLUMN_5_TYPE) .columnType(COLUMN_5_TYPE)
.isNullAllowed(COLUMN_5_NULL) .isNullAllowed(COLUMN_5_NULL)
.isUnique(COLUMN_5_UNIQUE)
.isPrimaryKey(COLUMN_5_PRIMARY) .isPrimaryKey(COLUMN_5_PRIMARY)
.enumValues(COLUMN_5_ENUM_VALUES) .enumValues(COLUMN_5_ENUM_VALUES)
.build()); .build());
...@@ -253,6 +286,7 @@ public abstract class BaseUnitTest { ...@@ -253,6 +286,7 @@ public abstract class BaseUnitTest {
.id(TABLE_1_ID) .id(TABLE_1_ID)
.created(Instant.now()) .created(Instant.now())
.internalName(TABLE_1_INTERNALNAME) .internalName(TABLE_1_INTERNALNAME)
.description(TABLE_1_DESCRIPTION)
.name(TABLE_1_NAME) .name(TABLE_1_NAME)
.lastModified(Instant.now()) .lastModified(Instant.now())
.columns(TABLE_1_COLUMNS) .columns(TABLE_1_COLUMNS)
...@@ -260,6 +294,17 @@ public abstract class BaseUnitTest { ...@@ -260,6 +294,17 @@ public abstract class BaseUnitTest {
.topic(TABLE_1_TOPIC) .topic(TABLE_1_TOPIC)
.build(); .build();
public final static Table TABLE_3 = Table.builder()
.id(TABLE_3_ID)
.created(Instant.now())
.internalName(TABLE_3_INTERNALNAME)
.description(TABLE_3_DESCRIPTION)
.name(TABLE_3_NAME)
.lastModified(Instant.now())
.tdbid(DATABASE_3_ID)
.topic(TABLE_3_TOPIC)
.build();
public final static Database DATABASE_1 = Database.builder() public final static Database DATABASE_1 = Database.builder()
.id(DATABASE_1_ID) .id(DATABASE_1_ID)
.created(Instant.now().minus(1, HOURS)) .created(Instant.now().minus(1, HOURS))
...@@ -272,6 +317,18 @@ public abstract class BaseUnitTest { ...@@ -272,6 +317,18 @@ public abstract class BaseUnitTest {
.exchange(DATABASE_1_EXCHANGE) .exchange(DATABASE_1_EXCHANGE)
.build(); .build();
public final static Database DATABASE_3 = Database.builder()
.id(DATABASE_3_ID)
.created(Instant.now().minus(1, HOURS))
.lastModified(Instant.now())
.isPublic(false)
.name(DATABASE_3_NAME)
.container(CONTAINER_3)
.tables(List.of(TABLE_3))
.internalName(DATABASE_3_INTERNALNAME)
.exchange(DATABASE_3_EXCHANGE)
.build();
/* no connection */ /* no connection */
public final static Database DATABASE_2 = Database.builder() public final static Database DATABASE_2 = Database.builder()
.id(DATABASE_2_ID) .id(DATABASE_2_ID)
...@@ -285,7 +342,7 @@ public abstract class BaseUnitTest { ...@@ -285,7 +342,7 @@ public abstract class BaseUnitTest {
.exchange(DATABASE_2_EXCHANGE) .exchange(DATABASE_2_EXCHANGE)
.build(); .build();
public final static ColumnCreateDto[] COLUMNS5 = new ColumnCreateDto[]{ public final static ColumnCreateDto[] COLUMNS_CSV01 = new ColumnCreateDto[]{
ColumnCreateDto.builder() ColumnCreateDto.builder()
.type(COLUMN_1_TYPE_DTO) .type(COLUMN_1_TYPE_DTO)
.name(COLUMN_1_NAME) .name(COLUMN_1_NAME)
...@@ -322,52 +379,16 @@ public abstract class BaseUnitTest { ...@@ -322,52 +379,16 @@ public abstract class BaseUnitTest {
.unique(COLUMN_5_UNIQUE) .unique(COLUMN_5_UNIQUE)
.build()}; .build()};
public final static ColumnDto[] COLUMNS5_DTO = new ColumnDto[]{ public final static TableCreateDto TABLE_1_CREATE_DTO = TableCreateDto.builder()
ColumnDto.builder() .name(TABLE_1_NAME)
.name(COLUMN_1_NAME) .description(TABLE_1_DESCRIPTION)
.columnType(COLUMN_1_TYPE_DTO) .columns(COLUMNS_CSV01)
.isNullAllowed(COLUMN_1_NULL) .build();
.isPrimaryKey(COLUMN_1_PRIMARY)
.foreignKey(COLUMN_1_FOREIGN_KEY)
.checkExpression(COLUMN_1_CHECK)
.build(),
ColumnDto.builder()
.name(COLUMN_2_NAME)
.columnType(COLUMN_2_TYPE_DTO)
.isNullAllowed(COLUMN_2_NULL)
.isPrimaryKey(COLUMN_2_PRIMARY)
.foreignKey(COLUMN_2_FOREIGN_KEY)
.checkExpression(COLUMN_2_CHECK)
.build(),
ColumnDto.builder()
.name(COLUMN_3_NAME)
.columnType(COLUMN_3_TYPE_DTO)
.isNullAllowed(COLUMN_3_NULL)
.isPrimaryKey(COLUMN_3_PRIMARY)
.foreignKey(COLUMN_3_FOREIGN_KEY)
.checkExpression(COLUMN_3_CHECK)
.build(),
ColumnDto.builder()
.name(COLUMN_4_NAME)
.columnType(COLUMN_4_TYPE_DTO)
.isNullAllowed(COLUMN_4_NULL)
.isPrimaryKey(COLUMN_4_PRIMARY)
.foreignKey(COLUMN_4_FOREIGN_KEY)
.checkExpression(COLUMN_4_CHECK)
.build(),
ColumnDto.builder()
.name(COLUMN_5_NAME)
.columnType(COLUMN_5_TYPE_DTO)
.isNullAllowed(COLUMN_5_NULL)
.isPrimaryKey(COLUMN_5_PRIMARY)
.foreignKey(COLUMN_5_FOREIGN_KEY)
.checkExpression(COLUMN_5_CHECK)
.build()};
public final static TableCreateDto TABLE_2_CREATE_DTO = TableCreateDto.builder() public final static TableCreateDto TABLE_2_CREATE_DTO = TableCreateDto.builder()
.name(TABLE_2_NAME) .name(TABLE_2_NAME)
.description(TABLE_2_DESCRIPTION) .description(TABLE_2_DESCRIPTION)
.columns(COLUMNS5) .columns(COLUMNS_CSV01)
.build(); .build();
} }
fda-table-service/rest-service/src/test/java/at/tuwien/CsvUnitTest.java 0 → 100644
+ 53
− 0
View file @ 8be84589
package at.tuwien;
import at.tuwien.api.database.table.columns.ColumnCreateDto;
import at.tuwien.api.database.table.columns.ColumnTypeDto;
public abstract class CsvUnitTest {
public final static ColumnCreateDto[] COLUMNS_CSV02 = new ColumnCreateDto[]{
ColumnCreateDto.builder()
.type(ColumnTypeDto.NUMBER)
.name("id")
.nullAllowed(false)
.primaryKey(true)
.unique(true)
.build(),
ColumnCreateDto.builder()
.type(ColumnTypeDto.STRING)
.name("nouniquestr")
.nullAllowed(false)
.primaryKey(false)
.unique(false)
.build(),
ColumnCreateDto.builder()
.type(ColumnTypeDto.ENUM)
.name("method")
.nullAllowed(false)
.primaryKey(false)
.unique(false)
.enumValues(new String[] {"mk", "zf", "ac", "em"})
.build(),
ColumnCreateDto.builder()
.type(ColumnTypeDto.STRING)
.name("company")
.nullAllowed(false)
.primaryKey(false)
.unique(false)
.build(),
ColumnCreateDto.builder()
.type(ColumnTypeDto.NUMBER)
.name("measurements")
.nullAllowed(false)
.primaryKey(false)
.unique(false)
.build(),
ColumnCreateDto.builder()
.type(ColumnTypeDto.NUMBER)
.name("trialn")
.nullAllowed(false)
.primaryKey(false)
.unique(false)
.build(),};
}
fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/DataEndpointIntegrationTest.java 0 → 100644
+ 188
− 0
View file @ 8be84589
package at.tuwien.endpoint;
import at.tuwien.BaseUnitTest;
import at.tuwien.api.database.table.*;
import at.tuwien.config.ReadyConfig;
import at.tuwien.endpoints.DataEndpoint;
import at.tuwien.endpoints.TableEndpoint;
import at.tuwien.exception.*;
import at.tuwien.repository.jpa.DatabaseRepository;
import at.tuwien.repository.jpa.ImageRepository;
import at.tuwien.repository.jpa.TableRepository;
import at.tuwien.service.DataService;
import at.tuwien.service.JdbcConnector;
import at.tuwien.service.TableService;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.CreateContainerResponse;
import com.github.dockerjava.api.exception.NotModifiedException;
import com.github.dockerjava.api.model.HostConfig;
import com.github.dockerjava.api.model.Network;
import com.rabbitmq.client.Channel;
import lombok.extern.log4j.Log4j2;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import javax.transaction.Transactional;
import javax.ws.rs.core.Response;
import java.time.Instant;
import java.util.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.*;
@Log4j2
@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
@SpringBootTest
@ExtendWith(SpringExtension.class)
public class DataEndpointIntegrationTest extends BaseUnitTest {
@MockBean
private Channel channel;
@MockBean
private ReadyConfig readyConfig;
@Autowired
private HostConfig hostConfig;
@Autowired
private DockerClient dockerClient;
@Autowired
private ImageRepository imageRepository;
@Autowired
private DataService dataService;
@Autowired
private TableService tableService;
@Autowired
private TableRepository tableRepository;
@Autowired
private DatabaseRepository databaseRepository;
@Autowired
private DataEndpoint dataEndpoint;
private CreateContainerResponse request;
@Transactional
@BeforeEach
public void beforeEach() throws InterruptedException, TableMalformedException, ArbitraryPrimaryKeysException,
DatabaseNotFoundException, ImageNotSupportedException, DataProcessingException {
afterEach();
/* create network */
dockerClient.createNetworkCmd()
.withName("fda-userdb")
.withIpam(new Network.Ipam()
.withConfig(new Network.Ipam.Config()
.withSubnet("172.28.0.0/16")))
.withEnableIpv6(false)
.exec();
imageRepository.save(IMAGE_1);
/* create container */
request = dockerClient.createContainerCmd(IMAGE_1_REPOSITORY + ":" + IMAGE_1_TAG)
.withEnv(IMAGE_1_ENVIRONMENT)
.withHostConfig(hostConfig.withNetworkMode("fda-userdb"))
.withName(CONTAINER_1_INTERNALNAME)
.withIpv4Address(CONTAINER_1_IP)
.withHostName(CONTAINER_1_INTERNALNAME)
.exec();
/* start container */
dockerClient.startContainerCmd(request.getId()).exec();
Thread.sleep(3000);
databaseRepository.save(DATABASE_1);
tableService.createTable(DATABASE_1_ID, TABLE_1_CREATE_DTO);
tableRepository.save(TABLE_1);
}
@AfterEach
public void afterEach() {
/* stop containers and remove them */
dockerClient.listContainersCmd()
.withShowAll(true)
.exec()
.forEach(container -> {
log.info("Delete container {}", Arrays.asList(container.getNames()));
try {
dockerClient.stopContainerCmd(container.getId()).exec();
} catch (NotModifiedException e) {
// ignore
}
dockerClient.removeContainerCmd(container.getId()).exec();
});
/* remove networks */
dockerClient.listNetworksCmd()
.exec()
.stream()
.filter(n -> n.getName().startsWith("fda"))
.forEach(network -> {
log.info("Delete network {}", network.getName());
dockerClient.removeNetworkCmd(network.getId()).exec();
});
}
@Test
public void insertFromTuple_succeeds() {
final TableCsvDto request = TableCsvDto.builder()
.data(List.of(Map.of(COLUMN_1_NAME, 1L, COLUMN_2_NAME, Instant.now(), COLUMN_3_NAME, 35.2,
COLUMN_4_NAME, "Sydney", COLUMN_5_NAME, 10.2)))
.build();
/* test */
assertThrows(TableMalformedException.class, () -> {
dataEndpoint.insertFromTuple(DATABASE_1_ID, TABLE_1_ID, request);
});
}
@Test
public void insertFromTuple_empty_fails() {
final TableCsvDto request = TableCsvDto.builder()
.data(List.of())
.build();
/* test */
assertThrows(TableMalformedException.class, () -> {
dataEndpoint.insertFromTuple(DATABASE_1_ID, TABLE_1_ID, request);
});
}
@Test
public void insertFromTuple_empty2_fails() {
final TableCsvDto request = TableCsvDto.builder()
.data(List.of(Map.of()))
.build();
/* test */
assertThrows(TableMalformedException.class, () -> {
dataEndpoint.insertFromTuple(DATABASE_1_ID, TABLE_1_ID, request);
});
}
@Test
public void getAll_succeeds() {
/* test */
}
@Test
public void getAll_fails() {
/* test */
}
}
fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java
+ 4
− 4
View file @ 8be84589
...@@ -7,8 +7,8 @@ import at.tuwien.api.database.table.TableDto; ...@@ -7,8 +7,8 @@ import at.tuwien.api.database.table.TableDto;
import at.tuwien.config.ReadyConfig; import at.tuwien.config.ReadyConfig;
import at.tuwien.endpoints.TableEndpoint; import at.tuwien.endpoints.TableEndpoint;
import at.tuwien.exception.*; import at.tuwien.exception.*;
import at.tuwien.repository.DatabaseRepository; import at.tuwien.repository.jpa.DatabaseRepository;
import at.tuwien.repository.TableRepository; import at.tuwien.repository.jpa.TableRepository;
import at.tuwien.service.TableService; import at.tuwien.service.TableService;
import com.rabbitmq.client.Channel; import com.rabbitmq.client.Channel;
import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Disabled;
...@@ -69,7 +69,7 @@ public class TableEndpointUnitTest extends BaseUnitTest { ...@@ -69,7 +69,7 @@ public class TableEndpointUnitTest extends BaseUnitTest {
final TableCreateDto request = TableCreateDto.builder() final TableCreateDto request = TableCreateDto.builder()
.name(TABLE_1_NAME) .name(TABLE_1_NAME)
.description(TABLE_1_DESCRIPTION) .description(TABLE_1_DESCRIPTION)
.columns(COLUMNS5) .columns(COLUMNS_CSV01)
.build(); .build();
when(tableRepository.findById(TABLE_1_ID)) when(tableRepository.findById(TABLE_1_ID))
.thenReturn(Optional.of(TABLE_1)); .thenReturn(Optional.of(TABLE_1));
...@@ -87,7 +87,7 @@ public class TableEndpointUnitTest extends BaseUnitTest { ...@@ -87,7 +87,7 @@ public class TableEndpointUnitTest extends BaseUnitTest {
final TableCreateDto request = TableCreateDto.builder() final TableCreateDto request = TableCreateDto.builder()
.name(TABLE_1_NAME) .name(TABLE_1_NAME)
.description(TABLE_1_DESCRIPTION) .description(TABLE_1_DESCRIPTION)
.columns(COLUMNS5) .columns(COLUMNS_CSV01)
.build(); .build();
when(tableService.createTable(DATABASE_1_ID, request)) when(tableService.createTable(DATABASE_1_ID, request))
.thenAnswer(invocation -> { .thenAnswer(invocation -> {
......
fda-table-service/rest-service/src/test/java/at/tuwien/mapper/TableMapperIntegrationTest.java
+ 4
− 7
View file @ 8be84589
...@@ -8,8 +8,8 @@ import at.tuwien.config.ReadyConfig; ...@@ -8,8 +8,8 @@ import at.tuwien.config.ReadyConfig;
import at.tuwien.exception.ArbitraryPrimaryKeysException; import at.tuwien.exception.ArbitraryPrimaryKeysException;
import at.tuwien.exception.ImageNotSupportedException; import at.tuwien.exception.ImageNotSupportedException;
import at.tuwien.exception.TableMalformedException; import at.tuwien.exception.TableMalformedException;
import at.tuwien.repository.DatabaseRepository; import at.tuwien.repository.jpa.DatabaseRepository;
import at.tuwien.repository.ImageRepository; import at.tuwien.repository.jpa.ImageRepository;
import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.CreateContainerResponse; import com.github.dockerjava.api.command.CreateContainerResponse;
import com.github.dockerjava.api.exception.NotModifiedException; import com.github.dockerjava.api.exception.NotModifiedException;
...@@ -17,7 +17,6 @@ import com.github.dockerjava.api.model.HostConfig; ...@@ -17,7 +17,6 @@ import com.github.dockerjava.api.model.HostConfig;
import com.github.dockerjava.api.model.PortBinding; import com.github.dockerjava.api.model.PortBinding;
import com.rabbitmq.client.Channel; import com.rabbitmq.client.Channel;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import org.apache.commons.lang.SerializationUtils;
import org.jooq.*; import org.jooq.*;
import org.jooq.impl.DSL; import org.jooq.impl.DSL;
import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.AfterEach;
...@@ -32,12 +31,10 @@ import org.springframework.test.annotation.DirtiesContext; ...@@ -32,12 +31,10 @@ import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import javax.validation.constraints.NotNull;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.*; import java.util.*;
import java.util.stream.Collectors;
import static org.jooq.impl.DSL.table; import static org.jooq.impl.DSL.table;
import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.api.Assertions.*;
...@@ -435,8 +432,8 @@ public class TableMapperIntegrationTest extends BaseUnitTest { ...@@ -435,8 +432,8 @@ public class TableMapperIntegrationTest extends BaseUnitTest {
ArbitraryPrimaryKeysException, ImageNotSupportedException, TableMalformedException { ArbitraryPrimaryKeysException, ImageNotSupportedException, TableMalformedException {
final DSLContext context = open(); final DSLContext context = open();
final TableCreateDto TABLE_2_CREATE_DTO = instance(); final TableCreateDto TABLE_2_CREATE_DTO = instance();
final ColumnCreateDto[] columns = new ColumnCreateDto[]{COLUMNS5[0], COLUMNS5[1], COLUMNS5[2], final ColumnCreateDto[] columns = new ColumnCreateDto[]{COLUMNS_CSV01[0], COLUMNS_CSV01[1], COLUMNS_CSV01[2],
COLUMNS5[3], COLUMNS5[4], ColumnCreateDto.builder() COLUMNS_CSV01[3], COLUMNS_CSV01[4], ColumnCreateDto.builder()
.name("Gender") .name("Gender")
.nullAllowed(false) .nullAllowed(false)
.primaryKey(false) .primaryKey(false)
......
fda-table-service/rest-service/src/test/java/at/tuwien/service/ImportServiceIntegrationTest.java → fda-table-service/rest-service/src/test/java/at/tuwien/service/DataServiceIntegrationTest.java
+ 90
− 35
View file @ 8be84589
...@@ -5,17 +5,19 @@ import at.tuwien.api.database.table.TableCreateDto; ...@@ -5,17 +5,19 @@ import at.tuwien.api.database.table.TableCreateDto;
import at.tuwien.api.database.table.TableInsertDto; import at.tuwien.api.database.table.TableInsertDto;
import at.tuwien.config.ReadyConfig; import at.tuwien.config.ReadyConfig;
import at.tuwien.entities.database.table.Table; import at.tuwien.entities.database.table.Table;
import at.tuwien.entities.database.table.columns.TableColumn;
import at.tuwien.exception.*; import at.tuwien.exception.*;
import at.tuwien.repository.ContainerRepository; import at.tuwien.repository.jpa.ContainerRepository;
import at.tuwien.repository.DatabaseRepository; import at.tuwien.repository.jpa.DatabaseRepository;
import at.tuwien.repository.ImageRepository; import at.tuwien.repository.jpa.ImageRepository;
import at.tuwien.repository.TableRepository; import at.tuwien.repository.jpa.TableRepository;
import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.CreateContainerResponse; import com.github.dockerjava.api.command.CreateContainerResponse;
import com.github.dockerjava.api.exception.NotModifiedException; import com.github.dockerjava.api.exception.NotModifiedException;
import com.github.dockerjava.api.model.HostConfig; import com.github.dockerjava.api.model.HostConfig;
import com.github.dockerjava.api.model.Network; import com.github.dockerjava.api.model.Network;
import com.rabbitmq.client.Channel; import com.rabbitmq.client.Channel;
import lombok.extern.log4j.Log4j2;
import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Disabled;
...@@ -24,25 +26,20 @@ import org.junit.jupiter.api.extension.ExtendWith; ...@@ -24,25 +26,20 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.ResourceUtils;
import org.springframework.web.multipart.MultipartFile;
import org.synchronoss.cloud.nio.multipart.Multipart;
import java.io.IOException;
import java.nio.file.Files;
import java.util.Arrays; import java.util.Arrays;
import java.util.Optional; import java.util.Optional;
import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.api.Assertions.*;
@Log4j2
@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) @DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
@ExtendWith(SpringExtension.class) @ExtendWith(SpringExtension.class)
@SpringBootTest @SpringBootTest
public class ImportServiceIntegrationTest extends BaseUnitTest { public class DataServiceIntegrationTest extends BaseUnitTest {
@MockBean @MockBean
private Channel channel; private Channel channel;
...@@ -74,7 +71,7 @@ public class ImportServiceIntegrationTest extends BaseUnitTest { ...@@ -74,7 +71,7 @@ public class ImportServiceIntegrationTest extends BaseUnitTest {
@Autowired @Autowired
private DataService dataService; private DataService dataService;
private CreateContainerResponse request; private CreateContainerResponse request1, request2;
@Transactional @Transactional
@BeforeEach @BeforeEach
...@@ -90,19 +87,28 @@ public class ImportServiceIntegrationTest extends BaseUnitTest { ...@@ -90,19 +87,28 @@ public class ImportServiceIntegrationTest extends BaseUnitTest {
.exec(); .exec();
imageRepository.save(IMAGE_1); imageRepository.save(IMAGE_1);
/* create container */ /* create container */
request = dockerClient.createContainerCmd(IMAGE_1_REPOSITORY + ":" + IMAGE_1_TAG) request1 = dockerClient.createContainerCmd(IMAGE_1_REPOSITORY + ":" + IMAGE_1_TAG)
.withEnv(IMAGE_1_ENVIRONMENT) .withEnv(IMAGE_1_ENVIRONMENT)
.withHostConfig(hostConfig.withNetworkMode("fda-userdb")) .withHostConfig(hostConfig.withNetworkMode("fda-userdb"))
.withName(CONTAINER_1_INTERNALNAME) .withName(CONTAINER_1_INTERNALNAME)
.withIpv4Address(CONTAINER_1_IP) .withIpv4Address(CONTAINER_1_IP)
.withHostName(CONTAINER_1_INTERNALNAME) .withHostName(CONTAINER_1_INTERNALNAME)
.exec(); .exec();
request2 = dockerClient.createContainerCmd(IMAGE_1_REPOSITORY + ":" + IMAGE_1_TAG)
.withEnv(IMAGE_3_ENVIRONMENT)
.withHostConfig(hostConfig.withNetworkMode("fda-userdb"))
.withName(CONTAINER_3_INTERNALNAME)
.withIpv4Address(CONTAINER_3_IP)
.withHostName(CONTAINER_3_INTERNALNAME)
.exec();
/* start container */ /* start container */
dockerClient.startContainerCmd(request.getId()).exec(); dockerClient.startContainerCmd(request1.getId()).exec();
Thread.sleep(3000); dockerClient.startContainerCmd(request2.getId()).exec();
CONTAINER_1_HASH = request.getId(); Thread.sleep(5 * 1000);
CONTAINER_1_HASH = request1.getId();
databaseRepository.save(DATABASE_1); databaseRepository.save(DATABASE_1);
databaseRepository.save(DATABASE_2); databaseRepository.save(DATABASE_2);
databaseRepository.save(DATABASE_3); /* csv_02 */
} }
@Transactional @Transactional
...@@ -113,7 +119,7 @@ public class ImportServiceIntegrationTest extends BaseUnitTest { ...@@ -113,7 +119,7 @@ public class ImportServiceIntegrationTest extends BaseUnitTest {
.withShowAll(true) .withShowAll(true)
.exec() .exec()
.forEach(container -> { .forEach(container -> {
System.out.println("DELETE CONTAINER " + Arrays.toString(container.getNames())); log.info("Delete container {}", Arrays.asList(container.getNames()));
try { try {
dockerClient.stopContainerCmd(container.getId()).exec(); dockerClient.stopContainerCmd(container.getId()).exec();
} catch (NotModifiedException e) { } catch (NotModifiedException e) {
...@@ -127,73 +133,122 @@ public class ImportServiceIntegrationTest extends BaseUnitTest { ...@@ -127,73 +133,122 @@ public class ImportServiceIntegrationTest extends BaseUnitTest {
.stream() .stream()
.filter(n -> n.getName().startsWith("fda")) .filter(n -> n.getName().startsWith("fda"))
.forEach(network -> { .forEach(network -> {
System.out.println("DELETE NETWORK " + network.getName()); log.info("Delete network {}", network.getName());
dockerClient.removeNetworkCmd(network.getId()).exec(); dockerClient.removeNetworkCmd(network.getId()).exec();
}); });
} }
private void create_table() throws ArbitraryPrimaryKeysException, DatabaseNotFoundException, ImageNotSupportedException, DataProcessingException, TableMalformedException { @Test
final Table response = tableService.createTable(DATABASE_1_ID, TableCreateDto.builder() public void insertFromFile_succeeds() throws TableMalformedException,
DatabaseNotFoundException, ImageNotSupportedException,
ArbitraryPrimaryKeysException, DataProcessingException, TableNotFoundException, FileStorageException {
tableService.createTable(DATABASE_1_ID, TableCreateDto.builder()
.name(TABLE_1_NAME) .name(TABLE_1_NAME)
.description(TABLE_1_DESCRIPTION) .description(TABLE_1_DESCRIPTION)
.columns(COLUMNS5) .columns(COLUMNS_CSV01)
.build()); .build());
final TableInsertDto request = TableInsertDto.builder()
.delimiter(';')
.skipHeader(true)
.nullElement("NA")
.csvLocation("test:src/test/resources/csv/csv_01.csv")
.build();
/* test */
dataService.insertCsv(DATABASE_1_ID, TABLE_1_ID, request);
final Optional<Table> response = tableRepository.findByDatabaseAndId(DATABASE_1, TABLE_1_ID);
assertTrue(response.isPresent());
assertEquals(TABLE_1, response.get());
} }
@Test @Test
@Disabled public void insertFromFileCsv01_nonUnique_succeeds() throws TableMalformedException,
public void insertFromFile_succeeds() throws TableMalformedException,
DatabaseNotFoundException, ImageNotSupportedException, DatabaseNotFoundException, ImageNotSupportedException,
ArbitraryPrimaryKeysException, DataProcessingException, TableNotFoundException, FileStorageException { ArbitraryPrimaryKeysException, DataProcessingException, TableNotFoundException, FileStorageException {
create_table(); COLUMNS_CSV01[0].setUnique(false);
tableService.createTable(DATABASE_1_ID, TableCreateDto.builder()
.name(TABLE_1_NAME)
.description(TABLE_1_DESCRIPTION)
.columns(COLUMNS_CSV01)
.build());
final TableInsertDto request = TableInsertDto.builder() final TableInsertDto request = TableInsertDto.builder()
.delimiter(';') .delimiter(';')
.skipHeader(true) .skipHeader(true)
.nullElement("NA") .nullElement("NA")
.csvLocation("test:src/test/resources/weather-small.csv") .csvLocation("test:src/test/resources/csv/csv_01.csv")
.build(); .build();
/* test */ /* test */
dataService.insertCsv(DATABASE_1_ID, TABLE_1_ID, request); dataService.insertCsv(DATABASE_1_ID, TABLE_1_ID, request);
final Optional<Table> response = tableRepository.findByDatabaseAndId(DATABASE_1, TABLE_1_ID); final Optional<Table> response = tableRepository.findByDatabaseAndId(DATABASE_1, TABLE_1_ID);
assertTrue(response.isPresent()); assertTrue(response.isPresent());
assertEquals(TABLE_1_ID, response.get().getId()); assertEquals(TABLE_1, response.get());
assertEquals(TABLE_1_NAME, response.get().getName()); }
assertEquals(TABLE_1_DESCRIPTION, response.get().getDescription());
@Test
@Disabled
public void insertFromFileCsv02_succeeds() throws TableMalformedException,
DatabaseNotFoundException, ImageNotSupportedException,
ArbitraryPrimaryKeysException, DataProcessingException, TableNotFoundException, FileStorageException {
tableService.createTable(DATABASE_3_ID, TableCreateDto.builder()
.name(TABLE_3_NAME)
.description(TABLE_3_DESCRIPTION)
.columns(COLUMNS_CSV02)
.build());
final TableInsertDto request = TableInsertDto.builder()
.delimiter(',')
.skipHeader(true)
.nullElement(null)
.csvLocation("test:src/test/resources/csv/csv_02.csv")
.build();
/* test */
dataService.insertCsv(DATABASE_3_ID, TABLE_3_ID, request);
final Optional<Table> response = tableRepository.findByDatabaseAndId(DATABASE_3, TABLE_3_ID);
assertTrue(response.isPresent());
assertArrayEquals(COLUMNS_CSV02, response.get().getColumns().toArray(new TableColumn[0]));
} }
@Test @Test
public void insertFromFile_columnNumberDiffers_fails() throws DatabaseNotFoundException, ImageNotSupportedException, public void insertFromFile_columnNumberDiffers_fails() throws DatabaseNotFoundException, ImageNotSupportedException,
ArbitraryPrimaryKeysException, DataProcessingException, TableMalformedException { ArbitraryPrimaryKeysException, DataProcessingException, TableMalformedException {
create_table(); tableService.createTable(DATABASE_1_ID, TableCreateDto.builder()
.name(TABLE_1_NAME)
.description(TABLE_1_DESCRIPTION)
.columns(COLUMNS_CSV01)
.build());
final TableInsertDto request = TableInsertDto.builder() final TableInsertDto request = TableInsertDto.builder()
.delimiter(';') .delimiter(';')
.skipHeader(true) .skipHeader(true)
.nullElement("NA") .nullElement("NA")
.csvLocation("test:src/test/resources/namen.csv") .csvLocation("test:src/test/resources/csv/csv_09.csv")
.build(); .build();
/* test */ /* test */
assertThrows(FileStorageException.class, () -> { assertThrows(FileStorageException.class, () -> {
dataService.insertCsv(DATABASE_1_ID, TABLE_2_ID, request); dataService.insertCsv(DATABASE_1_ID, TABLE_1_ID, request);
}); });
} }
@Test @Test
public void insertFromFile_notRunning_fails() throws DatabaseNotFoundException, ImageNotSupportedException, public void insertFromFile_notRunning_fails() throws DatabaseNotFoundException, ImageNotSupportedException,
ArbitraryPrimaryKeysException, DataProcessingException, TableMalformedException { ArbitraryPrimaryKeysException, DataProcessingException, TableMalformedException {
create_table(); tableService.createTable(DATABASE_1_ID, TableCreateDto.builder()
dockerClient.stopContainerCmd(request.getId()).exec(); .name(TABLE_1_NAME)
.description(TABLE_1_DESCRIPTION)
.columns(COLUMNS_CSV01)
.build());
dockerClient.stopContainerCmd(CONTAINER_1_HASH).exec();
final TableInsertDto request = TableInsertDto.builder() final TableInsertDto request = TableInsertDto.builder()
.delimiter(';') .delimiter(';')
.skipHeader(true) .skipHeader(true)
.nullElement("NA") .nullElement("NA")
.csvLocation("test:src/test/resources/weather-small.csv") .csvLocation("test:src/test/resources/csv/csv_01.csv")
.build(); .build();
/* test */ /* test */
assertThrows(TableMalformedException.class, () -> { assertThrows(TableMalformedException.class, () -> {
dataService.insertCsv(DATABASE_1_ID, TABLE_2_ID, request); dataService.insertCsv(DATABASE_1_ID, TABLE_1_ID, request);
}); });
} }
......
fda-table-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationTest.java
+ 10
− 16
View file @ 8be84589
...@@ -2,40 +2,34 @@ package at.tuwien.service; ...@@ -2,40 +2,34 @@ package at.tuwien.service;
import at.tuwien.BaseUnitTest; import at.tuwien.BaseUnitTest;
import at.tuwien.api.database.table.TableCreateDto; import at.tuwien.api.database.table.TableCreateDto;
import at.tuwien.api.database.table.TableInsertDto;
import at.tuwien.config.ReadyConfig; import at.tuwien.config.ReadyConfig;
import at.tuwien.entities.database.table.Table; import at.tuwien.entities.database.table.Table;
import at.tuwien.exception.*; import at.tuwien.exception.*;
import at.tuwien.repository.ContainerRepository; import at.tuwien.repository.jpa.ContainerRepository;
import at.tuwien.repository.DatabaseRepository; import at.tuwien.repository.jpa.DatabaseRepository;
import at.tuwien.repository.ImageRepository; import at.tuwien.repository.jpa.ImageRepository;
import at.tuwien.repository.TableRepository; import at.tuwien.repository.jpa.TableRepository;
import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.CreateContainerResponse; import com.github.dockerjava.api.command.CreateContainerResponse;
import com.github.dockerjava.api.exception.NotModifiedException; import com.github.dockerjava.api.exception.NotModifiedException;
import com.github.dockerjava.api.model.HealthCheck;
import com.github.dockerjava.api.model.HostConfig; import com.github.dockerjava.api.model.HostConfig;
import com.github.dockerjava.api.model.Network; import com.github.dockerjava.api.model.Network;
import com.rabbitmq.client.Channel; import com.rabbitmq.client.Channel;
import lombok.extern.log4j.Log4j2;
import org.junit.jupiter.api.*; import org.junit.jupiter.api.*;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.ResourceUtils;
import java.io.IOException;
import java.nio.file.Files;
import java.util.Arrays; import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.api.Assertions.*;
@Log4j2
@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) @DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
@ExtendWith(SpringExtension.class) @ExtendWith(SpringExtension.class)
@SpringBootTest @SpringBootTest
...@@ -105,7 +99,7 @@ public class TableServiceIntegrationTest extends BaseUnitTest { ...@@ -105,7 +99,7 @@ public class TableServiceIntegrationTest extends BaseUnitTest {
.withShowAll(true) .withShowAll(true)
.exec() .exec()
.forEach(container -> { .forEach(container -> {
System.out.println("DELETE CONTAINER " + Arrays.toString(container.getNames())); log.info("Delete container {}", Arrays.asList(container.getNames()));
try { try {
dockerClient.stopContainerCmd(container.getId()).exec(); dockerClient.stopContainerCmd(container.getId()).exec();
} catch (NotModifiedException e) { } catch (NotModifiedException e) {
...@@ -119,7 +113,7 @@ public class TableServiceIntegrationTest extends BaseUnitTest { ...@@ -119,7 +113,7 @@ public class TableServiceIntegrationTest extends BaseUnitTest {
.stream() .stream()
.filter(n -> n.getName().startsWith("fda")) .filter(n -> n.getName().startsWith("fda"))
.forEach(network -> { .forEach(network -> {
System.out.println("DELETE NETWORK " + network.getName()); log.info("Delete network {}", network.getName());
dockerClient.removeNetworkCmd(network.getId()).exec(); dockerClient.removeNetworkCmd(network.getId()).exec();
}); });
} }
...@@ -130,7 +124,7 @@ public class TableServiceIntegrationTest extends BaseUnitTest { ...@@ -130,7 +124,7 @@ public class TableServiceIntegrationTest extends BaseUnitTest {
final TableCreateDto request = TableCreateDto.builder() final TableCreateDto request = TableCreateDto.builder()
.name(TABLE_2_NAME) .name(TABLE_2_NAME)
.description(TABLE_2_DESCRIPTION) .description(TABLE_2_DESCRIPTION)
.columns(COLUMNS5) .columns(COLUMNS_CSV01)
.build(); .build();
/* test */ /* test */
...@@ -139,7 +133,7 @@ public class TableServiceIntegrationTest extends BaseUnitTest { ...@@ -139,7 +133,7 @@ public class TableServiceIntegrationTest extends BaseUnitTest {
assertEquals(TABLE_2_INTERNALNAME, response.getInternalName()); assertEquals(TABLE_2_INTERNALNAME, response.getInternalName());
assertEquals(TABLE_2_DESCRIPTION, response.getDescription()); assertEquals(TABLE_2_DESCRIPTION, response.getDescription());
assertEquals(DATABASE_1_ID, response.getTdbid()); assertEquals(DATABASE_1_ID, response.getTdbid());
assertEquals(COLUMNS5.length, response.getColumns().size()); assertEquals(COLUMNS_CSV01.length, response.getColumns().size());
} }
@Test @Test
......
fda-table-service/rest-service/src/test/java/at/tuwien/service/TableServiceUnitTest.java
+ 6
− 6
View file @ 8be84589
...@@ -6,8 +6,8 @@ import at.tuwien.api.database.table.TableInsertDto; ...@@ -6,8 +6,8 @@ import at.tuwien.api.database.table.TableInsertDto;
import at.tuwien.config.ReadyConfig; import at.tuwien.config.ReadyConfig;
import at.tuwien.entities.database.table.Table; import at.tuwien.entities.database.table.Table;
import at.tuwien.exception.*; import at.tuwien.exception.*;
import at.tuwien.repository.DatabaseRepository; import at.tuwien.repository.jpa.DatabaseRepository;
import at.tuwien.repository.TableRepository; import at.tuwien.repository.jpa.TableRepository;
import com.opencsv.exceptions.CsvException; import com.opencsv.exceptions.CsvException;
import com.rabbitmq.client.Channel; import com.rabbitmq.client.Channel;
import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeAll;
...@@ -103,7 +103,7 @@ public class TableServiceUnitTest extends BaseUnitTest { ...@@ -103,7 +103,7 @@ public class TableServiceUnitTest extends BaseUnitTest {
} }
@Test @Test
public void findById_succeeds() throws TableNotFoundException, DatabaseNotFoundException, ImageNotSupportedException { public void findById_succeeds() throws TableNotFoundException, DatabaseNotFoundException {
when(databaseRepository.findById(DATABASE_1_ID)) when(databaseRepository.findById(DATABASE_1_ID))
.thenReturn(Optional.of(DATABASE_1)); .thenReturn(Optional.of(DATABASE_1));
when(tableRepository.findByDatabaseAndId(DATABASE_1, TABLE_1_ID)) when(tableRepository.findByDatabaseAndId(DATABASE_1, TABLE_1_ID))
...@@ -130,7 +130,7 @@ public class TableServiceUnitTest extends BaseUnitTest { ...@@ -130,7 +130,7 @@ public class TableServiceUnitTest extends BaseUnitTest {
@Test @Test
public void readCsv_succeeds() throws IOException, CsvException { public void readCsv_succeeds() throws IOException, CsvException {
final MultipartFile file = new MockMultipartFile("weather-small", Files.readAllBytes(ResourceUtils.getFile("classpath:weather-small.csv").toPath())); final MultipartFile file = new MockMultipartFile("csv_01", Files.readAllBytes(ResourceUtils.getFile("classpath:csv/csv_01.csv").toPath()));
final TableInsertDto request = TableInsertDto.builder() final TableInsertDto request = TableInsertDto.builder()
.delimiter(';') .delimiter(';')
.skipHeader(true) .skipHeader(true)
...@@ -144,7 +144,7 @@ public class TableServiceUnitTest extends BaseUnitTest { ...@@ -144,7 +144,7 @@ public class TableServiceUnitTest extends BaseUnitTest {
@Test @Test
public void readCsv_nullElement_succeeds() throws IOException, CsvException { public void readCsv_nullElement_succeeds() throws IOException, CsvException {
final MultipartFile file = new MockMultipartFile("weather-small", Files.readAllBytes(ResourceUtils.getFile("classpath:weather-small.csv").toPath())); final MultipartFile file = new MockMultipartFile("csv_01", Files.readAllBytes(ResourceUtils.getFile("classpath:csv/csv_01.csv").toPath()));
final TableInsertDto request = TableInsertDto.builder() final TableInsertDto request = TableInsertDto.builder()
.delimiter(';') .delimiter(';')
.skipHeader(true) .skipHeader(true)
...@@ -158,7 +158,7 @@ public class TableServiceUnitTest extends BaseUnitTest { ...@@ -158,7 +158,7 @@ public class TableServiceUnitTest extends BaseUnitTest {
@Test @Test
public void readCsv_skipheader_succeeds() throws IOException, CsvException { public void readCsv_skipheader_succeeds() throws IOException, CsvException {
final MultipartFile file = new MockMultipartFile("weather-small", Files.readAllBytes(ResourceUtils.getFile("classpath:weather-small.csv").toPath())); final MultipartFile file = new MockMultipartFile("csv_01", Files.readAllBytes(ResourceUtils.getFile("classpath:csv/csv_01.csv").toPath()));
final TableInsertDto request = TableInsertDto.builder() final TableInsertDto request = TableInsertDto.builder()
.delimiter(';') .delimiter(';')
.skipHeader(false) .skipHeader(false)
......
fda-table-service/rest-service/src/test/resources/csv/csv_02.csv 0 → 100644
+ 15660
− 0
View file @ 8be84589
This diff is collapsed.
fda-table-service/rest-service/src/test/resources/csv/csv_03.csv 0 → 100644
+ 15277
− 0
View file @ 8be84589
This diff is collapsed.
fda-table-service/rest-service/src/test/resources/csv/csv_04.csv 0 → 100644
+ 15145
− 0
View file @ 8be84589
This diff is collapsed.
fda-table-service/rest-service/src/test/resources/csv/csv_05.csv 0 → 100644
+ 9136
− 0
View file @ 8be84589
This diff is collapsed.
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment