Skip to content
Snippets Groups Projects
Commit ed2001ff authored by Martin Weise's avatar Martin Weise
Browse files

Added store tests 

Former-commit-id: 93296830
parent 4df3ec43
No related branches found
No related tags found
1 merge request!42Fixed the query service tests
Show whitespace changes
Inline Side-by-side
Showing
with 614 additions and 130 deletions
fda-metadata-db/api/src/main/java/at/tuwien/api/database/query/QueryDto.java
+ 20
− 0
View file @ ed2001ff
......@@ -8,6 +8,7 @@ import lombok.*;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import java.sql.Timestamp;
import java.time.Instant;
import java.util.List;
......@@ -67,4 +68,23 @@ public class QueryDto {
@NotNull
private Instant lastModified;
/**
* Returns the ordered list of prepared values for the {@link org.hibernate.query.NativeQuery}.
*
* @return The ordered list of prepared values
*/
public String[] getPreparedValues() {
return new String[]{
this.getDoi(),
this.getTitle(),
this.getDescription(),
this.getQuery(),
this.getQueryHash(),
Timestamp.from(this.getExecutionTimestamp())
.toString(),
this.getResultHash(),
String.valueOf(this.getResultNumber())
};
}
}
fda-query-service/pom.xml
+ 2
− 0
View file @ ed2001ff
......@@ -210,6 +210,8 @@
<exclude>at/tuwien/handlers/**/*</exclude>
<exclude>at/tuwien/exception/**/*</exclude>
<exclude>at/tuwien/config/**/*</exclude>
<exclude>**/RabbitMqServiceImpl.class</exclude>
<exclude>**/ServiceSeeder.class</exclude>
<exclude>**/FdaQueryServiceApplication.class</exclude>
</excludes>
</configuration>
......
fda-query-service/rest-service/src/main/resources/application-docker.yml
+ 1
− 0
View file @ ed2001ff
......@@ -21,6 +21,7 @@ logging:
level:
root: warn
at.tuwien.: debug
at.tuwien.mapper.: trace
eureka:
instance.hostname: fda-query-service
client.serviceUrl.defaultZone: http://fda-discovery-service:9090/eureka/
......
fda-query-service/rest-service/src/main/resources/application.yml
+ 1
− 0
View file @ ed2001ff
......@@ -21,6 +21,7 @@ logging:
level:
root: warn
at.tuwien.: debug
at.tuwien.mapper.: trace
eureka:
instance.hostname: fda-query-service
client.serviceUrl.defaultZone: http://localhost:9090/eureka/
......
fda-query-service/rest-service/src/test/java/at/tuwien/service/CsvServiceIntegrationTest.java
+ 37
− 6
View file @ ed2001ff
package at.tuwien.service;
import at.tuwien.BaseUnitTest;
import at.tuwien.api.database.table.TableCsvDto;
import at.tuwien.config.DockerConfig;
import at.tuwien.config.ReadyConfig;
import at.tuwien.exception.*;
import at.tuwien.repository.elastic.DatabaseRepository;
import at.tuwien.repository.jpa.TableRepository;
import com.github.dockerjava.api.command.CreateContainerResponse;
import com.github.dockerjava.api.exception.NotModifiedException;
import com.github.dockerjava.api.model.Bind;
import com.github.dockerjava.api.model.Network;
import com.opencsv.exceptions.CsvException;
import com.rabbitmq.client.Channel;
import lombok.extern.log4j.Log4j2;
import org.junit.jupiter.api.AfterAll;
......@@ -24,11 +27,15 @@ import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Optional;
import static at.tuwien.config.DockerConfig.dockerClient;
import static at.tuwien.config.DockerConfig.hostConfig;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.when;
@Log4j2
@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
......@@ -48,12 +55,9 @@ public class CsvServiceIntegrationTest extends BaseUnitTest {
@Autowired
private TableRepository tableRepository;
/**
* We need a container to test the CRUD operations as of now it is unfeasible to determine the correctness of the
* operations without a live container
*
* @throws InterruptedException Sleep interrupted.
*/
@Autowired
private DatabaseRepository databaseRepository;
@BeforeAll
public static void beforeAll() throws InterruptedException {
afterAll();
......@@ -65,6 +69,13 @@ public class CsvServiceIntegrationTest extends BaseUnitTest {
.withSubnet("172.28.0.0/16")))
.withEnableIpv6(false)
.exec();
dockerClient.createNetworkCmd()
.withName("fda-public")
.withIpam(new Network.Ipam()
.withConfig(new Network.Ipam.Config()
.withSubnet("172.29.0.0/16")))
.withEnableIpv6(false)
.exec();
/* create container */
final String bind = new File("./src/test/resources/weather").toPath().toAbsolutePath() + ":/docker-entrypoint-initdb.d";
log.trace("container bind {}", bind);
......@@ -76,9 +87,20 @@ public class CsvServiceIntegrationTest extends BaseUnitTest {
.withEnv("MARIADB_USER=mariadb", "MARIADB_PASSWORD=mariadb", "MARIADB_ROOT_PASSWORD=mariadb", "MARIADB_DATABASE=weather")
.withBinds(Bind.parse(bind))
.exec();
final String bind2 = new File("./src/test/resources/webserver").toPath().toAbsolutePath() + ":/usr/share/nginx/html:ro";
log.trace("container bind {}", bind2);
final CreateContainerResponse response2 = dockerClient.createContainerCmd(CONTAINER_NGINX_IMAGE + ":" + CONTAINER_NGINX_TAG)
.withHostConfig(hostConfig.withNetworkMode("fda-public"))
.withName(CONTAINER_NGINX_NAME)
.withIpv4Address(CONTAINER_NGINX_IP)
.withHostName(CONTAINER_NGINX_INTERNALNAME)
.withBinds(Bind.parse(bind2))
.exec();
/* start */
CONTAINER_1.setHash(response.getId());
CONTAINER_2.setHash(response2.getId());
DockerConfig.startContainer(CONTAINER_1);
DockerConfig.startContainer(CONTAINER_2);
}
@AfterAll
......@@ -123,4 +145,13 @@ public class CsvServiceIntegrationTest extends BaseUnitTest {
assertTrue(response.exists());
}
@Test
public void read_url_succeeds() throws IOException, CsvException, TableNotFoundException, DatabaseNotFoundException {
final String location = "http://" + CONTAINER_NGINX_IP + "/weather_aus.csv";
/* test */
final TableCsvDto response = dataService.read(DATABASE_1_ID, TABLE_1_ID, location);
assertEquals(3, response.getData().size());
}
}
fda-query-service/rest-service/src/test/java/at/tuwien/service/CsvServiceUnitTest.java
+ 15
− 6
View file @ ed2001ff
......@@ -55,12 +55,6 @@ public class CsvServiceUnitTest extends BaseUnitTest {
@MockBean
private TableRepository tableRepository;
/**
* We need a container to test the CRUD operations as of now it is unfeasible to determine the correctness of the
* operations without a live container
*
* @throws InterruptedException Sleep interrupted.
*/
@BeforeAll
public static void beforeAll() throws InterruptedException {
afterAll();
......@@ -141,6 +135,21 @@ public class CsvServiceUnitTest extends BaseUnitTest {
assertEquals(1000, response.getData().size());
}
@Test
public void read2_succeeds() throws IOException, CsvException, TableNotFoundException, DatabaseNotFoundException {
final String location = "test:csv/csv_01.csv";
/* mock */
when(databaseRepository.findById(DATABASE_1_ID))
.thenReturn(Optional.of(DATABASE_1));
when(tableRepository.findByDatabaseAndId(DATABASE_1, TABLE_1_ID))
.thenReturn(Optional.of(TABLE_1));
/* test */
final TableCsvDto response = dataService.read(DATABASE_1_ID, TABLE_1_ID, location);
assertEquals(1001, response.getData().size());
}
@Test
public void read_nullElement_succeeds() throws IOException, CsvException, TableNotFoundException,
DatabaseNotFoundException {
......
fda-query-service/rest-service/src/test/java/at/tuwien/service/MessageQueueServiceUnitTest.java 0 → 100644
+ 97
− 0
View file @ ed2001ff
package at.tuwien.service;
import at.tuwien.BaseUnitTest;
import at.tuwien.config.ReadyConfig;
import at.tuwien.exception.*;
import com.rabbitmq.client.Channel;
import lombok.extern.log4j.Log4j2;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import java.io.IOException;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doThrow;
@SpringBootTest
@ExtendWith(SpringExtension.class)
@Log4j2
public class MessageQueueServiceUnitTest extends BaseUnitTest {
@MockBean
private ReadyConfig readyConfig;
@MockBean
private Channel channel;
@MockBean
private QueryService queryService;
@Autowired
private MessageQueueService messageQueueService;
@BeforeEach
public void beforeEach() {
TABLE_1.setDatabase(DATABASE_1);
TABLE_2.setDatabase(DATABASE_2);
}
@Test
public void createUserConsumer_imageNotSupported_succeeds() throws IOException, TableNotFoundException, TableMalformedException,
DatabaseNotFoundException, ImageNotSupportedException {
/* mock */
doThrow(ImageNotSupportedException.class)
.when(queryService)
.insert(eq(DATABASE_1_ID), eq(TABLE_1_ID), any());
/* test */
messageQueueService.createUserConsumer(TABLE_1);
}
@Test
public void createUserConsumer_tableFailed_succeeds() throws IOException, TableNotFoundException, TableMalformedException,
DatabaseNotFoundException, ImageNotSupportedException {
/* mock */
doThrow(TableMalformedException.class)
.when(queryService)
.insert(eq(DATABASE_1_ID), eq(TABLE_1_ID), any());
/* test */
messageQueueService.createUserConsumer(TABLE_1);
}
@Test
public void createUserConsumer_dbNotFound_succeeds() throws IOException, TableNotFoundException, TableMalformedException,
DatabaseNotFoundException, ImageNotSupportedException {
/* mock */
doThrow(DatabaseNotFoundException.class)
.when(queryService)
.insert(eq(DATABASE_1_ID), eq(TABLE_1_ID), any());
/* test */
messageQueueService.createUserConsumer(TABLE_1);
}
@Test
public void createUserConsumer_tableNotFound_succeeds() throws IOException, TableNotFoundException, TableMalformedException,
DatabaseNotFoundException, ImageNotSupportedException {
/* mock */
doThrow(TableNotFoundException.class)
.when(queryService)
.insert(eq(DATABASE_1_ID), eq(TABLE_1_ID), any());
/* test */
messageQueueService.createUserConsumer(TABLE_1);
}
}
fda-query-service/rest-service/src/test/java/at/tuwien/service/QueryStoreServiceUnitTest.java deleted 100644 → 0
+ 0
− 33
View file @ 4df3ec43
package at.tuwien.service;
import at.tuwien.BaseUnitTest;
import at.tuwien.config.ReadyConfig;
import at.tuwien.repository.jpa.DatabaseRepository;
import at.tuwien.service.impl.StoreServiceImpl;
import com.rabbitmq.client.Channel;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.junit.jupiter.SpringExtension;
@ExtendWith(SpringExtension.class)
@SpringBootTest
public class QueryStoreServiceUnitTest extends BaseUnitTest {
@MockBean
private Channel channel;
@MockBean
private ReadyConfig readyConfig;
@Autowired
private QueryService queryService;
@Autowired
private StoreServiceImpl queryStoreService;
@MockBean
private DatabaseRepository databaseRepository;
}
fda-query-service/rest-service/src/test/java/at/tuwien/service/StoreServiceIntegrationTest.java 0 → 100644
+ 283
− 0
View file @ ed2001ff
package at.tuwien.service;
import at.tuwien.BaseUnitTest;
import at.tuwien.api.database.query.ExecuteQueryDto;
import at.tuwien.api.database.query.QueryDto;
import at.tuwien.api.database.query.QueryResultDto;
import at.tuwien.config.DockerConfig;
import at.tuwien.config.ReadyConfig;
import at.tuwien.exception.DatabaseNotFoundException;
import at.tuwien.exception.ImageNotSupportedException;
import at.tuwien.exception.QueryNotFoundException;
import at.tuwien.exception.QueryStoreException;
import at.tuwien.repository.jpa.TableRepository;
import com.github.dockerjava.api.command.CreateContainerResponse;
import com.github.dockerjava.api.exception.NotModifiedException;
import com.github.dockerjava.api.model.Bind;
import com.github.dockerjava.api.model.Network;
import com.rabbitmq.client.Channel;
import lombok.extern.log4j.Log4j2;
import org.junit.jupiter.api.*;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.springframework.transaction.annotation.Transactional;
import java.io.File;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import static at.tuwien.config.DockerConfig.dockerClient;
import static at.tuwien.config.DockerConfig.hostConfig;
import static java.lang.Thread.activeCount;
import static org.junit.jupiter.api.Assertions.*;
@Log4j2
@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
@ExtendWith(SpringExtension.class)
@SpringBootTest
public class StoreServiceIntegrationTest extends BaseUnitTest {
@MockBean
private Channel channel;
@MockBean
private ReadyConfig readyConfig;
@Autowired
private StoreService storeService;
@Autowired
private TableRepository tableRepository;
@BeforeAll
public static void beforeAll() throws InterruptedException {
afterAll();
/* create network */
dockerClient.createNetworkCmd()
.withName("fda-userdb")
.withIpam(new Network.Ipam()
.withConfig(new Network.Ipam.Config()
.withSubnet("172.28.0.0/16")))
.withEnableIpv6(false)
.exec();
/* create container */
final String bind = new File("./src/test/resources/weather").toPath().toAbsolutePath() + ":/docker-entrypoint-initdb.d";
log.trace("container bind {}", bind);
final CreateContainerResponse response = dockerClient.createContainerCmd(IMAGE_1_REPOSITORY + ":" + IMAGE_1_TAG)
.withHostConfig(hostConfig.withNetworkMode("fda-userdb"))
.withName(CONTAINER_1_INTERNALNAME)
.withIpv4Address(CONTAINER_1_IP)
.withHostName(CONTAINER_1_INTERNALNAME)
.withEnv("MARIADB_USER=mariadb", "MARIADB_PASSWORD=mariadb", "MARIADB_ROOT_PASSWORD=mariadb", "MARIADB_DATABASE=weather")
.withBinds(Bind.parse(bind))
.exec();
/* start */
CONTAINER_1.setHash(response.getId());
DockerConfig.startContainer(CONTAINER_1);
}
@AfterAll
public static void afterAll() {
/* stop containers and remove them */
dockerClient.listContainersCmd()
.withShowAll(true)
.exec()
.forEach(container -> {
log.info("Delete container {}", Arrays.asList(container.getNames()));
try {
dockerClient.stopContainerCmd(container.getId()).exec();
} catch (NotModifiedException e) {
// ignore
}
dockerClient.removeContainerCmd(container.getId()).exec();
});
/* remove networks */
dockerClient.listNetworksCmd()
.exec()
.stream()
.filter(n -> n.getName().startsWith("fda"))
.forEach(network -> {
log.info("Delete network {}", network.getName());
dockerClient.removeNetworkCmd(network.getId()).exec();
});
}
@BeforeEach
@Transactional
public void beforeEach() {
TABLE_1.setDatabase(DATABASE_1);
TABLE_2.setDatabase(DATABASE_2);
tableRepository.save(TABLE_1);
tableRepository.save(TABLE_2);
}
@Test
public void findAll_succeeds() throws QueryStoreException, DatabaseNotFoundException, ImageNotSupportedException {
/* mock */
storeService.delete(DATABASE_1_ID);
storeService.create(DATABASE_1_ID);
/* test */
final List<QueryDto> response = storeService.findAll(DATABASE_1_ID);
assertEquals(0, response.size());
}
@Test
public void findAll_noStore_fails() throws DatabaseNotFoundException, ImageNotSupportedException {
/* mock */
storeService.delete(DATABASE_1_ID);
/* test */
assertThrows(QueryStoreException.class, () -> {
storeService.findAll(DATABASE_1_ID);
});
}
@Test
public void findOne_succeeds() throws DatabaseNotFoundException, ImageNotSupportedException, QueryStoreException,
QueryNotFoundException {
final QueryResultDto request = QueryResultDto.builder()
.result(List.of(Map.of("key", "val")))
.build();
final ExecuteQueryDto query = ExecuteQueryDto.builder()
.title(QUERY_1_TITLE)
.description(QUERY_1_DESCRIPTION)
.query(QUERY_1_STATEMENT)
.build();
/* mock */
storeService.delete(DATABASE_1_ID);
storeService.create(DATABASE_1_ID);
storeService.insert(DATABASE_1_ID, request, query);
/* test */
final QueryDto response = storeService.findOne(DATABASE_1_ID, QUERY_1_ID);
assertEquals(QUERY_1_ID, response.getId());
assertEquals(QUERY_1_TITLE, response.getTitle());
assertEquals(QUERY_1_DESCRIPTION, response.getDescription());
assertEquals(QUERY_1_STATEMENT, response.getQuery());
assertNotNull(response.getQueryHash());
}
@Test
public void findOne_notFound_fails() throws DatabaseNotFoundException, ImageNotSupportedException {
storeService.create(DATABASE_1_ID);
/* test */
assertThrows(QueryNotFoundException.class, () -> {
storeService.findOne(DATABASE_1_ID, QUERY_1_ID);
});
}
@Test
public void create_succeeds() throws DatabaseNotFoundException, ImageNotSupportedException {
/* mock */
storeService.delete(DATABASE_1_ID);
/* test */
storeService.create(DATABASE_1_ID);
}
@Test
public void create_dbNotFound() throws DatabaseNotFoundException, ImageNotSupportedException {
/* mock */
storeService.delete(DATABASE_1_ID);
/* test */
assertThrows(DatabaseNotFoundException.class, () -> {
storeService.create(9999L);
});
}
@Test
public void delete_succeeds() throws DatabaseNotFoundException, ImageNotSupportedException {
/* test */
storeService.delete(DATABASE_1_ID);
}
@Test
public void delete_dbNotFound_succeeds() {
/* test */
assertThrows(DatabaseNotFoundException.class, () -> {
storeService.delete(9999L);
});
}
// FIXME somehow inserts 3 tuples at once
@Test
@Disabled
public void insert_succeeds() throws DatabaseNotFoundException, ImageNotSupportedException, QueryStoreException {
final QueryResultDto request = QueryResultDto.builder()
.result(List.of(Map.of("id", "1")))
.build();
final ExecuteQueryDto query = ExecuteQueryDto.builder()
.title(QUERY_1_TITLE)
.description(QUERY_1_DESCRIPTION)
.query(QUERY_1_STATEMENT)
.build();
/* mock */
storeService.delete(DATABASE_1_ID);
storeService.create(DATABASE_1_ID);
/* test */
final QueryDto response = storeService.insert(DATABASE_1_ID, request, query);
assertEquals(QUERY_1_ID, response.getId());
assertEquals(QUERY_1_TITLE, response.getTitle());
assertEquals(QUERY_1_DESCRIPTION, response.getDescription());
assertEquals(QUERY_1_STATEMENT, response.getQuery());
}
@Test
public void insert_dbNotFound_fails() throws DatabaseNotFoundException, ImageNotSupportedException {
final QueryResultDto request = QueryResultDto.builder()
.result(List.of(Map.of("id", "1")))
.build();
final ExecuteQueryDto query = ExecuteQueryDto.builder()
.title(QUERY_1_TITLE)
.description(QUERY_1_DESCRIPTION)
.query(QUERY_1_STATEMENT)
.build();
/* mock */
storeService.delete(DATABASE_1_ID);
storeService.create(DATABASE_1_ID);
/* test */
assertThrows(DatabaseNotFoundException.class, () -> {
storeService.insert(9999L, request, query);
});
}
@Test
public void insert_noStore_fails() throws DatabaseNotFoundException, ImageNotSupportedException {
final QueryResultDto request = QueryResultDto.builder()
.result(List.of(Map.of("id", "1")))
.build();
final ExecuteQueryDto query = ExecuteQueryDto.builder()
.title(QUERY_1_TITLE)
.description(QUERY_1_DESCRIPTION)
.query(QUERY_1_STATEMENT)
.build();
/* mock */
storeService.delete(DATABASE_1_ID);
/* test */
assertThrows(QueryStoreException.class, () -> {
storeService.insert(DATABASE_1_ID, request, query);
});
}
}
fda-query-service/rest-service/src/test/resources/webserver/weather_aus.csv 0 → 100644
+ 3
− 0
View file @ ed2001ff
1,2008-12-01,Albury,13.4,0.6
2,2008-12-02,Albury,7.4,0
3,2008-12-03,Albury,12.9,0
\ No newline at end of file
fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java
+ 23
− 27
View file @ ed2001ff
......@@ -11,20 +11,17 @@ import at.tuwien.entities.database.table.Table;
import at.tuwien.entities.database.table.columns.TableColumn;
import at.tuwien.exception.ImageNotSupportedException;
import at.tuwien.exception.QueryStoreException;
import org.apache.commons.codec.digest.DigestUtils;
import org.mapstruct.Mapper;
import org.mapstruct.Mapping;
import org.mapstruct.Mappings;
import org.mapstruct.Named;
import org.mariadb.jdbc.MariaDbBlob;
import javax.validation.Valid;
import javax.validation.constraints.NotNull;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.math.BigInteger;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.sql.Timestamp;
import java.text.Normalizer;
import java.time.*;
import java.time.format.DateTimeFormatter;
......@@ -62,7 +59,7 @@ public interface QueryMapper {
return slug.toLowerCase(Locale.ENGLISH);
}
default List<QueryDto> resultListToQueryStoreQueryList(List<?> data) throws QueryStoreException {
default List<QueryDto> resultListToQueryStoreQueryList(List<?> data) {
final List<QueryDto> queries = new LinkedList<>();
final Iterator<?> iterator = data.iterator();
while (iterator.hasNext()) {
......@@ -71,12 +68,15 @@ public interface QueryMapper {
.id(Long.valueOf(String.valueOf(row[0])))
.doi(String.valueOf(row[1]))
.title(String.valueOf(row[2]))
.query(String.valueOf(row[3]))
.queryHash(String.valueOf(row[4]))
.executionTimestamp(Instant.parse(String.valueOf(row[5])))
.resultHash(String.valueOf(row[6]))
.resultNumber(Long.valueOf(String.valueOf(row[7])))
.created(Instant.parse(String.valueOf(row[8])))
.description(String.valueOf(row[3]))
.query(String.valueOf(row[4]))
.queryHash(String.valueOf(row[5]))
.executionTimestamp(Timestamp.valueOf(String.valueOf(row[6]))
.toInstant())
.resultHash(String.valueOf(row[7]))
.resultNumber(Long.valueOf(String.valueOf(row[8])))
.created(Timestamp.valueOf(String.valueOf(row[9]))
.toInstant())
.build());
}
return queries;
......@@ -217,21 +217,17 @@ public interface QueryMapper {
}
}
default QueryDto queryResultDtoToQueryDto(QueryResultDto data, ExecuteQueryDto metadata) throws QueryStoreException {
try {
return QueryDto.builder()
default QueryDto queryResultDtoToQueryDto(QueryResultDto data, ExecuteQueryDto metadata) {
final QueryDto query = QueryDto.builder()
.title(metadata.getTitle())
.description(metadata.getDescription())
.query(metadata.getQuery())
.resultNumber(Long.parseLong(String.valueOf(data.getResult().size())))
.resultHash(Arrays.toString(MessageDigest.getInstance("SHA256")
.digest(data.getResult()
.toString()
.getBytes())))
.resultHash(DigestUtils.sha256Hex(data.getResult().toString()))
.executionTimestamp(Instant.now())
.build();
} catch (NoSuchAlgorithmException e) {
throw new QueryStoreException("Failed to find sha256 algorithm", e);
}
log.trace("map to query {}", query);
return query;
}
}
fda-query-service/services/src/main/java/at/tuwien/mapper/StoreMapper.java
+ 40
− 12
View file @ ed2001ff
......@@ -3,52 +3,80 @@ package at.tuwien.mapper;
import at.tuwien.api.database.query.QueryDto;
import org.mapstruct.Mapper;
import java.sql.Timestamp;
import java.util.List;
@Mapper(componentModel = "spring")
public interface StoreMapper {
org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(StoreMapper.class);
default String createRawQueryStoreSequenceQuery() {
return "CREATE SEQUENCE IF NOT EXISTS `seq_querystore_id` START WITH 1 INCREMENT BY 1;";
final String query = "CREATE SEQUENCE IF NOT EXISTS `seq_querystore_id` START WITH 1 INCREMENT BY 1;";
log.trace("create store seq '{}'", query);
return query;
}
default String createRawQueryStoreQuery() {
final StringBuilder query = new StringBuilder("CREATE TABLE IF NOT EXISTS `userdb_querystore` (")
.append("id BIGINT NOT NULL PRIMARY KEY DEFAULT NEXTVAL(`seq_querystore_id`),")
.append("doi VARCHAR(255) UNIQUE NOT NULL, ")
.append("doi VARCHAR(255), ")
.append("title VARCHAR(255) NOT NULL, ")
.append("description TEXT NOT NULL, ")
.append("query TEXT NOT NULL, ")
.append("query_hash VARCHAR(255), ")
.append("execution_timestamp TIMESTAMP WITH TIMEZONE, ")
.append("execution_timestamp TIMESTAMP, ")
.append("result_hash VARCHAR(255), ")
.append("result_number BIGINT, ")
.append("created_at TIMESTAMP WITH TIMEZONE DEFAULT NOW());");
.append("created_at TIMESTAMP DEFAULT NOW());");
log.trace("create store '{}'", query);
return query.toString();
}
default String findOneRawQueryStoreQuery() {
final StringBuilder query = new StringBuilder("SELECT * FROM `userdb_querystore` WHERE id = ?1;");
log.trace("find one query '{}'", query);
final StringBuilder query = new StringBuilder("SELECT ")
.append(String.join(",", List.of("`id`", "`doi`", "`title`", "`description`", "`query`", "`query_hash`", "`execution_timestamp`",
"`result_hash`", "`result_number`", "`created_at`")))
.append(" FROM `userdb_querystore` WHERE `id` = ?1");
log.trace("find one '{}'", query);
return query.toString();
}
default String findAllRawQueryStoreQuery() {
final StringBuilder query = new StringBuilder("SELECT * FROM `userdb_querystore`;");
log.trace("find all query '{}'", query);
final StringBuilder query = new StringBuilder("SELECT ")
.append(String.join(",", List.of("`id`", "`doi`", "`title`", "`description`", "`query`", "`query_hash`", "`execution_timestamp`",
"`result_hash`", "`result_number`", "`created_at`")))
.append(" FROM `userdb_querystore`;");
log.trace("find all '{}'", query);
return query.toString();
}
default String deleteRawQueryStoreSequenceQuery() {
return "DROP SEQUENCE `seq_querystore_id`;";
final String query = "DROP SEQUENCE IF EXISTS `seq_querystore_id`;";
log.trace("delete store seq '{}'", query);
return query;
}
default String insertRawQueryStoreQuery(QueryDto data) {
return "";
default String insertRawQueryStoreQuery() {
final StringBuilder query = new StringBuilder("INSERT INTO `userdb_querystore` (")
.append(String.join(",", List.of("`doi`", "`title`", "`description`", "`query`", "`query_hash`", "`execution_timestamp`",
"`result_hash`", "`result_number`")))
.append(") VALUES (?1) RETURNING id");
log.trace("insert '{}'", query);
return query.toString();
}
default String quote(String data) {
if (data == null) {
return null;
}
return "'" + data + "'";
}
default String deleteRawQueryStoreQuery() {
return "DROP TABLE `userdb_querystore`;";
final String query = "DROP TABLE IF EXISTS `userdb_querystore`;";
log.trace("delete store '{}'", query);
return query;
}
}
fda-query-service/services/src/main/java/at/tuwien/service/MessageQueueService.java
+ 0
− 1
View file @ ed2001ff
package at.tuwien.service;
import at.tuwien.entities.database.table.Table;
import org.springframework.transaction.annotation.Transactional;
import java.io.IOException;
......
fda-query-service/services/src/main/java/at/tuwien/service/StoreService.java
+ 11
− 0
View file @ ed2001ff
......@@ -55,6 +55,17 @@ public interface StoreService {
*/
void delete(Long databaseId) throws ImageNotSupportedException, DatabaseNotFoundException;
/**
* Inserts a query and metadata to the query store of a given database id
*
* @param databaseId The database id.
* @param query The query.
* @param metadata The metadata.
* @return The stored query on success
* @throws QueryStoreException The query store raised some error
* @throws DatabaseNotFoundException The database id was not found in the metadata database
* @throws ImageNotSupportedException The image is not supported
*/
QueryDto insert(Long databaseId, QueryResultDto query, ExecuteQueryDto metadata) throws QueryStoreException,
DatabaseNotFoundException, ImageNotSupportedException;
}
fda-query-service/services/src/main/java/at/tuwien/service/impl/FlatFileServiceImpl.java
+ 2
− 0
View file @ ed2001ff
......@@ -50,12 +50,14 @@ public class FlatFileServiceImpl implements DataService {
}
@Override
@Transactional
public TableCsvDto read(Long databaseId, Long tableId, String location) throws TableNotFoundException,
DatabaseNotFoundException, IOException, CsvException {
return read(databaseId, tableId, location, ',', false, null, "0", "1");
}
@Override
@Transactional
public TableCsvDto read(Long databaseId, Long tableId, String location, Character separator, Boolean skipHeader, String nullElement,
String falseElement, String trueElement) throws TableNotFoundException,
DatabaseNotFoundException, IOException, CsvException {
......
fda-query-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java
+ 1
− 0
View file @ ed2001ff
......@@ -44,6 +44,7 @@ public abstract class HibernateConnector {
.setProperty("hibernate.connection.password", password)
.setProperty("hibernate.connection.driver_class", database.getContainer().getImage().getDriverClass())
.setProperty("hibernate.connection.pool_size", String.valueOf(POOL_SIZE))
.setProperty("hibernate.generate_statistics", "true")
.setProperty("hibernate.dialect", database.getContainer().getImage().getDialect())
.setProperty("hibernate.current_session_context_class", SESSION_CONTEXT);
return configuration.buildSessionFactory();
......
fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java
+ 12
− 10
View file @ ed2001ff
......@@ -14,7 +14,6 @@ import at.tuwien.service.TableService;
import lombok.NonNull;
import lombok.extern.log4j.Log4j2;
import org.hibernate.Session;
import org.hibernate.Transaction;
import org.hibernate.exception.SQLGrammarException;
import org.hibernate.query.NativeQuery;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -22,7 +21,6 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.persistence.PersistenceException;
import javax.validation.Valid;
import java.time.DateTimeException;
import java.time.Instant;
......@@ -64,12 +62,13 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
/* run query */
final Session session = getSessionFactory(table.getDatabase())
.openSession();
final Transaction transaction = session.beginTransaction();
session.beginTransaction();
/* prepare the statement */
final NativeQuery<?> query = session.createSQLQuery(data.getQuery());
try {
log.info("Query affected {} rows", query.executeUpdate());
transaction.commit();
session.getTransaction()
.commit();
} catch (SQLGrammarException e) {
throw new QueryMalformedException("Query not valid for this database", e);
}
......@@ -81,7 +80,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
}
@Override
@javax.transaction.Transactional
@Transactional
public QueryResultDto findAll(@NonNull Long databaseId, @NonNull Long tableId, Instant timestamp, Long page,
Long size) throws TableNotFoundException, DatabaseNotFoundException,
ImageNotSupportedException, DatabaseConnectionException, TableMalformedException, PaginationException {
......@@ -102,11 +101,12 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
/* run query */
final Session session = getSessionFactory(database)
.openSession();
final Transaction transaction = session.beginTransaction();
session.beginTransaction();
final NativeQuery<?> query = session.createSQLQuery(queryMapper.tableToRawFindAllQuery(table, timestamp, size,
page));
query.executeUpdate();
transaction.commit();
session.getTransaction()
.commit();
final QueryResultDto result;
try {
result = queryMapper.queryTableToQueryResultDto(query.getResultList(), table);
......@@ -128,7 +128,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
if (data.getData().size() == 0 || data.getData().get(0).size() == 0) return;
final Session session = getSessionFactory(database)
.openSession();
final Transaction transaction = session.beginTransaction();
session.beginTransaction();
/* prepare the statement */
final InsertTableRawQuery raw = queryMapper.tableTableCsvDtoToRawInsertQuery(table, data);
final NativeQuery<?> query = session.createSQLQuery(raw.getQuery());
......@@ -139,10 +139,12 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
log.info("Inserted {} tuples", query.executeUpdate());
} catch (PersistenceException e) {
log.error("Could not insert data");
transaction.rollback();
session.getTransaction()
.rollback();
throw new TableMalformedException("Could not insert data", e);
}
transaction.commit();
session.getTransaction()
.commit();
session.close();
}
......
fda-query-service/services/src/main/java/at/tuwien/service/impl/StoreServiceImpl.java
+ 65
− 34
View file @ ed2001ff
......@@ -11,11 +11,13 @@ import at.tuwien.service.DatabaseService;
import at.tuwien.service.StoreService;
import lombok.extern.log4j.Log4j2;
import org.hibernate.Session;
import org.hibernate.Transaction;
import org.hibernate.query.NativeQuery;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.persistence.PersistenceException;
import java.math.BigInteger;
import java.util.List;
@Log4j2
......@@ -34,50 +36,60 @@ public class StoreServiceImpl extends HibernateConnector implements StoreService
}
@Override
@Transactional
public List<QueryDto> findAll(Long databaseId) throws DatabaseNotFoundException, ImageNotSupportedException,
QueryStoreException {
/* find */
final Database database = databaseService.find(databaseId);
if (database.getContainer().getImage().getRepository().equals("mariadb")) {
if (!database.getContainer().getImage().getRepository().equals("mariadb")) {
throw new ImageNotSupportedException("Currently only MariaDB is supported");
}
/* run query */
final Session session = getSessionFactory(database)
.openSession();
final Transaction transaction = session.beginTransaction();
session.beginTransaction();
/* prepare the statement */
final NativeQuery<?> query = session.createSQLQuery(storeMapper.findAllRawQueryStoreQuery());
try {
log.info("Found {} query(s)", query.executeUpdate());
transaction.commit();
} catch (PersistenceException e) {
log.error("Query execution failed");
throw new QueryStoreException("Failed to execute query", e);
}
session.getTransaction()
.commit();
final List<QueryDto> queries = queryMapper.resultListToQueryStoreQueryList(query.getResultList());
session.close();
return queries;
}
@Override
@Transactional
public QueryDto findOne(Long databaseId, Long queryId) throws DatabaseNotFoundException, ImageNotSupportedException,
QueryStoreException, QueryNotFoundException {
/* find */
final Database database = databaseService.find(databaseId);
if (database.getContainer().getImage().getRepository().equals("mariadb")) {
if (!database.getContainer().getImage().getRepository().equals("mariadb")) {
throw new ImageNotSupportedException("Currently only MariaDB is supported");
}
/* run query */
final Session session = getSessionFactory(database)
.openSession();
final Transaction transaction = session.beginTransaction();
session.beginTransaction();
/* prepare the statement */
final NativeQuery<?> query = session.createSQLQuery(storeMapper.findOneRawQueryStoreQuery());
query.setParameter(1, queryId);
try {
log.info("Found {} query(s)", query.executeUpdate());
transaction.commit();
final List<QueryDto> queries = queryMapper.resultListToQueryStoreQueryList(query.getResultList());
if (queries.size() == 0) {
throw new QueryNotFoundException("Query was not found");
} catch (PersistenceException e) {
log.error("Query execution failed");
throw new QueryStoreException("Failed to execute query", e);
}
session.getTransaction()
.commit();
final List<QueryDto> queries = queryMapper.resultListToQueryStoreQueryList(query.getResultList());
if (queries.size() != 1) {
log.error("Invalid state, either 1 or 0 results, got {}", queries.size());
throw new QueryStoreException("Impossible state reached");
throw new QueryNotFoundException("Query was not found");
}
session.close();
return queries.get(0);
......@@ -85,60 +97,79 @@ public class StoreServiceImpl extends HibernateConnector implements StoreService
@Override
@Transactional
public void create(Long databaseId) throws ImageNotSupportedException, DatabaseNotFoundException {
/* find */
final Database database = databaseService.find(databaseId);
if (database.getContainer().getImage().getRepository().equals("mariadb")) {
if (!database.getContainer().getImage().getRepository().equals("mariadb")) {
throw new ImageNotSupportedException("Currently only MariaDB is supported");
}
/* run query */
final Session session = getSessionFactory(database)
.openSession();
final Transaction transaction = session.beginTransaction();
session.beginTransaction();
/* prepare the statement */
session.createSQLQuery(storeMapper.createRawQueryStoreSequenceQuery());
session.createSQLQuery(storeMapper.createRawQueryStoreQuery());
transaction.commit();
session.createSQLQuery(storeMapper.createRawQueryStoreSequenceQuery())
.executeUpdate();
session.createSQLQuery(storeMapper.createRawQueryStoreQuery())
.executeUpdate();
session.getTransaction()
.commit();
session.close();
}
@Override
@Transactional
public void delete(Long databaseId) throws ImageNotSupportedException, DatabaseNotFoundException {
/* find */
final Database database = databaseService.find(databaseId);
if (database.getContainer().getImage().getRepository().equals("mariadb")) {
if (!database.getContainer().getImage().getRepository().equals("mariadb")) {
throw new ImageNotSupportedException("Currently only MariaDB is supported");
}
/* run query */
final Session session = getSessionFactory(database)
.openSession();
final Transaction transaction = session.beginTransaction();
session.beginTransaction();
/* prepare the statement */
session.createSQLQuery(storeMapper.deleteRawQueryStoreSequenceQuery());
session.createSQLQuery(storeMapper.deleteRawQueryStoreQuery());
transaction.commit();
session.createSQLQuery(storeMapper.deleteRawQueryStoreSequenceQuery())
.executeUpdate();
session.createSQLQuery(storeMapper.deleteRawQueryStoreQuery())
.executeUpdate();
session.getTransaction()
.commit();
session.close();
}
@Override
public QueryDto insert(Long databaseId, QueryResultDto query, ExecuteQueryDto metadata) throws QueryStoreException,
@Transactional
public QueryDto insert(Long databaseId, QueryResultDto result, ExecuteQueryDto metadata) throws QueryStoreException,
DatabaseNotFoundException, ImageNotSupportedException {
/* find */
final Database database = databaseService.find(databaseId);
if (database.getContainer().getImage().getRepository().equals("mariadb")) {
if (!database.getContainer().getImage().getRepository().equals("mariadb")) {
throw new ImageNotSupportedException("Currently only MariaDB is supported");
}
/* run query */
final Session session = getSessionFactory(database)
.openSession();
final Transaction transaction = session.beginTransaction();
/* prepare the statement */
final QueryDto data = queryMapper.queryResultDtoToQueryDto(query, metadata);
final NativeQuery<?> query1 = session.createSQLQuery(storeMapper.insertRawQueryStoreQuery(data));
log.debug("Inserted {} query(s)", query1.executeUpdate());
transaction.commit();
final Object[] row = (Object[]) query1.getResultList().get(0);
data.setId(Long.valueOf(String.valueOf(row[0])));
session.beginTransaction();
/* execute the statement */
final QueryDto data = queryMapper.queryResultDtoToQueryDto(result, metadata);
/* store the result in the query store */
final NativeQuery<?> query = session.createSQLQuery(storeMapper.insertRawQueryStoreQuery());
query.setParameterList(1, data.getPreparedValues());
try {
log.debug("Inserted {} query(s)", query.executeUpdate());
} catch (PersistenceException e) {
log.error("Query execution failed");
throw new QueryStoreException("Failed to execute query", e);
}
session.getTransaction()
.commit();
if (query.getResultList().size() != 1) {
throw new QueryStoreException("Failed to get query result");
}
data.setId(((BigInteger) query.getResultList().get(0)).longValue());
session.close();
return data;
}
......
fda-table-service/rest-service/src/test/resources/application.properties
+ 1
− 1
View file @ ed2001ff
......@@ -17,7 +17,7 @@ spring.jpa.hibernate.ddl-auto=create-drop
spring.jpa.show-sql=false
# logging
logging.level.at.tuwien.service.impl.JdbcConnector=debug
logging.level.at.tuwien.service.impl.HibernateConnector=debug
logging.level.at.tuwien.=trace
# disable elasticsearch
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment