Skip to content
Snippets Groups Projects
Commit 5164e17b authored by Martin Weise's avatar Martin Weise
Browse files

Added some amqp test

Former-commit-id: c8fbeadd
parent d14df3fb
No related branches found
No related tags found
1 merge request!42Fixed the query service tests
Showing
with 359 additions and 81 deletions
......@@ -15,6 +15,7 @@ config-docker:
docker image pull -q mysql:8.0 || true > /dev/null
docker image pull -q mariadb:10.5 || true > /dev/null
docker image pull -q rabbitmq:3-alpine || true > /dev/null
docker image pull -q nginx:1.20-alpine || true > /dev/null
config: config-backend config-docker config-frontend
......@@ -181,7 +182,17 @@ registry-staging: registry-staging-tag registry-staging-push
logs:
docker-compose -f docker-compose.prod.yml logs
clean:
clean-maven:
mvn -f ./fda-authentication-service/pom.xml clean
#mvn -f ./fda-citation-service/pom.xml clean
mvn -f ./fda-container-service/pom.xml clean
mvn -f ./fda-database-service/pom.xml clean
mvn -f ./fda-discovery-service/pom.xml clean
mvn -f ./fda-gateway-service/pom.xml clean
mvn -f ./fda-query-service/pom.xml clean
mvn -f ./fda-table-service/pom.xml clean
clean: clean-maven
docker-compose down || true
docker volume rm fda-services_fda-metadata-db-data || true
docker volume rm fda-public || true
......
......@@ -22,7 +22,7 @@ import java.util.List;
@EntityListeners(AuditingEntityListener.class)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
@javax.persistence.Table(name = "mdb_columns")
public class TableColumn {
public class TableColumn implements Comparable<TableColumn> {
@Id
@EqualsAndHashCode.Include
......@@ -104,4 +104,8 @@ public class TableColumn {
@LastModifiedDate
private Instant lastModified;
@Override
public int compareTo(TableColumn tableColumn) {
return Integer.compare(this.ordinalPosition, tableColumn.getOrdinalPosition());
}
}
......@@ -34,7 +34,7 @@
<opencsv.version>5.4</opencsv.version>
<super-csv.version>2.4.0</super-csv.version>
<btrace-client.version>2.2.0</btrace-client.version>
<unleash.version>5.0.1</unleash.version>
<random-utils.version>2.0.0</random-utils.version>
</properties>
<dependencies>
......@@ -101,6 +101,12 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<dependency>
<groupId>com.aventrix.jnanoid</groupId>
<artifactId>jnanoid</artifactId>
<scope>test</scope>
<version>${random-utils.version}</version>
</dependency>
<!-- AMPQ -->
<dependency>
<groupId>com.rabbitmq</groupId>
......
package at.tuwien;
import at.tuwien.api.container.image.ImageEnvItemDto;
import at.tuwien.api.container.image.ImageEnvItemTypeDto;
import at.tuwien.api.database.table.TableCreateDto;
import at.tuwien.api.database.table.columns.ColumnCreateDto;
import at.tuwien.api.database.table.columns.ColumnTypeDto;
......@@ -23,6 +21,13 @@ import static java.time.temporal.ChronoUnit.HOURS;
@TestPropertySource(locations = "classpath:application.properties")
public abstract class BaseUnitTest extends CsvUnitTest {
public final static String DATABASE_NET = "fda-userdb";
public final static String BROKER_IMAGE = "fda-broker-service:latest";
public final static String BROKER_INTERNALNAME = "fda-broker-service";
public final static String BROKER_NET = "fda-public";
public final static String BROKER_IP = "172.29.0.2";
public final static Long DATABASE_1_ID = 1L;
public final static String DATABASE_1_NAME = "Weather";
public final static String DATABASE_1_INTERNALNAME = "weather";
......@@ -60,7 +65,7 @@ public abstract class BaseUnitTest extends CsvUnitTest {
public final static Integer COLUMN_1_1_ORDINALPOS = 0;
public final static Boolean COLUMN_1_1_PRIMARY = true;
public final static String COLUMN_1_1_NAME = "id";
public final static String COLUMN_1_1_INTERNAL_NAME = "mdb_id";
public final static String COLUMN_1_1_INTERNAL_NAME = "id";
public final static TableColumnType COLUMN_1_1_TYPE = TableColumnType.NUMBER;
public final static ColumnTypeDto COLUMN_1_1_TYPE_DTO = ColumnTypeDto.NUMBER;
public final static Boolean COLUMN_1_1_NULL = false;
......@@ -74,7 +79,7 @@ public abstract class BaseUnitTest extends CsvUnitTest {
public final static Integer COLUMN_1_2_ORDINALPOS = 1;
public final static Boolean COLUMN_1_2_PRIMARY = false;
public final static String COLUMN_1_2_NAME = "Date";
public final static String COLUMN_1_2_INTERNAL_NAME = "mdb_date";
public final static String COLUMN_1_2_INTERNAL_NAME = "date";
public final static TableColumnType COLUMN_1_2_TYPE = TableColumnType.DATE;
public final static ColumnTypeDto COLUMN_1_2_TYPE_DTO = ColumnTypeDto.DATE;
public final static Boolean COLUMN_1_2_NULL = true;
......@@ -88,7 +93,7 @@ public abstract class BaseUnitTest extends CsvUnitTest {
public final static Integer COLUMN_1_3_ORDINALPOS = 2;
public final static Boolean COLUMN_1_3_PRIMARY = false;
public final static String COLUMN_1_3_NAME = "Location";
public final static String COLUMN_1_3_INTERNAL_NAME = "mdb_location";
public final static String COLUMN_1_3_INTERNAL_NAME = "location";
public final static TableColumnType COLUMN_1_3_TYPE = TableColumnType.STRING;
public final static ColumnTypeDto COLUMN_1_3_TYPE_DTO = ColumnTypeDto.STRING;
public final static Boolean COLUMN_1_3_NULL = true;
......@@ -102,7 +107,7 @@ public abstract class BaseUnitTest extends CsvUnitTest {
public final static Integer COLUMN_1_4_ORDINALPOS = 3;
public final static Boolean COLUMN_1_4_PRIMARY = false;
public final static String COLUMN_1_4_NAME = "MinTemp";
public final static String COLUMN_1_4_INTERNAL_NAME = "mdb_mintemp";
public final static String COLUMN_1_4_INTERNAL_NAME = "mintemp";
public final static TableColumnType COLUMN_1_4_TYPE = TableColumnType.STRING;
public final static ColumnTypeDto COLUMN_1_4_TYPE_DTO = ColumnTypeDto.STRING;
public final static Boolean COLUMN_1_4_NULL = true;
......@@ -116,7 +121,7 @@ public abstract class BaseUnitTest extends CsvUnitTest {
public final static Integer COLUMN_1_5_ORDINALPOS = 4;
public final static Boolean COLUMN_1_5_PRIMARY = false;
public final static String COLUMN_1_5_NAME = "Rainfall";
public final static String COLUMN_1_5_INTERNAL_NAME = "mdb_rainfall";
public final static String COLUMN_1_5_INTERNAL_NAME = "rainfall";
public final static TableColumnType COLUMN_1_5_TYPE = TableColumnType.NUMBER;
public final static ColumnTypeDto COLUMN_1_5_TYPE_DTO = ColumnTypeDto.NUMBER;
public final static Boolean COLUMN_1_5_NULL = true;
......@@ -130,7 +135,7 @@ public abstract class BaseUnitTest extends CsvUnitTest {
public final static Integer COLUMN_3_1_ORDINALPOS = 0;
public final static Boolean COLUMN_3_1_PRIMARY = false;
public final static String COLUMN_3_1_NAME = "qu";
public final static String COLUMN_3_1_INTERNAL_NAME = "mdb_qu";
public final static String COLUMN_3_1_INTERNAL_NAME = "qu";
public final static TableColumnType COLUMN_3_1_TYPE = TableColumnType.STRING;
public final static ColumnTypeDto COLUMN_3_1_TYPE_DTO = ColumnTypeDto.STRING;
public final static Boolean COLUMN_3_1_NULL = false;
......@@ -144,7 +149,7 @@ public abstract class BaseUnitTest extends CsvUnitTest {
public final static Integer COLUMN_3_2_ORDINALPOS = 1;
public final static Boolean COLUMN_3_2_PRIMARY = false;
public final static String COLUMN_3_2_NAME = "species";
public final static String COLUMN_3_2_INTERNAL_NAME = "mdb_species";
public final static String COLUMN_3_2_INTERNAL_NAME = "species";
public final static TableColumnType COLUMN_3_2_TYPE = TableColumnType.ENUM;
public final static ColumnTypeDto COLUMN_3_2_TYPE_DTO = ColumnTypeDto.ENUM;
public final static Boolean COLUMN_3_2_NULL = false;
......@@ -158,7 +163,7 @@ public abstract class BaseUnitTest extends CsvUnitTest {
public final static Integer COLUMN_3_3_ORDINALPOS = 2;
public final static Boolean COLUMN_3_3_PRIMARY = false;
public final static String COLUMN_3_3_NAME = "score";
public final static String COLUMN_3_3_INTERNAL_NAME = "mdb_score";
public final static String COLUMN_3_3_INTERNAL_NAME = "score";
public final static TableColumnType COLUMN_3_3_TYPE = TableColumnType.STRING;
public final static ColumnTypeDto COLUMN_3_3_TYPE_DTO = ColumnTypeDto.STRING;
public final static Boolean COLUMN_3_3_NULL = false;
......@@ -283,6 +288,16 @@ public abstract class BaseUnitTest extends CsvUnitTest {
public final static String CONTAINER_3_IP = "172.28.0.7";
public final static Instant CONTAINER_3_CREATED = Instant.now().minus(1, HOURS);
public final static Long CONTAINER_NGINX_ID = 4L;
public final static String CONTAINER_NGINX_HASH = "deadbeef";
public final static String CONTAINER_NGINX_IMAGE = "nginx";
public final static String CONTAINER_NGINX_TAG = "1.20-alpine";
public final static String CONTAINER_NGINX_NET = "fda-public";
public final static String CONTAINER_NGINX_NAME = "file-service";
public final static String CONTAINER_NGINX_INTERNALNAME = "fda-test-file-service";
public final static String CONTAINER_NGINX_IP = "172.29.0.3";
public final static Instant CONTAINER_NGINX_CREATED = Instant.now().minus(3, HOURS);
public final static Container CONTAINER_1 = Container.builder()
.id(CONTAINER_1_ID)
.name(CONTAINER_1_NAME)
......@@ -310,6 +325,14 @@ public abstract class BaseUnitTest extends CsvUnitTest {
.containerCreated(CONTAINER_3_CREATED)
.build();
public final static Container CONTAINER_NGINX = Container.builder()
.id(CONTAINER_NGINX_ID)
.name(CONTAINER_NGINX_NAME)
.internalName(CONTAINER_NGINX_INTERNALNAME)
.hash(CONTAINER_NGINX_HASH)
.containerCreated(CONTAINER_NGINX_CREATED)
.build();
public final static List<TableColumn> TABLE_3_COLUMNS = List.of(TableColumn.builder()
.id(COLUMN_3_1_ID)
.ordinalPosition(COLUMN_3_1_ORDINALPOS)
......
......@@ -44,11 +44,20 @@ public class DockerConfig {
dockerClient.startContainerCmd(container.getHash())
.exec();
Thread.sleep(12 * 1000L);
log.debug("container {} was started", container.getHash());
}
public static void stopContainer(Container container) {
final InspectContainerResponse inspect = dockerClient.inspectContainerCmd(container.getHash())
.exec();
log.trace("container {} state {}", container.getHash(), inspect.getState().getStatus());
if (!Objects.equals(inspect.getState().getStatus(), "running")) {
return;
}
log.trace("container {} needs to be stopped", container.getHash());
dockerClient.stopContainerCmd(container.getHash())
.exec();
log.debug("container {} was stopped", container.getHash());
}
}
......@@ -5,10 +5,7 @@ import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Configurable;
import org.springframework.context.annotation.Bean;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.*;
import java.util.Properties;
@Log4j2
......@@ -32,4 +29,15 @@ public class MariaDbConfig {
connection.close();
}
public static boolean contains(Table table, String column, Object expected) throws SQLException {
final String jdbc = "jdbc:mariadb://" + table.getDatabase().getContainer().getInternalName() + "/" + table.getDatabase().getInternalName();
log.trace("connect to database {}", jdbc);
final Connection connection = DriverManager.getConnection(jdbc, "mariadb", "mariadb");
final Statement statement = connection.createStatement();
final ResultSet result = statement.executeQuery("SELECT `" + column + "` FROM " + table.getInternalName() +
" WHERE `" + column + "` = " + expected.toString() + ";");
connection.close();
return result.next();
}
}
package at.tuwien.endpoint;
import at.tuwien.BaseUnitTest;
import at.tuwien.api.database.table.TableCsvDto;
import at.tuwien.config.DockerConfig;
import at.tuwien.config.MariaDbConfig;
import at.tuwien.config.ReadyConfig;
import at.tuwien.endpoints.DataEndpoint;
import at.tuwien.entities.container.Container;
import at.tuwien.exception.*;
import at.tuwien.repository.jpa.ImageRepository;
import at.tuwien.repository.jpa.TableRepository;
import com.aventrix.jnanoid.jnanoid.NanoIdUtils;
import com.github.dockerjava.api.command.CreateContainerResponse;
import com.github.dockerjava.api.exception.NotModifiedException;
import com.github.dockerjava.api.model.Bind;
import com.github.dockerjava.api.model.Network;
import com.rabbitmq.client.Channel;
import lombok.extern.log4j.Log4j2;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import javax.transaction.Transactional;
import java.io.File;
import java.sql.SQLException;
import java.util.*;
import static at.tuwien.config.DockerConfig.dockerClient;
import static at.tuwien.config.DockerConfig.hostConfig;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
@SpringBootTest
@ExtendWith(SpringExtension.class)
@Log4j2
public class DataEndpointQueueIntegrationTest extends BaseUnitTest {
@MockBean
private Channel channel;
@MockBean
private ReadyConfig readyConfig;
@Autowired
private DataEndpoint dataEndpoint;
@Autowired
private TableRepository tableRepository;
@Autowired
private ImageRepository imageRepository;
/**
* We need a container to test the CRUD operations as of now it is unfeasible to determine the correctness of the
* operations without a live container
*
* @throws InterruptedException Sleep interrupted.
*/
@BeforeAll
public static void beforeAll() throws InterruptedException {
afterAll();
/* create networks */
dockerClient.createNetworkCmd()
.withName(BROKER_NET)
.withIpam(new Network.Ipam()
.withConfig(new Network.Ipam.Config()
.withSubnet("172.29.0.0/16")))
.withEnableIpv6(false)
.exec();
dockerClient.createNetworkCmd()
.withName(DATABASE_NET)
.withIpam(new Network.Ipam()
.withConfig(new Network.Ipam.Config()
.withSubnet("172.28.0.0/16")))
.withEnableIpv6(false)
.exec();
/* create broker container */
final CreateContainerResponse response1 = dockerClient.createContainerCmd(BROKER_IMAGE)
.withHostConfig(hostConfig.withNetworkMode(BROKER_NET))
.withName(BROKER_INTERNALNAME)
.withIpv4Address(BROKER_IP)
.withHostName(BROKER_INTERNALNAME)
.withEnv("TZ=Europe/Vienna")
.exec();
dockerClient.startContainerCmd(response1.getId())
.exec();
/* create table container */
final String bind = new File("./src/test/resources/weather").toPath().toAbsolutePath() + ":/docker-entrypoint-initdb.d";
log.trace("container bind {}", bind);
final CreateContainerResponse response2 = dockerClient.createContainerCmd(IMAGE_2_REPOSITORY + ":" + IMAGE_2_TAG)
.withHostConfig(hostConfig.withNetworkMode(DATABASE_NET))
.withName(CONTAINER_1_INTERNALNAME)
.withIpv4Address(CONTAINER_1_IP)
.withHostName(CONTAINER_1_INTERNALNAME)
.withEnv("MARIADB_USER=mariadb", "MARIADB_PASSWORD=mariadb", "MARIADB_ROOT_PASSWORD=mariadb", "MARIADB_DATABASE=weather")
.withBinds(Bind.parse(bind))
.exec();
/* wait */
CONTAINER_1.setHash(response2.getId());
Thread.sleep(10 * 1000);
}
@AfterAll
public static void afterAll() {
/* stop containers and remove them */
dockerClient.listContainersCmd()
.withShowAll(true)
.exec()
.forEach(container -> {
log.info("Delete container {}", Arrays.asList(container.getNames()));
try {
dockerClient.stopContainerCmd(container.getId()).exec();
} catch (NotModifiedException e) {
// ignore
}
dockerClient.removeContainerCmd(container.getId()).exec();
});
/* remove networks */
dockerClient.listNetworksCmd()
.exec()
.stream()
.filter(n -> n.getName().startsWith("fda"))
.forEach(network -> {
log.info("Delete network {}", network.getName());
dockerClient.removeNetworkCmd(network.getId()).exec();
});
}
@BeforeEach
@Transactional
public void beforeEach() {
imageRepository.save(IMAGE_1);
TABLE_1.setDatabase(DATABASE_1);
tableRepository.save(TABLE_1);
}
@Test
public void insertFromTuple_succeeds() throws TableNotFoundException, TableMalformedException,
DatabaseNotFoundException, ImageNotSupportedException, InterruptedException, SQLException {
final TableCsvDto request = TableCsvDto.builder()
.data(List.of(new LinkedHashMap<>() {{
put(COLUMN_1_1_NAME, 1L);
put(COLUMN_1_2_NAME, "2020-12-01");
put(COLUMN_1_3_NAME, "Somewhere");
put(COLUMN_1_4_NAME, 15.0);
put(COLUMN_1_5_NAME, 20.0);
}}))
.build();
/* mock */
DockerConfig.startContainer(CONTAINER_1);
MariaDbConfig.clearDatabase(TABLE_1);
/* test */
dataEndpoint.insertFromTuple(DATABASE_1_ID, TABLE_1_ID, request);
assertTrue(MariaDbConfig.contains(TABLE_1, COLUMN_1_1_NAME, 1L));
assertFalse(MariaDbConfig.contains(TABLE_1, COLUMN_1_1_NAME, 2L));
}
@Test
public void insertFromTuple_10k_succeeds() throws TableNotFoundException, TableMalformedException,
DatabaseNotFoundException, ImageNotSupportedException, InterruptedException, SQLException {
/* config */
final long limit = 10_000L;
/* mock */
final Random random = new Random();
final List<Map<String, Object>> data = new LinkedList<>();
for (long i = 0L; i <= limit; i++) {
final Long id = i;
data.add(new LinkedHashMap<>() {{
put(COLUMN_1_1_NAME, id);
put(COLUMN_1_2_NAME, "2020-01-01");
put(COLUMN_1_3_NAME, NanoIdUtils.randomNanoId());
put(COLUMN_1_4_NAME, random.nextDouble());
put(COLUMN_1_5_NAME, random.nextDouble());
}});
}
final TableCsvDto request = TableCsvDto.builder()
.data(data)
.build();
DockerConfig.startContainer(CONTAINER_1);
MariaDbConfig.clearDatabase(TABLE_1);
/* test */
final long start = System.currentTimeMillis();
dataEndpoint.insertFromTuple(DATABASE_1_ID, TABLE_1_ID, request);
final long end = System.currentTimeMillis();
log.info("Inserted {}k records in {} seconds", limit / 1000, (end - start) / 1000.0);
assertTrue(MariaDbConfig.contains(TABLE_1, COLUMN_1_1_NAME, 1L), "id 1 missing");
assertTrue(MariaDbConfig.contains(TABLE_1, COLUMN_1_1_NAME, 1_000L), "id 1k missing");
assertTrue(MariaDbConfig.contains(TABLE_1, COLUMN_1_1_NAME, 10_000L), "id 10k missing");
assertTrue(MariaDbConfig.contains(TABLE_1, COLUMN_1_1_NAME, limit), "id 100k missing");
}
}
......@@ -49,7 +49,7 @@ import static org.mockito.ArgumentMatchers.any;
@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
@SpringBootTest
@ExtendWith(SpringExtension.class)
public class DataEndpointIntegrationTest extends BaseUnitTest {
public class DataEndpointRestIntegrationTest extends BaseUnitTest {
@MockBean
private Channel channel;
......@@ -100,6 +100,7 @@ public class DataEndpointIntegrationTest extends BaseUnitTest {
public void beforeEach() {
imageRepository.save(IMAGE_1);
databaseRepository.save(DATABASE_1);
TABLE_1.setDatabase(DATABASE_1);
tableRepository.save(TABLE_1);
}
......
......@@ -9,6 +9,7 @@ import at.tuwien.endpoints.TableEndpoint;
import at.tuwien.exception.*;
import at.tuwien.repository.jpa.DatabaseRepository;
import at.tuwien.repository.jpa.TableRepository;
import at.tuwien.service.MessageQueueService;
import at.tuwien.service.impl.TableServiceImpl;
import com.rabbitmq.client.Channel;
import org.junit.jupiter.api.Disabled;
......@@ -21,6 +22,7 @@ import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
......@@ -48,6 +50,9 @@ public class TableEndpointUnitTest extends BaseUnitTest {
@MockBean
private DatabaseRepository databaseRepository;
@MockBean
private MessageQueueService messageQueueService;
@Autowired
private TableEndpoint tableEndpoint;
......@@ -63,18 +68,22 @@ public class TableEndpointUnitTest extends BaseUnitTest {
}
@Test
@Disabled("only works in integration test")
public void create_succeeds() throws DatabaseNotFoundException, ImageNotSupportedException,
TableNotFoundException, DataProcessingException, ArbitraryPrimaryKeysException, TableMalformedException, AmqpException {
TableNotFoundException, DataProcessingException, ArbitraryPrimaryKeysException, TableMalformedException, AmqpException, IOException {
final TableCreateDto request = TableCreateDto.builder()
.name(TABLE_1_NAME)
.description(TABLE_1_DESCRIPTION)
.columns(COLUMNS_CSV01)
.build();
/* mock */
when(tableRepository.findById(TABLE_1_ID))
.thenReturn(Optional.of(TABLE_1));
when(tableService.findById(DATABASE_1_ID, TABLE_1_ID))
.thenReturn(TABLE_1);
doNothing()
.when(messageQueueService)
.create(TABLE_1);
/* test */
final ResponseEntity<TableBriefDto> response = tableEndpoint.create(DATABASE_1_ID, request);
......
......@@ -67,9 +67,9 @@ public class DataServiceIntegrationTest extends BaseUnitTest {
private MariaDataService dataService;
@BeforeAll
public static void beforeAll() throws InterruptedException {
public static void beforeAll() {
afterAll();
/* create network */
/* create networks */
dockerClient.createNetworkCmd()
.withName("fda-userdb")
.withIpam(new Network.Ipam()
......@@ -77,7 +77,14 @@ public class DataServiceIntegrationTest extends BaseUnitTest {
.withSubnet("172.28.0.0/16")))
.withEnableIpv6(false)
.exec();
/* create container */
dockerClient.createNetworkCmd()
.withName("fda-public")
.withIpam(new Network.Ipam()
.withConfig(new Network.Ipam.Config()
.withSubnet("172.29.0.0/16")))
.withEnableIpv6(false)
.exec();
/* create weather container */
final String bind = new File("./src/test/resources/weather").toPath().toAbsolutePath() + ":/docker-entrypoint-initdb.d";
log.trace("container bind {}", bind);
final CreateContainerResponse request = dockerClient.createContainerCmd(IMAGE_2_REPOSITORY + ":" + IMAGE_2_TAG)
......@@ -88,19 +95,19 @@ public class DataServiceIntegrationTest extends BaseUnitTest {
.withEnv("MARIADB_USER=mariadb", "MARIADB_PASSWORD=mariadb", "MARIADB_ROOT_PASSWORD=mariadb", "MARIADB_DATABASE=weather")
.withBinds(Bind.parse(bind))
.exec();
final String bind3 = new File("./src/test/resources/species").toPath().toAbsolutePath() + ":/docker-entrypoint-initdb.d";
log.trace("container 3 bind {}", bind3);
final CreateContainerResponse request3 = dockerClient.createContainerCmd(IMAGE_2_REPOSITORY + ":" + IMAGE_2_TAG)
.withHostConfig(hostConfig.withNetworkMode("fda-userdb"))
.withName(CONTAINER_3_INTERNALNAME)
.withIpv4Address(CONTAINER_3_IP)
.withHostName(CONTAINER_3_INTERNALNAME)
.withEnv("MARIADB_USER=mariadb", "MARIADB_PASSWORD=mariadb", "MARIADB_ROOT_PASSWORD=mariadb", "MARIADB_DATABASE=biomedical")
.withBinds(Bind.parse(bind3))
/* create file container */
final String bind2 = new File("./src/test/resources/csv").toPath().toAbsolutePath() + ":/usr/share/nginx/html:ro";
log.trace("container bind2 {}", bind2);
final CreateContainerResponse request2 = dockerClient.createContainerCmd(CONTAINER_NGINX_IMAGE + ":" + CONTAINER_NGINX_TAG)
.withHostConfig(hostConfig.withNetworkMode(CONTAINER_NGINX_NET))
.withName(CONTAINER_NGINX_NAME)
.withIpv4Address(CONTAINER_NGINX_IP)
.withHostName(CONTAINER_NGINX_INTERNALNAME)
.withBinds(Bind.parse(bind2))
.exec();
/* set hash */
CONTAINER_1.setHash(request.getId());
CONTAINER_3.setHash(request3.getId());
CONTAINER_NGINX.setHash(request2.getId());
}
@AfterAll
......@@ -272,15 +279,17 @@ public class DataServiceIntegrationTest extends BaseUnitTest {
SQLException {
final TableInsertDto request = TableInsertDto.builder()
.skipHeader(true)
.csvLocation("https://sandbox.zenodo.org/api/files/6aca3421-add3-489b-8c4a-35228fe5c683/species_id.csv")
.nullElement("NA")
.csvLocation("http://172.29.0.3/csv_01.csv")
.build();
/* mock */
DockerConfig.startContainer(CONTAINER_3);
MariaDbConfig.clearDatabase(TABLE_3);
DockerConfig.startContainer(CONTAINER_1);
DockerConfig.startContainer(CONTAINER_NGINX);
MariaDbConfig.clearDatabase(TABLE_1);
/* test */
dataService.insertCsv(DATABASE_3_ID, TABLE_3_ID, request);
dataService.insertCsv(DATABASE_1_ID, TABLE_1_ID, request);
}
}
......@@ -17,6 +17,7 @@ spring.jpa.hibernate.ddl-auto=create-drop
spring.jpa.show-sql=false
# logging
logging.level.at.tuwien.service.impl.JdbcConnector=debug
logging.level.at.tuwien.=trace
# disable elasticsearch
......
/* https://sandbox.zenodo.org/api/files/6aca3421-add3-489b-8c4a-35228fe5c683/species_id.csv */
CREATE TABLE maldi_ms_data
(
mdb_qu VARCHAR(255) NOT NULL,
mdb_species VARCHAR(255) NOT NULL,
mdb_score VARCHAR(255) NOT NULL
qu VARCHAR(255) NOT NULL,
species VARCHAR(255) NOT NULL,
score VARCHAR(255) NOT NULL
) WITH SYSTEM VERSIONING;
\ No newline at end of file
/* https://www.kaggle.com/jsphyg/weather-dataset-rattle-package */
CREATE TABLE weather_aus
(
mdb_id BIGINT NOT NULL PRIMARY KEY,
mdb_date DATE NOT NULL,
mdb_mintemp DOUBLE PRECISION NULL,
mdb_location VARCHAR(255) NULL,
mdb_rainfall DOUBLE PRECISION NULL
id BIGINT NOT NULL PRIMARY KEY,
date DATE NOT NULL,
location VARCHAR(255) NULL,
mintemp DOUBLE PRECISION NULL,
rainfall DOUBLE PRECISION NULL
) WITH SYSTEM VERSIONING;
INSERT INTO weather_aus (mdb_id, mdb_date, mdb_location, mdb_mintemp, mdb_rainfall)
VALUES (1, '2008-12-01'::DATE, 13.4, 'Albury', 0.6),
(2, '2008-12-02'::DATE, 7.4, 'Albury', 0),
(3, '2008-12-03'::DATE, 12.9, 'Albury', 0);
\ No newline at end of file
INSERT INTO weather_aus (id, date, location, mintemp, rainfall)
VALUES (1, '2008-12-01'::DATE, 'Albury', 13.4, 0.6),
(2, '2008-12-02'::DATE, 'Albury', 7.4, 0),
(3, '2008-12-03'::DATE, 'Albury', 12.9, 0);
\ No newline at end of file
......@@ -139,7 +139,9 @@ public interface TableMapper {
/* create sequence nonetheless, if it is used or not */
default CreateSequenceFlagsStep tableCreateDtoToCreateSequenceFlagsStep(DSLContext context, TableCreateDto data) {
return context.createSequenceIfNotExists(tableNameToSequenceName(data.getName()));
final String sequenceName = tableNameToSequenceName(data.getName());
log.trace("create sequence {}", sequenceName);
return context.createSequenceIfNotExists(sequenceName);
}
default CreateTableColumnStep tableCreateDtoToCreateTableColumnStep(DSLContext context, TableCreateDto data)
......@@ -238,7 +240,7 @@ public interface TableMapper {
default List<Field<?>> tableToFieldList(Table data) {
return data.getColumns()
.stream()
.sorted(Comparator.comparing(TableColumn::getOrdinalPosition))
.sorted()
.map(c -> field(c.getInternalName()))
.collect(Collectors.toList());
}
......@@ -250,15 +252,6 @@ public interface TableMapper {
.collect(Collectors.toList());
}
default ForcedType columnCreateDtoToForcedType(TableCreateDto data, ColumnCreateDto column) {
final String name = columnCreateDtoToEnumTypeName(data, column);
final ForcedType type = new ForcedType()
.withName(name)
.withTypes("varchar")
.withExpression(".*" + name + ".*");
return type;
}
default DataType<?> columnTypeDtoToDataType(TableCreateDto table, ColumnCreateDto data) {
if (data.getPrimaryKey()) {
if (data.getType().equals(ColumnTypeDto.NUMBER)) {
......
......@@ -13,26 +13,19 @@ import at.tuwien.mapper.ImageMapper;
import at.tuwien.mapper.TableMapper;
import at.tuwien.service.DatabaseConnector;
import lombok.extern.log4j.Log4j2;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.jooq.*;
import org.jooq.Record;
import org.jooq.exception.DataAccessException;
import org.jooq.impl.DSL;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.web.servlet.server.Encoding;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.ResourceUtils;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigInteger;
import java.net.URI;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
......@@ -76,7 +69,6 @@ public abstract class JdbcConnector implements DatabaseConnector {
final CreateSequenceFlagsStep createSequenceFlagsStep = tableMapper.tableCreateDtoToCreateSequenceFlagsStep(context,
createDto);
createSequenceFlagsStep.execute();
log.debug("created id sequence");
final CreateTableColumnStep createTableColumnStep = tableMapper.tableCreateDtoToCreateTableColumnStep(context,
createDto);
log.trace("before execution: {} ", createTableColumnStep.getSQL());
......@@ -112,15 +104,13 @@ public abstract class JdbcConnector implements DatabaseConnector {
log.warn("No data provided.");
throw new TableMalformedException("No data provided");
}
log.info("First row {}", data.getData().get(0));
log.info("Table columns {}", table.getColumns());
if (data.getData().get(0).size() != table.getColumns().size()) {
log.error("Provided columns differ from table columns found in metadata db.");
throw new TableMalformedException("Provided columns differ from table columns found in metadata db.");
}
final List<Field<?>> headers = tableMapper.tableToFieldList(table);
log.trace("headers received {}", headers.stream().map(Field::getName).collect(Collectors.toList()));
log.trace("first row received {}", data.getData().size() > 0 ? data.getData().get(0) : null);
log.trace("first row {}", data.getData()
.get(0));
log.trace("table columns {}", table.getColumns()
.stream()
.map(TableColumn::getInternalName)
.collect(Collectors.toList()));
final DSLContext context = open(table.getDatabase());
final List<InsertValuesStepN<Record>> statements = new LinkedList<>();
final Optional<TableColumn> idxColumn = table.getColumns()
......@@ -136,15 +126,19 @@ public abstract class JdbcConnector implements DatabaseConnector {
log.trace("set auto-generated sequence value {}", idVal);
row.set(idx, idVal);
}
try {
statements.add(context.insertInto(table(table.getInternalName()), headers)
.values(row));
} catch (IllegalArgumentException e) {
throw new TableMalformedException("metadata field number differs from actual data field number", e);
}
}
try {
log.trace("insertCsv statements {}", statements);
context.batch(statements)
.execute();
} catch (DataAccessException e) {
log.error("DataAccessException {}", e);
log.error("data access failed {}", e.getMessage());
throw new TableMalformedException("Columns seem to differ or other problem with jOOQ mapper, most commonly it is a data type issue try with type 'STRING'", e);
}
}
......
......@@ -123,7 +123,7 @@ public class MariaDataService extends JdbcConnector implements DataService {
}
protected TableCsvDto readCsv(Table table, TableInsertDto data) throws IOException, CsvException,
ArrayIndexOutOfBoundsException, TableMalformedException, FileStorageException, SQLException, ImageNotSupportedException {
ArrayIndexOutOfBoundsException, TableMalformedException, SQLException {
log.trace("insert into table {} with params {}", table, data);
if (data.getDelimiter() == null) {
log.warn("No delimiter provided, using comma ','");
......@@ -171,7 +171,12 @@ public class MariaDataService extends JdbcConnector implements DataService {
.build();
final List<List<String>> rows = new LinkedList<>();
reader.readAll()
.forEach(x -> rows.add(new ArrayList<>(List.of(x))));
.forEach(x -> {
final ArrayList<String> row = new ArrayList<>();
Collections.addAll(row, x);
log.trace("add row {}", row);
rows.add(row);
});
log.trace("csv rows {}", rows.size());
/* generic header, ref issue #95 */
List<String> headers = TableUtils.fill(0, rows.get(0).size());
......
......@@ -84,6 +84,7 @@ public class RabbitMqService implements MessageQueueService {
log.debug("declare fanout exchange {}", database.getExchange());
channel.exchangeBind(database.getExchange(), AMQP_EXCHANGE, database.getExchange());
log.debug("bind exchange {} to {}", database.getExchange(), AMQP_EXCHANGE);
log.info("Declared database exchange {} and bound to root exchange {}", database.getExchange(), AMQP_EXCHANGE);
}
@Override
......@@ -93,6 +94,7 @@ public class RabbitMqService implements MessageQueueService {
log.debug("declare queue {}", table.getTopic());
channel.queueBind(table.getTopic(), table.getDatabase().getExchange(), table.getTopic());
log.debug("bind queue {} to {}", table.getTopic(), table.getDatabase().getExchange());
log.info("Declared queue {} and bound to database exchange {}", table.getTopic(), table.getDatabase().getExchange());
}
@Override
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment