From 5164e17b5d29fab389bf1a74f0ca5bb942cbab11 Mon Sep 17 00:00:00 2001
From: Martin Weise <martin.weise@tuwien.ac.at>
Date: Mon, 20 Dec 2021 16:39:11 +0100
Subject: [PATCH] Added some amqp test
Former-commit-id: c8fbeadd41e59552f1c714d070513260bbd8a5c7
---
Makefile | 13 +-
.../database/table/columns/TableColumn.java | 6 +-
fda-table-service/pom.xml | 8 +-
.../src/test/java/at/tuwien/BaseUnitTest.java | 43 +++-
.../java/at/tuwien/config/DockerConfig.java | 9 +
.../java/at/tuwien/config/MariaDbConfig.java | 16 +-
.../DataEndpointQueueIntegrationTest.java | 203 ++++++++++++++++++
...a => DataEndpointRestIntegrationTest.java} | 3 +-
.../endpoint/TableEndpointUnitTest.java | 13 +-
.../service/DataServiceIntegrationTest.java | 43 ++--
.../src/test/resources/application.properties | 1 +
.../src/test/resources/species/Species.sql | 6 +-
.../src/test/resources/weather/WeatherAus.sql | 18 +-
.../java/at/tuwien/mapper/TableMapper.java | 15 +-
.../at/tuwien/service/impl/JdbcConnector.java | 32 ++-
.../tuwien/service/impl/MariaDataService.java | 9 +-
.../tuwien/service/impl/RabbitMqService.java | 2 +
17 files changed, 359 insertions(+), 81 deletions(-)
create mode 100644 fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/DataEndpointQueueIntegrationTest.java
rename fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/{DataEndpointIntegrationTest.java => DataEndpointRestIntegrationTest.java} (98%)
diff --git a/Makefile b/Makefile
index 7552d2b4aa..3f1b6cf5c8 100644
--- a/Makefile
+++ b/Makefile
@@ -15,6 +15,7 @@ config-docker:
docker image pull -q mysql:8.0 || true > /dev/null
docker image pull -q mariadb:10.5 || true > /dev/null
docker image pull -q rabbitmq:3-alpine || true > /dev/null
+ docker image pull -q nginx:1.20-alpine || true > /dev/null
config: config-backend config-docker config-frontend
@@ -181,7 +182,17 @@ registry-staging: registry-staging-tag registry-staging-push
logs:
docker-compose -f docker-compose.prod.yml logs
-clean:
+clean-maven:
+ mvn -f ./fda-authentication-service/pom.xml clean
+ #mvn -f ./fda-citation-service/pom.xml clean
+ mvn -f ./fda-container-service/pom.xml clean
+ mvn -f ./fda-database-service/pom.xml clean
+ mvn -f ./fda-discovery-service/pom.xml clean
+ mvn -f ./fda-gateway-service/pom.xml clean
+ mvn -f ./fda-query-service/pom.xml clean
+ mvn -f ./fda-table-service/pom.xml clean
+
+clean: clean-maven
docker-compose down || true
docker volume rm fda-services_fda-metadata-db-data || true
docker volume rm fda-public || true
diff --git a/fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java b/fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java
index 100c478285..4089b6fce0 100644
--- a/fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java
+++ b/fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java
@@ -22,7 +22,7 @@ import java.util.List;
@EntityListeners(AuditingEntityListener.class)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
@javax.persistence.Table(name = "mdb_columns")
-public class TableColumn {
+public class TableColumn implements Comparable<TableColumn> {
@Id
@EqualsAndHashCode.Include
@@ -104,4 +104,8 @@ public class TableColumn {
@LastModifiedDate
private Instant lastModified;
+ @Override
+ public int compareTo(TableColumn tableColumn) {
+ return Integer.compare(this.ordinalPosition, tableColumn.getOrdinalPosition());
+ }
}
diff --git a/fda-table-service/pom.xml b/fda-table-service/pom.xml
index f99ca04ee7..df29f30282 100644
--- a/fda-table-service/pom.xml
+++ b/fda-table-service/pom.xml
@@ -34,7 +34,7 @@
<opencsv.version>5.4</opencsv.version>
<super-csv.version>2.4.0</super-csv.version>
<btrace-client.version>2.2.0</btrace-client.version>
- <unleash.version>5.0.1</unleash.version>
+ <random-utils.version>2.0.0</random-utils.version>
</properties>
<dependencies>
@@ -101,6 +101,12 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.aventrix.jnanoid</groupId>
+ <artifactId>jnanoid</artifactId>
+ <scope>test</scope>
+ <version>${random-utils.version}</version>
+ </dependency>
<!-- AMPQ -->
<dependency>
<groupId>com.rabbitmq</groupId>
diff --git a/fda-table-service/rest-service/src/test/java/at/tuwien/BaseUnitTest.java b/fda-table-service/rest-service/src/test/java/at/tuwien/BaseUnitTest.java
index 037ef13738..63fdb2bb96 100644
--- a/fda-table-service/rest-service/src/test/java/at/tuwien/BaseUnitTest.java
+++ b/fda-table-service/rest-service/src/test/java/at/tuwien/BaseUnitTest.java
@@ -1,7 +1,5 @@
package at.tuwien;
-import at.tuwien.api.container.image.ImageEnvItemDto;
-import at.tuwien.api.container.image.ImageEnvItemTypeDto;
import at.tuwien.api.database.table.TableCreateDto;
import at.tuwien.api.database.table.columns.ColumnCreateDto;
import at.tuwien.api.database.table.columns.ColumnTypeDto;
@@ -23,6 +21,13 @@ import static java.time.temporal.ChronoUnit.HOURS;
@TestPropertySource(locations = "classpath:application.properties")
public abstract class BaseUnitTest extends CsvUnitTest {
+ public final static String DATABASE_NET = "fda-userdb";
+
+ public final static String BROKER_IMAGE = "fda-broker-service:latest";
+ public final static String BROKER_INTERNALNAME = "fda-broker-service";
+ public final static String BROKER_NET = "fda-public";
+ public final static String BROKER_IP = "172.29.0.2";
+
public final static Long DATABASE_1_ID = 1L;
public final static String DATABASE_1_NAME = "Weather";
public final static String DATABASE_1_INTERNALNAME = "weather";
@@ -60,7 +65,7 @@ public abstract class BaseUnitTest extends CsvUnitTest {
public final static Integer COLUMN_1_1_ORDINALPOS = 0;
public final static Boolean COLUMN_1_1_PRIMARY = true;
public final static String COLUMN_1_1_NAME = "id";
- public final static String COLUMN_1_1_INTERNAL_NAME = "mdb_id";
+ public final static String COLUMN_1_1_INTERNAL_NAME = "id";
public final static TableColumnType COLUMN_1_1_TYPE = TableColumnType.NUMBER;
public final static ColumnTypeDto COLUMN_1_1_TYPE_DTO = ColumnTypeDto.NUMBER;
public final static Boolean COLUMN_1_1_NULL = false;
@@ -74,7 +79,7 @@ public abstract class BaseUnitTest extends CsvUnitTest {
public final static Integer COLUMN_1_2_ORDINALPOS = 1;
public final static Boolean COLUMN_1_2_PRIMARY = false;
public final static String COLUMN_1_2_NAME = "Date";
- public final static String COLUMN_1_2_INTERNAL_NAME = "mdb_date";
+ public final static String COLUMN_1_2_INTERNAL_NAME = "date";
public final static TableColumnType COLUMN_1_2_TYPE = TableColumnType.DATE;
public final static ColumnTypeDto COLUMN_1_2_TYPE_DTO = ColumnTypeDto.DATE;
public final static Boolean COLUMN_1_2_NULL = true;
@@ -88,7 +93,7 @@ public abstract class BaseUnitTest extends CsvUnitTest {
public final static Integer COLUMN_1_3_ORDINALPOS = 2;
public final static Boolean COLUMN_1_3_PRIMARY = false;
public final static String COLUMN_1_3_NAME = "Location";
- public final static String COLUMN_1_3_INTERNAL_NAME = "mdb_location";
+ public final static String COLUMN_1_3_INTERNAL_NAME = "location";
public final static TableColumnType COLUMN_1_3_TYPE = TableColumnType.STRING;
public final static ColumnTypeDto COLUMN_1_3_TYPE_DTO = ColumnTypeDto.STRING;
public final static Boolean COLUMN_1_3_NULL = true;
@@ -102,7 +107,7 @@ public abstract class BaseUnitTest extends CsvUnitTest {
public final static Integer COLUMN_1_4_ORDINALPOS = 3;
public final static Boolean COLUMN_1_4_PRIMARY = false;
public final static String COLUMN_1_4_NAME = "MinTemp";
- public final static String COLUMN_1_4_INTERNAL_NAME = "mdb_mintemp";
+ public final static String COLUMN_1_4_INTERNAL_NAME = "mintemp";
public final static TableColumnType COLUMN_1_4_TYPE = TableColumnType.STRING;
public final static ColumnTypeDto COLUMN_1_4_TYPE_DTO = ColumnTypeDto.STRING;
public final static Boolean COLUMN_1_4_NULL = true;
@@ -116,7 +121,7 @@ public abstract class BaseUnitTest extends CsvUnitTest {
public final static Integer COLUMN_1_5_ORDINALPOS = 4;
public final static Boolean COLUMN_1_5_PRIMARY = false;
public final static String COLUMN_1_5_NAME = "Rainfall";
- public final static String COLUMN_1_5_INTERNAL_NAME = "mdb_rainfall";
+ public final static String COLUMN_1_5_INTERNAL_NAME = "rainfall";
public final static TableColumnType COLUMN_1_5_TYPE = TableColumnType.NUMBER;
public final static ColumnTypeDto COLUMN_1_5_TYPE_DTO = ColumnTypeDto.NUMBER;
public final static Boolean COLUMN_1_5_NULL = true;
@@ -130,7 +135,7 @@ public abstract class BaseUnitTest extends CsvUnitTest {
public final static Integer COLUMN_3_1_ORDINALPOS = 0;
public final static Boolean COLUMN_3_1_PRIMARY = false;
public final static String COLUMN_3_1_NAME = "qu";
- public final static String COLUMN_3_1_INTERNAL_NAME = "mdb_qu";
+ public final static String COLUMN_3_1_INTERNAL_NAME = "qu";
public final static TableColumnType COLUMN_3_1_TYPE = TableColumnType.STRING;
public final static ColumnTypeDto COLUMN_3_1_TYPE_DTO = ColumnTypeDto.STRING;
public final static Boolean COLUMN_3_1_NULL = false;
@@ -144,7 +149,7 @@ public abstract class BaseUnitTest extends CsvUnitTest {
public final static Integer COLUMN_3_2_ORDINALPOS = 1;
public final static Boolean COLUMN_3_2_PRIMARY = false;
public final static String COLUMN_3_2_NAME = "species";
- public final static String COLUMN_3_2_INTERNAL_NAME = "mdb_species";
+ public final static String COLUMN_3_2_INTERNAL_NAME = "species";
public final static TableColumnType COLUMN_3_2_TYPE = TableColumnType.ENUM;
public final static ColumnTypeDto COLUMN_3_2_TYPE_DTO = ColumnTypeDto.ENUM;
public final static Boolean COLUMN_3_2_NULL = false;
@@ -158,7 +163,7 @@ public abstract class BaseUnitTest extends CsvUnitTest {
public final static Integer COLUMN_3_3_ORDINALPOS = 2;
public final static Boolean COLUMN_3_3_PRIMARY = false;
public final static String COLUMN_3_3_NAME = "score";
- public final static String COLUMN_3_3_INTERNAL_NAME = "mdb_score";
+ public final static String COLUMN_3_3_INTERNAL_NAME = "score";
public final static TableColumnType COLUMN_3_3_TYPE = TableColumnType.STRING;
public final static ColumnTypeDto COLUMN_3_3_TYPE_DTO = ColumnTypeDto.STRING;
public final static Boolean COLUMN_3_3_NULL = false;
@@ -283,6 +288,16 @@ public abstract class BaseUnitTest extends CsvUnitTest {
public final static String CONTAINER_3_IP = "172.28.0.7";
public final static Instant CONTAINER_3_CREATED = Instant.now().minus(1, HOURS);
+ public final static Long CONTAINER_NGINX_ID = 4L;
+ public final static String CONTAINER_NGINX_HASH = "deadbeef";
+ public final static String CONTAINER_NGINX_IMAGE = "nginx";
+ public final static String CONTAINER_NGINX_TAG = "1.20-alpine";
+ public final static String CONTAINER_NGINX_NET = "fda-public";
+ public final static String CONTAINER_NGINX_NAME = "file-service";
+ public final static String CONTAINER_NGINX_INTERNALNAME = "fda-test-file-service";
+ public final static String CONTAINER_NGINX_IP = "172.29.0.3";
+ public final static Instant CONTAINER_NGINX_CREATED = Instant.now().minus(3, HOURS);
+
public final static Container CONTAINER_1 = Container.builder()
.id(CONTAINER_1_ID)
.name(CONTAINER_1_NAME)
@@ -310,6 +325,14 @@ public abstract class BaseUnitTest extends CsvUnitTest {
.containerCreated(CONTAINER_3_CREATED)
.build();
+ public final static Container CONTAINER_NGINX = Container.builder()
+ .id(CONTAINER_NGINX_ID)
+ .name(CONTAINER_NGINX_NAME)
+ .internalName(CONTAINER_NGINX_INTERNALNAME)
+ .hash(CONTAINER_NGINX_HASH)
+ .containerCreated(CONTAINER_NGINX_CREATED)
+ .build();
+
public final static List<TableColumn> TABLE_3_COLUMNS = List.of(TableColumn.builder()
.id(COLUMN_3_1_ID)
.ordinalPosition(COLUMN_3_1_ORDINALPOS)
diff --git a/fda-table-service/rest-service/src/test/java/at/tuwien/config/DockerConfig.java b/fda-table-service/rest-service/src/test/java/at/tuwien/config/DockerConfig.java
index 057e57d64d..81d5bc8afe 100644
--- a/fda-table-service/rest-service/src/test/java/at/tuwien/config/DockerConfig.java
+++ b/fda-table-service/rest-service/src/test/java/at/tuwien/config/DockerConfig.java
@@ -44,11 +44,20 @@ public class DockerConfig {
dockerClient.startContainerCmd(container.getHash())
.exec();
Thread.sleep(12 * 1000L);
+ log.debug("container {} was started", container.getHash());
}
public static void stopContainer(Container container) {
+ final InspectContainerResponse inspect = dockerClient.inspectContainerCmd(container.getHash())
+ .exec();
+ log.trace("container {} state {}", container.getHash(), inspect.getState().getStatus());
+ if (!Objects.equals(inspect.getState().getStatus(), "running")) {
+ return;
+ }
+ log.trace("container {} needs to be stopped", container.getHash());
dockerClient.stopContainerCmd(container.getHash())
.exec();
+ log.debug("container {} was stopped", container.getHash());
}
}
diff --git a/fda-table-service/rest-service/src/test/java/at/tuwien/config/MariaDbConfig.java b/fda-table-service/rest-service/src/test/java/at/tuwien/config/MariaDbConfig.java
index 39926a27fc..9ebb89afcf 100644
--- a/fda-table-service/rest-service/src/test/java/at/tuwien/config/MariaDbConfig.java
+++ b/fda-table-service/rest-service/src/test/java/at/tuwien/config/MariaDbConfig.java
@@ -5,10 +5,7 @@ import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Configurable;
import org.springframework.context.annotation.Bean;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.sql.Statement;
+import java.sql.*;
import java.util.Properties;
@Log4j2
@@ -32,4 +29,15 @@ public class MariaDbConfig {
connection.close();
}
+ public static boolean contains(Table table, String column, Object expected) throws SQLException {
+ final String jdbc = "jdbc:mariadb://" + table.getDatabase().getContainer().getInternalName() + "/" + table.getDatabase().getInternalName();
+ log.trace("connect to database {}", jdbc);
+ final Connection connection = DriverManager.getConnection(jdbc, "mariadb", "mariadb");
+ final Statement statement = connection.createStatement();
+ final ResultSet result = statement.executeQuery("SELECT `" + column + "` FROM " + table.getInternalName() +
+ " WHERE `" + column + "` = " + expected.toString() + ";");
+ connection.close();
+ return result.next();
+ }
+
}
diff --git a/fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/DataEndpointQueueIntegrationTest.java b/fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/DataEndpointQueueIntegrationTest.java
new file mode 100644
index 0000000000..688da3bc5f
--- /dev/null
+++ b/fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/DataEndpointQueueIntegrationTest.java
@@ -0,0 +1,203 @@
+package at.tuwien.endpoint;
+
+import at.tuwien.BaseUnitTest;
+import at.tuwien.api.database.table.TableCsvDto;
+import at.tuwien.config.DockerConfig;
+import at.tuwien.config.MariaDbConfig;
+import at.tuwien.config.ReadyConfig;
+import at.tuwien.endpoints.DataEndpoint;
+import at.tuwien.entities.container.Container;
+import at.tuwien.exception.*;
+import at.tuwien.repository.jpa.ImageRepository;
+import at.tuwien.repository.jpa.TableRepository;
+import com.aventrix.jnanoid.jnanoid.NanoIdUtils;
+import com.github.dockerjava.api.command.CreateContainerResponse;
+import com.github.dockerjava.api.exception.NotModifiedException;
+import com.github.dockerjava.api.model.Bind;
+import com.github.dockerjava.api.model.Network;
+import com.rabbitmq.client.Channel;
+import lombok.extern.log4j.Log4j2;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import javax.transaction.Transactional;
+import java.io.File;
+import java.sql.SQLException;
+import java.util.*;
+
+import static at.tuwien.config.DockerConfig.dockerClient;
+import static at.tuwien.config.DockerConfig.hostConfig;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@SpringBootTest
+@ExtendWith(SpringExtension.class)
+@Log4j2
+public class DataEndpointQueueIntegrationTest extends BaseUnitTest {
+
+ @MockBean
+ private Channel channel;
+
+ @MockBean
+ private ReadyConfig readyConfig;
+
+ @Autowired
+ private DataEndpoint dataEndpoint;
+
+ @Autowired
+ private TableRepository tableRepository;
+
+ @Autowired
+ private ImageRepository imageRepository;
+
+ /**
+ * We need a container to test the CRUD operations as of now it is unfeasible to determine the correctness of the
+ * operations without a live container
+ *
+ * @throws InterruptedException Sleep interrupted.
+ */
+ @BeforeAll
+ public static void beforeAll() throws InterruptedException {
+ afterAll();
+ /* create networks */
+ dockerClient.createNetworkCmd()
+ .withName(BROKER_NET)
+ .withIpam(new Network.Ipam()
+ .withConfig(new Network.Ipam.Config()
+ .withSubnet("172.29.0.0/16")))
+ .withEnableIpv6(false)
+ .exec();
+ dockerClient.createNetworkCmd()
+ .withName(DATABASE_NET)
+ .withIpam(new Network.Ipam()
+ .withConfig(new Network.Ipam.Config()
+ .withSubnet("172.28.0.0/16")))
+ .withEnableIpv6(false)
+ .exec();
+ /* create broker container */
+ final CreateContainerResponse response1 = dockerClient.createContainerCmd(BROKER_IMAGE)
+ .withHostConfig(hostConfig.withNetworkMode(BROKER_NET))
+ .withName(BROKER_INTERNALNAME)
+ .withIpv4Address(BROKER_IP)
+ .withHostName(BROKER_INTERNALNAME)
+ .withEnv("TZ=Europe/Vienna")
+ .exec();
+ dockerClient.startContainerCmd(response1.getId())
+ .exec();
+ /* create table container */
+ final String bind = new File("./src/test/resources/weather").toPath().toAbsolutePath() + ":/docker-entrypoint-initdb.d";
+ log.trace("container bind {}", bind);
+ final CreateContainerResponse response2 = dockerClient.createContainerCmd(IMAGE_2_REPOSITORY + ":" + IMAGE_2_TAG)
+ .withHostConfig(hostConfig.withNetworkMode(DATABASE_NET))
+ .withName(CONTAINER_1_INTERNALNAME)
+ .withIpv4Address(CONTAINER_1_IP)
+ .withHostName(CONTAINER_1_INTERNALNAME)
+ .withEnv("MARIADB_USER=mariadb", "MARIADB_PASSWORD=mariadb", "MARIADB_ROOT_PASSWORD=mariadb", "MARIADB_DATABASE=weather")
+ .withBinds(Bind.parse(bind))
+ .exec();
+ /* wait */
+ CONTAINER_1.setHash(response2.getId());
+ Thread.sleep(10 * 1000);
+ }
+
+ @AfterAll
+ public static void afterAll() {
+ /* stop containers and remove them */
+ dockerClient.listContainersCmd()
+ .withShowAll(true)
+ .exec()
+ .forEach(container -> {
+ log.info("Delete container {}", Arrays.asList(container.getNames()));
+ try {
+ dockerClient.stopContainerCmd(container.getId()).exec();
+ } catch (NotModifiedException e) {
+ // ignore
+ }
+ dockerClient.removeContainerCmd(container.getId()).exec();
+ });
+ /* remove networks */
+ dockerClient.listNetworksCmd()
+ .exec()
+ .stream()
+ .filter(n -> n.getName().startsWith("fda"))
+ .forEach(network -> {
+ log.info("Delete network {}", network.getName());
+ dockerClient.removeNetworkCmd(network.getId()).exec();
+ });
+ }
+
+ @BeforeEach
+ @Transactional
+ public void beforeEach() {
+ imageRepository.save(IMAGE_1);
+ TABLE_1.setDatabase(DATABASE_1);
+ tableRepository.save(TABLE_1);
+ }
+
+ @Test
+ public void insertFromTuple_succeeds() throws TableNotFoundException, TableMalformedException,
+ DatabaseNotFoundException, ImageNotSupportedException, InterruptedException, SQLException {
+ final TableCsvDto request = TableCsvDto.builder()
+ .data(List.of(new LinkedHashMap<>() {{
+ put(COLUMN_1_1_NAME, 1L);
+ put(COLUMN_1_2_NAME, "2020-12-01");
+ put(COLUMN_1_3_NAME, "Somewhere");
+ put(COLUMN_1_4_NAME, 15.0);
+ put(COLUMN_1_5_NAME, 20.0);
+ }}))
+ .build();
+
+ /* mock */
+ DockerConfig.startContainer(CONTAINER_1);
+ MariaDbConfig.clearDatabase(TABLE_1);
+
+ /* test */
+ dataEndpoint.insertFromTuple(DATABASE_1_ID, TABLE_1_ID, request);
+ assertTrue(MariaDbConfig.contains(TABLE_1, COLUMN_1_1_NAME, 1L));
+ assertFalse(MariaDbConfig.contains(TABLE_1, COLUMN_1_1_NAME, 2L));
+ }
+
+ @Test
+ public void insertFromTuple_10k_succeeds() throws TableNotFoundException, TableMalformedException,
+ DatabaseNotFoundException, ImageNotSupportedException, InterruptedException, SQLException {
+ /* config */
+ final long limit = 10_000L;
+
+ /* mock */
+ final Random random = new Random();
+ final List<Map<String, Object>> data = new LinkedList<>();
+ for (long i = 0L; i <= limit; i++) {
+ final Long id = i;
+ data.add(new LinkedHashMap<>() {{
+ put(COLUMN_1_1_NAME, id);
+ put(COLUMN_1_2_NAME, "2020-01-01");
+ put(COLUMN_1_3_NAME, NanoIdUtils.randomNanoId());
+ put(COLUMN_1_4_NAME, random.nextDouble());
+ put(COLUMN_1_5_NAME, random.nextDouble());
+ }});
+ }
+ final TableCsvDto request = TableCsvDto.builder()
+ .data(data)
+ .build();
+ DockerConfig.startContainer(CONTAINER_1);
+ MariaDbConfig.clearDatabase(TABLE_1);
+
+ /* test */
+ final long start = System.currentTimeMillis();
+ dataEndpoint.insertFromTuple(DATABASE_1_ID, TABLE_1_ID, request);
+ final long end = System.currentTimeMillis();
+ log.info("Inserted {}k records in {} seconds", limit / 1000, (end - start) / 1000.0);
+ assertTrue(MariaDbConfig.contains(TABLE_1, COLUMN_1_1_NAME, 1L), "id 1 missing");
+ assertTrue(MariaDbConfig.contains(TABLE_1, COLUMN_1_1_NAME, 1_000L), "id 1k missing");
+ assertTrue(MariaDbConfig.contains(TABLE_1, COLUMN_1_1_NAME, 10_000L), "id 10k missing");
+ assertTrue(MariaDbConfig.contains(TABLE_1, COLUMN_1_1_NAME, limit), "id 100k missing");
+ }
+
+}
diff --git a/fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/DataEndpointIntegrationTest.java b/fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/DataEndpointRestIntegrationTest.java
similarity index 98%
rename from fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/DataEndpointIntegrationTest.java
rename to fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/DataEndpointRestIntegrationTest.java
index c4f126ee54..4fd54a3086 100644
--- a/fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/DataEndpointIntegrationTest.java
+++ b/fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/DataEndpointRestIntegrationTest.java
@@ -49,7 +49,7 @@ import static org.mockito.ArgumentMatchers.any;
@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
@SpringBootTest
@ExtendWith(SpringExtension.class)
-public class DataEndpointIntegrationTest extends BaseUnitTest {
+public class DataEndpointRestIntegrationTest extends BaseUnitTest {
@MockBean
private Channel channel;
@@ -100,6 +100,7 @@ public class DataEndpointIntegrationTest extends BaseUnitTest {
public void beforeEach() {
imageRepository.save(IMAGE_1);
databaseRepository.save(DATABASE_1);
+ TABLE_1.setDatabase(DATABASE_1);
tableRepository.save(TABLE_1);
}
diff --git a/fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java b/fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java
index f2b59ec195..a98a1552d9 100644
--- a/fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java
+++ b/fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java
@@ -9,6 +9,7 @@ import at.tuwien.endpoints.TableEndpoint;
import at.tuwien.exception.*;
import at.tuwien.repository.jpa.DatabaseRepository;
import at.tuwien.repository.jpa.TableRepository;
+import at.tuwien.service.MessageQueueService;
import at.tuwien.service.impl.TableServiceImpl;
import com.rabbitmq.client.Channel;
import org.junit.jupiter.api.Disabled;
@@ -21,6 +22,7 @@ import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.test.context.junit.jupiter.SpringExtension;
+import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
@@ -48,6 +50,9 @@ public class TableEndpointUnitTest extends BaseUnitTest {
@MockBean
private DatabaseRepository databaseRepository;
+ @MockBean
+ private MessageQueueService messageQueueService;
+
@Autowired
private TableEndpoint tableEndpoint;
@@ -63,18 +68,22 @@ public class TableEndpointUnitTest extends BaseUnitTest {
}
@Test
- @Disabled("only works in integration test")
public void create_succeeds() throws DatabaseNotFoundException, ImageNotSupportedException,
- TableNotFoundException, DataProcessingException, ArbitraryPrimaryKeysException, TableMalformedException, AmqpException {
+ TableNotFoundException, DataProcessingException, ArbitraryPrimaryKeysException, TableMalformedException, AmqpException, IOException {
final TableCreateDto request = TableCreateDto.builder()
.name(TABLE_1_NAME)
.description(TABLE_1_DESCRIPTION)
.columns(COLUMNS_CSV01)
.build();
+
+ /* mock */
when(tableRepository.findById(TABLE_1_ID))
.thenReturn(Optional.of(TABLE_1));
when(tableService.findById(DATABASE_1_ID, TABLE_1_ID))
.thenReturn(TABLE_1);
+ doNothing()
+ .when(messageQueueService)
+ .create(TABLE_1);
/* test */
final ResponseEntity<TableBriefDto> response = tableEndpoint.create(DATABASE_1_ID, request);
diff --git a/fda-table-service/rest-service/src/test/java/at/tuwien/service/DataServiceIntegrationTest.java b/fda-table-service/rest-service/src/test/java/at/tuwien/service/DataServiceIntegrationTest.java
index f2d80aa71e..04241bf9cb 100644
--- a/fda-table-service/rest-service/src/test/java/at/tuwien/service/DataServiceIntegrationTest.java
+++ b/fda-table-service/rest-service/src/test/java/at/tuwien/service/DataServiceIntegrationTest.java
@@ -67,9 +67,9 @@ public class DataServiceIntegrationTest extends BaseUnitTest {
private MariaDataService dataService;
@BeforeAll
- public static void beforeAll() throws InterruptedException {
+ public static void beforeAll() {
afterAll();
- /* create network */
+ /* create networks */
dockerClient.createNetworkCmd()
.withName("fda-userdb")
.withIpam(new Network.Ipam()
@@ -77,7 +77,14 @@ public class DataServiceIntegrationTest extends BaseUnitTest {
.withSubnet("172.28.0.0/16")))
.withEnableIpv6(false)
.exec();
- /* create container */
+ dockerClient.createNetworkCmd()
+ .withName("fda-public")
+ .withIpam(new Network.Ipam()
+ .withConfig(new Network.Ipam.Config()
+ .withSubnet("172.29.0.0/16")))
+ .withEnableIpv6(false)
+ .exec();
+ /* create weather container */
final String bind = new File("./src/test/resources/weather").toPath().toAbsolutePath() + ":/docker-entrypoint-initdb.d";
log.trace("container bind {}", bind);
final CreateContainerResponse request = dockerClient.createContainerCmd(IMAGE_2_REPOSITORY + ":" + IMAGE_2_TAG)
@@ -88,19 +95,19 @@ public class DataServiceIntegrationTest extends BaseUnitTest {
.withEnv("MARIADB_USER=mariadb", "MARIADB_PASSWORD=mariadb", "MARIADB_ROOT_PASSWORD=mariadb", "MARIADB_DATABASE=weather")
.withBinds(Bind.parse(bind))
.exec();
- final String bind3 = new File("./src/test/resources/species").toPath().toAbsolutePath() + ":/docker-entrypoint-initdb.d";
- log.trace("container 3 bind {}", bind3);
- final CreateContainerResponse request3 = dockerClient.createContainerCmd(IMAGE_2_REPOSITORY + ":" + IMAGE_2_TAG)
- .withHostConfig(hostConfig.withNetworkMode("fda-userdb"))
- .withName(CONTAINER_3_INTERNALNAME)
- .withIpv4Address(CONTAINER_3_IP)
- .withHostName(CONTAINER_3_INTERNALNAME)
- .withEnv("MARIADB_USER=mariadb", "MARIADB_PASSWORD=mariadb", "MARIADB_ROOT_PASSWORD=mariadb", "MARIADB_DATABASE=biomedical")
- .withBinds(Bind.parse(bind3))
+ /* create file container */
+ final String bind2 = new File("./src/test/resources/csv").toPath().toAbsolutePath() + ":/usr/share/nginx/html:ro";
+ log.trace("container bind2 {}", bind2);
+ final CreateContainerResponse request2 = dockerClient.createContainerCmd(CONTAINER_NGINX_IMAGE + ":" + CONTAINER_NGINX_TAG)
+ .withHostConfig(hostConfig.withNetworkMode(CONTAINER_NGINX_NET))
+ .withName(CONTAINER_NGINX_NAME)
+ .withIpv4Address(CONTAINER_NGINX_IP)
+ .withHostName(CONTAINER_NGINX_INTERNALNAME)
+ .withBinds(Bind.parse(bind2))
.exec();
/* set hash */
CONTAINER_1.setHash(request.getId());
- CONTAINER_3.setHash(request3.getId());
+ CONTAINER_NGINX.setHash(request2.getId());
}
@AfterAll
@@ -272,15 +279,17 @@ public class DataServiceIntegrationTest extends BaseUnitTest {
SQLException {
final TableInsertDto request = TableInsertDto.builder()
.skipHeader(true)
- .csvLocation("https://sandbox.zenodo.org/api/files/6aca3421-add3-489b-8c4a-35228fe5c683/species_id.csv")
+ .nullElement("NA")
+ .csvLocation("http://172.29.0.3/csv_01.csv")
.build();
/* mock */
- DockerConfig.startContainer(CONTAINER_3);
- MariaDbConfig.clearDatabase(TABLE_3);
+ DockerConfig.startContainer(CONTAINER_1);
+ DockerConfig.startContainer(CONTAINER_NGINX);
+ MariaDbConfig.clearDatabase(TABLE_1);
/* test */
- dataService.insertCsv(DATABASE_3_ID, TABLE_3_ID, request);
+ dataService.insertCsv(DATABASE_1_ID, TABLE_1_ID, request);
}
}
diff --git a/fda-table-service/rest-service/src/test/resources/application.properties b/fda-table-service/rest-service/src/test/resources/application.properties
index 05fbac4c44..29b0c3c7ab 100644
--- a/fda-table-service/rest-service/src/test/resources/application.properties
+++ b/fda-table-service/rest-service/src/test/resources/application.properties
@@ -17,6 +17,7 @@ spring.jpa.hibernate.ddl-auto=create-drop
spring.jpa.show-sql=false
# logging
+logging.level.at.tuwien.service.impl.JdbcConnector=debug
logging.level.at.tuwien.=trace
# disable elasticsearch
diff --git a/fda-table-service/rest-service/src/test/resources/species/Species.sql b/fda-table-service/rest-service/src/test/resources/species/Species.sql
index aaa14c1ad8..957368fcc7 100644
--- a/fda-table-service/rest-service/src/test/resources/species/Species.sql
+++ b/fda-table-service/rest-service/src/test/resources/species/Species.sql
@@ -1,7 +1,7 @@
/* https://sandbox.zenodo.org/api/files/6aca3421-add3-489b-8c4a-35228fe5c683/species_id.csv */
CREATE TABLE maldi_ms_data
(
- mdb_qu VARCHAR(255) NOT NULL,
- mdb_species VARCHAR(255) NOT NULL,
- mdb_score VARCHAR(255) NOT NULL
+ qu VARCHAR(255) NOT NULL,
+ species VARCHAR(255) NOT NULL,
+ score VARCHAR(255) NOT NULL
) WITH SYSTEM VERSIONING;
\ No newline at end of file
diff --git a/fda-table-service/rest-service/src/test/resources/weather/WeatherAus.sql b/fda-table-service/rest-service/src/test/resources/weather/WeatherAus.sql
index 2b78c2210c..b53010acbb 100644
--- a/fda-table-service/rest-service/src/test/resources/weather/WeatherAus.sql
+++ b/fda-table-service/rest-service/src/test/resources/weather/WeatherAus.sql
@@ -1,14 +1,14 @@
/* https://www.kaggle.com/jsphyg/weather-dataset-rattle-package */
CREATE TABLE weather_aus
(
- mdb_id BIGINT NOT NULL PRIMARY KEY,
- mdb_date DATE NOT NULL,
- mdb_mintemp DOUBLE PRECISION NULL,
- mdb_location VARCHAR(255) NULL,
- mdb_rainfall DOUBLE PRECISION NULL
+ id BIGINT NOT NULL PRIMARY KEY,
+ date DATE NOT NULL,
+ location VARCHAR(255) NULL,
+ mintemp DOUBLE PRECISION NULL,
+ rainfall DOUBLE PRECISION NULL
) WITH SYSTEM VERSIONING;
-INSERT INTO weather_aus (mdb_id, mdb_date, mdb_location, mdb_mintemp, mdb_rainfall)
-VALUES (1, '2008-12-01'::DATE, 13.4, 'Albury', 0.6),
- (2, '2008-12-02'::DATE, 7.4, 'Albury', 0),
- (3, '2008-12-03'::DATE, 12.9, 'Albury', 0);
\ No newline at end of file
+INSERT INTO weather_aus (id, date, location, mintemp, rainfall)
+VALUES (1, '2008-12-01'::DATE, 'Albury', 13.4, 0.6),
+ (2, '2008-12-02'::DATE, 'Albury', 7.4, 0),
+ (3, '2008-12-03'::DATE, 'Albury', 12.9, 0);
\ No newline at end of file
diff --git a/fda-table-service/services/src/main/java/at/tuwien/mapper/TableMapper.java b/fda-table-service/services/src/main/java/at/tuwien/mapper/TableMapper.java
index c9b789f4e9..08ad55f934 100644
--- a/fda-table-service/services/src/main/java/at/tuwien/mapper/TableMapper.java
+++ b/fda-table-service/services/src/main/java/at/tuwien/mapper/TableMapper.java
@@ -139,7 +139,9 @@ public interface TableMapper {
/* create sequence nonetheless, if it is used or not */
default CreateSequenceFlagsStep tableCreateDtoToCreateSequenceFlagsStep(DSLContext context, TableCreateDto data) {
- return context.createSequenceIfNotExists(tableNameToSequenceName(data.getName()));
+ final String sequenceName = tableNameToSequenceName(data.getName());
+ log.trace("create sequence {}", sequenceName);
+ return context.createSequenceIfNotExists(sequenceName);
}
default CreateTableColumnStep tableCreateDtoToCreateTableColumnStep(DSLContext context, TableCreateDto data)
@@ -238,7 +240,7 @@ public interface TableMapper {
default List<Field<?>> tableToFieldList(Table data) {
return data.getColumns()
.stream()
- .sorted(Comparator.comparing(TableColumn::getOrdinalPosition))
+ .sorted()
.map(c -> field(c.getInternalName()))
.collect(Collectors.toList());
}
@@ -250,15 +252,6 @@ public interface TableMapper {
.collect(Collectors.toList());
}
- default ForcedType columnCreateDtoToForcedType(TableCreateDto data, ColumnCreateDto column) {
- final String name = columnCreateDtoToEnumTypeName(data, column);
- final ForcedType type = new ForcedType()
- .withName(name)
- .withTypes("varchar")
- .withExpression(".*" + name + ".*");
- return type;
- }
-
default DataType<?> columnTypeDtoToDataType(TableCreateDto table, ColumnCreateDto data) {
if (data.getPrimaryKey()) {
if (data.getType().equals(ColumnTypeDto.NUMBER)) {
diff --git a/fda-table-service/services/src/main/java/at/tuwien/service/impl/JdbcConnector.java b/fda-table-service/services/src/main/java/at/tuwien/service/impl/JdbcConnector.java
index 81193a9f36..4ed1df95c2 100644
--- a/fda-table-service/services/src/main/java/at/tuwien/service/impl/JdbcConnector.java
+++ b/fda-table-service/services/src/main/java/at/tuwien/service/impl/JdbcConnector.java
@@ -13,26 +13,19 @@ import at.tuwien.mapper.ImageMapper;
import at.tuwien.mapper.TableMapper;
import at.tuwien.service.DatabaseConnector;
import lombok.extern.log4j.Log4j2;
-import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.jooq.*;
import org.jooq.Record;
import org.jooq.exception.DataAccessException;
import org.jooq.impl.DSL;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.web.servlet.server.Encoding;
import org.springframework.core.io.ClassPathResource;
-import org.springframework.core.io.Resource;
-import org.springframework.core.io.ResourceLoader;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
-import org.springframework.util.ResourceUtils;
-import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigInteger;
-import java.net.URI;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
@@ -76,7 +69,6 @@ public abstract class JdbcConnector implements DatabaseConnector {
final CreateSequenceFlagsStep createSequenceFlagsStep = tableMapper.tableCreateDtoToCreateSequenceFlagsStep(context,
createDto);
createSequenceFlagsStep.execute();
- log.debug("created id sequence");
final CreateTableColumnStep createTableColumnStep = tableMapper.tableCreateDtoToCreateTableColumnStep(context,
createDto);
log.trace("before execution: {} ", createTableColumnStep.getSQL());
@@ -112,15 +104,13 @@ public abstract class JdbcConnector implements DatabaseConnector {
log.warn("No data provided.");
throw new TableMalformedException("No data provided");
}
- log.info("First row {}", data.getData().get(0));
- log.info("Table columns {}", table.getColumns());
- if (data.getData().get(0).size() != table.getColumns().size()) {
- log.error("Provided columns differ from table columns found in metadata db.");
- throw new TableMalformedException("Provided columns differ from table columns found in metadata db.");
- }
final List<Field<?>> headers = tableMapper.tableToFieldList(table);
- log.trace("headers received {}", headers.stream().map(Field::getName).collect(Collectors.toList()));
- log.trace("first row received {}", data.getData().size() > 0 ? data.getData().get(0) : null);
+ log.trace("first row {}", data.getData()
+ .get(0));
+ log.trace("table columns {}", table.getColumns()
+ .stream()
+ .map(TableColumn::getInternalName)
+ .collect(Collectors.toList()));
final DSLContext context = open(table.getDatabase());
final List<InsertValuesStepN<Record>> statements = new LinkedList<>();
final Optional<TableColumn> idxColumn = table.getColumns()
@@ -136,15 +126,19 @@ public abstract class JdbcConnector implements DatabaseConnector {
log.trace("set auto-generated sequence value {}", idVal);
row.set(idx, idVal);
}
- statements.add(context.insertInto(table(table.getInternalName()), headers)
- .values(row));
+ try {
+ statements.add(context.insertInto(table(table.getInternalName()), headers)
+ .values(row));
+ } catch (IllegalArgumentException e) {
+ throw new TableMalformedException("metadata field number differs from actual data field number", e);
+ }
}
try {
log.trace("insertCsv statements {}", statements);
context.batch(statements)
.execute();
} catch (DataAccessException e) {
- log.error("DataAccessException {}", e);
+ log.error("data access failed {}", e.getMessage());
throw new TableMalformedException("Columns seem to differ or other problem with jOOQ mapper, most commonly it is a data type issue try with type 'STRING'", e);
}
}
diff --git a/fda-table-service/services/src/main/java/at/tuwien/service/impl/MariaDataService.java b/fda-table-service/services/src/main/java/at/tuwien/service/impl/MariaDataService.java
index 1becaf0a13..196af30707 100644
--- a/fda-table-service/services/src/main/java/at/tuwien/service/impl/MariaDataService.java
+++ b/fda-table-service/services/src/main/java/at/tuwien/service/impl/MariaDataService.java
@@ -123,7 +123,7 @@ public class MariaDataService extends JdbcConnector implements DataService {
}
protected TableCsvDto readCsv(Table table, TableInsertDto data) throws IOException, CsvException,
- ArrayIndexOutOfBoundsException, TableMalformedException, FileStorageException, SQLException, ImageNotSupportedException {
+ ArrayIndexOutOfBoundsException, TableMalformedException, SQLException {
log.trace("insert into table {} with params {}", table, data);
if (data.getDelimiter() == null) {
log.warn("No delimiter provided, using comma ','");
@@ -171,7 +171,12 @@ public class MariaDataService extends JdbcConnector implements DataService {
.build();
final List<List<String>> rows = new LinkedList<>();
reader.readAll()
- .forEach(x -> rows.add(new ArrayList<>(List.of(x))));
+ .forEach(x -> {
+ final ArrayList<String> row = new ArrayList<>();
+ Collections.addAll(row, x);
+ log.trace("add row {}", row);
+ rows.add(row);
+ });
log.trace("csv rows {}", rows.size());
/* generic header, ref issue #95 */
List<String> headers = TableUtils.fill(0, rows.get(0).size());
diff --git a/fda-table-service/services/src/main/java/at/tuwien/service/impl/RabbitMqService.java b/fda-table-service/services/src/main/java/at/tuwien/service/impl/RabbitMqService.java
index 5d35123746..fc1f75c3e8 100644
--- a/fda-table-service/services/src/main/java/at/tuwien/service/impl/RabbitMqService.java
+++ b/fda-table-service/services/src/main/java/at/tuwien/service/impl/RabbitMqService.java
@@ -84,6 +84,7 @@ public class RabbitMqService implements MessageQueueService {
log.debug("declare fanout exchange {}", database.getExchange());
channel.exchangeBind(database.getExchange(), AMQP_EXCHANGE, database.getExchange());
log.debug("bind exchange {} to {}", database.getExchange(), AMQP_EXCHANGE);
+ log.info("Declared database exchange {} and bound to root exchange {}", database.getExchange(), AMQP_EXCHANGE);
}
@Override
@@ -93,6 +94,7 @@ public class RabbitMqService implements MessageQueueService {
log.debug("declare queue {}", table.getTopic());
channel.queueBind(table.getTopic(), table.getDatabase().getExchange(), table.getTopic());
log.debug("bind queue {} to {}", table.getTopic(), table.getDatabase().getExchange());
+ log.info("Declared queue {} and bound to database exchange {}", table.getTopic(), table.getDatabase().getExchange());
}
@Override
--
GitLab