Merge branch 'dev' into 291-modify-primary-keys-not-possible

.gitlab-ci.yml

@@ -298,7 +298,7 @@ test-frontend:
     - build-frontend
   script:
     - "make test-frontend"
-    - "cat ./fda-analyse-service/coverage.txt | grep -o 'TOTAL[^%]*%'"
+    - "cat ./fda-ui/coverage.txt | grep -o 'Lines[^%]*%'"
   artifacts:
     when: always
     paths:

.junit/teardown

+#!/bin/bash
+echo "=== [ Stopping dbrepo-* ] ==="
+docker container stop $(docker container ls -aq -f name=^/dbrepo-.*) || true
+echo "=== [ Removing dbrepo-* ] ==="
+docker container rm $(docker container ls -aq -f name=^/dbrepo-.*) || true
+docker volume rm $(docker volume ls -q -f name=^dbrepo-.*) || true
+docker network rm $(docker network ls -q -f name=^dbrepo-.*) || true
+echo "=== [ Stopping * ] ==="
+docker container stop $(docker container ls -aq -f name=.*-service) || true
+docker container stop ui ui-proxy metadata-db || true
+echo "=== [ Removing * ] ==="
+docker container rm $(docker container ls -aq -f name=.*-service) || true
+docker container rm ui ui-proxy metadata-db || true
+docker volume rm $(docker volume ls -q) || true
+docker network rm core public userdb || true
+echo "=== [ Stopping fda-* ] ==="
+docker container stop $(docker container ls -aq -f name=^/fda-.*) || true
+echo "=== [ Removing fda-* ] ==="
+docker container rm $(docker container ls -aq -f name=^/fda-.*) || true
+docker volume rm $(docker volume ls -q -f name=^fda-.*) || true
+docker network rm $(docker network ls -q -f name=^fda-.*) || true
+echo "=== [ Stopping tuw-* ] ==="
+docker container stop $(docker container ls -aq -f name=^/tuw-.*) || true
+echo "=== [ Removing tuw-* ] ==="
+docker container rm $(docker container ls -aq -f name=^/tuw-.*) || true
+docker volume rm $(docker volume ls -q -f name=^tuw-.*) || true
+docker network rm $(docker network ls -q -f name=^tuw-.*) || true
\ No newline at end of file

Makefile

@@ -189,10 +189,8 @@ coverage-frontend: build-frontend
 
 test-frontend: clean build-frontend
 	yarn --cwd ./fda-ui install
-	cp .env.unix.example .env
-	docker compose up -d
-	yarn --cwd ./fda-ui run test:e2e || true
-	docker compose down
+	yarn --cwd ./fda-ui run test:unit || true
+	yarn --cwd ./fda-ui run coverage || true
 
 clean:
 	docker system prune -f --volumes
@@ -203,4 +201,4 @@ test-clients:
 test: test-backend test-frontend
 
 teardown:
-	./.dbrepo2/teardown
+	./.junit/teardown

fda-query-service/rest-service/src/main/resources/application-local.yml

@@ -32,7 +32,7 @@ logging:
   pattern.console: "%d %highlight(%-5level) %msg%n"
   level:
     root: warn
-    at.tuwien.: info
+    at.tuwien.: trace
     org.springframework.web.servlet.mvc.method.annotation.ExceptionHandlerExceptionResolver: debug
 eureka:
   instance.hostname: query-service

fda-query-service/rest-service/src/test/java/at/tuwien/BaseUnitTest.java

@@ -300,6 +300,16 @@ public abstract class BaseUnitTest {
             .tag(IMAGE_ELASTIC_TAG)
             .build();
 
+    public final static Long IMAGE_PROXY_ID = 4L;
+    public final static String IMAGE_PROXY_REPOSITORY = "nginx";
+    public final static String IMAGE_PROXY_TAG = "latest";
+
+    public final static ContainerImage IMAGE_PROXY = ContainerImage.builder()
+            .id(IMAGE_PROXY_ID)
+            .repository(IMAGE_PROXY_REPOSITORY)
+            .tag(IMAGE_PROXY_TAG)
+            .build();
+
     public final static Long CONTAINER_1_ID = 1L;
     public final static String CONTAINER_1_HASH = "deadbeef";
     public final static ContainerImage CONTAINER_1_IMAGE = IMAGE_1;
@@ -434,6 +444,28 @@ public abstract class BaseUnitTest {
             .creator(USER_1)
             .build();
 
+    public final static Long CONTAINER_PROXY_ID = 7L;
+    public final static String CONTAINER_PROXY_NAME = "dbrepo-proxy";
+    public final static String CONTAINER_PROXY_INTERNAL_NAME = "dbrepo-proxy";
+    public final static String CONTAINER_PROXY_IP = "172.31.0.4";
+    public final static String CONTAINER_PROXY_HASH = "deadbeef";
+    public final static Instant CONTAINER_PROXY_CREATED = Instant.now().minus(1, HOURS);
+    public final static String[] CONTAINER_PROXY_ENV = new String[]{};
+    public final static HealthCheck CONTAINER_PROXY_HEALTHCHECK = new HealthCheck()
+            .withTest(List.of("CMD", "service", "nginx", "status"));
+
+    public final static Container CONTAINER_PROXY = Container.builder()
+            .id(CONTAINER_PROXY_ID)
+            .name(CONTAINER_PROXY_NAME)
+            .internalName(CONTAINER_PROXY_INTERNAL_NAME)
+            .imageId(IMAGE_PROXY_ID)
+            .image(IMAGE_PROXY)
+            .hash(CONTAINER_PROXY_HASH)
+            .ipAddress(CONTAINER_PROXY_IP)
+            .created(CONTAINER_PROXY_CREATED)
+            .creator(USER_1)
+            .build();
+
     public final static Long DATABASE_1_ID = 1L;
     public final static String DATABASE_1_NAME = "Weather";
     public final static String DATABASE_1_INTERNALNAME = "weather";

fda-query-service/rest-service/src/test/java/at/tuwien/config/DockerConfig.java

@@ -90,12 +90,16 @@ public class DockerConfig extends BaseUnitTest {
         final HostConfig hostConfig1;
         final String network = (container.getInternalName().contains("userdb") ? "fda-userdb" : "fda-public");
         if (bind == null) {
+            log.trace("map standard binding /tmp:/tmp");
             hostConfig1 = hostConfig.withNetworkMode(network)
                     .withBinds(Bind.parse("/tmp:/tmp"));
         } else {
-            hostConfig1 = hostConfig.withNetworkMode(network).withBinds(Bind.parse(bind), Bind.parse("/tmp:/tmp"));
+            log.trace("map non-standard binding {}, /tmp:/tmp", bind);
+            hostConfig1 = hostConfig.withNetworkMode(network)
+                    .withBinds(Bind.parse(bind), Bind.parse("/tmp:/tmp"));
         }
         if (port != null) {
+            log.trace("map port binding {}:{}", port, port);
             hostConfig1.withPortBindings(PortBinding.parse(port + ":" + port));
         }
         final CreateContainerCmd cmd = dockerClient.createContainerCmd(container.getImage().getRepository() + ":" + container.getImage().getTag())
@@ -191,6 +195,9 @@ public class DockerConfig extends BaseUnitTest {
             case 5:
                 log.debug("container with id {} has a health check config", containerId);
                 return CONTAINER_BROKER_HEALTHCHECK;
+            case 7:
+                log.debug("container with id {} has a health check config", containerId);
+                return CONTAINER_PROXY_HEALTHCHECK;
         }
         log.trace("container with id {} does not have a healthcheck config", containerId);
         return null;

fda-query-service/rest-service/src/test/java/at/tuwien/listener/RabbitMqListenerIntegrationTest.java

@@ -3,9 +3,6 @@ package at.tuwien.listener;
 import at.tuwien.BaseUnitTest;
 import at.tuwien.api.amqp.ConsumerDto;
 import at.tuwien.config.*;
-import at.tuwien.exception.*;
-import at.tuwien.gateway.BrokerServiceGateway;
-import at.tuwien.listener.impl.RabbitMqListenerImpl;
 import at.tuwien.repository.jpa.*;
 import com.rabbitmq.client.BuiltinExchangeType;
 import com.rabbitmq.client.Channel;
@@ -18,6 +15,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.ActiveProfiles;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import java.io.IOException;
@@ -25,10 +23,10 @@ import java.util.*;
 import java.util.stream.Collectors;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.mockito.Mockito.when;
 
 
 @Log4j2
+@ActiveProfiles(profiles = "junit")
 @DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
 @SpringBootTest
 @ExtendWith(SpringExtension.class)
@@ -40,9 +38,6 @@ public class RabbitMqListenerIntegrationTest extends BaseUnitTest {
     @MockBean
     private IndexConfig indexConfig;
 
-    @MockBean
-    private BrokerServiceGateway brokerServiceGateway;
-
     @Autowired
     private Channel channel;
 
@@ -58,9 +53,6 @@ public class RabbitMqListenerIntegrationTest extends BaseUnitTest {
     @Autowired
     private TableRepository tableRepository;
 
-    @Autowired
-    private RabbitMqListenerImpl rabbitMqListener;
-
     @Autowired
     private H2Utils h2Utils;
 
@@ -71,12 +63,14 @@ public class RabbitMqListenerIntegrationTest extends BaseUnitTest {
     private AmqpConfig amqpConfig;
 
     @Rule
-    public Timeout globalTimeout = Timeout.seconds(60);
+    public Timeout globalTimeout = Timeout.seconds(300);
 
     @BeforeAll
     public static void beforeAll() throws InterruptedException {
         afterAll();
+        /* create networks */
         DockerConfig.createAllNetworks();
+        /* create containers */
         DockerConfig.createContainer(null, CONTAINER_BROKER, 15672, CONTAINER_BROKER_ENV);
         DockerConfig.startContainer(CONTAINER_BROKER);
     }
@@ -102,9 +96,10 @@ public class RabbitMqListenerIntegrationTest extends BaseUnitTest {
     }
 
     @Test
-    public void updateConsumers_succeeds() throws AmqpException, IOException, InterruptedException {
+    public void updateConsumers_succeeds() throws IOException, InterruptedException {
 
         /* pre-condition */
+        assertEquals(0, getConsumers().size());
         assertEquals(2, amqpConfig.getAmqpConsumers());
 
         /* mock */
@@ -115,19 +110,23 @@ public class RabbitMqListenerIntegrationTest extends BaseUnitTest {
         channel.queueBind(TABLE_2_QUEUE_NAME, DATABASE_1_EXCHANGE, TABLE_2_ROUTING_KEY);
         channel.queueDeclare(TABLE_3_QUEUE_NAME, true, false, false, null);
         channel.queueBind(TABLE_3_QUEUE_NAME, DATABASE_1_EXCHANGE, TABLE_3_ROUTING_KEY);
-        when(brokerServiceGateway.findAllConsumers())
-                .thenReturn(List.of());
 
         /* test */
-        rabbitMqListener.updateConsumers();
-        Thread.sleep(10 * 1000);
-        final List<ConsumerDto> response = rabbitMqConfig.findAllConsumers()
+        Thread.sleep(30 * 1000) /* wait for scheduled insert */;
+        final List<ConsumerDto> response = getConsumers();
+        final List<ConsumerDto> consumers1 = response.stream().filter(c -> c.getQueue().getName().equals(TABLE_1_QUEUE_NAME)).collect(Collectors.toList());
+        assertEquals(2, consumers1.size());
+        final List<ConsumerDto> consumers2 = response.stream().filter(c -> c.getQueue().getName().equals(TABLE_2_QUEUE_NAME)).collect(Collectors.toList());
+        assertEquals(2, consumers2.size());
+        final List<ConsumerDto> consumers3 = response.stream().filter(c -> c.getQueue().getName().equals(TABLE_3_QUEUE_NAME)).collect(Collectors.toList());
+        assertEquals(2, consumers3.size());
+    }
+
+    private List<ConsumerDto> getConsumers() throws IOException {
+        return rabbitMqConfig.findAllConsumers()
                 .stream()
                 .filter(c -> List.of(TABLE_1_QUEUE_NAME, TABLE_2_QUEUE_NAME, TABLE_3_QUEUE_NAME).contains(c.getQueue().getName()))
                 .collect(Collectors.toList());
-        assertEquals(2, (int) response.stream().filter(c -> c.getQueue().getName().equals(TABLE_1_QUEUE_NAME)).count());
-        assertEquals(2, (int) response.stream().filter(c -> c.getQueue().getName().equals(TABLE_2_QUEUE_NAME)).count());
-        assertEquals(2, (int) response.stream().filter(c -> c.getQueue().getName().equals(TABLE_3_QUEUE_NAME)).count());
     }
 
 }

fda-query-service/rest-service/src/test/resources/application.properties

@@ -19,7 +19,7 @@ spring.jpa.show-sql=false
 
 # additional logging
 logging.level.root=error
-logging.level.at.tuwien.=info
+logging.level.at.tuwien.=trace
 
 # broker service
 spring.rabbitmq.host=dbrepo-broker-service
@@ -27,4 +27,6 @@ spring.rabbitmq.username=guest
 spring.rabbitmq.password=guest
 
 # search service
+fda.consumers=2
+fda.gateway.endpoint: http://localhost:15672
 fda.elastic.endpoint=dbrepo-search-service:9200
\ No newline at end of file

fda-query-service/rest-service/src/test/resources/webserver/weather_aus.csv

-1,2008-12-01,Albury,13.4,0.6
-2,2008-12-02,Albury,7.4,0
-3,2008-12-03,Albury,12.9,0
\ No newline at end of file

fda-query-service/services/src/main/java/at/tuwien/gateway/impl/BrokerServiceGatewayImpl.java

@@ -7,7 +7,9 @@ import at.tuwien.gateway.BrokerServiceGateway;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.codec.binary.Base64;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.ApplicationContext;
 import org.springframework.core.ParameterizedTypeReference;
+import org.springframework.core.env.Environment;
 import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpMethod;
@@ -17,6 +19,7 @@ import org.springframework.web.client.RestTemplate;
 
 import java.net.URI;
 import java.nio.charset.Charset;
+import java.util.Arrays;
 import java.util.List;
 
 @Slf4j
@@ -26,20 +29,30 @@ public class BrokerServiceGatewayImpl implements BrokerServiceGateway {
     private final AmqpConfig amqpConfig;
     private final RestTemplate restTemplate;
     private final GatewayConfig gatewayConfig;
+    private final Environment environment;
 
     private final static String VIRTUAL_SERVER = "%2F";
 
     @Autowired
-    public BrokerServiceGatewayImpl(AmqpConfig amqpConfig, RestTemplate restTemplate, GatewayConfig gatewayConfig) {
+    public BrokerServiceGatewayImpl(AmqpConfig amqpConfig, RestTemplate restTemplate, GatewayConfig gatewayConfig,
+                                    Environment environment) {
         this.amqpConfig = amqpConfig;
         this.restTemplate = restTemplate;
         this.gatewayConfig = gatewayConfig;
+        this.environment = environment;
     }
 
     @Override
     public List<ConsumerDto> findAllConsumers() {
+        final StringBuilder urlBuilder = new StringBuilder(gatewayConfig.getGatewayEndpoint())
+                .append("/api");
+        if (Arrays.stream(environment.getActiveProfiles()).noneMatch(p -> p.equals("junit"))) {
+            urlBuilder.append("/broker");
+        }
+        urlBuilder.append("/consumers/")
+                .append(VIRTUAL_SERVER);
         log.trace("gateway broker find all consumers, virtual server={}", VIRTUAL_SERVER);
-        final URI findUri = URI.create(gatewayConfig.getGatewayEndpoint() + "/api/broker/consumers/" + VIRTUAL_SERVER);
+        final URI findUri = URI.create(urlBuilder.toString());
         final ResponseEntity<List<ConsumerDto>> response = restTemplate.exchange(findUri, HttpMethod.GET,
                 new HttpEntity<>(null, getHeaders()), new ParameterizedTypeReference<>() {
                 });

fda-ui/.gitignore

@@ -21,6 +21,7 @@ lib-cov
 
 # Coverage directory used by tools like istanbul
 coverage/
+coverage.txt
 
 # nyc test coverage
 .nyc_output

fda-ui/.nycrc

@@ -6,7 +6,7 @@
   "branches": 35,
   "check-coverage": true,
   "include": [ "components/**/*.vue", "layouts/**/*.vue", "pages/**/*.vue", "server-middleware/**/*.js", "store/**/*.js"],
-  "exclude": ["node_modules"],
+  "exclude": ["node_modules", "server-middleware", "store"],
   "extension": [".js", ".vue"],
   "reporter": [
     "lcov",

fda-ui/package.json

@@ -12,7 +12,7 @@
     "ver": "nuxt --version",
     "lint:js": "eslint --ext .js,.vue --ignore-path .gitignore .",
     "lint": "yarn lint:js",
-    "coverage": "nyc ava test/unit/**/* test/specs/**/*",
+    "coverage": "nyc ava test/unit/**/* test/specs/**/* > ./coverage.txt",
     "test:unit": "ava test/unit/**/* test/specs/**/*",
     "test:watch": "ava --watch test/unit/**/* test/specs/**/*",
     "test:e2e": "yarn clean && cross-env PORT=3001 ava --timeout=2h --fail-fast test/e2e/**",