Improved images

- Analyse service severely less CVEs
- Semantics service too
- Removed discovery service
- Removed gateway service
- Use nginx as gateway servic- Removed discovery service
- Removed gateway service
- Use nginx as gateway servicee

.env.unix.example

@@ -15,7 +15,7 @@ KEYCLOAK_ADMIN=fda
 KEYCLOAK_ADMIN_PASSWORD=fda
 BROKER_CONSUMERS=2
 WEBSITE=http://localhost
-GATEWAY_ENDPOINT=http://gateway-service:9095
+GATEWAY_ENDPOINT=http://gateway-service
 TOKEN_MAX=5
 LOG_LEVEL=trace # error, warning, info, debug, trace
 DOI_URL="https://doi.org"

dbrepo-analyse-service/Dockerfile

@@ -4,7 +4,7 @@ MAINTAINER Martin Weise <martin.weise@tuwien.ac.at>
 RUN apk update && apk --no-cache add build-base gcc python3-dev libpq-dev libffi-dev py3-pandas \
     py3-sqlalchemy py3-requests py3-gevent py3-psycopg2
 
-COPY ./requirements.docker.txt ./requirements.txt
+COPY ./requirements.txt ./requirements.txt
 RUN pip install -r ./requirements.txt
 
 WORKDIR /app
@@ -14,7 +14,6 @@ ENV FLASK_RUN_HOST=0.0.0.0
 ENV PORT_APP=5000
 ENV FLASK_ENV=production
 ENV HOSTNAME=analyse-service
-ENV EUREKA_SERVER=http://discovery-service:9090/eureka/
 
 HEALTHCHECK --interval=10s --timeout=5s --retries=12 CMD ./service_ready
 

dbrepo-analyse-service/app.py

-import os
 from _csv import Error
 
 from flask import Flask, request, Response
 from determine_dt import determine_datatypes
 from determine_pk import determine_pk
 import logging
-import py_eureka_client.eureka_client as eureka_client
 from flasgger import Swagger
 from flasgger.utils import swag_from
 from flasgger import LazyJSONEncoder
@@ -148,11 +146,6 @@ def determinepk():
 
 
 rest_server_port = 5000
-eureka_client.init(eureka_server=os.getenv('EUREKA_SERVER', 'http://localhost:9090/eureka/'),
-                   app_name=os.getenv('HOSTNAME', 'analyse-service'),
-                   instance_ip=os.getenv('HOSTNAME', 'analyse-service'),
-                   instance_host=os.getenv('HOSTNAME', 'analyse-service'),
-                   instance_port=rest_server_port)
 
 if __name__ == '__main__':
     http_server = WSGIServer(('', 5000), app)

dbrepo-analyse-service/requirements.docker.txt

-messytables==0.15.2
-flask==2.3.2
-flasgger==0.9.5
-py-eureka-client==0.9.1
-docker==5.0.0
-prometheus_flask_exporter==0.21.0
-pytest==7.2.1
-coverage==7.1.0
\ No newline at end of file

dbrepo-analyse-service/requirements.txt

+attrs==23.1.0
+certifi==2023.5.7
+chardet==5.1.0
+charset-normalizer==2.0.12
+click==8.1.3
+coverage==7.1.0
+docker==5.0.0
+exceptiongroup==1.1.1
+flasgger==0.9.5
+Flask==2.2.2
+gevent==21.8.0
+greenlet==1.1.3.post0
+html5lib==1.1
+idna==3.4
+importlib-metadata==6.6.0
+iniconfig==2.0.0
+itsdangerous==2.1.2
+Jinja2==3.1.2
+json-table-schema==0.2.1
+jsonschema==4.17.3
+lxml==4.9.2
+MarkupSafe==2.1.2
 messytables==0.15.2
+mistune==2.0.5
+numpy==1.24.3
+packaging==23.1
 pandas==1.2.3
-flask==2.3.2
-flasgger==0.9.5
+pluggy==1.0.0
+prometheus-client==0.16.0
+prometheus-flask-exporter==0.21.0
 psycopg2-binary==2.8.6
-py-eureka-client==0.9.1
-docker==5.0.0
-sqlalchemy==1.4.15
-requests==2.26.0
-gevent==21.8.0
-prometheus_flask_exporter==0.21.0
+pyrsistent==0.19.3
 pytest==7.2.1
-coverage==7.1.0
+python-dateutil==2.8.2
\ No newline at end of file
+python-magic==0.4.27
+pytz==2023.3
+PyYAML==6.0
+requests==2.26.0
+six==1.16.0
+SQLAlchemy==1.4.15
+tomli==2.0.1
+urllib3==1.26.15
+webencodings==0.5.1
+websocket-client==1.5.1
+Werkzeug==2.3.3
+xlrd==2.0.1
+zipp==3.15.0
+zope.event==4.6
+zope.interface==6.0

dbrepo-authentication-service/Dockerfile

@@ -35,8 +35,6 @@ COPY ./dbrepo-realm.json /opt/keycloak/data/import/dbrepo-realm.json
 
 WORKDIR /app
 
-COPY ./service-register.sh ./service-register.sh
-COPY ./docker-entrypoint.sh ./docker-entrypoint.sh
 COPY ./healthcheck.sh ./healthcheck.sh
 
 ENV METADATA_USERNAME=root
@@ -54,4 +52,4 @@ ENV KEYCLOAK_ADMIN_PASSWORD=fda
 
 HEALTHCHECK --interval=10s --timeout=5s --retries=12 CMD ["bash", "/app/healthcheck.sh"]
 
-ENTRYPOINT ["bash", "/app/docker-entrypoint.sh"]
+ENTRYPOINT ["/opt/keycloak/bin/kc.sh", "start-dev", "--import-realm"]

dbrepo-authentication-service/docker-entrypoint.sh

-#!/bin/bash
-
-/app/service-register.sh authentication-service 8443 8080
-(while sleep 60; do bash /app/service-register.sh authentication-service 8443 8080; done) &
-
-/opt/keycloak/bin/kc.sh start-dev --import-realm

dbrepo-authentication-service/service-register.sh

-#!/bin/bash
-
-# $1 is used as the host name.
-
-EUREKA_HOST="discovery-service"
-EUREKA_PORT="9090"
-EUREKA_URI="http://$EUREKA_HOST:$EUREKA_PORT"
-
-SERVICE_NAME="$1"
-SERVICE_PROTOCOL="http"
-SERVICE_HOST="$1"
-SECURE_PORT="${2:-9000}"
-SERVICE_PORT="${3:-9000}"
-
-SERVICE_URI="$SERVICE_PROTOCOL://$SERVICE_HOST:$SERVICE_PORT"
-HOME_URI="$SERVICE_URI/realms/dbrepo"
-HEALTH_URI="$SERVICE_URI/health"
-
-# This is the URL shown in the "status" field in the
-# instances section of the eureka dashboard.
-#
-# It's up to you to decide what the URL points to. Some
-# information or status endpoint might be good.
-STATUS_URI="$SERVICE_URI/health"
-
-# This is the name displayed to the right of the status
-# on the eureka dashbard. If the app (FAKE_SERVICE) is
-# registered with more than one hostname, they will be
-# displayed as a comma-separated list. This hostname
-# is part of the heartbeat message.
-#
-# If you'll have more than one host per service,
-# make sure they have different host names.
-HOST_NAME="${1:-fake01}"
-
-# Everyone of these parameters seem to be required. I don't know
-# anything about secureVipAddress and vipAddress.
-#
-# dataCenterInfo must have a name of "MyOwn" or "Amazon".
-#
-# status can be UP, DOWN, STARTING, OUT_OF_SERVICE, UNKNOWN.
-#   if the registration status is STARTING, then the service
-#   will never be evicted. Also, simply sending a Heartbeat
-#   does not change the status.
-#
-# The metadata fields can be any information you want associated
-# with a service. I recommend keeping it short.
-#
-
-cat <<EOF > /tmp/json.json
-{
-  "instance": {
-    "instanceId": "$SERVICE_NAME:$SERVICE_NAME:$SERVICE_PORT",
-    "hostName": "$HOST_NAME",
-    "app": "$SERVICE_NAME",
-    "ipAddr": "$SERVICE_HOST",
-    "status": "UP",
-    "dataCenterInfo": {
-      "@class": "com.netflix.appinfo.MyDataCenterInfo",
-      "name": "MyOwn"
-    },
-    "healthCheckUrl": "$HEALTH_URI",
-    "homePageUrl": "$HOME_URI",
-    "leaseInfo": {
-      "evictionDurationInSecs": 90
-    },
-    "metadata": {
-      "zone": "default",
-      "management.port": "8443"
-    },
-    "port": {
-      "\$": "$SERVICE_PORT",
-      "@enabled": "true"
-    },
-    "securePort": {
-      "\$": "$SECURE_PORT",
-      "@enabled": "true"
-    },
-    "vipAddress": "$SERVICE_HOST",
-    "secureVipAddress": "$SERVICE_HOST",
-    "statusPageUrl": "$STATUS_URI"
-  }
-}
-EOF
-
-curl --header "content-type: application/json" --data-binary @/tmp/json.json --silent $EUREKA_URI/eureka/apps/$SERVICE_NAME

dbrepo-broker-service/Dockerfile

@@ -16,12 +16,9 @@ COPY ./rabbitmq.conf /etc/rabbitmq/rabbitmq.conf
 WORKDIR /app
 
 COPY ./init.sh ./init.sh
-COPY ./service-register.sh ./service-register.sh
 COPY ./service_ready /usr/bin/service_ready
 COPY ./docker-entrypoint.sh ./docker-entrypoint.sh
 
-RUN chmod +x ./service-register.sh
-
 HEALTHCHECK --interval=10s --timeout=5s --retries=12 CMD service_ready
 
 ENTRYPOINT [ "bash", "/app/docker-entrypoint.sh" ]
\ No newline at end of file

dbrepo-broker-service/docker-entrypoint.sh

@@ -6,8 +6,4 @@ bash ./init.sh
 # enable prometheus plugin
 (sleep 10; rabbitmq-plugins enable rabbitmq_prometheus rabbitmq_mqtt rabbitmq_auth_backend_oauth2 rabbitmq_auth_mechanism_ssl; touch /ready) &
 
-# register with discovery service
-/app/service-register.sh broker-service 15672 15672
-(while sleep 60; do /app/service-register.sh broker-service 15672 15672; done) &
-
 rabbitmq-server
\ No newline at end of file

dbrepo-broker-service/service-register.sh

-#!/bin/bash
-
-# $1 is used as the host name.
-
-EUREKA_HOST="discovery-service"
-EUREKA_PORT="9090"
-EUREKA_URI="http://$EUREKA_HOST:$EUREKA_PORT"
-
-SERVICE_NAME="$1"
-SERVICE_PROTOCOL="http"
-SERVICE_HOST="$1"
-SECURE_PORT="${2:-9000}"
-SERVICE_PORT="${3:-9000}"
-
-SERVICE_URI="$SERVICE_PROTOCOL://$SERVICE_HOST:$SERVICE_PORT"
-HOME_URI="$SERVICE_URI/realms/dbrepo"
-HEALTH_URI="$SERVICE_URI/health"
-
-# This is the URL shown in the "status" field in the
-# instances section of the eureka dashboard.
-#
-# It's up to you to decide what the URL points to. Some
-# information or status endpoint might be good.
-STATUS_URI="$SERVICE_URI/health"
-
-# This is the name displayed to the right of the status
-# on the eureka dashbard. If the app (FAKE_SERVICE) is
-# registered with more than one hostname, they will be
-# displayed as a comma-separated list. This hostname
-# is part of the heartbeat message.
-#
-# If you'll have more than one host per service,
-# make sure they have different host names.
-HOST_NAME="${1:-fake01}"
-
-# Everyone of these parameters seem to be required. I don't know
-# anything about secureVipAddress and vipAddress.
-#
-# dataCenterInfo must have a name of "MyOwn" or "Amazon".
-#
-# status can be UP, DOWN, STARTING, OUT_OF_SERVICE, UNKNOWN.
-#   if the registration status is STARTING, then the service
-#   will never be evicted. Also, simply sending a Heartbeat
-#   does not change the status.
-#
-# The metadata fields can be any information you want associated
-# with a service. I recommend keeping it short.
-#
-
-cat <<EOF > /tmp/json.json
-{
-  "instance": {
-    "instanceId": "$SERVICE_NAME:$SERVICE_NAME:$SERVICE_PORT",
-    "hostName": "$HOST_NAME",
-    "app": "$SERVICE_NAME",
-    "ipAddr": "$SERVICE_HOST",
-    "status": "UP",
-    "dataCenterInfo": {
-      "@class": "com.netflix.appinfo.MyDataCenterInfo",
-      "name": "MyOwn"
-    },
-    "healthCheckUrl": "$HEALTH_URI",
-    "homePageUrl": "$HOME_URI",
-    "leaseInfo": {
-      "evictionDurationInSecs": 90
-    },
-    "metadata": {
-      "zone": "default",
-      "management.port": "15672"
-    },
-    "port": {
-      "\$": "$SERVICE_PORT",
-      "@enabled": "true"
-    },
-    "securePort": {
-      "\$": "$SECURE_PORT",
-      "@enabled": "false"
-    },
-    "vipAddress": "$SERVICE_HOST",
-    "secureVipAddress": "$SERVICE_HOST",
-    "statusPageUrl": "$STATUS_URI"
-  }
-}
-EOF
-
-curl --header "content-type: application/json" --data-binary @/tmp/json.json --silent $EUREKA_URI/eureka/apps/$SERVICE_NAME

dbrepo-container-service/rest-service/src/main/resources/application-local.yml

@@ -36,11 +36,6 @@ logging:
     root: warn
     at.tuwien.: trace
     org.springframework.web.servlet.mvc.method.annotation.ExceptionHandlerExceptionResolver: debug
-eureka:
-  instance:
-    hostname: container-service
-    non-secure-port: 9091
-  client.serviceUrl.defaultZone: http://localhost:9090/eureka/
 fda:
   network: userdb
   mount.path: /tmp
@@ -50,4 +45,4 @@ fda:
     public_key: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB
   client_secret: client-secret
   client_id: dbrepo-client
-  gateway.endpoint: http://localhost:9095
+  gateway.endpoint: http://localhost
\ No newline at end of file

dbrepo-container-service/rest-service/src/main/resources/application.yml

@@ -36,11 +36,6 @@ logging:
     root: warn
     at.tuwien.: "${LOG_LEVEL}"
     org.springframework.web.servlet.mvc.method.annotation.ExceptionHandlerExceptionResolver: debug
-eureka:
-  instance:
-    hostname: container-service
-    non-secure-port: 9091
-  client.serviceUrl.defaultZone: http://discovery-service:9090/eureka/
 fda:
   network: "${USER_NETWORK}"
   mount.path: "${SHARED_FILESYSTEM}"
@@ -50,4 +45,4 @@ fda:
     public_key: "${JWT_PUBKEY}"
   client_secret: "${DBREPO_CLIENT_SECRET}"
   client_id: "${CLIENT_ID}"
-  gateway.endpoint: http://gateway-service:9095
+  gateway.endpoint: http://gateway-service
\ No newline at end of file

dbrepo-database-service/Dockerfile

@@ -29,7 +29,7 @@ ENV BROKER_PASSWORD=fda
 ENV SEARCH_ENDPOINT=search-service
 ENV SEARCH_USERNAME=elastic
 ENV SEARCH_PASSWORD=elastic
-ENV GATEWAY_ENDPOINT=http://gateway-service:9095
+ENV GATEWAY_ENDPOINT=http://gateway-service
 ENV LOG_LEVEL=debug
 ENV DBREPO_CLIENT_SECRET=client-secret
 ENV CLIENT_ID=dbrepo-client

dbrepo-database-service/rest-service/src/main/resources/application-local.yml

@@ -35,11 +35,6 @@ logging:
     root: warn
     at.tuwien.: info
     org.springframework.web.servlet.mvc.method.annotation.ExceptionHandlerExceptionResolver: debug
-eureka:
-  instance:
-    hostname: database-service
-    non-secure-port: 9092
-  client.serviceUrl.defaultZone: http://localhost:9090/eureka/
 fda:
   elastic:
     endpoint: localhost:9200
@@ -51,4 +46,4 @@ fda:
     public_key: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB
   client_secret: client-secret
   client_id: dbrepo-client
-  gateway.endpoint: http://localhost:9095
+  gateway.endpoint: http://localhost
\ No newline at end of file

dbrepo-database-service/rest-service/src/main/resources/application.yml

@@ -35,11 +35,6 @@ logging:
     root: warn
     at.tuwien.: "${LOG_LEVEL}"
     org.springframework.web.servlet.mvc.method.annotation.ExceptionHandlerExceptionResolver: debug
-eureka:
-  instance:
-    hostname: database-service
-    non-secure-port: 9092
-  client.serviceUrl.defaultZone: http://discovery-service:9090/eureka/
 fda:
   elastic:
     endpoint: search-service:9200

dbrepo-discovery-service/.gitignore

-HELP.md
-target/
-!.mvn/wrapper/maven-wrapper.jar
-!**/src/main/**/target/
-!**/src/test/**/target/
-
-## Generated
-rest-service/ready
-
-### STS ###
-.apt_generated
-.classpath
-.factorypath
-.project
-.settings
-.springBeans
-.sts4-cache
-
-### IntelliJ IDEA ###
-.idea
-*.iws
-*.iml
-*.ipr
-
-### NetBeans ###
-/nbproject/private/
-/nbbuild/
-/dist/
-/nbdist/
-/.nb-gradle/
-build/
-!**/src/main/**/build/
-!**/src/test/**/build/
-
-### VS Code ###
-.vscode/

dbrepo-discovery-service/.mvn/wrapper/MavenWrapperDownloader.java

-/*
- * Copyright 2007-present the original author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.net.*;
-import java.io.*;
-import java.nio.channels.*;
-import java.util.Properties;
-
-public class MavenWrapperDownloader {
-
-    private static final String WRAPPER_VERSION = "0.5.6";
-    /**
-     * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
-     */
-    private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/"
-            + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
-
-    /**
-     * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
-     * use instead of the default one.
-     */
-    private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
-            ".mvn/wrapper/maven-wrapper.properties";
-
-    /**
-     * Path where the maven-wrapper.jar will be saved to.
-     */
-    private static final String MAVEN_WRAPPER_JAR_PATH =
-            ".mvn/wrapper/maven-wrapper.jar";
-
-    /**
-     * Name of the property which should be used to override the default download url for the wrapper.
-     */
-    private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
-
-    public static void main(String args[]) {
-        System.out.println("- Downloader started");
-        File baseDirectory = new File(args[0]);
-        System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
-
-        // If the maven-wrapper.properties exists, read it and check if it contains a custom
-        // wrapperUrl parameter.
-        File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
-        String url = DEFAULT_DOWNLOAD_URL;
-        if (mavenWrapperPropertyFile.exists()) {
-            FileInputStream mavenWrapperPropertyFileInputStream = null;
-            try {
-                mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
-                Properties mavenWrapperProperties = new Properties();
-                mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
-                url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
-            } catch (IOException e) {
-                System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
-            } finally {
-                try {
-                    if (mavenWrapperPropertyFileInputStream != null) {
-                        mavenWrapperPropertyFileInputStream.close();
-                    }
-                } catch (IOException e) {
-                    // Ignore ...
-                }
-            }
-        }
-        System.out.println("- Downloading from: " + url);
-
-        File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
-        if (!outputFile.getParentFile().exists()) {
-            if (!outputFile.getParentFile().mkdirs()) {
-                System.out.println(
-                        "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'");
-            }
-        }
-        System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
-        try {
-            downloadFileFromURL(url, outputFile);
-            System.out.println("Done");
-            System.exit(0);
-        } catch (Throwable e) {
-            System.out.println("- Error downloading");
-            e.printStackTrace();
-            System.exit(1);
-        }
-    }
-
-    private static void downloadFileFromURL(String urlString, File destination) throws Exception {
-        if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) {
-            String username = System.getenv("MVNW_USERNAME");
-            char[] password = System.getenv("MVNW_PASSWORD").toCharArray();
-            Authenticator.setDefault(new Authenticator() {
-                @Override
-                protected PasswordAuthentication getPasswordAuthentication() {
-                    return new PasswordAuthentication(username, password);
-                }
-            });
-        }
-        URL website = new URL(urlString);
-        ReadableByteChannel rbc;
-        rbc = Channels.newChannel(website.openStream());
-        FileOutputStream fos = new FileOutputStream(destination);
-        fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
-        fos.close();
-        rbc.close();
-    }
-
-}

dbrepo-discovery-service/.mvn/wrapper/maven-wrapper.properties

-distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip
-wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar