Skip to content
Snippets Groups Projects
Commit f8c58ea7 authored by Martin Weise's avatar Martin Weise
Browse files

Merge branch 'dev' into 67-database-query-frontend

parents 4668c7c3 c77dc057
Branches
Tags
2 merge requests!81New stable release,!68Raw SQL
Showing
with 256 additions and 146 deletions
REGISTRY="docker.example.com"
API="http://fda-gateway-service:9095"
KEY_STORE_PASSWORD="12345"
MAIL_USERNAME="eMATRIKELNUMMER" // TU student e-mail server
MAIL_PASSWORD="PASSWORD" // TU student e-mail server
ADMIN_PASSWORD="admin"
\ No newline at end of file
#!/bin/bash
FILES=$(cd /tmp && find -exec basename '{}' ';' 2>/dev/null | egrep '^.{32}$' | egrep "^([a-z0-9]+)$")
for file in $FILES; do
rm -f /tmp/$file
echo "Removed /tmp/${file}"
done
\ No newline at end of file
......@@ -152,10 +152,13 @@ clean-ide:
clean-frontend:
rm -f ./fda-ui/videos/*.webm
clean-tmp:
./.fda-deployment/clean-tmp
clean-docker:
./.fda-deployment/clean
clean: clean-ide clean-frontend clean-docker
clean: clean-ide clean-frontend clean-docker clean-tmp
teardown:
./.fda-deployment/teardown
......@@ -28,7 +28,6 @@ services:
image: fda-metadata-db
networks:
fda-public:
# ipv4_address: 172.29.0.5
volumes:
- fda-metadata-db-data:/var/lib/postgresql/data
ports:
......@@ -49,7 +48,6 @@ services:
image: fda-discovery-service
networks:
fda-public:
# ipv4_address: 172.29.0.4
environment:
SPRING_PROFILES_ACTIVE: docker
TZ: Europe/Vienna
......@@ -69,7 +67,6 @@ services:
image: fda-gateway-service
networks:
fda-public:
# ipv4_address: 172.29.0.2
environment:
SPRING_PROFILES_ACTIVE: docker
TZ: Europe/Vienna
......@@ -87,7 +84,6 @@ services:
networks:
fda-userdb:
fda-public:
# ipv4_address: 172.29.0.9
environment:
SPRING_PROFILES_ACTIVE: docker
TZ: Europe/Vienna
......@@ -98,6 +94,10 @@ services:
depends_on:
fda-container-service:
condition: service_healthy
fda-broker-service:
condition: service_healthy
fda-authentication-service:
condition: service_healthy
logging:
driver: json-file
......@@ -109,7 +109,6 @@ services:
image: fda-container-service
networks:
fda-public:
# ipv4_address: 172.29.0.8
environment:
SPRING_PROFILES_ACTIVE: docker
TZ: Europe/Vienna
......@@ -131,17 +130,19 @@ services:
build: fda-authentication-service
networks:
fda-public:
# ipv4_address: 172.29.0.14
environment:
SPRING_PROFILES_ACTIVE: docker
SMTP_USERNAME: ${MAIL_USERNAME:-local}
SMTP_PASSWORD: ${MAIL_PASSWORD:-local}
SYSTEM_PASSWORD: ${ADMIN_PASSWORD:-admin}
TZ: Europe/Vienna
ports:
- "9097:9097"
depends_on:
fda-discovery-service:
condition: service_healthy
fda-metadata-db:
condition: service_healthy
logging:
driver: json-file
......@@ -153,7 +154,6 @@ services:
image: fda-query-service
networks:
fda-public:
# ipv4_address: 172.29.0.12
fda-userdb:
environment:
SPRING_PROFILES_ACTIVE: docker
......@@ -165,6 +165,8 @@ services:
depends_on:
fda-table-service:
condition: service_healthy
fda-authentication-service:
condition: service_healthy
logging:
driver: json-file
......@@ -176,10 +178,10 @@ services:
image: fda-table-service
networks:
fda-public:
# ipv4_address: 172.29.0.11
fda-userdb:
environment:
SPRING_PROFILES_ACTIVE: docker
SYSTEM_PASSWORD: ${ADMIN_PASSWORD:-admin}
multipart.location: /tmp
TZ: Europe/Vienna
ports:
......@@ -188,7 +190,11 @@ services:
- /var/run/docker.sock:/var/run/docker.sock
- /tmp:/tmp
depends_on:
fda-database-service:
fda-authentication-service:
condition: service_healthy
fda-search-service:
condition: service_started
fda-broker-service:
condition: service_healthy
logging:
driver: json-file
......@@ -201,7 +207,6 @@ services:
image: fda-identifier-service
networks:
fda-public:
# ipv4_address: 172.29.0.13
environment:
SPRING_PROFILES_ACTIVE: docker
TZ: Europe/Vienna
......@@ -210,6 +215,10 @@ services:
depends_on:
fda-query-service:
condition: service_healthy
fda-authentication-service:
condition: service_healthy
logging:
driver: json-file
fda-analyse-service:
restart: on-failure
......@@ -219,7 +228,6 @@ services:
image: fda-analyse-service
networks:
fda-public:
# ipv4_address: 172.29.0.10
fda-userdb:
command: sh -c "/wait && flask run" # docker-compose should not test the implementation
environment:
......@@ -243,7 +251,6 @@ services:
image: fda-units-service
networks:
fda-public:
# ipv4_address: 172.29.0.7
environment:
EUREKA_SERVER: http://fda-discovery-service:9090/eureka/
TZ: Europe/Vienna
......@@ -268,7 +275,6 @@ services:
TZ: Europe/Vienna
networks:
fda-public:
# ipv4_address: 172.29.0.3
ports:
- "5672:5672"
- "15672:15672"
......@@ -288,7 +294,6 @@ services:
command: [ "elasticsearch" ]
networks:
fda-public:
# ipv4_address: 172.29.0.6
environment:
- discovery.type=single-node
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
......@@ -310,7 +315,6 @@ services:
image: fda-ui
networks:
fda-public:
# ipv4_address: 172.29.0.15
ports:
- "443:3000"
volumes:
......
......@@ -134,6 +134,7 @@ services:
SPRING_PROFILES_ACTIVE: docker
SMTP_USERNAME: ${MAIL_USERNAME:-local}
SMTP_PASSWORD: ${MAIL_PASSWORD:-local}
SYSTEM_PASSWORD: ${ADMIN_PASSWORD:-admin}
TZ: Europe/Vienna
ports:
- "9097:9097"
......@@ -180,6 +181,7 @@ services:
fda-userdb:
environment:
SPRING_PROFILES_ACTIVE: docker
SYSTEM_PASSWORD: ${ADMIN_PASSWORD:-admin}
multipart.location: /tmp
TZ: Europe/Vienna
ports:
......
import os
from flask import Flask, flash, request, redirect, url_for, Response, abort, jsonify
from determine_dt import determine_datatypes
from analysecsv import analysecsv
from insert_mdb_db import insert_mdb_db
from import_db import import_db
from update_mdb_db_ispublic import insert_mdb_db_pub
from insert_mdb_tbl import insert_mdb_tbl
from insert_mdb_col import insert_mdb_col, update_mdb_col
from insert_mdb_col import update_mdb_siunit
from update_mdb_data import update_mdb_data
from insert_mdb_col import update_mdb_col
from determine_pk import determine_pk
#from werkzeug.utils import secure_filename
#from werkzeug import cached_property
import logging
import sys
import py_eureka_client.eureka_client as eureka_client
from os import environ
#from flask_swagger_ui import get_swaggerui_blueprint
#from flask_restful_swagger_3 import Api, Resource, swagger, Schema
import json
from flasgger import Swagger
from flasgger.utils import swag_from
from flasgger import LazyString, LazyJSONEncoder
from gevent.pywsgi import WSGIServer
#logging.basicConfig()
logging.basicConfig(level=logging.DEBUG)
#UPLOAD_FOLDER = '.'
#ALLOWED_EXTENSIONS = {'csv'}
from logging.config import dictConfig
dictConfig({
'version': 1,
'formatters': {'default': {
'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
}},
'handlers': {'wsgi': {
'class': 'logging.StreamHandler',
'stream': 'ext://flask.logging.wsgi_errors_stream',
'formatter': 'default'
}},
'root': {
'level': 'INFO',
'handlers': ['wsgi']
}
})
app = Flask(__name__)
app.config["SWAGGER"] = {"title": "FDA-Analyse-Service", "uiversion": 3}
......@@ -79,10 +86,12 @@ def determinedt():
if 'separator' in input_json:
separator = str(input_json['separator'])
res = determine_datatypes(filepath,enum,enum_tol,separator)
except Exception as e:
print(e)
res = {"success": False, "message": "Unknown error"}
logging.info('Determined datatypes successfully: %s',res)
return jsonify(res), 200
except Exception as e:
logging.error(e)
res = {"success": False, "message": str(e)}
return jsonify(res), 500
@app.route('/api/analyse/determinepk', methods=["POST"], endpoint='analyze_determinepk')
@swag_from('/as-yml/determinepk.yml')
......@@ -94,113 +103,12 @@ def determinepk():
if 'seperator' in input_json:
seperator = str(input_json['seperator'])
res = determine_pk(filepath,seperator)
except Exception as e:
print(e)
res = {"success": False, "message": "Unknown error"}
logging.info('Determined list of primary keys: %s', res)
return jsonify(res), 200
@app.route('/api/analyse/checkcsv', methods=["POST"], endpoint='analyze_checkcsv')
@swag_from('/as-yml/checkcsv.yml')
def checkcsv():
input_json = request.get_json()
try:
filepath = str(input_json['filepath'])
intdbname = str(input_json['intdbname'])
dbhost = str(input_json['dbhost'])
dbid = int(input_json['dbid'])
tname = str(input_json['tname'])
header = True
if 'header' in input_json:
header = bool(input_json['header'])
seperator = ','
if 'seperator' in input_json:
seperator = str(input_json['seperator'])
res = analysecsv(filepath,seperator,intdbname, dbhost, dbid, tname, header)
except Exception as e:
print(e)
res = {"success": False, "message": "Unknown error"}
return jsonify(res), 200
@app.route('/api/analyse/update_mdb_db', methods=["POST"], endpoint='mdb_update_db')
@swag_from('/as-yml/importdb.yml')
def importdb():
input_json = request.get_json()
try:
dbid=int(input_json['dbid'])
resourcetype = str(input_json['resourcetype'])
description = str(input_json['description'])
publisher = str(input_json['publisher'])
res = insert_mdb_db(dbid, resourcetype, description, publisher)
except Exception as e:
print(e)
res = {"success": False, "message": "Unknown error"}
return jsonify(res), 200
@app.route('/api/analyse/update_mdb_tbl', methods=["POST"], endpoint='mdb_update_tbl')
@swag_from('/as-yml/importtbl.yml')
def importtbl():
input_json = request.get_json()
try:
dbid=int(input_json['dbid'])
res = insert_mdb_tbl(dbid)
except Exception as e:
print(e)
res = {"success": False, "message": "Unknown error"}
return jsonify(res), 200
@app.route('/api/analyse/update_mdb_db_ispublic', methods=["POST"], endpoint='mdb_ispublic')
@swag_from('/as-yml/updateispub.yml')
def updateispublic():
input_json = request.get_json()
try:
dbid=int(input_json['dbid'])
ispublic=bool(input_json['is_public'])
res = insert_mdb_db_pub(dbid, ispublic)
except Exception as e:
print(e)
res = {"success": False, "message": "Unknown error"}
return jsonify(res), 200
@app.route('/api/analyse/update_mdb_columns_num_siunit', methods=["POST"], endpoint='mdb_columns_num')
@swag_from('/as-yml/updatesiunit.yml')
def updatesiunit():
input_json = request.get_json()
try:
dbid=int(input_json['dbid'])
tid=int(input_json['tid'])
cid=int(input_json['cid'])
siunit=str(input_json['siunit'])
res = update_mdb_siunit(dbid,tid,cid,siunit)
except Exception as e:
print(e)
res = {"success": False, "message": "Unknown error"}
return jsonify(res), 200
@app.route('/api/analyse/update_mdb_data_provenance', methods=["POST"], endpoint='mdb_update_data_provenance')
@swag_from('/as-yml/updatedata.yml')
def updatesdataprovenance():
input_json = request.get_json()
try:
dataid=int(input_json['dataid'])
prov=str(input_json['provenance'])
res = update_mdb_data(dataid,prov)
except Exception as e:
print(e)
res = {"success": False, "message": "Unknown error"}
return jsonify(res), 200
#@app.route('/api/analyse/insert_mdb_col', methods=["POST"], endpoint='mdb_insert_col')
#@swag_from('/as-yml/importcol.yml')
#def importcol():
# input_json = request.get_json()
# try:
# dbid=int(input_json['dbid'])
# tid=int(input_json['tid'])
# res = insert_mdb_col(dbid,tid)
# except Exception as e:
# print(e)
# res = {"success": False, "message": "Unknown error"}
# return jsonify(res), 200
logging.error(e)
res = {"success": False, "message": str(e)}
return jsonify(res), 500
@app.route('/api/analyse/update_mdb_col', methods=["POST"], endpoint='mdb_update_col')
@swag_from('/as-yml/updatecol.yml')
......@@ -211,10 +119,12 @@ def updatecol():
tid=int(input_json['tid'])
cid=int(input_json['cid'])
res = update_mdb_col(dbid,tid,cid)
logging.info('Update metadata database entity mdb_columns')
return jsonify(res), 200
except Exception as e:
print(e)
logging.error(e)
res = {"success": False, "message": "Unknown error"}
return jsonify(res), 200
return jsonify(res), 500
rest_server_port = 5000
eureka_client.init(eureka_server=os.getenv('EUREKA_SERVER', 'http://localhost:9090/eureka/'),
......
id#name
1#c
2#a
3#b
messytables==0.15.2
pandas==1.2.3
flask==1.1.4
flask==2.1.2
flasgger==0.9.5
psycopg2-binary==2.8.6
py-eureka-client==0.9.1
......
package at.tuwien.config;
import at.tuwien.seeder.Seeder;
import com.google.common.io.Files;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Value;
......@@ -17,8 +18,15 @@ public class ReadyConfig {
@Value("${fda.ready.path}")
private String readyPath;
private final Seeder userSeederImpl;
public ReadyConfig(Seeder userSeederImpl) {
this.userSeederImpl = userSeederImpl;
}
@EventListener(ApplicationReadyEvent.class)
public void init() throws IOException {
userSeederImpl.seed();
Files.touch(new File(readyPath));
}
......
......@@ -41,6 +41,7 @@ eureka:
fda:
ready.path: ./ready
mail.prefix: /
system.passwd: "${SYSTEM_PASSWORD}"
mail.from: DBREPO <noreply@dbrepo.ossdip.at>
mail.replyto: Martin Weise <martin.weise@tuwien.ac.at>
jwt:
......
......@@ -41,6 +41,7 @@ eureka:
fda:
ready.path: ./ready
mail.prefix: /
system.passwd: "${SYSTEM_PASSWORD}"
mail.from: DBREPO <noreply@dbrepo.ossdip.at>
mail.replyto: Martin Weise <martin.weise@tuwien.ac.at>
jwt:
......
package at.tuwien.config;
import lombok.Getter;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.thymeleaf.spring5.SpringTemplateEngine;
import org.thymeleaf.templatemode.TemplateMode;
import org.thymeleaf.templateresolver.ClassLoaderTemplateResolver;
import java.nio.charset.StandardCharsets;
@Log4j2
@Getter
@Configuration
public class SecurityConfig {
@Value("${fda.system.passwd}")
private String systemPassword;
}
package at.tuwien.seeder;
public interface Seeder {
void seed();
}
package at.tuwien.seeder.impl;
import at.tuwien.entities.user.RoleType;
import at.tuwien.entities.user.User;
import java.util.List;
public abstract class AbstractSeeder {
public static final String USER_1_USERNAME = "system";
public static final String USER_1_EMAIL = "noreply@dbrepo.ossdip.at";
public static final List<RoleType> USER_1_ROLES = List.of(RoleType.ROLE_RESEARCHER);
public static final User USER_1 = User.builder()
.username(USER_1_USERNAME)
.email(USER_1_EMAIL)
.roles(USER_1_ROLES)
.build();
}
package at.tuwien.seeder.impl;
import at.tuwien.config.SecurityConfig;
import at.tuwien.entities.user.User;
import at.tuwien.repositories.UserRepository;
import at.tuwien.seeder.Seeder;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.stereotype.Component;
@Log4j2
@Component
public class UserSeederImpl extends AbstractSeeder implements Seeder {
private final UserRepository userRepository;
private final SecurityConfig securityConfig;
private final PasswordEncoder passwordEncoder;
@Autowired
public UserSeederImpl(UserRepository userRepository, SecurityConfig securityConfig,
PasswordEncoder passwordEncoder) {
this.userRepository = userRepository;
this.securityConfig = securityConfig;
this.passwordEncoder = passwordEncoder;
}
@Override
public void seed() {
if (userRepository.findAll().size() > 0) {
log.warn("Already seeded. Skip.");
return;
}
USER_1.setPassword(passwordEncoder.encode(securityConfig.getSystemPassword()));
final User user1 = userRepository.save(USER_1);
log.info("Seeded user with username {}", user1.getUsername());
log.debug("seeded user {}", user1);
}
}
......@@ -3,7 +3,6 @@ package at.tuwien.service.impl;
import at.tuwien.api.auth.SignupRequestDto;
import at.tuwien.entities.user.RoleType;
import at.tuwien.entities.user.User;
import at.tuwien.exception.RoleNotFoundException;
import at.tuwien.exception.UserEmailExistsException;
import at.tuwien.exception.UserNameExistsException;
import at.tuwien.exception.UserNotFoundException;
......
#!/usr/bin/env python3
import pika
import json
import time
import datetime
from sys import argv, exit
def mock(channel, exchange, routing_key, size):
data = {'status': 'provisorisch', 'datum': '2022-06-12 00:00:00', 'parameter': None,
'intervall': 'h1', 'wert': 0, 'einheit': 'mg/m3', 'standort': 'Vienna'}
dump = json.dumps(data)
if len(dump) > size:
print('ERROR: data skeleton is too big:', len(dump))
exit(1)
fill = 'A' * (size - len(dump))
data['parameter'] = fill
channel.basic_publish(exchange=exchange,
routing_key=routing_key,
body=bytes(json.dumps(data), encoding='utf8'))
method, properties, body = channel.basic_get(queue=routing_key)
if method is not None:
return 1
else:
return 0
def send(exchange, routing_key):
creds = pika.credentials.PlainCredentials('fda', 'fda')
connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost', credentials=creds))
channel = connection.channel()
size = 200
rate = 200 # tuple/s
wait = 60 # seconds
print('submit tuple size', size / 1000, '(kB) data at rate', rate, '(tuple/s) ...')
now = datetime.datetime.now()
stop = now + datetime.timedelta(0, wait)
while datetime.datetime.now() < stop:
res = mock(channel, exchange, routing_key, size)
if res == 1:
print('WARN: queue is not empty, exit after', (datetime.datetime.now() - now).total_seconds(), 'seconds.')
exit(2)
time.sleep(1 / rate)
print('waited', wait, 'seconds. Goodbye!')
connection.close()
def usage():
print('USAGE: ./amqp exchange routing_key')
if __name__ == "__main__":
if len(argv) != 3:
usage()
exit(1)
send(exchange=argv[1], routing_key=argv[2])
timestamp,location,value
2022-05-24 12:00:00,Vienna,24.0
2022-05-24 13:00:00,Vienna,25.0
2022-05-24 14:00:00,Vienna,24.5
2022-05-24 15:00:00,Vienna,24.0
\ No newline at end of file
......@@ -107,6 +107,13 @@ public abstract class AbstractSeeder {
public final static String IMAGE_DATE_7_EXAMPLE = "2022-01-30 13:44:25";
public final static Boolean IMAGE_DATE_7_HAS_TIME = true;
public final static Long IMAGE_DATE_8_ID = 8L;
public final static Long IMAGE_DATE_8_IMAGE_ID = IMAGE_1_ID;
public final static String IMAGE_DATE_8_DATABASE_FORMAT = "%d.%c.%Y %H:%i:%S";
public final static String IMAGE_DATE_8_UNIX_FORMAT = "dd.MM.yyyy HH:mm:ss";
public final static String IMAGE_DATE_8_EXAMPLE = "30.01.2022 13:44:25";
public final static Boolean IMAGE_DATE_8_HAS_TIME = true;
public final static ContainerImageDate IMAGE_DATE_1 = ContainerImageDate.builder()
.id(IMAGE_DATE_1_ID)
.iid(IMAGE_DATE_1_IMAGE_ID)
......@@ -170,6 +177,15 @@ public abstract class AbstractSeeder {
.hasTime(IMAGE_DATE_7_HAS_TIME)
.build();
public final static ContainerImageDate IMAGE_DATE_8 = ContainerImageDate.builder()
.id(IMAGE_DATE_8_ID)
.iid(IMAGE_DATE_8_IMAGE_ID)
.databaseFormat(IMAGE_DATE_8_DATABASE_FORMAT)
.unixFormat(IMAGE_DATE_8_UNIX_FORMAT)
.example(IMAGE_DATE_8_EXAMPLE)
.hasTime(IMAGE_DATE_8_HAS_TIME)
.build();
public final static ContainerImage IMAGE_1 = ContainerImage.builder()
.dialect(IMAGE_1_DIALECT)
.driverClass(IMAGE_1_DRIVER)
......
......@@ -46,6 +46,8 @@ public class ImageSeederImpl extends AbstractSeeder implements Seeder {
log.info("Seeded image date id {}", date6.getId());
final ContainerImageDate date7 = imageDateRepository.save(IMAGE_DATE_7);
log.info("Seeded image date id {}", date7.getId());
final ContainerImageDate date8 = imageDateRepository.save(IMAGE_DATE_8);
log.info("Seeded image date id {}", date8.getId());
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment