Skip to content
Snippets Groups Projects
Unverified Commit 35d0696a authored by Martin Weise's avatar Martin Weise
Browse files

Added Broker API again for Python, fixed some bugs and moved the download .csv...

Added Broker API again for Python, fixed some bugs and moved the download .csv button down to time-versioned area
parent a4c3e17c
Branches
Tags
No related merge requests found
Showing
with 284 additions and 213 deletions
...@@ -19,4 +19,4 @@ SMTP_PORT= ...@@ -19,4 +19,4 @@ SMTP_PORT=
SMTP_USERNAME= SMTP_USERNAME=
SMTP_PASSWORD= SMTP_PASSWORD=
EUREKA_SERVER=http://discovery-service:9090/eureka/ EUREKA_SERVER=http://discovery-service:9090/eureka/
SHARED_FILESYSTEM=%localappdata%\Temp SHARED_FILESYSTEM=C:\tmp
...@@ -8,7 +8,6 @@ target/ ...@@ -8,7 +8,6 @@ target/
.jupyter/.idea/ .jupyter/.idea/
.jupyter/.ipynb_checkpoints .jupyter/.ipynb_checkpoints
.jupyter/api_authentication .jupyter/api_authentication
.jupyter/api_broker
.jupyter/api_container .jupyter/api_container
.jupyter/api_database .jupyter/api_database
.jupyter/api_identifier .jupyter/api_identifier
......
# Python
__pycache__ __pycache__
\ No newline at end of file
venv/
publish.py
./swagger-codegen-cli.jar
\ No newline at end of file
import json
from pika import BlockingConnection, ConnectionParameters
from pika.credentials import PlainCredentials
class BrokerServiceClient:
def __init__(self, exchange, routing_key, host, username, password):
self.exchange = exchange
self.routing_key = routing_key
self.host = host
self.username = username
self.password = password
self.connection = BlockingConnection(ConnectionParameters(self.host,
credentials=PlainCredentials(
username=self.username,
password=self.password)))
def send(self, message):
channel = self.connection.channel()
payload = json.dumps(message)
print("... sending tuple %s" % payload)
channel.basic_publish(exchange=self.exchange, routing_key=self.routing_key,
body=bytes(payload, encoding='utf8'))
print("... sent tuple")
channel.close()
.jupyter/dev.py 100644 → 100755
#!/bin/env python3 #!/bin/env python3
import time
from api_broker.BrokerServiceClient import BrokerServiceClient from api_broker.BrokerServiceClient import BrokerServiceClient
broker = BrokerServiceClient(exchange="airquality2", routing_key="airquality2", host="localhost", username="user", broker = BrokerServiceClient(exchange="airquality_6f28efee-2377-11ed-9491-8c8caada74c3", routing_key="airquality_6f510ee8-2377-11ed-9491-8c8caada74c3", host="localhost", username="test1",
password="user") password="test1")
payload = {'date': '2022-07-21', 'location': 'Kuala Lumpur', 'parameter': 'T', 'interval': 'h1', 'unit': 'deg-celsius', payload = {"date": "2021-01-01", "location": "Stampfenbachstrasse", "parameter": "CO", "interval": "h1", "unit": "mg/m3", "value": 0.44, "status": "tentative"}
'value': 33.0, 'status': 'tentative'}
response = broker.send(payload)
print(payload)
# faulty date
payload = {'date': '2022-07', 'location': 'Kuala Lumpur', 'parameter': 'T', 'interval': 'h1', 'unit': 'deg-celsius',
'value': 33.0, 'status': 'tentative'}
response = broker.send(payload)
print(payload)
# faulty number
payload = {'date': '2022-07-01', 'location': 'Kuala Lumpur', 'parameter': 'T', 'interval': 'h1', 'unit': 'deg-celsius',
'value': 'hello', 'status': 'tentative'}
response = broker.send(payload)
print(payload)
# faulty string
payload = {'date': '2022-07-01', 'location': 'Kuala Lumpur', 'parameter': 'T', 'interval': 'h1', 'unit': 'deg-celsius',
'value': 33.0, 'status': 88}
response = broker.send(payload)
print(payload)
while True:
response = broker.send(payload) response = broker.send(payload)
print(payload)
time.sleep(1)
...@@ -8,6 +8,7 @@ import requests as rq ...@@ -8,6 +8,7 @@ import requests as rq
from postgres import Postgres from postgres import Postgres
import api_query.rest import api_query.rest
from api_broker.BrokerServiceClient import BrokerServiceClient
from api_authentication.api.authentication_endpoint_api import AuthenticationEndpointApi from api_authentication.api.authentication_endpoint_api import AuthenticationEndpointApi
from api_authentication.api.user_endpoint_api import UserEndpointApi from api_authentication.api.user_endpoint_api import UserEndpointApi
from api_container.api.container_endpoint_api import ContainerEndpointApi from api_container.api.container_endpoint_api import ContainerEndpointApi
...@@ -99,6 +100,12 @@ def create_database(container_id, is_public=True): ...@@ -99,6 +100,12 @@ def create_database(container_id, is_public=True):
return response return response
def find_database(container_id, database_id):
response = database.find_by_id(container_id, database_id)
print("found database with id %d" % response.id)
return response
def update_database(container_id, database_id, is_public=True): def update_database(container_id, database_id, is_public=True):
response = database.update({ response = database.update({
"description": "This dataset includes daily values from 1983 to the current day, divided into annual files. This includes the maximum hourly average and the number of times the hourly average limit value for ozone was exceeded and the daily averages for sulfur dioxide (SO2), carbon monoxide (CO), nitrogen oxide (NOx), nitrogen monoxide (NO), nitrogen dioxide (NO2), particulate matter (PM10 and PM2.5). ) and particle number (PN), provided that they are of sufficient quality. The values of the completed day for the current year are updated every 30 minutes after midnight (UTC+1).", "description": "This dataset includes daily values from 1983 to the current day, divided into annual files. This includes the maximum hourly average and the number of times the hourly average limit value for ozone was exceeded and the daily averages for sulfur dioxide (SO2), carbon monoxide (CO), nitrogen oxide (NOx), nitrogen monoxide (NO), nitrogen dioxide (NO2), particulate matter (PM10 and PM2.5). ) and particle number (PN), provided that they are of sufficient quality. The values of the completed day for the current year are updated every 30 minutes after midnight (UTC+1).",
...@@ -295,6 +302,14 @@ def download_identifier_metadata(container_id, database_id, identifier_id): ...@@ -295,6 +302,14 @@ def download_identifier_metadata(container_id, database_id, identifier_id):
return response return response
def send_tuple(exchange, routing_key, username, password, payload):
broker = BrokerServiceClient(exchange=exchange, routing_key=routing_key, host="localhost", username=username,
password=password)
response = broker.send(payload)
print("sent tuple to exchange with routing key %s" % routing_key)
return response
if __name__ == '__main__': if __name__ == '__main__':
# #
# create 1 user and 3 containers (public, private, public) # create 1 user and 3 containers (public, private, public)
...@@ -349,6 +364,7 @@ if __name__ == '__main__': ...@@ -349,6 +364,7 @@ if __name__ == '__main__':
cid = create_container().id cid = create_container().id
start_container(cid) start_container(cid)
dbid = create_database(cid).id dbid = create_database(cid).id
dbexchange = find_database(cid, dbid).exchange
update_database(cid, dbid) update_database(cid, dbid)
create_table(cid, dbid, columns=[]) create_table(cid, dbid, columns=[])
create_table(cid, dbid, columns=[{ create_table(cid, dbid, columns=[{
...@@ -365,6 +381,10 @@ if __name__ == '__main__': ...@@ -365,6 +381,10 @@ if __name__ == '__main__':
"primary_key": True, "primary_key": True,
"null_allowed": False, "null_allowed": False,
}]).id }]).id
ttopic = find_table(cid, dbid, tid).topic
send_tuple(dbexchange, ttopic, "test1", "test1", {"primary": 1})
send_tuple(dbexchange, ttopic, "test1", "test1", {"primary": 2})
send_tuple(dbexchange, ttopic, "test1", "test1", {"primary": 3})
create_table(cid, dbid, columns=[{ create_table(cid, dbid, columns=[{
"name": "primary", "name": "primary",
"type": "date", "type": "date",
...@@ -400,3 +420,4 @@ if __name__ == '__main__': ...@@ -400,3 +420,4 @@ if __name__ == '__main__':
auth_user("test3") auth_user("test3")
update_user(uid) update_user(uid)
update_theme(uid) update_theme(uid)
print("FINISHED")
requests>=2.28.0 requests>=2.28.0
pandas>=1.4.3 pandas>=1.4.3
postgres>=4.0 postgres>=4.0
pika>=1.3.0
\ No newline at end of file
Date,Location,Parameter,Interval,Unit,Value,Status
2021-01-01,Stampfenbachstrasse,CO,h1,mg/m3,0.44,tentative 2021-01-01,Stampfenbachstrasse,CO,h1,mg/m3,0.44,tentative
2021-01-01,Stampfenbachstrasse,SO2,h1,µg/m3,4.88,tentative 2021-01-01,Stampfenbachstrasse,SO2,h1,µg/m3,4.88,tentative
2021-01-01,Stampfenbachstrasse,NOx,h1,ppb,29.46,tentative 2021-01-01,Stampfenbachstrasse,NOx,h1,ppb,29.46,tentative
This diff is collapsed.
...@@ -117,6 +117,7 @@ services: ...@@ -117,6 +117,7 @@ services:
METADATA_PASSWORD: ${METADATA_PASSWORD} METADATA_PASSWORD: ${METADATA_PASSWORD}
BROKER_USERNAME: ${BROKER_USERNAME} BROKER_USERNAME: ${BROKER_USERNAME}
BROKER_PASSWORD: ${BROKER_PASSWORD} BROKER_PASSWORD: ${BROKER_PASSWORD}
SHARED_FILESYSTEM: ${SHARED_FILESYSTEM}
volumes: volumes:
- /var/run/docker.sock:/var/run/docker.sock - /var/run/docker.sock:/var/run/docker.sock
depends_on: depends_on:
...@@ -176,7 +177,7 @@ services: ...@@ -176,7 +177,7 @@ services:
BROKER_USERNAME: ${BROKER_USERNAME} BROKER_USERNAME: ${BROKER_USERNAME}
BROKER_PASSWORD: ${BROKER_PASSWORD} BROKER_PASSWORD: ${BROKER_PASSWORD}
volumes: volumes:
- /tmp:/tmp - ${SHARED_FILESYSTEM}:/tmp
depends_on: depends_on:
table-service: table-service:
condition: service_healthy condition: service_healthy
...@@ -266,7 +267,7 @@ services: ...@@ -266,7 +267,7 @@ services:
ports: ports:
- "5010:5010" - "5010:5010"
volumes: volumes:
- /tmp:/tmp - ${SHARED_FILESYSTEM}:/tmp
- /var/run/docker.sock:/var/run/docker.sock - /var/run/docker.sock:/var/run/docker.sock
depends_on: depends_on:
discovery-service: discovery-service:
...@@ -343,5 +344,6 @@ services: ...@@ -343,5 +344,6 @@ services:
environment: environment:
BROKER_USERNAME: ${BROKER_USERNAME} BROKER_USERNAME: ${BROKER_USERNAME}
BROKER_PASSWORD: ${BROKER_PASSWORD} BROKER_PASSWORD: ${BROKER_PASSWORD}
SHARED_FILESYSTEM: ${SHARED_FILESYSTEM}
logging: logging:
driver: json-file driver: json-file
...@@ -79,8 +79,6 @@ services: ...@@ -79,8 +79,6 @@ services:
userdb: userdb:
public: public:
environment: environment:
GATEWAY_ENDPOINT: http://fda-gateway-service:9095
SEARCH_ENDPOINT: fda-search-service:9200
SPRING_PROFILES_ACTIVE: docker SPRING_PROFILES_ACTIVE: docker
ports: ports:
- "9092:9092" - "9092:9092"
...@@ -105,7 +103,6 @@ services: ...@@ -105,7 +103,6 @@ services:
networks: networks:
public: public:
environment: environment:
GATEWAY_ENDPOINT: http://fda-gateway-service:9095
SPRING_PROFILES_ACTIVE: docker SPRING_PROFILES_ACTIVE: docker
ports: ports:
- "9091:9091" - "9091:9091"
...@@ -126,13 +123,6 @@ services: ...@@ -126,13 +123,6 @@ services:
networks: networks:
public: public:
environment: environment:
GATEWAY_ENDPOINT: http://fda-gateway-service:9095
WEBSITE: http://localhost:3000
MAIL_FROM: Database Repository <noreply@dbrepo.ossdip.at>
MAIL_REPLY_TO: Martin Weise <martin.weise@tuwien.ac.at>
JWT_ISSUER: fda-dbrepo
JWT_SECRET: fda-secret
JWT_EXPIRATION: 86400000
SPRING_PROFILES_ACTIVE: docker SPRING_PROFILES_ACTIVE: docker
ports: ports:
- "9097:9097" - "9097:9097"
...@@ -157,7 +147,6 @@ services: ...@@ -157,7 +147,6 @@ services:
userdb: userdb:
environment: environment:
SPRING_PROFILES_ACTIVE: docker SPRING_PROFILES_ACTIVE: docker
GATEWAY_ENDPOINT: http://fda-gateway-service:9095
ports: ports:
- "9093:9093" - "9093:9093"
volumes: volumes:
...@@ -208,7 +197,6 @@ services: ...@@ -208,7 +197,6 @@ services:
networks: networks:
public: public:
environment: environment:
GATEWAY_ENDPOINT: http://fda-gateway-service:9095
SPRING_PROFILES_ACTIVE: docker SPRING_PROFILES_ACTIVE: docker
ports: ports:
- "9096:9096" - "9096:9096"
...@@ -231,7 +219,6 @@ services: ...@@ -231,7 +219,6 @@ services:
userdb: userdb:
command: sh -c "/wait && flask run" # docker-compose should not test the implementation command: sh -c "/wait && flask run" # docker-compose should not test the implementation
environment: environment:
EUREKA_SERVER: http://fda-discovery-service:9090/eureka/
hostname: analyse-service hostname: analyse-service
ports: ports:
- "5000:5000" - "5000:5000"
...@@ -273,7 +260,6 @@ services: ...@@ -273,7 +260,6 @@ services:
image: fda-broker-service image: fda-broker-service
environment: environment:
SPRING_PROFILES_ACTIVE: docker SPRING_PROFILES_ACTIVE: docker
GATEWAY_ENDPOINT: http://fda-gateway-service:9095
networks: networks:
public: public:
ports: ports:
......
...@@ -32,7 +32,6 @@ public class AuthTokenFilter extends OncePerRequestFilter { ...@@ -32,7 +32,6 @@ public class AuthTokenFilter extends OncePerRequestFilter {
final String jwt = parseJwt(request); final String jwt = parseJwt(request);
if (jwt != null && jwtUtils.validateJwtToken(jwt)) { if (jwt != null && jwtUtils.validateJwtToken(jwt)) {
final String username = jwtUtils.getUserNameFromJwtToken(jwt); final String username = jwtUtils.getUserNameFromJwtToken(jwt);
final UserDetails userDetails = userDetailsService.loadUserByUsername(username); final UserDetails userDetails = userDetailsService.loadUserByUsername(username);
final UsernamePasswordAuthenticationToken authentication = new UsernamePasswordAuthenticationToken( final UsernamePasswordAuthenticationToken authentication = new UsernamePasswordAuthenticationToken(
userDetails, null, userDetails.getAuthorities()); userDetails, null, userDetails.getAuthorities());
......
...@@ -35,7 +35,7 @@ public class UserDetailsServiceImpl implements UserDetailsService { ...@@ -35,7 +35,7 @@ public class UserDetailsServiceImpl implements UserDetailsService {
try { try {
user = userService.findByUsername(username); user = userService.findByUsername(username);
} catch (UserNotFoundException e) { } catch (UserNotFoundException e) {
log.error("Failed to find user"); log.error("Failed to find user with username {}", username);
throw new UsernameNotFoundException("Failed to find user", e); throw new UsernameNotFoundException("Failed to find user", e);
} }
log.trace("loaded user {}", user); log.trace("loaded user {}", user);
......
...@@ -26,6 +26,7 @@ ENV METADATA_USERNAME=postgres ...@@ -26,6 +26,7 @@ ENV METADATA_USERNAME=postgres
ENV METADATA_PASSWORD=postgres ENV METADATA_PASSWORD=postgres
ENV BROKER_USERNAME=fda ENV BROKER_USERNAME=fda
ENV BROKER_PASSWORD=fda ENV BROKER_PASSWORD=fda
ENV SHARED_FILESYSTEM=/tmp
COPY ./service_ready /usr/bin COPY ./service_ready /usr/bin
RUN chmod +x /usr/bin/service_ready RUN chmod +x /usr/bin/service_ready
......
...@@ -27,7 +27,7 @@ ...@@ -27,7 +27,7 @@
<mapstruct.version>1.4.2.Final</mapstruct.version> <mapstruct.version>1.4.2.Final</mapstruct.version>
<docker.version>3.2.7</docker.version> <docker.version>3.2.7</docker.version>
<testcontainers.version>1.15.2</testcontainers.version> <testcontainers.version>1.15.2</testcontainers.version>
<swagger.version>2.1.7</swagger.version> <swagger.version>2.2.2</swagger.version>
<jacoco.version>0.8.7</jacoco.version> <jacoco.version>0.8.7</jacoco.version>
<sqlserver.version>9.2.1.jre11</sqlserver.version> <sqlserver.version>9.2.1.jre11</sqlserver.version>
</properties> </properties>
...@@ -59,6 +59,12 @@ ...@@ -59,6 +59,12 @@
<artifactId>spring-security-test</artifactId> <artifactId>spring-security-test</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<!-- Swagger -->
<dependency>
<groupId>io.swagger.core.v3</groupId>
<artifactId>swagger-core</artifactId>
<version>2.2.2</version>
</dependency>
<!-- Data Source --> <!-- Data Source -->
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
......
...@@ -34,4 +34,4 @@ eureka: ...@@ -34,4 +34,4 @@ eureka:
fda: fda:
mount.path: /tmp mount.path: /tmp
ready.path: /ready ready.path: /ready
gateway.endpoint: "${GATEWAY_ENDPOINT}" gateway.endpoint: http://gateway-service:9095
\ No newline at end of file \ No newline at end of file
app.version: '@project.version@'
spring:
main.banner-mode: off
datasource:
url: jdbc:postgresql://localhost:5432/fda
driver-class-name: org.postgresql.Driver
username: postgres
password: postgres
jpa:
show-sql: false
database-platform: org.hibernate.dialect.PostgreSQLDialect
hibernate:
ddl-auto: validate
open-in-view: false
properties:
hibernate:
jdbc:
time_zone: UTC
application:
name: container-service
cloud:
loadbalancer.ribbon.enabled: false
springdoc.swagger-ui.enabled: true
server.port: 9091
logging:
pattern.console: "%d %highlight(%-5level) %msg%n"
level:
root: warn
at.tuwien.: debug
org.springframework.web.servlet.mvc.method.annotation.ExceptionHandlerExceptionResolver: debug
eureka:
instance.hostname: container-service
client.serviceUrl.defaultZone: http://localhost:9090/eureka/
fda:
mount.path: /tmp
ready.path: ./ready
gateway.endpoint: http://localhost:9095
\ No newline at end of file
...@@ -32,6 +32,6 @@ eureka: ...@@ -32,6 +32,6 @@ eureka:
instance.hostname: container-service instance.hostname: container-service
client.serviceUrl.defaultZone: http://discovery-service:9090/eureka/ client.serviceUrl.defaultZone: http://discovery-service:9090/eureka/
fda: fda:
mount.path: /tmp mount.path: "${SHARED_FILESYSTEM}"
ready.path: /ready ready.path: /ready
gateway.endpoint: http://gateway-service:9095 gateway.endpoint: http://gateway-service:9095
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment