Skip to content
Snippets Groups Projects
Verified Commit 267b4f98 authored by Martin Weise's avatar Martin Weise
Browse files

WIP 


Signed-off-by: Martin Weise's avatarMartin Weise <martin.weise@tuwien.ac.at>
parent 1fec58fd
No related branches found
No related tags found
2 merge requests!395Library,!394Grafana
This commit is part of merge request !394. Comments created here will be created in the context of that merge request.
Show whitespace changes
Inline Side-by-side
Showing
with 66 additions and 302 deletions
dbrepo-auth-service/listeners/target/create-event-listener.jar
+ 0
− 0
View file @ 267b4f98
No preview for this file type
dbrepo-dashboard-service/app.py
+ 1
− 1
View file @ 267b4f98
...@@ -216,7 +216,7 @@ def create_dashboard(): ...@@ -216,7 +216,7 @@ def create_dashboard():
logging.debug( logging.debug(
f"endpoint create dashboard, is_public={is_public}, is_schema_public={is_schema_public}, owner_username={owner_username}") f"endpoint create dashboard, is_public={is_public}, is_schema_public={is_schema_public}, owner_username={owner_username}")
try: try:
db = dashboard.create(request.json['database_name'], request.json['owner_username']) db = dashboard.create(request.json['database_name'])
access.update_anonymous_read_access(db['uid'], is_public, is_schema_public) access.update_anonymous_read_access(db['uid'], is_public, is_schema_public)
return Response(dumps(db)), 201, headers return Response(dumps(db)), 201, headers
except GrafanaClientError as e: except GrafanaClientError as e:
......
dbrepo-dashboard-service/dashboard.py
+ 29
− 277
View file @ 267b4f98
import logging import logging
import os import os
from dbrepo.api.dto import Database, View from dbrepo.api.dto import Database
from dbrepo.api.exceptions import MalformedError
from grafana_client.client import GrafanaClientError
from api.exceptions import DashboardNotFound
from clients.grafana_client import GrafanaClient from clients.grafana_client import GrafanaClient
from panel import get_panels
statistics_row_title = '${view_id}'
base_url = os.getenv('BASE_URL', 'http://localhost') base_url = os.getenv('BASE_URL', 'http://localhost')
datasource_uid = os.getenv('JSON_DATASOURCE_NAME', 'dbrepojson0') datasource_uid = os.getenv('JSON_DATASOURCE_NAME', 'dbrepojson0')
def map_link(title: str, url: str) -> dict: def map_link(title: str, url: str, icon: str = 'info') -> dict:
return dict(targetBlank=True, return dict(targetBlank=True,
asDropdown=False, asDropdown=False,
includeVars=False, includeVars=False,
keepTime=False, keepTime=False,
tags=[], tags=[],
type='link', type='link',
icon='info', icon=icon,
title=title, title=title,
url=url) url=url)
def map_statistics_row(dashboard: dict) -> dict | None:
filtered_panels = [panel for panel in dashboard['panels'] if
panel['type'] == 'row' and panel['title'] == statistics_row_title]
if len(filtered_panels) == 0:
logging.warning(f"Failed to find statistics row title {statistics_row_title} in: {filtered_panels}")
return None
return filtered_panels[0]
def map_links(database: Database) -> [dict]: def map_links(database: Database) -> [dict]:
links = [] links = []
if len(database.identifiers) > 0: if len(database.identifiers) > 0:
...@@ -41,275 +34,30 @@ def map_links(database: Database) -> [dict]: ...@@ -41,275 +34,30 @@ def map_links(database: Database) -> [dict]:
return links return links
def map_templating(database: Database) -> dict:
options = [dict(selected=False,
text=view.name,
value=str(view.id)) for view in database.views]
selected = dict(selected=True,
text=[view.name for view in database.views],
value=[str(view.id) for view in database.views])
datasource = dict(uid=datasource_uid,
type='yesoreyeram-infinity-datasource')
return dict(list=[dict(description='',
name='view_id',
hide=0,
includeAll=True,
multi=True,
datasource=datasource,
refresh=1,
regex='',
sort=0,
definition='dbrepo-json- (infinity) json',
query=dict(queryType='infinity',
query='',
infinityQuery=dict(format='table',
filters=[],
parser='backend',
refId='variable',
root_selector='',
source='url',
type='json',
url=f"/api/database/{database.id}/view",
columns=[dict(selector='id',
text='value',
type='string'),
dict(
selector='internal_name',
text='name',
type='string')],
url_options=dict(data='',
method='GET'))),
label='Datasource',
skipUrlSync=False,
type='query',
current=selected,
options=options)])
def map_timeseries_panel(database: Database, view: View) -> dict:
datasource = dict(uid=datasource_uid,
type='yesoreyeram-infinity-datasource')
return dict(
title=view['name'],
type='timeseries',
datasource=datasource,
targets=[dict(datasource=datasource,
format='table',
global_query_id='',
hide=False,
refId='A',
root_selector='',
source='url',
type='json',
url=f"/api/database/{database['id']}/view/{view['id']}",
url_options=dict(data='',
method='GET'))],
gridPos=dict(h=8,
w=12,
x=0,
y=0),
options=dict(legend=dict(displayMode='list',
placement='bottom',
showLegend=True),
tooltip=dict(mode='single',
sort='none')),
fieldConfig=dict(
defaults=dict(color=dict(mode='palette-classic'),
custom=dict(
axisBorderShow=False,
axisCenteredZero=False,
axisColorMode='text',
axisLabel='',
axisPlacement='auto',
barAlignment=0,
drawStyle='line',
fillOpacity=0,
gradientMode='none',
hideFrom=dict(legend=False,
tooltip=False,
viz=False),
insertNulls=False,
lineInterpolation='linear',
lineWidth=1,
pointSize=5,
scaleDistribution=dict(type='linear'),
showPoints='auto',
spanNulls=False,
stacking=dict(group='A',
mode='none'),
thresholdsStyle=dict(mode='absolute')))))
def map_statistics_panel(database_id: str, view: View) -> dict:
datasource = dict(uid=datasource_uid,
type='yesoreyeram-infinity-datasource')
return dict(
title=view.name,
type='table',
datasource=datasource,
targets=[dict(datasource=datasource,
columns=[],
filters=[],
format='table',
global_query_id='',
hide=False,
refId='A',
root_selector='',
source='url',
type='json',
url=f"/api/database/{database_id}/view/{view.id}/data",
url_options=dict(data='',
method='GET'))],
options=dict(cellHeight="sm",
showHeader=True,
footer=dict(countRows=False,
fields="",
reducer=["sum"],
show=False)),
gridPos=dict(h=8,
w=12,
x=12,
y=0),
transformations=dict(id="organize",
options=dict(excludeByName=dict(),
includeByName=dict(),
indexByName=dict(
HEADER_AVG=3,
HEADER_COL=0,
HEADER_STDDEV=4,
HEADER_MAX=2,
HEADER_MIN=1))),
fieldConfig=dict(defaults=dict(custom=dict(align="auto",
filterable="true",
cellOptions=dict(type="auto"),
inspect=False),
mappings=[],
thresholds=dict(mode="absolute",
steps=[dict(color="green",
value=None),
dict(color="red",
value=80)
])),
overrides=[dict(matcher=dict(id="byName",
options="HEADER_COL"),
properties=[dict(id="custom.align",
value="center")]),
dict(matcher=dict(id="byName",
options="HEADER_MIN"),
properties=[dict(id="custom.width",
value=115)]),
dict(matcher=dict(id="byName",
options="HEADER_MAX"),
properties=[dict(id="custom.width",
value=115)]),
dict(matcher=dict(id="byName",
options="HEADER_AVG"),
properties=[dict(id="custom.width",
value=115)]),
dict(matcher=dict(id="byName",
options="HEADER_STDDEV"),
properties=[dict(id="custom.width",
value=115)])
]))
def map_overview_panel(database_id: str) -> dict:
datasource = dict(uid=datasource_uid,
type='yesoreyeram-infinity-datasource')
return dict(title='Preview',
type='table',
fieldConfig=dict(
defaults=dict(
color=dict(mode='palette-classic'),
custom=dict(axisBorderShow=False,
axisCenteredZero=False,
axisColorMode='text',
axisLabel='',
axisPlacement='auto',
barAlignment=0,
drawStyle='line',
fillOpacity=0,
gradientMode='none',
hideFrom=dict(
legend=False,
tooltip=False,
viz=False),
insertNulls=False,
lineInterpolation='linear',
lineWidth=1,
pointSize=5,
scaleDistribution=dict(
type='linear'),
showPoints='auto',
spanNulls=False,
stacking=dict(group='A',
mode='none'),
thresholdsStyle=dict(
mode='off'))),
overrides=[]),
options=dict(legend=dict(displayMode='list',
placement='bottom',
showLegend=True,
calcs=[]),
tooltip=dict(mode='single',
sort='none')),
targets=[dict(format='json',
columns=[],
datasource=datasource,
filters=[],
global_query_id='',
refId='A',
root_selector='',
source='url',
type='json',
url='/api/database/' + database_id + '/view/${view_id}/data',
url_options=dict(data='',
method='GET'))],
datasource=datasource,
gridPos=dict(h=4,
w=12,
x=0,
y=0))
def map_row() -> dict:
datasource = dict(uid=datasource_uid,
type='yesoreyeram-infinity-datasource')
return dict(collapsed=False,
repeat='view_id',
repeatDirection='h',
title=statistics_row_title,
type='row',
panels=[],
targets=[dict(refId='A',
datasource=datasource)],
gridPos=dict(h=1,
w=24,
x=0,
y=0))
def map_panels(dashboard: dict, database: Database) -> [dict]:
if map_statistics_row(dashboard) is None:
dashboard['panels'].append(map_row())
dashboard['panels'].append(map_overview_panel(database.id))
for view in database.views:
dashboard['panels'].append(map_statistics_panel(database.id, view))
return dashboard['panels']
def find(uid: str): def find(uid: str):
"""
Finds a dashboard with the given uid.
@return The dashboard, if successful. Otherwise, `None`.
"""
if uid is None:
return None
grafana = GrafanaClient().connect() grafana = GrafanaClient().connect()
try:
return grafana.dashboard.get_dashboard(uid) return grafana.dashboard.get_dashboard(uid)
except GrafanaClientError:
logging.warning(f"Failed to find dashboard with uid: {uid}")
return None
def create(database_name: str, uid: str = '') -> dict: def create(database_name: str, uid: str = '') -> dict:
grafana = GrafanaClient().connect() grafana = GrafanaClient().connect()
dashboard = dict(uid=uid, dashboard = dict(uid=uid,
title=f'{database_name} Overview', title=f'{database_name} Overview',
tags=['dbrepo'], tags=['managed'],
timezone='browser', timezone='browser',
fiscalYearStartMonth=1, refresh='30m',
preload=False,
panels=[]) panels=[])
dashboard['panels'] = [] dashboard['panels'] = []
payload = dict(folderUid='', payload = dict(folderUid='',
...@@ -327,16 +75,20 @@ def delete(uid: str) -> None: ...@@ -327,16 +75,20 @@ def delete(uid: str) -> None:
def update(database: Database) -> None: def update(database: Database) -> None:
grafana = GrafanaClient().connect() grafana = GrafanaClient().connect()
dashboard = find(database.dashboard_uid)['dashboard'] dashboard = find(database.dashboard_uid)
if dashboard is None:
raise DashboardNotFound(f'Dashboard {database.dashboard_uid} not found')
dashboard = dashboard['dashboard']
# update metadata # update metadata
if not database.is_dashboard_enabled and 'managed' in dashboard['tags']:
dashboard['tags'].remove('managed')
if len(database.identifiers) > 0 and len(database.identifiers[0].titles) > 0: if len(database.identifiers) > 0 and len(database.identifiers[0].titles) > 0:
dashboard['title'] = database.identifiers[0].titles[0].title dashboard['title'] = database.identifiers[0].titles[0].title
if len(database.identifiers) > 0 and len(database.identifiers[0].descriptions) > 0: if len(database.identifiers) > 0 and len(database.identifiers[0].descriptions) > 0:
dashboard['description'] = database.identifiers[0].descriptions[0].description dashboard['description'] = database.identifiers[0].descriptions[0].description
dashboard['links'] = map_links(database) dashboard['links'] = map_links(database)
dashboard['templating'] = map_templating(database)
# update panels # update panels
dashboard['panels'] = map_panels(dashboard, database) dashboard['panels'] = get_panels(dashboard, database)
payload = dict(folderUid='', payload = dict(folderUid='',
overwrite=True, overwrite=True,
dashboard=dashboard) dashboard=dashboard)
......
dbrepo-dashboard-service/init/dashboard.py
+ 2
− 2
View file @ 267b4f98
...@@ -50,7 +50,7 @@ def find(uid: str): ...@@ -50,7 +50,7 @@ def find(uid: str):
return None return None
def create(database_name: str, uid: str = '') -> str: def create(database_name: str, uid: str = '') -> dict:
grafana = GrafanaClient().connect() grafana = GrafanaClient().connect()
dashboard = dict(uid=uid, dashboard = dict(uid=uid,
title=f'{database_name} Overview', title=f'{database_name} Overview',
...@@ -65,7 +65,7 @@ def create(database_name: str, uid: str = '') -> str: ...@@ -65,7 +65,7 @@ def create(database_name: str, uid: str = '') -> str:
dashboard=dashboard) dashboard=dashboard)
dashboard = grafana.dashboard.update_dashboard(payload) dashboard = grafana.dashboard.update_dashboard(payload)
logging.info(f"Created dashboard with uid: {dashboard['uid']}") logging.info(f"Created dashboard with uid: {dashboard['uid']}")
return dashboard['uid'] return dashboard
def delete(uid: str) -> None: def delete(uid: str) -> None:
......
dbrepo-dashboard-service/init/tests/test_integration_dashboard.py
+ 1
− 1
View file @ 267b4f98
...@@ -108,7 +108,7 @@ class DashboardIntegrationTest(unittest.TestCase): ...@@ -108,7 +108,7 @@ class DashboardIntegrationTest(unittest.TestCase):
def test_create_succeeds(self): def test_create_succeeds(self):
# test # test
dashboard.create('some_database', 'foobar') dashboard.create('some_database')
def test_create_with_uid_succeeds(self): def test_create_with_uid_succeeds(self):
# test # test
......
dbrepo-data-service/Dockerfile
+ 3
− 3
View file @ 267b4f98
###### FIRST STAGE ###### ###### FIRST STAGE ######
FROM dbrepo-metadata-service:build AS dependency FROM dbrepo-core:build AS dependency
LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at" LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at"
###### SECOND STAGE ###### ###### SECOND STAGE ######
...@@ -10,7 +10,7 @@ COPY ./pom.xml ./ ...@@ -10,7 +10,7 @@ COPY ./pom.xml ./
RUN mvn -fn dependency:go-offline RUN mvn -fn dependency:go-offline
COPY --from=dependency /root/.m2/repository/at/tuwien /root/.m2/repository/at/tuwien COPY --from=dependency /root/.m2/repository/at/ac/tuwien/ifs/dbrepo /root/.m2/repository/at/ac/tuwien/ifs/dbrepo
COPY ./querystore ./querystore COPY ./querystore ./querystore
COPY ./report ./report COPY ./report ./report
...@@ -28,7 +28,7 @@ RUN apk add --no-cache curl bash jq ...@@ -28,7 +28,7 @@ RUN apk add --no-cache curl bash jq
WORKDIR /app WORKDIR /app
RUN adduser -S -u 1001 data-service RUN adduser -S -u 1001 dbrepo
USER 1001 USER 1001
......
dbrepo-metadata-service/Dockerfile
+ 16
− 17
View file @ 267b4f98
###### FIRST STAGE ###### ###### FIRST STAGE ######
FROM dbrepo-core:build AS dependency
LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at"
###### SECOND STAGE ######
FROM maven:3-amazoncorretto-17 AS build FROM maven:3-amazoncorretto-17 AS build
LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at" LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at"
COPY ./pom.xml ./ COPY ./pom.xml ./
COPY ./api/pom.xml ./api/
COPY ./entities/pom.xml ./entities/ RUN mvn -fn dependency:go-offline
COPY ./oai/pom.xml ./oai/
COPY ./report/pom.xml ./report/ COPY --from=dependency /root/.m2/repository/at/ac/tuwien/ifs/dbrepo /root/.m2/repository/at/ac/tuwien/ifs/dbrepo
COPY ./repositories/pom.xml ./repositories/
COPY ./rest-service/pom.xml ./rest-service/
COPY ./services/pom.xml ./services/
COPY ./test/pom.xml ./test/
RUN mvn dependency:go-offline
COPY ./api ./api
COPY ./entities ./entities
COPY ./oai ./oai COPY ./oai ./oai
COPY ./report ./report COPY ./report ./report
COPY ./repositories ./repositories COPY ./repositories ./repositories
COPY ./rest-service ./rest-service COPY ./rest-service ./rest-service
COPY ./services ./services COPY ./services ./services
COPY ./test ./test
# Make sure it compiles # Make sure it compiles
RUN mvn clean install -DskipTests RUN mvn -fn clean package -DskipTests
###### SECOND STAGE ###### ###### THIRD STAGE ######
FROM amazoncorretto:17-alpine3.19 AS runtime FROM amazoncorretto:17-alpine3.19 AS runtime
LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at" LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at"
...@@ -34,6 +29,8 @@ RUN apk add --no-cache curl bash jq ...@@ -34,6 +29,8 @@ RUN apk add --no-cache curl bash jq
WORKDIR /app WORKDIR /app
RUN adduser -S -u 1001 dbrepo
USER 1001 USER 1001
COPY --from=build --chown=1001 ./rest-service/target/dbrepo-metadata-service-rest-service-*.jar ./metadata-service.jar COPY --from=build --chown=1001 ./rest-service/target/dbrepo-metadata-service-rest-service-*.jar ./metadata-service.jar
...@@ -41,4 +38,6 @@ COPY --from=build --chown=1001 ./rest-service/target/dbrepo-metadata-service-res ...@@ -41,4 +38,6 @@ COPY --from=build --chown=1001 ./rest-service/target/dbrepo-metadata-service-res
# non-root port # non-root port
EXPOSE 8080 EXPOSE 8080
ENTRYPOINT ["java", "-Dlog4j2.formatMsgNoLookups=true", "-jar", "./metadata-service.jar"] ENV JAVA_OPTS="-Dlog4j2.formatMsgNoLookups=true"
ENTRYPOINT exec java $JAVA_OPTS -jar ./metadata-service.jar
\ No newline at end of file
helm/dbrepo/files/create-event-listener.jar
+ 0
− 0
View file @ 267b4f98
No preview for this file type
lib/java/dbrepo-core/Dockerfile 0 → 100644
+ 12
− 0
View file @ 267b4f98
###### FIRST STAGE ######
FROM maven:3-amazoncorretto-17 AS build
LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at"
COPY ./pom.xml ./
RUN mvn dependency:go-offline
COPY ./src/ ./src/
# Make sure it compiles
RUN mvn clean install -DskipTests
\ No newline at end of file
make/build.mk
+ 2
− 1
View file @ 267b4f98
...@@ -2,8 +2,9 @@ ...@@ -2,8 +2,9 @@
.PHONY: build-images .PHONY: build-images
build-images: ## Build Docker images. build-images: ## Build Docker images.
docker build --network=host -t dbrepo-metadata-service:build --target build dbrepo-metadata-service docker build --network=host -t dbrepo-core:build --target build ./lib/java/dbrepo-core
docker build --network=host -t dbrepo-data-service:build --target build dbrepo-data-service docker build --network=host -t dbrepo-data-service:build --target build dbrepo-data-service
docker build --network=host -t dbrepo-metadata-service:build --target build dbrepo-metadata-service
docker compose build --parallel docker compose build --parallel
.PHONY: build-data-service .PHONY: build-data-service
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment