From 93a4976af9b59a8fa2e02e413ba10e89b2753860 Mon Sep 17 00:00:00 2001
From: Martin Weise <martin.weise@tuwien.ac.at>
Date: Thu, 26 Dec 2024 06:56:45 +0000
Subject: [PATCH] Master
---
.../dashboards/system.json | 287 +++++++++---------
.../at/tuwien/endpoints/SubsetEndpoint.java | 13 +-
.../at/tuwien/endpoints/TableEndpoint.java | 20 +-
.../at/tuwien/endpoints/ViewEndpoint.java | 26 +-
.../src/main/resources/application.yml | 1 +
.../java/at/tuwien/config/CacheConfig.java | 51 +++-
.../java/at/tuwien/config/MetricsConfig.java | 24 +-
.../gateway/impl/KeycloakGatewayImpl.java | 17 +-
.../java/at/tuwien/mapper/MetadataMapper.java | 18 ++
.../at/tuwien/service/MetricsService.java | 10 +
.../impl/MetricsServicePrometheusImpl.java | 47 +++
.../impl/QueueServiceRabbitMqImpl.java | 7 +-
.../java/at/tuwien/mapper/MetadataMapper.java | 5 +-
.../at/tuwien/endpoints/TableEndpoint.java | 3 +-
.../tuwien/service/impl/TableServiceImpl.java | 2 +-
dbrepo-ui/components/subset/SubsetList.vue | 46 ++-
dbrepo-ui/components/view/ViewList.vue | 3 -
dbrepo-ui/components/view/ViewToolbar.vue | 2 +-
.../[database_id]/view/[view_id]/data.vue | 2 +-
19 files changed, 354 insertions(+), 230 deletions(-)
create mode 100644 dbrepo-data-service/services/src/main/java/at/tuwien/service/MetricsService.java
create mode 100644 dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/MetricsServicePrometheusImpl.java
diff --git a/dbrepo-dashboard-service/dashboards/system.json b/dbrepo-dashboard-service/dashboards/system.json
index edee464f62..e6f81bda40 100644
--- a/dbrepo-dashboard-service/dashboards/system.json
+++ b/dbrepo-dashboard-service/dashboards/system.json
@@ -18,6 +18,7 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
+ "id": 3,
"links": [
{
"asDropdown": false,
@@ -109,7 +110,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.3",
+ "pluginVersion": "10.4.9",
"targets": [
{
"datasource": {
@@ -179,7 +180,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.3",
+ "pluginVersion": "10.4.9",
"targets": [
{
"datasource": {
@@ -247,7 +248,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.3",
+ "pluginVersion": "10.4.9",
"targets": [
{
"datasource": {
@@ -315,7 +316,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.3",
+ "pluginVersion": "10.4.9",
"targets": [
{
"datasource": {
@@ -431,7 +432,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.3",
+ "pluginVersion": "10.4.9",
"targets": [
{
"datasource": {
@@ -453,13 +454,83 @@
"title": "Data Volume",
"type": "stat"
},
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "P18F45E9DC7E75912"
+ },
+ "description": "Top 10 by number of accesses",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ }
+ },
+ "mappings": []
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 4
+ },
+ "id": 38,
+ "options": {
+ "displayLabels": [
+ "percent"
+ ],
+ "legend": {
+ "calcs": [],
+ "displayMode": "hidden",
+ "placement": "right",
+ "showLegend": false,
+ "values": []
+ },
+ "pieType": "pie",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "P18F45E9DC7E75912"
+ },
+ "editorMode": "code",
+ "expr": "topk(10, dbrepo_datasource_data_get_total)",
+ "instant": false,
+ "legendFormat": "{{uri}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Popular Data Sources",
+ "type": "piechart"
+ },
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 4
+ "y": 11
},
"id": 22,
"panels": [],
@@ -506,7 +577,7 @@
"h": 3,
"w": 4,
"x": 0,
- "y": 5
+ "y": 12
},
"id": 17,
"options": {
@@ -526,7 +597,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.3",
+ "pluginVersion": "10.4.9",
"targets": [
{
"datasource": {
@@ -588,7 +659,7 @@
"h": 3,
"w": 4,
"x": 4,
- "y": 5
+ "y": 12
},
"id": 24,
"options": {
@@ -608,7 +679,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.3",
+ "pluginVersion": "10.4.9",
"targets": [
{
"datasource": {
@@ -658,7 +729,7 @@
"h": 3,
"w": 4,
"x": 8,
- "y": 5
+ "y": 12
},
"id": 25,
"options": {
@@ -678,7 +749,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.3",
+ "pluginVersion": "10.4.9",
"targets": [
{
"datasource": {
@@ -728,7 +799,7 @@
"h": 3,
"w": 4,
"x": 12,
- "y": 5
+ "y": 12
},
"id": 26,
"options": {
@@ -748,7 +819,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.3",
+ "pluginVersion": "10.4.9",
"targets": [
{
"datasource": {
@@ -800,7 +871,7 @@
"h": 3,
"w": 4,
"x": 16,
- "y": 5
+ "y": 12
},
"id": 27,
"options": {
@@ -819,7 +890,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.3",
+ "pluginVersion": "10.4.9",
"targets": [
{
"datasource": {
@@ -881,7 +952,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 8
+ "y": 15
},
"id": 20,
"options": {
@@ -902,7 +973,7 @@
"sizing": "auto",
"valueMode": "color"
},
- "pluginVersion": "10.4.3",
+ "pluginVersion": "10.4.9",
"targets": [
{
"datasource": {
@@ -988,7 +1059,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 8
+ "y": 15
},
"id": 21,
"options": {
@@ -1030,7 +1101,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 15
+ "y": 22
},
"id": 31,
"panels": [],
@@ -1052,8 +1123,7 @@
"mode": "absolute",
"steps": [
{
- "color": "purple",
- "value": null
+ "color": "purple"
},
{
"color": "red",
@@ -1081,7 +1151,7 @@
"h": 3,
"w": 4,
"x": 0,
- "y": 16
+ "y": 23
},
"id": 32,
"options": {
@@ -1101,7 +1171,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.3",
+ "pluginVersion": "10.4.9",
"targets": [
{
"datasource": {
@@ -1138,8 +1208,7 @@
"mode": "absolute",
"steps": [
{
- "color": "blue",
- "value": null
+ "color": "blue"
}
]
},
@@ -1151,7 +1220,7 @@
"h": 3,
"w": 4,
"x": 4,
- "y": 16
+ "y": 23
},
"id": 29,
"options": {
@@ -1171,7 +1240,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.3",
+ "pluginVersion": "10.4.9",
"targets": [
{
"datasource": {
@@ -1208,8 +1277,7 @@
"mode": "absolute",
"steps": [
{
- "color": "blue",
- "value": null
+ "color": "blue"
}
]
},
@@ -1221,7 +1289,7 @@
"h": 3,
"w": 4,
"x": 8,
- "y": 16
+ "y": 23
},
"id": 30,
"options": {
@@ -1241,7 +1309,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.3",
+ "pluginVersion": "10.4.9",
"targets": [
{
"datasource": {
@@ -1278,8 +1346,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "#EAB839",
@@ -1303,7 +1370,7 @@
"h": 3,
"w": 4,
"x": 12,
- "y": 16
+ "y": 23
},
"id": 35,
"options": {
@@ -1323,7 +1390,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.3",
+ "pluginVersion": "10.4.9",
"targets": [
{
"datasource": {
@@ -1360,8 +1427,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "#EAB839",
@@ -1385,7 +1451,7 @@
"h": 3,
"w": 4,
"x": 16,
- "y": 16
+ "y": 23
},
"id": 36,
"options": {
@@ -1405,7 +1471,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.3",
+ "pluginVersion": "10.4.9",
"targets": [
{
"datasource": {
@@ -1442,8 +1508,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "#EAB839",
@@ -1467,7 +1532,7 @@
"h": 3,
"w": 4,
"x": 20,
- "y": 16
+ "y": 23
},
"id": 37,
"options": {
@@ -1487,7 +1552,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "10.4.3",
+ "pluginVersion": "10.4.9",
"targets": [
{
"datasource": {
@@ -1557,8 +1622,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -1601,7 +1665,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 19
+ "y": 26
},
"id": 33,
"options": {
@@ -1644,7 +1708,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 26
+ "y": 33
},
"id": 2,
"panels": [],
@@ -1699,8 +1763,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -1715,7 +1778,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 27
+ "y": 34
},
"id": 23,
"options": {
@@ -1790,8 +1853,7 @@
"mode": "absolute",
"steps": [
{
- "color": "red",
- "value": null
+ "color": "red"
},
{
"color": "green",
@@ -1806,7 +1868,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 27
+ "y": 34
},
"id": 16,
"options": {
@@ -1891,8 +1953,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -1965,7 +2026,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 34
+ "y": 41
},
"id": 6,
"options": {
@@ -2052,8 +2113,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -2126,7 +2186,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 34
+ "y": 41
},
"id": 7,
"options": {
@@ -2169,104 +2229,52 @@
"type": "prometheus",
"uid": "P18F45E9DC7E75912"
},
+ "description": "Top 10 by frequency of access",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 25,
- "gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "smooth",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
}
},
"mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
"unit": "reqps"
},
- "overrides": [
- {
- "matcher": {
- "id": "byRegexp",
- "options": "/.*search-service.*/"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "orange",
- "mode": "fixed"
- }
- }
- ]
- },
- {
- "matcher": {
- "id": "byRegexp",
- "options": "/.*analyse-service.*/"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "super-light-orange",
- "mode": "fixed"
- }
- }
- ]
- }
- ]
+ "overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
- "y": 41
+ "y": 48
},
"id": 18,
"options": {
+ "displayLabels": [
+ "percent"
+ ],
"legend": {
"calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
+ "displayMode": "hidden",
+ "placement": "right",
+ "showLegend": false,
+ "values": []
+ },
+ "pieType": "pie",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
},
"tooltip": {
- "mode": "multi",
+ "mode": "single",
"sort": "none"
}
},
@@ -2277,15 +2285,15 @@
"uid": "P18F45E9DC7E75912"
},
"editorMode": "code",
- "expr": "rate(flask_http_request_duration_seconds_count{status=~\"200|201|202\",path!=\"/health\"}[$__rate_interval])",
+ "expr": "topk(10, rate(dbrepo_table_data_get_total[$__range]))",
"instant": false,
- "legendFormat": "{{method}} {{instance}} {{path}} ({{status}})",
+ "legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
- "title": "Successful API Requests",
- "type": "timeseries"
+ "title": "Popular Datasources",
+ "type": "piechart"
},
{
"datasource": {
@@ -2334,8 +2342,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -2378,7 +2385,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 41
+ "y": 48
},
"id": 19,
"options": {
@@ -2421,13 +2428,13 @@
"list": []
},
"time": {
- "from": "now-30m",
+ "from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "DBRepo - Overview",
"uid": "bdz20owu8zn5se",
- "version": 1,
+ "version": 8,
"weekStart": ""
}
\ No newline at end of file
diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java
index 13cfc5c560..4f1b5d59c9 100644
--- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java
+++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java
@@ -10,10 +10,7 @@ import at.tuwien.api.database.query.QueryPersistDto;
import at.tuwien.api.error.ApiErrorDto;
import at.tuwien.exception.*;
import at.tuwien.mapper.MetadataMapper;
-import at.tuwien.service.CredentialService;
-import at.tuwien.service.SchemaService;
-import at.tuwien.service.StorageService;
-import at.tuwien.service.SubsetService;
+import at.tuwien.service.*;
import at.tuwien.utils.UserUtil;
import at.tuwien.validation.EndpointValidator;
import io.micrometer.observation.annotation.Observed;
@@ -55,17 +52,19 @@ public class SubsetEndpoint extends AbstractEndpoint {
private final SchemaService schemaService;
private final SubsetService subsetService;
private final MetadataMapper metadataMapper;
+ private final MetricsService metricsService;
private final StorageService storageService;
private final CredentialService credentialService;
private final EndpointValidator endpointValidator;
@Autowired
public SubsetEndpoint(SchemaService schemaService, SubsetService subsetService, MetadataMapper metadataMapper,
- StorageService storageService, CredentialService credentialService,
- EndpointValidator endpointValidator) {
+ MetricsService metricsService, StorageService storageService,
+ CredentialService credentialService, EndpointValidator endpointValidator) {
this.schemaService = schemaService;
this.subsetService = subsetService;
this.metadataMapper = metadataMapper;
+ this.metricsService = metricsService;
this.storageService = storageService;
this.credentialService = credentialService;
this.endpointValidator = endpointValidator;
@@ -188,6 +187,7 @@ public class SubsetEndpoint extends AbstractEndpoint {
log.trace("accept header matches csv");
try {
final Dataset<Row> dataset = subsetService.getData(database, subset, null, null);
+ metricsService.countSubsetGetData(databaseId, subsetId);
final ExportResourceDto resource = storageService.transformDataset(dataset);
final HttpHeaders headers = new HttpHeaders();
headers.add("Content-Disposition", "attachment; filename=\"" + resource.getFilename() + "\"");
@@ -366,6 +366,7 @@ public class SubsetEndpoint extends AbstractEndpoint {
.build();
}
final Dataset<Row> dataset = subsetService.getData(database, subset, page, size);
+ metricsService.countSubsetGetData(databaseId, subsetId);
final ViewDto view = schemaService.inspectView(database, metadataMapper.queryDtoToViewName(subset));
headers.set("Access-Control-Expose-Headers", "X-Id X-Headers");
headers.set("X-Headers", String.join(",", view.getColumns().stream().map(ViewColumnDto::getInternalName).toList()));
diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
index 4191726dd9..11720b4906 100644
--- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
+++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
@@ -12,10 +12,7 @@ import at.tuwien.api.database.table.internal.TableCreateDto;
import at.tuwien.api.error.ApiErrorDto;
import at.tuwien.exception.*;
import at.tuwien.gateway.MetadataServiceGateway;
-import at.tuwien.service.CredentialService;
-import at.tuwien.service.SchemaService;
-import at.tuwien.service.StorageService;
-import at.tuwien.service.TableService;
+import at.tuwien.service.*;
import at.tuwien.utils.UserUtil;
import at.tuwien.validation.EndpointValidator;
import io.micrometer.observation.annotation.Observed;
@@ -55,17 +52,19 @@ public class TableEndpoint extends AbstractEndpoint {
private final TableService tableService;
private final SchemaService schemaService;
+ private final MetricsService metricsService;
private final StorageService storageService;
private final CredentialService credentialService;
private final EndpointValidator endpointValidator;
private final MetadataServiceGateway metadataServiceGateway;
@Autowired
- public TableEndpoint(TableService tableService, SchemaService schemaService, StorageService storageService,
- CredentialService credentialService, EndpointValidator endpointValidator,
- MetadataServiceGateway metadataServiceGateway) {
+ public TableEndpoint(TableService tableService, SchemaService schemaService, MetricsService metricsService,
+ StorageService storageService, CredentialService credentialService,
+ EndpointValidator endpointValidator, MetadataServiceGateway metadataServiceGateway) {
this.tableService = tableService;
this.schemaService = schemaService;
+ this.metricsService = metricsService;
this.storageService = storageService;
this.credentialService = credentialService;
this.endpointValidator = endpointValidator;
@@ -291,10 +290,12 @@ public class TableEndpoint extends AbstractEndpoint {
}
headers.set("Access-Control-Expose-Headers", "X-Headers");
headers.set("X-Headers", String.join(",", table.getColumns().stream().map(ColumnDto::getInternalName).toList()));
+ final Dataset<Row> dataset = tableService.getData(table.getDatabase(), table.getInternalName(), timestamp,
+ null, null, null, null);
+ metricsService.countTableGetData(databaseId, tableId);
return ResponseEntity.ok()
.headers(headers)
- .body(transform(tableService.getData(table.getDatabase(), table.getInternalName(), timestamp,
- null, null, null, null)));
+ .body(transform(dataset));
} catch (SQLException | QueryMalformedException e) {
log.error("Failed to establish connection to database: {}", e.getMessage());
throw new DatabaseUnavailableException("Failed to establish connection to database: " + e.getMessage(), e);
@@ -625,6 +626,7 @@ public class TableEndpoint extends AbstractEndpoint {
}
final Dataset<Row> dataset = tableService.getData(table.getDatabase(), table.getInternalName(), timestamp, null,
null, null, null);
+ metricsService.countTableGetData(databaseId, tableId);
final ExportResourceDto resource = storageService.transformDataset(dataset);
final HttpHeaders headers = new HttpHeaders();
headers.add("Content-Disposition", "attachment; filename=\"" + resource.getFilename() + "\"");
diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java
index b08c300b45..c8da423986 100644
--- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java
+++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java
@@ -8,10 +8,7 @@ import at.tuwien.api.database.internal.PrivilegedDatabaseDto;
import at.tuwien.api.database.internal.PrivilegedViewDto;
import at.tuwien.api.error.ApiErrorDto;
import at.tuwien.exception.*;
-import at.tuwien.service.CredentialService;
-import at.tuwien.service.StorageService;
-import at.tuwien.service.TableService;
-import at.tuwien.service.ViewService;
+import at.tuwien.service.*;
import at.tuwien.utils.UserUtil;
import at.tuwien.validation.EndpointValidator;
import io.micrometer.observation.annotation.Observed;
@@ -26,6 +23,8 @@ import jakarta.servlet.http.HttpServletRequest;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotNull;
import lombok.extern.log4j.Log4j2;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.InputStreamResource;
import org.springframework.http.HttpHeaders;
@@ -48,15 +47,18 @@ public class ViewEndpoint extends AbstractEndpoint {
private final ViewService viewService;
private final TableService tableService;
+ private final MetricsService metricsService;
private final StorageService storageService;
private final CredentialService credentialService;
private final EndpointValidator endpointValidator;
@Autowired
- public ViewEndpoint(ViewService viewService, TableService tableService, StorageService storageService,
- CredentialService credentialService, EndpointValidator endpointValidator) {
+ public ViewEndpoint(ViewService viewService, TableService tableService, MetricsService metricsService,
+ StorageService storageService, CredentialService credentialService,
+ EndpointValidator endpointValidator) {
this.viewService = viewService;
this.tableService = tableService;
+ this.metricsService = metricsService;
this.storageService = storageService;
this.credentialService = credentialService;
this.endpointValidator = endpointValidator;
@@ -286,10 +288,12 @@ public class ViewEndpoint extends AbstractEndpoint {
}
headers.set("Access-Control-Expose-Headers", "X-Headers");
headers.set("X-Headers", String.join(",", view.getColumns().stream().map(ViewColumnDto::getInternalName).toList()));
+ final Dataset<Row> dataset = tableService.getData(view.getDatabase(), view.getInternalName(), timestamp,
+ page, size, null, null);
+ metricsService.countViewGetData(databaseId, viewId);
return ResponseEntity.ok()
.headers(headers)
- .body(transform(tableService.getData(view.getDatabase(), view.getInternalName(), timestamp, page,
- size, null, null)));
+ .body(transform(dataset));
} catch (SQLException e) {
log.error("Failed to establish connection to database: {}", e.getMessage());
throw new DatabaseUnavailableException("Failed to establish connection to database: " + e.getMessage(), e);
@@ -349,9 +353,11 @@ public class ViewEndpoint extends AbstractEndpoint {
}
credentialService.getAccess(databaseId, UserUtil.getId(principal));
}
+ final Dataset<Row> dataset = tableService.getData(view.getDatabase(), view.getInternalName(), timestamp, null,
+ null, null, null);
+ metricsService.countViewGetData(databaseId, viewId);
+ final ExportResourceDto resource = storageService.transformDataset(dataset);
final HttpHeaders headers = new HttpHeaders();
- final ExportResourceDto resource = storageService.transformDataset(tableService.getData(view.getDatabase(),
- view.getInternalName(), timestamp, null, null, null, null));
headers.add("Content-Disposition", "attachment; filename=\"" + resource.getFilename() + "\"");
log.trace("export table resulted in resource {}", resource);
return ResponseEntity.ok()
diff --git a/dbrepo-data-service/rest-service/src/main/resources/application.yml b/dbrepo-data-service/rest-service/src/main/resources/application.yml
index 9848928b7b..03d89895cb 100644
--- a/dbrepo-data-service/rest-service/src/main/resources/application.yml
+++ b/dbrepo-data-service/rest-service/src/main/resources/application.yml
@@ -82,3 +82,4 @@ dbrepo:
exchangeName: "${BROKER_EXCHANGE_NAME:dbrepo}"
routingKey: "${BROKER_ROUTING_KEY:#}"
connectionTimeout: "${SPARQL_CONNECTION_TIMEOUT:10000}"
+ baseUrl: "${BASE_URL:http://localhost}"
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/config/CacheConfig.java b/dbrepo-data-service/services/src/main/java/at/tuwien/config/CacheConfig.java
index a77af353d3..45654157d1 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/config/CacheConfig.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/config/CacheConfig.java
@@ -1,25 +1,64 @@
package at.tuwien.config;
-import at.tuwien.api.PrivilegedObjectDto;
+import at.tuwien.api.container.internal.PrivilegedContainerDto;
+import at.tuwien.api.database.DatabaseAccessDto;
+import at.tuwien.api.database.internal.PrivilegedDatabaseDto;
+import at.tuwien.api.database.internal.PrivilegedViewDto;
+import at.tuwien.api.database.table.internal.PrivilegedTableDto;
+import at.tuwien.api.user.internal.PrivilegedUserDto;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
+import java.util.UUID;
import java.util.concurrent.TimeUnit;
@Configuration
-public class CacheConfig<K, T extends PrivilegedObjectDto> {
+public class CacheConfig {
@Value("${dbrepo.credentialCacheTimeout}")
private Long credentialCacheTimeout;
@Bean
- public Cache<K, T> cache() {
- return Caffeine.newBuilder()
- .expireAfterWrite(credentialCacheTimeout, TimeUnit.SECONDS)
- .build();
+ public Cache<UUID, PrivilegedUserDto> userCache() {
+ return new ExpiryCache<UUID, PrivilegedUserDto>().build();
+ }
+
+ @Bean
+ public Cache<Long, PrivilegedViewDto> viewCache() {
+ return new ExpiryCache<Long, PrivilegedViewDto>().build();
+ }
+
+ @Bean
+ public Cache<Long, DatabaseAccessDto> accessCache() {
+ return new ExpiryCache<Long, DatabaseAccessDto>().build();
+ }
+
+ @Bean
+ public Cache<Long, PrivilegedTableDto> tableCache() {
+ return new ExpiryCache<Long, PrivilegedTableDto>().build();
+ }
+
+ @Bean
+ public Cache<Long, PrivilegedDatabaseDto> databaseCache() {
+ return new ExpiryCache<Long, PrivilegedDatabaseDto>().build();
+ }
+
+ @Bean
+ public Cache<Long, PrivilegedContainerDto> containerCache() {
+ return new ExpiryCache<Long, PrivilegedContainerDto>().build();
+ }
+
+ class ExpiryCache<K, T> {
+
+ Cache<K, T> build() {
+ return Caffeine.newBuilder()
+ .expireAfterWrite(credentialCacheTimeout, TimeUnit.SECONDS)
+ .build();
+ }
+
}
}
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/config/MetricsConfig.java b/dbrepo-data-service/services/src/main/java/at/tuwien/config/MetricsConfig.java
index 9ff09ab42b..af9ea49f9f 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/config/MetricsConfig.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/config/MetricsConfig.java
@@ -1,33 +1,21 @@
package at.tuwien.config;
-import io.micrometer.core.instrument.Counter;
-import io.micrometer.core.instrument.Metrics;
import io.micrometer.observation.ObservationRegistry;
import io.micrometer.observation.aop.ObservedAspect;
+import lombok.Getter;
+import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
+@Getter
@Configuration
public class MetricsConfig {
+ @Value("${dbrepo.baseUrl}")
+ private String baseUrl;
+
@Bean
public ObservedAspect observedAspect(ObservationRegistry observationRegistry) {
return new ObservedAspect(observationRegistry);
}
-
- @Bean
- public Counter httpDataAccessCounter() {
- return Counter.builder("dbrepo.data.access")
- .tag("protocol", "http")
- .description("The total number of accessed data sources")
- .register(Metrics.globalRegistry);
- }
-
- @Bean
- public Counter amqpDataAccessCounter() {
- return Counter.builder("dbrepo.data.access")
- .tag("protocol", "amqp")
- .description("The total number of accessed data sources")
- .register(Metrics.globalRegistry);
- }
}
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java
index fe1b2e3611..1e235de4c6 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java
@@ -1,9 +1,10 @@
package at.tuwien.gateway.impl;
-import at.tuwien.api.auth.KeycloakErrorDto;
import at.tuwien.api.keycloak.TokenDto;
import at.tuwien.config.KeycloakConfig;
-import at.tuwien.exception.*;
+import at.tuwien.exception.AccountNotSetupException;
+import at.tuwien.exception.AuthServiceConnectionException;
+import at.tuwien.exception.CredentialsInvalidException;
import at.tuwien.gateway.KeycloakGateway;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired;
@@ -40,18 +41,6 @@ public class KeycloakGatewayImpl implements KeycloakGateway {
payload.add("client_secret", keycloakConfig.getKeycloakClientSecret());
final String url = keycloakConfig.getKeycloakEndpoint() + "/realms/dbrepo/protocol/openid-connect/token";
log.trace("request user token from url: {}", url);
- log.trace("request username: {}", username);
- if (password.isEmpty() || password.isBlank()) {
- log.warn("request password: (empty)");
- } else {
- log.trace("request password: (set)");
- }
- log.trace("request client_id: {}", keycloakConfig.getKeycloakClient());
- if (keycloakConfig.getKeycloakClientSecret().isEmpty() || keycloakConfig.getKeycloakClientSecret().isBlank()) {
- log.warn("request client_secret: (empty)");
- } else {
- log.trace("request client_secret: (set)");
- }
final ResponseEntity<TokenDto> response;
try {
response = new RestTemplate()
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MetadataMapper.java b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MetadataMapper.java
index 884732bdc1..0adfafa8f9 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MetadataMapper.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MetadataMapper.java
@@ -64,4 +64,22 @@ public interface MetadataMapper {
IdentifierBriefDto identifierDtoToIdentifierBriefDto(IdentifierDto data);
+ default String metricToUri(String baseUrl, Long databaseId, Long tableId, Long subsetId, Long viewId) {
+ final StringBuilder uri = new StringBuilder(baseUrl)
+ .append("/database/")
+ .append(databaseId);
+ if (tableId != null) {
+ uri.append("/table/")
+ .append(tableId);
+ } else if (subsetId != null) {
+ uri.append("/subset/")
+ .append(subsetId);
+ } else if (viewId != null) {
+ uri.append("/view/")
+ .append(viewId);
+ }
+ log.trace("count uri: {}", uri);
+ return uri.toString();
+ }
+
}
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/MetricsService.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/MetricsService.java
new file mode 100644
index 0000000000..131bae7287
--- /dev/null
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/MetricsService.java
@@ -0,0 +1,10 @@
+package at.tuwien.service;
+
+public interface MetricsService {
+
+ void countTableGetData(Long databaseId, Long tableId);
+
+ void countSubsetGetData(Long databaseId, Long subsetId);
+
+ void countViewGetData(Long databaseId, Long viewId);
+}
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/MetricsServicePrometheusImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/MetricsServicePrometheusImpl.java
new file mode 100644
index 0000000000..73754c9f5f
--- /dev/null
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/MetricsServicePrometheusImpl.java
@@ -0,0 +1,47 @@
+package at.tuwien.service.impl;
+
+import at.tuwien.config.MetricsConfig;
+import at.tuwien.mapper.MetadataMapper;
+import at.tuwien.service.MetricsService;
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.Metrics;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.stereotype.Service;
+
+@Log4j2
+@Service
+public class MetricsServicePrometheusImpl implements MetricsService {
+
+ private final MetricsConfig metricsConfig;
+ private final MetadataMapper metadataMapper;
+
+ public MetricsServicePrometheusImpl(MetricsConfig metricsConfig, MetadataMapper metadataMapper) {
+ this.metricsConfig = metricsConfig;
+ this.metadataMapper = metadataMapper;
+ }
+
+ @Override
+ public void countTableGetData(Long databaseId, Long tableId) {
+ countGetData(databaseId, tableId, null, null);
+ }
+
+ @Override
+ public void countSubsetGetData(Long databaseId, Long subsetId) {
+ countGetData(databaseId, null, subsetId, null);
+ }
+
+ @Override
+ public void countViewGetData(Long databaseId, Long viewId) {
+ countGetData(databaseId, null, null, viewId);
+ }
+
+ public void countGetData(Long databaseId, Long tableId, Long subsetId, Long viewId) {
+ Counter.builder("dbrepo.datasource.data.get")
+ .tag("uri", metadataMapper.metricToUri(metricsConfig.getBaseUrl(), databaseId, tableId, subsetId, viewId))
+ .tag("protocol", "http")
+ .description("The total number of accessed data sources")
+ .register(Metrics.globalRegistry)
+ .increment();
+ }
+
+}
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java
index aff8513787..d5127e050e 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java
@@ -6,7 +6,6 @@ import at.tuwien.mapper.DataMapper;
import at.tuwien.mapper.MetadataMapper;
import at.tuwien.service.QueueService;
import com.mchange.v2.c3p0.ComboPooledDataSource;
-import io.micrometer.core.instrument.Counter;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -21,14 +20,11 @@ import java.util.Optional;
@Service
public class QueueServiceRabbitMqImpl extends HibernateConnector implements QueueService {
- private final Counter amqpDataAccessCounter;
private final DataMapper dataMapper;
private final MetadataMapper metadataMapper;
@Autowired
- public QueueServiceRabbitMqImpl(Counter amqpDataAccessCounter, DataMapper dataMapper,
- MetadataMapper metadataMapper) {
- this.amqpDataAccessCounter = amqpDataAccessCounter;
+ public QueueServiceRabbitMqImpl(DataMapper dataMapper, MetadataMapper metadataMapper) {
this.dataMapper = dataMapper;
this.metadataMapper = metadataMapper;
}
@@ -54,7 +50,6 @@ public class QueueServiceRabbitMqImpl extends HibernateConnector implements Queu
preparedStatement.executeUpdate();
log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
log.trace("successfully inserted tuple");
- amqpDataAccessCounter.increment();
} finally {
dataSource.close();
}
diff --git a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java
index 98c77c7b0d..f63c387c98 100644
--- a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java
+++ b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java
@@ -483,7 +483,6 @@ public interface MetadataMapper {
@Mappings({
@Mapping(target = "databaseId", source = "tdbid"),
- @Mapping(target = "isPublic", source = "database.isPublic"),
})
TableBriefDto tableToTableBriefDto(Table data);
@@ -535,7 +534,7 @@ public interface MetadataMapper {
.internalName(data.getInternalName())
.owner(userToUserBriefDto(data.getOwner()))
.tdbid(data.getTdbid())
- .isPublic(data.getDatabase().getIsPublic())
+ .isPublic(data.getIsPublic())
.isSchemaPublic(data.getIsSchemaPublic())
.isVersioned(true)
.description(data.getDescription())
@@ -720,7 +719,7 @@ public interface MetadataMapper {
@Mappings({
@Mapping(target = "tableId", source = "table.id"),
@Mapping(target = "databaseId", source = "table.database.id"),
- @Mapping(target = "isPublic", source = "table.database.isPublic"),
+ @Mapping(target = "isPublic", source = "table.isSchemaPublic"),
@Mapping(target = "description", source = "description"),
@Mapping(target = "table", ignore = true),
@Mapping(target = "views", ignore = true)
diff --git a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
index 3dbc450735..a517c39d10 100644
--- a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
+++ b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
@@ -403,7 +403,8 @@ public class TableEndpoint {
@NotNull Principal principal) throws NotAllowedException,
DataServiceException, DataServiceConnectionException, DatabaseNotFoundException, TableNotFoundException,
SearchServiceException, SearchServiceConnectionException {
- log.debug("endpoint update table, databaseId={}, data.is_public={}", databaseId, data.getIsPublic());
+ log.debug("endpoint update table, databaseId={}, data.is_public={}, data.is_schema_public={}", databaseId,
+ data.getIsPublic(), data.getIsSchemaPublic());
final Table table = tableService.findById(databaseId, tableId);
if (!table.getOwner().equals(principal)) {
log.error("Failed to update table: not owner");
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java
index 4a4a9ccaca..57b546fcea 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java
@@ -220,8 +220,8 @@ public class TableServiceImpl implements TableService {
}
final Table tableEntity = optional.get();
tableEntity.setIsPublic(data.getIsPublic());
- tableEntity.setDescription(data.getDescription());
tableEntity.setIsSchemaPublic(data.getIsSchemaPublic());
+ tableEntity.setDescription(data.getDescription());
final Database database = databaseRepository.save(table.getDatabase());
/* update in search service */
searchServiceGateway.update(database);
diff --git a/dbrepo-ui/components/subset/SubsetList.vue b/dbrepo-ui/components/subset/SubsetList.vue
index b977daffa5..f57dc68a88 100644
--- a/dbrepo-ui/components/subset/SubsetList.vue
+++ b/dbrepo-ui/components/subset/SubsetList.vue
@@ -26,6 +26,21 @@
:to="link(item)"
:href="link(item)">
<template v-slot:append>
+ <v-chip
+ v-if="database.is_public"
+ size="small"
+ class="ml-2"
+ color="success"
+ :text="$t('toolbars.database.public')"
+ variant="outlined" />
+ <v-chip
+ v-if="!database.is_public"
+ size="small"
+ class="ml-2"
+ :color="colorVariant"
+ variant="outlined"
+ :text="$t('toolbars.database.private')"
+ flat />
<v-tooltip
v-if="hasPublishedIdentifier(item)"
:text="$t('pages.identifier.pid.title')"
@@ -65,6 +80,15 @@ export default {
},
database () {
return this.cacheStore.getDatabase
+ },
+ isContrastTheme () {
+ return this.$vuetify.theme.global.name.toLowerCase().endsWith('contrast')
+ },
+ isDarkTheme () {
+ return this.$vuetify.theme.global.name.toLowerCase().startsWith('dark')
+ },
+ colorVariant () {
+ return this.isContrastTheme ? '' : (this.isDarkTheme ? 'tertiary' : 'secondary')
}
},
mounted () {
@@ -88,25 +112,25 @@ export default {
toast.error(this.$t(code))
})
},
- title (query) {
- if (query.identifiers.length === 0) {
- return formatTimestampUTCLabel(query.created)
+ title (subset) {
+ if (subset.identifiers.length === 0) {
+ return subset.query
}
const identifierService = useIdentifierService()
- return identifierService.identifierPreferEnglishTitle(query.identifiers[0])
+ return identifierService.identifierPreferEnglishTitle(subset.identifiers[0])
},
- subtitle (query) {
- if (query.identifiers.length === 0) {
+ subtitle (subset) {
+ if (subset.identifiers.length === 0) {
return null
}
const identifierService = useIdentifierService()
- return identifierService.identifierPreferEnglishDescription(query.identifiers[0])
+ return identifierService.identifierPreferEnglishDescription(subset.identifiers[0])
},
- link (query) {
- return `/database/${this.$route.params.database_id}/subset/${query.id}/info`
+ link (subset) {
+ return `/database/${this.$route.params.database_id}/subset/${subset.id}/info`
},
- clazz (view) {
- return this.hasPublishedIdentifier(view) ? 'primary-text' : null
+ clazz (subset) {
+ return this.hasPublishedIdentifier(subset) ? 'primary-text' : null
},
hasPublishedIdentifier (subset) {
if (!subset.identifiers) {
diff --git a/dbrepo-ui/components/view/ViewList.vue b/dbrepo-ui/components/view/ViewList.vue
index 6fe8451903..543a8746af 100644
--- a/dbrepo-ui/components/view/ViewList.vue
+++ b/dbrepo-ui/components/view/ViewList.vue
@@ -61,9 +61,6 @@ export default {
}
},
computed: {
- loadingColor () {
- return this.error ? 'red lighten-2' : 'primary'
- },
user () {
return this.userStore.getUser
},
diff --git a/dbrepo-ui/components/view/ViewToolbar.vue b/dbrepo-ui/components/view/ViewToolbar.vue
index 1f122cc7a4..64ea3f1029 100644
--- a/dbrepo-ui/components/view/ViewToolbar.vue
+++ b/dbrepo-ui/components/view/ViewToolbar.vue
@@ -187,7 +187,7 @@ export default {
return this.access.type === 'read' || this.access.type === 'write_own' || this.access.type === 'write_all'
},
canReadData () {
- if (!this.view) {
+ if (!this.cachedView) {
return false
}
if (this.cachedView.is_public) {
diff --git a/dbrepo-ui/pages/database/[database_id]/view/[view_id]/data.vue b/dbrepo-ui/pages/database/[database_id]/view/[view_id]/data.vue
index 6aed9307c4..9751fce5b7 100644
--- a/dbrepo-ui/pages/database/[database_id]/view/[view_id]/data.vue
+++ b/dbrepo-ui/pages/database/[database_id]/view/[view_id]/data.vue
@@ -96,7 +96,7 @@ export default {
return this.access.type === 'read' || this.access.type === 'write_own' || this.access.type === 'write_all'
},
canReadData () {
- if (!this.view) {
+ if (!this.cachedView) {
return false
}
if (this.cachedView.is_public) {
--
GitLab