diff --git a/dbrepo-dashboard-service/dashboards/system.json b/dbrepo-dashboard-service/dashboards/system.json index edee464f623bde864422058dc1edf31d98631162..e6f81bda403f662339d11600d46abfd6d18af81a 100644 --- a/dbrepo-dashboard-service/dashboards/system.json +++ b/dbrepo-dashboard-service/dashboards/system.json @@ -18,6 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, + "id": 3, "links": [ { "asDropdown": false, @@ -109,7 +110,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.3", + "pluginVersion": "10.4.9", "targets": [ { "datasource": { @@ -179,7 +180,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.3", + "pluginVersion": "10.4.9", "targets": [ { "datasource": { @@ -247,7 +248,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.3", + "pluginVersion": "10.4.9", "targets": [ { "datasource": { @@ -315,7 +316,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.3", + "pluginVersion": "10.4.9", "targets": [ { "datasource": { @@ -431,7 +432,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.3", + "pluginVersion": "10.4.9", "targets": [ { "datasource": { @@ -453,13 +454,83 @@ "title": "Data Volume", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "P18F45E9DC7E75912" + }, + "description": "Top 10 by number of accesses", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 4 + }, + "id": 38, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false, + "values": [] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P18F45E9DC7E75912" + }, + "editorMode": "code", + "expr": "topk(10, dbrepo_datasource_data_get_total)", + "instant": false, + "legendFormat": "{{uri}}", + "range": true, + "refId": "A" + } + ], + "title": "Popular Data Sources", + "type": "piechart" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 4 + "y": 11 }, "id": 22, "panels": [], @@ -506,7 +577,7 @@ "h": 3, "w": 4, "x": 0, - "y": 5 + "y": 12 }, "id": 17, "options": { @@ -526,7 +597,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.3", + "pluginVersion": "10.4.9", "targets": [ { "datasource": { @@ -588,7 +659,7 @@ "h": 3, "w": 4, "x": 4, - "y": 5 + "y": 12 }, "id": 24, "options": { @@ -608,7 +679,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.3", + "pluginVersion": "10.4.9", "targets": [ { "datasource": { @@ -658,7 +729,7 @@ "h": 3, "w": 4, "x": 8, - "y": 5 + "y": 12 }, "id": 25, "options": { @@ -678,7 +749,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.3", + "pluginVersion": "10.4.9", "targets": [ { "datasource": { @@ -728,7 +799,7 @@ "h": 3, "w": 4, "x": 12, - "y": 5 + "y": 12 }, "id": 26, "options": { @@ -748,7 +819,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.3", + "pluginVersion": "10.4.9", "targets": [ { "datasource": { @@ -800,7 +871,7 @@ "h": 3, "w": 4, "x": 16, - "y": 5 + "y": 12 }, "id": 27, "options": { @@ -819,7 +890,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.3", + "pluginVersion": "10.4.9", "targets": [ { "datasource": { @@ -881,7 +952,7 @@ "h": 7, "w": 12, "x": 0, - "y": 8 + "y": 15 }, "id": 20, "options": { @@ -902,7 +973,7 @@ "sizing": "auto", "valueMode": "color" }, - "pluginVersion": "10.4.3", + "pluginVersion": "10.4.9", "targets": [ { "datasource": { @@ -988,7 +1059,7 @@ "h": 7, "w": 12, "x": 12, - "y": 8 + "y": 15 }, "id": 21, "options": { @@ -1030,7 +1101,7 @@ "h": 1, "w": 24, "x": 0, - "y": 15 + "y": 22 }, "id": 31, "panels": [], @@ -1052,8 +1123,7 @@ "mode": "absolute", "steps": [ { - "color": "purple", - "value": null + "color": "purple" }, { "color": "red", @@ -1081,7 +1151,7 @@ "h": 3, "w": 4, "x": 0, - "y": 16 + "y": 23 }, "id": 32, "options": { @@ -1101,7 +1171,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.3", + "pluginVersion": "10.4.9", "targets": [ { "datasource": { @@ -1138,8 +1208,7 @@ "mode": "absolute", "steps": [ { - "color": "blue", - "value": null + "color": "blue" } ] }, @@ -1151,7 +1220,7 @@ "h": 3, "w": 4, "x": 4, - "y": 16 + "y": 23 }, "id": 29, "options": { @@ -1171,7 +1240,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.3", + "pluginVersion": "10.4.9", "targets": [ { "datasource": { @@ -1208,8 +1277,7 @@ "mode": "absolute", "steps": [ { - "color": "blue", - "value": null + "color": "blue" } ] }, @@ -1221,7 +1289,7 @@ "h": 3, "w": 4, "x": 8, - "y": 16 + "y": 23 }, "id": 30, "options": { @@ -1241,7 +1309,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.3", + "pluginVersion": "10.4.9", "targets": [ { "datasource": { @@ -1278,8 +1346,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "#EAB839", @@ -1303,7 +1370,7 @@ "h": 3, "w": 4, "x": 12, - "y": 16 + "y": 23 }, "id": 35, "options": { @@ -1323,7 +1390,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.3", + "pluginVersion": "10.4.9", "targets": [ { "datasource": { @@ -1360,8 +1427,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "#EAB839", @@ -1385,7 +1451,7 @@ "h": 3, "w": 4, "x": 16, - "y": 16 + "y": 23 }, "id": 36, "options": { @@ -1405,7 +1471,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.3", + "pluginVersion": "10.4.9", "targets": [ { "datasource": { @@ -1442,8 +1508,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "#EAB839", @@ -1467,7 +1532,7 @@ "h": 3, "w": 4, "x": 20, - "y": 16 + "y": 23 }, "id": 37, "options": { @@ -1487,7 +1552,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.3", + "pluginVersion": "10.4.9", "targets": [ { "datasource": { @@ -1557,8 +1622,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -1601,7 +1665,7 @@ "h": 7, "w": 12, "x": 0, - "y": 19 + "y": 26 }, "id": 33, "options": { @@ -1644,7 +1708,7 @@ "h": 1, "w": 24, "x": 0, - "y": 26 + "y": 33 }, "id": 2, "panels": [], @@ -1699,8 +1763,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1715,7 +1778,7 @@ "h": 7, "w": 12, "x": 0, - "y": 27 + "y": 34 }, "id": 23, "options": { @@ -1790,8 +1853,7 @@ "mode": "absolute", "steps": [ { - "color": "red", - "value": null + "color": "red" }, { "color": "green", @@ -1806,7 +1868,7 @@ "h": 7, "w": 12, "x": 12, - "y": 27 + "y": 34 }, "id": 16, "options": { @@ -1891,8 +1953,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -1965,7 +2026,7 @@ "h": 7, "w": 12, "x": 0, - "y": 34 + "y": 41 }, "id": 6, "options": { @@ -2052,8 +2113,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -2126,7 +2186,7 @@ "h": 7, "w": 12, "x": 12, - "y": 34 + "y": 41 }, "id": 7, "options": { @@ -2169,104 +2229,52 @@ "type": "prometheus", "uid": "P18F45E9DC7E75912" }, + "description": "Top 10 by frequency of access", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, "unit": "reqps" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*search-service.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*analyse-service.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "super-light-orange", - "mode": "fixed" - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 41 + "y": 48 }, "id": 18, "options": { + "displayLabels": [ + "percent" + ], "legend": { "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "displayMode": "hidden", + "placement": "right", + "showLegend": false, + "values": [] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, "tooltip": { - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -2277,15 +2285,15 @@ "uid": "P18F45E9DC7E75912" }, "editorMode": "code", - "expr": "rate(flask_http_request_duration_seconds_count{status=~\"200|201|202\",path!=\"/health\"}[$__rate_interval])", + "expr": "topk(10, rate(dbrepo_table_data_get_total[$__range]))", "instant": false, - "legendFormat": "{{method}} {{instance}} {{path}} ({{status}})", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Successful API Requests", - "type": "timeseries" + "title": "Popular Datasources", + "type": "piechart" }, { "datasource": { @@ -2334,8 +2342,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -2378,7 +2385,7 @@ "h": 7, "w": 12, "x": 12, - "y": 41 + "y": 48 }, "id": 19, "options": { @@ -2421,13 +2428,13 @@ "list": [] }, "time": { - "from": "now-30m", + "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "DBRepo - Overview", "uid": "bdz20owu8zn5se", - "version": 1, + "version": 8, "weekStart": "" } \ No newline at end of file diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java index 13cfc5c56011512f6137bb728933fe5f41055c35..4f1b5d59c94cee09b4666a8171367cd9d737ea27 100644 --- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java +++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java @@ -10,10 +10,7 @@ import at.tuwien.api.database.query.QueryPersistDto; import at.tuwien.api.error.ApiErrorDto; import at.tuwien.exception.*; import at.tuwien.mapper.MetadataMapper; -import at.tuwien.service.CredentialService; -import at.tuwien.service.SchemaService; -import at.tuwien.service.StorageService; -import at.tuwien.service.SubsetService; +import at.tuwien.service.*; import at.tuwien.utils.UserUtil; import at.tuwien.validation.EndpointValidator; import io.micrometer.observation.annotation.Observed; @@ -55,17 +52,19 @@ public class SubsetEndpoint extends AbstractEndpoint { private final SchemaService schemaService; private final SubsetService subsetService; private final MetadataMapper metadataMapper; + private final MetricsService metricsService; private final StorageService storageService; private final CredentialService credentialService; private final EndpointValidator endpointValidator; @Autowired public SubsetEndpoint(SchemaService schemaService, SubsetService subsetService, MetadataMapper metadataMapper, - StorageService storageService, CredentialService credentialService, - EndpointValidator endpointValidator) { + MetricsService metricsService, StorageService storageService, + CredentialService credentialService, EndpointValidator endpointValidator) { this.schemaService = schemaService; this.subsetService = subsetService; this.metadataMapper = metadataMapper; + this.metricsService = metricsService; this.storageService = storageService; this.credentialService = credentialService; this.endpointValidator = endpointValidator; @@ -188,6 +187,7 @@ public class SubsetEndpoint extends AbstractEndpoint { log.trace("accept header matches csv"); try { final Dataset<Row> dataset = subsetService.getData(database, subset, null, null); + metricsService.countSubsetGetData(databaseId, subsetId); final ExportResourceDto resource = storageService.transformDataset(dataset); final HttpHeaders headers = new HttpHeaders(); headers.add("Content-Disposition", "attachment; filename=\"" + resource.getFilename() + "\""); @@ -366,6 +366,7 @@ public class SubsetEndpoint extends AbstractEndpoint { .build(); } final Dataset<Row> dataset = subsetService.getData(database, subset, page, size); + metricsService.countSubsetGetData(databaseId, subsetId); final ViewDto view = schemaService.inspectView(database, metadataMapper.queryDtoToViewName(subset)); headers.set("Access-Control-Expose-Headers", "X-Id X-Headers"); headers.set("X-Headers", String.join(",", view.getColumns().stream().map(ViewColumnDto::getInternalName).toList())); diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java index 4191726dd9a5b10cf26055bbe91c323c73e75450..11720b49063e9ba0c1dbc1b1f844aa215a6b3b4d 100644 --- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java +++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java @@ -12,10 +12,7 @@ import at.tuwien.api.database.table.internal.TableCreateDto; import at.tuwien.api.error.ApiErrorDto; import at.tuwien.exception.*; import at.tuwien.gateway.MetadataServiceGateway; -import at.tuwien.service.CredentialService; -import at.tuwien.service.SchemaService; -import at.tuwien.service.StorageService; -import at.tuwien.service.TableService; +import at.tuwien.service.*; import at.tuwien.utils.UserUtil; import at.tuwien.validation.EndpointValidator; import io.micrometer.observation.annotation.Observed; @@ -55,17 +52,19 @@ public class TableEndpoint extends AbstractEndpoint { private final TableService tableService; private final SchemaService schemaService; + private final MetricsService metricsService; private final StorageService storageService; private final CredentialService credentialService; private final EndpointValidator endpointValidator; private final MetadataServiceGateway metadataServiceGateway; @Autowired - public TableEndpoint(TableService tableService, SchemaService schemaService, StorageService storageService, - CredentialService credentialService, EndpointValidator endpointValidator, - MetadataServiceGateway metadataServiceGateway) { + public TableEndpoint(TableService tableService, SchemaService schemaService, MetricsService metricsService, + StorageService storageService, CredentialService credentialService, + EndpointValidator endpointValidator, MetadataServiceGateway metadataServiceGateway) { this.tableService = tableService; this.schemaService = schemaService; + this.metricsService = metricsService; this.storageService = storageService; this.credentialService = credentialService; this.endpointValidator = endpointValidator; @@ -291,10 +290,12 @@ public class TableEndpoint extends AbstractEndpoint { } headers.set("Access-Control-Expose-Headers", "X-Headers"); headers.set("X-Headers", String.join(",", table.getColumns().stream().map(ColumnDto::getInternalName).toList())); + final Dataset<Row> dataset = tableService.getData(table.getDatabase(), table.getInternalName(), timestamp, + null, null, null, null); + metricsService.countTableGetData(databaseId, tableId); return ResponseEntity.ok() .headers(headers) - .body(transform(tableService.getData(table.getDatabase(), table.getInternalName(), timestamp, - null, null, null, null))); + .body(transform(dataset)); } catch (SQLException | QueryMalformedException e) { log.error("Failed to establish connection to database: {}", e.getMessage()); throw new DatabaseUnavailableException("Failed to establish connection to database: " + e.getMessage(), e); @@ -625,6 +626,7 @@ public class TableEndpoint extends AbstractEndpoint { } final Dataset<Row> dataset = tableService.getData(table.getDatabase(), table.getInternalName(), timestamp, null, null, null, null); + metricsService.countTableGetData(databaseId, tableId); final ExportResourceDto resource = storageService.transformDataset(dataset); final HttpHeaders headers = new HttpHeaders(); headers.add("Content-Disposition", "attachment; filename=\"" + resource.getFilename() + "\""); diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java index b08c300b450c4035d7fdd5975503a67afe9ff414..c8da4239861264539b2140c64fde2b95a8fb8bc4 100644 --- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java +++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java @@ -8,10 +8,7 @@ import at.tuwien.api.database.internal.PrivilegedDatabaseDto; import at.tuwien.api.database.internal.PrivilegedViewDto; import at.tuwien.api.error.ApiErrorDto; import at.tuwien.exception.*; -import at.tuwien.service.CredentialService; -import at.tuwien.service.StorageService; -import at.tuwien.service.TableService; -import at.tuwien.service.ViewService; +import at.tuwien.service.*; import at.tuwien.utils.UserUtil; import at.tuwien.validation.EndpointValidator; import io.micrometer.observation.annotation.Observed; @@ -26,6 +23,8 @@ import jakarta.servlet.http.HttpServletRequest; import jakarta.validation.Valid; import jakarta.validation.constraints.NotNull; import lombok.extern.log4j.Log4j2; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.InputStreamResource; import org.springframework.http.HttpHeaders; @@ -48,15 +47,18 @@ public class ViewEndpoint extends AbstractEndpoint { private final ViewService viewService; private final TableService tableService; + private final MetricsService metricsService; private final StorageService storageService; private final CredentialService credentialService; private final EndpointValidator endpointValidator; @Autowired - public ViewEndpoint(ViewService viewService, TableService tableService, StorageService storageService, - CredentialService credentialService, EndpointValidator endpointValidator) { + public ViewEndpoint(ViewService viewService, TableService tableService, MetricsService metricsService, + StorageService storageService, CredentialService credentialService, + EndpointValidator endpointValidator) { this.viewService = viewService; this.tableService = tableService; + this.metricsService = metricsService; this.storageService = storageService; this.credentialService = credentialService; this.endpointValidator = endpointValidator; @@ -286,10 +288,12 @@ public class ViewEndpoint extends AbstractEndpoint { } headers.set("Access-Control-Expose-Headers", "X-Headers"); headers.set("X-Headers", String.join(",", view.getColumns().stream().map(ViewColumnDto::getInternalName).toList())); + final Dataset<Row> dataset = tableService.getData(view.getDatabase(), view.getInternalName(), timestamp, + page, size, null, null); + metricsService.countViewGetData(databaseId, viewId); return ResponseEntity.ok() .headers(headers) - .body(transform(tableService.getData(view.getDatabase(), view.getInternalName(), timestamp, page, - size, null, null))); + .body(transform(dataset)); } catch (SQLException e) { log.error("Failed to establish connection to database: {}", e.getMessage()); throw new DatabaseUnavailableException("Failed to establish connection to database: " + e.getMessage(), e); @@ -349,9 +353,11 @@ public class ViewEndpoint extends AbstractEndpoint { } credentialService.getAccess(databaseId, UserUtil.getId(principal)); } + final Dataset<Row> dataset = tableService.getData(view.getDatabase(), view.getInternalName(), timestamp, null, + null, null, null); + metricsService.countViewGetData(databaseId, viewId); + final ExportResourceDto resource = storageService.transformDataset(dataset); final HttpHeaders headers = new HttpHeaders(); - final ExportResourceDto resource = storageService.transformDataset(tableService.getData(view.getDatabase(), - view.getInternalName(), timestamp, null, null, null, null)); headers.add("Content-Disposition", "attachment; filename=\"" + resource.getFilename() + "\""); log.trace("export table resulted in resource {}", resource); return ResponseEntity.ok() diff --git a/dbrepo-data-service/rest-service/src/main/resources/application.yml b/dbrepo-data-service/rest-service/src/main/resources/application.yml index 9848928b7bd5d5d8b66802437ed3f2e0886396e7..03d89895cb607cee20bc84cd8cd3195c029a9ab8 100644 --- a/dbrepo-data-service/rest-service/src/main/resources/application.yml +++ b/dbrepo-data-service/rest-service/src/main/resources/application.yml @@ -82,3 +82,4 @@ dbrepo: exchangeName: "${BROKER_EXCHANGE_NAME:dbrepo}" routingKey: "${BROKER_ROUTING_KEY:#}" connectionTimeout: "${SPARQL_CONNECTION_TIMEOUT:10000}" + baseUrl: "${BASE_URL:http://localhost}" diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/config/CacheConfig.java b/dbrepo-data-service/services/src/main/java/at/tuwien/config/CacheConfig.java index a77af353d3b48f84706bbfff80cdcde1f374b414..45654157d1a7564d6504e08bd6d6d5fa5d24b206 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/config/CacheConfig.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/config/CacheConfig.java @@ -1,25 +1,64 @@ package at.tuwien.config; -import at.tuwien.api.PrivilegedObjectDto; +import at.tuwien.api.container.internal.PrivilegedContainerDto; +import at.tuwien.api.database.DatabaseAccessDto; +import at.tuwien.api.database.internal.PrivilegedDatabaseDto; +import at.tuwien.api.database.internal.PrivilegedViewDto; +import at.tuwien.api.database.table.internal.PrivilegedTableDto; +import at.tuwien.api.user.internal.PrivilegedUserDto; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import java.util.UUID; import java.util.concurrent.TimeUnit; @Configuration -public class CacheConfig<K, T extends PrivilegedObjectDto> { +public class CacheConfig { @Value("${dbrepo.credentialCacheTimeout}") private Long credentialCacheTimeout; @Bean - public Cache<K, T> cache() { - return Caffeine.newBuilder() - .expireAfterWrite(credentialCacheTimeout, TimeUnit.SECONDS) - .build(); + public Cache<UUID, PrivilegedUserDto> userCache() { + return new ExpiryCache<UUID, PrivilegedUserDto>().build(); + } + + @Bean + public Cache<Long, PrivilegedViewDto> viewCache() { + return new ExpiryCache<Long, PrivilegedViewDto>().build(); + } + + @Bean + public Cache<Long, DatabaseAccessDto> accessCache() { + return new ExpiryCache<Long, DatabaseAccessDto>().build(); + } + + @Bean + public Cache<Long, PrivilegedTableDto> tableCache() { + return new ExpiryCache<Long, PrivilegedTableDto>().build(); + } + + @Bean + public Cache<Long, PrivilegedDatabaseDto> databaseCache() { + return new ExpiryCache<Long, PrivilegedDatabaseDto>().build(); + } + + @Bean + public Cache<Long, PrivilegedContainerDto> containerCache() { + return new ExpiryCache<Long, PrivilegedContainerDto>().build(); + } + + class ExpiryCache<K, T> { + + Cache<K, T> build() { + return Caffeine.newBuilder() + .expireAfterWrite(credentialCacheTimeout, TimeUnit.SECONDS) + .build(); + } + } } diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/config/MetricsConfig.java b/dbrepo-data-service/services/src/main/java/at/tuwien/config/MetricsConfig.java index 9ff09ab42b2c2e62536e1d46fa986057ff664d53..af9ea49f9f5fbcbf97ef06a82bfb2113423ae6c6 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/config/MetricsConfig.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/config/MetricsConfig.java @@ -1,33 +1,21 @@ package at.tuwien.config; -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.Metrics; import io.micrometer.observation.ObservationRegistry; import io.micrometer.observation.aop.ObservedAspect; +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +@Getter @Configuration public class MetricsConfig { + @Value("${dbrepo.baseUrl}") + private String baseUrl; + @Bean public ObservedAspect observedAspect(ObservationRegistry observationRegistry) { return new ObservedAspect(observationRegistry); } - - @Bean - public Counter httpDataAccessCounter() { - return Counter.builder("dbrepo.data.access") - .tag("protocol", "http") - .description("The total number of accessed data sources") - .register(Metrics.globalRegistry); - } - - @Bean - public Counter amqpDataAccessCounter() { - return Counter.builder("dbrepo.data.access") - .tag("protocol", "amqp") - .description("The total number of accessed data sources") - .register(Metrics.globalRegistry); - } } diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java index fe1b2e36116052f41952d93afe0f7c65fa408514..1e235de4c63d883f2807c67e94480105750f5019 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java @@ -1,9 +1,10 @@ package at.tuwien.gateway.impl; -import at.tuwien.api.auth.KeycloakErrorDto; import at.tuwien.api.keycloak.TokenDto; import at.tuwien.config.KeycloakConfig; -import at.tuwien.exception.*; +import at.tuwien.exception.AccountNotSetupException; +import at.tuwien.exception.AuthServiceConnectionException; +import at.tuwien.exception.CredentialsInvalidException; import at.tuwien.gateway.KeycloakGateway; import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Autowired; @@ -40,18 +41,6 @@ public class KeycloakGatewayImpl implements KeycloakGateway { payload.add("client_secret", keycloakConfig.getKeycloakClientSecret()); final String url = keycloakConfig.getKeycloakEndpoint() + "/realms/dbrepo/protocol/openid-connect/token"; log.trace("request user token from url: {}", url); - log.trace("request username: {}", username); - if (password.isEmpty() || password.isBlank()) { - log.warn("request password: (empty)"); - } else { - log.trace("request password: (set)"); - } - log.trace("request client_id: {}", keycloakConfig.getKeycloakClient()); - if (keycloakConfig.getKeycloakClientSecret().isEmpty() || keycloakConfig.getKeycloakClientSecret().isBlank()) { - log.warn("request client_secret: (empty)"); - } else { - log.trace("request client_secret: (set)"); - } final ResponseEntity<TokenDto> response; try { response = new RestTemplate() diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MetadataMapper.java b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MetadataMapper.java index 884732bdc1a4be32da0d18a26498ca569ee7a2bb..0adfafa8f96c1c63dafea3a8edd6f45230f88a3d 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MetadataMapper.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MetadataMapper.java @@ -64,4 +64,22 @@ public interface MetadataMapper { IdentifierBriefDto identifierDtoToIdentifierBriefDto(IdentifierDto data); + default String metricToUri(String baseUrl, Long databaseId, Long tableId, Long subsetId, Long viewId) { + final StringBuilder uri = new StringBuilder(baseUrl) + .append("/database/") + .append(databaseId); + if (tableId != null) { + uri.append("/table/") + .append(tableId); + } else if (subsetId != null) { + uri.append("/subset/") + .append(subsetId); + } else if (viewId != null) { + uri.append("/view/") + .append(viewId); + } + log.trace("count uri: {}", uri); + return uri.toString(); + } + } diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/MetricsService.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/MetricsService.java new file mode 100644 index 0000000000000000000000000000000000000000..131bae7287292192931ea09e70cc8709512c9029 --- /dev/null +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/MetricsService.java @@ -0,0 +1,10 @@ +package at.tuwien.service; + +public interface MetricsService { + + void countTableGetData(Long databaseId, Long tableId); + + void countSubsetGetData(Long databaseId, Long subsetId); + + void countViewGetData(Long databaseId, Long viewId); +} diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/MetricsServicePrometheusImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/MetricsServicePrometheusImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..73754c9f5f2183d24f2576548b57a3f8a672b7a7 --- /dev/null +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/MetricsServicePrometheusImpl.java @@ -0,0 +1,47 @@ +package at.tuwien.service.impl; + +import at.tuwien.config.MetricsConfig; +import at.tuwien.mapper.MetadataMapper; +import at.tuwien.service.MetricsService; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Metrics; +import lombok.extern.log4j.Log4j2; +import org.springframework.stereotype.Service; + +@Log4j2 +@Service +public class MetricsServicePrometheusImpl implements MetricsService { + + private final MetricsConfig metricsConfig; + private final MetadataMapper metadataMapper; + + public MetricsServicePrometheusImpl(MetricsConfig metricsConfig, MetadataMapper metadataMapper) { + this.metricsConfig = metricsConfig; + this.metadataMapper = metadataMapper; + } + + @Override + public void countTableGetData(Long databaseId, Long tableId) { + countGetData(databaseId, tableId, null, null); + } + + @Override + public void countSubsetGetData(Long databaseId, Long subsetId) { + countGetData(databaseId, null, subsetId, null); + } + + @Override + public void countViewGetData(Long databaseId, Long viewId) { + countGetData(databaseId, null, null, viewId); + } + + public void countGetData(Long databaseId, Long tableId, Long subsetId, Long viewId) { + Counter.builder("dbrepo.datasource.data.get") + .tag("uri", metadataMapper.metricToUri(metricsConfig.getBaseUrl(), databaseId, tableId, subsetId, viewId)) + .tag("protocol", "http") + .description("The total number of accessed data sources") + .register(Metrics.globalRegistry) + .increment(); + } + +} diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java index aff85137870749a600d1e4c76ef317a978c6c54e..d5127e050e521b69e5130c4af836680ffaae013d 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java @@ -6,7 +6,6 @@ import at.tuwien.mapper.DataMapper; import at.tuwien.mapper.MetadataMapper; import at.tuwien.service.QueueService; import com.mchange.v2.c3p0.ComboPooledDataSource; -import io.micrometer.core.instrument.Counter; import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -21,14 +20,11 @@ import java.util.Optional; @Service public class QueueServiceRabbitMqImpl extends HibernateConnector implements QueueService { - private final Counter amqpDataAccessCounter; private final DataMapper dataMapper; private final MetadataMapper metadataMapper; @Autowired - public QueueServiceRabbitMqImpl(Counter amqpDataAccessCounter, DataMapper dataMapper, - MetadataMapper metadataMapper) { - this.amqpDataAccessCounter = amqpDataAccessCounter; + public QueueServiceRabbitMqImpl(DataMapper dataMapper, MetadataMapper metadataMapper) { this.dataMapper = dataMapper; this.metadataMapper = metadataMapper; } @@ -54,7 +50,6 @@ public class QueueServiceRabbitMqImpl extends HibernateConnector implements Queu preparedStatement.executeUpdate(); log.trace("executed statement in {} ms", System.currentTimeMillis() - start); log.trace("successfully inserted tuple"); - amqpDataAccessCounter.increment(); } finally { dataSource.close(); } diff --git a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java index 98c77c7b0d64d8ced9436ae67cda9fca73bf07b1..f63c387c9865079aca8eb8a3dedf74dcccdc3bba 100644 --- a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java +++ b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java @@ -483,7 +483,6 @@ public interface MetadataMapper { @Mappings({ @Mapping(target = "databaseId", source = "tdbid"), - @Mapping(target = "isPublic", source = "database.isPublic"), }) TableBriefDto tableToTableBriefDto(Table data); @@ -535,7 +534,7 @@ public interface MetadataMapper { .internalName(data.getInternalName()) .owner(userToUserBriefDto(data.getOwner())) .tdbid(data.getTdbid()) - .isPublic(data.getDatabase().getIsPublic()) + .isPublic(data.getIsPublic()) .isSchemaPublic(data.getIsSchemaPublic()) .isVersioned(true) .description(data.getDescription()) @@ -720,7 +719,7 @@ public interface MetadataMapper { @Mappings({ @Mapping(target = "tableId", source = "table.id"), @Mapping(target = "databaseId", source = "table.database.id"), - @Mapping(target = "isPublic", source = "table.database.isPublic"), + @Mapping(target = "isPublic", source = "table.isSchemaPublic"), @Mapping(target = "description", source = "description"), @Mapping(target = "table", ignore = true), @Mapping(target = "views", ignore = true) diff --git a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java index 3dbc450735cc1db936d1351cf89087411ee7dba8..a517c39d10f9c0463a756239c40fa14818d3a6b2 100644 --- a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java +++ b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java @@ -403,7 +403,8 @@ public class TableEndpoint { @NotNull Principal principal) throws NotAllowedException, DataServiceException, DataServiceConnectionException, DatabaseNotFoundException, TableNotFoundException, SearchServiceException, SearchServiceConnectionException { - log.debug("endpoint update table, databaseId={}, data.is_public={}", databaseId, data.getIsPublic()); + log.debug("endpoint update table, databaseId={}, data.is_public={}, data.is_schema_public={}", databaseId, + data.getIsPublic(), data.getIsSchemaPublic()); final Table table = tableService.findById(databaseId, tableId); if (!table.getOwner().equals(principal)) { log.error("Failed to update table: not owner"); diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java index 4a4a9ccaca7dedfc88f15db944c474cc5882e400..57b546fcea0b9cfe558ab935022b79d78c83edca 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java @@ -220,8 +220,8 @@ public class TableServiceImpl implements TableService { } final Table tableEntity = optional.get(); tableEntity.setIsPublic(data.getIsPublic()); - tableEntity.setDescription(data.getDescription()); tableEntity.setIsSchemaPublic(data.getIsSchemaPublic()); + tableEntity.setDescription(data.getDescription()); final Database database = databaseRepository.save(table.getDatabase()); /* update in search service */ searchServiceGateway.update(database); diff --git a/dbrepo-ui/components/subset/SubsetList.vue b/dbrepo-ui/components/subset/SubsetList.vue index b977daffa5aaf71dbe2980b98761244e8a3627b0..f57dc68a88fd77c06b858b423565b4ccac7d46f0 100644 --- a/dbrepo-ui/components/subset/SubsetList.vue +++ b/dbrepo-ui/components/subset/SubsetList.vue @@ -26,6 +26,21 @@ :to="link(item)" :href="link(item)"> <template v-slot:append> + <v-chip + v-if="database.is_public" + size="small" + class="ml-2" + color="success" + :text="$t('toolbars.database.public')" + variant="outlined" /> + <v-chip + v-if="!database.is_public" + size="small" + class="ml-2" + :color="colorVariant" + variant="outlined" + :text="$t('toolbars.database.private')" + flat /> <v-tooltip v-if="hasPublishedIdentifier(item)" :text="$t('pages.identifier.pid.title')" @@ -65,6 +80,15 @@ export default { }, database () { return this.cacheStore.getDatabase + }, + isContrastTheme () { + return this.$vuetify.theme.global.name.toLowerCase().endsWith('contrast') + }, + isDarkTheme () { + return this.$vuetify.theme.global.name.toLowerCase().startsWith('dark') + }, + colorVariant () { + return this.isContrastTheme ? '' : (this.isDarkTheme ? 'tertiary' : 'secondary') } }, mounted () { @@ -88,25 +112,25 @@ export default { toast.error(this.$t(code)) }) }, - title (query) { - if (query.identifiers.length === 0) { - return formatTimestampUTCLabel(query.created) + title (subset) { + if (subset.identifiers.length === 0) { + return subset.query } const identifierService = useIdentifierService() - return identifierService.identifierPreferEnglishTitle(query.identifiers[0]) + return identifierService.identifierPreferEnglishTitle(subset.identifiers[0]) }, - subtitle (query) { - if (query.identifiers.length === 0) { + subtitle (subset) { + if (subset.identifiers.length === 0) { return null } const identifierService = useIdentifierService() - return identifierService.identifierPreferEnglishDescription(query.identifiers[0]) + return identifierService.identifierPreferEnglishDescription(subset.identifiers[0]) }, - link (query) { - return `/database/${this.$route.params.database_id}/subset/${query.id}/info` + link (subset) { + return `/database/${this.$route.params.database_id}/subset/${subset.id}/info` }, - clazz (view) { - return this.hasPublishedIdentifier(view) ? 'primary-text' : null + clazz (subset) { + return this.hasPublishedIdentifier(subset) ? 'primary-text' : null }, hasPublishedIdentifier (subset) { if (!subset.identifiers) { diff --git a/dbrepo-ui/components/view/ViewList.vue b/dbrepo-ui/components/view/ViewList.vue index 6fe84519032aee28de5b02b62289195328cbaa1a..543a8746affd8cbbed495647fc963c8db1534072 100644 --- a/dbrepo-ui/components/view/ViewList.vue +++ b/dbrepo-ui/components/view/ViewList.vue @@ -61,9 +61,6 @@ export default { } }, computed: { - loadingColor () { - return this.error ? 'red lighten-2' : 'primary' - }, user () { return this.userStore.getUser }, diff --git a/dbrepo-ui/components/view/ViewToolbar.vue b/dbrepo-ui/components/view/ViewToolbar.vue index 1f122cc7a49a92ff9e7d71db3620f204c227d7b8..64ea3f1029407e3143bc6dd9b5cdfcb634aa9b94 100644 --- a/dbrepo-ui/components/view/ViewToolbar.vue +++ b/dbrepo-ui/components/view/ViewToolbar.vue @@ -187,7 +187,7 @@ export default { return this.access.type === 'read' || this.access.type === 'write_own' || this.access.type === 'write_all' }, canReadData () { - if (!this.view) { + if (!this.cachedView) { return false } if (this.cachedView.is_public) { diff --git a/dbrepo-ui/pages/database/[database_id]/view/[view_id]/data.vue b/dbrepo-ui/pages/database/[database_id]/view/[view_id]/data.vue index 6aed9307c4f9d8cbebfd6bc85d0a80be40405f63..9751fce5b7843b27043ea9e103003a01a3dcfbb4 100644 --- a/dbrepo-ui/pages/database/[database_id]/view/[view_id]/data.vue +++ b/dbrepo-ui/pages/database/[database_id]/view/[view_id]/data.vue @@ -96,7 +96,7 @@ export default { return this.access.type === 'read' || this.access.type === 'write_own' || this.access.type === 'write_all' }, canReadData () { - if (!this.view) { + if (!this.cachedView) { return false } if (this.cachedView.is_public) {