From f749bb65826e33aa9c7065e0854f34568279abdb Mon Sep 17 00:00:00 2001 From: Martin Weise <martin.weise@tuwien.ac.at> Date: Mon, 7 Oct 2024 08:41:41 +0200 Subject: [PATCH] Removed date --- .docker/config/dbrepo.conf | 37 +- .docker/config/rabbitmq.conf | 1 - .docker/docker-compose.yml | 3 +- .docs/api/broker-service.md | 2 +- dbrepo-broker-service/rabbitmq.conf | 1 - dbrepo-dashboard-service/Dockerfile | 2 +- .../dashboards/system.json | 458 +++++++++++++++++- .../at/tuwien/endpoints/SubsetEndpoint.java | 7 +- .../src/main/resources/init/querystore.sql | 2 +- .../src/test/resources/init/querystore.sql | 2 +- .../impl/SubsetServiceMariaDbImpl.java | 8 +- dbrepo-gateway-service/dbrepo.conf | 13 +- dbrepo-metadata-db/1_setup-schema.sql | 178 ++++--- .../table/columns/ColumnCreateDto.java | 3 - .../container/image/ContainerImage.java | 3 - .../container/image/ContainerImageDate.java | 59 --- .../image/ContainerImageDateKey.java | 14 - .../tuwien/entities/database/ViewColumn.java | 6 - .../database/table/columns/TableColumn.java | 6 - .../src/main/resources/application-local.yml | 2 +- .../DatabaseServicePersistenceTest.java | 4 - .../service/TableServicePersistenceTest.java | 3 - .../tuwien/service/TableServiceUnitTest.java | 1 - .../validator/EndpointValidatorUnitTest.java | 37 -- .../tuwien/service/impl/TableServiceImpl.java | 16 - .../main/java/at/tuwien/test/BaseTest.java | 129 +---- dbrepo-metric-db/prometheus.yml | 8 +- dbrepo-ui/components/subset/Results.vue | 1 + dbrepo-ui/nuxt.config.ts | 4 - .../[database_id]/table/[table_id]/data.vue | 1 + dbrepo-ui/pages/semantic/index.vue | 5 +- docker-compose.yml | 8 +- 32 files changed, 610 insertions(+), 414 deletions(-) delete mode 100644 dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImageDate.java delete mode 100644 dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImageDateKey.java diff --git a/.docker/config/dbrepo.conf b/.docker/config/dbrepo.conf index 8ac239d464..7c98728994 100644 --- a/.docker/config/dbrepo.conf +++ b/.docker/config/dbrepo.conf @@ -1,3 +1,9 @@ +# This is required to proxy Grafana Live WebSocket connections. +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + client_max_body_size 20G; resolver 127.0.0.11 valid=30s; # docker dns @@ -34,17 +40,32 @@ upstream upload { server upload-service:8080; } +upstream dashboard-service { + server dashboard-service:3000; +} + server { listen 80 default_server; server_name _; - location /admin/broker { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass http://broker; - proxy_read_timeout 90; + location /dashboard { + rewrite ^/dashboard/(.*) /$1 break; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://dashboard-service; + proxy_read_timeout 90; + } + + # Proxy Grafana Live WebSocket connections. + location /dashboard/api/live { + rewrite ^/dashboard/(.*) /$1 break; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + proxy_pass http://dashboard-service; } location /api/search { @@ -57,7 +78,7 @@ server { } location /api/broker { - rewrite /api/broker/(.*) /admin/broker/api/$1 break; + rewrite /api/broker/(.*) /api/$1 break; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/.docker/config/rabbitmq.conf b/.docker/config/rabbitmq.conf index 635c0476f6..8503942950 100644 --- a/.docker/config/rabbitmq.conf +++ b/.docker/config/rabbitmq.conf @@ -6,7 +6,6 @@ default_user_tags.administrator = false listeners.tcp.1 = 0.0.0.0:5672 # management prefix (https://www.rabbitmq.com/management.html#path-prefix) -management.path_prefix = /admin/broker management.load_definitions = /app/definitions.json # logging diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml index 1c6d964357..87dab50c91 100644 --- a/.docker/docker-compose.yml +++ b/.docker/docker-compose.yml @@ -8,6 +8,7 @@ volumes: storage-service-data: identity-service-data: metric-db-data: + dashboard-service-data: services: dbrepo-metadata-db: @@ -473,7 +474,7 @@ services: BROKER_HOST: "${BROKER_ENDPOINT:-broker-service}" BROKER_PASSWORD: "${SYSTEM_PASSWORD:-admin}" BROKER_PORT: ${BROKER_PORT:-5672} - BROKER_SERVICE_ENDPOINT: "${BROKER_SERVICE_ENDPOINT:-http://gateway-service/admin/broker}" + BROKER_SERVICE_ENDPOINT: "${BROKER_SERVICE_ENDPOINT:-http://broker-service:15672}" BROKER_USERNAME: "${SYSTEM_USERNAME:-admin}" BROKER_VIRTUALHOST: "${BROKER_VIRTUALHOST:-dbrepo}" CONNECTION_TIMEOUT: ${CONNECTION_TIMEOUT:-60000} diff --git a/.docs/api/broker-service.md b/.docs/api/broker-service.md index e923ff9b9f..41e2dc10b1 100644 --- a/.docs/api/broker-service.md +++ b/.docs/api/broker-service.md @@ -11,7 +11,7 @@ author: Martin Weise * Ports: 5672/tcp, 15672/tcp, 15692/tcp * AMQP: `amqp://<hostname>:5672` * Prometheus: `http://<hostname>:15692/metrics` - * Management: `http://<hostname>/admin/broker` + * Management: `http://<hostname>:15672` ## Overview diff --git a/dbrepo-broker-service/rabbitmq.conf b/dbrepo-broker-service/rabbitmq.conf index 635c0476f6..8503942950 100644 --- a/dbrepo-broker-service/rabbitmq.conf +++ b/dbrepo-broker-service/rabbitmq.conf @@ -6,7 +6,6 @@ default_user_tags.administrator = false listeners.tcp.1 = 0.0.0.0:5672 # management prefix (https://www.rabbitmq.com/management.html#path-prefix) -management.path_prefix = /admin/broker management.load_definitions = /app/definitions.json # logging diff --git a/dbrepo-dashboard-service/Dockerfile b/dbrepo-dashboard-service/Dockerfile index 00fc60339b..3c1f201870 100644 --- a/dbrepo-dashboard-service/Dockerfile +++ b/dbrepo-dashboard-service/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/bitnami/grafana:10.4.9-debian-12-r0 AS runtime +FROM docker.io/grafana/grafana-oss:11.2.2 AS runtime LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at" WORKDIR /app diff --git a/dbrepo-dashboard-service/dashboards/system.json b/dbrepo-dashboard-service/dashboards/system.json index 4d5d4e4b00..a1414496c1 100644 --- a/dbrepo-dashboard-service/dashboards/system.json +++ b/dbrepo-dashboard-service/dashboards/system.json @@ -18,7 +18,6 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": 3, "links": [ { "asDropdown": false, @@ -93,7 +92,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.9", + "pluginVersion": "11.2.2", "targets": [ { "datasource": { @@ -161,7 +160,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.9", + "pluginVersion": "11.2.2", "targets": [ { "datasource": { @@ -277,7 +276,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.9", + "pluginVersion": "11.2.2", "targets": [ { "datasource": { @@ -377,7 +376,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.9", + "pluginVersion": "11.2.2", "targets": [ { "datasource": { @@ -539,7 +538,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.9", + "pluginVersion": "11.2.2", "targets": [ { "datasource": { @@ -578,6 +577,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -588,7 +588,7 @@ }, "insertNulls": false, "lineInterpolation": "smooth", - "lineWidth": 1, + "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -743,6 +743,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -752,8 +753,8 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -884,6 +885,443 @@ ], "title": "JVM Memory Usage", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P18F45E9DC7E75912" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*search-service.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*analyse-service.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "super-light-orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P18F45E9DC7E75912" + }, + "editorMode": "code", + "expr": "rate(flask_http_request_duration_seconds_count{status=~\"200|201|202\",path!=\"/health\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{method}} {{instance}} {{path}} ({{status}})", + "range": true, + "refId": "A" + } + ], + "title": "Successful API Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P18F45E9DC7E75912" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*search-service.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*analyse-service.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "super-light-orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P18F45E9DC7E75912" + }, + "editorMode": "code", + "expr": "rate(flask_http_request_duration_seconds_count{status!~\"200|201|202\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{method}} {{instance}} ({{status}})", + "range": true, + "refId": "A" + } + ], + "title": "Failed API Requests", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "P18F45E9DC7E75912" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "DELETE" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "GET" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "HEAD" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PATCH" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OPTIONS" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "POST" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P18F45E9DC7E75912" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "tusd_requests_total", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Storage Service Requests", + "type": "timeseries" } ], "refresh": "10s", @@ -903,6 +1341,6 @@ "timezone": "browser", "title": "DBRepo - Overview", "uid": "bdz20owu8zn5se", - "version": 20, + "version": 1, "weekStart": "" } \ No newline at end of file diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java index 87471c74c2..1637878a67 100644 --- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java +++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java @@ -250,12 +250,7 @@ public class SubsetEndpoint { endpointValidator.validateDataParams(page, size); endpointValidator.validateForbiddenStatements(data.getStatement()); /* parameters */ - final UUID userId; - if (principal == null) { - userId = metadataServiceGateway.getSystemUserId(); - } else { - userId = UserUtil.getId(principal); - } + final UUID userId = principal != null ? UserUtil.getId(principal) : null; if (page == null) { page = 0L; log.debug("page not set: default to {}", page); diff --git a/dbrepo-data-service/rest-service/src/main/resources/init/querystore.sql b/dbrepo-data-service/rest-service/src/main/resources/init/querystore.sql index c1df44d1b0..3e7471df3e 100644 --- a/dbrepo-data-service/rest-service/src/main/resources/init/querystore.sql +++ b/dbrepo-data-service/rest-service/src/main/resources/init/querystore.sql @@ -1,5 +1,5 @@ CREATE SEQUENCE `qs_queries_seq` NOCACHE; -CREATE TABLE `qs_queries` ( `id` bigint not null primary key default nextval(`qs_queries_seq`), `created` datetime not null default now(), `executed` datetime not null default now(), `created_by` varchar(36) not null, `query` text not null, `query_normalized` text not null, `is_persisted` boolean not null, `query_hash` varchar(255) not null, `result_hash` varchar(255), `result_number` bigint ) WITH SYSTEM VERSIONING; +CREATE TABLE `qs_queries` ( `id` bigint not null primary key default nextval(`qs_queries_seq`), `created` datetime not null default now(), `executed` datetime not null default now(), `created_by` varchar(36), `query` text not null, `query_normalized` text not null, `is_persisted` boolean not null, `query_hash` varchar(255) not null, `result_hash` varchar(255), `result_number` bigint ) WITH SYSTEM VERSIONING; CREATE PROCEDURE hash_table(IN name VARCHAR(255), OUT hash VARCHAR(255), OUT count BIGINT) BEGIN DECLARE _sql TEXT; SELECT CONCAT('SELECT SHA2(GROUP_CONCAT(CONCAT_WS(\'\',', GROUP_CONCAT(CONCAT('`', column_name, '`') ORDER BY column_name), ') SEPARATOR \',\'), 256) AS hash, COUNT(*) AS count FROM `', name, '` INTO @hash, @count;') FROM `information_schema`.`columns` WHERE `table_schema` = DATABASE() AND `table_name` = name INTO _sql; PREPARE stmt FROM _sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; SET hash = @hash; SET count = @count; END; CREATE PROCEDURE store_query(IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _username varchar(255) DEFAULT REGEXP_REPLACE(current_user(), '@.*', ''); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END; CREATE DEFINER = 'root' PROCEDURE _store_query(IN _username VARCHAR(255), IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END; \ No newline at end of file diff --git a/dbrepo-data-service/rest-service/src/test/resources/init/querystore.sql b/dbrepo-data-service/rest-service/src/test/resources/init/querystore.sql index c1df44d1b0..3e7471df3e 100644 --- a/dbrepo-data-service/rest-service/src/test/resources/init/querystore.sql +++ b/dbrepo-data-service/rest-service/src/test/resources/init/querystore.sql @@ -1,5 +1,5 @@ CREATE SEQUENCE `qs_queries_seq` NOCACHE; -CREATE TABLE `qs_queries` ( `id` bigint not null primary key default nextval(`qs_queries_seq`), `created` datetime not null default now(), `executed` datetime not null default now(), `created_by` varchar(36) not null, `query` text not null, `query_normalized` text not null, `is_persisted` boolean not null, `query_hash` varchar(255) not null, `result_hash` varchar(255), `result_number` bigint ) WITH SYSTEM VERSIONING; +CREATE TABLE `qs_queries` ( `id` bigint not null primary key default nextval(`qs_queries_seq`), `created` datetime not null default now(), `executed` datetime not null default now(), `created_by` varchar(36), `query` text not null, `query_normalized` text not null, `is_persisted` boolean not null, `query_hash` varchar(255) not null, `result_hash` varchar(255), `result_number` bigint ) WITH SYSTEM VERSIONING; CREATE PROCEDURE hash_table(IN name VARCHAR(255), OUT hash VARCHAR(255), OUT count BIGINT) BEGIN DECLARE _sql TEXT; SELECT CONCAT('SELECT SHA2(GROUP_CONCAT(CONCAT_WS(\'\',', GROUP_CONCAT(CONCAT('`', column_name, '`') ORDER BY column_name), ') SEPARATOR \',\'), 256) AS hash, COUNT(*) AS count FROM `', name, '` INTO @hash, @count;') FROM `information_schema`.`columns` WHERE `table_schema` = DATABASE() AND `table_name` = name INTO _sql; PREPARE stmt FROM _sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; SET hash = @hash; SET count = @count; END; CREATE PROCEDURE store_query(IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _username varchar(255) DEFAULT REGEXP_REPLACE(current_user(), '@.*', ''); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END; CREATE DEFINER = 'root' PROCEDURE _store_query(IN _username VARCHAR(255), IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END; \ No newline at end of file diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java index e691bee25e..8bfdc0089a 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java @@ -17,8 +17,8 @@ import at.tuwien.gateway.MetadataServiceGateway; import at.tuwien.mapper.DataMapper; import at.tuwien.mapper.MariaDbMapper; import at.tuwien.mapper.MetadataMapper; -import at.tuwien.service.SubsetService; import at.tuwien.service.StorageService; +import at.tuwien.service.SubsetService; import com.mchange.v2.c3p0.ComboPooledDataSource; import io.micrometer.core.instrument.Counter; import lombok.extern.log4j.Log4j2; @@ -287,7 +287,11 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs /* insert query into query store */ final long start = System.currentTimeMillis(); final CallableStatement callableStatement = connection.prepareCall(mariaDbMapper.queryStoreStoreQueryRawQuery()); - callableStatement.setString(1, String.valueOf(userId)); + if (userId != null) { + callableStatement.setString(1, String.valueOf(userId)); + } else { + callableStatement.setNull(1, Types.VARCHAR); + } callableStatement.setString(2, query); callableStatement.setTimestamp(3, Timestamp.from(timestamp)); callableStatement.registerOutParameter(4, Types.BIGINT); diff --git a/dbrepo-gateway-service/dbrepo.conf b/dbrepo-gateway-service/dbrepo.conf index 49e5ce6496..e10a65bfae 100644 --- a/dbrepo-gateway-service/dbrepo.conf +++ b/dbrepo-gateway-service/dbrepo.conf @@ -48,17 +48,7 @@ server { listen 80 default_server; server_name _; - location /admin/broker { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass http://broker; - proxy_read_timeout 90; - } - location /dashboard { - rewrite ^/dashboard/(.*) /$1 break; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -69,7 +59,6 @@ server { # Proxy Grafana Live WebSocket connections. location /dashboard/api/live { - rewrite ^/dashboard/(.*) /$1 break; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; @@ -87,7 +76,7 @@ server { } location /api/broker { - rewrite /api/broker/(.*) /admin/broker/api/$1 break; + rewrite /api/broker/(.*) /api/$1 break; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/dbrepo-metadata-db/1_setup-schema.sql b/dbrepo-metadata-db/1_setup-schema.sql index 80ff2240a1..7e5caf6b6f 100644 --- a/dbrepo-metadata-db/1_setup-schema.sql +++ b/dbrepo-metadata-db/1_setup-schema.sql @@ -53,8 +53,7 @@ CREATE TABLE IF NOT EXISTS `mdb_containers` privileged_username character varying(255) NOT NULL, privileged_password character varying(255) NOT NULL, quota integer NOT NULL DEFAULT 50, - PRIMARY KEY (id), - FOREIGN KEY (image_id) REFERENCES mdb_images (id) + PRIMARY KEY (id) ) WITH SYSTEM VERSIONING; CREATE TABLE IF NOT EXISTS `mdb_data` @@ -80,7 +79,7 @@ CREATE TABLE IF NOT EXISTS `mdb_licenses` CREATE TABLE IF NOT EXISTS `mdb_databases` ( id SERIAL, - cid bigint NOT NULL, + cid BIGINT UNSIGNED NOT NULL, name character varying(255) NOT NULL, internal_name character varying(255) NOT NULL, exchange_name character varying(255) NOT NULL, @@ -94,7 +93,7 @@ CREATE TABLE IF NOT EXISTS `mdb_databases` created timestamp NOT NULL DEFAULT NOW(), last_modified timestamp, PRIMARY KEY (id), - FOREIGN KEY (cid) REFERENCES mdb_containers (id) /* currently we only support one-to-one */, + FOREIGN KEY (cid) REFERENCES mdb_containers (id), FOREIGN KEY (created_by) REFERENCES mdb_users (id), FOREIGN KEY (owned_by) REFERENCES mdb_users (id), FOREIGN KEY (contact_person) REFERENCES mdb_users (id) @@ -110,7 +109,7 @@ CREATE TABLE IF NOT EXISTS `mdb_databases_subjects` CREATE TABLE IF NOT EXISTS `mdb_tables` ( ID SERIAL, - tDBID bigint NOT NULL, + tDBID BIGINT UNSIGNED NOT NULL, tName VARCHAR(64) NOT NULL, internal_name VARCHAR(64) NOT NULL, queue_name VARCHAR(255) NOT NULL, @@ -141,26 +140,25 @@ CREATE TABLE IF NOT EXISTS `mdb_tables` CREATE TABLE IF NOT EXISTS `mdb_columns` ( - ID BIGINT NOT NULL AUTO_INCREMENT, - tID BIGINT NOT NULL, - dfID BIGINT, + ID SERIAL, + tID BIGINT UNSIGNED NOT NULL, cName VARCHAR(64), - internal_name VARCHAR(64) NOT NULL, + internal_name VARCHAR(64) NOT NULL, Datatype ENUM ('CHAR','VARCHAR','BINARY','VARBINARY','TINYBLOB','TINYTEXT','TEXT','BLOB','MEDIUMTEXT','MEDIUMBLOB','LONGTEXT','LONGBLOB','ENUM','SET','BIT','TINYINT','BOOL','SMALLINT','MEDIUMINT','INT','BIGINT','FLOAT','DOUBLE','DECIMAL','DATE','DATETIME','TIMESTAMP','TIME','YEAR'), - length BIGINT NULL, - ordinal_position INTEGER NOT NULL, - index_length BIGINT NULL, + length BIGINT UNSIGNED NULL, + ordinal_position INTEGER NOT NULL, + index_length BIGINT UNSIGNED NULL, description VARCHAR(2048), - size BIGINT, - d BIGINT, - auto_generated BOOLEAN DEFAULT false, - is_null_allowed BOOLEAN NOT NULL DEFAULT true, - val_min NUMERIC NULL, - val_max NUMERIC NULL, - mean NUMERIC NULL, - median NUMERIC NULL, - std_dev Numeric NULL, - created timestamp NOT NULL DEFAULT NOW(), + size BIGINT UNSIGNED, + d BIGINT UNSIGNED, + auto_generated BOOLEAN DEFAULT false, + is_null_allowed BOOLEAN NOT NULL DEFAULT true, + val_min NUMERIC NULL, + val_max NUMERIC NULL, + mean NUMERIC NULL, + median NUMERIC NULL, + std_dev Numeric NULL, + created timestamp NOT NULL DEFAULT NOW(), last_modified timestamp, FOREIGN KEY (tID) REFERENCES mdb_tables (ID) ON DELETE CASCADE, PRIMARY KEY (ID), @@ -170,7 +168,7 @@ CREATE TABLE IF NOT EXISTS `mdb_columns` CREATE TABLE IF NOT EXISTS `mdb_columns_enums` ( id SERIAL, - column_id bigint NOT NULL, + column_id BIGINT UNSIGNED NOT NULL, value CHARACTER VARYING(255) NOT NULL, FOREIGN KEY (column_id) REFERENCES mdb_columns (ID) ON DELETE CASCADE, PRIMARY KEY (id) @@ -179,7 +177,7 @@ CREATE TABLE IF NOT EXISTS `mdb_columns_enums` CREATE TABLE IF NOT EXISTS `mdb_columns_sets` ( id SERIAL, - column_id bigint NOT NULL, + column_id BIGINT UNSIGNED NOT NULL, value CHARACTER VARYING(255) NOT NULL, FOREIGN KEY (column_id) REFERENCES mdb_columns (ID) ON DELETE CASCADE, PRIMARY KEY (id) @@ -187,8 +185,8 @@ CREATE TABLE IF NOT EXISTS `mdb_columns_sets` CREATE TABLE IF NOT EXISTS `mdb_columns_nom` ( - cID bigint, - tID bigint, + cID BIGINT UNSIGNED, + tID BIGINT UNSIGNED, maxlength INTEGER, last_modified timestamp, created timestamp NOT NULL DEFAULT NOW(), @@ -198,8 +196,8 @@ CREATE TABLE IF NOT EXISTS `mdb_columns_nom` CREATE TABLE IF NOT EXISTS `mdb_columns_cat` ( - cID bigint, - tID bigint, + cID BIGINT UNSIGNED, + tID BIGINT UNSIGNED, num_cat INTEGER, -- cat_array TEXT[], last_modified timestamp, @@ -210,13 +208,13 @@ CREATE TABLE IF NOT EXISTS `mdb_columns_cat` CREATE TABLE IF NOT EXISTS `mdb_constraints_foreign_key` ( - fkid BIGINT NOT NULL AUTO_INCREMENT, - tid BIGINT NOT NULL, - rtid BIGINT NOT NULL, - name VARCHAR(255) NOT NULL, - on_update VARCHAR(50) NULL, - on_delete VARCHAR(50) NULL, - position INT NULL, + fkid BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + tid BIGINT UNSIGNED NOT NULL, + rtid BIGINT UNSIGNED NOT NULL, + name VARCHAR(255) NOT NULL, + on_update VARCHAR(50) NULL, + on_delete VARCHAR(50) NULL, + position INT NULL, PRIMARY KEY (fkid), FOREIGN KEY (tid) REFERENCES mdb_tables (id) ON DELETE CASCADE, FOREIGN KEY (rtid) REFERENCES mdb_tables (id) @@ -224,9 +222,9 @@ CREATE TABLE IF NOT EXISTS `mdb_constraints_foreign_key` CREATE TABLE IF NOT EXISTS `mdb_constraints_primary_key` ( - pkid BIGINT NOT NULL AUTO_INCREMENT, - tID BIGINT NOT NULL, - cid BIGINT NOT NULL, + pkid SERIAL, + tID BIGINT UNSIGNED NOT NULL, + cid BIGINT UNSIGNED NOT NULL, PRIMARY KEY (pkid), FOREIGN KEY (tID) REFERENCES mdb_tables (id) ON DELETE CASCADE, FOREIGN KEY (cid) REFERENCES mdb_columns (id) ON DELETE CASCADE @@ -234,10 +232,10 @@ CREATE TABLE IF NOT EXISTS `mdb_constraints_primary_key` CREATE TABLE IF NOT EXISTS `mdb_constraints_foreign_key_reference` ( - id BIGINT NOT NULL AUTO_INCREMENT, - fkid BIGINT NOT NULL, - cid BIGINT NOT NULL, - rcid BIGINT NOT NULL, + id SERIAL, + fkid BIGINT UNSIGNED NOT NULL, + cid BIGINT UNSIGNED NOT NULL, + rcid BIGINT UNSIGNED NOT NULL, PRIMARY KEY (id), UNIQUE (fkid, cid, rcid), FOREIGN KEY (fkid) REFERENCES mdb_constraints_foreign_key (fkid) ON UPDATE CASCADE, @@ -247,19 +245,19 @@ CREATE TABLE IF NOT EXISTS `mdb_constraints_foreign_key_reference` CREATE TABLE IF NOT EXISTS `mdb_constraints_unique` ( - uid BIGINT NOT NULL AUTO_INCREMENT, - name VARCHAR(255) NOT NULL, - tid BIGINT NOT NULL, - position INT NULL, + uid BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + name VARCHAR(255) NOT NULL, + tid BIGINT UNSIGNED NOT NULL, + position INT NULL, PRIMARY KEY (uid), FOREIGN KEY (tid) REFERENCES mdb_tables (id) ON DELETE CASCADE ); CREATE TABLE IF NOT EXISTS `mdb_constraints_unique_columns` ( - id BIGINT NOT NULL AUTO_INCREMENT, - uid BIGINT NOT NULL, - cid BIGINT NOT NULL, + id SERIAL, + uid BIGINT UNSIGNED NOT NULL, + cid BIGINT UNSIGNED NOT NULL, PRIMARY KEY (id), FOREIGN KEY (uid) REFERENCES mdb_constraints_unique (uid), FOREIGN KEY (cid) REFERENCES mdb_columns (id) ON DELETE CASCADE @@ -267,9 +265,9 @@ CREATE TABLE IF NOT EXISTS `mdb_constraints_unique_columns` CREATE TABLE IF NOT EXISTS `mdb_constraints_checks` ( - id BIGINT NOT NULL AUTO_INCREMENT, - tid BIGINT NOT NULL, - checks VARCHAR(255) NOT NULL, + id SERIAL, + tid BIGINT UNSIGNED NOT NULL, + checks VARCHAR(255) NOT NULL, PRIMARY KEY (id), FOREIGN KEY (tid) REFERENCES mdb_tables (id) ON DELETE CASCADE ) WITH SYSTEM VERSIONING; @@ -299,18 +297,18 @@ CREATE TABLE IF NOT EXISTS `mdb_units` CREATE TABLE IF NOT EXISTS `mdb_columns_concepts` ( - id bigint NOT NULL, - cID bigint NOT NULL, - created timestamp NOT NULL DEFAULT NOW(), + id BIGINT UNSIGNED NOT NULL, + cID BIGINT UNSIGNED NOT NULL, + created timestamp NOT NULL DEFAULT NOW(), PRIMARY KEY (id, cid), FOREIGN KEY (cID) REFERENCES mdb_columns (ID) ) WITH SYSTEM VERSIONING; CREATE TABLE IF NOT EXISTS `mdb_columns_units` ( - id bigint NOT NULL, - cID bigint NOT NULL, - created timestamp NOT NULL DEFAULT NOW(), + id BIGINT UNSIGNED NOT NULL, + cID BIGINT UNSIGNED NOT NULL, + created timestamp NOT NULL DEFAULT NOW(), PRIMARY KEY (id, cID), FOREIGN KEY (cID) REFERENCES mdb_columns (ID) ) WITH SYSTEM VERSIONING; @@ -318,7 +316,7 @@ CREATE TABLE IF NOT EXISTS `mdb_columns_units` CREATE TABLE IF NOT EXISTS `mdb_view` ( id SERIAL, - vdbid bigint NOT NULL, + vdbid BIGINT UNSIGNED NOT NULL, vName VARCHAR(64) NOT NULL, internal_name VARCHAR(64) NOT NULL, Query TEXT NOT NULL, @@ -362,28 +360,28 @@ CREATE TABLE IF NOT EXISTS `mdb_ontologies` CREATE TABLE IF NOT EXISTS `mdb_view_columns` ( - id BIGINT NOT NULL AUTO_INCREMENT, - view_id BIGINT NOT NULL, - dfID BIGINT, + id SERIAL, + view_id BIGINT UNSIGNED NOT NULL, + dfID BIGINT UNSIGNED, name VARCHAR(64), - internal_name VARCHAR(64) NOT NULL, + internal_name VARCHAR(64) NOT NULL, column_type ENUM ('CHAR','VARCHAR','BINARY','VARBINARY','TINYBLOB','TINYTEXT','TEXT','BLOB','MEDIUMTEXT','MEDIUMBLOB','LONGTEXT','LONGBLOB','ENUM','SET','BIT','TINYINT','BOOL','SMALLINT','MEDIUMINT','INT','BIGINT','FLOAT','DOUBLE','DECIMAL','DATE','DATETIME','TIMESTAMP','TIME','YEAR'), - ordinal_position INTEGER NOT NULL, - size BIGINT, - d BIGINT, - auto_generated BOOLEAN DEFAULT false, - is_null_allowed BOOLEAN NOT NULL DEFAULT true, + ordinal_position INTEGER NOT NULL, + size BIGINT UNSIGNED, + d BIGINT UNSIGNED, + auto_generated BOOLEAN DEFAULT false, + is_null_allowed BOOLEAN NOT NULL DEFAULT true, PRIMARY KEY (id), FOREIGN KEY (view_id) REFERENCES mdb_view (id) ) WITH SYSTEM VERSIONING; CREATE TABLE IF NOT EXISTS `mdb_identifiers` ( - id BIGINT NOT NULL AUTO_INCREMENT, - dbid BIGINT NOT NULL, - qid BIGINT, - vid BIGINT, - tid BIGINT, + id SERIAL, + dbid BIGINT UNSIGNED NOT NULL, + qid BIGINT UNSIGNED, + vid BIGINT UNSIGNED, + tid BIGINT UNSIGNED, publisher VARCHAR(255) NOT NULL, language VARCHAR(2), publication_year INTEGER NOT NULL, @@ -408,8 +406,8 @@ CREATE TABLE IF NOT EXISTS `mdb_identifiers` CREATE TABLE IF NOT EXISTS `mdb_identifier_licenses` ( - pid bigint NOT NULL, - license_id VARCHAR(255) NOT NULL, + pid BIGINT UNSIGNED NOT NULL, + license_id VARCHAR(255) NOT NULL, PRIMARY KEY (pid, license_id), FOREIGN KEY (pid) REFERENCES mdb_identifiers (id), FOREIGN KEY (license_id) REFERENCES mdb_licenses (identifier) @@ -418,8 +416,8 @@ CREATE TABLE IF NOT EXISTS `mdb_identifier_licenses` CREATE TABLE IF NOT EXISTS `mdb_identifier_titles` ( id SERIAL, - pid bigint NOT NULL, - title text NOT NULL, + pid BIGINT UNSIGNED NOT NULL, + title text NOT NULL, title_type ENUM ('ALTERNATIVE_TITLE', 'SUBTITLE', 'TRANSLATED_TITLE', 'OTHER'), language VARCHAR(2), PRIMARY KEY (id), @@ -429,8 +427,8 @@ CREATE TABLE IF NOT EXISTS `mdb_identifier_titles` CREATE TABLE IF NOT EXISTS `mdb_identifier_funders` ( id SERIAL, - pid bigint NOT NULL, - funder_name VARCHAR(255) NOT NULL, + pid BIGINT UNSIGNED NOT NULL, + funder_name VARCHAR(255) NOT NULL, funder_identifier TEXT, funder_identifier_type ENUM ('CROSSREF_FUNDER_ID', 'GRID', 'ISNI', 'ROR', 'OTHER'), scheme_uri text, @@ -444,8 +442,8 @@ CREATE TABLE IF NOT EXISTS `mdb_identifier_funders` CREATE TABLE IF NOT EXISTS `mdb_identifier_descriptions` ( id SERIAL, - pid bigint NOT NULL, - description text NOT NULL, + pid BIGINT UNSIGNED NOT NULL, + description text NOT NULL, description_type ENUM ('ABSTRACT', 'METHODS', 'SERIES_INFORMATION', 'TABLE_OF_CONTENTS', 'TECHNICAL_INFO', 'OTHER'), language VARCHAR(2), PRIMARY KEY (id), @@ -455,10 +453,10 @@ CREATE TABLE IF NOT EXISTS `mdb_identifier_descriptions` CREATE TABLE IF NOT EXISTS `mdb_related_identifiers` ( id SERIAL, - pid bigint NOT NULL, - value varchar(255) NOT NULL, - type varchar(255) NOT NULL, - relation varchar(255) NOT NULL, + pid BIGINT UNSIGNED NOT NULL, + value varchar(255) NOT NULL, + type varchar(255) NOT NULL, + relation varchar(255) NOT NULL, PRIMARY KEY (id), /* must be a single id from persistent identifier concept */ FOREIGN KEY (pid) REFERENCES mdb_identifiers (id), UNIQUE (pid, value) @@ -467,10 +465,10 @@ CREATE TABLE IF NOT EXISTS `mdb_related_identifiers` CREATE TABLE IF NOT EXISTS `mdb_identifier_creators` ( id SERIAL, - pid bigint NOT NULL, + pid BIGINT UNSIGNED NOT NULL, given_names text, family_name text, - creator_name VARCHAR(255) NOT NULL, + creator_name VARCHAR(255) NOT NULL, name_type ENUM ('PERSONAL', 'ORGANIZATIONAL') default 'PERSONAL', name_identifier text, name_identifier_scheme ENUM ('ROR', 'GRID', 'ISNI', 'ORCID'), @@ -486,7 +484,7 @@ CREATE TABLE IF NOT EXISTS `mdb_identifier_creators` CREATE TABLE IF NOT EXISTS `mdb_update` ( uUserID character varying(255) NOT NULL, - uDBID bigint NOT NULL, + uDBID BIGINT UNSIGNED NOT NULL, created timestamp NOT NULL DEFAULT NOW(), PRIMARY KEY (uUserID, uDBID), FOREIGN KEY (uDBID) REFERENCES mdb_databases (id) @@ -495,7 +493,7 @@ CREATE TABLE IF NOT EXISTS `mdb_update` CREATE TABLE IF NOT EXISTS `mdb_access` ( aUserID character varying(255) NOT NULL, - aDBID bigint REFERENCES mdb_databases (id), + aDBID BIGINT UNSIGNED REFERENCES mdb_databases (id), attime TIMESTAMP, download BOOLEAN, created timestamp NOT NULL DEFAULT NOW(), @@ -505,7 +503,7 @@ CREATE TABLE IF NOT EXISTS `mdb_access` CREATE TABLE IF NOT EXISTS `mdb_have_access` ( user_id character varying(36) NOT NULL, - database_id bigint REFERENCES mdb_databases (id), + database_id BIGINT UNSIGNED REFERENCES mdb_databases (id), access_type ENUM ('READ', 'WRITE_OWN', 'WRITE_ALL') NOT NULL, created timestamp NOT NULL DEFAULT NOW(), PRIMARY KEY (user_id, database_id), diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnCreateDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnCreateDto.java index 37aa493020..2d0696abc0 100644 --- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnCreateDto.java +++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnCreateDto.java @@ -52,9 +52,6 @@ public class ColumnCreateDto { @JsonProperty("unit_uri") private String unitUri; - @Schema(description = "date format id") - private Long dfid; - @Schema(description = "enum values, only considered when type = ENUM") private List<String> enums; diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImage.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImage.java index 8250b6ae43..9989b911ba 100644 --- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImage.java +++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImage.java @@ -57,9 +57,6 @@ public class ContainerImage { @Column(nullable = false, unique = true, columnDefinition = "BOOLEAN DEFAULT FALSE") private Boolean isDefault = false; - @OneToMany(fetch = FetchType.LAZY, cascade = {CascadeType.ALL, CascadeType.PERSIST}, mappedBy = "image") - private List<ContainerImageDate> dateFormats; - @ToString.Exclude @OneToMany(fetch = FetchType.LAZY, mappedBy = "image") private List<Container> containers; diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImageDate.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImageDate.java deleted file mode 100644 index 5b370ecc06..0000000000 --- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImageDate.java +++ /dev/null @@ -1,59 +0,0 @@ -package at.tuwien.entities.container.image; - -import com.fasterxml.jackson.annotation.JsonFormat; -import lombok.*; -import org.hibernate.annotations.GenericGenerator; -import org.springframework.data.annotation.CreatedDate; -import org.springframework.data.jpa.domain.support.AuditingEntityListener; - -import jakarta.persistence.*; - -import java.time.Instant; - -@Data -@Entity -@Builder -@ToString -@AllArgsConstructor -@NoArgsConstructor -@EntityListeners(AuditingEntityListener.class) -@EqualsAndHashCode(onlyExplicitlyIncluded = true) -@Table(name = "mdb_images_date", uniqueConstraints = @UniqueConstraint(columnNames = {"database_format"})) -public class ContainerImageDate { - - @Id - @EqualsAndHashCode.Include - @GeneratedValue(generator = "dates-sequence") - @GenericGenerator(name = "dates-sequence", strategy = "increment") - @Column(updatable = false, nullable = false) - private Long id; - - @EqualsAndHashCode.Include - @Column(name = "iid") - private Long iid; - - @ToString.Exclude - @ManyToOne(fetch = FetchType.LAZY, cascade = {CascadeType.MERGE}) - @JoinColumns({ - @JoinColumn(name = "iid", insertable = false, updatable = false) - }) - private ContainerImage image; - - @Column(name = "example", nullable = false) - private String example; - - @Column(name = "has_time", nullable = false) - private Boolean hasTime; - - @Column(name = "database_format", nullable = false) - private String databaseFormat; - - @Column(name = "unix_format", nullable = false) - private String unixFormat; - - @CreatedDate - @Column(name = "created_at", nullable = false, updatable = false, columnDefinition = "TIMESTAMP") - @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", timezone = "UTC") - private Instant createdAt; - -} diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImageDateKey.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImageDateKey.java deleted file mode 100644 index c5ae9598f7..0000000000 --- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImageDateKey.java +++ /dev/null @@ -1,14 +0,0 @@ -package at.tuwien.entities.container.image; - -import lombok.EqualsAndHashCode; - -import java.io.Serializable; - -@EqualsAndHashCode -public class ContainerImageDateKey implements Serializable { - - private Long id; - - private Long iid; - -} diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/ViewColumn.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/ViewColumn.java index ff18c0137d..b9eff2c694 100644 --- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/ViewColumn.java +++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/ViewColumn.java @@ -1,6 +1,5 @@ package at.tuwien.entities.database; -import at.tuwien.entities.container.image.ContainerImageDate; import at.tuwien.entities.database.table.columns.TableColumnType; import lombok.*; import org.hibernate.annotations.GenericGenerator; @@ -25,11 +24,6 @@ public class ViewColumn implements Comparable<ViewColumn> { @Column(updatable = false, nullable = false) private Long id; - @ToString.Exclude - @OneToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "dfid", referencedColumnName = "id") - private ContainerImageDate dateFormat; - @ToString.Exclude @ManyToOne(fetch = FetchType.LAZY, cascade = {CascadeType.MERGE}) @JoinColumns({ diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java index dd59e21003..9ddf89c453 100644 --- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java +++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java @@ -1,6 +1,5 @@ package at.tuwien.entities.database.table.columns; -import at.tuwien.entities.container.image.ContainerImageDate; import at.tuwien.entities.database.table.Table; import com.fasterxml.jackson.annotation.JsonFormat; import lombok.*; @@ -38,11 +37,6 @@ public class TableColumn implements Comparable<TableColumn> { @Column(updatable = false, nullable = false) private Long id; - @ToString.Exclude - @OneToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "dfid", referencedColumnName = "id") - private ContainerImageDate dateFormat; - @ToString.Exclude @ManyToOne(fetch = FetchType.LAZY, cascade = {CascadeType.MERGE}) @JoinColumns({ diff --git a/dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml b/dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml index e2c9df6a59..793a2b6021 100644 --- a/dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml +++ b/dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml @@ -66,7 +66,7 @@ dbrepo: searchService: http://localhost analyseService: http://localhost dataService: http://localhost:9093 - brokerService: http://localhost/admin/broker + brokerService: http://localhost:15672 authService: http://localhost/api/auth storageService: http://localhost/api/storage rorService: https://api.ror.org diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/DatabaseServicePersistenceTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/DatabaseServicePersistenceTest.java index b4bd67f88d..51b6df4d27 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/DatabaseServicePersistenceTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/DatabaseServicePersistenceTest.java @@ -82,8 +82,6 @@ public class DatabaseServicePersistenceTest extends AbstractUnitTest { assertEquals(IMAGE_1_DRIVER, response.getContainer().getImage().getDriverClass()); assertEquals(IMAGE_1_REGISTRY, response.getContainer().getImage().getRegistry()); assertEquals(IMAGE_1_PORT, response.getContainer().getImage().getDefaultPort()); - assertNotNull(response.getContainer().getImage().getDateFormats()); - assertEquals(4, response.getContainer().getImage().getDateFormats().size()); /* creator */ assertNotNull(response.getCreator()); assertEquals(USER_1_ID, response.getCreator().getId()); @@ -124,8 +122,6 @@ public class DatabaseServicePersistenceTest extends AbstractUnitTest { assertEquals(IMAGE_1_DRIVER, response.getContainer().getImage().getDriverClass()); assertEquals(IMAGE_1_REGISTRY, response.getContainer().getImage().getRegistry()); assertEquals(IMAGE_1_PORT, response.getContainer().getImage().getDefaultPort()); - assertNotNull(response.getContainer().getImage().getDateFormats()); - assertEquals(4, response.getContainer().getImage().getDateFormats().size()); /* creator */ assertNotNull(response.getCreator()); assertEquals(USER_1_ID, response.getCreator().getId()); diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServicePersistenceTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServicePersistenceTest.java index e2d7d33896..7aa22159c3 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServicePersistenceTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServicePersistenceTest.java @@ -92,7 +92,6 @@ public class TableServicePersistenceTest extends AbstractUnitTest { .name("date") .nullAllowed(true) .type(ColumnTypeDto.DATE) - .dfid(3L) .build())) .constraints(ConstraintsCreateDto.builder() .checks(Set.of()) @@ -124,8 +123,6 @@ public class TableServicePersistenceTest extends AbstractUnitTest { assertEquals("date", date.getName()); assertEquals("date", date.getInternalName()); assertEquals(TableColumnType.DATE, date.getColumnType()); - assertNotNull(date.getDateFormat()); - assertEquals(3L, date.getDateFormat().getId()); assertTrue(date.getIsNullAllowed()); assertNotNull(response.getConstraints()); final List<Unique> uniques = response.getConstraints().getUniques(); diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceUnitTest.java index 59419f9bad..db02454e70 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceUnitTest.java @@ -202,7 +202,6 @@ public class TableServiceUnitTest extends AbstractUnitTest { .name("date") .nullAllowed(true) .type(ColumnTypeDto.DATE) - .dfid(9999L) .build())) .constraints(ConstraintsCreateDto.builder() .checks(Set.of()) diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/validator/EndpointValidatorUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/validator/EndpointValidatorUnitTest.java index f6cf4e887d..66ae6777e9 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/validator/EndpointValidatorUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/validator/EndpointValidatorUnitTest.java @@ -72,14 +72,6 @@ public class EndpointValidatorUnitTest extends AbstractUnitTest { ); } - public static Stream<Arguments> needDateFormat_parameters() { - return Stream.of( - Arguments.arguments(ColumnTypeDto.DATETIME), - Arguments.arguments(ColumnTypeDto.TIMESTAMP), - Arguments.arguments(ColumnTypeDto.TIME) - ); - } - @BeforeEach public void beforeEach() { genesis(); @@ -355,35 +347,6 @@ public class EndpointValidatorUnitTest extends AbstractUnitTest { }); } - @ParameterizedTest - @MethodSource("needDateFormat_parameters") - public void validateColumnCreateConstraints_needDateFormat_fails(ColumnTypeDto type) { - final TableCreateDto request = TableCreateDto.builder() - .columns(List.of(ColumnCreateDto.builder() - .type(type) - .dfid(null) // <<<<<<< - .build())) - .build(); - - /* test */ - assertThrows(MalformedException.class, () -> { - endpointValidator.validateColumnCreateConstraints(request); - }); - } - - @Test - public void validateColumnCreateConstraints_dateFormatEmpty_succeeds() throws MalformedException { - final TableCreateDto request = TableCreateDto.builder() - .columns(List.of(ColumnCreateDto.builder() - .type(ColumnTypeDto.DATE) - .dfid(null) // <<<<<<< - .build())) - .build(); - - /* test */ - endpointValidator.validateColumnCreateConstraints(request); - } - @Test public void validateOnlyOwnerOrWriteAll_onlyReadAccess_fails() throws DatabaseNotFoundException, TableNotFoundException, AccessNotFoundException { diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java index f88ba5c28f..8d24691467 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java @@ -4,10 +4,8 @@ import at.tuwien.api.database.table.TableCreateDto; import at.tuwien.api.database.table.TableStatisticDto; import at.tuwien.api.database.table.columns.ColumnCreateDto; import at.tuwien.api.database.table.columns.ColumnStatisticDto; -import at.tuwien.api.database.table.columns.ColumnTypeDto; import at.tuwien.api.database.table.columns.concepts.ColumnSemanticsUpdateDto; import at.tuwien.config.RabbitConfig; -import at.tuwien.entities.container.image.ContainerImageDate; import at.tuwien.entities.database.Database; import at.tuwien.entities.database.table.Table; import at.tuwien.entities.database.table.columns.TableColumn; @@ -146,20 +144,6 @@ public class TableServiceImpl implements TableService { } column.setConcept(concept); } - if (List.of(TableColumnType.TIME, TableColumnType.TIMESTAMP, TableColumnType.DATE, TableColumnType.DATETIME).contains(column.getColumnType())) { - final Optional<ContainerImageDate> optional = database.getContainer() - .getImage() - .getDateFormats() - .stream() - .filter(df -> df.getId().equals(c.getDfid())) - .findFirst(); - if (optional.isEmpty()) { - log.error("Failed to find date format with id {} in metadata database", c.getDfid()); - throw new IllegalArgumentException("Failed to find date format in metadata database"); - } - column.setDateFormat(optional.get()); - log.debug("column is of temporal type: added date format with id {}", column.getDateFormat().getId()); - } table.getColumns() .add(column); } diff --git a/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java b/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java index adff51d164..f372879434 100644 --- a/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java +++ b/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java @@ -1,12 +1,18 @@ package at.tuwien.test; import at.tuwien.ExportResourceDto; -import at.tuwien.api.amqp.*; +import at.tuwien.api.amqp.CreateVirtualHostDto; +import at.tuwien.api.amqp.ExchangeDto; +import at.tuwien.api.amqp.GrantVirtualHostPermissionsDto; +import at.tuwien.api.amqp.QueueDto; import at.tuwien.api.auth.LoginRequestDto; import at.tuwien.api.auth.SignupRequestDto; import at.tuwien.api.container.ContainerBriefDto; import at.tuwien.api.container.ContainerDto; -import at.tuwien.api.container.image.*; +import at.tuwien.api.container.image.ImageBriefDto; +import at.tuwien.api.container.image.ImageChangeDto; +import at.tuwien.api.container.image.ImageCreateDto; +import at.tuwien.api.container.image.ImageDto; import at.tuwien.api.container.internal.PrivilegedContainerDto; import at.tuwien.api.database.*; import at.tuwien.api.database.internal.CreateDatabaseDto; @@ -20,7 +26,10 @@ import at.tuwien.api.database.table.TableCreateDto; import at.tuwien.api.database.table.TableDto; import at.tuwien.api.database.table.TableStatisticDto; import at.tuwien.api.database.table.columns.*; -import at.tuwien.api.database.table.columns.concepts.*; +import at.tuwien.api.database.table.columns.concepts.ConceptDto; +import at.tuwien.api.database.table.columns.concepts.ConceptSaveDto; +import at.tuwien.api.database.table.columns.concepts.UnitDto; +import at.tuwien.api.database.table.columns.concepts.UnitSaveDto; import at.tuwien.api.database.table.constraints.ConstraintsCreateDto; import at.tuwien.api.database.table.constraints.ConstraintsDto; import at.tuwien.api.database.table.constraints.foreign.*; @@ -51,11 +60,9 @@ import at.tuwien.api.orcid.person.name.OrcidValueDto; import at.tuwien.api.semantics.OntologyCreateDto; import at.tuwien.api.semantics.OntologyModifyDto; import at.tuwien.api.user.*; -import at.tuwien.api.user.UserDetailsDto; import at.tuwien.api.user.internal.UpdateUserPasswordDto; import at.tuwien.entities.container.Container; import at.tuwien.entities.container.image.ContainerImage; -import at.tuwien.entities.container.image.ContainerImageDate; import at.tuwien.entities.database.*; import at.tuwien.entities.database.table.Table; import at.tuwien.entities.database.table.columns.TableColumn; @@ -897,22 +904,6 @@ public abstract class BaseTest { public final static Integer IMAGE_1_PORT = 3306; public final static Boolean IMAGE_1_IS_DEFAULT = true; - public final static Long IMAGE_DATE_1_ID = 1L; - public final static Long IMAGE_DATE_1_IMAGE_ID = IMAGE_1_ID; - public final static String IMAGE_DATE_1_UNIX_FORMAT = "yyyy-MM-dd"; - public final static String IMAGE_DATE_1_DATABASE_FORMAT = "%Y-%c-%d"; - public final static String IMAGE_DATE_1_EXAMPLE = "2022-01-30"; - public final static Boolean IMAGE_DATE_1_HAS_TIME = false; - - public final static ContainerImageDate IMAGE_DATE_1 = ContainerImageDate.builder() - .id(IMAGE_DATE_1_ID) - .iid(IMAGE_DATE_1_IMAGE_ID) - .unixFormat(IMAGE_DATE_1_UNIX_FORMAT) - .databaseFormat(IMAGE_DATE_1_DATABASE_FORMAT) - .example(IMAGE_DATE_1_EXAMPLE) - .hasTime(IMAGE_DATE_1_HAS_TIME) - .build(); - public final static ImageCreateDto IMAGE_1_CREATE_DTO = ImageCreateDto.builder() .registry(IMAGE_1_REGISTRY) .name(IMAGE_1_NAME) @@ -931,54 +922,6 @@ public abstract class BaseTest { .defaultPort(IMAGE_1_PORT) .build(); - public final static Long IMAGE_DATE_2_ID = 2L; - public final static Long IMAGE_DATE_2_IMAGE_ID = IMAGE_1_ID; - public final static String IMAGE_DATE_2_UNIX_FORMAT = "dd.MM.yy"; - public final static String IMAGE_DATE_2_DATABASE_FORMAT = "%d.%c.%y"; - public final static String IMAGE_DATE_2_EXAMPLE = "30.01.2022"; - public final static Boolean IMAGE_DATE_2_HAS_TIME = false; - - public final static ContainerImageDate IMAGE_DATE_2 = ContainerImageDate.builder() - .id(IMAGE_DATE_2_ID) - .iid(IMAGE_DATE_2_IMAGE_ID) - .unixFormat(IMAGE_DATE_2_UNIX_FORMAT) - .databaseFormat(IMAGE_DATE_2_DATABASE_FORMAT) - .example(IMAGE_DATE_2_EXAMPLE) - .hasTime(IMAGE_DATE_2_HAS_TIME) - .build(); - - public final static Long IMAGE_DATE_3_ID = 3L; - public final static Long IMAGE_DATE_3_IMAGE_ID = IMAGE_1_ID; - public final static String IMAGE_DATE_3_UNIX_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSSSS"; - public final static String IMAGE_DATE_3_DATABASE_FORMAT = "%Y-%c-%dT%H:%i:%S.%f"; - public final static String IMAGE_DATE_3_EXAMPLE = "2022-01-30T13:44:25.499"; - public final static Boolean IMAGE_DATE_3_HAS_TIME = true; - - public final static ContainerImageDate IMAGE_DATE_3 = ContainerImageDate.builder() - .id(IMAGE_DATE_3_ID) - .iid(IMAGE_DATE_3_IMAGE_ID) - .unixFormat(IMAGE_DATE_3_UNIX_FORMAT) - .databaseFormat(IMAGE_DATE_3_DATABASE_FORMAT) - .example(IMAGE_DATE_3_EXAMPLE) - .hasTime(IMAGE_DATE_3_HAS_TIME) - .build(); - - public final static Long IMAGE_DATE_4_ID = 4L; - public final static Long IMAGE_DATE_4_IMAGE_ID = IMAGE_1_ID; - public final static String IMAGE_DATE_4_UNIX_FORMAT = "HH:mm:ss"; - public final static String IMAGE_DATE_4_DATABASE_FORMAT = "%H:%i:%S"; - public final static String IMAGE_DATE_4_EXAMPLE = "14:44:25"; - public final static Boolean IMAGE_DATE_4_HAS_TIME = true; - - public final static ContainerImageDate IMAGE_DATE_4 = ContainerImageDate.builder() - .id(IMAGE_DATE_4_ID) - .iid(IMAGE_DATE_4_IMAGE_ID) - .unixFormat(IMAGE_DATE_4_UNIX_FORMAT) - .databaseFormat(IMAGE_DATE_4_DATABASE_FORMAT) - .example(IMAGE_DATE_4_EXAMPLE) - .hasTime(IMAGE_DATE_4_HAS_TIME) - .build(); - public final static ContainerImage IMAGE_1 = ContainerImage.builder() .id(IMAGE_1_ID) .name(IMAGE_1_NAME) @@ -989,7 +932,6 @@ public abstract class BaseTest { .driverClass(IMAGE_1_DRIVER) .defaultPort(IMAGE_1_PORT) .isDefault(IMAGE_1_IS_DEFAULT) - .dateFormats(new LinkedList<>(List.of(IMAGE_DATE_1, IMAGE_DATE_2, IMAGE_DATE_3, IMAGE_DATE_4))) .build(); public final static ImageDto IMAGE_1_DTO = ImageDto.builder() @@ -1452,25 +1394,21 @@ public abstract class BaseTest { .name("col25") .type(ColumnTypeDto.DATE) .nullAllowed(true) - .dfid(IMAGE_DATE_1_ID) .build(), ColumnCreateDto.builder() .name("col26") .type(ColumnTypeDto.DATETIME) .nullAllowed(true) - .dfid(IMAGE_DATE_3_ID) .build(), ColumnCreateDto.builder() .name("col27") .type(ColumnTypeDto.TIMESTAMP) .nullAllowed(true) - .dfid(IMAGE_DATE_3_ID) .build(), ColumnCreateDto.builder() .name("col28") .type(ColumnTypeDto.TIME) .nullAllowed(true) - .dfid(IMAGE_DATE_4_ID) .build(), ColumnCreateDto.builder() .name("col29") @@ -2167,7 +2105,6 @@ public abstract class BaseTest { public final static List<ColumnCreateDto> TABLE_4_COLUMNS_CREATE_DTO = List.of(ColumnCreateDto.builder() .name("Timestamp") .type(ColumnTypeDto.TIMESTAMP) - .dfid(IMAGE_DATE_4_ID) .nullAllowed(false) .build(), ColumnCreateDto.builder() @@ -2844,7 +2781,6 @@ public abstract class BaseTest { .name("Date") .internalName("date") .columnType(TableColumnType.DATE) - .dateFormat(IMAGE_DATE_1) .isNullAllowed(true) .autoGenerated(false) .build(), @@ -2897,14 +2833,12 @@ public abstract class BaseTest { .name("Date") .type(ColumnTypeDto.DATE) .nullAllowed(true) - .dfid(IMAGE_DATE_1_ID) .build(), ColumnCreateDto.builder() .name("Location") .type(ColumnTypeDto.VARCHAR) .size(255L) .nullAllowed(true) - .dfid(IMAGE_DATE_1_ID) .build(), ColumnCreateDto.builder() .name("MinTemp") @@ -2912,7 +2846,6 @@ public abstract class BaseTest { .size(10L) .d(0L) .nullAllowed(true) - .dfid(IMAGE_DATE_1_ID) .build(), ColumnCreateDto.builder() .name("Rainfall") @@ -2920,7 +2853,6 @@ public abstract class BaseTest { .size(10L) .d(0L) .nullAllowed(true) - .dfid(IMAGE_DATE_1_ID) .conceptUri(CONCEPT_1_URI) .unitUri(UNIT_1_URI) .build()); @@ -3157,7 +3089,6 @@ public abstract class BaseTest { .name("id") .internalName("id") .isNullAllowed(false) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3170,7 +3101,6 @@ public abstract class BaseTest { .name("linie") .internalName("linie") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3183,7 +3113,6 @@ public abstract class BaseTest { .name("richtung") .internalName("richtung") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3208,7 +3137,6 @@ public abstract class BaseTest { .name("fahrzeug") .internalName("fahrzeug") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3221,7 +3149,6 @@ public abstract class BaseTest { .name("kurs") .internalName("kurs") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3234,7 +3161,6 @@ public abstract class BaseTest { .name("seq_von") .internalName("seq_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3247,7 +3173,6 @@ public abstract class BaseTest { .name("halt_diva_von") .internalName("halt_diva_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3260,7 +3185,6 @@ public abstract class BaseTest { .name("halt_punkt_diva_von") .internalName("halt_punkt_diva_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3273,7 +3197,6 @@ public abstract class BaseTest { .name("halt_kurz_von1") .internalName("halt_kurz_von1") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3298,7 +3221,6 @@ public abstract class BaseTest { .name("soll_an_von") .internalName("soll_an_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3311,7 +3233,6 @@ public abstract class BaseTest { .name("ist_an_von") .internalName("ist_an_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3324,7 +3245,6 @@ public abstract class BaseTest { .name("soll_ab_von") .internalName("soll_ab_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3337,7 +3257,6 @@ public abstract class BaseTest { .name("ist_ab_von") .internalName("ist_ab_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3350,7 +3269,6 @@ public abstract class BaseTest { .name("seq_nach") .internalName("seq_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3363,7 +3281,6 @@ public abstract class BaseTest { .name("halt_diva_nach") .internalName("halt_diva_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3376,7 +3293,6 @@ public abstract class BaseTest { .name("halt_punkt_diva_nach") .internalName("halt_punkt_diva_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3389,7 +3305,6 @@ public abstract class BaseTest { .name("halt_kurz_nach1") .internalName("halt_kurz_nach1") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3414,7 +3329,6 @@ public abstract class BaseTest { .name("soll_an_nach") .internalName("soll_an_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3427,7 +3341,6 @@ public abstract class BaseTest { .name("ist_an_nach1") .internalName("ist_an_nach1") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3440,7 +3353,6 @@ public abstract class BaseTest { .name("soll_ab_nach") .internalName("soll_ab_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3453,7 +3365,6 @@ public abstract class BaseTest { .name("ist_ab_nach") .internalName("ist_ab_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3466,7 +3377,6 @@ public abstract class BaseTest { .name("fahrt_id") .internalName("fahrt_id") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3479,7 +3389,6 @@ public abstract class BaseTest { .name("fahrweg_id") .internalName("fahrweg_id") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3492,7 +3401,6 @@ public abstract class BaseTest { .name("fw_no") .internalName("fw_no") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3505,7 +3413,6 @@ public abstract class BaseTest { .name("fw_typ") .internalName("fw_typ") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3518,7 +3425,6 @@ public abstract class BaseTest { .name("fw_kurz") .internalName("fw_kurz") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3531,7 +3437,6 @@ public abstract class BaseTest { .name("fw_lang") .internalName("fw_lang") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3544,7 +3449,6 @@ public abstract class BaseTest { .name("umlauf_von") .internalName("umlauf_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3557,7 +3461,6 @@ public abstract class BaseTest { .name("halt_id_von") .internalName("halt_id_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3570,7 +3473,6 @@ public abstract class BaseTest { .name("halt_id_nach") .internalName("halt_id_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3583,7 +3485,6 @@ public abstract class BaseTest { .name("halt_punkt_id_von") .internalName("halt_punkt_id_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3596,7 +3497,6 @@ public abstract class BaseTest { .name("halt_punkt_id_nach") .internalName("halt_punkt_id_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build()); @@ -4733,7 +4633,6 @@ public abstract class BaseTest { .name("reminder") .internalName("reminder") .columnType(TableColumnType.TIME) - .dateFormat(IMAGE_DATE_4) .isNullAllowed(false) .autoGenerated(false) .build(), @@ -4965,7 +4864,7 @@ public abstract class BaseTest { .size(22L) .isNullAllowed(true) .autoGenerated(false) - .build() ); + .build()); public final static View VIEW_1 = View.builder() .id(VIEW_1_ID) @@ -5168,7 +5067,6 @@ public abstract class BaseTest { .name("Date") .internalName("date") .columnType(TableColumnType.DATE) - .dateFormat(IMAGE_DATE_1) .isNullAllowed(true) .autoGenerated(false) .view(VIEW_2) @@ -5376,7 +5274,6 @@ public abstract class BaseTest { .name("Date") .internalName("date") .columnType(TableColumnType.DATE) - .dateFormat(IMAGE_DATE_1) .isNullAllowed(true) .autoGenerated(false) .view(VIEW_3) diff --git a/dbrepo-metric-db/prometheus.yml b/dbrepo-metric-db/prometheus.yml index f7338b0113..08d130b6ef 100644 --- a/dbrepo-metric-db/prometheus.yml +++ b/dbrepo-metric-db/prometheus.yml @@ -1,5 +1,5 @@ global: - scrape_interval: 1m + scrape_interval: 15s rule_files: [] @@ -16,4 +16,8 @@ scrape_configs: - job_name: 'metrics scrape' metrics_path: '/metrics' static_configs: - - targets: ['auth-service:8080', 'analyse-service:8080', 'search-service:8080', 'data-db-sidecar:8080', 'broker-service:15692', 'dashboard-service:3000', 'storage-service:9090', 'upload-service:8080'] + - targets: ['auth-service:8080', 'analyse-service:8080', 'search-service:8080', 'data-db-sidecar:8080', 'broker-service:15692', 'storage-service:9090', 'upload-service:8080'] + - job_name: 'dashboard scrape' + metrics_path: '/dashboard/metrics' + static_configs: + - targets: ['dashboard-service:3000'] diff --git a/dbrepo-ui/components/subset/Results.vue b/dbrepo-ui/components/subset/Results.vue index 95becef12c..4ba414309c 100644 --- a/dbrepo-ui/components/subset/Results.vue +++ b/dbrepo-ui/components/subset/Results.vue @@ -8,6 +8,7 @@ :items="result.rows" :items-length="total" :footer-props="footerProps" + :items-per-page-options="footerProps.itemsPerPageOptions" @update:options="updateOptions" /> </div> </template> diff --git a/dbrepo-ui/nuxt.config.ts b/dbrepo-ui/nuxt.config.ts index bb30b673f4..f07d9aa0dd 100644 --- a/dbrepo-ui/nuxt.config.ts +++ b/dbrepo-ui/nuxt.config.ts @@ -102,10 +102,6 @@ export default defineNuxtConfig({ endpoint: 'https://doi.org' }, links: { - rabbitmq: { - text: 'Broker Service', - href: '/admin/broker/' - }, keycloak: { text: 'Auth Service', href: '/api/auth/' diff --git a/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue b/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue index bcab9b60be..72e7501d16 100644 --- a/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue +++ b/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue @@ -75,6 +75,7 @@ :loading="loadingData || loadingCount" :options.sync="options" :footer-props="footerProps" + :items-per-page-options="footerProps.itemsPerPageOptions" @update:options="loadData"> <template v-for="(blobColumn, idx) in blobColumns" diff --git a/dbrepo-ui/pages/semantic/index.vue b/dbrepo-ui/pages/semantic/index.vue index f6b6721b17..c5f12d48f4 100644 --- a/dbrepo-ui/pages/semantic/index.vue +++ b/dbrepo-ui/pages/semantic/index.vue @@ -27,7 +27,8 @@ :items="rows" :options.sync="options" :server-items-length="total" - :footer-props="footerProps"> + :footer-props="footerProps" + :items-per-page-options="footerProps.itemsPerPageOptions"> <template v-slot:item.uri="{ item }"> <a :href="item.uri" target="_blank" v-text="item.uri" /> </template> @@ -83,7 +84,7 @@ export default { }, total: -1, footerProps: { - 'items-per-page-options': [10, 20, 30, 40, 50] + itemsPerPageOptions: [10, 25, 50, 100] }, tab: 0, tabs: [ diff --git a/docker-compose.yml b/docker-compose.yml index b60c7b0d94..cb45ec51d0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -211,6 +211,7 @@ services: hostname: broker-service image: docker.io/bitnami/rabbitmq:3.13.7-debian-12-r4 ports: + - 15672:15672 - 5672:5672 - 1883:1883 volumes: @@ -219,6 +220,8 @@ services: - ./dbrepo-broker-service/enabled_plugins:/etc/rabbitmq/enabled_plugins - ./dbrepo-broker-service/definitions.json:/app/definitions.json - broker-service-data:/bitnami/rabbitmq/mnesia + environment: + RABBITMQ_FEATURE_FLAGS: mqtt_v5 depends_on: dbrepo-identity-service: condition: service_healthy @@ -458,12 +461,13 @@ services: environment: GF_SERVER_DOMAIN: "dashboard-service" GF_SERVER_ROOT_URL: "${BASE_URL:-http://localhost}/dashboard/" + GF_SERVER_SERVE_FROM_SUB_PATH: "true" GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION: "true" LDAP_ADMIN_USERNAME: "${IDENTITY_SERVICE_ADMIN_USERNAME:-admin}" LDAP_ADMIN_PASSWORD: "${IDENTITY_SERVICE_ADMIN_PASSWORD:-admin}" LDAP_ROOT: "${IDENTITY_SERVICE_ROOT:-dc=dbrepo,dc=at}" healthcheck: - test: test -f /opt/bitnami/grafana/tmp/grafana.pid + test: test -f /var/lib/grafana/grafana.db interval: 10s timeout: 5s retries: 12 @@ -539,7 +543,7 @@ services: BROKER_HOST: "${BROKER_ENDPOINT:-broker-service}" BROKER_PASSWORD: "${SYSTEM_PASSWORD:-admin}" BROKER_PORT: ${BROKER_PORT:-5672} - BROKER_SERVICE_ENDPOINT: "${BROKER_SERVICE_ENDPOINT:-http://gateway-service/admin/broker}" + BROKER_SERVICE_ENDPOINT: "${BROKER_SERVICE_ENDPOINT:-http://broker-service:15672}" BROKER_USERNAME: "${SYSTEM_USERNAME:-admin}" BROKER_VIRTUALHOST: "${BROKER_VIRTUALHOST:-dbrepo}" CONNECTION_TIMEOUT: ${CONNECTION_TIMEOUT:-60000} -- GitLab