diff --git a/.docker/config/dbrepo.conf b/.docker/config/dbrepo.conf index 8ac239d464398ec381b98d71c91192ea6ba061ab..7c987289940be7a71bb957af6f0a8f7ca0dd1725 100644 --- a/.docker/config/dbrepo.conf +++ b/.docker/config/dbrepo.conf @@ -1,3 +1,9 @@ +# This is required to proxy Grafana Live WebSocket connections. +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + client_max_body_size 20G; resolver 127.0.0.11 valid=30s; # docker dns @@ -34,17 +40,32 @@ upstream upload { server upload-service:8080; } +upstream dashboard-service { + server dashboard-service:3000; +} + server { listen 80 default_server; server_name _; - location /admin/broker { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass http://broker; - proxy_read_timeout 90; + location /dashboard { + rewrite ^/dashboard/(.*) /$1 break; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://dashboard-service; + proxy_read_timeout 90; + } + + # Proxy Grafana Live WebSocket connections. + location /dashboard/api/live { + rewrite ^/dashboard/(.*) /$1 break; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + proxy_pass http://dashboard-service; } location /api/search { @@ -57,7 +78,7 @@ server { } location /api/broker { - rewrite /api/broker/(.*) /admin/broker/api/$1 break; + rewrite /api/broker/(.*) /api/$1 break; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/.docker/config/enabled_plugins b/.docker/config/enabled_plugins index 95f1c0014dd4ee232580adea29176756a25274ed..db0ae888499ea44c2dd7d40f5ac9c8fcc0ca0567 100644 --- a/.docker/config/enabled_plugins +++ b/.docker/config/enabled_plugins @@ -1 +1 @@ -[rabbitmq_prometheus,rabbitmq_auth_backend_ldap,rabbitmq_auth_mechanism_ssl,rabbitmq_management]. \ No newline at end of file +[rabbitmq_prometheus,rabbitmq_auth_backend_ldap,rabbitmq_auth_mechanism_ssl,rabbitmq_management,rabbitmq_mqtt]. \ No newline at end of file diff --git a/.docker/config/rabbitmq.conf b/.docker/config/rabbitmq.conf index ff592bb3ecd4b003d180dbb44d8bd9acc5a70394..8503942950feb8c701ac8765872d005249ee612b 100644 --- a/.docker/config/rabbitmq.conf +++ b/.docker/config/rabbitmq.conf @@ -6,7 +6,6 @@ default_user_tags.administrator = false listeners.tcp.1 = 0.0.0.0:5672 # management prefix (https://www.rabbitmq.com/management.html#path-prefix) -management.path_prefix = /admin/broker management.load_definitions = /app/definitions.json # logging @@ -14,6 +13,11 @@ log.console = true log.console.level = warning auth_ldap.log = true +# MQTT +mqtt.vhost = dbrepo +mqtt.exchange = dbrepo +mqtt.prefetch = 10 + # Obviously your authentication server cannot vouch for itself, so you'll need another backend with at least one user in # it. You should probably use the internal database auth_backends.1.authn = ldap diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml index ec7a87d25295fd3284589152120263b6204b183a..87dab50c910aa24671b50db413f4838e48d10369 100644 --- a/.docker/docker-compose.yml +++ b/.docker/docker-compose.yml @@ -7,6 +7,8 @@ volumes: search-db-data: storage-service-data: identity-service-data: + metric-db-data: + dashboard-service-data: services: dbrepo-metadata-db: @@ -191,6 +193,7 @@ services: image: docker.io/bitnami/rabbitmq:3.12-debian-12 ports: - 5672:5672 + - 1883:1883 volumes: - ./config/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf - ./config/advanced.config:/etc/rabbitmq/advanced.config @@ -279,11 +282,6 @@ services: NUXT_PUBLIC_API_CLIENT: "${BASE_URL:-http://localhost}" NUXT_PUBLIC_API_SERVER: "${BASE_URL:-http://localhost}" NUXT_PUBLIC_UPLOAD_CLIENT: "${BASE_URL:-http://localhost}/api/upload/files/" - depends_on: - dbrepo-search-service: - condition: service_started - dbrepo-storage-service: - condition: service_healthy healthcheck: test: wget -qO- localhost:3000 | grep "Database Repository" || exit 1 interval: 10s @@ -386,6 +384,7 @@ services: image: bitnami/prometheus:2.54.1-debian-12-r4 volumes: - ./config/prometheus.yml:/etc/prometheus/prometheus.yml + - metric-db-data:/opt/bitnami/prometheus/data healthcheck: test: promtool check healthy interval: 10s @@ -475,7 +474,7 @@ services: BROKER_HOST: "${BROKER_ENDPOINT:-broker-service}" BROKER_PASSWORD: "${SYSTEM_PASSWORD:-admin}" BROKER_PORT: ${BROKER_PORT:-5672} - BROKER_SERVICE_ENDPOINT: "${BROKER_SERVICE_ENDPOINT:-http://gateway-service/admin/broker}" + BROKER_SERVICE_ENDPOINT: "${BROKER_SERVICE_ENDPOINT:-http://broker-service:15672}" BROKER_USERNAME: "${SYSTEM_USERNAME:-admin}" BROKER_VIRTUALHOST: "${BROKER_VIRTUALHOST:-dbrepo}" CONNECTION_TIMEOUT: ${CONNECTION_TIMEOUT:-60000} diff --git a/.docs/api/broker-service.md b/.docs/api/broker-service.md index f2f684c4a944f4e726a53f4d050bb19b2f368a8d..41e2dc10b15ccd137a934cd9c162c85b32dffc22 100644 --- a/.docs/api/broker-service.md +++ b/.docs/api/broker-service.md @@ -11,7 +11,7 @@ author: Martin Weise * Ports: 5672/tcp, 15672/tcp, 15692/tcp * AMQP: `amqp://<hostname>:5672` * Prometheus: `http://<hostname>:15692/metrics` - * Management: `http://<hostname>/admin/broker` + * Management: `http://<hostname>:15672` ## Overview @@ -19,6 +19,13 @@ It holds exchanges and topics responsible for holding AMQP messages for later co use [RabbitMQ](https://www.rabbitmq.com/) in the implementation. By default, the endpoint listens to the insecure port `5672` for incoming AMQP tuples and insecure port `15672` for the management UI. +## Supported Protocols + +* AMQP (v0.9.1, v1.0), see [RabbitMQ docs](https://www.rabbitmq.com/docs/next/amqp). +* MQTT (v3.1, v3.1.1, v5), see [RabbitMQ docs](https://www.rabbitmq.com/docs/mqtt). + +## Authentication + The default configuration allows any user in the `cn=system,ou=users,dc=dbrepo,dc=at` from the [Identity Service](../identity-service) to access the Broker Service as user with `administrator` role, i.e. the `cn=admin,dc=dbrepo,dc=at` user that is created by default. @@ -28,6 +35,8 @@ The Broker Service allows two ways of authentication for AMQP tuples: 1. LDAP 2. Plain (RabbitMQ's internal authentication) +## Architecture + The queue architecture of the Broker Service is very simple. There is only one durable, topic exchange `dbrepo` and one quorum queue `dbrepo`, connected with a binding of `dbrepo.#` which routes all tuples with routing key prefix `dbrepo.` to this queue. diff --git a/dbrepo-broker-service/README.md b/dbrepo-broker-service/README.md index 95e5afaefdfc73751db6856526ca8c5e3a8f4c7c..6cff53bb912b21c916f30b2a07f6a7c90a76dc5a 100644 --- a/dbrepo-broker-service/README.md +++ b/dbrepo-broker-service/README.md @@ -1,5 +1,7 @@ # Broker Service +Supports MQTT v3, v4 and v5 (https://www.rabbitmq.com/blog/2023/07/21/mqtt5) + ## Advanced Config https://www.rabbitmq.com/docs/ldap \ No newline at end of file diff --git a/dbrepo-broker-service/advanced.config b/dbrepo-broker-service/advanced.config index 4445ea601954e5c93c32edeba1638135c5af5e59..584bcc52325e61a8d6019d2f092590bb7f989530 100644 --- a/dbrepo-broker-service/advanced.config +++ b/dbrepo-broker-service/advanced.config @@ -1,4 +1,9 @@ [ + { + rabbit, [ + {forced_feature_flags_on_init, [quorum_queue, mqtt_v5]} + ] + }, { rabbitmq_auth_backend_ldap, [ diff --git a/dbrepo-broker-service/enabled_plugins b/dbrepo-broker-service/enabled_plugins index 95f1c0014dd4ee232580adea29176756a25274ed..db0ae888499ea44c2dd7d40f5ac9c8fcc0ca0567 100644 --- a/dbrepo-broker-service/enabled_plugins +++ b/dbrepo-broker-service/enabled_plugins @@ -1 +1 @@ -[rabbitmq_prometheus,rabbitmq_auth_backend_ldap,rabbitmq_auth_mechanism_ssl,rabbitmq_management]. \ No newline at end of file +[rabbitmq_prometheus,rabbitmq_auth_backend_ldap,rabbitmq_auth_mechanism_ssl,rabbitmq_management,rabbitmq_mqtt]. \ No newline at end of file diff --git a/dbrepo-broker-service/rabbitmq.conf b/dbrepo-broker-service/rabbitmq.conf index ff592bb3ecd4b003d180dbb44d8bd9acc5a70394..8503942950feb8c701ac8765872d005249ee612b 100644 --- a/dbrepo-broker-service/rabbitmq.conf +++ b/dbrepo-broker-service/rabbitmq.conf @@ -6,7 +6,6 @@ default_user_tags.administrator = false listeners.tcp.1 = 0.0.0.0:5672 # management prefix (https://www.rabbitmq.com/management.html#path-prefix) -management.path_prefix = /admin/broker management.load_definitions = /app/definitions.json # logging @@ -14,6 +13,11 @@ log.console = true log.console.level = warning auth_ldap.log = true +# MQTT +mqtt.vhost = dbrepo +mqtt.exchange = dbrepo +mqtt.prefetch = 10 + # Obviously your authentication server cannot vouch for itself, so you'll need another backend with at least one user in # it. You should probably use the internal database auth_backends.1.authn = ldap diff --git a/dbrepo-dashboard-service/dashboards/system.json b/dbrepo-dashboard-service/dashboards/system.json index 68279f8fe4ffc11a52e7a53de0d21f07797f80d5..4eb40a1a2ea5311ed634f406f0603e82d4c1ee91 100644 --- a/dbrepo-dashboard-service/dashboards/system.json +++ b/dbrepo-dashboard-service/dashboards/system.json @@ -18,7 +18,6 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": 2, "links": [ { "asDropdown": false, @@ -55,34 +54,17 @@ }, "fieldConfig": { "defaults": { - "displayName": "QoS", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "purple", + "color": "blue", "value": null - }, - { - "color": "red", - "value": 0 - }, - { - "color": "orange", - "value": 60 - }, - { - "color": "#EAB839", - "value": 80 - }, - { - "color": "green", - "value": 100 } ] }, - "unit": "percent" + "unit": "short" }, "overrides": [] }, @@ -92,7 +74,7 @@ "x": 0, "y": 1 }, - "id": 9, + "id": 4, "options": { "colorMode": "background", "graphMode": "none", @@ -110,7 +92,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "", + "pluginVersion": "11.2.2", "targets": [ { "datasource": { @@ -118,17 +100,18 @@ "uid": "P18F45E9DC7E75912" }, "disableTextWrap": false, - "editorMode": "code", - "expr": "sum(up)*100/count(up)", + "editorMode": "builder", + "expr": "dbrepo_database_count", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "Services Running", + "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false } ], + "title": "Databases", "type": "stat" }, { @@ -139,34 +122,17 @@ }, "fieldConfig": { "defaults": { - "displayName": "UI Response Time (Mean)", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "purple", + "color": "blue", "value": null - }, - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 200 - }, - { - "color": "orange", - "value": 400 - }, - { - "color": "red", - "value": 600 } ] }, - "unit": "ms" + "unit": "short" }, "overrides": [] }, @@ -176,7 +142,7 @@ "x": 4, "y": 1 }, - "id": 17, + "id": 5, "options": { "colorMode": "background", "graphMode": "none", @@ -194,22 +160,71 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "", + "pluginVersion": "11.2.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "P18F45E9DC7E75912" }, - "editorMode": "code", - "expr": "avg(page_total_time)", - "format": "table", - "intervalFactor": 3, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "dbrepo_view_count", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "Views", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "P18F45E9DC7E75912" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "dbrepo_subset_count", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "Subsets", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "P18F45E9DC7E75912" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "dbrepo_table_count", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, "legendFormat": "__auto", "range": true, - "refId": "B", - "step": 15, - "target": "dev.grafana.cb-office.alerting.active_alerts" + "refId": "Tables", + "useBackend": false + } + ], + "title": "Datasources", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "mode": "reduceRow", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } } ], "type": "stat" @@ -220,9 +235,9 @@ "type": "prometheus", "uid": "P18F45E9DC7E75912" }, + "description": "", "fieldConfig": { "defaults": { - "displayName": "Databases", "mappings": [], "thresholds": { "mode": "absolute", @@ -233,7 +248,7 @@ } ] }, - "unit": "short" + "unit": "decbytes" }, "overrides": [] }, @@ -243,7 +258,7 @@ "x": 8, "y": 1 }, - "id": 4, + "id": 8, "options": { "colorMode": "background", "graphMode": "none", @@ -261,7 +276,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "", + "pluginVersion": "11.2.2", "targets": [ { "datasource": { @@ -270,7 +285,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "dbrepo_database_count", + "expr": "dbrepo_volume_sum", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -280,38 +295,70 @@ "useBackend": false } ], + "title": "Data Volume", "type": "stat" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 2, + "panels": [], + "title": "Services", + "type": "row" + }, { "datasource": { "default": true, "type": "prometheus", "uid": "P18F45E9DC7E75912" }, + "description": "Quality of Service", "fieldConfig": { "defaults": { - "displayName": "Datasources", "mappings": [], + "max": 100, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "blue", + "color": "purple", "value": null + }, + { + "color": "red", + "value": 0 + }, + { + "color": "orange", + "value": 60 + }, + { + "color": "#EAB839", + "value": 80 + }, + { + "color": "green", + "value": 100 } ] }, - "unit": "short" + "unit": "percent" }, "overrides": [] }, "gridPos": { - "h": 3, + "h": 4, "w": 4, - "x": 12, - "y": 1 + "x": 0, + "y": 5 }, - "id": 5, + "id": 9, "options": { "colorMode": "background", "graphMode": "none", @@ -329,7 +376,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "", + "pluginVersion": "11.2.2", "targets": [ { "datasource": { @@ -337,33 +384,92 @@ "uid": "P18F45E9DC7E75912" }, "disableTextWrap": false, - "editorMode": "builder", - "expr": "dbrepo_view_count", + "editorMode": "code", + "expr": "sum(up)*100/count(up)", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "__auto", + "legendFormat": "Services Running", "range": true, - "refId": "Views", + "refId": "A", "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "P18F45E9DC7E75912" + } + ], + "title": "QoS", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P18F45E9DC7E75912" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "dbrepo_subset_count", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "Subsets", - "useBackend": false + "custom": { + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [ + { + "options": { + "0": { + "index": 0, + "text": "DOWN" + }, + "1": { + "index": 1, + "text": "UP" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 20, + "x": 4, + "y": 5 + }, + "id": 16, + "options": { + "colWidth": 0.9, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "rowHeight": 0.9, + "showValue": "auto", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", @@ -371,42 +477,30 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "dbrepo_table_count", + "expr": "up", "fullMetaSearch": false, - "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{instance}}", "range": true, - "refId": "Tables", + "refId": "A", "useBackend": false } ], - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "mode": "reduceRow", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" + "title": "Service QoS", + "type": "status-history" }, { "datasource": { - "default": true, "type": "prometheus", "uid": "P18F45E9DC7E75912" }, + "description": "Total used disk space in Storage Service", "fieldConfig": { "defaults": { - "displayName": "Volume", "mappings": [], + "max": 100, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -421,12 +515,12 @@ "overrides": [] }, "gridPos": { - "h": 3, + "h": 4, "w": 4, - "x": 16, - "y": 1 + "x": 0, + "y": 9 }, - "id": 8, + "id": 17, "options": { "colorMode": "background", "graphMode": "none", @@ -444,7 +538,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "", + "pluginVersion": "11.2.2", "targets": [ { "datasource": { @@ -452,8 +546,8 @@ "uid": "P18F45E9DC7E75912" }, "disableTextWrap": false, - "editorMode": "builder", - "expr": "dbrepo_volume_sum", + "editorMode": "code", + "expr": "SeaweedFS_volumeServer_total_disk_size", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -463,21 +557,9 @@ "useBackend": false } ], + "title": "S3 Volume", "type": "stat" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 2, - "panels": [], - "title": "Services", - "type": "row" - }, { "datasource": { "type": "prometheus", @@ -486,7 +568,40 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { @@ -496,50 +611,97 @@ "color": "green", "value": null }, - { - "color": "#EAB839", - "value": 200 - }, - { - "color": "orange", - "value": 400 - }, { "color": "red", - "value": 600 + "value": 80 } ] }, - "unit": "ms" + "unit": "none" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "auth-service:8080" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "data-service:8080" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "metadata-service:8080" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "metadata-service:80" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { - "h": 9, - "w": 24, + "h": 7, + "w": 12, "x": 0, - "y": 5 + "y": 13 }, - "id": 13, + "id": 6, "options": { - "displayMode": "gradient", - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "pluginVersion": "", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -548,23 +710,19 @@ }, "disableTextWrap": false, "editorMode": "builder", - "exemplar": false, - "expr": "page_total_time{path!=\"empty: empty\"}", - "format": "time_series", + "expr": "process_cpu_usage", "fullMetaSearch": false, + "hide": false, "includeNullMetadata": true, "instant": false, - "intervalFactor": 3, - "legendFormat": "{{path}}", + "legendFormat": "{{instance}}", "range": true, - "refId": "B", - "step": 15, - "target": "dev.grafana.cb-office.alerting.active_alerts", + "refId": "process_cpu_usage", "useBackend": false } ], - "title": "UI Response Time", - "type": "bargauge" + "title": "CPU Usage", + "type": "timeseries" }, { "datasource": { @@ -572,6 +730,7 @@ "type": "prometheus", "uid": "P18F45E9DC7E75912" }, + "description": "Heap and non-heap memory summed", "fieldConfig": { "defaults": { "color": { @@ -584,6 +743,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -592,9 +752,9 @@ "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -616,13 +776,10 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "decbytes" }, "overrides": [ { @@ -643,13 +800,13 @@ { "matcher": { "id": "byName", - "options": "broker-service:15692" + "options": "data-service:8080" }, "properties": [ { "id": "color", "value": { - "fixedColor": "light-blue", + "fixedColor": "blue", "mode": "fixed" } } @@ -670,36 +827,6 @@ } ] }, - { - "matcher": { - "id": "byName", - "options": "analyse-service:80" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "super-light-red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "auth-service-metrics:8080" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, { "matcher": { "id": "byName", @@ -714,61 +841,16 @@ } } ] - }, - { - "matcher": { - "id": "byName", - "options": "data-service:80" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "search-service:80" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-purple", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "ui:80" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "super-light-purple", - "mode": "fixed" - } - } - ] } ] }, "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 14 + "h": 7, + "w": 12, + "x": 12, + "y": 13 }, - "id": 1, + "id": 7, "options": { "legend": { "calcs": [], @@ -777,7 +859,7 @@ "showLegend": true }, "tooltip": { - "mode": "single", + "mode": "multi", "sort": "none" } }, @@ -790,10 +872,10 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "up", + "expr": "sum by(instance) (jvm_memory_used_bytes)", "fullMetaSearch": false, "hide": false, - "includeNullMetadata": true, + "includeNullMetadata": false, "instant": false, "legendFormat": "{{instance}}", "range": true, @@ -801,12 +883,11 @@ "useBackend": false } ], - "title": "Service Instances Running", + "title": "JVM Memory Usage", "type": "timeseries" }, { "datasource": { - "default": true, "type": "prometheus", "uid": "P18F45E9DC7E75912" }, @@ -822,6 +903,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -832,7 +914,7 @@ }, "insertNulls": false, "lineInterpolation": "smooth", - "lineWidth": 1, + "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -841,7 +923,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" @@ -854,25 +936,22 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "reqps" }, "overrides": [ { "matcher": { - "id": "byName", - "options": "auth-service:8080" + "id": "byRegexp", + "options": "/.*search-service.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "yellow", + "fixedColor": "orange", "mode": "fixed" } } @@ -880,29 +959,123 @@ }, { "matcher": { - "id": "byName", - "options": "data-service:8080" + "id": "byRegexp", + "options": "/.*analyse-service.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "blue", + "fixedColor": "super-light-orange", "mode": "fixed" } } ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P18F45E9DC7E75912" + }, + "editorMode": "code", + "expr": "rate(flask_http_request_duration_seconds_count{status=~\"200|201|202\",path!=\"/health\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{method}} {{instance}} {{path}} ({{status}})", + "range": true, + "refId": "A" + } + ], + "title": "Successful API Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P18F45E9DC7E75912" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] }, + "unit": "reqps" + }, + "overrides": [ { "matcher": { - "id": "byName", - "options": "metadata-service:8080" + "id": "byRegexp", + "options": "/.*search-service.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "purple", + "fixedColor": "orange", "mode": "fixed" } } @@ -910,14 +1083,14 @@ }, { "matcher": { - "id": "byName", - "options": "metadata-service:80" + "id": "byRegexp", + "options": "/.*analyse-service.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "blue", + "fixedColor": "super-light-orange", "mode": "fixed" } } @@ -926,12 +1099,12 @@ ] }, "gridPos": { - "h": 6, - "w": 24, - "x": 0, + "h": 7, + "w": 12, + "x": 12, "y": 20 }, - "id": 6, + "id": 19, "options": { "legend": { "calcs": [], @@ -944,27 +1117,21 @@ "sort": "none" } }, - "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "P18F45E9DC7E75912" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "process_cpu_usage", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, + "editorMode": "code", + "expr": "rate(flask_http_request_duration_seconds_count{status!~\"200|201|202\"}[$__rate_interval])", "instant": false, - "legendFormat": "{{instance}}", + "legendFormat": "{{method}} {{instance}} ({{status}})", "range": true, - "refId": "Java", - "useBackend": false + "refId": "A" } ], - "title": "CPU Usage", + "title": "Failed API Requests", "type": "timeseries" }, { @@ -973,7 +1140,6 @@ "type": "prometheus", "uid": "P18F45E9DC7E75912" }, - "description": "Heap and non-heap memory summed", "fieldConfig": { "defaults": { "color": { @@ -986,8 +1152,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 25, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1005,7 +1172,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" @@ -1020,20 +1187,19 @@ "value": null } ] - }, - "unit": "decbytes" + } }, "overrides": [ { "matcher": { "id": "byName", - "options": "auth-service:8080" + "options": "DELETE" }, "properties": [ { "id": "color", "value": { - "fixedColor": "yellow", + "fixedColor": "red", "mode": "fixed" } } @@ -1042,7 +1208,7 @@ { "matcher": { "id": "byName", - "options": "data-service:8080" + "options": "GET" }, "properties": [ { @@ -1057,7 +1223,7 @@ { "matcher": { "id": "byName", - "options": "metadata-service:8080" + "options": "HEAD" }, "properties": [ { @@ -1072,13 +1238,42 @@ { "matcher": { "id": "byName", - "options": "metadata-service:80" + "options": "PATCH" }, "properties": [ { "id": "color", "value": { - "fixedColor": "blue", + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OPTIONS" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "POST" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", "mode": "fixed" } } @@ -1087,12 +1282,12 @@ ] }, "gridPos": { - "h": 6, - "w": 24, + "h": 7, + "w": 12, "x": 0, - "y": 26 + "y": 27 }, - "id": 7, + "id": 20, "options": { "legend": { "calcs": [], @@ -1105,7 +1300,6 @@ "sort": "none" } }, - "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -1114,18 +1308,17 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "sum by(instance) (jvm_memory_used_bytes)", + "expr": "tusd_requests_total", "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, + "includeNullMetadata": true, "instant": false, - "legendFormat": "{{instance}}", + "legendFormat": "{{method}}", "range": true, "refId": "A", "useBackend": false } ], - "title": "JVM Memory Usage", + "title": "Storage Service Requests", "type": "timeseries" } ], diff --git a/dbrepo-dashboard-service/grafana.ini b/dbrepo-dashboard-service/grafana.ini index cc2f5d41a63f1106d0e6b59bba175c4becb16a73..1f8d9c1ef376dfd487ebf122789b557237134e35 100644 --- a/dbrepo-dashboard-service/grafana.ini +++ b/dbrepo-dashboard-service/grafana.ini @@ -1,4 +1,7 @@ [server] +protocol = http +domain = localhost +root_url = http://%(domain)s/dashboard/ http_port = 3000 [security] diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/AccessEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/AccessEndpoint.java index 133bee769c792602ae89114a218381dd913f5c93..4059a37a92ba138ef39a6f53365fd70558951e56 100644 --- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/AccessEndpoint.java +++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/AccessEndpoint.java @@ -18,6 +18,7 @@ import jakarta.validation.Valid; import jakarta.validation.constraints.NotBlank; import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.*; @@ -88,7 +89,7 @@ public class AccessEndpoint { } try { accessService.create(database, user, data.getType()); - return ResponseEntity.accepted() + return ResponseEntity.status(HttpStatus.CREATED) .build(); } catch (SQLException e) { log.error("Failed to establish connection to database: {}", e.getMessage()); diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java index 87471c74c2e21d45dc86a28e76409e3cb9564906..1637878a6785f595a35643471aa2b2b71f027399 100644 --- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java +++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java @@ -250,12 +250,7 @@ public class SubsetEndpoint { endpointValidator.validateDataParams(page, size); endpointValidator.validateForbiddenStatements(data.getStatement()); /* parameters */ - final UUID userId; - if (principal == null) { - userId = metadataServiceGateway.getSystemUserId(); - } else { - userId = UserUtil.getId(principal); - } + final UUID userId = principal != null ? UserUtil.getId(principal) : null; if (page == null) { page = 0L; log.debug("page not set: default to {}", page); diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java index 4af577bed5771e66502ab48e78509c096666d151..548558cafd8ccba785bda4c52dde303932c24a84 100644 --- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java +++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java @@ -4,8 +4,7 @@ import at.tuwien.ExportResourceDto; import at.tuwien.api.database.DatabaseAccessDto; import at.tuwien.api.database.DatabaseDto; import at.tuwien.api.database.internal.PrivilegedDatabaseDto; -import at.tuwien.api.database.query.ImportCsvDto; -import at.tuwien.api.database.query.QueryDto; +import at.tuwien.api.database.query.ImportDto; import at.tuwien.api.database.query.QueryResultDto; import at.tuwien.api.database.table.*; import at.tuwien.api.database.table.internal.PrivilegedTableDto; @@ -13,6 +12,7 @@ import at.tuwien.api.database.table.internal.TableCreateDto; import at.tuwien.api.error.ApiErrorDto; import at.tuwien.exception.*; import at.tuwien.gateway.MetadataServiceGateway; +import at.tuwien.service.SchemaService; import at.tuwien.service.TableService; import at.tuwien.utils.UserUtil; import at.tuwien.validation.EndpointValidator; @@ -33,7 +33,6 @@ import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.InputStreamResource; import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; @@ -51,13 +50,15 @@ import java.util.List; public class TableEndpoint { private final TableService tableService; + private final SchemaService schemaService; private final EndpointValidator endpointValidator; private final MetadataServiceGateway metadataServiceGateway; @Autowired - public TableEndpoint(TableService tableService, EndpointValidator endpointValidator, + public TableEndpoint(TableService tableService, SchemaService schemaService, EndpointValidator endpointValidator, MetadataServiceGateway metadataServiceGateway) { this.tableService = tableService; + this.schemaService = schemaService; this.endpointValidator = endpointValidator; this.metadataServiceGateway = metadataServiceGateway; } @@ -107,8 +108,9 @@ public class TableEndpoint { /* create */ final PrivilegedDatabaseDto database = metadataServiceGateway.getDatabaseById(databaseId); try { + final TableDto table = tableService.createTable(database, data); return ResponseEntity.status(HttpStatus.CREATED) - .body(tableService.createTable(database, data)); + .body(schemaService.inspectTable(database, table.getInternalName())); } catch (SQLException e) { log.error("Failed to establish connection to database: {}", e.getMessage()); throw new DatabaseUnavailableException("Failed to establish connection to database: " + e.getMessage(), e); @@ -614,7 +616,7 @@ public class TableEndpoint { }) public ResponseEntity<Void> importDataset(@NotBlank @PathVariable("databaseId") Long databaseId, @NotBlank @PathVariable("tableId") Long tableId, - @Valid @RequestBody ImportCsvDto data, + @Valid @RequestBody ImportDto data, @NotNull Principal principal) throws DatabaseUnavailableException, RemoteUnavailableException, TableNotFoundException, QueryMalformedException, StorageNotFoundException, SidecarImportException, NotAllowedException, @@ -623,10 +625,6 @@ public class TableEndpoint { final PrivilegedTableDto table = metadataServiceGateway.getTableById(databaseId, tableId); final DatabaseAccessDto access = metadataServiceGateway.getAccess(databaseId, UserUtil.getId(principal)); endpointValidator.validateOnlyWriteOwnOrWriteAllAccess(access.getType(), table.getOwner().getId(), UserUtil.getId(principal)); - if (data.getNullElement() == null) { - data.setNullElement(""); - log.debug("null element not present, default to empty string"); - } if (data.getLineTermination() == null) { data.setLineTermination("\\r\\n"); log.debug("line termination not present, default to {}", data.getLineTermination()); diff --git a/dbrepo-data-service/rest-service/src/main/resources/init/querystore.sql b/dbrepo-data-service/rest-service/src/main/resources/init/querystore.sql index c1df44d1b0766fb04d081f3b5b3679039d8ba72f..3e7471df3e1eb3a1bb79db7246ecb805a06f495c 100644 --- a/dbrepo-data-service/rest-service/src/main/resources/init/querystore.sql +++ b/dbrepo-data-service/rest-service/src/main/resources/init/querystore.sql @@ -1,5 +1,5 @@ CREATE SEQUENCE `qs_queries_seq` NOCACHE; -CREATE TABLE `qs_queries` ( `id` bigint not null primary key default nextval(`qs_queries_seq`), `created` datetime not null default now(), `executed` datetime not null default now(), `created_by` varchar(36) not null, `query` text not null, `query_normalized` text not null, `is_persisted` boolean not null, `query_hash` varchar(255) not null, `result_hash` varchar(255), `result_number` bigint ) WITH SYSTEM VERSIONING; +CREATE TABLE `qs_queries` ( `id` bigint not null primary key default nextval(`qs_queries_seq`), `created` datetime not null default now(), `executed` datetime not null default now(), `created_by` varchar(36), `query` text not null, `query_normalized` text not null, `is_persisted` boolean not null, `query_hash` varchar(255) not null, `result_hash` varchar(255), `result_number` bigint ) WITH SYSTEM VERSIONING; CREATE PROCEDURE hash_table(IN name VARCHAR(255), OUT hash VARCHAR(255), OUT count BIGINT) BEGIN DECLARE _sql TEXT; SELECT CONCAT('SELECT SHA2(GROUP_CONCAT(CONCAT_WS(\'\',', GROUP_CONCAT(CONCAT('`', column_name, '`') ORDER BY column_name), ') SEPARATOR \',\'), 256) AS hash, COUNT(*) AS count FROM `', name, '` INTO @hash, @count;') FROM `information_schema`.`columns` WHERE `table_schema` = DATABASE() AND `table_name` = name INTO _sql; PREPARE stmt FROM _sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; SET hash = @hash; SET count = @count; END; CREATE PROCEDURE store_query(IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _username varchar(255) DEFAULT REGEXP_REPLACE(current_user(), '@.*', ''); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END; CREATE DEFINER = 'root' PROCEDURE _store_query(IN _username VARCHAR(255), IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END; \ No newline at end of file diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/config/MariaDbConfig.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/config/MariaDbConfig.java index b0c332a63b5b87e786ca928daa7bc8bfa8a7ed3c..691d96006bb273bc257e6439b0f828cae70b9d58 100644 --- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/config/MariaDbConfig.java +++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/config/MariaDbConfig.java @@ -6,13 +6,11 @@ import at.tuwien.api.database.internal.PrivilegedDatabaseDto; import at.tuwien.api.database.query.QueryDto; import at.tuwien.api.database.table.columns.ColumnTypeDto; import at.tuwien.api.database.table.internal.PrivilegedTableDto; -import at.tuwien.querystore.Query; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Configuration; import org.springframework.core.io.ClassPathResource; import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; -import java.io.IOException; import java.sql.*; import java.time.Instant; import java.util.*; diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/AccessEndpointUnitTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/AccessEndpointUnitTest.java index 3beb5626b3b8a2a001ac365a376cffea68b7b3ee..a2cbae3ea89ef15be88121a5ced8c47374601319 100644 --- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/AccessEndpointUnitTest.java +++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/AccessEndpointUnitTest.java @@ -15,12 +15,14 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.security.test.context.support.WithMockUser; import org.springframework.test.context.junit.jupiter.SpringExtension; import java.sql.SQLException; -import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; @Log4j2 @@ -54,7 +56,9 @@ public class AccessEndpointUnitTest extends AbstractUnitTest { .thenReturn(USER_4_PRIVILEGED_DTO); /* test */ - accessEndpoint.create(DATABASE_1_ID, USER_4_ID, UPDATE_DATABASE_ACCESS_READ_DTO); + final ResponseEntity<Void> response = accessEndpoint.create(DATABASE_1_ID, USER_4_ID, UPDATE_DATABASE_ACCESS_READ_DTO); + assertEquals(HttpStatus.CREATED, response.getStatusCode()); + assertNull(response.getBody()); } @Test @@ -150,7 +154,9 @@ public class AccessEndpointUnitTest extends AbstractUnitTest { .thenReturn(USER_1_DTO); /* test */ - accessEndpoint.update(DATABASE_1_ID, USER_1_ID, UPDATE_DATABASE_ACCESS_READ_DTO); + final ResponseEntity<Void> response = accessEndpoint.update(DATABASE_1_ID, USER_1_ID, UPDATE_DATABASE_ACCESS_READ_DTO); + assertEquals(HttpStatus.ACCEPTED, response.getStatusCode()); + assertNull(response.getBody()); } @Test @@ -250,7 +256,9 @@ public class AccessEndpointUnitTest extends AbstractUnitTest { .delete(any(PrivilegedDatabaseDto.class), any(UserDto.class)); /* test */ - accessEndpoint.revoke(DATABASE_1_ID, USER_1_ID); + final ResponseEntity<Void> response = accessEndpoint.revoke(DATABASE_1_ID, USER_1_ID); + assertEquals(HttpStatus.ACCEPTED, response.getStatusCode()); + assertNull(response.getBody()); } @Test diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java index 358a04008f9faeca8daf5b898df238fa5142a14f..9ab7082d0eb40da175947314c13b669de9f02476 100644 --- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java +++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java @@ -368,9 +368,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest { /* mock */ when(metadataServiceGateway.getDatabaseById(DATABASE_3_ID)) .thenReturn(DATABASE_3_PRIVILEGED_DTO); - when(metadataServiceGateway.getSystemUserId()) - .thenReturn(USER_LOCAL_ADMIN_ID); - when(subsetService.execute(eq(DATABASE_3_PRIVILEGED_DTO), anyString(), any(Instant.class), eq(USER_LOCAL_ADMIN_ID), eq(0L), eq(10L), eq(null), eq(null))) + when(subsetService.execute(eq(DATABASE_3_PRIVILEGED_DTO), anyString(), any(Instant.class), eq(null), eq(0L), eq(10L), eq(null), eq(null))) .thenReturn(QUERY_5_RESULT_DTO); /* test */ diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java index f03f4c3f189c98013e6fa5d4bb01a5cfd4fe211f..cc4b957226a77fbe4f39aef09193b29def575959 100644 --- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java +++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java @@ -2,13 +2,14 @@ package at.tuwien.endpoint; import at.tuwien.ExportResourceDto; import at.tuwien.api.database.DatabaseAccessDto; -import at.tuwien.api.database.query.ImportCsvDto; +import at.tuwien.api.database.query.ImportDto; import at.tuwien.api.database.query.QueryResultDto; import at.tuwien.api.database.table.*; import at.tuwien.api.database.table.internal.PrivilegedTableDto; import at.tuwien.endpoints.TableEndpoint; import at.tuwien.exception.*; import at.tuwien.gateway.MetadataServiceGateway; +import at.tuwien.service.SchemaService; import at.tuwien.service.TableService; import at.tuwien.test.AbstractUnitTest; import jakarta.servlet.http.HttpServletRequest; @@ -54,6 +55,9 @@ public class TableEndpointUnitTest extends AbstractUnitTest { @MockBean private TableService tableService; + @MockBean + private SchemaService schemaService; + @MockBean private MetadataServiceGateway metadataServiceGateway; @@ -89,6 +93,8 @@ public class TableEndpointUnitTest extends AbstractUnitTest { .thenReturn(DATABASE_1_PRIVILEGED_DTO); when(tableService.createTable(DATABASE_1_PRIVILEGED_DTO, TABLE_4_CREATE_INTERNAL_DTO)) .thenReturn(TABLE_4_DTO); + when(schemaService.inspectTable(DATABASE_1_PRIVILEGED_DTO, TABLE_4_INTERNALNAME)) + .thenReturn(TABLE_4_DTO); /* test */ final ResponseEntity<TableDto> response = tableEndpoint.create(DATABASE_1_ID, TABLE_4_CREATE_INTERNAL_DTO); @@ -1261,7 +1267,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest { public void importDataset_succeeds() throws DatabaseUnavailableException, TableNotFoundException, SidecarImportException, NotAllowedException, QueryMalformedException, RemoteUnavailableException, StorageNotFoundException, SQLException, MetadataServiceException { - final ImportCsvDto request = ImportCsvDto.builder() + final ImportDto request = ImportDto.builder() .skipLines(1L) .lineTermination(null) .location("deadbeef") @@ -1287,7 +1293,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest { @Test @WithMockUser(username = USER_4_USERNAME) public void importDataset_noRole_fails() { - final ImportCsvDto request = ImportCsvDto.builder() + final ImportDto request = ImportDto.builder() .skipLines(1L) .lineTermination("\\n") .location("deadbeef") @@ -1303,7 +1309,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest { @WithMockUser(username = USER_3_USERNAME, authorities = {"insert-table-data"}) public void importDataset_tableNotFound_fails() throws TableNotFoundException, RemoteUnavailableException, MetadataServiceException { - final ImportCsvDto request = ImportCsvDto.builder() + final ImportDto request = ImportDto.builder() .skipLines(1L) .lineTermination("\\n") .location("deadbeef") @@ -1325,7 +1331,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest { public void importDataset_unavailable_fails() throws RemoteUnavailableException, SidecarImportException, SQLException, QueryMalformedException, StorageNotFoundException, TableNotFoundException, MetadataServiceException, NotAllowedException { - final ImportCsvDto request = ImportCsvDto.builder() + final ImportDto request = ImportDto.builder() .skipLines(1L) .lineTermination("\\n") .location("deadbeef") @@ -1351,7 +1357,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest { public void importDataset_writeOwnAccess_fails() throws RemoteUnavailableException, SidecarImportException, SQLException, QueryMalformedException, StorageNotFoundException, TableNotFoundException, MetadataServiceException, NotAllowedException { - final ImportCsvDto request = ImportCsvDto.builder() + final ImportDto request = ImportDto.builder() .skipLines(1L) .lineTermination("\\n") .location("deadbeef") @@ -1376,7 +1382,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest { @WithMockUser(username = USER_3_USERNAME, authorities = {"insert-table-data"}) public void importDataset_readAccess_fails() throws TableNotFoundException, RemoteUnavailableException, NotAllowedException, MetadataServiceException { - final ImportCsvDto request = ImportCsvDto.builder() + final ImportDto request = ImportDto.builder() .skipLines(1L) .lineTermination("\\n") .location("deadbeef") @@ -1399,7 +1405,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest { public void importDataset_writeOwnAccess_succeeds() throws TableNotFoundException, RemoteUnavailableException, NotAllowedException, DatabaseUnavailableException, SidecarImportException, QueryMalformedException, StorageNotFoundException, MetadataServiceException { - final ImportCsvDto request = ImportCsvDto.builder() + final ImportDto request = ImportDto.builder() .skipLines(1L) .lineTermination("\\n") .location("deadbeef") @@ -1419,7 +1425,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest { @WithMockUser(username = USER_3_USERNAME, authorities = {"insert-table-data"}) public void importDataset_writeOwnAccessForeign_fails() throws TableNotFoundException, RemoteUnavailableException, NotAllowedException, MetadataServiceException { - final ImportCsvDto request = ImportCsvDto.builder() + final ImportDto request = ImportDto.builder() .skipLines(1L) .lineTermination("\\n") .location("deadbeef") @@ -1442,7 +1448,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest { public void importDataset_writeAllAccessForeign_succeeds() throws TableNotFoundException, RemoteUnavailableException, NotAllowedException, DatabaseUnavailableException, SidecarImportException, QueryMalformedException, StorageNotFoundException, MetadataServiceException { - final ImportCsvDto request = ImportCsvDto.builder() + final ImportDto request = ImportDto.builder() .skipLines(1L) .lineTermination("\\n") .location("deadbeef") @@ -1463,7 +1469,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest { public void importDataset_privateForeign_succeeds() throws TableNotFoundException, RemoteUnavailableException, NotAllowedException, DatabaseUnavailableException, SidecarImportException, QueryMalformedException, StorageNotFoundException, MetadataServiceException { - final ImportCsvDto request = ImportCsvDto.builder() + final ImportDto request = ImportDto.builder() .skipLines(1L) .lineTermination("\\n") .location("deadbeef") @@ -1484,7 +1490,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest { public void importDataset_private_succeeds() throws TableNotFoundException, RemoteUnavailableException, NotAllowedException, DatabaseUnavailableException, SidecarImportException, QueryMalformedException, StorageNotFoundException, MetadataServiceException { - final ImportCsvDto request = ImportCsvDto.builder() + final ImportDto request = ImportDto.builder() .skipLines(1L) .lineTermination("\\n") .location("deadbeef") @@ -1504,7 +1510,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest { @WithMockUser(username = USER_2_USERNAME, authorities = {"insert-table-data"}) public void importDataset_privateForeign_fails() throws TableNotFoundException, RemoteUnavailableException, NotAllowedException, MetadataServiceException { - final ImportCsvDto request = ImportCsvDto.builder() + final ImportDto request = ImportDto.builder() .skipLines(1L) .lineTermination("\\n") .location("deadbeef") @@ -1526,7 +1532,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest { @WithMockUser(username = USER_2_USERNAME, authorities = {"insert-table-data"}) public void importDataset_privateReadAccess_fails() throws TableNotFoundException, RemoteUnavailableException, NotAllowedException, MetadataServiceException { - final ImportCsvDto request = ImportCsvDto.builder() + final ImportDto request = ImportDto.builder() .skipLines(1L) .lineTermination("\\n") .location("deadbeef") diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/gateway/MetadataServiceGatewayUnitTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/gateway/MetadataServiceGatewayUnitTest.java index c224af4cb28604c2554da1942a782932db8d7390..44b01b1e9e2919721e446d7e183fbbcb739ea813 100644 --- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/gateway/MetadataServiceGatewayUnitTest.java +++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/gateway/MetadataServiceGatewayUnitTest.java @@ -175,8 +175,12 @@ public class MetadataServiceGatewayUnitTest extends AbstractUnitTest { MetadataServiceException { /* mock */ + final HttpHeaders headers = new HttpHeaders(); + headers.set("X-Username", CONTAINER_1_PRIVILEGED_USERNAME); + headers.set("X-Password", CONTAINER_1_PRIVILEGED_PASSWORD); when(restTemplate.exchange(anyString(), eq(HttpMethod.GET), eq(HttpEntity.EMPTY), eq(PrivilegedDatabaseDto[].class))) .thenReturn(ResponseEntity.status(HttpStatus.OK) + .headers(headers) .body(new PrivilegedDatabaseDto[]{DATABASE_1_PRIVILEGED_DTO})); /* test */ @@ -221,7 +225,7 @@ public class MetadataServiceGatewayUnitTest extends AbstractUnitTest { .build()); /* test */ - assertThrows(MetadataServiceException.class, () -> { + assertThrows(DatabaseNotFoundException.class, () -> { metadataServiceGateway.getDatabaseByInternalName(DATABASE_1_INTERNALNAME); }); } diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java index b331a03fa60915290380b5ff233fd51a0996ee36..f4bd429a9039ab2f2fb36c4fc2fce653ceda8054 100644 --- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java +++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java @@ -1,7 +1,7 @@ package at.tuwien.mvc; import at.tuwien.api.database.query.ExecuteStatementDto; -import at.tuwien.api.database.query.ImportCsvDto; +import at.tuwien.api.database.query.ImportDto; import at.tuwien.api.database.query.QueryPersistDto; import at.tuwien.api.database.table.TupleDeleteDto; import at.tuwien.api.database.table.TupleDto; @@ -201,7 +201,7 @@ public class PrometheusEndpointMvcTest extends AbstractUnitTest { /* ignore */ } try { - tableEndpoint.importDataset(DATABASE_1_ID, TABLE_1_ID, ImportCsvDto.builder().build(), USER_1_PRINCIPAL); + tableEndpoint.importDataset(DATABASE_1_ID, TABLE_1_ID, ImportDto.builder().build(), USER_1_PRINCIPAL); } catch (Exception e) { /* ignore */ } diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/SchemaServiceIntegrationTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/SchemaServiceIntegrationTest.java index d69ea86e683faf960e7dd0b64ab01b10144c44c3..85e2b8071174adf3bb401a286cf1e595713e279c 100644 --- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/SchemaServiceIntegrationTest.java +++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/SchemaServiceIntegrationTest.java @@ -1,6 +1,5 @@ package at.tuwien.service; -import at.tuwien.api.container.image.ImageDateDto; import at.tuwien.api.database.ViewColumnDto; import at.tuwien.api.database.ViewDto; import at.tuwien.api.database.table.TableBriefDto; @@ -16,7 +15,8 @@ import at.tuwien.api.database.table.constraints.unique.UniqueDto; import at.tuwien.api.identifier.IdentifierDto; import at.tuwien.config.MariaDbConfig; import at.tuwien.config.MariaDbContainerConfig; -import at.tuwien.exception.*; +import at.tuwien.exception.TableNotFoundException; +import at.tuwien.exception.ViewNotFoundException; import at.tuwien.test.AbstractUnitTest; import lombok.extern.log4j.Log4j2; import org.junit.jupiter.api.BeforeEach; @@ -74,11 +74,11 @@ public class SchemaServiceIntegrationTest extends AbstractUnitTest { final List<ColumnDto> columns = response.getColumns(); assertNotNull(columns); assertEquals(5, columns.size()); - assertColumn(columns.get(0), null, null, DATABASE_1_ID, "id", "id", ColumnTypeDto.BIGINT, 19L, 0L, false, null, null); - assertColumn(columns.get(1), null, null, DATABASE_1_ID, "given_name", "given_name", ColumnTypeDto.VARCHAR, 255L, null, false, null, null); - assertColumn(columns.get(2), null, null, DATABASE_1_ID, "middle_name", "middle_name", ColumnTypeDto.VARCHAR, 255L, null, true, null, null); - assertColumn(columns.get(3), null, null, DATABASE_1_ID, "family_name", "family_name", ColumnTypeDto.VARCHAR, 255L, null, false, null, null); - assertColumn(columns.get(4), null, null, DATABASE_1_ID, "age", "age", ColumnTypeDto.INT, 10L, 0L, false, null, null); + assertColumn(columns.get(0), null, null, DATABASE_1_ID, "id", "id", ColumnTypeDto.BIGINT, 19L, 0L, false, null); + assertColumn(columns.get(1), null, null, DATABASE_1_ID, "given_name", "given_name", ColumnTypeDto.VARCHAR, 255L, null, false, null); + assertColumn(columns.get(2), null, null, DATABASE_1_ID, "middle_name", "middle_name", ColumnTypeDto.VARCHAR, 255L, null, true, null); + assertColumn(columns.get(3), null, null, DATABASE_1_ID, "family_name", "family_name", ColumnTypeDto.VARCHAR, 255L, null, false, null); + assertColumn(columns.get(4), null, null, DATABASE_1_ID, "age", "age", ColumnTypeDto.INT, 10L, 0L, false, null); final ConstraintsDto constraints = response.getConstraints(); assertNotNull(constraints); final Set<PrimaryKeyDto> primaryKey = constraints.getPrimaryKey(); @@ -127,11 +127,11 @@ public class SchemaServiceIntegrationTest extends AbstractUnitTest { final List<ColumnDto> columns = response.getColumns(); assertNotNull(columns); assertEquals(3, columns.size()); - assertColumn(columns.get(0), null, null, DATABASE_2_ID, "id", "id", ColumnTypeDto.BIGINT, 19L, 0L, false, null, null); - assertColumn(columns.get(1), null, null, DATABASE_2_ID, "mode", "mode", ColumnTypeDto.ENUM, 3L, null, false, null, null); + assertColumn(columns.get(0), null, null, DATABASE_2_ID, "id", "id", ColumnTypeDto.BIGINT, 19L, 0L, false, null); + assertColumn(columns.get(1), null, null, DATABASE_2_ID, "mode", "mode", ColumnTypeDto.ENUM, 3L, null, false, null); assertEquals(2, columns.get(1).getEnums().size()); assertEquals(List.of("ABC", "DEF"), columns.get(1).getEnums()); - assertColumn(columns.get(2), null, null, DATABASE_2_ID, "seq", "seq", ColumnTypeDto.SET, 5L, null, true, null, null); + assertColumn(columns.get(2), null, null, DATABASE_2_ID, "seq", "seq", ColumnTypeDto.SET, 5L, null, true, null); assertEquals(3, columns.get(2).getSets().size()); assertEquals(List.of("1", "2", "3"), columns.get(2).getSets()); /* ignore rest (constraints) */ @@ -167,11 +167,11 @@ public class SchemaServiceIntegrationTest extends AbstractUnitTest { final List<ColumnDto> columns = response.getColumns(); assertNotNull(columns); assertEquals(5, columns.size()); - assertColumn(columns.get(0), null, null, DATABASE_1_ID, "id", "id", ColumnTypeDto.BIGINT, 19L, 0L, false, null, null); - assertColumn(columns.get(1), null, null, DATABASE_1_ID, "date", "date", ColumnTypeDto.DATE, null, null, false, IMAGE_DATE_1_ID, null); - assertColumn(columns.get(2), null, null, DATABASE_1_ID, "location", "location", ColumnTypeDto.VARCHAR, 255L, null, true, null, "Closest city"); - assertColumn(columns.get(3), null, null, DATABASE_1_ID, "mintemp", "mintemp", ColumnTypeDto.DOUBLE, 22L, null, true, null, null); - assertColumn(columns.get(4), null, null, DATABASE_1_ID, "rainfall", "rainfall", ColumnTypeDto.DOUBLE, 22L, null, true, null, null); + assertColumn(columns.get(0), null, null, DATABASE_1_ID, "id", "id", ColumnTypeDto.BIGINT, 19L, 0L, false, null); + assertColumn(columns.get(1), null, null, DATABASE_1_ID, "date", "date", ColumnTypeDto.DATE, null, null, false, null); + assertColumn(columns.get(2), null, null, DATABASE_1_ID, "location", "location", ColumnTypeDto.VARCHAR, 255L, null, true, "Closest city"); + assertColumn(columns.get(3), null, null, DATABASE_1_ID, "mintemp", "mintemp", ColumnTypeDto.DOUBLE, 22L, null, true, null); + assertColumn(columns.get(4), null, null, DATABASE_1_ID, "rainfall", "rainfall", ColumnTypeDto.DOUBLE, 22L, null, true, null); final ConstraintsDto constraints = response.getConstraints(); final List<PrimaryKeyDto> primaryKey = new LinkedList<>(constraints.getPrimaryKey()); assertEquals(1, primaryKey.size()); @@ -359,9 +359,9 @@ public class SchemaServiceIntegrationTest extends AbstractUnitTest { assertEquals(ColumnTypeDto.BOOL, pk0.getColumn().getColumnType()); final List<ColumnDto> columns = response.getColumns(); assertEquals(3, columns.size()); - assertColumn(columns.get(0), null, null, DATABASE_1_ID, "bool_default", "bool_default", ColumnTypeDto.BOOL, null, 0L, false, null, null); - assertColumn(columns.get(1), null, null, DATABASE_1_ID, "bool_tinyint", "bool_tinyint", ColumnTypeDto.BOOL, null, 0L, false, null, null); - assertColumn(columns.get(2), null, null, DATABASE_1_ID, "bool_tinyint_unsigned", "bool_tinyint_unsigned", ColumnTypeDto.BOOL, null, 0L, false, null, null); + assertColumn(columns.get(0), null, null, DATABASE_1_ID, "bool_default", "bool_default", ColumnTypeDto.BOOL, null, 0L, false, null); + assertColumn(columns.get(1), null, null, DATABASE_1_ID, "bool_tinyint", "bool_tinyint", ColumnTypeDto.BOOL, null, 0L, false, null); + assertColumn(columns.get(2), null, null, DATABASE_1_ID, "bool_tinyint_unsigned", "bool_tinyint_unsigned", ColumnTypeDto.BOOL, null, 0L, false, null); } @Test @@ -398,9 +398,9 @@ public class SchemaServiceIntegrationTest extends AbstractUnitTest { assertEquals(DATABASE_1_ID, column3.getDatabaseId()); } - protected static void assertViewColumn(ViewColumnDto column, Long id, Long databaseId, String name, String internalName, - ColumnTypeDto type, Long size, Long d, Boolean nullAllowed, - ImageDateDto dateFormat, String description) { + protected static void assertViewColumn(ViewColumnDto column, Long id, Long databaseId, String name, + String internalName, ColumnTypeDto type, Long size, Long d, + Boolean nullAllowed, String description) { log.trace("assert column: {}", internalName); assertNotNull(column); assertEquals(id, column.getId()); @@ -412,17 +412,11 @@ public class SchemaServiceIntegrationTest extends AbstractUnitTest { assertEquals(d, column.getD()); assertEquals(nullAllowed, column.getIsNullAllowed()); assertEquals(description, column.getDescription()); - if (dateFormat != null) { - assertNotNull(column.getDateFormat()); - assertEquals(dateFormat.getId(), column.getDateFormat().getId()); - } else { - assertNull(column.getDateFormat()); - } } protected static void assertColumn(ColumnDto column, Long id, Long tableId, Long databaseId, String name, String internalName, ColumnTypeDto type, Long size, Long d, Boolean nullAllowed, - Long dfid, String description) { + String description) { log.trace("assert column: {}", internalName); assertNotNull(column); assertEquals(id, column.getId()); @@ -437,12 +431,6 @@ public class SchemaServiceIntegrationTest extends AbstractUnitTest { assertEquals(d, column.getD()); assertEquals(nullAllowed, column.getIsNullAllowed()); assertEquals(description, column.getDescription()); - if (dfid != null) { - assertNotNull(column.getDateFormat()); - assertEquals(dfid, column.getDateFormat().getId()); - } else { - assertNull(column.getDateFormat()); - } } } diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationTest.java index 4ebaba4931327a80ba319ae6407e3dc3547e1dc7..f21fc0e5399610af4d28d217d9866de9687a00d8 100644 --- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationTest.java +++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationTest.java @@ -1,7 +1,7 @@ package at.tuwien.service; import at.tuwien.ExportResourceDto; -import at.tuwien.api.database.query.ImportCsvDto; +import at.tuwien.api.database.query.ImportDto; import at.tuwien.api.database.query.QueryResultDto; import at.tuwien.api.database.table.*; import at.tuwien.api.database.table.columns.ColumnCreateDto; @@ -19,7 +19,6 @@ import at.tuwien.api.database.table.internal.TableCreateDto; import at.tuwien.config.MariaDbConfig; import at.tuwien.config.MariaDbContainerConfig; import at.tuwien.config.QueryConfig; -import at.tuwien.config.S3Config; import at.tuwien.exception.*; import at.tuwien.gateway.DataDatabaseSidecarGateway; import at.tuwien.gateway.MetadataServiceGateway; @@ -45,15 +44,11 @@ import java.io.IOException; import java.io.InputStream; import java.math.BigDecimal; import java.math.BigInteger; -import java.nio.charset.Charset; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermissions; import java.sql.SQLException; import java.time.Instant; import java.util.*; import static at.tuwien.service.SchemaServiceIntegrationTest.assertColumn; -import static at.tuwien.service.SchemaServiceIntegrationTest.assertViewColumn; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.doNothing; @@ -383,9 +378,9 @@ public class TableServiceIntegrationTest extends AbstractUnitTest { final List<ColumnDto> columns0 = table0.getColumns(); assertNotNull(columns0); Assertions.assertEquals(3, columns0.size()); - assertColumn(columns0.get(0), null, null, DATABASE_1_ID, "id", "id", ColumnTypeDto.BIGINT, 19L, 0L, false, null, null); - assertColumn(columns0.get(1), null, null, DATABASE_1_ID, "weather_id", "weather_id", ColumnTypeDto.BIGINT, 19L, 0L, false, null, null); - assertColumn(columns0.get(2), null, null, DATABASE_1_ID, "other_id", "other_id", ColumnTypeDto.BIGINT, 19L, 0L, false, null, null); + assertColumn(columns0.get(0), null, null, DATABASE_1_ID, "id", "id", ColumnTypeDto.BIGINT, 19L, 0L, false, null); + assertColumn(columns0.get(1), null, null, DATABASE_1_ID, "weather_id", "weather_id", ColumnTypeDto.BIGINT, 19L, 0L, false, null); + assertColumn(columns0.get(2), null, null, DATABASE_1_ID, "other_id", "other_id", ColumnTypeDto.BIGINT, 19L, 0L, false, null); final ConstraintsDto constraints0 = table0.getConstraints(); assertNotNull(constraints0); assertEquals(1, constraints0.getPrimaryKey().size()); @@ -430,8 +425,8 @@ public class TableServiceIntegrationTest extends AbstractUnitTest { final List<ColumnDto> columns1 = table1.getColumns(); assertNotNull(columns1); Assertions.assertEquals(2, columns1.size()); - assertColumn(columns1.get(0), null, null, DATABASE_1_ID, "id", "id", ColumnTypeDto.BIGINT, 19L, 0L, false, null, null); - assertColumn(columns1.get(1), null, null, DATABASE_1_ID, "other_id", "other_id", ColumnTypeDto.BIGINT, 19L, 0L, false, null, null); + assertColumn(columns1.get(0), null, null, DATABASE_1_ID, "id", "id", ColumnTypeDto.BIGINT, 19L, 0L, false, null); + assertColumn(columns1.get(1), null, null, DATABASE_1_ID, "other_id", "other_id", ColumnTypeDto.BIGINT, 19L, 0L, false, null); final ConstraintsDto constraints1 = table1.getConstraints(); assertNotNull(constraints1); assertEquals(2, constraints1.getPrimaryKey().size()); @@ -458,9 +453,9 @@ public class TableServiceIntegrationTest extends AbstractUnitTest { final List<ColumnDto> columns2 = table2.getColumns(); assertNotNull(columns2); Assertions.assertEquals(3, columns2.size()); - assertColumn(columns2.get(0), null, null, DATABASE_1_ID, "bool_default", "bool_default", ColumnTypeDto.BOOL, null, 0L, false, null, null); - assertColumn(columns2.get(1), null, null, DATABASE_1_ID, "bool_tinyint", "bool_tinyint", ColumnTypeDto.BOOL, null, 0L, false, null, null); - assertColumn(columns2.get(2), null, null, DATABASE_1_ID, "bool_tinyint_unsigned", "bool_tinyint_unsigned", ColumnTypeDto.BOOL, null, 0L, false, null, null); + assertColumn(columns2.get(0), null, null, DATABASE_1_ID, "bool_default", "bool_default", ColumnTypeDto.BOOL, null, 0L, false, null); + assertColumn(columns2.get(1), null, null, DATABASE_1_ID, "bool_tinyint", "bool_tinyint", ColumnTypeDto.BOOL, null, 0L, false, null); + assertColumn(columns2.get(2), null, null, DATABASE_1_ID, "bool_tinyint_unsigned", "bool_tinyint_unsigned", ColumnTypeDto.BOOL, null, 0L, false, null); final ConstraintsDto constraints2 = table2.getConstraints(); assertNotNull(constraints2); final Set<PrimaryKeyDto> primaryKey2 = constraints2.getPrimaryKey(); @@ -479,11 +474,11 @@ public class TableServiceIntegrationTest extends AbstractUnitTest { final List<ColumnDto> columns3 = table3.getColumns(); assertNotNull(columns3); Assertions.assertEquals(5, columns3.size()); - assertColumn(columns3.get(0), null, null, DATABASE_1_ID, "id", "id", ColumnTypeDto.BIGINT, 19L, 0L, false, null, null); - assertColumn(columns3.get(1), null, null, DATABASE_1_ID, "given_name", "given_name", ColumnTypeDto.VARCHAR, 255L, null, false, null, null); - assertColumn(columns3.get(2), null, null, DATABASE_1_ID, "middle_name", "middle_name", ColumnTypeDto.VARCHAR, 255L, null, true, null, null); - assertColumn(columns3.get(3), null, null, DATABASE_1_ID, "family_name", "family_name", ColumnTypeDto.VARCHAR, 255L, null, false, null, null); - assertColumn(columns3.get(4), null, null, DATABASE_1_ID, "age", "age", ColumnTypeDto.INT, 10L, 0L, false, null, null); + assertColumn(columns3.get(0), null, null, DATABASE_1_ID, "id", "id", ColumnTypeDto.BIGINT, 19L, 0L, false, null); + assertColumn(columns3.get(1), null, null, DATABASE_1_ID, "given_name", "given_name", ColumnTypeDto.VARCHAR, 255L, null, false, null); + assertColumn(columns3.get(2), null, null, DATABASE_1_ID, "middle_name", "middle_name", ColumnTypeDto.VARCHAR, 255L, null, true, null); + assertColumn(columns3.get(3), null, null, DATABASE_1_ID, "family_name", "family_name", ColumnTypeDto.VARCHAR, 255L, null, false, null); + assertColumn(columns3.get(4), null, null, DATABASE_1_ID, "age", "age", ColumnTypeDto.INT, 10L, 0L, false, null); final ConstraintsDto constraints3 = table3.getConstraints(); assertNotNull(constraints3); final Set<PrimaryKeyDto> primaryKey3 = constraints3.getPrimaryKey(); @@ -509,8 +504,8 @@ public class TableServiceIntegrationTest extends AbstractUnitTest { assertEquals(TABLE_4_INTERNALNAME, response.getInternalName()); final List<ColumnDto> columns = response.getColumns(); assertEquals(TABLE_4_COLUMNS.size(), columns.size()); - assertColumn(columns.get(0), null, null, DATABASE_1_ID, "timestamp", "timestamp", ColumnTypeDto.TIMESTAMP, null, null, false, queryConfig.getDefaultTimestampFormatId(), null); - assertColumn(columns.get(1), null, null, DATABASE_1_ID, "value", "value", ColumnTypeDto.DECIMAL, 10L, 10L, true, null, null); + assertColumn(columns.get(0), null, null, DATABASE_1_ID, "timestamp", "timestamp", ColumnTypeDto.TIMESTAMP, null, null, false, null); + assertColumn(columns.get(1), null, null, DATABASE_1_ID, "value", "value", ColumnTypeDto.DECIMAL, 10L, 10L, true, null); final ConstraintsDto constraints = response.getConstraints(); assertNotNull(constraints); final Set<PrimaryKeyDto> primaryKey = constraints.getPrimaryKey(); @@ -619,9 +614,9 @@ public class TableServiceIntegrationTest extends AbstractUnitTest { assertEquals("composite_primary_key", response.getInternalName()); final List<ColumnDto> columns = response.getColumns(); assertEquals(3, columns.size()); - assertColumn(columns.get(0), null, null, DATABASE_1_ID, "name", "name", ColumnTypeDto.VARCHAR, 255L, null, false, null, null); - assertColumn(columns.get(1), null, null, DATABASE_1_ID, "lat", "lat", ColumnTypeDto.DECIMAL, 10L, 10L, false, null, null); - assertColumn(columns.get(2), null, null, DATABASE_1_ID, "lng", "lng", ColumnTypeDto.DECIMAL, 10L, 10L, false, null, null); + assertColumn(columns.get(0), null, null, DATABASE_1_ID, "name", "name", ColumnTypeDto.VARCHAR, 255L, null, false, null); + assertColumn(columns.get(1), null, null, DATABASE_1_ID, "lat", "lat", ColumnTypeDto.DECIMAL, 10L, 10L, false, null); + assertColumn(columns.get(2), null, null, DATABASE_1_ID, "lng", "lng", ColumnTypeDto.DECIMAL, 10L, 10L, false, null); final ConstraintsDto constraints = response.getConstraints(); assertNotNull(constraints); final Set<String> checks = constraints.getChecks(); @@ -768,53 +763,6 @@ public class TableServiceIntegrationTest extends AbstractUnitTest { }); } - @Test - public void importDataset_withSeparatorAndQuoteAndNullElement_succeeds() throws SidecarImportException, - SQLException, QueryMalformedException, RemoteUnavailableException, StorageNotFoundException, IOException { - final ImportCsvDto request = ImportCsvDto.builder() - .location("weather_aus.csv") - .separator(';') - .quote('"') - .nullElement("NA") - .build(); - - /* mock */ - final File source = new File("src/test/resources/csv/weather_aus.csv"); - final File target = new File("/tmp/weather_aus.csv") /* must be /tmp */; - log.trace("copy dataset from {} to {}", source.toPath().toAbsolutePath(), target.toPath().toAbsolutePath()); - FileUtils.copyFile(source, target); - doNothing() - .when(dataDatabaseSidecarGateway) - .importFile(anyString(), anyInt(), eq("weather_aus.csv")); - - /* test */ - tableService.importDataset(TABLE_1_PRIVILEGED_DTO, request); - } - - @Test - public void importDataset_malformedData_fails() throws RemoteUnavailableException, StorageNotFoundException, - IOException, SidecarImportException { - final ImportCsvDto request = ImportCsvDto.builder() - .location("weather_aus.csv") - .separator(';') - .quote('"') - .build(); - - /* mock */ - final File source = new File("src/test/resources/csv/weather_aus.csv"); - final File target = new File("/tmp/weather_aus.csv"); - log.trace("copy dataset from {} to {}", source.toPath().toAbsolutePath(), target.toPath().toAbsolutePath()); - FileUtils.copyFile(source, target); - doNothing() - .when(dataDatabaseSidecarGateway) - .importFile(anyString(), anyInt(), eq("weather_aus.csv")); - - /* test */ - assertThrows(QueryMalformedException.class, () -> { - tableService.importDataset(TABLE_1_PRIVILEGED_DTO, request); - }); - } - @Test public void exportDataset_succeeds() throws SQLException, QueryMalformedException, RemoteUnavailableException, StorageNotFoundException, StorageUnavailableException, SidecarExportException { diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/ViewServiceIntegrationTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/ViewServiceIntegrationTest.java index 20a769f8ce8e8b8a5f4cdf1f4e42610e0e3f561d..5f20464eb9f252164c037a2155cd661d24d9019a 100644 --- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/ViewServiceIntegrationTest.java +++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/ViewServiceIntegrationTest.java @@ -3,7 +3,6 @@ package at.tuwien.service; import at.tuwien.api.database.ViewColumnDto; import at.tuwien.api.database.ViewDto; import at.tuwien.api.database.query.QueryResultDto; -import at.tuwien.api.database.table.columns.ColumnDto; import at.tuwien.config.MariaDbConfig; import at.tuwien.config.MariaDbContainerConfig; import at.tuwien.exception.*; @@ -24,7 +23,6 @@ import java.time.Instant; import java.util.List; import java.util.Map; -import static at.tuwien.service.SchemaServiceIntegrationTest.assertViewColumn; import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -68,11 +66,11 @@ public class ViewServiceIntegrationTest extends AbstractUnitTest { final List<ViewColumnDto> columns = response.getColumns(); assertEquals(VIEW_1_COLUMNS.size(), columns.size()); ViewColumnDto ref = VIEW_1_COLUMNS_DTO.get(0); - SchemaServiceIntegrationTest.assertViewColumn(columns.get(0), null, ref.getDatabaseId(), ref.getName(), ref.getInternalName(), ref.getColumnType(), ref.getSize(), ref.getD(), ref.getIsNullAllowed(), ref.getDateFormat(), ref.getDescription()); + SchemaServiceIntegrationTest.assertViewColumn(columns.get(0), null, ref.getDatabaseId(), ref.getName(), ref.getInternalName(), ref.getColumnType(), ref.getSize(), ref.getD(), ref.getIsNullAllowed(), ref.getDescription()); ref = VIEW_1_COLUMNS_DTO.get(1); - SchemaServiceIntegrationTest.assertViewColumn(columns.get(1), null, ref.getDatabaseId(), ref.getName(), ref.getInternalName(), ref.getColumnType(), ref.getSize(), ref.getD(), ref.getIsNullAllowed(), ref.getDateFormat(), ref.getDescription()); + SchemaServiceIntegrationTest.assertViewColumn(columns.get(1), null, ref.getDatabaseId(), ref.getName(), ref.getInternalName(), ref.getColumnType(), ref.getSize(), ref.getD(), ref.getIsNullAllowed(), ref.getDescription()); ref = VIEW_1_COLUMNS_DTO.get(2); - SchemaServiceIntegrationTest.assertViewColumn(columns.get(2), null, ref.getDatabaseId(), ref.getName(), ref.getInternalName(), ref.getColumnType(), ref.getSize(), ref.getD(), ref.getIsNullAllowed(), ref.getDateFormat(), ref.getDescription()); + SchemaServiceIntegrationTest.assertViewColumn(columns.get(2), null, ref.getDatabaseId(), ref.getName(), ref.getInternalName(), ref.getColumnType(), ref.getSize(), ref.getD(), ref.getIsNullAllowed(), ref.getDescription()); } diff --git a/dbrepo-data-service/rest-service/src/test/resources/init/querystore.sql b/dbrepo-data-service/rest-service/src/test/resources/init/querystore.sql index c1df44d1b0766fb04d081f3b5b3679039d8ba72f..3e7471df3e1eb3a1bb79db7246ecb805a06f495c 100644 --- a/dbrepo-data-service/rest-service/src/test/resources/init/querystore.sql +++ b/dbrepo-data-service/rest-service/src/test/resources/init/querystore.sql @@ -1,5 +1,5 @@ CREATE SEQUENCE `qs_queries_seq` NOCACHE; -CREATE TABLE `qs_queries` ( `id` bigint not null primary key default nextval(`qs_queries_seq`), `created` datetime not null default now(), `executed` datetime not null default now(), `created_by` varchar(36) not null, `query` text not null, `query_normalized` text not null, `is_persisted` boolean not null, `query_hash` varchar(255) not null, `result_hash` varchar(255), `result_number` bigint ) WITH SYSTEM VERSIONING; +CREATE TABLE `qs_queries` ( `id` bigint not null primary key default nextval(`qs_queries_seq`), `created` datetime not null default now(), `executed` datetime not null default now(), `created_by` varchar(36), `query` text not null, `query_normalized` text not null, `is_persisted` boolean not null, `query_hash` varchar(255) not null, `result_hash` varchar(255), `result_number` bigint ) WITH SYSTEM VERSIONING; CREATE PROCEDURE hash_table(IN name VARCHAR(255), OUT hash VARCHAR(255), OUT count BIGINT) BEGIN DECLARE _sql TEXT; SELECT CONCAT('SELECT SHA2(GROUP_CONCAT(CONCAT_WS(\'\',', GROUP_CONCAT(CONCAT('`', column_name, '`') ORDER BY column_name), ') SEPARATOR \',\'), 256) AS hash, COUNT(*) AS count FROM `', name, '` INTO @hash, @count;') FROM `information_schema`.`columns` WHERE `table_schema` = DATABASE() AND `table_name` = name INTO _sql; PREPARE stmt FROM _sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; SET hash = @hash; SET count = @count; END; CREATE PROCEDURE store_query(IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _username varchar(255) DEFAULT REGEXP_REPLACE(current_user(), '@.*', ''); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END; CREATE DEFINER = 'root' PROCEDURE _store_query(IN _username VARCHAR(255), IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END; \ No newline at end of file diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java index 640ef7172ac2b750815e17a0bdf11b02b5ed997b..11a90afde703a353e9f5b539394806d6356fe73c 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java @@ -66,8 +66,11 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { log.error("Failed to find container with id {}: service responded unsuccessful: {}", containerId, response.getStatusCode()); throw new MetadataServiceException("Failed to find container: service responded unsuccessful: " + response.getStatusCode()); } - if (!response.getHeaders().keySet().containsAll(List.of("X-Username", "X-Password"))) { + final List<String> expectedHeaders = List.of("X-Username", "X-Password"); + if (!response.getHeaders().keySet().containsAll(expectedHeaders)) { log.error("Failed to find all privileged container headers"); + log.debug("expected headers: {}", expectedHeaders); + log.debug("found headers: {}", response.getHeaders().keySet()); throw new MetadataServiceException("Failed to find all privileged container headers"); } if (response.getBody() == null) { @@ -98,8 +101,11 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { log.error("Failed to find database with id {}: service responded unsuccessful: {}", id, response.getStatusCode()); throw new MetadataServiceException("Failed to find database: service responded unsuccessful: " + response.getStatusCode()); } - if (!response.getHeaders().keySet().containsAll(List.of("X-Username", "X-Password"))) { + final List<String> expectedHeaders = List.of("X-Username", "X-Password"); + if (!response.getHeaders().keySet().containsAll(expectedHeaders)) { log.error("Failed to find all privileged database headers"); + log.debug("expected headers: {}", expectedHeaders); + log.debug("found headers: {}", response.getHeaders().keySet()); throw new MetadataServiceException("Failed to find all privileged database headers"); } if (response.getBody() == null) { @@ -123,14 +129,22 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { log.error("Failed to find database with internal name {}: {}", internalName, e.getMessage()); throw new RemoteUnavailableException("Failed to find database: " + e.getMessage(), e); } - if (!response.getStatusCode().equals(HttpStatus.OK) || response.getBody() == null) { + if (!response.getStatusCode().equals(HttpStatus.OK)) { log.error("Failed to find database with internal name {}: service responded unsuccessful: {}", internalName, response.getStatusCode()); throw new MetadataServiceException("Failed to find database: service responded unsuccessful: " + response.getStatusCode()); } - if (response.getBody().length != 1) { + /* body first, then headers next */ + if (response.getBody() == null || response.getBody().length != 1) { log.error("Failed to find database with internal name {}: body is empty", internalName); throw new DatabaseNotFoundException("Failed to find database: body is empty"); } + final List<String> expectedHeaders = List.of("X-Username", "X-Password"); + if (!response.getHeaders().keySet().containsAll(expectedHeaders)) { + log.error("Failed to find all privileged database headers"); + log.debug("expected headers: {}", expectedHeaders); + log.debug("found headers: {}", response.getHeaders().keySet()); + throw new MetadataServiceException("Failed to find all privileged database headers"); + } return response.getBody()[0]; } @@ -151,8 +165,11 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { log.error("Failed to find table with id {}: service responded unsuccessful: {}", id, response.getStatusCode()); throw new MetadataServiceException("Failed to find table: service responded unsuccessful: " + response.getStatusCode()); } - if (!response.getHeaders().keySet().containsAll(List.of("X-Type", "X-Host", "X-Port", "X-Username", "X-Password", "X-Database", "X-Sidecar-Host", "X-Sidecar-Port"))) { + final List<String> expectedHeaders = List.of("X-Type", "X-Host", "X-Port", "X-Username", "X-Password", "X-Database", "X-Sidecar-Host", "X-Sidecar-Port"); + if (!response.getHeaders().keySet().containsAll(expectedHeaders)) { log.error("Failed to find all privileged table headers"); + log.debug("expected headers: {}", expectedHeaders); + log.debug("found headers: {}", response.getHeaders().keySet()); throw new MetadataServiceException("Failed to find all privileged table headers"); } if (response.getBody() == null) { @@ -189,8 +206,11 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { log.error("Failed to find view with id {}: service responded unsuccessful: {}", id, response.getStatusCode()); throw new MetadataServiceException("Failed to find view: service responded unsuccessful: " + response.getStatusCode()); } - if (!response.getHeaders().keySet().containsAll(List.of("X-Type", "X-Host", "X-Port", "X-Username", "X-Password", "X-Database"))) { + final List<String> expectedHeaders = List.of("X-Type", "X-Host", "X-Port", "X-Username", "X-Password", "X-Database"); + if (!response.getHeaders().keySet().containsAll(expectedHeaders)) { log.error("Failed to find all privileged view headers"); + log.debug("expected headers: {}", expectedHeaders); + log.debug("found headers: {}", response.getHeaders().keySet()); throw new MetadataServiceException("Failed to find all privileged view headers"); } if (response.getBody() == null) { @@ -276,8 +296,11 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { log.error("Failed to find user with id {}: service responded unsuccessful: {}", userId, response.getStatusCode()); throw new MetadataServiceException("Failed to find user: service responded unsuccessful: " + response.getStatusCode()); } - if (!response.getHeaders().keySet().containsAll(List.of("X-Username", "X-Password"))) { + final List<String> expectedHeaders = List.of("X-Username", "X-Password"); + if (!response.getHeaders().keySet().containsAll(expectedHeaders)) { log.error("Failed to find all privileged user headers"); + log.debug("expected headers: {}", expectedHeaders); + log.debug("found headers: {}", response.getHeaders().keySet()); throw new MetadataServiceException("Failed to find all privileged user headers"); } if (response.getBody() == null) { diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/DataMapper.java b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/DataMapper.java index b69d06b0aabd9bd76b686e9526a3cad52298c752..01854e0cb7ac4dfccadd6faa6004c88f1e4d3903 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/DataMapper.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/DataMapper.java @@ -1,6 +1,5 @@ package at.tuwien.mapper; -import at.tuwien.api.container.image.ImageDateDto; import at.tuwien.api.database.DatabaseDto; import at.tuwien.api.database.ViewColumnDto; import at.tuwien.api.database.ViewDto; @@ -45,7 +44,6 @@ import java.io.StringReader; import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.sql.*; -import java.sql.Date; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; @@ -53,7 +51,6 @@ import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; import java.util.*; import java.util.stream.Collectors; -import java.util.stream.Stream; @Mapper(componentModel = "spring") public interface DataMapper { @@ -177,19 +174,6 @@ public interface DataMapper { } else if (resultSet.getString(6) != null) { column.setSize(resultSet.getLong(6)); } - if (column.getColumnType().equals(ColumnTypeDto.TIMESTAMP) || column.getColumnType().equals(ColumnTypeDto.DATETIME)) { - column.setDateFormat(ImageDateDto.builder() - .id(queryConfig.getDefaultTimestampFormatId()) - .build()); - } else if (column.getColumnType().equals(ColumnTypeDto.DATE)) { - column.setDateFormat(ImageDateDto.builder() - .id(queryConfig.getDefaultDateFormatId()) - .build()); - } else if (column.getColumnType().equals(ColumnTypeDto.TIME)) { - column.setDateFormat(ImageDateDto.builder() - .id(queryConfig.getDefaultTimeFormatId()) - .build()); - } /* constraints */ if (resultSet.getString(9) != null && resultSet.getString(9).equals("PRI")) { table.getConstraints().getPrimaryKey().add(PrimaryKeyDto.builder() @@ -221,19 +205,6 @@ public interface DataMapper { } else if (resultSet.getString(6) != null) { column.setSize(resultSet.getLong(6)); } - if (column.getColumnType().equals(ColumnTypeDto.TIMESTAMP) || column.getColumnType().equals(ColumnTypeDto.DATETIME)) { - column.setDateFormat(ImageDateDto.builder() - .id(queryConfig.getDefaultTimestampFormatId()) - .build()); - } else if (column.getColumnType().equals(ColumnTypeDto.DATE)) { - column.setDateFormat(ImageDateDto.builder() - .id(queryConfig.getDefaultDateFormatId()) - .build()); - } else if (column.getColumnType().equals(ColumnTypeDto.TIME)) { - column.setDateFormat(ImageDateDto.builder() - .id(queryConfig.getDefaultTimeFormatId()) - .build()); - } view.getColumns() .add(column); log.trace("parsed view {}.{} column: {}", view.getDatabase().getInternalName(), view.getInternalName(), column.getInternalName()); @@ -562,10 +533,6 @@ public interface DataMapper { } switch (column.getColumnType()) { case DATE -> { - if (column.getDateFormat() == null) { - log.error("Missing date format for column {}", column.getId()); - throw new IllegalArgumentException("Missing date format"); - } final DateTimeFormatter formatter = new DateTimeFormatterBuilder() .parseCaseInsensitive() /* case insensitive to parse JAN and FEB */ .appendPattern("yyyy-MM-dd") @@ -575,10 +542,6 @@ public interface DataMapper { .toInstant(); } case TIMESTAMP, DATETIME -> { - if (column.getDateFormat() == null) { - log.error("Missing date format for column {}", column.getId()); - throw new IllegalArgumentException("Missing date format"); - } return Timestamp.valueOf(data.toString()) .toInstant(); } @@ -678,7 +641,7 @@ public interface DataMapper { ps.setNull(idx, Types.DATE); break; } - ps.setDate(idx, Date.valueOf(String.valueOf(value))); + ps.setString(idx, String.valueOf(value)); break; case BIGINT: if (value == null) { @@ -743,28 +706,7 @@ public interface DataMapper { } ps.setBoolean(idx, Boolean.parseBoolean(String.valueOf(value))); break; - case TIMESTAMP: - if (value == null) { - ps.setNull(idx, Types.TIMESTAMP); - break; - } - ps.setTimestamp(idx, Timestamp.valueOf(String.valueOf(value))); - break; - case DATETIME: - if (value == null) { - ps.setNull(idx, Types.TIMESTAMP); - break; - } - ps.setTimestamp(idx, Timestamp.valueOf(String.valueOf(value))); - break; - case TIME: - if (value == null) { - ps.setNull(idx, Types.TIME); - break; - } - ps.setTime(idx, Time.valueOf(String.valueOf(value))); - break; - case YEAR: + case TIME, DATETIME, TIMESTAMP, YEAR: if (value == null) { ps.setNull(idx, Types.TIME); break; diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java index d870215771628283ddb9e115a755c176d7f69240..0e0a94f4279b9be83f8ce660a9b5e6948228f4b4 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java @@ -1,22 +1,27 @@ package at.tuwien.mapper; -import at.tuwien.api.database.query.ImportCsvDto; -import at.tuwien.api.database.table.*; -import at.tuwien.api.database.table.columns.*; +import at.tuwien.api.database.query.ImportDto; +import at.tuwien.api.database.table.TupleDeleteDto; +import at.tuwien.api.database.table.TupleDto; +import at.tuwien.api.database.table.TupleUpdateDto; +import at.tuwien.api.database.table.columns.ColumnCreateDto; +import at.tuwien.api.database.table.columns.ColumnDto; +import at.tuwien.api.database.table.columns.ColumnTypeDto; import at.tuwien.api.database.table.internal.PrivilegedTableDto; -import at.tuwien.exception.*; +import at.tuwien.exception.QueryMalformedException; +import at.tuwien.exception.TableMalformedException; import at.tuwien.utils.MariaDbUtil; import org.mapstruct.Mapper; import org.mapstruct.Named; -import java.io.*; -import java.math.BigInteger; -import java.sql.*; +import java.io.ByteArrayInputStream; +import java.io.InputStream; import java.sql.Date; +import java.sql.*; import java.text.Normalizer; -import java.time.*; +import java.time.Instant; +import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; import java.util.*; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -548,7 +553,7 @@ public interface MariaDbMapper { return statement.toString(); } - default String datasetToRawInsertQuery(String databaseName, PrivilegedTableDto table, ImportCsvDto data) { + default String datasetToRawInsertQuery(String databaseName, PrivilegedTableDto table, ImportDto data) { final StringBuilder statement = new StringBuilder("LOAD DATA INFILE '") .append(data.getLocation()) .append("' REPLACE INTO TABLE `") @@ -563,11 +568,17 @@ public interface MariaDbMapper { .append(data.getQuote()) .append("'"); } - statement.append(" LINES TERMINATED BY '") - .append(data.getLineTermination()) - .append("'") - .append(data.getSkipLines() != null ? (" IGNORE " + data.getSkipLines() + " LINES") : "") - .append(" ("); + if (data.getLineTermination() != null) { + statement.append(" LINES TERMINATED BY '") + .append(data.getLineTermination()) + .append("'"); + } + if (data.getSkipLines() != null) { + statement.append(" IGNORE") + .append(data.getSkipLines()) + .append(" LINES"); + } + statement.append(" ("); final StringBuilder set = new StringBuilder(); int[] idx = new int[]{0}; table.getColumns() @@ -580,20 +591,10 @@ public interface MariaDbMapper { /* format as variable */ statement.append("@") .append(column.getInternalName()); - if (column.getDateFormat() != null) { - /* reformat dates */ - columnToDateSet(data, column, set); - } else if (column.getColumnType().equals(ColumnTypeDto.BOOL)) { - /* reformat booleans */ - columnToBoolSet(data, column, set); - } else { - /* reformat others */ - columnToTextSet(data, column, set); - } idx[0]++; }); statement.append(")") - .append(set.length() != 0 ? (" SET " + set) : "") + .append(!set.isEmpty() ? (" SET " + set) : "") .append(";"); log.trace("mapped insert statement: {}", statement); return statement.toString(); @@ -710,125 +711,6 @@ public interface MariaDbMapper { return statement.toString(); } - default void columnToDateSet(ImportCsvDto data, ColumnDto column, StringBuilder set) { - log.trace("import column has date format, need to format it: {}", column.getDateFormat().getUnixFormat()); - set.append(!set.isEmpty() ? ", " : "") - .append("`") - .append(column.getInternalName()) - .append("` = STR_TO_DATE("); - if (data.getNullElement() != null) { - set.append("IF(STRCMP(@") - .append(column.getInternalName()) - .append(",'") - .append(data.getNullElement()) - .append("'), @") - .append(column.getInternalName()) - .append(", NULL), '") - .append(column.getDateFormat() - .getDatabaseFormat() - .replace('\'', '\\')) - .append("')"); - return; - } - set.append("@") - .append(column.getInternalName()) - .append(", '") - .append(column.getDateFormat() - .getDatabaseFormat() - .replace('\'', '\\')) - .append("')"); - } - - default void columnToBoolSet(ImportCsvDto data, ColumnDto column, StringBuilder set) { - set.append(!set.isEmpty() ? ", " : "") - .append("`") - .append(column.getInternalName()) - .append("` = "); - if (data.getNullElement() != null) { - set.append("IF(!STRCMP(@") - .append(column.getInternalName()) - .append(",'") - .append(data.getNullElement()) - .append("'),NULL,"); - columnToBoolSet2(data, column, set); - set.append(")"); - return; - } - columnToBoolSet2(data, column, set); - } - - default void columnToBoolSet2(ImportCsvDto data, ColumnDto column, StringBuilder set) { - if (data.getTrueElement() != null) { - set.append("IF(!STRCMP(@") - .append(column.getInternalName()) - .append(",'") - .append(data.getTrueElement()) - .append("'),TRUE,"); - if (data.getFalseElement() != null) { - log.trace("import has false element present (both true and false)"); - /* can map both true/false */ - set.append("IF(!STRCMP(@") - .append(column.getInternalName()) - .append(",'") - .append(data.getFalseElement()) - .append("'),FALSE,@") - .append(column.getInternalName()) - .append("))"); - } else { - /* can only map true */ - set.append("@") - .append(column.getInternalName()) - .append(")"); - } - return; - } - if (data.getFalseElement() != null) { - set.append("IF(!STRCMP(@") - .append(column.getInternalName()) - .append(",'") - .append(data.getFalseElement()) - .append("'),FALSE,"); - if (data.getTrueElement() != null) { - log.trace("import has true element present (both true and false)"); - /* can map both true/false */ - set.append("IF(!STRCMP(@") - .append(column.getInternalName()) - .append(",'") - .append(data.getTrueElement()) - .append("'),TRUE,@") - .append(column.getInternalName()) - .append("))"); - } else { - /* can only map true */ - set.append("@") - .append(column.getInternalName()) - .append(")"); - } - return; - } - set.append("@") - .append(column.getInternalName()); - } - - default void columnToTextSet(ImportCsvDto data, ColumnDto column, StringBuilder set) { - set.append(!set.isEmpty() ? ", " : "") - .append("`") - .append(column.getInternalName()) - .append("` = "); - if (data.getNullElement() != null) { - set.append("IF(STRCMP(@") - .append(column.getInternalName()) - .append(",'") - .append(data.getNullElement()) - .append("'), @") - .append(column.getInternalName()) - .append(", NULL)"); - return; - } - set.append("@") - .append(column.getInternalName()); - } - default void prepareStatementWithColumnTypeObject(PreparedStatement statement, ColumnTypeDto columnType, int idx, String columnName, Object value) throws SQLException { switch (columnType) { diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/TableService.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/TableService.java index 765a3b7e2e630b70b4e52d99aaddecd8559754a3..33ef0026a8b3df7e9feddbe22d57da33be0af39d 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/TableService.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/TableService.java @@ -2,7 +2,7 @@ package at.tuwien.service; import at.tuwien.ExportResourceDto; import at.tuwien.api.database.internal.PrivilegedDatabaseDto; -import at.tuwien.api.database.query.ImportCsvDto; +import at.tuwien.api.database.query.ImportDto; import at.tuwien.api.database.query.QueryResultDto; import at.tuwien.api.database.table.*; import at.tuwien.api.database.table.internal.PrivilegedTableDto; @@ -104,7 +104,7 @@ public interface TableService { Long getCount(PrivilegedTableDto table, Instant timestamp) throws SQLException, QueryMalformedException; - void importDataset(PrivilegedTableDto table, ImportCsvDto data) throws SidecarImportException, + void importDataset(PrivilegedTableDto table, ImportDto data) throws SidecarImportException, StorageNotFoundException, SQLException, QueryMalformedException, RemoteUnavailableException; void deleteTuple(PrivilegedTableDto table, TupleDeleteDto data) throws SQLException, diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java index e691bee25e4513c335b440d802cfc20d8b6c52a2..8bfdc0089a9d31ce9faa8af40be151eb1044532f 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java @@ -17,8 +17,8 @@ import at.tuwien.gateway.MetadataServiceGateway; import at.tuwien.mapper.DataMapper; import at.tuwien.mapper.MariaDbMapper; import at.tuwien.mapper.MetadataMapper; -import at.tuwien.service.SubsetService; import at.tuwien.service.StorageService; +import at.tuwien.service.SubsetService; import com.mchange.v2.c3p0.ComboPooledDataSource; import io.micrometer.core.instrument.Counter; import lombok.extern.log4j.Log4j2; @@ -287,7 +287,11 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs /* insert query into query store */ final long start = System.currentTimeMillis(); final CallableStatement callableStatement = connection.prepareCall(mariaDbMapper.queryStoreStoreQueryRawQuery()); - callableStatement.setString(1, String.valueOf(userId)); + if (userId != null) { + callableStatement.setString(1, String.valueOf(userId)); + } else { + callableStatement.setNull(1, Types.VARCHAR); + } callableStatement.setString(2, query); callableStatement.setTimestamp(3, Timestamp.from(timestamp)); callableStatement.registerOutParameter(4, Types.BIGINT); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java index cdcde4f2c2c785702284022ab28c9237846f0ec8..3ab02bbb016ff28469301a699f2ebd65bfd76f6c 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java @@ -2,7 +2,7 @@ package at.tuwien.service.impl; import at.tuwien.ExportResourceDto; import at.tuwien.api.database.internal.PrivilegedDatabaseDto; -import at.tuwien.api.database.query.ImportCsvDto; +import at.tuwien.api.database.query.ImportDto; import at.tuwien.api.database.query.QueryResultDto; import at.tuwien.api.database.table.*; import at.tuwien.api.database.table.columns.ColumnDto; @@ -273,7 +273,7 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table } @Override - public void importDataset(PrivilegedTableDto table, ImportCsvDto data) throws StorageNotFoundException, + public void importDataset(PrivilegedTableDto table, ImportDto data) throws StorageNotFoundException, SQLException, QueryMalformedException, RemoteUnavailableException, SidecarImportException { /* import .csv from blob storage to sidecar */ dataDatabaseSidecarGateway.importFile(table.getDatabase().getContainer().getSidecarHost(), table.getDatabase().getContainer().getSidecarPort(), data.getLocation()); diff --git a/dbrepo-gateway-service/dbrepo.conf b/dbrepo-gateway-service/dbrepo.conf index 49e5ce649631baf8ffb1945f3cbc1037de099d28..e61d202266b9a064efb96244f9c3eb0dbbf1ac13 100644 --- a/dbrepo-gateway-service/dbrepo.conf +++ b/dbrepo-gateway-service/dbrepo.conf @@ -48,16 +48,7 @@ server { listen 80 default_server; server_name _; - location /admin/broker { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass http://broker; - proxy_read_timeout 90; - } - - location /dashboard { + location /dashboard/ { rewrite ^/dashboard/(.*) /$1 break; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; @@ -68,7 +59,7 @@ server { } # Proxy Grafana Live WebSocket connections. - location /dashboard/api/live { + location /dashboard/api/live/ { rewrite ^/dashboard/(.*) /$1 break; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; @@ -87,7 +78,7 @@ server { } location /api/broker { - rewrite /api/broker/(.*) /admin/broker/api/$1 break; + rewrite /api/broker/(.*) /api/$1 break; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/dbrepo-metadata-db/1_setup-schema.sql b/dbrepo-metadata-db/1_setup-schema.sql index 9c21d44c0ec15b39f4c0a4205755ae3e904ab521..db90e94c1fa7989b390f09ba8d2bc78fd92c2fdd 100644 --- a/dbrepo-metadata-db/1_setup-schema.sql +++ b/dbrepo-metadata-db/1_setup-schema.sql @@ -19,7 +19,7 @@ CREATE TABLE IF NOT EXISTS `mdb_users` CREATE TABLE IF NOT EXISTS `mdb_images` ( - id bigint NOT NULL AUTO_INCREMENT, + id SERIAL, registry character varying(255) NOT NULL DEFAULT 'docker.io', name character varying(255) NOT NULL, version character varying(255) NOT NULL, @@ -35,23 +35,9 @@ CREATE TABLE IF NOT EXISTS `mdb_images` UNIQUE (is_default) ) WITH SYSTEM VERSIONING; -CREATE TABLE IF NOT EXISTS `mdb_images_date` -( - id bigint NOT NULL AUTO_INCREMENT, - iid bigint NOT NULL, - database_format character varying(255) NOT NULL, - unix_format character varying(255) NOT NULL, - example character varying(255) NOT NULL, - has_time boolean NOT NULL, - created_at timestamp NOT NULL DEFAULT NOW(), - PRIMARY KEY (id), - FOREIGN KEY (iid) REFERENCES mdb_images (id), - UNIQUE (database_format, unix_format, example) -) WITH SYSTEM VERSIONING; - CREATE TABLE IF NOT EXISTS `mdb_containers` ( - id bigint NOT NULL AUTO_INCREMENT, + id SERIAL, internal_name character varying(255) NOT NULL, name character varying(255) NOT NULL, host character varying(255) NOT NULL, @@ -67,13 +53,12 @@ CREATE TABLE IF NOT EXISTS `mdb_containers` privileged_username character varying(255) NOT NULL, privileged_password character varying(255) NOT NULL, quota integer NOT NULL DEFAULT 50, - PRIMARY KEY (id), - FOREIGN KEY (image_id) REFERENCES mdb_images (id) + PRIMARY KEY (id) ) WITH SYSTEM VERSIONING; CREATE TABLE IF NOT EXISTS `mdb_data` ( - ID bigint NOT NULL AUTO_INCREMENT, + ID SERIAL, PROVENANCE text, FileEncoding text, FileType character varying(100), @@ -93,8 +78,8 @@ CREATE TABLE IF NOT EXISTS `mdb_licenses` CREATE TABLE IF NOT EXISTS `mdb_databases` ( - id bigint NOT NULL AUTO_INCREMENT, - cid bigint NOT NULL, + id SERIAL, + cid BIGINT UNSIGNED NOT NULL, name character varying(255) NOT NULL, internal_name character varying(255) NOT NULL, exchange_name character varying(255) NOT NULL, @@ -108,7 +93,7 @@ CREATE TABLE IF NOT EXISTS `mdb_databases` created timestamp NOT NULL DEFAULT NOW(), last_modified timestamp, PRIMARY KEY (id), - FOREIGN KEY (cid) REFERENCES mdb_containers (id) /* currently we only support one-to-one */, + FOREIGN KEY (cid) REFERENCES mdb_containers (id), FOREIGN KEY (created_by) REFERENCES mdb_users (id), FOREIGN KEY (owned_by) REFERENCES mdb_users (id), FOREIGN KEY (contact_person) REFERENCES mdb_users (id) @@ -123,8 +108,8 @@ CREATE TABLE IF NOT EXISTS `mdb_databases_subjects` CREATE TABLE IF NOT EXISTS `mdb_tables` ( - ID bigint NOT NULL AUTO_INCREMENT, - tDBID bigint NOT NULL, + ID SERIAL, + tDBID BIGINT UNSIGNED NOT NULL, tName VARCHAR(64) NOT NULL, internal_name VARCHAR(64) NOT NULL, queue_name VARCHAR(255) NOT NULL, @@ -155,26 +140,25 @@ CREATE TABLE IF NOT EXISTS `mdb_tables` CREATE TABLE IF NOT EXISTS `mdb_columns` ( - ID BIGINT NOT NULL AUTO_INCREMENT, - tID BIGINT NOT NULL, - dfID BIGINT, + ID SERIAL, + tID BIGINT UNSIGNED NOT NULL, cName VARCHAR(64), - internal_name VARCHAR(64) NOT NULL, + internal_name VARCHAR(64) NOT NULL, Datatype ENUM ('CHAR','VARCHAR','BINARY','VARBINARY','TINYBLOB','TINYTEXT','TEXT','BLOB','MEDIUMTEXT','MEDIUMBLOB','LONGTEXT','LONGBLOB','ENUM','SET','BIT','TINYINT','BOOL','SMALLINT','MEDIUMINT','INT','BIGINT','FLOAT','DOUBLE','DECIMAL','DATE','DATETIME','TIMESTAMP','TIME','YEAR'), - length BIGINT NULL, - ordinal_position INTEGER NOT NULL, - index_length BIGINT NULL, + length BIGINT UNSIGNED NULL, + ordinal_position INTEGER NOT NULL, + index_length BIGINT UNSIGNED NULL, description VARCHAR(2048), - size BIGINT, - d BIGINT, - auto_generated BOOLEAN DEFAULT false, - is_null_allowed BOOLEAN NOT NULL DEFAULT true, - val_min NUMERIC NULL, - val_max NUMERIC NULL, - mean NUMERIC NULL, - median NUMERIC NULL, - std_dev Numeric NULL, - created timestamp NOT NULL DEFAULT NOW(), + size BIGINT UNSIGNED, + d BIGINT UNSIGNED, + auto_generated BOOLEAN DEFAULT false, + is_null_allowed BOOLEAN NOT NULL DEFAULT true, + val_min NUMERIC NULL, + val_max NUMERIC NULL, + mean NUMERIC NULL, + median NUMERIC NULL, + std_dev Numeric NULL, + created timestamp NOT NULL DEFAULT NOW(), last_modified timestamp, FOREIGN KEY (tID) REFERENCES mdb_tables (ID) ON DELETE CASCADE, PRIMARY KEY (ID), @@ -183,8 +167,8 @@ CREATE TABLE IF NOT EXISTS `mdb_columns` CREATE TABLE IF NOT EXISTS `mdb_columns_enums` ( - id bigint NOT NULL AUTO_INCREMENT, - column_id bigint NOT NULL, + id SERIAL, + column_id BIGINT UNSIGNED NOT NULL, value CHARACTER VARYING(255) NOT NULL, FOREIGN KEY (column_id) REFERENCES mdb_columns (ID) ON DELETE CASCADE, PRIMARY KEY (id) @@ -192,8 +176,8 @@ CREATE TABLE IF NOT EXISTS `mdb_columns_enums` CREATE TABLE IF NOT EXISTS `mdb_columns_sets` ( - id bigint NOT NULL AUTO_INCREMENT, - column_id bigint NOT NULL, + id SERIAL, + column_id BIGINT UNSIGNED NOT NULL, value CHARACTER VARYING(255) NOT NULL, FOREIGN KEY (column_id) REFERENCES mdb_columns (ID) ON DELETE CASCADE, PRIMARY KEY (id) @@ -201,8 +185,8 @@ CREATE TABLE IF NOT EXISTS `mdb_columns_sets` CREATE TABLE IF NOT EXISTS `mdb_columns_nom` ( - cID bigint, - tID bigint, + cID BIGINT UNSIGNED, + tID BIGINT UNSIGNED, maxlength INTEGER, last_modified timestamp, created timestamp NOT NULL DEFAULT NOW(), @@ -212,8 +196,8 @@ CREATE TABLE IF NOT EXISTS `mdb_columns_nom` CREATE TABLE IF NOT EXISTS `mdb_columns_cat` ( - cID bigint, - tID bigint, + cID BIGINT UNSIGNED, + tID BIGINT UNSIGNED, num_cat INTEGER, -- cat_array TEXT[], last_modified timestamp, @@ -224,13 +208,13 @@ CREATE TABLE IF NOT EXISTS `mdb_columns_cat` CREATE TABLE IF NOT EXISTS `mdb_constraints_foreign_key` ( - fkid BIGINT NOT NULL AUTO_INCREMENT, - tid BIGINT NOT NULL, - rtid BIGINT NOT NULL, - name VARCHAR(255) NOT NULL, - on_update VARCHAR(50) NULL, - on_delete VARCHAR(50) NULL, - position INT NULL, + fkid BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + tid BIGINT UNSIGNED NOT NULL, + rtid BIGINT UNSIGNED NOT NULL, + name VARCHAR(255) NOT NULL, + on_update VARCHAR(50) NULL, + on_delete VARCHAR(50) NULL, + position INT NULL, PRIMARY KEY (fkid), FOREIGN KEY (tid) REFERENCES mdb_tables (id) ON DELETE CASCADE, FOREIGN KEY (rtid) REFERENCES mdb_tables (id) @@ -238,9 +222,9 @@ CREATE TABLE IF NOT EXISTS `mdb_constraints_foreign_key` CREATE TABLE IF NOT EXISTS `mdb_constraints_primary_key` ( - pkid BIGINT NOT NULL AUTO_INCREMENT, - tID BIGINT NOT NULL, - cid BIGINT NOT NULL, + pkid SERIAL, + tID BIGINT UNSIGNED NOT NULL, + cid BIGINT UNSIGNED NOT NULL, PRIMARY KEY (pkid), FOREIGN KEY (tID) REFERENCES mdb_tables (id) ON DELETE CASCADE, FOREIGN KEY (cid) REFERENCES mdb_columns (id) ON DELETE CASCADE @@ -248,10 +232,10 @@ CREATE TABLE IF NOT EXISTS `mdb_constraints_primary_key` CREATE TABLE IF NOT EXISTS `mdb_constraints_foreign_key_reference` ( - id BIGINT NOT NULL AUTO_INCREMENT, - fkid BIGINT NOT NULL, - cid BIGINT NOT NULL, - rcid BIGINT NOT NULL, + id SERIAL, + fkid BIGINT UNSIGNED NOT NULL, + cid BIGINT UNSIGNED NOT NULL, + rcid BIGINT UNSIGNED NOT NULL, PRIMARY KEY (id), UNIQUE (fkid, cid, rcid), FOREIGN KEY (fkid) REFERENCES mdb_constraints_foreign_key (fkid) ON UPDATE CASCADE, @@ -261,19 +245,19 @@ CREATE TABLE IF NOT EXISTS `mdb_constraints_foreign_key_reference` CREATE TABLE IF NOT EXISTS `mdb_constraints_unique` ( - uid BIGINT NOT NULL AUTO_INCREMENT, - name VARCHAR(255) NOT NULL, - tid BIGINT NOT NULL, - position INT NULL, + uid BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + name VARCHAR(255) NOT NULL, + tid BIGINT UNSIGNED NOT NULL, + position INT NULL, PRIMARY KEY (uid), FOREIGN KEY (tid) REFERENCES mdb_tables (id) ON DELETE CASCADE ); CREATE TABLE IF NOT EXISTS `mdb_constraints_unique_columns` ( - id BIGINT NOT NULL AUTO_INCREMENT, - uid BIGINT NOT NULL, - cid BIGINT NOT NULL, + id SERIAL, + uid BIGINT UNSIGNED NOT NULL, + cid BIGINT UNSIGNED NOT NULL, PRIMARY KEY (id), FOREIGN KEY (uid) REFERENCES mdb_constraints_unique (uid), FOREIGN KEY (cid) REFERENCES mdb_columns (id) ON DELETE CASCADE @@ -281,9 +265,9 @@ CREATE TABLE IF NOT EXISTS `mdb_constraints_unique_columns` CREATE TABLE IF NOT EXISTS `mdb_constraints_checks` ( - id BIGINT NOT NULL AUTO_INCREMENT, - tid BIGINT NOT NULL, - checks VARCHAR(255) NOT NULL, + id SERIAL, + tid BIGINT UNSIGNED NOT NULL, + checks VARCHAR(255) NOT NULL, PRIMARY KEY (id), FOREIGN KEY (tid) REFERENCES mdb_tables (id) ON DELETE CASCADE ) WITH SYSTEM VERSIONING; @@ -291,7 +275,7 @@ CREATE TABLE IF NOT EXISTS `mdb_constraints_checks` CREATE TABLE IF NOT EXISTS `mdb_concepts` ( - id bigint NOT NULL AUTO_INCREMENT, + id SERIAL, uri text not null, name VARCHAR(255) null, description TEXT null, @@ -302,7 +286,7 @@ CREATE TABLE IF NOT EXISTS `mdb_concepts` CREATE TABLE IF NOT EXISTS `mdb_units` ( - id bigint NOT NULL AUTO_INCREMENT, + id SERIAL, uri text not null, name VARCHAR(255) null, description TEXT null, @@ -313,26 +297,26 @@ CREATE TABLE IF NOT EXISTS `mdb_units` CREATE TABLE IF NOT EXISTS `mdb_columns_concepts` ( - id bigint NOT NULL, - cID bigint NOT NULL, - created timestamp NOT NULL DEFAULT NOW(), + id BIGINT UNSIGNED NOT NULL, + cID BIGINT UNSIGNED NOT NULL, + created timestamp NOT NULL DEFAULT NOW(), PRIMARY KEY (id, cid), FOREIGN KEY (cID) REFERENCES mdb_columns (ID) ) WITH SYSTEM VERSIONING; CREATE TABLE IF NOT EXISTS `mdb_columns_units` ( - id bigint NOT NULL, - cID bigint NOT NULL, - created timestamp NOT NULL DEFAULT NOW(), + id BIGINT UNSIGNED NOT NULL, + cID BIGINT UNSIGNED NOT NULL, + created timestamp NOT NULL DEFAULT NOW(), PRIMARY KEY (id, cID), FOREIGN KEY (cID) REFERENCES mdb_columns (ID) ) WITH SYSTEM VERSIONING; CREATE TABLE IF NOT EXISTS `mdb_view` ( - id bigint NOT NULL AUTO_INCREMENT, - vdbid bigint NOT NULL, + id SERIAL, + vdbid BIGINT UNSIGNED NOT NULL, vName VARCHAR(64) NOT NULL, internal_name VARCHAR(64) NOT NULL, Query TEXT NOT NULL, @@ -349,7 +333,7 @@ CREATE TABLE IF NOT EXISTS `mdb_view` CREATE TABLE IF NOT EXISTS `mdb_banner_messages` ( - id bigint NOT NULL AUTO_INCREMENT, + id SERIAL, type ENUM ('ERROR', 'WARNING', 'INFO') NOT NULL default 'INFO', message TEXT NOT NULL, link TEXT NULL, @@ -361,7 +345,7 @@ CREATE TABLE IF NOT EXISTS `mdb_banner_messages` CREATE TABLE IF NOT EXISTS `mdb_ontologies` ( - id bigint NOT NULL AUTO_INCREMENT, + id SERIAL, prefix VARCHAR(8) NOT NULL, uri TEXT NOT NULL, uri_pattern TEXT, @@ -376,28 +360,28 @@ CREATE TABLE IF NOT EXISTS `mdb_ontologies` CREATE TABLE IF NOT EXISTS `mdb_view_columns` ( - id BIGINT NOT NULL AUTO_INCREMENT, - view_id BIGINT NOT NULL, - dfID BIGINT, + id SERIAL, + view_id BIGINT UNSIGNED NOT NULL, + dfID BIGINT UNSIGNED, name VARCHAR(64), - internal_name VARCHAR(64) NOT NULL, + internal_name VARCHAR(64) NOT NULL, column_type ENUM ('CHAR','VARCHAR','BINARY','VARBINARY','TINYBLOB','TINYTEXT','TEXT','BLOB','MEDIUMTEXT','MEDIUMBLOB','LONGTEXT','LONGBLOB','ENUM','SET','BIT','TINYINT','BOOL','SMALLINT','MEDIUMINT','INT','BIGINT','FLOAT','DOUBLE','DECIMAL','DATE','DATETIME','TIMESTAMP','TIME','YEAR'), - ordinal_position INTEGER NOT NULL, - size BIGINT, - d BIGINT, - auto_generated BOOLEAN DEFAULT false, - is_null_allowed BOOLEAN NOT NULL DEFAULT true, + ordinal_position INTEGER NOT NULL, + size BIGINT UNSIGNED, + d BIGINT UNSIGNED, + auto_generated BOOLEAN DEFAULT false, + is_null_allowed BOOLEAN NOT NULL DEFAULT true, PRIMARY KEY (id), FOREIGN KEY (view_id) REFERENCES mdb_view (id) ) WITH SYSTEM VERSIONING; CREATE TABLE IF NOT EXISTS `mdb_identifiers` ( - id BIGINT NOT NULL AUTO_INCREMENT, - dbid BIGINT NOT NULL, - qid BIGINT, - vid BIGINT, - tid BIGINT, + id SERIAL, + dbid BIGINT UNSIGNED NOT NULL, + qid BIGINT UNSIGNED, + vid BIGINT UNSIGNED, + tid BIGINT UNSIGNED, publisher VARCHAR(255) NOT NULL, language VARCHAR(2), publication_year INTEGER NOT NULL, @@ -422,8 +406,8 @@ CREATE TABLE IF NOT EXISTS `mdb_identifiers` CREATE TABLE IF NOT EXISTS `mdb_identifier_licenses` ( - pid bigint NOT NULL, - license_id VARCHAR(255) NOT NULL, + pid BIGINT UNSIGNED NOT NULL, + license_id VARCHAR(255) NOT NULL, PRIMARY KEY (pid, license_id), FOREIGN KEY (pid) REFERENCES mdb_identifiers (id), FOREIGN KEY (license_id) REFERENCES mdb_licenses (identifier) @@ -431,9 +415,9 @@ CREATE TABLE IF NOT EXISTS `mdb_identifier_licenses` CREATE TABLE IF NOT EXISTS `mdb_identifier_titles` ( - id bigint NOT NULL AUTO_INCREMENT, - pid bigint NOT NULL, - title text NOT NULL, + id SERIAL, + pid BIGINT UNSIGNED NOT NULL, + title text NOT NULL, title_type ENUM ('ALTERNATIVE_TITLE', 'SUBTITLE', 'TRANSLATED_TITLE', 'OTHER'), language VARCHAR(2), PRIMARY KEY (id), @@ -442,9 +426,9 @@ CREATE TABLE IF NOT EXISTS `mdb_identifier_titles` CREATE TABLE IF NOT EXISTS `mdb_identifier_funders` ( - id bigint NOT NULL AUTO_INCREMENT, - pid bigint NOT NULL, - funder_name VARCHAR(255) NOT NULL, + id SERIAL, + pid BIGINT UNSIGNED NOT NULL, + funder_name VARCHAR(255) NOT NULL, funder_identifier TEXT, funder_identifier_type ENUM ('CROSSREF_FUNDER_ID', 'GRID', 'ISNI', 'ROR', 'OTHER'), scheme_uri text, @@ -457,9 +441,9 @@ CREATE TABLE IF NOT EXISTS `mdb_identifier_funders` CREATE TABLE IF NOT EXISTS `mdb_identifier_descriptions` ( - id bigint NOT NULL AUTO_INCREMENT, - pid bigint NOT NULL, - description text NOT NULL, + id SERIAL, + pid BIGINT UNSIGNED NOT NULL, + description text NOT NULL, description_type ENUM ('ABSTRACT', 'METHODS', 'SERIES_INFORMATION', 'TABLE_OF_CONTENTS', 'TECHNICAL_INFO', 'OTHER'), language VARCHAR(2), PRIMARY KEY (id), @@ -468,11 +452,11 @@ CREATE TABLE IF NOT EXISTS `mdb_identifier_descriptions` CREATE TABLE IF NOT EXISTS `mdb_related_identifiers` ( - id bigint NOT NULL AUTO_INCREMENT, - pid bigint NOT NULL, - value varchar(255) NOT NULL, - type varchar(255) NOT NULL, - relation varchar(255) NOT NULL, + id SERIAL, + pid BIGINT UNSIGNED NOT NULL, + value varchar(255) NOT NULL, + type varchar(255) NOT NULL, + relation varchar(255) NOT NULL, PRIMARY KEY (id), /* must be a single id from persistent identifier concept */ FOREIGN KEY (pid) REFERENCES mdb_identifiers (id), UNIQUE (pid, value) @@ -480,11 +464,11 @@ CREATE TABLE IF NOT EXISTS `mdb_related_identifiers` CREATE TABLE IF NOT EXISTS `mdb_identifier_creators` ( - id bigint NOT NULL AUTO_INCREMENT, - pid bigint NOT NULL, + id SERIAL, + pid BIGINT UNSIGNED NOT NULL, given_names text, family_name text, - creator_name VARCHAR(255) NOT NULL, + creator_name VARCHAR(255) NOT NULL, name_type ENUM ('PERSONAL', 'ORGANIZATIONAL') default 'PERSONAL', name_identifier text, name_identifier_scheme ENUM ('ROR', 'GRID', 'ISNI', 'ORCID'), @@ -500,7 +484,7 @@ CREATE TABLE IF NOT EXISTS `mdb_identifier_creators` CREATE TABLE IF NOT EXISTS `mdb_update` ( uUserID character varying(255) NOT NULL, - uDBID bigint NOT NULL, + uDBID BIGINT UNSIGNED NOT NULL, created timestamp NOT NULL DEFAULT NOW(), PRIMARY KEY (uUserID, uDBID), FOREIGN KEY (uDBID) REFERENCES mdb_databases (id) @@ -509,7 +493,7 @@ CREATE TABLE IF NOT EXISTS `mdb_update` CREATE TABLE IF NOT EXISTS `mdb_access` ( aUserID character varying(255) NOT NULL, - aDBID bigint REFERENCES mdb_databases (id), + aDBID BIGINT UNSIGNED REFERENCES mdb_databases (id), attime TIMESTAMP, download BOOLEAN, created timestamp NOT NULL DEFAULT NOW(), @@ -519,14 +503,40 @@ CREATE TABLE IF NOT EXISTS `mdb_access` CREATE TABLE IF NOT EXISTS `mdb_have_access` ( user_id character varying(36) NOT NULL, - database_id bigint REFERENCES mdb_databases (id), + database_id BIGINT UNSIGNED REFERENCES mdb_databases (id), access_type ENUM ('READ', 'WRITE_OWN', 'WRITE_ALL') NOT NULL, created timestamp NOT NULL DEFAULT NOW(), PRIMARY KEY (user_id, database_id), FOREIGN KEY (user_id) REFERENCES mdb_users (id) ) WITH SYSTEM VERSIONING; +CREATE TABLE IF NOT EXISTS `mdb_image_types` +( + id SERIAL, + image_id BIGINT UNSIGNED NOT NULL, + display_name varchar(255) NOT NULL, + value varchar(255) NOT NULL, + size_min INT UNSIGNED, + size_max INT UNSIGNED, + size_default INT UNSIGNED, + size_required BOOLEAN COMMENT 'When setting NULL, the service assumes the data type has no size', + size_step INT UNSIGNED, + d_min INT UNSIGNED, + d_max INT UNSIGNED, + d_default INT UNSIGNED, + d_required BOOLEAN COMMENT 'When setting NULL, the service assumes the data type has no d', + d_step INT UNSIGNED, + hint TEXT, + documentation TEXT NOT NULL, + is_quoted BOOLEAN NOT NULL, + is_buildable BOOLEAN NOT NULL, + PRIMARY KEY (id), + FOREIGN KEY (image_id) REFERENCES `mdb_images` (`id`), + UNIQUE (value) +) WITH SYSTEM VERSIONING; + COMMIT; + BEGIN; INSERT INTO `mdb_licenses` (identifier, uri, description) @@ -539,14 +549,71 @@ INSERT INTO `mdb_images` (name, registry, version, default_port, dialect, driver VALUES ('mariadb', 'docker.io', '11.1.3', 3306, 'org.hibernate.dialect.MariaDBDialect', 'org.mariadb.jdbc.Driver', 'mariadb'); -INSERT INTO `mdb_images_date` (iid, database_format, unix_format, example, has_time) -VALUES (1, '%Y-%c-%d %H:%i:%S.%f', 'yyyy-MM-dd HH:mm:ss.SSSSSS', '2022-01-30 13:44:25.499', true), - (1, '%Y-%c-%d %H:%i:%S', 'yyyy-MM-dd HH:mm:ss', '2022-01-30 13:44:25', true), - (1, '%Y-%c-%d', 'yyyy-MM-dd', '2022-01-30', false), - (1, '%H:%i:%S', 'HH:mm:ss', '13:44:25', true), - (1, '%d.%c.%Y', 'dd.MM.yyyy', '30.01.2022', false); - -INSERT INTO `mdb_ontologies` (prefix, uri, uri_pattern, sparql_endpoint, rdf_path) +INSERT INTO `mdb_image_types` (image_id, display_name, value, size_min, size_max, size_default, size_required, + size_step, d_min, d_max, d_default, d_required, d_step, hint, documentation, is_quoted, + is_buildable) +VALUES (1, 'BIGINT(size)', 'bigint', 0, null, null, false, 1, null, null, null, null, null, null, + 'https://mariadb.com/kb/en/bigint/', false, true), + (1, 'BINARY(size)', 'binary', 0, 255, 255, true, 1, null, null, null, null, null, 'size in Bytes', + 'https://mariadb.com/kb/en/binary/', false, true), + (1, 'BIT(size)', 'bit', 0, 64, null, false, 1, null, null, null, null, null, null, + 'https://mariadb.com/kb/en/bit/', false, true), + (1, 'BLOB(size)', 'blob', 0, 65535, null, false, 1, null, null, null, null, null, 'size in Bytes', + 'https://mariadb.com/kb/en/blob/', false, false), + (1, 'BOOL', 'bool', null, null, null, null, null, null, null, null, null, null, null, + 'https://mariadb.com/kb/en/bool/', false, true), + (1, 'CHAR(size)', 'char', 0, 255, 255, false, 1, null, null, null, null, null, null, + 'https://mariadb.com/kb/en/char/', false, true), + (1, 'DATE', 'date', null, null, null, null, null, null, null, null, null, null, + 'min. 1000-01-01, max. 9999-12-31', 'https://mariadb.com/kb/en/date/', true, true), + (1, 'DATETIME(fsp)', 'datetime', 0, 6, null, null, 1, null, null, null, null, null, + 'fsp=microsecond precision, min. 1000-01-01 00:00:00.0, max. 9999-12-31 23:59:59.9', + 'https://mariadb.com/kb/en/datetime/', true, true), + (1, 'DECIMAL(size, d)', 'decimal', 0, 65, null, false, 1, 0, 38, null, false, null, null, + 'https://mariadb.com/kb/en/decimal/', false, true), + (1, 'DOUBLE(size, d)', 'double', null, null, null, false, null, null, null, null, false, null, null, + 'https://mariadb.com/kb/en/double/', false, true), + (1, 'ENUM(v1,v2,...)', 'enum', null, null, null, null, null, null, null, null, null, null, null, + 'https://mariadb.com/kb/en/enum/', true, true), + (1, 'FLOAT(size)', 'float', null, null, null, false, null, null, null, null, null, null, null, + 'https://mariadb.com/kb/en/float/', false, true), + (1, 'INT(size)', 'int', null, null, null, false, null, null, null, null, null, null, 'size in Bytes', + 'https://mariadb.com/kb/en/int/', false, true), + (1, 'LONGBLOB', 'longblob', null, null, null, null, null, null, null, null, null, null, 'max. 3.999 GiB', + 'https://mariadb.com/kb/en/longblob/', false, true), + (1, 'LONGTEXT', 'longtext', null, null, null, null, null, null, null, null, null, null, 'max. 3.999 GiB', + 'https://mariadb.com/kb/en/longtext/', true, true), + (1, 'MEDIUMBLOB', 'mediumblob', null, null, null, null, null, null, null, null, null, null, 'max. 15.999 MiB', + 'https://mariadb.com/kb/en/mediumblob/', false, true), + (1, 'MEDIUMINT', 'mediumint', null, null, null, null, null, null, null, null, null, null, 'size in Bytes', + 'https://mariadb.com/kb/en/mediumint/', false, true), + (1, 'MEDIUMTEXT', 'mediumtext', null, null, null, null, null, null, null, null, null, null, 'size in Bytes', + 'https://mariadb.com/kb/en/mediumtext/', true, true), + (1, 'SET(v1,v2,...)', 'set', null, null, null, null, null, null, null, null, null, null, null, + 'https://mariadb.com/kb/en/set/', true, true), + (1, 'SMALLINT(size)', 'smallint', 0, null, null, false, null, null, null, null, null, null, 'size in Bytes', + 'https://mariadb.com/kb/en/smallint/', false, true), + (1, 'TEXT(size)', 'text', 0, null, null, false, null, null, null, null, null, null, 'size in Bytes', + 'https://mariadb.com/kb/en/text/', true, true), + (1, 'TIME(fsp)', 'time', 0, 6, 0, false, null, null, null, null, null, null, + 'fsp=microsecond precision, min. 0, max. 6', 'https://mariadb.com/kb/en/time/', true, true), + (1, 'TIMESTAMP(fsp)', 'timestamp', 0, 6, 0, false, null, null, null, null, null, null, + 'fsp=microsecond precision, min. 0, max. 6', 'https://mariadb.com/kb/en/timestamp/', true, true), + (1, 'TINYBLOB', 'tinyblob', null, null, null, null, null, null, null, null, null, null, + 'fsp=microsecond precision, min. 0, max. 6', 'https://mariadb.com/kb/en/timestamp/', false, true), + (1, 'TINYINT(size)', 'tinyint', 0, null, null, false, null, null, null, null, null, null, + 'size in Bytes', 'https://mariadb.com/kb/en/tinyint/', false, true), + (1, 'TINYTEXT', 'tinytext', null, null, null, null, null, null, null, null, null, null, + 'max. 255 characters', 'https://mariadb.com/kb/en/tinytext/', true, true), + (1, 'YEAR', 'year', 2, 4, null, false, 2, null, null, null, null, null, 'min. 1901, max. 2155', + 'https://mariadb.com/kb/en/year/', false, true), + (1, 'VARBINARY(size)', 'varbinary', 0, null, null, true, null, null, null, null, null, null, + null, 'https://mariadb.com/kb/en/varbinary/', false, true), + (1, 'VARCHAR(size)', 'varchar', 0, 65532, 255, true, null, null, null, null, null, null, + null, 'https://mariadb.com/kb/en/varchar/', false, true); + +INSERT +INTO `mdb_ontologies` (prefix, uri, uri_pattern, sparql_endpoint, rdf_path) VALUES ('om', 'http://www.ontology-of-units-of-measure.org/resource/om-2/', 'http://www.ontology-of-units-of-measure.org/resource/om-2/.*', null, 'rdf/om-2.0.rdf'), ('wd', 'http://www.wikidata.org/', 'http://www.wikidata.org/entity/.*', 'https://query.wikidata.org/sparql', diff --git a/dbrepo-metadata-service/api/pom.xml b/dbrepo-metadata-service/api/pom.xml index 4722e9c3c91fbc07e86af58622d2e3d0cc53de0b..328fc8ee57f64a7dd1d8215785d8d1decabd15ea 100644 --- a/dbrepo-metadata-service/api/pom.xml +++ b/dbrepo-metadata-service/api/pom.xml @@ -13,7 +13,14 @@ <name>dbrepo-metadata-service-api</name> <version>1.4.6</version> - <dependencies/> + <dependencies> + <dependency> + <groupId>at.tuwien</groupId> + <artifactId>dbrepo-metadata-service-entities</artifactId> + <version>1.4.6</version> + <scope>compile</scope> + </dependency> + </dependencies> <build> <plugins> diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/image/DataTypeDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/image/DataTypeDto.java new file mode 100644 index 0000000000000000000000000000000000000000..cd31aa225531b4331e77144b51445a8daeec0046 --- /dev/null +++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/image/DataTypeDto.java @@ -0,0 +1,66 @@ +package at.tuwien.api.container.image; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.*; +import lombok.extern.jackson.Jacksonized; + +@Getter +@Setter +@Builder +@NoArgsConstructor +@AllArgsConstructor +@Jacksonized +@ToString +public class DataTypeDto { + + @NotBlank + @JsonProperty("display_name") + @Schema(example = "BIGINT") + private String displayName; + + @NotBlank + @Schema(example = "bigint") + private String value; + + @JsonProperty("size_min") + private Integer sizeMin; + + @JsonProperty("size_max") + private Integer sizeMax; + + @JsonProperty("size_default") + private Integer sizeDefault; + + @JsonProperty("size_required") + private Boolean sizeRequired; + + @JsonProperty("d_min") + private Integer dMin; + + @JsonProperty("d_max") + private Integer dMax; + + @JsonProperty("d_default") + private Integer dDefault; + + @JsonProperty("d_required") + private Boolean dRequired; + + @NotNull + @Schema(example = "https://mariadb.com/kb/en/bigint/") + private String documentation; + + @NotNull + @Schema(description = "frontend needs to quote this data type") + @JsonProperty("is_quoted") + private Boolean quoted; + + @NotNull + @JsonProperty("is_buildable") + @Schema(description = "frontend can build this data type") + private Boolean buildable; + +} diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/image/ImageDateDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/image/ImageDateDto.java deleted file mode 100644 index 6fc25ad3cb3ddf860d11e9b4a5ac4ae75b98c277..0000000000000000000000000000000000000000 --- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/image/ImageDateDto.java +++ /dev/null @@ -1,48 +0,0 @@ -package at.tuwien.api.container.image; - -import com.fasterxml.jackson.annotation.JsonFormat; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.swagger.v3.oas.annotations.media.Schema; -import lombok.*; - -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; -import lombok.extern.jackson.Jacksonized; - -import java.time.Instant; - -@Getter -@Setter -@Builder -@NoArgsConstructor -@AllArgsConstructor -@Jacksonized -@ToString -public class ImageDateDto { - - @NotNull - private Long id; - - @NotBlank - @JsonProperty("database_format") - @Schema(example = "%d.%c.%Y") - private String databaseFormat; - - @NotBlank - @JsonProperty("unix_format") - @Schema(example = "dd.MM.YYYY") - private String unixFormat; - - @NotNull - @JsonProperty("has_time") - @Schema(example = "false") - private Boolean hasTime; - - - @NotNull - @Schema(example = "2021-03-12T15:26:21Z") - @JsonProperty("created_at") - @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", timezone = "UTC") - private Instant createdAt; - -} diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/image/ImageDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/image/ImageDto.java index c0cf7f3bceebc8aefa73c267e422d0cb7e9fa3d0..743f1f2b0a8ab67795a58b060d6ef30597d36837 100644 --- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/image/ImageDto.java +++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/image/ImageDto.java @@ -38,9 +38,6 @@ public class ImageDto { @Schema(example = "org.mariadb.jdbc.Driver") private String driverClass; - @JsonProperty("date_formats") - private List<ImageDateDto> dateFormats; - @NotBlank @Schema(example = "org.hibernate.dialect.MariaDBDialect") private String dialect; @@ -60,4 +57,8 @@ public class ImageDto { @Schema(example = "3306") private Integer defaultPort; + @NotNull + @JsonProperty("data_types") + private List<DataTypeDto> dataTypes; + } diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/internal/PrivilegedContainerDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/internal/PrivilegedContainerDto.java index 8de17a48f3221ed4ace1d837ec67183186731381..8bda16bf41630376ad105399ca1aca048d1514e6 100644 --- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/internal/PrivilegedContainerDto.java +++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/internal/PrivilegedContainerDto.java @@ -1,6 +1,5 @@ package at.tuwien.api.container.internal; -import at.tuwien.api.container.image.ImageDateDto; import at.tuwien.api.container.image.ImageDto; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/ViewColumnDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/ViewColumnDto.java index 337a61a637ea03437f74462c3b1c245e6943999d..75b0a3d68452e5ac223a85f351fb6078f04fc09e 100644 --- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/ViewColumnDto.java +++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/ViewColumnDto.java @@ -1,6 +1,5 @@ package at.tuwien.api.database; -import at.tuwien.api.container.image.ImageDateDto; import at.tuwien.api.database.table.columns.ColumnTypeDto; import at.tuwien.api.database.table.columns.concepts.ConceptDto; import at.tuwien.api.database.table.columns.concepts.UnitDto; @@ -47,9 +46,6 @@ public class ViewColumnDto { @Schema private String alias; - @JsonProperty("date_format") - private ImageDateDto dateFormat; - @NotNull @JsonProperty("auto_generated") @Schema(example = "false") diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/query/ImportCsvDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/query/ImportCsvDto.java deleted file mode 100644 index eac536143e1b59efce8617f740334acc74b77f5a..0000000000000000000000000000000000000000 --- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/query/ImportCsvDto.java +++ /dev/null @@ -1,49 +0,0 @@ -package at.tuwien.api.database.query; - -import com.fasterxml.jackson.annotation.JsonProperty; -import io.swagger.v3.oas.annotations.media.Schema; -import lombok.*; - -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; -import lombok.extern.jackson.Jacksonized; - -@Getter -@Setter -@Builder -@NoArgsConstructor -@AllArgsConstructor -@Jacksonized -@ToString -public class ImportCsvDto { - - @NotBlank - @Schema(example = "file.csv") - private String location; - - @Min(value = 0L) - @JsonProperty("skip_lines") - private Long skipLines; - - @JsonProperty("false_element") - private String falseElement; - - @JsonProperty("true_element") - private String trueElement; - - @JsonProperty("null_element") - @Schema(example = "NA") - private String nullElement; - - @NotNull - @Schema(example = ",") - private Character separator; - - @Schema(example = "\"") - private Character quote; - - @JsonProperty("line_termination") - @Schema(example = "\\r\\n") - private String lineTermination; -} diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/query/ImportDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/query/ImportDto.java index b865f7892cf74fcf1ae4a0feeec3075c87fa5817..043e3bc3eea9c0c438fb34f2e1b9427c0607bb18 100644 --- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/query/ImportDto.java +++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/query/ImportDto.java @@ -27,16 +27,6 @@ public class ImportDto { @JsonProperty("skip_lines") private Long skipLines; - @JsonProperty("false_element") - private String falseElement; - - @JsonProperty("true_element") - private String trueElement; - - @JsonProperty("null_element") - @Schema(example = "NA") - private String nullElement; - @NotNull @Schema(example = ",") private Character separator; diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnCreateDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnCreateDto.java index 37aa493020845255694d4d0fb0ac5cc7900803f2..2d0696abc044db7dbffe8ace7444df0b9f7127c4 100644 --- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnCreateDto.java +++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnCreateDto.java @@ -52,9 +52,6 @@ public class ColumnCreateDto { @JsonProperty("unit_uri") private String unitUri; - @Schema(description = "date format id") - private Long dfid; - @Schema(description = "enum values, only considered when type = ENUM") private List<String> enums; diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java index a506dbca82e9e64d4a0cec4fec12640452f56456..4e95fefc3e993d53c13107c48112939f55c0e03e 100644 --- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java +++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java @@ -1,6 +1,5 @@ package at.tuwien.api.database.table.columns; -import at.tuwien.api.container.image.ImageDateDto; import at.tuwien.api.database.ViewDto; import at.tuwien.api.database.table.TableDto; import at.tuwien.api.database.table.columns.concepts.ConceptDto; @@ -59,9 +58,6 @@ public class ColumnDto { @Schema private String alias; - @JsonProperty("date_format") - private ImageDateDto dateFormat; - @NotNull @JsonProperty("auto_generated") @Schema(example = "false") diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImage.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImage.java index 8250b6ae4350c48eca7c5dc1d0ec0a08935e9196..080a843aada9982608cb534c0e24027439e9d9c0 100644 --- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImage.java +++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImage.java @@ -57,9 +57,6 @@ public class ContainerImage { @Column(nullable = false, unique = true, columnDefinition = "BOOLEAN DEFAULT FALSE") private Boolean isDefault = false; - @OneToMany(fetch = FetchType.LAZY, cascade = {CascadeType.ALL, CascadeType.PERSIST}, mappedBy = "image") - private List<ContainerImageDate> dateFormats; - @ToString.Exclude @OneToMany(fetch = FetchType.LAZY, mappedBy = "image") private List<Container> containers; @@ -73,4 +70,8 @@ public class ContainerImage { @Column(columnDefinition = "TIMESTAMP") private Instant lastModified; + @ToString.Exclude + @OneToMany(fetch = FetchType.LAZY, mappedBy = "image") + private List<DataType> dataTypes; + } diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImageDate.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImageDate.java deleted file mode 100644 index 5b370ecc0664fa438e34511d606c75d0219ceb9f..0000000000000000000000000000000000000000 --- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImageDate.java +++ /dev/null @@ -1,59 +0,0 @@ -package at.tuwien.entities.container.image; - -import com.fasterxml.jackson.annotation.JsonFormat; -import lombok.*; -import org.hibernate.annotations.GenericGenerator; -import org.springframework.data.annotation.CreatedDate; -import org.springframework.data.jpa.domain.support.AuditingEntityListener; - -import jakarta.persistence.*; - -import java.time.Instant; - -@Data -@Entity -@Builder -@ToString -@AllArgsConstructor -@NoArgsConstructor -@EntityListeners(AuditingEntityListener.class) -@EqualsAndHashCode(onlyExplicitlyIncluded = true) -@Table(name = "mdb_images_date", uniqueConstraints = @UniqueConstraint(columnNames = {"database_format"})) -public class ContainerImageDate { - - @Id - @EqualsAndHashCode.Include - @GeneratedValue(generator = "dates-sequence") - @GenericGenerator(name = "dates-sequence", strategy = "increment") - @Column(updatable = false, nullable = false) - private Long id; - - @EqualsAndHashCode.Include - @Column(name = "iid") - private Long iid; - - @ToString.Exclude - @ManyToOne(fetch = FetchType.LAZY, cascade = {CascadeType.MERGE}) - @JoinColumns({ - @JoinColumn(name = "iid", insertable = false, updatable = false) - }) - private ContainerImage image; - - @Column(name = "example", nullable = false) - private String example; - - @Column(name = "has_time", nullable = false) - private Boolean hasTime; - - @Column(name = "database_format", nullable = false) - private String databaseFormat; - - @Column(name = "unix_format", nullable = false) - private String unixFormat; - - @CreatedDate - @Column(name = "created_at", nullable = false, updatable = false, columnDefinition = "TIMESTAMP") - @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", timezone = "UTC") - private Instant createdAt; - -} diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImageDateKey.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImageDateKey.java deleted file mode 100644 index c5ae9598f7baf4de7df1487561dacca93d4c52f5..0000000000000000000000000000000000000000 --- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/ContainerImageDateKey.java +++ /dev/null @@ -1,14 +0,0 @@ -package at.tuwien.entities.container.image; - -import lombok.EqualsAndHashCode; - -import java.io.Serializable; - -@EqualsAndHashCode -public class ContainerImageDateKey implements Serializable { - - private Long id; - - private Long iid; - -} diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/DataType.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/DataType.java new file mode 100644 index 0000000000000000000000000000000000000000..8333470ad3ba7fdf6014caf32f786ce4b496b0e0 --- /dev/null +++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/container/image/DataType.java @@ -0,0 +1,73 @@ +package at.tuwien.entities.container.image; + +import jakarta.persistence.*; +import lombok.*; +import org.hibernate.annotations.GenericGenerator; +import org.springframework.data.jpa.domain.support.AuditingEntityListener; + +import java.util.List; + +@Data +@Entity +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +@EntityListeners(AuditingEntityListener.class) +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +@Table(name = "mdb_image_types") +public class DataType { + + @Id + @EqualsAndHashCode.Include + @GeneratedValue(generator = "image-type-sequence") + @GenericGenerator(name = "image-type-sequence", strategy = "increment") + @Column(updatable = false, nullable = false) + public Long id; + + @Column(name = "display_name", nullable = false) + private String displayName; + + @Column(name = "value", nullable = false, unique = true) + private String value; + + @Column(name = "size_min", nullable = false) + private Integer sizeMin; + + @Column(name = "size_max") + private Integer sizeMax; + + @Column(name = "size_default") + private Integer sizeDefault; + + @Column(name = "size_required", nullable = false) + private Boolean sizeRequired; + + @Column(name = "d_min") + private Integer dMin; + + @Column(name = "d_max") + private Integer dMax; + + @Column(name = "d_default") + private Integer dDefault; + + @Column(name = "d_required", nullable = false) + private Boolean dRequired; + + @Column(nullable = false) + private String documentation; + + @Column(name = "is_quoted", nullable = false) + private Boolean quoted; + + @Column(name = "is_buildable", nullable = false) + private Boolean buildable; + + @ManyToOne(fetch = FetchType.LAZY, cascade = {CascadeType.ALL, CascadeType.PERSIST}) + @JoinColumns({ + @JoinColumn(name = "image_id", referencedColumnName = "id") + }) + private ContainerImage image; + +} diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/ViewColumn.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/ViewColumn.java index ff18c0137d06964bcc49c02367f95eb49a3b4033..b9eff2c6947de4e8f69693467b57f67db720a8b4 100644 --- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/ViewColumn.java +++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/ViewColumn.java @@ -1,6 +1,5 @@ package at.tuwien.entities.database; -import at.tuwien.entities.container.image.ContainerImageDate; import at.tuwien.entities.database.table.columns.TableColumnType; import lombok.*; import org.hibernate.annotations.GenericGenerator; @@ -25,11 +24,6 @@ public class ViewColumn implements Comparable<ViewColumn> { @Column(updatable = false, nullable = false) private Long id; - @ToString.Exclude - @OneToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "dfid", referencedColumnName = "id") - private ContainerImageDate dateFormat; - @ToString.Exclude @ManyToOne(fetch = FetchType.LAZY, cascade = {CascadeType.MERGE}) @JoinColumns({ diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java index dd59e210033886535cbf6b708175b47dbc0f8341..9ddf89c453617c6f345c73b4c98cb9eb2678e3a0 100644 --- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java +++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java @@ -1,6 +1,5 @@ package at.tuwien.entities.database.table.columns; -import at.tuwien.entities.container.image.ContainerImageDate; import at.tuwien.entities.database.table.Table; import com.fasterxml.jackson.annotation.JsonFormat; import lombok.*; @@ -38,11 +37,6 @@ public class TableColumn implements Comparable<TableColumn> { @Column(updatable = false, nullable = false) private Long id; - @ToString.Exclude - @OneToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "dfid", referencedColumnName = "id") - private ContainerImageDate dateFormat; - @ToString.Exclude @ManyToOne(fetch = FetchType.LAZY, cascade = {CascadeType.MERGE}) @JoinColumns({ diff --git a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java index f6c96f133d31e59bd72214133d5f707d8fdf0468..0555e1c62c5d2ee91ba7e0b7f59f68b5372ceb02 100644 --- a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java +++ b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java @@ -4,6 +4,7 @@ import at.tuwien.api.auth.SignupRequestDto; import at.tuwien.api.container.ContainerBriefDto; import at.tuwien.api.container.ContainerCreateDto; import at.tuwien.api.container.ContainerDto; +import at.tuwien.api.container.image.DataTypeDto; import at.tuwien.api.container.image.ImageBriefDto; import at.tuwien.api.container.image.ImageCreateDto; import at.tuwien.api.container.image.ImageDto; @@ -55,6 +56,7 @@ import at.tuwien.api.user.external.ExternalResultType; import at.tuwien.api.user.external.affiliation.ExternalAffiliationDto; import at.tuwien.entities.container.Container; import at.tuwien.entities.container.image.ContainerImage; +import at.tuwien.entities.container.image.DataType; import at.tuwien.entities.database.*; import at.tuwien.entities.database.table.Table; import at.tuwien.entities.database.table.columns.TableColumn; @@ -86,6 +88,14 @@ public interface MetadataMapper { org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(MetadataMapper.class); + @Mappings({ + @Mapping(target = "dMin", source = "DMin"), + @Mapping(target = "dMax", source = "DMax"), + @Mapping(target = "dDefault", source = "DDefault"), + @Mapping(target = "dRequired", source = "DRequired") + }) + DataTypeDto dataTypeToDataTypeDto(DataType data); + BannerMessageDto bannerMessageToBannerMessageDto(BannerMessage data); BannerMessageBriefDto bannerMessageToBannerMessageBriefDto(BannerMessage data); diff --git a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/AccessEndpoint.java b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/AccessEndpoint.java index 9cdcfdedf9e88fd0138e236db6f15afb959ff2dc..f50f916ac4e2a900127244d619814d345c84b809 100644 --- a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/AccessEndpoint.java +++ b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/AccessEndpoint.java @@ -101,18 +101,19 @@ public class AccessEndpoint { data.getType()); final Database database = databaseService.findById(databaseId); final User user = userService.findByUsername(principal.getName()); - if (database.getOwner().equals(user)) { - log.error("Failed to give access to user with id {}: not owner", userId); - throw new NotAllowedException("Failed to give access to user with id " + userId + ": not owner"); + if (!database.getOwner().equals(user)) { + log.error("Failed to create access: not owner"); + throw new NotAllowedException("Failed to create access: not owner"); } + final User otherUser = userService.findById(userId); try { - accessService.find(database, user); - log.error("Failed to give access to user with id {}: already has access", userId); - throw new NotAllowedException("Failed to give access to user with id " + userId + ": already has access"); + accessService.find(database, otherUser); + log.error("Failed to create access to user with id {}: already has access", userId); + throw new NotAllowedException("Failed to create access to user with id " + userId + ": already has access"); } catch (AccessNotFoundException e) { /* ignore */ } - accessService.create(database, user, data.getType()); + accessService.create(database, otherUser, data.getType()); return ResponseEntity.accepted() .build(); } @@ -163,12 +164,13 @@ public class AccessEndpoint { data.getType()); final Database database = databaseService.findById(databaseId); final User user = userService.findByUsername(principal.getName()); - if (database.getOwner().equals(user)) { - log.error("Failed to give access to user with id {}: not owner", userId); - throw new NotAllowedException("Failed to give access to user with id " + userId + ": not owner"); + if (!database.getOwner().equals(user)) { + log.error("Failed to update access: not owner"); + throw new NotAllowedException("Failed to update access: not owner"); } - accessService.find(database, user); - accessService.update(database, user, data.getType()); + final User otherUser = userService.findById(userId); + accessService.find(database, otherUser); + accessService.update(database, otherUser, data.getType()); return ResponseEntity.accepted() .build(); } @@ -211,8 +213,8 @@ public class AccessEndpoint { log.trace("principal is allowed to check foreign user access"); } final Database database = databaseService.findById(databaseId); - final User user = userService.findById(userId); - final DatabaseAccess access = accessService.find(database, user); + final User otherUser = userService.findById(userId); + final DatabaseAccess access = accessService.find(database, otherUser); final DatabaseAccessDto dto = databaseMapper.databaseAccessToDatabaseAccessDto(access); log.trace("check access resulted in dto {}", dto); return ResponseEntity.ok(dto); @@ -263,11 +265,12 @@ public class AccessEndpoint { final Database database = databaseService.findById(databaseId); final User user = userService.findByUsername(principal.getName()); if (!database.getOwner().equals(user)) { - log.error("Failed to revoke access to user with id {}: not owner", user.getId()); - throw new NotAllowedException("Failed to revoke access to user with id " + user.getId() + ": not owner"); + log.error("Failed to revoke access: not owner"); + throw new NotAllowedException("Failed to revoke access: not owner"); } - accessService.find(database, user); - accessService.delete(database, user); + final User otherUser = userService.findById(userId); + accessService.find(database, otherUser); + accessService.delete(database, otherUser); return ResponseEntity.accepted() .build(); } diff --git a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/DatabaseEndpoint.java b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/DatabaseEndpoint.java index e89a38b6f6fbc9a021bc62b5a31be44fa0d86040..d5c316fed960a8600352df7201871a370654c021 100644 --- a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/DatabaseEndpoint.java +++ b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/DatabaseEndpoint.java @@ -3,6 +3,7 @@ package at.tuwien.endpoints; import at.tuwien.api.database.*; import at.tuwien.api.error.ApiErrorDto; import at.tuwien.entities.container.Container; +import at.tuwien.entities.container.image.DataType; import at.tuwien.entities.database.Database; import at.tuwien.entities.database.DatabaseAccess; import at.tuwien.entities.user.User; diff --git a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java index 4fb8240b1d00f00f39a233cd3492e0b06940314a..738e30d4e48aba2dc52703d0ca2bd3cb53b2b63a 100644 --- a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java +++ b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java @@ -3,9 +3,7 @@ package at.tuwien.endpoints; import at.tuwien.api.database.table.TableBriefDto; import at.tuwien.api.database.table.TableCreateDto; import at.tuwien.api.database.table.TableDto; -import at.tuwien.api.database.table.columns.ColumnCreateDto; import at.tuwien.api.database.table.columns.ColumnDto; -import at.tuwien.api.database.table.columns.ColumnTypeDto; import at.tuwien.api.database.table.columns.concepts.ColumnSemanticsUpdateDto; import at.tuwien.api.error.ApiErrorDto; import at.tuwien.api.semantics.EntityDto; @@ -16,7 +14,10 @@ import at.tuwien.entities.database.table.columns.TableColumn; import at.tuwien.entities.user.User; import at.tuwien.exception.*; import at.tuwien.mapper.MetadataMapper; -import at.tuwien.service.*; +import at.tuwien.service.DatabaseService; +import at.tuwien.service.EntityService; +import at.tuwien.service.TableService; +import at.tuwien.service.UserService; import at.tuwien.utils.UserUtil; import at.tuwien.validation.EndpointValidator; import io.micrometer.observation.annotation.Observed; @@ -39,7 +40,7 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.web.bind.annotation.*; import java.security.Principal; -import java.util.*; +import java.util.List; import java.util.stream.Collectors; @Log4j2 @@ -349,15 +350,6 @@ public class TableEndpoint { final Database database = databaseService.findById(databaseId); endpointValidator.validateOnlyAccess(database, principal, true); endpointValidator.validateColumnCreateConstraints(data); - final List<ColumnCreateDto> failedDateColumns = data.getColumns() - .stream() - .filter(column -> List.of(ColumnTypeDto.DATE, ColumnTypeDto.DATETIME, ColumnTypeDto.TIME, ColumnTypeDto.TIMESTAMP).contains(column.getType())) - .filter(column -> Objects.isNull(column.getDfid())) - .toList(); - if (!failedDateColumns.isEmpty()) { - log.error("Failed to create table: date column(s) {} do not contain date format id", failedDateColumns.stream().map(ColumnCreateDto::getName).toList()); - throw new MalformedException("Failed to create table: date column(s) " + failedDateColumns.stream().map(ColumnCreateDto::getName).toList() + " do not contain date format id"); - } final Table table = tableService.createTable(database, data, principal); final TableDto dto = metadataMapper.customTableToTableDto(table); log.info("Created table with id {}", dto.getId()); diff --git a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/UserEndpoint.java b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/UserEndpoint.java index b81a8142f727972c95ce0ea5573b99e3d1e59be9..34082d18c14b4c6c92b9d353bedbb8892b571fe8 100644 --- a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/UserEndpoint.java +++ b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/UserEndpoint.java @@ -5,7 +5,10 @@ import at.tuwien.api.auth.RefreshTokenRequestDto; import at.tuwien.api.auth.SignupRequestDto; import at.tuwien.api.error.ApiErrorDto; import at.tuwien.api.keycloak.TokenDto; -import at.tuwien.api.user.*; +import at.tuwien.api.user.UserBriefDto; +import at.tuwien.api.user.UserDto; +import at.tuwien.api.user.UserPasswordDto; +import at.tuwien.api.user.UserUpdateDto; import at.tuwien.entities.database.Database; import at.tuwien.entities.user.User; import at.tuwien.exception.*; @@ -26,6 +29,7 @@ import jakarta.validation.Valid; import jakarta.validation.constraints.NotNull; import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; @@ -284,7 +288,13 @@ public class UserEndpoint { } } final UserDto dto = userMapper.userToUserDto(user); - return ResponseEntity.ok() + final HttpHeaders headers = new HttpHeaders(); + if (UserUtil.isSystem(principal)) { + headers.set("X-Username", user.getUsername()); + headers.set("X-Password", user.getMariadbPassword()); + } + return ResponseEntity.status(HttpStatus.OK) + .headers(headers) .body(dto); } diff --git a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/validation/EndpointValidator.java b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/validation/EndpointValidator.java index 7f05bf84a58ad90788476850fed0765394f80ae2..a3fb6833ce05abf5cd8c2dd4f5842722e17785a2 100644 --- a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/validation/EndpointValidator.java +++ b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/validation/EndpointValidator.java @@ -68,9 +68,9 @@ public class EndpointValidator { if (data == null) { throw new MalformedException("Validation failed: table data is null"); } - final List<ColumnTypeDto> needSize = List.of(ColumnTypeDto.CHAR, ColumnTypeDto.VARCHAR, ColumnTypeDto.BINARY, ColumnTypeDto.VARBINARY, ColumnTypeDto.BIT, ColumnTypeDto.TINYINT, ColumnTypeDto.SMALLINT, ColumnTypeDto.MEDIUMINT, ColumnTypeDto.INT); - final List<ColumnTypeDto> needSizeAndD = List.of(ColumnTypeDto.DOUBLE, ColumnTypeDto.DECIMAL); - final List<ColumnTypeDto> needDateFormat = List.of(ColumnTypeDto.DATETIME, ColumnTypeDto.TIMESTAMP, ColumnTypeDto.TIME); + final List<ColumnTypeDto> needSize = List.of(ColumnTypeDto.VARCHAR, ColumnTypeDto.BINARY, ColumnTypeDto.VARBINARY); + final List<ColumnTypeDto> canHaveSize = List.of(ColumnTypeDto.CHAR, ColumnTypeDto.VARCHAR, ColumnTypeDto.BINARY, ColumnTypeDto.VARBINARY, ColumnTypeDto.BIT, ColumnTypeDto.TINYINT, ColumnTypeDto.SMALLINT, ColumnTypeDto.MEDIUMINT, ColumnTypeDto.INT); + final List<ColumnTypeDto> canHaveSizeAndD = List.of(ColumnTypeDto.DOUBLE, ColumnTypeDto.DECIMAL); /* check size */ final Optional<ColumnCreateDto> optional0 = data.getColumns() .stream() @@ -78,36 +78,37 @@ public class EndpointValidator { .filter(c -> needSize.contains(c.getType())) .findFirst(); if (optional0.isPresent()) { - log.error("Validation failed: column {} needs size parameter", optional0.get().getName()); - throw new MalformedException("Validation failed: column " + optional0.get().getName() + " needs size parameter"); + log.error("Validation failed: column {} need size parameter", optional0.get().getName()); + throw new MalformedException("Validation failed: column " + optional0.get().getName() + " need size parameter"); } - /* check size and d */ - final Optional<ColumnCreateDto> optional1 = data.getColumns() + final Optional<ColumnCreateDto> optional0a = data.getColumns() .stream() - .filter(c -> needSizeAndD.contains(c.getType())) - .filter(c -> Objects.isNull(c.getSize()) || Objects.isNull(c.getD())) + .filter(c -> !Objects.isNull(c.getSize())) + .filter(c -> canHaveSize.contains(c.getType())) + .filter(c -> c.getSize() < 0) .findFirst(); - if (optional1.isPresent()) { - log.error("Validation failed: column {} needs size and d parameter", optional1.get().getName()); - throw new MalformedException("Validation failed: column " + optional1.get().getName() + " needs size and d parameter"); + if (optional0a.isPresent()) { + log.error("Validation failed: column {} needs positive size parameter", optional0a.get().getName()); + throw new MalformedException("Validation failed: column " + optional0a.get().getName() + " needs positive size parameter"); } - final Optional<ColumnCreateDto> optional1a = data.getColumns() + final Optional<ColumnCreateDto> optional0b = data.getColumns() .stream() - .filter(c -> needSizeAndD.contains(c.getType())) - .filter(c -> c.getSize() > 65 || c.getD() > 38) + .filter(c -> !Objects.isNull(c.getSize())) + .filter(c -> !canHaveSize.contains(c.getType())) .findFirst(); - if (optional1a.isPresent()) { - log.error("Validation failed: column {} needs size (max 65) and d (max 30)", optional1a.get().getName()); - throw new MalformedException("Validation failed: column " + optional1a.get().getName() + " needs size (max 65) and d (max 30)"); + if (optional0b.isPresent()) { + log.error("Validation failed: column {} cannot have size parameter", optional0b.get().getName()); + throw new MalformedException("Validation failed: column " + optional0b.get().getName() + " cannot have size parameter"); } - final Optional<ColumnCreateDto> optional1b = data.getColumns() + /* check size and d */ + final Optional<ColumnCreateDto> optional1 = data.getColumns() .stream() - .filter(c -> needSizeAndD.contains(c.getType())) - .filter(c -> c.getSize() < c.getD()) + .filter(c -> Objects.isNull(c.getSize()) ^ Objects.isNull(c.getD())) + .filter(c -> canHaveSizeAndD.contains(c.getType())) .findFirst(); - if (optional1b.isPresent()) { - log.error("Validation failed: column {} needs size >= d", optional1b.get().getName()); - throw new MalformedException("Validation failed: column " + optional1b.get().getName() + " needs size >= d"); + if (optional1.isPresent()) { + log.error("Validation failed: column {} either needs both size and d parameter or none (use default)", optional1.get().getName()); + throw new MalformedException("Validation failed: column " + optional1.get().getName() + " either needs both size and d parameter or none (use default)"); } /* check enum */ final Optional<ColumnCreateDto> optional2 = data.getColumns() @@ -129,16 +130,6 @@ public class EndpointValidator { log.error("Validation failed: column {} needs at least 1 allowed set value", optional3.get().getName()); throw new MalformedException("Validation failed: column " + optional3.get().getName() + " needs at least 1 allowed set value"); } - /* check date */ - final Optional<ColumnCreateDto> optional4 = data.getColumns() - .stream() - .filter(c -> needDateFormat.contains(c.getType())) - .filter(c -> Objects.isNull(c.getDfid())) - .findFirst(); - if (optional4.isPresent()) { - log.error("Validation failed: column {} needs a format", optional4.get().getName()); - throw new MalformedException("Validation failed: column " + optional4.get().getName() + " needs a format"); - } } public boolean validateOnlyMineOrWriteAccessOrHasRole(User owner, Principal principal, DatabaseAccess access, String role) { diff --git a/dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml b/dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml index e2c9df6a59d8454661ca42b72617f2b67d6c1492..793a2b6021a270dfd6b23971e40675889b3e4980 100644 --- a/dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml +++ b/dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml @@ -66,7 +66,7 @@ dbrepo: searchService: http://localhost analyseService: http://localhost dataService: http://localhost:9093 - brokerService: http://localhost/admin/broker + brokerService: http://localhost:15672 authService: http://localhost/api/auth storageService: http://localhost/api/storage rorService: https://api.ror.org diff --git a/dbrepo-metadata-service/rest-service/src/main/resources/datatypes.json b/dbrepo-metadata-service/rest-service/src/main/resources/datatypes.json new file mode 100644 index 0000000000000000000000000000000000000000..3779d12cbe32b67fa163cf6b0285b9e01f7dc681 --- /dev/null +++ b/dbrepo-metadata-service/rest-service/src/main/resources/datatypes.json @@ -0,0 +1,15 @@ +[ + { + "name": "", + "size": { + "min": 0, + "required": true + }, + "d": { + "required": false + }, + "documentation": "https://mariadb.com/kb/en/bigint/", + "quoted": false, + "buildable": true + } +] \ No newline at end of file diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/AccessEndpointUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/AccessEndpointUnitTest.java index 69d817afb763fa47b878b71fb315bb8dbb02f750..7c6061ed1e663c27d9f2a72bc64e309eaee0ffc1 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/AccessEndpointUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/AccessEndpointUnitTest.java @@ -1,17 +1,18 @@ package at.tuwien.endpoints; -import at.tuwien.mapper.MetadataMapper; -import at.tuwien.test.AbstractUnitTest; import at.tuwien.api.database.AccessTypeDto; import at.tuwien.api.database.DatabaseAccessDto; import at.tuwien.entities.database.Database; import at.tuwien.entities.database.DatabaseAccess; import at.tuwien.entities.user.User; import at.tuwien.exception.*; +import at.tuwien.mapper.MetadataMapper; import at.tuwien.repository.DatabaseRepository; import at.tuwien.repository.UserRepository; import at.tuwien.service.AccessService; +import at.tuwien.test.AbstractUnitTest; import lombok.extern.log4j.Log4j2; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -51,13 +52,18 @@ public class AccessEndpointUnitTest extends AbstractUnitTest { @Autowired private MetadataMapper metadataMapper; + @BeforeEach + public void beforeEach() { + genesis(); + } + @Test @WithAnonymousUser public void create_anonymous_fails() { /* test */ assertThrows(org.springframework.security.access.AccessDeniedException.class, () -> { - generic_create(null, USER_2_ID, null, null); + generic_create(null, null, null, null); }); } @@ -67,7 +73,7 @@ public class AccessEndpointUnitTest extends AbstractUnitTest { /* test */ assertThrows(org.springframework.security.access.AccessDeniedException.class, () -> { - generic_create(USER_2_PRINCIPAL, USER_4_ID, USER_4_USERNAME, USER_4); + generic_create(USER_2_PRINCIPAL, USER_2, USER_4_ID, USER_4); }); } @@ -82,7 +88,7 @@ public class AccessEndpointUnitTest extends AbstractUnitTest { .thenReturn(DATABASE_1_USER_1_READ_ACCESS); /* test */ - generic_create(USER_2_PRINCIPAL, USER_2_ID, USER_2_USERNAME, USER_2); + generic_create(USER_1_PRINCIPAL, USER_1, USER_2_ID, USER_2); } @Test @@ -129,7 +135,7 @@ public class AccessEndpointUnitTest extends AbstractUnitTest { /* test */ assertThrows(org.springframework.security.access.AccessDeniedException.class, () -> { - generic_update(null, USER_4_USERNAME, USER_4, null, null); + generic_update(null, null, null, null, null); }); } @@ -138,8 +144,8 @@ public class AccessEndpointUnitTest extends AbstractUnitTest { public void update_hasRoleNoAccess_fails() { /* test */ - assertThrows(NotAllowedException.class, () -> { - generic_update(null, USER_4_USERNAME, USER_4, USER_1_PRINCIPAL, USER_1); + assertThrows(AccessNotFoundException.class, () -> { + generic_update(USER_1_PRINCIPAL, USER_1, USER_4_ID, USER_4, null); }); } @@ -149,7 +155,7 @@ public class AccessEndpointUnitTest extends AbstractUnitTest { /* test */ assertThrows(org.springframework.security.access.AccessDeniedException.class, () -> { - generic_update(null, USER_4_USERNAME, USER_4, USER_4_PRINCIPAL, USER_4); + generic_update(USER_4_PRINCIPAL, USER_4, USER_1_ID, USER_1, null); }); } @@ -165,7 +171,7 @@ public class AccessEndpointUnitTest extends AbstractUnitTest { .update(eq(DATABASE_1), eq(USER_2), any(AccessTypeDto.class)); /* test */ - generic_update(DATABASE_1_USER_2_WRITE_OWN_ACCESS, USER_2_USERNAME, USER_2, USER_2_PRINCIPAL, USER_2); + generic_update(USER_1_PRINCIPAL, USER_1, USER_2_ID, USER_2, DATABASE_1_USER_2_WRITE_OWN_ACCESS); } @Test @@ -174,7 +180,7 @@ public class AccessEndpointUnitTest extends AbstractUnitTest { /* test */ assertThrows(org.springframework.security.access.AccessDeniedException.class, () -> { - generic_revoke(USER_1_PRINCIPAL, USER_1); + generic_revoke(null, null, USER_1_ID, USER_1); }); } @@ -184,7 +190,7 @@ public class AccessEndpointUnitTest extends AbstractUnitTest { /* test */ assertThrows(org.springframework.security.access.AccessDeniedException.class, () -> { - generic_revoke(USER_4_PRINCIPAL, USER_4); + generic_revoke(USER_4_PRINCIPAL, USER_4, USER_1_ID, USER_1); }); } @@ -200,14 +206,14 @@ public class AccessEndpointUnitTest extends AbstractUnitTest { .delete(DATABASE_1, USER_2); /* test */ - generic_revoke(USER_1_PRINCIPAL, USER_1); + generic_revoke(USER_1_PRINCIPAL, USER_1, USER_2_ID, USER_2); } /* ################################################################################################### */ /* ## GENERIC TEST CASES ## */ /* ################################################################################################### */ - protected void generic_create(Principal principal, UUID userId, String username, User user) + protected void generic_create(Principal principal, User principalUser, UUID userId, User user) throws NotAllowedException, DataServiceException, DataServiceConnectionException, UserNotFoundException, DatabaseNotFoundException, AccessNotFoundException, SearchServiceException, SearchServiceConnectionException { @@ -218,11 +224,18 @@ public class AccessEndpointUnitTest extends AbstractUnitTest { doThrow(AccessNotFoundException.class) .when(accessService) .find(DATABASE_1, user); + if (principalUser != null) { + when(userRepository.findByUsername(principal.getName())) + .thenReturn(Optional.of(principalUser)); + } else { + when(userRepository.findByUsername(anyString())) + .thenReturn(Optional.empty()); + } if (user != null) { - when(userRepository.findByUsername(username)) + when(userRepository.findById(userId)) .thenReturn(Optional.of(user)); } else { - when(userRepository.findByUsername(anyString())) + when(userRepository.findById(any(UUID.class))) .thenReturn(Optional.empty()); } @@ -268,61 +281,62 @@ public class AccessEndpointUnitTest extends AbstractUnitTest { } } - protected void generic_update(DatabaseAccess access, String otherUsername, User otherUser, Principal principal, - User user) throws NotAllowedException, DataServiceException, DataServiceConnectionException, - AccessNotFoundException, UserNotFoundException, DatabaseNotFoundException, SearchServiceException, - SearchServiceConnectionException { + protected void generic_update(Principal principal, User principalUser, UUID userId, User user, + DatabaseAccess access) throws NotAllowedException, DataServiceException, + DataServiceConnectionException, AccessNotFoundException, UserNotFoundException, DatabaseNotFoundException, + SearchServiceException, SearchServiceConnectionException { /* mock */ when(databaseRepository.findById(DATABASE_1_ID)) .thenReturn(Optional.of(DATABASE_1)); if (access != null) { - log.trace("mock access {} for user with id {} for database with id {}", access.getType(), USER_1_ID, DATABASE_1_ID); - when(accessService.find(DATABASE_1, USER_1)) + log.trace("mock access {} for user with id {} for database with id {}", access.getType(), userId, DATABASE_1_ID); + when(accessService.find(DATABASE_1, user)) .thenReturn(access); } else { - log.trace("mock no access for user with id {} for database with id {}", USER_1_ID, DATABASE_1_ID); + log.trace("mock no access for user with id {} for database with id {}", userId, DATABASE_1_ID); doThrow(AccessNotFoundException.class) .when(accessService) - .find(DATABASE_1, USER_1); + .find(DATABASE_1, user); } - if (otherUsername != null) { - when(userRepository.findByUsername(otherUsername)) - .thenReturn(Optional.of(otherUser)); + if (userId != null) { + when(userRepository.findById(userId)) + .thenReturn(Optional.of(user)); } else { - when(userRepository.findByUsername(anyString())) + when(userRepository.findById(any(UUID.class))) .thenReturn(Optional.empty()); } if (principal != null) { when(userRepository.findByUsername(principal.getName())) - .thenReturn(Optional.of(user)); + .thenReturn(Optional.of(principalUser)); } else { when(userRepository.findByUsername(anyString())) .thenReturn(Optional.empty()); } /* test */ - final ResponseEntity<?> response = accessEndpoint.update(DATABASE_1_ID, USER_1_ID, UPDATE_DATABASE_ACCESS_READ_DTO, principal); + final ResponseEntity<?> response = accessEndpoint.update(DATABASE_1_ID, userId, UPDATE_DATABASE_ACCESS_READ_DTO, principal); assertEquals(HttpStatus.ACCEPTED, response.getStatusCode()); assertNull(response.getBody()); } - protected void generic_revoke(Principal principal, User user) throws DataServiceConnectionException, - NotAllowedException, DataServiceException, UserNotFoundException, DatabaseNotFoundException, - AccessNotFoundException, SearchServiceException, SearchServiceConnectionException { + protected void generic_revoke(Principal principal, User principalUser, UUID userId, User user) + throws DataServiceConnectionException, NotAllowedException, DataServiceException, UserNotFoundException, + DatabaseNotFoundException, AccessNotFoundException, SearchServiceException, + SearchServiceConnectionException { /* mock */ - when(accessService.find(any(Database.class), eq(user))) - .thenReturn(DATABASE_1_USER_1_READ_ACCESS); when(databaseRepository.findById(DATABASE_1_ID)) .thenReturn(Optional.of(DATABASE_1)); if (principal != null) { when(userRepository.findByUsername(principal.getName())) - .thenReturn(Optional.of(user)); + .thenReturn(Optional.of(principalUser)); } + when(userRepository.findById(userId)) + .thenReturn(Optional.of(user)); /* test */ - final ResponseEntity<?> response = accessEndpoint.revoke(DATABASE_1_ID, USER_1_ID, principal); + final ResponseEntity<?> response = accessEndpoint.revoke(DATABASE_1_ID, userId, principal); assertEquals(HttpStatus.ACCEPTED, response.getStatusCode()); assertNull(response.getBody()); } diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/ActuatorComponentTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/ActuatorComponentTest.java index 238dec0db1736eccca0976a0d0c89badabd58dcd..78b7f086c300b546c808ed161abef2e3712e64c3 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/ActuatorComponentTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/ActuatorComponentTest.java @@ -2,6 +2,7 @@ package at.tuwien.endpoints; import at.tuwien.test.AbstractUnitTest; import lombok.extern.log4j.Log4j2; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -24,6 +25,11 @@ public class ActuatorComponentTest extends AbstractUnitTest { @Autowired private MockMvc mockMvc; + @BeforeEach + public void beforeEach() { + genesis(); + } + @Test public void actuatorInfo_succeeds() throws Exception { this.mockMvc.perform(get("/actuator/info")) diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/ConceptEndpointUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/ConceptEndpointUnitTest.java index 6698be6995ac00573e348274e5cb411f08755023..d48317f119cbbdf51087b16fc4c8df999bd519bb 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/ConceptEndpointUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/ConceptEndpointUnitTest.java @@ -4,6 +4,7 @@ import at.tuwien.test.AbstractUnitTest; import at.tuwien.api.database.table.columns.concepts.ConceptDto; import at.tuwien.service.ConceptService; import lombok.extern.log4j.Log4j2; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -31,6 +32,11 @@ public class ConceptEndpointUnitTest extends AbstractUnitTest { @Autowired private ConceptEndpoint conceptEndpoint; + @BeforeEach + public void beforeEach() { + genesis(); + } + @Test @WithAnonymousUser public void findAllConcepts_anonymous_succeeds() { diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/ImageEndpointUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/ImageEndpointUnitTest.java index 3d1c37d36348cd90e5239129957506758cc63f1e..c5c3c24cfd37ea0986079132cf67a4e527669892 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/ImageEndpointUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/ImageEndpointUnitTest.java @@ -9,6 +9,7 @@ import at.tuwien.entities.container.image.ContainerImage; import at.tuwien.exception.*; import at.tuwien.service.impl.ImageServiceImpl; import lombok.extern.log4j.Log4j2; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -37,6 +38,11 @@ public class ImageEndpointUnitTest extends AbstractUnitTest { @Autowired private ImageEndpoint imageEndpoint; + @BeforeEach + public void beforeEach() { + genesis(); + } + @Test @WithAnonymousUser public void findAll_anonymous_succeeds() { diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/LicenseEndpointUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/LicenseEndpointUnitTest.java index 5be4624021400c25bd130fab4ef150bffb9e328b..f45dd85fb4d13423451599e8ffc3cd046a8ebdca 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/LicenseEndpointUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/LicenseEndpointUnitTest.java @@ -4,6 +4,7 @@ import at.tuwien.test.AbstractUnitTest; import at.tuwien.api.database.LicenseDto; import at.tuwien.repository.LicenseRepository; import lombok.extern.log4j.Log4j2; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -30,6 +31,11 @@ public class LicenseEndpointUnitTest extends AbstractUnitTest { @Autowired private LicenseEndpoint licenseEndpoint; + @BeforeEach + public void beforeEach() { + genesis(); + } + @Test public void list_succeeds() { diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/MessageEndpointUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/MessageEndpointUnitTest.java index cea67bc5103de5eeac619f24d0f15c7042d9b115..59166b720076f1541774f6ed8ac3dcc816da9e9a 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/MessageEndpointUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/MessageEndpointUnitTest.java @@ -8,6 +8,7 @@ import at.tuwien.api.maintenance.BannerMessageUpdateDto; import at.tuwien.entities.maintenance.BannerMessage; import at.tuwien.service.BannerMessageService; import lombok.extern.log4j.Log4j2; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -35,6 +36,11 @@ public class MessageEndpointUnitTest extends AbstractUnitTest { @Autowired private MessageEndpoint messageEndpoint; + @BeforeEach + public void beforeEach() { + genesis(); + } + @Test @WithAnonymousUser public void list_anonymous_succeeds() { diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/MetadataEndpointUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/MetadataEndpointUnitTest.java index d024978449f6269615839a13067e42c3a02ab784..7ed994534287e13f0f3b33e0c19abcfb360e3f50 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/MetadataEndpointUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/MetadataEndpointUnitTest.java @@ -6,6 +6,7 @@ import at.tuwien.oaipmh.OaiRecordParameters; import at.tuwien.repository.IdentifierRepository; import at.tuwien.utils.XmlUtils; import lombok.extern.log4j.Log4j2; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -33,6 +34,11 @@ public class MetadataEndpointUnitTest extends AbstractUnitTest { @Autowired private MetadataEndpoint metadataEndpoint; + @BeforeEach + public void beforeEach() { + genesis(); + } + @Test @WithAnonymousUser public void identify_succeeds() { diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/OntologyEndpointUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/OntologyEndpointUnitTest.java index fc20a0b9e308774f0e8ef1c3558b63454f2be088..b7af494253ebc97fa51f3b5da86f72a9515a4703 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/OntologyEndpointUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/OntologyEndpointUnitTest.java @@ -12,6 +12,7 @@ import lombok.extern.log4j.Log4j2; import org.apache.jena.sys.JenaSystem; import org.hibernate.HibernateException; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -51,6 +52,11 @@ public class OntologyEndpointUnitTest extends AbstractUnitTest { JenaSystem.init(); } + @BeforeEach + public void beforeEach() { + genesis(); + } + @Test @WithAnonymousUser public void findAll_anonymous_succeeds() { diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableEndpointUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableEndpointUnitTest.java index 154ebda86c3a32554c2c83a3330846ebdee8f939..69f9a8dadd385f2a679b855b839be24f3b8101ef 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableEndpointUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableEndpointUnitTest.java @@ -167,101 +167,6 @@ public class TableEndpointUnitTest extends AbstractUnitTest { }); } - @Test - @WithMockUser(username = USER_3_USERNAME, authorities = {"create-table"}) - public void create_publicDecimalColumnSizeMissing_fails() { - final TableCreateDto request = TableCreateDto.builder() - .name("Some Table") - .description("Some Description") - .columns(List.of(ColumnCreateDto.builder() - .name("ID") - .type(ColumnTypeDto.DECIMAL) - .build())) - .constraints(null) - .build(); - - /* test */ - assertThrows(MalformedException.class, () -> { - generic_create(DATABASE_3_ID, DATABASE_3, request, USER_1_PRINCIPAL, USER_1, DATABASE_3_USER_1_WRITE_OWN_ACCESS); - }); - } - - @Test - @WithMockUser(username = USER_3_USERNAME, authorities = {"create-table"}) - public void create_publicDateFormatMissing_fails() { - final TableCreateDto request = TableCreateDto.builder() - .name("Some Table") - .description("Some Description") - .columns(List.of(ColumnCreateDto.builder() - .name("timestamp") - .type(ColumnTypeDto.DATE) - .build())) - .constraints(null) - .build(); - - /* test */ - assertThrows(MalformedException.class, () -> { - generic_create(DATABASE_3_ID, DATABASE_3, request, USER_1_PRINCIPAL, USER_1, DATABASE_3_USER_1_WRITE_OWN_ACCESS); - }); - } - - @Test - @WithMockUser(username = USER_3_USERNAME, authorities = {"create-table"}) - public void create_publicDateTimeFormatMissing_fails() { - final TableCreateDto request = TableCreateDto.builder() - .name("Some Table") - .description("Some Description") - .columns(List.of(ColumnCreateDto.builder() - .name("timestamp") - .type(ColumnTypeDto.DATETIME) - .build())) - .constraints(null) - .build(); - - /* test */ - assertThrows(MalformedException.class, () -> { - generic_create(DATABASE_3_ID, DATABASE_3, request, USER_1_PRINCIPAL, USER_1, DATABASE_3_USER_1_WRITE_OWN_ACCESS); - }); - } - - @Test - @WithMockUser(username = USER_3_USERNAME, authorities = {"create-table"}) - public void create_publicTimeFormatMissing_fails() { - final TableCreateDto request = TableCreateDto.builder() - .name("Some Table") - .description("Some Description") - .columns(List.of(ColumnCreateDto.builder() - .name("timestamp") - .type(ColumnTypeDto.TIME) - .build())) - .constraints(null) - .build(); - - /* test */ - assertThrows(MalformedException.class, () -> { - generic_create(DATABASE_3_ID, DATABASE_3, request, USER_1_PRINCIPAL, USER_1, DATABASE_3_USER_1_WRITE_OWN_ACCESS); - }); - } - - @Test - @WithMockUser(username = USER_3_USERNAME, authorities = {"create-table"}) - public void create_publicTimestampFormatMissing_fails() { - final TableCreateDto request = TableCreateDto.builder() - .name("Some Table") - .description("Some Description") - .columns(List.of(ColumnCreateDto.builder() - .name("timestamp") - .type(ColumnTypeDto.TIMESTAMP) - .build())) - .constraints(null) - .build(); - - /* test */ - assertThrows(MalformedException.class, () -> { - generic_create(DATABASE_3_ID, DATABASE_3, request, USER_1_PRINCIPAL, USER_1, DATABASE_3_USER_1_WRITE_OWN_ACCESS); - }); - } - @Test @WithMockUser(username = USER_3_USERNAME, authorities = {"create-table"}) public void create_publicDecimalColumnSizeTooSmall_fails() { diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/UnitEndpointUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/UnitEndpointUnitTest.java index b5fc19681ccb34f41fce526c12a0de7238a1bfa7..7d74e615be3a24dfd16af29c825774361f1a7ffc 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/UnitEndpointUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/UnitEndpointUnitTest.java @@ -4,6 +4,7 @@ import at.tuwien.test.AbstractUnitTest; import at.tuwien.api.database.table.columns.concepts.UnitDto; import at.tuwien.service.UnitService; import lombok.extern.log4j.Log4j2; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -31,6 +32,11 @@ public class UnitEndpointUnitTest extends AbstractUnitTest { @Autowired private UnitEndpoint unitEndpoint; + @BeforeEach + public void beforeEach() { + genesis(); + } + @Test @WithAnonymousUser public void findAllUnits_anonymous_succeeds() { diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/UserEndpointUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/UserEndpointUnitTest.java index ad4cef5bbea6cb262607ebbad44f6ec8148b6f9b..da86dd9344606e59c55a9037111877fc55fcb1c4 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/UserEndpointUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/UserEndpointUnitTest.java @@ -8,6 +8,7 @@ import at.tuwien.exception.*; import at.tuwien.service.AuthenticationService; import at.tuwien.service.UserService; import lombok.extern.log4j.Log4j2; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -43,6 +44,11 @@ public class UserEndpointUnitTest extends AbstractUnitTest { @Autowired private UserEndpoint userEndpoint; + @BeforeEach + public void beforeEach() { + genesis(); + } + @Test @WithAnonymousUser public void findAll_anonymous_succeeds() throws UserNotFoundException { diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/ViewEndpointUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/ViewEndpointUnitTest.java index 74e418f42b1ca4fc4cbceec451f645ec05c1223d..b4291ce902652eeeea7aa7e12f6aac26d444bbe1 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/ViewEndpointUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/ViewEndpointUnitTest.java @@ -14,6 +14,7 @@ import at.tuwien.service.DatabaseService; import at.tuwien.service.UserService; import at.tuwien.service.ViewService; import lombok.extern.log4j.Log4j2; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -52,6 +53,11 @@ public class ViewEndpointUnitTest extends AbstractUnitTest { @Autowired private ViewEndpoint viewEndpoint; + @BeforeEach + public void beforeEach() { + genesis(); + } + @Test @WithAnonymousUser public void findAll_publicAnonymous_succeeds() throws ViewNotFoundException, UserNotFoundException, diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/gateway/BrokerServiceGatewayUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/gateway/BrokerServiceGatewayUnitTest.java index a812bd5de49af37c5eca041a2fb36653d243dafc..fc2a9c2c220c4fa601138ee9ee8333d3cd67f493 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/gateway/BrokerServiceGatewayUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/gateway/BrokerServiceGatewayUnitTest.java @@ -1,5 +1,6 @@ package at.tuwien.gateway; +import at.tuwien.api.amqp.GrantExchangePermissionsDto; import at.tuwien.test.AbstractUnitTest; import at.tuwien.exception.*; import lombok.extern.log4j.Log4j2; @@ -33,6 +34,12 @@ public class BrokerServiceGatewayUnitTest extends AbstractUnitTest { @Autowired private BrokerServiceGateway brokerServiceGateway; + private final GrantExchangePermissionsDto WRITE_ALL_PERMISSIONS = GrantExchangePermissionsDto.builder() + .exchange("dbrepo") + .read("^(dbrepo\\.1\\..*)$") /* WRITE_ALL */ + .write("^(dbrepo\\.1\\..*)$") + .build(); + @Test public void grantTopicPermission_exchangeNoRightsBefore_succeeds() throws BrokerServiceException, BrokerServiceConnectionException { @@ -178,43 +185,40 @@ public class BrokerServiceGatewayUnitTest extends AbstractUnitTest { @Test public void grantExchangePermission_succeeds() throws BrokerServiceException, BrokerServiceConnectionException { - final ResponseEntity<Void> mock = ResponseEntity.status(HttpStatus.CREATED) - .build(); /* mock */ when(restTemplate.exchange(anyString(), eq(HttpMethod.PUT), any(HttpEntity.class), eq(Void.class))) - .thenReturn(mock); + .thenReturn(ResponseEntity.status(HttpStatus.CREATED) + .build()); /* test */ - brokerServiceGateway.grantExchangePermission(USER_1_USERNAME, USER_1_RABBITMQ_GRANT_TOPIC_DTO); + brokerServiceGateway.grantExchangePermission(USER_1_USERNAME, WRITE_ALL_PERMISSIONS); } @Test public void grantExchangePermission_exists_succeeds() throws BrokerServiceException, BrokerServiceConnectionException { - final ResponseEntity<Void> mock = ResponseEntity.status(HttpStatus.NO_CONTENT) - .build(); /* mock */ when(restTemplate.exchange(anyString(), eq(HttpMethod.PUT), any(HttpEntity.class), eq(Void.class))) - .thenReturn(mock); + .thenReturn(ResponseEntity.status(HttpStatus.NO_CONTENT) + .build()); /* test */ - brokerServiceGateway.grantExchangePermission(USER_1_USERNAME, USER_1_RABBITMQ_GRANT_TOPIC_DTO); + brokerServiceGateway.grantExchangePermission(USER_1_USERNAME, WRITE_ALL_PERMISSIONS); } @Test public void grantExchangePermission_unexpected2_fails() { - final ResponseEntity<Void> mock = ResponseEntity.status(HttpStatus.BAD_GATEWAY) - .build(); /* mock */ when(restTemplate.exchange(anyString(), eq(HttpMethod.PUT), any(HttpEntity.class), eq(Void.class))) - .thenReturn(mock); + .thenReturn(ResponseEntity.status(HttpStatus.BAD_GATEWAY) + .build()); /* test */ assertThrows(BrokerServiceException.class, () -> { - brokerServiceGateway.grantExchangePermission(USER_1_USERNAME, USER_1_RABBITMQ_GRANT_TOPIC_DTO); + brokerServiceGateway.grantExchangePermission(USER_1_USERNAME, WRITE_ALL_PERMISSIONS); }); } @@ -228,7 +232,7 @@ public class BrokerServiceGatewayUnitTest extends AbstractUnitTest { /* test */ assertThrows(BrokerServiceConnectionException.class, () -> { - brokerServiceGateway.grantExchangePermission(USER_1_USERNAME, USER_1_RABBITMQ_GRANT_TOPIC_DTO); + brokerServiceGateway.grantExchangePermission(USER_1_USERNAME, WRITE_ALL_PERMISSIONS); }); } @@ -242,7 +246,7 @@ public class BrokerServiceGatewayUnitTest extends AbstractUnitTest { /* test */ assertThrows(BrokerServiceException.class, () -> { - brokerServiceGateway.grantExchangePermission(USER_1_USERNAME, USER_1_RABBITMQ_GRANT_TOPIC_DTO); + brokerServiceGateway.grantExchangePermission(USER_1_USERNAME, WRITE_ALL_PERMISSIONS); }); } diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/AuthenticationServiceIntegrationTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/AuthenticationServiceIntegrationTest.java index 4125529155b135dae929fa7192db073e20dc9f55..fa1cd5d4beeb1fe8fb40d8f59fa974b5d2501dba 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/AuthenticationServiceIntegrationTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/AuthenticationServiceIntegrationTest.java @@ -6,6 +6,7 @@ import at.tuwien.exception.*; import at.tuwien.gateway.KeycloakGateway; import dasniko.testcontainers.keycloak.KeycloakContainer; import lombok.extern.log4j.Log4j2; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -31,6 +32,11 @@ public class AuthenticationServiceIntegrationTest extends AbstractUnitTest { @Autowired private KeycloakGateway keycloakGateway; + @BeforeEach + public void beforeEach() { + genesis(); + } + @Container private static KeycloakContainer keycloakContainer = new KeycloakContainer("quay.io/keycloak/keycloak:24.0") .withImagePullPolicy(PullPolicy.alwaysPull()) diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/BrokerServiceIntegrationTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/BrokerServiceIntegrationTest.java index c9a2ad62b61d324934236c885bf6be2aca586e9e..d04409c87b095cfcdc41a478df87a733cdf1bb82 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/BrokerServiceIntegrationTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/BrokerServiceIntegrationTest.java @@ -1,13 +1,15 @@ package at.tuwien.service; -import at.tuwien.config.RabbitConfig; -import at.tuwien.exception.*; -import at.tuwien.test.AbstractUnitTest; import at.tuwien.api.amqp.GrantExchangePermissionsDto; +import at.tuwien.api.amqp.GrantVirtualHostPermissionsDto; import at.tuwien.api.amqp.TopicPermissionDto; import at.tuwien.api.amqp.VirtualHostPermissionDto; +import at.tuwien.config.RabbitConfig; import at.tuwien.entities.database.DatabaseAccess; import at.tuwien.entities.user.User; +import at.tuwien.exception.BrokerServiceConnectionException; +import at.tuwien.exception.BrokerServiceException; +import at.tuwien.test.AbstractUnitTest; import at.tuwien.utils.AmqpUtils; import lombok.extern.log4j.Log4j2; import org.junit.jupiter.api.BeforeEach; @@ -27,7 +29,7 @@ import org.testcontainers.junit.jupiter.Testcontainers; import java.util.List; import java.util.Set; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; @Log4j2 @Testcontainers @@ -167,10 +169,15 @@ public class BrokerServiceIntegrationTest extends AbstractUnitTest { protected VirtualHostPermissionDto setVirtualHostPermissions_generic() throws BrokerServiceException, BrokerServiceConnectionException { + final GrantVirtualHostPermissionsDto permissions = GrantVirtualHostPermissionsDto.builder() + .configure("") + .read("") + .write("") + .build(); final AmqpUtils amqpUtils = new AmqpUtils(rabbitContainer.getHttpUrl()); /* mock */ - amqpUtils.setVirtualHostPermissions(REALM_DBREPO_NAME, USER_1_USERNAME, USER_1_RABBITMQ_GRANT_DTO); + amqpUtils.setVirtualHostPermissions(REALM_DBREPO_NAME, USER_1_USERNAME, permissions); /* test */ brokerService.setVirtualHostPermissions(USER_1); diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/DatabaseServicePersistenceTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/DatabaseServicePersistenceTest.java index b4bd67f88d225a5cfc5398fac6c485b48c674ac2..51b6df4d27b7af3332e5cb592c627e1e2ae03995 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/DatabaseServicePersistenceTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/DatabaseServicePersistenceTest.java @@ -82,8 +82,6 @@ public class DatabaseServicePersistenceTest extends AbstractUnitTest { assertEquals(IMAGE_1_DRIVER, response.getContainer().getImage().getDriverClass()); assertEquals(IMAGE_1_REGISTRY, response.getContainer().getImage().getRegistry()); assertEquals(IMAGE_1_PORT, response.getContainer().getImage().getDefaultPort()); - assertNotNull(response.getContainer().getImage().getDateFormats()); - assertEquals(4, response.getContainer().getImage().getDateFormats().size()); /* creator */ assertNotNull(response.getCreator()); assertEquals(USER_1_ID, response.getCreator().getId()); @@ -124,8 +122,6 @@ public class DatabaseServicePersistenceTest extends AbstractUnitTest { assertEquals(IMAGE_1_DRIVER, response.getContainer().getImage().getDriverClass()); assertEquals(IMAGE_1_REGISTRY, response.getContainer().getImage().getRegistry()); assertEquals(IMAGE_1_PORT, response.getContainer().getImage().getDefaultPort()); - assertNotNull(response.getContainer().getImage().getDateFormats()); - assertEquals(4, response.getContainer().getImage().getDateFormats().size()); /* creator */ assertNotNull(response.getCreator()); assertEquals(USER_1_ID, response.getCreator().getId()); diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceUnitTest.java index 68d6e1a93d5fe594d207df5ae0edc9da1b5c80b3..1e7633b851f9c052fd8fe60d14193698565421f3 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceUnitTest.java @@ -1,6 +1,5 @@ package at.tuwien.service; -import at.tuwien.test.AbstractUnitTest; import at.tuwien.api.database.DatabaseModifyVisibilityDto; import at.tuwien.api.database.internal.CreateDatabaseDto; import at.tuwien.entities.database.Database; @@ -8,8 +7,8 @@ import at.tuwien.entities.user.User; import at.tuwien.exception.*; import at.tuwien.gateway.DataServiceGateway; import at.tuwien.gateway.SearchServiceGateway; -import at.tuwien.repository.ContainerRepository; import at.tuwien.repository.DatabaseRepository; +import at.tuwien.test.AbstractUnitTest; import lombok.extern.log4j.Log4j2; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -23,10 +22,10 @@ import java.util.List; import java.util.Optional; import static org.junit.jupiter.api.Assertions.*; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.when; @Log4j2 @SpringBootTest @@ -212,7 +211,11 @@ public class DatabaseServiceUnitTest extends AbstractUnitTest { SearchServiceConnectionException { /* test */ - generic_modifyOwner(DATABASE_1, USER_1); + final Database response = generic_modifyOwner(DATABASE_1, USER_2); + assertEquals(USER_2, response.getOwner()); + assertEquals(USER_2_ID, response.getOwnedBy()); + assertEquals(USER_2, response.getContact()); + assertEquals(USER_2_ID, response.getContactPerson()); } @Test diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/ImageServiceUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/ImageServiceUnitTest.java index 524c5715b4fd59e24c3f972a3acf2bd08453d99b..725d956f570e7ed6678047b4bb9d7baf85cbe33b 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/ImageServiceUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/ImageServiceUnitTest.java @@ -10,6 +10,7 @@ import at.tuwien.exception.ImageAlreadyExistsException; import at.tuwien.repository.ImageRepository; import at.tuwien.service.impl.ImageServiceImpl; import jakarta.validation.ConstraintViolationException; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -34,6 +35,11 @@ public class ImageServiceUnitTest extends AbstractUnitTest { @Autowired private ImageService imageService; + @BeforeEach + public void beforeEach() { + genesis(); + } + @Test public void getAll_succeeds() { diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/MetadataServiceUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/MetadataServiceUnitTest.java index 24ed0f686e8c9885fb4e4b637b6e02a5fc664a48..fa10d09a877c5a967086f855efd091eee1f5efe2 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/MetadataServiceUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/MetadataServiceUnitTest.java @@ -16,6 +16,7 @@ import at.tuwien.gateway.RorGateway; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.log4j.Log4j2; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -59,6 +60,11 @@ public class MetadataServiceUnitTest extends AbstractUnitTest { @Autowired private ObjectMapper objectMapper; + @BeforeEach + public void beforeEach() { + genesis(); + } + @Test public void identify_succeeds() { diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServicePersistenceTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServicePersistenceTest.java index e2d7d33896f6c1b13d0ce9ab5af3262c8668f0bc..7aa22159c395046548324a714d7d8274fc8114af 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServicePersistenceTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServicePersistenceTest.java @@ -92,7 +92,6 @@ public class TableServicePersistenceTest extends AbstractUnitTest { .name("date") .nullAllowed(true) .type(ColumnTypeDto.DATE) - .dfid(3L) .build())) .constraints(ConstraintsCreateDto.builder() .checks(Set.of()) @@ -124,8 +123,6 @@ public class TableServicePersistenceTest extends AbstractUnitTest { assertEquals("date", date.getName()); assertEquals("date", date.getInternalName()); assertEquals(TableColumnType.DATE, date.getColumnType()); - assertNotNull(date.getDateFormat()); - assertEquals(3L, date.getDateFormat().getId()); assertTrue(date.getIsNullAllowed()); assertNotNull(response.getConstraints()); final List<Unique> uniques = response.getConstraints().getUniques(); diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceUnitTest.java index 59419f9bad37d26687342c51da6155463d674e87..94636bb3c5122539d6bba533a80940203b80df5d 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceUnitTest.java @@ -5,16 +5,16 @@ import at.tuwien.api.database.table.columns.ColumnCreateDto; import at.tuwien.api.database.table.columns.ColumnTypeDto; import at.tuwien.api.database.table.constraints.ConstraintsCreateDto; import at.tuwien.api.database.table.constraints.foreign.ForeignKeyCreateDto; -import at.tuwien.entities.database.table.columns.TableColumnType; -import at.tuwien.entities.database.table.constraints.Constraints; -import at.tuwien.test.AbstractUnitTest; import at.tuwien.entities.database.Database; import at.tuwien.entities.database.table.Table; import at.tuwien.entities.database.table.columns.TableColumn; +import at.tuwien.entities.database.table.columns.TableColumnType; +import at.tuwien.entities.database.table.constraints.Constraints; import at.tuwien.exception.*; import at.tuwien.gateway.DataServiceGateway; import at.tuwien.gateway.SearchServiceGateway; import at.tuwien.repository.DatabaseRepository; +import at.tuwien.test.AbstractUnitTest; import lombok.extern.log4j.Log4j2; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -49,9 +49,6 @@ public class TableServiceUnitTest extends AbstractUnitTest { @MockBean private DataServiceGateway dataServiceGateway; - @MockBean - private OntologyService ontologyService; - @Autowired private TableService tableService; @@ -202,7 +199,6 @@ public class TableServiceUnitTest extends AbstractUnitTest { .name("date") .nullAllowed(true) .type(ColumnTypeDto.DATE) - .dfid(9999L) .build())) .constraints(ConstraintsCreateDto.builder() .checks(Set.of()) diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/UserServiceUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/UserServiceUnitTest.java index 5becb9225a42db3ab451dc054663e811e7c71629..a9fe4694cc69d8eb347e9fefd12498704b8797fd 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/UserServiceUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/UserServiceUnitTest.java @@ -5,6 +5,7 @@ import at.tuwien.entities.user.User; import at.tuwien.exception.*; import at.tuwien.gateway.KeycloakGateway; import at.tuwien.repository.UserRepository; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -32,6 +33,11 @@ public class UserServiceUnitTest extends AbstractUnitTest { @Autowired private UserService userService; + @BeforeEach + public void beforeEach() { + genesis(); + } + @Test public void findByUsername_succeeds() throws UserNotFoundException { diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/validator/EndpointValidatorUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/validator/EndpointValidatorUnitTest.java index f6cf4e887d4d25f2dfc201ae6d4d014f3b8c1547..8528d29f07d267bd4dcbe94de5379e47e13f1cef 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/validator/EndpointValidatorUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/validator/EndpointValidatorUnitTest.java @@ -53,30 +53,9 @@ public class EndpointValidatorUnitTest extends AbstractUnitTest { public static Stream<Arguments> needSize_parameters() { return Stream.of( - Arguments.arguments(ColumnTypeDto.CHAR), Arguments.arguments(ColumnTypeDto.VARCHAR), Arguments.arguments(ColumnTypeDto.BINARY), - Arguments.arguments(ColumnTypeDto.VARBINARY), - Arguments.arguments(ColumnTypeDto.BIT), - Arguments.arguments(ColumnTypeDto.TINYINT), - Arguments.arguments(ColumnTypeDto.SMALLINT), - Arguments.arguments(ColumnTypeDto.MEDIUMINT), - Arguments.arguments(ColumnTypeDto.INT) - ); - } - - public static Stream<Arguments> needSizeAndD_parameters() { - return Stream.of( - Arguments.arguments(ColumnTypeDto.DOUBLE), - Arguments.arguments(ColumnTypeDto.DECIMAL) - ); - } - - public static Stream<Arguments> needDateFormat_parameters() { - return Stream.of( - Arguments.arguments(ColumnTypeDto.DATETIME), - Arguments.arguments(ColumnTypeDto.TIMESTAMP), - Arguments.arguments(ColumnTypeDto.TIME) + Arguments.arguments(ColumnTypeDto.VARBINARY) ); } @@ -308,23 +287,6 @@ public class EndpointValidatorUnitTest extends AbstractUnitTest { }); } - @ParameterizedTest - @MethodSource("needSizeAndD_parameters") - public void validateColumnCreateConstraints_needSizeAndD_fails(ColumnTypeDto type) { - final TableCreateDto request = TableCreateDto.builder() - .columns(List.of(ColumnCreateDto.builder() - .type(type) - .size(10L) - .d(null) // <<<<<<< - .build())) - .build(); - - /* test */ - assertThrows(MalformedException.class, () -> { - endpointValidator.validateColumnCreateConstraints(request); - }); - } - @Test public void validateColumnCreateConstraints_needEnum_fails() { final TableCreateDto request = TableCreateDto.builder() @@ -355,35 +317,6 @@ public class EndpointValidatorUnitTest extends AbstractUnitTest { }); } - @ParameterizedTest - @MethodSource("needDateFormat_parameters") - public void validateColumnCreateConstraints_needDateFormat_fails(ColumnTypeDto type) { - final TableCreateDto request = TableCreateDto.builder() - .columns(List.of(ColumnCreateDto.builder() - .type(type) - .dfid(null) // <<<<<<< - .build())) - .build(); - - /* test */ - assertThrows(MalformedException.class, () -> { - endpointValidator.validateColumnCreateConstraints(request); - }); - } - - @Test - public void validateColumnCreateConstraints_dateFormatEmpty_succeeds() throws MalformedException { - final TableCreateDto request = TableCreateDto.builder() - .columns(List.of(ColumnCreateDto.builder() - .type(ColumnTypeDto.DATE) - .dfid(null) // <<<<<<< - .build())) - .build(); - - /* test */ - endpointValidator.validateColumnCreateConstraints(request); - } - @Test public void validateOnlyOwnerOrWriteAll_onlyReadAccess_fails() throws DatabaseNotFoundException, TableNotFoundException, AccessNotFoundException { diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/config/MetricsConfig.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/config/MetricsConfig.java index e366666244515dca7965663ef5ce5016a734dd4a..1b7578b4bcdb547eae2cc26690ef028c38367787 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/config/MetricsConfig.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/config/MetricsConfig.java @@ -1,12 +1,10 @@ package at.tuwien.config; -import at.tuwien.entities.database.Database; import at.tuwien.entities.database.table.Table; import at.tuwien.repository.DatabaseRepository; import at.tuwien.repository.IdentifierRepository; import at.tuwien.repository.TableRepository; import at.tuwien.repository.ViewRepository; -import at.tuwien.service.DatabaseService; import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.Metrics; import io.micrometer.observation.ObservationRegistry; @@ -16,8 +14,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import java.util.List; - @Log4j2 @Configuration public class MetricsConfig { @@ -70,7 +66,12 @@ public class MetricsConfig { @Bean public Gauge volumeSumGauge() { - return Gauge.builder("dbrepo.volume.sum", () -> tableRepository.findAll().stream().map(Table::getDataLength).mapToLong(d -> d).sum()) + return Gauge.builder("dbrepo.volume.sum", () -> { + if (tableRepository.findAll().isEmpty()) { + return 0; + } + return tableRepository.findAll().stream().map(Table::getDataLength).mapToLong(d -> d).sum(); + }) .description("The total volume of available research data") .strongReference(true) .register(Metrics.globalRegistry); diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/AccessServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/AccessServiceImpl.java index aaa50251c3dc0dd79e51af1c983bfda07affffaa..b6af9018118f0e502feb1452b8a4838e9baadb34 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/AccessServiceImpl.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/AccessServiceImpl.java @@ -73,6 +73,7 @@ public class AccessServiceImpl implements AccessService { .database(database) .huserid(user.getId()) .type(metadataMapper.accessTypeDtoToAccessType(type)) + .user(user) .build(); database.getAccesses() .add(access); @@ -95,8 +96,8 @@ public class AccessServiceImpl implements AccessService { .hdbid(database.getId()) .database(database) .huserid(user.getId()) - .user(user) .type(metadataMapper.accessTypeDtoToAccessType(access)) + .user(user) .build(); final int idx = database.getAccesses().indexOf(entity); if (idx == -1) { diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceImpl.java index 0a962f7d6edfe899365534a084a1faa265c6d864..330d4518dedba7e1e5e78f25a7433c18762fd579 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceImpl.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceImpl.java @@ -167,7 +167,10 @@ public class DatabaseServiceImpl implements DatabaseService { public Database modifyOwner(Database database, User user) throws DatabaseNotFoundException, SearchServiceException, SearchServiceConnectionException { /* update in metadata database */ + database.setOwner(user); database.setOwnedBy(user.getId()); + database.setContact(user); + database.setContactPerson(user.getId()); database = databaseRepository.save(database); /* save in search service */ searchServiceGateway.update(database); @@ -281,7 +284,7 @@ public class DatabaseServiceImpl implements DatabaseService { /* correct the unique constraint columns */ for (Table table : database.getTables()) { for (Unique uniqueConstraint : table.getConstraints().getUniques()) { - uniqueConstraint.setColumns(uniqueConstraint.getColumns() + uniqueConstraint.setColumns(new LinkedList<>(uniqueConstraint.getColumns() .stream() .map(column -> { final Optional<TableColumn> optional = table.getColumns() @@ -294,7 +297,7 @@ public class DatabaseServiceImpl implements DatabaseService { } return optional.get(); }) - .toList()); + .toList())); } } /* update in metadata database */ diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java index f88ba5c28f4d928757a501d8ed1a3f4effb018d8..0013f0156301e03872939957db011de70e98ef82 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java @@ -4,15 +4,12 @@ import at.tuwien.api.database.table.TableCreateDto; import at.tuwien.api.database.table.TableStatisticDto; import at.tuwien.api.database.table.columns.ColumnCreateDto; import at.tuwien.api.database.table.columns.ColumnStatisticDto; -import at.tuwien.api.database.table.columns.ColumnTypeDto; import at.tuwien.api.database.table.columns.concepts.ColumnSemanticsUpdateDto; import at.tuwien.config.RabbitConfig; -import at.tuwien.entities.container.image.ContainerImageDate; import at.tuwien.entities.database.Database; import at.tuwien.entities.database.table.Table; import at.tuwien.entities.database.table.columns.TableColumn; import at.tuwien.entities.database.table.columns.TableColumnConcept; -import at.tuwien.entities.database.table.columns.TableColumnType; import at.tuwien.entities.database.table.columns.TableColumnUnit; import at.tuwien.entities.user.User; import at.tuwien.exception.*; @@ -27,7 +24,10 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import java.security.Principal; -import java.util.*; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; @Log4j2 @Service @@ -146,20 +146,6 @@ public class TableServiceImpl implements TableService { } column.setConcept(concept); } - if (List.of(TableColumnType.TIME, TableColumnType.TIMESTAMP, TableColumnType.DATE, TableColumnType.DATETIME).contains(column.getColumnType())) { - final Optional<ContainerImageDate> optional = database.getContainer() - .getImage() - .getDateFormats() - .stream() - .filter(df -> df.getId().equals(c.getDfid())) - .findFirst(); - if (optional.isEmpty()) { - log.error("Failed to find date format with id {} in metadata database", c.getDfid()); - throw new IllegalArgumentException("Failed to find date format in metadata database"); - } - column.setDateFormat(optional.get()); - log.debug("column is of temporal type: added date format with id {}", column.getDateFormat().getId()); - } table.getColumns() .add(column); } @@ -172,6 +158,8 @@ public class TableServiceImpl implements TableService { for (int i = 0; i < data.getConstraints().getUniques().size(); i++) { if (data.getConstraints().getUniques().get(i).size() != table.getConstraints().getUniques().get(i).getColumns().size()) { log.error("Failed to create table: some unique constraint(s) reference non-existing table columns: {}", data.getConstraints().getUniques().get(i)); + log.debug("payload uniques: {}", data.getConstraints().getUniques()); + log.debug("mapped table uniques: {}", table.getConstraints().getUniques().stream().map(u -> List.of(u.getColumns().stream().map(TableColumn::getInternalName).toList())).toList()); throw new MalformedException("Failed to create table: some unique constraint(s) reference non-existing table columns"); } } diff --git a/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/AbstractUnitTest.java b/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/AbstractUnitTest.java index 996dbb9a7d71b00c7988d9da4dc1ddeb3876245e..924db22930ac5bbffc94d5cad5c768cb45406be3 100644 --- a/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/AbstractUnitTest.java +++ b/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/AbstractUnitTest.java @@ -22,6 +22,7 @@ public abstract class AbstractUnitTest extends BaseTest { /* USER_4 */ USER_5.setAccesses(new LinkedList<>()); /* DATABASE 1 */ + DATABASE_1.setOwner(USER_1); DATABASE_1.setSubsets(new LinkedList<>()); DATABASE_1.setAccesses(new LinkedList<>(List.of(DATABASE_1_USER_1_READ_ACCESS, DATABASE_1_USER_2_WRITE_OWN_ACCESS, DATABASE_1_USER_3_WRITE_ALL_ACCESS))); DATABASE_1_PRIVILEGED_DTO.setAccesses(new LinkedList<>(List.of(DATABASE_1_USER_1_READ_ACCESS_DTO, DATABASE_1_USER_2_WRITE_OWN_ACCESS_DTO, DATABASE_1_USER_3_WRITE_ALL_ACCESS_DTO))); diff --git a/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java b/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java index 6ade19c1ec0f6c742a97731d3a97482fea97923d..f372879434524410c299763e72041dcbb3dd1442 100644 --- a/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java +++ b/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java @@ -1,12 +1,18 @@ package at.tuwien.test; import at.tuwien.ExportResourceDto; -import at.tuwien.api.amqp.*; +import at.tuwien.api.amqp.CreateVirtualHostDto; +import at.tuwien.api.amqp.ExchangeDto; +import at.tuwien.api.amqp.GrantVirtualHostPermissionsDto; +import at.tuwien.api.amqp.QueueDto; import at.tuwien.api.auth.LoginRequestDto; import at.tuwien.api.auth.SignupRequestDto; import at.tuwien.api.container.ContainerBriefDto; import at.tuwien.api.container.ContainerDto; -import at.tuwien.api.container.image.*; +import at.tuwien.api.container.image.ImageBriefDto; +import at.tuwien.api.container.image.ImageChangeDto; +import at.tuwien.api.container.image.ImageCreateDto; +import at.tuwien.api.container.image.ImageDto; import at.tuwien.api.container.internal.PrivilegedContainerDto; import at.tuwien.api.database.*; import at.tuwien.api.database.internal.CreateDatabaseDto; @@ -20,7 +26,10 @@ import at.tuwien.api.database.table.TableCreateDto; import at.tuwien.api.database.table.TableDto; import at.tuwien.api.database.table.TableStatisticDto; import at.tuwien.api.database.table.columns.*; -import at.tuwien.api.database.table.columns.concepts.*; +import at.tuwien.api.database.table.columns.concepts.ConceptDto; +import at.tuwien.api.database.table.columns.concepts.ConceptSaveDto; +import at.tuwien.api.database.table.columns.concepts.UnitDto; +import at.tuwien.api.database.table.columns.concepts.UnitSaveDto; import at.tuwien.api.database.table.constraints.ConstraintsCreateDto; import at.tuwien.api.database.table.constraints.ConstraintsDto; import at.tuwien.api.database.table.constraints.foreign.*; @@ -51,11 +60,9 @@ import at.tuwien.api.orcid.person.name.OrcidValueDto; import at.tuwien.api.semantics.OntologyCreateDto; import at.tuwien.api.semantics.OntologyModifyDto; import at.tuwien.api.user.*; -import at.tuwien.api.user.UserDetailsDto; import at.tuwien.api.user.internal.UpdateUserPasswordDto; import at.tuwien.entities.container.Container; import at.tuwien.entities.container.image.ContainerImage; -import at.tuwien.entities.container.image.ContainerImageDate; import at.tuwien.entities.database.*; import at.tuwien.entities.database.table.Table; import at.tuwien.entities.database.table.columns.TableColumn; @@ -422,17 +429,6 @@ public abstract class BaseTest { public final static Instant USER_1_LAST_MODIFIED = USER_1_CREATED; public final static UUID USER_1_REALM_ID = REALM_DBREPO_ID; - public final static CreateUserDto USER_1_RABBITMQ_CREATE_DTO = CreateUserDto.builder() - .password("") - .tags("") - .build(); - - public final static GrantVirtualHostPermissionsDto USER_1_RABBITMQ_GRANT_DTO = GrantVirtualHostPermissionsDto.builder() - .configure("") - .read("") - .write("") - .build(); - public final static UpdateUserPasswordDto USER_1_UPDATE_PASSWORD_DTO = UpdateUserPasswordDto.builder() .username(USER_1_USERNAME) .password(USER_1_PASSWORD) @@ -908,29 +904,6 @@ public abstract class BaseTest { public final static Integer IMAGE_1_PORT = 3306; public final static Boolean IMAGE_1_IS_DEFAULT = true; - public final static Long IMAGE_DATE_1_ID = 1L; - public final static Long IMAGE_DATE_1_IMAGE_ID = IMAGE_1_ID; - public final static String IMAGE_DATE_1_UNIX_FORMAT = "yyyy-MM-dd"; - public final static String IMAGE_DATE_1_DATABASE_FORMAT = "%Y-%c-%d"; - public final static String IMAGE_DATE_1_EXAMPLE = "2022-01-30"; - public final static Boolean IMAGE_DATE_1_HAS_TIME = false; - - public final static ContainerImageDate IMAGE_DATE_1 = ContainerImageDate.builder() - .id(IMAGE_DATE_1_ID) - .iid(IMAGE_DATE_1_IMAGE_ID) - .unixFormat(IMAGE_DATE_1_UNIX_FORMAT) - .databaseFormat(IMAGE_DATE_1_DATABASE_FORMAT) - .example(IMAGE_DATE_1_EXAMPLE) - .hasTime(IMAGE_DATE_1_HAS_TIME) - .build(); - - public final static ImageDateDto IMAGE_DATE_1_DTO = ImageDateDto.builder() - .id(IMAGE_DATE_1_ID) - .unixFormat(IMAGE_DATE_1_UNIX_FORMAT) - .databaseFormat(IMAGE_DATE_1_DATABASE_FORMAT) - .hasTime(IMAGE_DATE_1_HAS_TIME) - .build(); - public final static ImageCreateDto IMAGE_1_CREATE_DTO = ImageCreateDto.builder() .registry(IMAGE_1_REGISTRY) .name(IMAGE_1_NAME) @@ -949,75 +922,6 @@ public abstract class BaseTest { .defaultPort(IMAGE_1_PORT) .build(); - public final static Long IMAGE_DATE_2_ID = 2L; - public final static Long IMAGE_DATE_2_IMAGE_ID = IMAGE_1_ID; - public final static String IMAGE_DATE_2_UNIX_FORMAT = "dd.MM.yy"; - public final static String IMAGE_DATE_2_DATABASE_FORMAT = "%d.%c.%y"; - public final static String IMAGE_DATE_2_EXAMPLE = "30.01.2022"; - public final static Boolean IMAGE_DATE_2_HAS_TIME = false; - - public final static ContainerImageDate IMAGE_DATE_2 = ContainerImageDate.builder() - .id(IMAGE_DATE_2_ID) - .iid(IMAGE_DATE_2_IMAGE_ID) - .unixFormat(IMAGE_DATE_2_UNIX_FORMAT) - .databaseFormat(IMAGE_DATE_2_DATABASE_FORMAT) - .example(IMAGE_DATE_2_EXAMPLE) - .hasTime(IMAGE_DATE_2_HAS_TIME) - .build(); - - public final static ImageDateDto IMAGE_DATE_2_DTO = ImageDateDto.builder() - .id(IMAGE_DATE_2_ID) - .unixFormat(IMAGE_DATE_2_UNIX_FORMAT) - .databaseFormat(IMAGE_DATE_2_DATABASE_FORMAT) - .hasTime(IMAGE_DATE_2_HAS_TIME) - .build(); - - public final static Long IMAGE_DATE_3_ID = 3L; - public final static Long IMAGE_DATE_3_IMAGE_ID = IMAGE_1_ID; - public final static String IMAGE_DATE_3_UNIX_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSSSS"; - public final static String IMAGE_DATE_3_DATABASE_FORMAT = "%Y-%c-%dT%H:%i:%S.%f"; - public final static String IMAGE_DATE_3_EXAMPLE = "2022-01-30T13:44:25.499"; - public final static Boolean IMAGE_DATE_3_HAS_TIME = true; - - public final static ContainerImageDate IMAGE_DATE_3 = ContainerImageDate.builder() - .id(IMAGE_DATE_3_ID) - .iid(IMAGE_DATE_3_IMAGE_ID) - .unixFormat(IMAGE_DATE_3_UNIX_FORMAT) - .databaseFormat(IMAGE_DATE_3_DATABASE_FORMAT) - .example(IMAGE_DATE_3_EXAMPLE) - .hasTime(IMAGE_DATE_3_HAS_TIME) - .build(); - - public final static ImageDateDto IMAGE_DATE_3_DTO = ImageDateDto.builder() - .id(IMAGE_DATE_3_ID) - .unixFormat(IMAGE_DATE_3_UNIX_FORMAT) - .databaseFormat(IMAGE_DATE_3_DATABASE_FORMAT) - .hasTime(IMAGE_DATE_3_HAS_TIME) - .build(); - - public final static Long IMAGE_DATE_4_ID = 4L; - public final static Long IMAGE_DATE_4_IMAGE_ID = IMAGE_1_ID; - public final static String IMAGE_DATE_4_UNIX_FORMAT = "HH:mm:ss"; - public final static String IMAGE_DATE_4_DATABASE_FORMAT = "%H:%i:%S"; - public final static String IMAGE_DATE_4_EXAMPLE = "14:44:25"; - public final static Boolean IMAGE_DATE_4_HAS_TIME = true; - - public final static ContainerImageDate IMAGE_DATE_4 = ContainerImageDate.builder() - .id(IMAGE_DATE_4_ID) - .iid(IMAGE_DATE_4_IMAGE_ID) - .unixFormat(IMAGE_DATE_4_UNIX_FORMAT) - .databaseFormat(IMAGE_DATE_4_DATABASE_FORMAT) - .example(IMAGE_DATE_4_EXAMPLE) - .hasTime(IMAGE_DATE_4_HAS_TIME) - .build(); - - public final static ImageDateDto IMAGE_DATE_4_DTO = ImageDateDto.builder() - .id(IMAGE_DATE_4_ID) - .unixFormat(IMAGE_DATE_4_UNIX_FORMAT) - .databaseFormat(IMAGE_DATE_4_DATABASE_FORMAT) - .hasTime(IMAGE_DATE_4_HAS_TIME) - .build(); - public final static ContainerImage IMAGE_1 = ContainerImage.builder() .id(IMAGE_1_ID) .name(IMAGE_1_NAME) @@ -1028,7 +932,6 @@ public abstract class BaseTest { .driverClass(IMAGE_1_DRIVER) .defaultPort(IMAGE_1_PORT) .isDefault(IMAGE_1_IS_DEFAULT) - .dateFormats(new LinkedList<>(List.of(IMAGE_DATE_1, IMAGE_DATE_2, IMAGE_DATE_3, IMAGE_DATE_4))) .build(); public final static ImageDto IMAGE_1_DTO = ImageDto.builder() @@ -1041,7 +944,6 @@ public abstract class BaseTest { .driverClass(IMAGE_1_DRIVER) .defaultPort(IMAGE_1_PORT) .isDefault(IMAGE_1_IS_DEFAULT) - .dateFormats(List.of(IMAGE_DATE_1_DTO, IMAGE_DATE_2_DTO, IMAGE_DATE_3_DTO)) .build(); public final static ImageBriefDto IMAGE_1_BRIEF_DTO = ImageBriefDto.builder() @@ -1253,12 +1155,6 @@ public abstract class BaseTest { public final static UserDto DATABASE_1_CREATOR_DTO = USER_1_DTO; public final static UserDto DATABASE_1_OWNER_DTO = USER_1_DTO; - public final static GrantExchangePermissionsDto USER_1_RABBITMQ_GRANT_TOPIC_DTO = GrantExchangePermissionsDto.builder() - .exchange("dbrepo") - .read("^(dbrepo\\." + DATABASE_1_INTERNALNAME + "\\..)$") - .write("^(dbrepo\\." + DATABASE_1_INTERNALNAME + "\\..)$") - .build(); - public final static DatabaseCreateDto DATABASE_1_CREATE = DatabaseCreateDto.builder() .name(DATABASE_1_NAME) .isPublic(DATABASE_1_PUBLIC) @@ -1498,25 +1394,21 @@ public abstract class BaseTest { .name("col25") .type(ColumnTypeDto.DATE) .nullAllowed(true) - .dfid(IMAGE_DATE_1_ID) .build(), ColumnCreateDto.builder() .name("col26") .type(ColumnTypeDto.DATETIME) .nullAllowed(true) - .dfid(IMAGE_DATE_3_ID) .build(), ColumnCreateDto.builder() .name("col27") .type(ColumnTypeDto.TIMESTAMP) .nullAllowed(true) - .dfid(IMAGE_DATE_3_ID) .build(), ColumnCreateDto.builder() .name("col28") .type(ColumnTypeDto.TIME) .nullAllowed(true) - .dfid(IMAGE_DATE_4_ID) .build(), ColumnCreateDto.builder() .name("col29") @@ -1632,7 +1524,6 @@ public abstract class BaseTest { .internalName("date") .ordinalPosition(1) .columnType(ColumnTypeDto.DATE) - .dateFormat(IMAGE_DATE_1_DTO) .isNullAllowed(true) .autoGenerated(false) .enums(null) @@ -2214,7 +2105,6 @@ public abstract class BaseTest { public final static List<ColumnCreateDto> TABLE_4_COLUMNS_CREATE_DTO = List.of(ColumnCreateDto.builder() .name("Timestamp") .type(ColumnTypeDto.TIMESTAMP) - .dfid(IMAGE_DATE_4_ID) .nullAllowed(false) .build(), ColumnCreateDto.builder() @@ -2253,7 +2143,6 @@ public abstract class BaseTest { .name("Timestamp") .internalName("timestamp") .columnType(ColumnTypeDto.TIMESTAMP) - .dateFormat(IMAGE_DATE_3_DTO) .isNullAllowed(false) .autoGenerated(false) .build(), @@ -2264,7 +2153,6 @@ public abstract class BaseTest { .name("Value") .internalName("value") .columnType(ColumnTypeDto.DECIMAL) - .dateFormat(null) .isNullAllowed(true) .autoGenerated(false) .build()); @@ -2893,7 +2781,6 @@ public abstract class BaseTest { .name("Date") .internalName("date") .columnType(TableColumnType.DATE) - .dateFormat(IMAGE_DATE_1) .isNullAllowed(true) .autoGenerated(false) .build(), @@ -2946,14 +2833,12 @@ public abstract class BaseTest { .name("Date") .type(ColumnTypeDto.DATE) .nullAllowed(true) - .dfid(IMAGE_DATE_1_ID) .build(), ColumnCreateDto.builder() .name("Location") .type(ColumnTypeDto.VARCHAR) .size(255L) .nullAllowed(true) - .dfid(IMAGE_DATE_1_ID) .build(), ColumnCreateDto.builder() .name("MinTemp") @@ -2961,7 +2846,6 @@ public abstract class BaseTest { .size(10L) .d(0L) .nullAllowed(true) - .dfid(IMAGE_DATE_1_ID) .build(), ColumnCreateDto.builder() .name("Rainfall") @@ -2969,7 +2853,6 @@ public abstract class BaseTest { .size(10L) .d(0L) .nullAllowed(true) - .dfid(IMAGE_DATE_1_ID) .conceptUri(CONCEPT_1_URI) .unitUri(UNIT_1_URI) .build()); @@ -3206,7 +3089,6 @@ public abstract class BaseTest { .name("id") .internalName("id") .isNullAllowed(false) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3219,7 +3101,6 @@ public abstract class BaseTest { .name("linie") .internalName("linie") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3232,7 +3113,6 @@ public abstract class BaseTest { .name("richtung") .internalName("richtung") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3257,7 +3137,6 @@ public abstract class BaseTest { .name("fahrzeug") .internalName("fahrzeug") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3270,7 +3149,6 @@ public abstract class BaseTest { .name("kurs") .internalName("kurs") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3283,7 +3161,6 @@ public abstract class BaseTest { .name("seq_von") .internalName("seq_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3296,7 +3173,6 @@ public abstract class BaseTest { .name("halt_diva_von") .internalName("halt_diva_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3309,7 +3185,6 @@ public abstract class BaseTest { .name("halt_punkt_diva_von") .internalName("halt_punkt_diva_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3322,7 +3197,6 @@ public abstract class BaseTest { .name("halt_kurz_von1") .internalName("halt_kurz_von1") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3347,7 +3221,6 @@ public abstract class BaseTest { .name("soll_an_von") .internalName("soll_an_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3360,7 +3233,6 @@ public abstract class BaseTest { .name("ist_an_von") .internalName("ist_an_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3373,7 +3245,6 @@ public abstract class BaseTest { .name("soll_ab_von") .internalName("soll_ab_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3386,7 +3257,6 @@ public abstract class BaseTest { .name("ist_ab_von") .internalName("ist_ab_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3399,7 +3269,6 @@ public abstract class BaseTest { .name("seq_nach") .internalName("seq_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3412,7 +3281,6 @@ public abstract class BaseTest { .name("halt_diva_nach") .internalName("halt_diva_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3425,7 +3293,6 @@ public abstract class BaseTest { .name("halt_punkt_diva_nach") .internalName("halt_punkt_diva_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3438,7 +3305,6 @@ public abstract class BaseTest { .name("halt_kurz_nach1") .internalName("halt_kurz_nach1") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3463,7 +3329,6 @@ public abstract class BaseTest { .name("soll_an_nach") .internalName("soll_an_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3476,7 +3341,6 @@ public abstract class BaseTest { .name("ist_an_nach1") .internalName("ist_an_nach1") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3489,7 +3353,6 @@ public abstract class BaseTest { .name("soll_ab_nach") .internalName("soll_ab_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3502,7 +3365,6 @@ public abstract class BaseTest { .name("ist_ab_nach") .internalName("ist_ab_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3515,7 +3377,6 @@ public abstract class BaseTest { .name("fahrt_id") .internalName("fahrt_id") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3528,7 +3389,6 @@ public abstract class BaseTest { .name("fahrweg_id") .internalName("fahrweg_id") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3541,7 +3401,6 @@ public abstract class BaseTest { .name("fw_no") .internalName("fw_no") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3554,7 +3413,6 @@ public abstract class BaseTest { .name("fw_typ") .internalName("fw_typ") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3567,7 +3425,6 @@ public abstract class BaseTest { .name("fw_kurz") .internalName("fw_kurz") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3580,7 +3437,6 @@ public abstract class BaseTest { .name("fw_lang") .internalName("fw_lang") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3593,7 +3449,6 @@ public abstract class BaseTest { .name("umlauf_von") .internalName("umlauf_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3606,7 +3461,6 @@ public abstract class BaseTest { .name("halt_id_von") .internalName("halt_id_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3619,7 +3473,6 @@ public abstract class BaseTest { .name("halt_id_nach") .internalName("halt_id_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3632,7 +3485,6 @@ public abstract class BaseTest { .name("halt_punkt_id_von") .internalName("halt_punkt_id_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3645,7 +3497,6 @@ public abstract class BaseTest { .name("halt_punkt_id_nach") .internalName("halt_punkt_id_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build()); @@ -3660,7 +3511,6 @@ public abstract class BaseTest { .name("id") .internalName("id") .isNullAllowed(false) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3674,7 +3524,6 @@ public abstract class BaseTest { .name("linie") .internalName("linie") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3688,7 +3537,6 @@ public abstract class BaseTest { .name("richtung") .internalName("richtung") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3702,7 +3550,6 @@ public abstract class BaseTest { .name("betriebsdatum") .internalName("betriebsdatum") .isNullAllowed(true) - .dateFormat(IMAGE_DATE_2_DTO) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3716,7 +3563,6 @@ public abstract class BaseTest { .name("fahrzeug") .internalName("fahrzeug") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3730,7 +3576,6 @@ public abstract class BaseTest { .name("kurs") .internalName("kurs") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3744,7 +3589,6 @@ public abstract class BaseTest { .name("seq_von") .internalName("seq_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3758,7 +3602,6 @@ public abstract class BaseTest { .name("halt_diva_von") .internalName("halt_diva_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3772,7 +3615,6 @@ public abstract class BaseTest { .name("halt_punkt_diva_von") .internalName("halt_punkt_diva_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3786,7 +3628,6 @@ public abstract class BaseTest { .name("halt_kurz_von1") .internalName("halt_kurz_von1") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3800,7 +3641,6 @@ public abstract class BaseTest { .name("datum_von") .internalName("datum_von") .isNullAllowed(true) - .dateFormat(IMAGE_DATE_2_DTO) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3814,7 +3654,6 @@ public abstract class BaseTest { .name("soll_an_von") .internalName("soll_an_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3828,7 +3667,6 @@ public abstract class BaseTest { .name("ist_an_von") .internalName("ist_an_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3842,7 +3680,6 @@ public abstract class BaseTest { .name("soll_ab_von") .internalName("soll_ab_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3856,7 +3693,6 @@ public abstract class BaseTest { .name("ist_ab_von") .internalName("ist_ab_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3870,7 +3706,6 @@ public abstract class BaseTest { .name("seq_nach") .internalName("seq_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3884,7 +3719,6 @@ public abstract class BaseTest { .name("halt_diva_nach") .internalName("halt_diva_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3898,7 +3732,6 @@ public abstract class BaseTest { .name("halt_punkt_diva_nach") .internalName("halt_punkt_diva_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3912,7 +3745,6 @@ public abstract class BaseTest { .name("halt_kurz_nach1") .internalName("halt_kurz_nach1") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3926,7 +3758,6 @@ public abstract class BaseTest { .name("datum_nach") .internalName("datum_nach") .isNullAllowed(true) - .dateFormat(IMAGE_DATE_2_DTO) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3940,7 +3771,6 @@ public abstract class BaseTest { .name("soll_an_nach") .internalName("soll_an_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3954,7 +3784,6 @@ public abstract class BaseTest { .name("ist_an_nach1") .internalName("ist_an_nach1") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3968,7 +3797,6 @@ public abstract class BaseTest { .name("soll_ab_nach") .internalName("soll_ab_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3982,7 +3810,6 @@ public abstract class BaseTest { .name("ist_ab_nach") .internalName("ist_ab_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -3996,7 +3823,6 @@ public abstract class BaseTest { .name("fahrt_id") .internalName("fahrt_id") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -4010,7 +3836,6 @@ public abstract class BaseTest { .name("fahrweg_id") .internalName("fahrweg_id") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -4024,7 +3849,6 @@ public abstract class BaseTest { .name("fw_no") .internalName("fw_no") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -4038,7 +3862,6 @@ public abstract class BaseTest { .name("fw_typ") .internalName("fw_typ") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -4052,7 +3875,6 @@ public abstract class BaseTest { .name("fw_kurz") .internalName("fw_kurz") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -4066,7 +3888,6 @@ public abstract class BaseTest { .name("fw_lang") .internalName("fw_lang") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -4080,7 +3901,6 @@ public abstract class BaseTest { .name("umlauf_von") .internalName("umlauf_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -4094,7 +3914,6 @@ public abstract class BaseTest { .name("halt_id_von") .internalName("halt_id_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -4108,7 +3927,6 @@ public abstract class BaseTest { .name("halt_id_nach") .internalName("halt_id_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -4122,7 +3940,6 @@ public abstract class BaseTest { .name("halt_punkt_id_von") .internalName("halt_punkt_id_von") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build(), @@ -4136,7 +3953,6 @@ public abstract class BaseTest { .name("halt_punkt_id_nach") .internalName("halt_punkt_id_nach") .isNullAllowed(true) - .dateFormat(null) .enums(new LinkedList<>()) .sets(new LinkedList<>()) .build()); @@ -4817,7 +4633,6 @@ public abstract class BaseTest { .name("reminder") .internalName("reminder") .columnType(TableColumnType.TIME) - .dateFormat(IMAGE_DATE_4) .isNullAllowed(false) .autoGenerated(false) .build(), @@ -4891,7 +4706,6 @@ public abstract class BaseTest { .name("reminder") .internalName("reminder") .columnType(ColumnTypeDto.TIME) - .dateFormat(IMAGE_DATE_4_DTO) .isNullAllowed(false) .autoGenerated(false) .build(), @@ -5050,7 +4864,7 @@ public abstract class BaseTest { .size(22L) .isNullAllowed(true) .autoGenerated(false) - .build() ); + .build()); public final static View VIEW_1 = View.builder() .id(VIEW_1_ID) @@ -5195,7 +5009,6 @@ public abstract class BaseTest { .internalName("date") .ordinalPosition(1) .columnType(ColumnTypeDto.DATE) - .dateFormat(IMAGE_DATE_1_DTO) .isNullAllowed(true) .autoGenerated(false) .build(), @@ -5254,7 +5067,6 @@ public abstract class BaseTest { .name("Date") .internalName("date") .columnType(TableColumnType.DATE) - .dateFormat(IMAGE_DATE_1) .isNullAllowed(true) .autoGenerated(false) .view(VIEW_2) @@ -5388,7 +5200,6 @@ public abstract class BaseTest { .internalName("date") .ordinalPosition(3) .columnType(ColumnTypeDto.DATE) - .dateFormat(IMAGE_DATE_1_DTO) .isNullAllowed(true) .autoGenerated(false) .build() @@ -5463,7 +5274,6 @@ public abstract class BaseTest { .name("Date") .internalName("date") .columnType(TableColumnType.DATE) - .dateFormat(IMAGE_DATE_1) .isNullAllowed(true) .autoGenerated(false) .view(VIEW_3) diff --git a/dbrepo-metric-db/prometheus.yml b/dbrepo-metric-db/prometheus.yml index 447396d167785e7973855314431230fb1193bd0d..82986b249082b38e12247ba865cab6a7cd3d440f 100644 --- a/dbrepo-metric-db/prometheus.yml +++ b/dbrepo-metric-db/prometheus.yml @@ -9,11 +9,11 @@ alerting: - targets: [] scrape_configs: - - job_name: 'spring boot scrape' + - job_name: 'actuator scrape' metrics_path: '/actuator/prometheus' static_configs: - - targets: ['data-service:8080', 'metadata-service:8080'] + - targets: ['data-service:8080', 'metadata-service:8080', 'ui:3000'] - job_name: 'metrics scrape' metrics_path: '/metrics' static_configs: - - targets: ['auth-service:8080', 'analyse-service:8080', 'search-service:8080', 'data-db-sidecar:8080', 'broker-service:15692', 'ui:3000', 'dashboard-service:3000', 'storage-service:9090', 'upload-service:8080'] + - targets: ['auth-service:8080', 'analyse-service:8080', 'search-service:8080', 'data-db-sidecar:8080', 'broker-service:15692', 'storage-service:9090', 'upload-service:8080', 'dashboard-service:3000'] diff --git a/dbrepo-search-service/init/Pipfile b/dbrepo-search-service/init/Pipfile index 517796af748f40cf55f52bac420a000c04c11b23..9647c2ca4e1866754e9b0d695023fb6ba252a61b 100644 --- a/dbrepo-search-service/init/Pipfile +++ b/dbrepo-search-service/init/Pipfile @@ -9,7 +9,7 @@ opensearch-py = "~=2.2" python-dotenv = "~=1.0" testcontainers-opensearch = "*" pytest = "*" -dbrepo = {path = "./lib/dbrepo-1.4.4.tar.gz"} +dbrepo = {path = "./lib/dbrepo-1.4.6.tar.gz"} [dev-packages] coverage = "*" diff --git a/dbrepo-search-service/init/lib/dbrepo-1.4.6rc1-py3-none-any.whl b/dbrepo-search-service/init/lib/dbrepo-1.4.6rc1-py3-none-any.whl deleted file mode 100644 index 83944ce88d8aec5a3b767aa09caf9a8700323104..0000000000000000000000000000000000000000 Binary files a/dbrepo-search-service/init/lib/dbrepo-1.4.6rc1-py3-none-any.whl and /dev/null differ diff --git a/dbrepo-search-service/init/lib/dbrepo-1.4.6rc1.tar.gz b/dbrepo-search-service/init/lib/dbrepo-1.4.6rc1.tar.gz deleted file mode 100644 index a0c8432134f3c21359cd7fb8ee1a341812a6c034..0000000000000000000000000000000000000000 Binary files a/dbrepo-search-service/init/lib/dbrepo-1.4.6rc1.tar.gz and /dev/null differ diff --git a/dbrepo-search-service/init/test/conftest.py b/dbrepo-search-service/init/test/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..2a21f689702d7f78e14e73b6170715753e32b49c --- /dev/null +++ b/dbrepo-search-service/init/test/conftest.py @@ -0,0 +1,49 @@ +import logging + +import pytest +from app import app +from flask import current_app + +from testcontainers.opensearch import OpenSearchContainer + + +@pytest.fixture(scope="session", autouse=True) +def session(request): + """ + Create one OpenSearch container per test run only (admin:admin) + :param request: / + :return: The OpenSearch container + """ + logging.debug("[fixture] creating opensearch container") + container = OpenSearchContainer() + logging.debug("[fixture] starting opensearch container") + container.start() + + with app.app_context(): + current_app.config['OPENSEARCH_HOST'] = container.get_container_host_ip() + current_app.config['OPENSEARCH_PORT'] = container.get_exposed_port(9200) + + # destructor + def stop_opensearch(): + container.stop() + + request.addfinalizer(stop_opensearch) + return container + +# @pytest.fixture(scope="function", autouse=True) +# def cleanup(request, session): +# """ +# Clean up after each test by removing the buckets and re-adding them (=so they are empty again) +# :param request: / +# :param session: / +# :return: +# """ +# logging.info("[fixture] truncate buckets") +# for bucket in ["dbrepo-upload", "dbrepo-download"]: +# objects = [] +# for obj in session.get_client().list_objects(bucket): +# objects.append(DeleteObject(obj.object_name)) +# logging.info(f'request to remove objects {objects}') +# errors = session.get_client().remove_objects(bucket, objects) +# for error in errors: +# raise ConnectionError(f'Failed to delete object with key {error.object_name} of bucket {bucket}') diff --git a/dbrepo-search-service/init/test/test_app.py b/dbrepo-search-service/init/test/test_app.py new file mode 100644 index 0000000000000000000000000000000000000000..a8e6d9755b63d35ac6af4c2b5f1d2d23e311e085 --- /dev/null +++ b/dbrepo-search-service/init/test/test_app.py @@ -0,0 +1,95 @@ +import datetime +import unittest + +from app import app + +from clients.opensearch_client import OpenSearchClient + + +class OpenSearchClientTest(unittest.TestCase): + + def test_index_exists_succeeds(self): + with app.app_context(): + client = RestClient(endpoint=self.metadata_service_endpoint) + # mock + client.update_database(database_id=1, data=req) + + # test + req.tables = [Table(id=1, + name="Test Table", + internal_name="test_table", + queue_name="dbrepo", + routing_key="dbrepo.test_tuw1.test_table", + is_public=True, + database_id=req.id, + constraints=Constraints(uniques=[], foreign_keys=[], checks=[], + primary_key=[PrimaryKey(id=1, + table=TableMinimal(id=1, database_id=1), + column=ColumnMinimal(id=1, table_id=1, + database_id=1))]), + is_versioned=True, + created_by="c6b71ef5-2d2f-48b2-9d79-b8f23a3a0502", + creator=User(id="c6b71ef5-2d2f-48b2-9d79-b8f23a3a0502", + username="foo", + attributes=UserAttributes(theme="dark")), + owner=User(id="c6b71ef5-2d2f-48b2-9d79-b8f23a3a0502", + username="foo", + attributes=UserAttributes(theme="dark")), + created=datetime.datetime(2024, 4, 25, 17, 44, tzinfo=datetime.timezone.utc), + columns=[Column(id=1, + name="ID", + internal_name="id", + database_id=req.id, + table_id=1, + auto_generated=True, + column_type=ColumnType.BIGINT, + is_public=True, + is_null_allowed=False)])] + database = client.update_database(database_id=1, data=req) + self.assertEqual(1, database.id) + self.assertEqual("Test", database.name) + self.assertEqual("test_tuw1", database.internal_name) + self.assertEqual("c6b71ef5-2d2f-48b2-9d79-b8f23a3a0502", database.creator.id) + self.assertEqual("foo", database.creator.username) + self.assertEqual("dark", database.creator.attributes.theme) + self.assertEqual("c6b71ef5-2d2f-48b2-9d79-b8f23a3a0502", database.owner.id) + self.assertEqual("foo", database.owner.username) + self.assertEqual("dark", database.owner.attributes.theme) + self.assertEqual("c6b71ef5-2d2f-48b2-9d79-b8f23a3a0502", database.contact.id) + self.assertEqual("foo", database.contact.username) + self.assertEqual("dark", database.contact.attributes.theme) + self.assertEqual(datetime.datetime(2024, 3, 25, 16, tzinfo=datetime.timezone.utc), database.created) + self.assertEqual("dbrepo", database.exchange_name) + self.assertEqual(True, database.is_public) + self.assertEqual(1, database.container.id) + # ... + self.assertEqual(1, database.container.image.id) + # ... + self.assertEqual(1, len(database.tables)) + self.assertEqual(1, database.tables[0].id) + self.assertEqual("Test Table", database.tables[0].name) + self.assertEqual("test_table", database.tables[0].internal_name) + self.assertEqual("dbrepo", database.tables[0].queue_name) + self.assertEqual("dbrepo.test_tuw1.test_table", database.tables[0].routing_key) + self.assertEqual(True, database.tables[0].is_public) + self.assertEqual(1, database.tables[0].database_id) + self.assertEqual(True, database.tables[0].is_versioned) + self.assertEqual("c6b71ef5-2d2f-48b2-9d79-b8f23a3a0502", database.tables[0].created_by) + self.assertEqual("c6b71ef5-2d2f-48b2-9d79-b8f23a3a0502", database.tables[0].creator.id) + self.assertEqual("foo", database.tables[0].creator.username) + self.assertEqual("dark", database.tables[0].creator.attributes.theme) + self.assertEqual("c6b71ef5-2d2f-48b2-9d79-b8f23a3a0502", database.tables[0].owner.id) + self.assertEqual("foo", database.tables[0].owner.username) + self.assertEqual("dark", database.tables[0].owner.attributes.theme) + self.assertEqual(datetime.datetime(2024, 4, 25, 17, 44, tzinfo=datetime.timezone.utc), + database.tables[0].created) + self.assertEqual(1, len(database.tables[0].columns)) + self.assertEqual(1, database.tables[0].columns[0].id) + self.assertEqual("ID", database.tables[0].columns[0].name) + self.assertEqual("id", database.tables[0].columns[0].internal_name) + self.assertEqual(ColumnType.BIGINT, database.tables[0].columns[0].column_type) + self.assertEqual(1, database.tables[0].columns[0].database_id) + self.assertEqual(1, database.tables[0].columns[0].table_id) + self.assertEqual(True, database.tables[0].columns[0].auto_generated) + self.assertEqual(True, database.tables[0].columns[0].is_public) + self.assertEqual(False, database.tables[0].columns[0].is_null_allowed) diff --git a/dbrepo-search-service/test/run_testindicies.py b/dbrepo-search-service/test/run_testindicies.py deleted file mode 100644 index b547573dd65369cd88be1750f1224f05c333a883..0000000000000000000000000000000000000000 --- a/dbrepo-search-service/test/run_testindicies.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -This script spins up docker containers running an opensearch db with predefined entries. -This is useful e.g. if you want to run tests on the functionality of the opensearch_client. - -note: The port of the test container should be 9200, but it's somehow kinda random, -and using environmet variables also doesn't really work, -so the correct port number is just saved in the .testpickle -""" - -from testcontainers.opensearch import OpenSearchContainer -import pprint -import time -import os -import pickle - - -doc1 = { - "author": "aaa", - "name": "Hi! My name is", - "description":"here's some description text", - "created": "2023-07-27", - "docID":1, - "public":True, - "details": { - "nestedObject1": "something", - "nestedObject2": "something else", - "evenMoreNested": { - "bla":"blib", - "blob":"blub" - } - } -} - -doc2 = { - "author": "max", - "name": "Bla Bla", - "public": False, - "description": "here's another description text, about a fictional entry with some random measurement data", - "created": "2023-07-27", - "docID":2, - "details": { - "nestedObject1": "something", - "nestedObject2": "something else" - } -} - -doc3 = { - "author": "mweise", - "name": "databaseName", - "public": True, - "description": "here is a really old entry", - "created":"2022-07-27", - "docID":3, - "details": { - "nestedObject1": "something", - "nestedObject2": "something else" - } -} -placeholderDoc = { - "blib":"blub", - "public": False -} - -with OpenSearchContainer(port_to_expose=9200) as opensearch: - client = opensearch.get_client() - creation_result = client.index(index="database", body=doc1) - creation_result = client.index(index="database", body=doc2) - creation_result = client.index(index="database", body=doc3) - creation_result = client.index(index="user", body=placeholderDoc) - creation_result = client.index(index="table", body=placeholderDoc) - creation_result = client.index(index="column", body=placeholderDoc) - creation_result = client.index(index="identifier", body=placeholderDoc) - refresh_result = client.indices.refresh(index="database") - search_result = client.search(index="database", body={"query": {"match_all": {}}}) - pp = pprint.PrettyPrinter(indent=1) - config = opensearch.get_config() - os.environ["TEST_OPENSEARCH_HOST"] = config["host"] - os.putenv("TEST_OPENSEARCH_HOST", config["host"]) - os.environ["TEST_OPENSEARCH_PORT"] = config["port"] - os.environ["TEST_OPENSEARCH_USERNAME"] = config["user"] - os.environ["TEST_OPENSEARCH_PASSWORD"] = config["password"] - - pickle_info = {} - pickle_info["port"] = config["port"] - pickle_info["host"] = config["host"] - with open(".testpickle", "ab") as outfile: - pickle.dump(pickle_info, outfile) - print(f"serving on port: {config['port']}") - while True: - time.sleep(1) - diff --git a/dbrepo-ui/Dockerfile b/dbrepo-ui/Dockerfile index 130ce0082d3a5a2c68300a6ce0ece873d81e162c..07f7f5d903e09c477753bdd0e52042cd61bcf6fb 100644 --- a/dbrepo-ui/Dockerfile +++ b/dbrepo-ui/Dockerfile @@ -1,10 +1,10 @@ -FROM node:18.20.4-alpine3.20 AS build +FROM oven/bun:1.1.20-alpine AS build WORKDIR /app COPY ./package.json ./package.json -RUN npm install +RUN bun install ENV NODE_ENV="production" @@ -22,9 +22,9 @@ COPY ./stores ./stores COPY ./utils ./utils COPY ./nuxt.config.ts ./nuxt.config.ts -RUN npm run build +RUN bun run build -FROM node:18.20.4-alpine3.20 AS runtime +FROM oven/bun:1.1.20-alpine AS runtime ARG APP_VERSION="latest" ARG COMMIT="" @@ -39,7 +39,8 @@ RUN chmod -R 755 /app/.output ENV NUXT_PUBLIC_VERSION="${APP_VERSION:-}" ENV NUXT_PUBLIC_COMMIT="${COMMIT:-}" +ENV NODE_OPTIONS="--max_old_space_size=4096" EXPOSE 3000 -ENTRYPOINT [ "node", ".output/server/index.mjs" ] +ENTRYPOINT [ "bun", "run", ".output/server/index.mjs" ] diff --git a/dbrepo-ui/bun.lockb b/dbrepo-ui/bun.lockb index 08343a9606e23719aa606a7bb102499777d9e001..d4a0ef58602cbb7a069d13bc894340f4b0d72465 100755 Binary files a/dbrepo-ui/bun.lockb and b/dbrepo-ui/bun.lockb differ diff --git a/dbrepo-ui/components/dialogs/EditAccess.vue b/dbrepo-ui/components/dialogs/EditAccess.vue index 8132adddf5935ae357893f8f91a2c4c597e6cf94..039b1c40e8e5d241b1d52f6145affad898581702 100644 --- a/dbrepo-ui/components/dialogs/EditAccess.vue +++ b/dbrepo-ui/components/dialogs/EditAccess.vue @@ -12,14 +12,13 @@ <v-col> <v-autocomplete v-if="!isModification" - v-model="modify.userId" + v-model="localUserId" :items="eligibleUsers" :disabled="loadingUsers" :loading="loadingUsers" :rules="[v => !!v || $t('validation.required')]" required :variant="inputVariant" - hide-no-data hide-selected hide-details item-value="id" @@ -56,7 +55,7 @@ :disabled="!valid || loading || accessType === modify.type" :color="buttonColor" type="submit" - :text="$t('pages.database.subpages.access.submit.text')" + :text="$t('navigation.modify')" :loading="loading" @click="updateAccess" /> </v-card-actions> @@ -90,6 +89,7 @@ export default { loadingUsers: false, users: [], error: false, + localUserId: null, types: [ { title: this.$t('pages.database.subpages.access.read'), value: 'read' }, { title: this.$t('pages.database.subpages.access.write-own'), value: 'write_own' }, @@ -169,36 +169,48 @@ export default { }, revokeAccess () { const accessService = useAccessService() - accessService.remove(this.$route.params.database_id, this.userId) + accessService.remove(this.$route.params.database_id, this.localUserId) .then(() => { const toast = useToastInstance() - toast.success(this.$t('notifications.access.revoked')) + toast.success(this.$t('success.access.revoked')) this.$emit('close-dialog', { success: true }) }) + .catch(({code, message}) => { + const toast = useToastInstance() + toast.error(message) + }) .finally(() => { this.loading = false }) }, modifyAccess () { const accessService = useAccessService() - accessService.modify(this.$route.params.database_id, this.userId, this.modify) + accessService.modify(this.$route.params.database_id, this.localUserId, this.modify) .then(() => { const toast = useToastInstance() - toast.success(this.$t('notifications.access.modified')) + toast.success(this.$t('success.access.modified')) this.$emit('close-dialog', { success: true }) }) + .catch(({code, message}) => { + const toast = useToastInstance() + toast.error(message) + }) .finally(() => { this.loading = false }) }, giveAccess () { const accessService = useAccessService() - accessService.create(this.$route.params.database_id, this.userId, this.modify) + accessService.create(this.$route.params.database_id, this.localUserId, this.modify) .then(() => { const toast = useToastInstance() - toast.success(this.$t('notifications.access.created')) + toast.success(this.$t('success.access.created')) this.$emit('close-dialog', { success: true }) }) + .catch(({code, message}) => { + const toast = useToastInstance() + toast.error(message) + }) .finally(() => { this.loading = false }) @@ -210,6 +222,10 @@ export default { .then((users) => { this.users = users.filter(u => u.username !== this.database.creator.username) }) + .catch(({code}) => { + const toast = useToastInstance() + toast.error(this.$t(code)) + }) .finally(() => { this.loadingUsers = false }) @@ -217,6 +233,8 @@ export default { init () { if (!this.userId) { this.loadUsers() + } else { + this.localUserId = this.userId } if (!this.accessType) { this.modify.type = null diff --git a/dbrepo-ui/components/dialogs/EditTuple.vue b/dbrepo-ui/components/dialogs/EditTuple.vue index 589c82b9b7aee5d2678e9ceab8249618878ced3f..da8000dae28372ec49fd8bfec8772a4bbe354fe0 100644 --- a/dbrepo-ui/components/dialogs/EditTuple.vue +++ b/dbrepo-ui/components/dialogs/EditTuple.vue @@ -179,6 +179,7 @@ export default { } }, mounted() { + this.$refs.form.validate() this.oldTuple = Object.assign({}, this.tuple) }, computed: { @@ -212,7 +213,7 @@ export default { hint += ' ' + this.$t('pages.table.subpages.data.primary-key.hint') } if (['double', 'decimal'].includes(column_type)) { - hint += ' ' + this.$t('pages.table.subpages.data.format.hint') + ` ${'d'.repeat(size)}.${'f'.repeat(d)}` + hint += ' ' + this.$t('pages.table.subpages.data.format.hint') + ' ddd.f' } if (['date', 'datetime', 'timestamp', 'time'].includes(column_type) && date_format) { hint += ' ' + this.$t('pages.table.subpages.data.format.hint') + ' ' + date_format.unix_format @@ -258,7 +259,7 @@ export default { rules.push(v => v !== null || this.$t('validation.required')) if (column.column_type === 'decimal' || column.column_type === 'double') { rules.push(v => !(!v || v.split('.')[0].length > column.size) || `${this.$t('pages.table.subpages.data.float.max')} ${column.size} ${this.$t('pages.table.subpages.data.float.before')}`) - rules.push(v => !(!v || v.split('.')[1].length > column.d) || `${this.$t('pages.table.subpages.data.float.max')} ${column.d} ${this.$t('pages.table.subpages.data.float.after')}`) + rules.push(v => !(!v || (column.d && v.split('.')[1].length > column.d)) || `${this.$t('pages.table.subpages.data.float.max')} ${column.d} ${this.$t('pages.table.subpages.data.float.after')}`) } return rules }, diff --git a/dbrepo-ui/components/subset/Builder.vue b/dbrepo-ui/components/subset/Builder.vue index 7c30fd8f5a488cee4173f710e50e50e0154e9210..9ae991620375976dd3c53eee4b5756c35734ad4e 100644 --- a/dbrepo-ui/components/subset/Builder.vue +++ b/dbrepo-ui/components/subset/Builder.vue @@ -370,6 +370,12 @@ export default { database () { return this.cacheStore.getDatabase }, + columnTypes () { + if (!this.database) { + return [] + } + return this.database.container.image.data_types + }, user () { return this.userStore.getUser }, @@ -533,7 +539,7 @@ export default { return } const queryService = useQueryService() - const { error, reason, column, raw, formatted } = queryService.build(this.table.internal_name, this.select, this.clauses) + const { error, reason, column, raw, formatted } = queryService.build(this.table.internal_name, this.select, this.columnTypes, this.clauses) if (error) { const toast = useToastInstance() toast.error(this.$t('error.query.' + reason) + ' ' + column) diff --git a/dbrepo-ui/components/subset/Results.vue b/dbrepo-ui/components/subset/Results.vue index 95becef12c4f0e5116345f1ade444fb36f350fb3..4ba414309c5009bdf88914150b6c1afe11e9a65e 100644 --- a/dbrepo-ui/components/subset/Results.vue +++ b/dbrepo-ui/components/subset/Results.vue @@ -8,6 +8,7 @@ :items="result.rows" :items-length="total" :footer-props="footerProps" + :items-per-page-options="footerProps.itemsPerPageOptions" @update:options="updateOptions" /> </div> </template> diff --git a/dbrepo-ui/components/subset/SubsetToolbar.vue b/dbrepo-ui/components/subset/SubsetToolbar.vue index 5c5081a2f8f5c3944928bbd59cd76dddc2c79d89..db9452feaac3c78e879e3ca157034ea8c87da6dc 100644 --- a/dbrepo-ui/components/subset/SubsetToolbar.vue +++ b/dbrepo-ui/components/subset/SubsetToolbar.vue @@ -166,11 +166,17 @@ export default { } return this.subset.creator.username === this.username }, + hasReadAccess () { + if (!this.access) { + return false + } + return this.access.type === 'read' || this.access.type === 'write_all' || this.access.type === 'write_own' + }, canGetPid () { if (!this.user || !this.subset || !this.database) { return false } - return this.database.owner.id === this.user.id || (this.subset.creator.id === this.user.id && UserUtils.hasReadAccess(this.access)) + return this.database.owner.id === this.user.id || (this.subset.creator.id === this.user.id && this.hasReadAccess) }, title () { if (!this.identifier) { diff --git a/dbrepo-ui/components/table/TableImport.vue b/dbrepo-ui/components/table/TableImport.vue index e55db4130e893810d22bf93ea4308c6e97a2b060..f1658a58bb7663ad2cf65725458b26ebb38cf59d 100644 --- a/dbrepo-ui/components/table/TableImport.vue +++ b/dbrepo-ui/components/table/TableImport.vue @@ -83,39 +83,6 @@ </v-select> </v-col> </v-row> - <v-row dense> - <v-col md="8"> - <v-text-field - v-model="tableImport.null_element" - clearable - persistent-hint - :variant="inputVariant" - :hint="$t('pages.table.subpages.import.null.hint')" - :label="$t('pages.table.subpages.import.null.label')"/> - </v-col> - </v-row> - <v-row dense> - <v-col md="8"> - <v-text-field - v-model="tableImport.true_element" - clearable - persistent-hint - :variant="inputVariant" - :hint="$t('pages.table.subpages.import.true.hint')" - :label="$t('pages.table.subpages.import.true.label')"/> - </v-col> - </v-row> - <v-row dense> - <v-col md="8"> - <v-text-field - v-model="tableImport.false_element" - clearable - persistent-hint - :variant="inputVariant" - :hint="$t('pages.table.subpages.import.false.hint')" - :label="$t('pages.table.subpages.import.false.label')"/> - </v-col> - </v-row> </v-container> </v-form> </v-stepper-window> @@ -356,9 +323,6 @@ export default { this.cacheStore.setUploadProgress(null) this.setQueryParamSafely('location') this.setQueryParamSafely('quote') - this.setQueryParamSafely('false_element') - this.setQueryParamSafely('true_element') - this.setQueryParamSafely('null_element') this.setQueryParamSafely('separator') this.setQueryParamSafely('line_termination') this.setQueryParamSafely('skip_lines') @@ -541,9 +505,6 @@ export default { separator: this.tableImport.separator, skip_lines: this.tableImport.skip_lines, quote: this.tableImport.quote, - null_element: this.tableImport.null_element, - true_element: this.tableImport.true_element, - false_element: this.tableImport.false_element }) this.loading = false }) diff --git a/dbrepo-ui/components/table/TableSchema.vue b/dbrepo-ui/components/table/TableSchema.vue index da30905fa30a9327ca90266bd2ee4088fab46c70..5e105577d7c5ab4f4c5d156a0a9a521240b39073 100644 --- a/dbrepo-ui/components/table/TableSchema.vue +++ b/dbrepo-ui/components/table/TableSchema.vue @@ -14,7 +14,10 @@ <v-text-field v-model="c.name" required - :rules="[v => !!v || $t('validation.required')]" + :rules="[ + v => !!v || $t('validation.required'), + v => this.columns.filter(column => column.name === v).length === 1 || $t('validation.column.exists') + ]" persistent-hint :variant="inputVariant" :label="$t('pages.table.subpages.schema.name.label')" @@ -25,7 +28,7 @@ <v-select v-model="c.type" :items="columnTypes" - item-title="text" + item-title="display_name" item-value="value" required :rules="[v => !!v || $t('validation.required')]" @@ -68,43 +71,43 @@ @focusout="formatValues(c)" /> </v-col> <v-col - v-if="defaultSize(c) !== false" + v-if="columnType(c) && columnType(c).size_required !== null" cols="1"> <v-text-field v-model.number="c.size" type="number" - required + :min="columnType(c).size_min !== null ? columnType(c).size_min : null" + :max="columnType(c).size_max !== null ? columnType(c).size_max : null" + :step="columnType(c).size_step" + :hint="sizeHint(c)" + :clearable="!columnType(c).size_required" + persistent-hint :variant="inputVariant" - :rules="[v => (v !== null && v !== '') || $t('validation.required')]" + :rules="[ + v => !(columnType(c).size_required && (v === null || v === '')) || $t('validation.required') + ]" :error-messages="sizeErrorMessages(c)" :label="$t('pages.table.subpages.schema.size.label')" /> </v-col> <v-col - v-if="defaultD(c) !== false" + v-if="columnType(c) && columnType(c).d_required !== null" cols="1"> <v-text-field v-model.number="c.d" type="number" - required + :min="columnType(c).d_min !== null ? columnType(c).d_min : null" + :max="columnType(c).d_max !== null ? columnType(c).d_max : null" + :step="columnType(c).d_step" + :hint="dHint(c)" + :clearable="!columnType(c).d_required" + persistent-hint :variant="inputVariant" - :rules="[v => (v !== null && v !== '') || $t('validation.required')]" + :rules="[ + v => !(columnType(c).d_required && (v === null || v === '')) || $t('validation.required') + ]" :error-messages="dErrorMessages(c)" :label="$t('pages.table.subpages.schema.d.label')" /> </v-col> - <v-col - cols="2" - v-if="hasDate(c)"> - <v-select - v-model="c.dfid" - required - :variant="inputVariant" - :disabled="disabled" - :rules="[v => !!v || $t('validation.required')]" - :items="filterDateFormats(c)" - item-title="unix_format" - item-value="id" - :label="$t('pages.table.subpages.schema.fsp.label')" /> - </v-col> <v-col v-if="shift(c)" :cols="shift(c)" /> @@ -219,7 +222,6 @@ export default { return { valid: false, tableColumns: [], - columnTypes: useQueryService().mySql8DataTypes(), cacheStore: useCacheStore() } }, @@ -227,6 +229,12 @@ export default { database () { return this.cacheStore.getDatabase }, + columnTypes () { + if (!this.database) { + return [] + } + return this.database.container.image.data_types + }, dateFormats () { if (!this.database || !('container' in this.database) || !('image' in this.database.container) || !('date_formats' in this.database.container.image)) { return [] @@ -258,16 +266,10 @@ export default { return false } let shift = 0 - if (this.hasDate(column) === false && this.columns.filter(c => this.hasDate(c) !== false).length > 0) { - shift++ - } - if (this.defaultSize(column) === false && this.columns.filter(c => this.defaultSize(c) !== false).length > 0) { - shift++ - } - if (this.defaultD(column) === false && this.columns.filter(c => this.defaultD(c) !== false).length > 0) { + if (!this.hasEnumOrSet(column) && (this.columnType(column).size_required === null || this.columnType(column).size_required === undefined) && this.columns.filter(c => (this.columnType(c).size_required !== null || this.columnType(c).size_required !== undefined)).length > 0) { shift++ } - if (this.hasEnumOrSet(column) === false && this.columns.filter(c => this.hasEnumOrSet(c) !== false).length > 0) { + if (!this.hasEnumOrSet(column) && (this.columnType(column).d_required === null || this.columnType(column).d_required === undefined) && this.columns.filter(c => (this.columnType(c).d_required !== null || this.columnType(c).d_required !== undefined)).length > 0) { shift++ } return shift @@ -298,7 +300,6 @@ export default { type, null_allowed, primary_key, - dfid: null, sets: [], sets_values: null, enums: [], @@ -321,31 +322,57 @@ export default { column.enums = column.enums_values.split(',').map(v => v.trim()) } }, - defaultSize (column) { + columnType (column) { const filter = this.columnTypes.filter(t => t.value === column.type) if (!filter || filter.length === 0) { return false } - if (filter[0].defaultSize === undefined || filter[0].defaultSize === null) { - return false + return filter[0] + }, + sizeHint (column) { + let hint = '' + if (this.columnType(column).size_min !== null) { + hint += `min. ${this.columnType(column).size_min}` + } + if (this.columnType(column).size_max) { + if (hint.length > 0) { + hint += ', ' + } + hint += `max. ${this.columnType(column).size_max}` } - return filter[0].defaultSize + if (!this.columnType(column).size_required) { + hint += ' (optional)' + } + return hint }, - defaultD (column) { - const filter = this.columnTypes.filter(t => t.value === column.type) - if (!filter || filter.length === 0) { - return false + dHint (column) { + let hint = '' + if (this.columnType(column).d_min !== null) { + hint += `min. ${this.columnType(column).d_min}` } - if (filter[0].defaultD === undefined || filter[0].defaultD === null) { - return false + if (this.columnType(column).d_max) { + if (hint.length > 0) { + hint += ', ' + } + hint += `max. ${this.columnType(column).d_max}` + } + if (!this.columnType(column).d_required) { + hint += ' (optional)' } - return filter[0].defaultD + return hint }, setDefaultSizeAndD (column) { - column.size = this.defaultSize(column) - column.d = this.defaultD(column) - column.dfid = null - console.debug('for column type', column.type, 'set default size', column.size, '& d', column.d, '& dfid', column.dfid) + if (this.columnType(column).size_default !== null) { + column.size = this.columnType(column).size_default + } else { + column.size = null + } + if (this.columnType(column).d_default !== null) { + column.d = this.columnType(column).d_default + } else { + column.d = null + } + console.debug('for column type', column.type, 'set default size', column.size, '& d', column.d) }, hasDate (column) { return column.type === 'date' || column.type === 'datetime' || column.type === 'timestamp' || column.type === 'time' diff --git a/dbrepo-ui/composables/access-service.ts b/dbrepo-ui/composables/access-service.ts index c08e5d0b9f6bac53b8d7eaeba993e06234435b31..056efec1171933d7dcf0921c7eff34717d574333 100644 --- a/dbrepo-ui/composables/access-service.ts +++ b/dbrepo-ui/composables/access-service.ts @@ -21,7 +21,7 @@ export const useAccessService = (): any => { const axios = useAxiosInstance() console.debug('create access for user with id', userId, 'of database with id', databaseId) return new Promise<DatabaseAccessDto>((resolve, reject) => { - axios.post<DatabaseAccessDto>(`/api/database/${databaseId}/access`, payload) + axios.post<DatabaseAccessDto>(`/api/database/${databaseId}/access/${userId}`, payload) .then((response) => { console.info('Created access for user with id', userId, 'of database with id', databaseId) resolve(response.data) diff --git a/dbrepo-ui/composables/query-service.ts b/dbrepo-ui/composables/query-service.ts index f5d805b958f27676f7f1031bfe56af16721b4378..b3c21c605344db722259df375d7eba98404c65de 100644 --- a/dbrepo-ui/composables/query-service.ts +++ b/dbrepo-ui/composables/query-service.ts @@ -126,7 +126,7 @@ export const useQueryService = (): any => { }) } - function build(table: TableDto, columns: ColumnDto[], clauses: any[]): QueryBuildResultDto { + function build(table: TableDto, columns: ColumnDto[], types: DataTypeDto[], clauses: any[]): QueryBuildResultDto { var sql = 'SELECT' for (let i = 0; i < columns.length; i++) { sql += `${i > 0 ? ',' : ''} \`${columns[i].internal_name}\`` @@ -140,8 +140,8 @@ export const useQueryService = (): any => { sql += ` ${clause.type.toUpperCase()} ` continue } - const fCol = columns.filter(c => c.internal_name === clause.params[0]) - if (fCol.length === 0) { + const filteredColumn = columns.filter(c => c.internal_name === clause.params[0]) + if (filteredColumn.length === 0) { return { error: true, reason: 'column.exists', @@ -151,26 +151,26 @@ export const useQueryService = (): any => { } } sql += ` \`${clause.params[0]}\` ${clause.params[1]} ` - const fCon = mySql8DataTypes().filter(t => t.value === fCol[0].column_type) - if (fCol.length === 0) { + const filteredType = types.filter(t => t.value === filteredColumn[0].column_type) + if (filteredType.length === 0) { return { error: true, - reason: 'type.exists', - column: fCol[0].column_type, + reason: 'exists', + column: filteredColumn[0].column_type, raw: null, formatted: null } } - if (!fCon[0].isBuildable) { + if (!filteredType[0].is_buildable) { return { error: true, - reason: 'type.build', - column: fCol[0].column_type, + reason: 'build', + column: filteredColumn[0].column_type, raw: null, formatted: null } } - if (fCon[0].quoted) { + if (filteredType[0].is_quoted) { sql += `'${clause.params[2]}'` } else { sql += `${clause.params[2]}` @@ -196,39 +196,5 @@ export const useQueryService = (): any => { return {timestamp, page, size} } - function mySql8DataTypes(): MySql8DataType[] { - return [ - {value: 'bigint', text: 'BIGINT(size)', defaultSize: 255, defaultD: null, quoted: false, isBuildable: true}, - {value: 'binary', text: 'BINARY(size)', defaultSize: 1, defaultD: null, quoted: false, isBuildable: false}, - {value: 'bit', text: 'BIT(size)', defaultSize: 1, defaultD: null, quoted: false, isBuildable: true}, - {value: 'blob', text: 'BLOB', defaultSize: null, defaultD: null, quoted: false, isBuildable: false}, - {value: 'bool', text: 'BOOL', defaultSize: null, defaultD: null, quoted: false, isBuildable: true}, - {value: 'char', text: 'CHAR(size)', defaultSize: 1, defaultD: null, quoted: true, isBuildable: true}, - {value: 'date', text: 'DATE', defaultSize: null, defaultD: null, quoted: true, isBuildable: true}, - {value: 'datetime', text: 'DATETIME(fsp)', defaultSize: null, defaultD: null, quoted: true, isBuildable: true}, - {value: 'decimal', text: 'DECIMAL(size, d)', defaultSize: 40, defaultD: 10, quoted: false, isBuildable: true}, - {value: 'double', text: 'DOUBLE(size, d)', defaultSize: 40, defaultD: 10, quoted: false, isBuildable: true}, - {value: 'enum', text: 'ENUM(val1,val2,...)', defaultSize: null, defaultD: null, quoted: true, isBuildable: true}, - {value: 'float', text: 'FLOAT(p)', defaultSize: 24, defaultD: null, quoted: false, isBuildable: true}, - {value: 'int', text: 'INT(size)', defaultSize: 255, defaultD: null, quoted: false, isBuildable: true}, - {value: 'longblob', text: 'LONGBLOB', defaultSize: null, defaultD: null, quoted: false, isBuildable: false}, - {value: 'longtext', text: 'LONGTEXT', defaultSize: null, defaultD: null, quoted: true, isBuildable: true}, - {value: 'mediumblob', text: 'MEDIUMBLOB', defaultSize: null, defaultD: null, quoted: false, isBuildable: false}, - {value: 'mediumint', text: 'MEDIUMINT(size)', defaultSize: 40, defaultD: null, quoted: false, isBuildable: true}, - {value: 'mediumtext', text: 'MEDIUMTEXT', defaultSize: null, defaultD: null, quoted: true, isBuildable: true}, - {value: 'set', text: 'SET(val1,val2,...)', defaultSize: null, defaultD: null, quoted: true, isBuildable: true}, - {value: 'smallint', text: 'SMALLINT(size)', defaultSize: 10, defaultD: null, quoted: false, isBuildable: true}, - {value: 'text', text: 'TEXT', defaultSize: null, defaultD: null, quoted: true, isBuildable: true}, - {value: 'time', text: 'TIME(fsp)', defaultSize: null, defaultD: null, quoted: true, isBuildable: true}, - {value: 'timestamp', text: 'TIMESTAMP(fsp)', defaultSize: null, defaultD: null, quoted: true, isBuildable: true}, - {value: 'tinyblob', text: 'TINYBLOB', defaultSize: null, defaultD: null, quoted: false, isBuildable: false}, - {value: 'tinyint', text: 'TINYINT(size)', defaultSize: 10, defaultD: null, quoted: false, isBuildable: true}, - {value: 'tinytext', text: 'TINYTEXT', defaultSize: null, defaultD: null, quoted: true, isBuildable: true}, - {value: 'year', text: 'YEAR', defaultSize: null, defaultD: null, quoted: true, isBuildable: true}, - {value: 'varbinary', text: 'VARBINARY(size)', defaultSize: 1, defaultD: null, quoted: false, isBuildable: false}, - {value: 'varchar', text: 'VARCHAR(size)', defaultSize: 255, defaultD: null, quoted: true, isBuildable: true} - ] - } - - return {findAll, findOne, update, exportCsv, execute, reExecuteData, reExecuteCount, build, mySql8DataTypes} + return {findAll, findOne, update, exportCsv, execute, reExecuteData, reExecuteCount, build} } diff --git a/dbrepo-ui/composables/table-service.ts b/dbrepo-ui/composables/table-service.ts index ffd7ebcd6074cc23d9bfa9412e19e48ef87f8f37..3d87e68d4febed8b825a56b1aada946bd6b0f4d5 100644 --- a/dbrepo-ui/composables/table-service.ts +++ b/dbrepo-ui/composables/table-service.ts @@ -210,7 +210,6 @@ export const useTableService = (): any => { type: c.type, size: c.size ? c.size : null, d: c.d ? c.d : null, - dfid: c.dfid ? c.dfid : null, enums: c.enums_values ? c.enums_values.split(',') : [], sets: c.sets_values ? c.sets_values.split(',') : [], index_length: c.index_length, diff --git a/dbrepo-ui/dto/index.ts b/dbrepo-ui/dto/index.ts index df0babcfe1f310bdff6415f6351a82020ec3cf4e..543bd0a3910a7f8ffbe2d0d10486234e2c446424 100644 --- a/dbrepo-ui/dto/index.ts +++ b/dbrepo-ui/dto/index.ts @@ -532,9 +532,6 @@ interface ImportCsv { separator: string; quote: string; skip_lines: number; - false_element: string; - true_element: string; - null_element: string; line_termination: string; } @@ -562,7 +559,6 @@ interface ColumnCreateDto { type: string; size: number | null; d: number | null; - dfid: number | null; enums: string[]; sets: string[]; index_length: number; @@ -574,7 +570,6 @@ interface InternalColumnDto { type: string; size: number; d: number; - dfid: number; enums: string[]; sets: string[]; primary_key: boolean; @@ -658,9 +653,6 @@ interface ImportDto { separator: string; quote: string; skip_lines: number; - false_element: string; - true_element: string; - null_element: string; line_termination: string; } diff --git a/dbrepo-ui/dto/mysql.ts b/dbrepo-ui/dto/mysql.ts index b100da017c241b94d84169d1c4545d9fb800e9fc..c366e43f649c9f3eebd14b5c114164f5c4c9c71c 100644 --- a/dbrepo-ui/dto/mysql.ts +++ b/dbrepo-ui/dto/mysql.ts @@ -1,8 +1,15 @@ -interface MySql8DataType { +interface DataTypeDto { + display_name: string; value: string; - text: string; - defaultSize: number | null; - defaultD: number | null; - quoted: boolean; - isBuildable: boolean; + size_min: number | null; + size_max: number | null; + size_default: number | null; + size_required: number | null; + d_min: number | null; + d_max: number | null; + d_default: number | null; + d_required: number | null; + documentation: string; + is_quoted: boolean; + is_buildable: boolean; } diff --git a/dbrepo-ui/locales/en-US.json b/dbrepo-ui/locales/en-US.json index 6abf715bc0ea6b7e8756acb78141672758ef9b54..db2c41bfd272feb51c5ef36fa6bf4a490e0d5e08 100644 --- a/dbrepo-ui/locales/en-US.json +++ b/dbrepo-ui/locales/en-US.json @@ -32,7 +32,8 @@ "no": "No", "mine": "(mine)", "loading": "Loading", - "view": "View" + "view": "View", + "modify": "Modify" }, "pages": { "identifier": { @@ -611,10 +612,10 @@ "access": { "title": "Database Access", "subtitle": "Overview on users with their access to the database", - "read": "You can read all contents", - "write-own": "You can write own tables and read all contents", - "write-all": "You can write own tables and read all contents", - "revoke": "Revoke", + "read": "Read all contents", + "write-own": "Read all contents & write own tables", + "write-all": "Read all contents & write all tables", + "revoke": "No access", "action": "Action", "username": { "label": "Username", @@ -623,9 +624,6 @@ "type": { "label": "Access Type", "hint": "Required" - }, - "submit": { - "text": "Modify" } }, "create": { @@ -1144,8 +1142,8 @@ "query": { "missing": "Failed to find query in data service", "invalid": "Query is invalid", - "type.exists": "Failed to build query: no such column type", - "type.build": "Failed to build query: currently no query build support for column type", + "exists": "Failed to build query: no such column type", + "build": "Failed to build query: currently no query build support for column type", "column.exists": "Failed to build query: data columns are missing column with name" }, "store": { @@ -1442,6 +1440,9 @@ "pattern": "Invalid URI", "exists": "URI exists" }, + "column": { + "exists": "Column with this name exists" + }, "user": { "pattern": "Only lowercase letters, min. 3 length", "exists": "This username is already taken" diff --git a/dbrepo-ui/nuxt.config.ts b/dbrepo-ui/nuxt.config.ts index f33c990a71ca2ca1b4e594a6b219b8ac5cbcb748..80b866f1c157fab3a87b12b9fcce42dad4b5c05b 100644 --- a/dbrepo-ui/nuxt.config.ts +++ b/dbrepo-ui/nuxt.config.ts @@ -102,13 +102,13 @@ export default defineNuxtConfig({ endpoint: 'https://doi.org' }, links: { - rabbitmq: { - text: 'RabbitMQ Admin', - href: '/admin/broker/' - }, keycloak: { - text: 'Keycloak Admin', - href: '/api/auth/' + text: 'Auth Service', + href: 'http://localhost/api/auth/' + }, + grafana: { + text: 'Dashboard Service', + href: 'http://localhost:3000/dashboards' } } } @@ -123,8 +123,7 @@ export default defineNuxtConfig({ modules: [ '@pinia/nuxt', '@pinia-plugin-persistedstate/nuxt', - '@nuxtjs/i18n', - '@artmizu/nuxt-prometheus' + '@nuxtjs/i18n' ], pinia: { diff --git a/dbrepo-ui/package.json b/dbrepo-ui/package.json index a1ed44b93a9887fbe98bfd878ea5dd9aaf628262..2bbe6696bc2e59845e4d1d30194a860923f2760d 100644 --- a/dbrepo-ui/package.json +++ b/dbrepo-ui/package.json @@ -11,7 +11,6 @@ "prod": "bun run .output/server/index.mjs" }, "dependencies": { - "@artmizu/nuxt-prometheus": "^2.4.0", "@fontsource/open-sans": "^5.0.24", "@mdi/font": "^7.4.47", "@nuxtjs/robots": "^3.0.0", diff --git a/dbrepo-ui/pages/database/[database_id]/settings.vue b/dbrepo-ui/pages/database/[database_id]/settings.vue index 09c3d8263bc46bc74eb0fe34b8651e41c6214ccc..1d2d9dca0c873f300ec0a1b3c41795595fd085ee 100644 --- a/dbrepo-ui/pages/database/[database_id]/settings.vue +++ b/dbrepo-ui/pages/database/[database_id]/settings.vue @@ -96,8 +96,9 @@ v-if="item && item.user && item.user.username !== user.username" size="x-small" variant="flat" + color="warning" :disabled="!canModifyAccess" - :text="$t('pages.database.subpages.access.submit.text')" + :text="$t('navigation.modify')" @click="modifyAccess(item)" /> </template> </v-data-table> @@ -107,7 +108,7 @@ variant="flat" :disabled="!canCreateAccess" color="warning" - :text="$t('pages.database.subpages.access.submit.text')" + :text="$t('navigation.create')" @click="giveAccess" /> </v-card-text> </v-card> @@ -422,7 +423,7 @@ export default { this.$refs.form.validate() }, closeDialog () { - this.reloadDatabase() + this.cacheStore.reloadDatabase() this.editAccessDialog = false }, updateDatabaseVisibility () { @@ -510,11 +511,11 @@ export default { updateDatabaseOwner () { this.loading = true const databaseService = useDatabaseService() - databaseService.updateOwner(this.$route.params.database_id, this.modifyOwner.id) + databaseService.updateOwner(this.$route.params.database_id, { id: this.modifyOwner.id }) .then(() => { const toast = useToastInstance() toast.success(this.$t('success.database.transfer')) - location.reload() + this.$router.push(`/database/${this.$route.params.database_id}/info`) }) .catch(() => { this.loading = false diff --git a/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue b/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue index bcab9b60be8812f270836644e5f521553a22040d..72e7501d16625e32d3c05e32072a3a3cb3a719e3 100644 --- a/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue +++ b/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue @@ -75,6 +75,7 @@ :loading="loadingData || loadingCount" :options.sync="options" :footer-props="footerProps" + :items-per-page-options="footerProps.itemsPerPageOptions" @update:options="loadData"> <template v-for="(blobColumn, idx) in blobColumns" diff --git a/dbrepo-ui/pages/database/[database_id]/table/[table_id]/schema.vue b/dbrepo-ui/pages/database/[database_id]/table/[table_id]/schema.vue index 3a821a730b50c6292f603b67c8b666add18aaa08..7e8aed6717ee9fd22190b56ef6b943f441348974 100644 --- a/dbrepo-ui/pages/database/[database_id]/table/[table_id]/schema.vue +++ b/dbrepo-ui/pages/database/[database_id]/table/[table_id]/schema.vue @@ -217,12 +217,20 @@ export default { }, methods: { extra (column) { - if (['date', 'datetime', 'timestamp', 'time'].includes(column.column_type)) { - return `fsp=${column.date_format.unix_format}` - } else if (column.column_type === 'float') { - return `p=${column.size}` + if (column.column_type === 'float') { + return `precision=${column.size}` } else if (['decimal', 'double'].includes(column.column_type)) { - return `size=${column.size} d=${column.d}` + let extra = '' + if (column.size !== null) { + extra += `size=${column.size}` + } + if (column.d !== null) { + if (extra.length > 0) { + extra += ', ' + } + extra += `d=${column.d}` + } + return extra } else if (column.column_type === 'enum') { return `(${column.enums.join(', ')})` } else if (column.column_type === 'set') { diff --git a/dbrepo-ui/pages/database/[database_id]/table/create/dataset.vue b/dbrepo-ui/pages/database/[database_id]/table/create/dataset.vue index c3b5a38c7ab8f95892ecf496325d8365dd5f1fcb..e1d71643ea84be7a727ba031715f72ce817b4209 100644 --- a/dbrepo-ui/pages/database/[database_id]/table/create/dataset.vue +++ b/dbrepo-ui/pages/database/[database_id]/table/create/dataset.vue @@ -241,9 +241,6 @@ export default { tableImport: { location: null, quote: '"', - false_element: null, - true_element: null, - null_element: '', separator: ',', line_termination: null, skip_lines: 1 @@ -364,9 +361,6 @@ export default { this.tableImport.separator = separator this.tableImport.skip_lines = skip_lines this.tableImport.quote = quote - this.tableImport.null_element = null_element - this.tableImport.true_element = true_element - this.tableImport.false_element = false_element if (filename) { this.step = 4 } diff --git a/dbrepo-ui/pages/database/[database_id]/table/create/schema.vue b/dbrepo-ui/pages/database/[database_id]/table/create/schema.vue index 6642e89bf41e31326a42b5bae1e1100e0f04f5fb..79493ec39f3a13054df2e4b605f7da3fdb442805 100644 --- a/dbrepo-ui/pages/database/[database_id]/table/create/schema.vue +++ b/dbrepo-ui/pages/database/[database_id]/table/create/schema.vue @@ -269,13 +269,13 @@ export default { this.cacheStore.reloadDatabase() this.table = table }) - .catch(({code}) => { + .catch(({code, message}) => { this.loading = false const toast = useToastInstance() if (typeof code !== 'string') { return } - toast.error(this.$t(code)) + toast.error(message) }) .finally(() => { this.loading = false diff --git a/dbrepo-ui/pages/semantic/index.vue b/dbrepo-ui/pages/semantic/index.vue index f6b6721b17c28d1bc5f30d2885ee253139d96a0c..c5f12d48f48efe9c60825af6f1078f8ae439feb8 100644 --- a/dbrepo-ui/pages/semantic/index.vue +++ b/dbrepo-ui/pages/semantic/index.vue @@ -27,7 +27,8 @@ :items="rows" :options.sync="options" :server-items-length="total" - :footer-props="footerProps"> + :footer-props="footerProps" + :items-per-page-options="footerProps.itemsPerPageOptions"> <template v-slot:item.uri="{ item }"> <a :href="item.uri" target="_blank" v-text="item.uri" /> </template> @@ -83,7 +84,7 @@ export default { }, total: -1, footerProps: { - 'items-per-page-options': [10, 20, 30, 40, 50] + itemsPerPageOptions: [10, 25, 50, 100] }, tab: 0, tabs: [ diff --git a/dbrepo-ui/server/routes/actuator/prometheus.ts b/dbrepo-ui/server/routes/actuator/prometheus.ts new file mode 100644 index 0000000000000000000000000000000000000000..3647103148b19761fe534d9a70ad004bd8e70199 --- /dev/null +++ b/dbrepo-ui/server/routes/actuator/prometheus.ts @@ -0,0 +1,4 @@ +export default defineEventHandler((event) => { + event.node.res.setHeader('Content-Type', 'text/plain'); + return 'service_started 1' +}) diff --git a/docker-compose.yml b/docker-compose.yml index 5effbfdb0b75bd013f09e2b85f3ac55a755e7b3a..9a13b7ca63e2b49ecc32e7004bce2e6ecdc704eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,8 @@ volumes: search-db-data: storage-service-data: identity-service-data: + metric-db-data: + dashboard-service-data: services: dbrepo-metadata-db: @@ -207,16 +209,19 @@ services: restart: "no" container_name: dbrepo-broker-service hostname: broker-service - image: docker.io/bitnami/rabbitmq:3.12-debian-12 + image: docker.io/bitnami/rabbitmq:3.13.7-debian-12-r4 ports: - 15672:15672 - 5672:5672 + - 1883:1883 volumes: - ./dbrepo-broker-service/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf - ./dbrepo-broker-service/advanced.config:/etc/rabbitmq/advanced.config - ./dbrepo-broker-service/enabled_plugins:/etc/rabbitmq/enabled_plugins - ./dbrepo-broker-service/definitions.json:/app/definitions.json - broker-service-data:/bitnami/rabbitmq/mnesia + environment: + RABBITMQ_FEATURE_FLAGS: mqtt_v5 depends_on: dbrepo-identity-service: condition: service_healthy @@ -432,6 +437,7 @@ services: image: bitnami/prometheus:2.54.1-debian-12-r4 volumes: - ./dbrepo-metric-db/prometheus.yml:/etc/prometheus/prometheus.yml + - metric-db-data:/opt/bitnami/prometheus/data ports: - 9090:9090 healthcheck: @@ -450,10 +456,9 @@ services: build: context: ./dbrepo-dashboard-service network: host + volumes: + - dashboard-service-data:/opt/bitnami/grafana/data environment: - GF_SERVER_DOMAIN: "dashboard-service" - GF_SERVER_ROOT_URL: "${BASE_URL:-http://localhost}/dashboard/" - GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION: "true" LDAP_ADMIN_USERNAME: "${IDENTITY_SERVICE_ADMIN_USERNAME:-admin}" LDAP_ADMIN_PASSWORD: "${IDENTITY_SERVICE_ADMIN_PASSWORD:-admin}" LDAP_ROOT: "${IDENTITY_SERVICE_ROOT:-dc=dbrepo,dc=at}" @@ -534,7 +539,7 @@ services: BROKER_HOST: "${BROKER_ENDPOINT:-broker-service}" BROKER_PASSWORD: "${SYSTEM_PASSWORD:-admin}" BROKER_PORT: ${BROKER_PORT:-5672} - BROKER_SERVICE_ENDPOINT: "${BROKER_SERVICE_ENDPOINT:-http://gateway-service/admin/broker}" + BROKER_SERVICE_ENDPOINT: "${BROKER_SERVICE_ENDPOINT:-http://broker-service:15672}" BROKER_USERNAME: "${SYSTEM_USERNAME:-admin}" BROKER_VIRTUALHOST: "${BROKER_VIRTUALHOST:-dbrepo}" CONNECTION_TIMEOUT: ${CONNECTION_TIMEOUT:-60000} diff --git a/helm/dbrepo/charts/dbrepo-mariadb-galera-1.4.6.tgz b/helm/dbrepo/charts/dbrepo-mariadb-galera-1.4.6.tgz index f5a16c907c311685e859a0f971aeae9e603300bc..4af22ff6e998199f69a8e1ff43fd96c3f55aa8ec 100644 Binary files a/helm/dbrepo/charts/dbrepo-mariadb-galera-1.4.6.tgz and b/helm/dbrepo/charts/dbrepo-mariadb-galera-1.4.6.tgz differ diff --git a/helm/dbrepo/values.schema.json b/helm/dbrepo/values.schema.json index 072efb6fe131830c6534580d7a31da547908a4aa..a8e7bbbf743a45c32d8a3a43c8785f4c279fc0ba 100644 --- a/helm/dbrepo/values.schema.json +++ b/helm/dbrepo/values.schema.json @@ -386,6 +386,23 @@ }, "service": { "properties": { + "extraPorts": { + "items": { + "properties": { + "name": { + "type": "string" + }, + "port": { + "type": "integer" + }, + "targetPort": { + "type": "integer" + } + }, + "type": "object" + }, + "type": "array" + }, "managerPortEnabled": { "type": "boolean" }, diff --git a/helm/dbrepo/values.yaml b/helm/dbrepo/values.yaml index d4133c84655da607738f728674877962ef75998a..d889cc85b7bb3e6efce433eaf96fc12409724662 100644 --- a/helm/dbrepo/values.yaml +++ b/helm/dbrepo/values.yaml @@ -262,7 +262,7 @@ brokerservice: enabled: true existingSecret: broker-service-secret ## @param brokerservice.extraPlugins The list of plugins to be activated. - extraPlugins: rabbitmq_prometheus rabbitmq_auth_backend_ldap rabbitmq_auth_mechanism_ssl + extraPlugins: rabbitmq_prometheus rabbitmq_auth_backend_ldap rabbitmq_auth_mechanism_ssl rabbitmq_mqtt persistence: ## @param brokerservice.persistence.enabled If set to true, a PVC will be created. enabled: false @@ -270,6 +270,10 @@ brokerservice: service: type: ClusterIP managerPortEnabled: true + extraPorts: + - name: mqtt + port: 1883 + targetPort: 1883 # loadBalancerIP: ## @param brokerservice.replicaCount The number of replicas. replicaCount: 1 diff --git a/install.sh b/install.sh index a59965816bd2200960b407a53aacbd3e0b879b0d..bce01f0ec311f524daa159bf70a995893e765aa5 100644 --- a/install.sh +++ b/install.sh @@ -58,7 +58,7 @@ fi # environment echo "[🚀] Gathering environment for version ${VERSION} ..." -curl -sSL -o ./dist.tar.gz "https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/${VERSION}/dist.tar.gz" +curl -ksSL -o ./dist.tar.gz "https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/${VERSION}/dist.tar.gz" tar xzfv ./dist.tar.gz if [[ $DOWNLOAD_ONLY -eq 1 ]]; then diff --git a/lib/python/dbrepo/AmqpClient.py b/lib/python/dbrepo/AmqpClient.py index 27f7fc4f0fd702e7f40135102f844bb8108971d2..cd0b1140996d0572586985ee0eb2e1285e556c27 100644 --- a/lib/python/dbrepo/AmqpClient.py +++ b/lib/python/dbrepo/AmqpClient.py @@ -4,6 +4,8 @@ import sys import json import logging +from dbrepo.api.exceptions import AuthenticationError + logging.basicConfig(format='%(asctime)s %(name)-12s %(levelname)-6s %(message)s', level=logging.INFO, stream=sys.stdout) @@ -14,9 +16,9 @@ class AmqpClient: via environment variables, e.g. set endpoint with DBREPO_ENDPOINT. You can override the constructor parameters \ with the environment variables. - :param broker_host: The AMQP API host. Optional. Default: "broker-service" - :param broker_port: The AMQP API port. Optional. Default: 5672 - :param broker_virtual_host: The AMQP API virtual host. Optional. Default: "/" + :param broker_host: The AMQP API host. Optional. Default: "localhost". + :param broker_port: The AMQP API port. Optional. Default: 5672, + :param broker_virtual_host: The AMQP API virtual host. Optional. Default: "dbrepo". :param username: The AMQP API username. Optional. :param password: The AMQP API password. Optional. """ @@ -27,9 +29,9 @@ class AmqpClient: password: str = None def __init__(self, - broker_host: str = 'broker-service', + broker_host: str = 'localhost', broker_port: int = 5672, - broker_virtual_host: str = '/', + broker_virtual_host: str = 'dbrepo', username: str = None, password: str = None) -> None: self.broker_host = os.environ.get('AMQP_API_HOST', broker_host) @@ -41,14 +43,16 @@ class AmqpClient: self.username = os.environ.get('AMQP_API_USERNAME', username) self.password = os.environ.get('AMQP_API_PASSWORD', password) - def publish(self, exchange: str, routing_key: str, data=dict) -> None: + def publish(self, routing_key: str, data=dict, exchange: str = 'dbrepo') -> None: """ Publishes data to a given exchange with the given routing key with a blocking connection. - :param exchange: The exchange name. :param routing_key: The routing key. :param data: The data. + :param exchange: The exchange name. Default: "dbrepo". """ + if self.username is None or self.password is None: + raise AuthenticationError(f"Failed to perform request: authentication required") parameters = pika.ConnectionParameters(host=self.broker_host, port=self.broker_port, virtual_host=self.broker_virtual_host, credentials=pika.credentials.PlainCredentials(self.username,