diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml
index e8978ea1eba7f48b7721d86b6501bb5b0c015f92..56c5f099a79672895fbfeed6a56a673991e36a10 100644
--- a/.docker/docker-compose.yml
+++ b/.docker/docker-compose.yml
@@ -131,7 +131,7 @@ services:
restart: "no"
container_name: dbrepo-data-db
hostname: data-db
- image: docker.io/bitnami/mariadb:${MARIADB_VERSION}
+ image: docker.io/bitnami/mariadb-galera:${MARIADB_VERSION}
volumes:
- data-db-data:/var/lib/mysql
- ./config/1_grant-user.sql:/docker-entrypoint-initdb.d/1_grant-user.sql
@@ -141,15 +141,42 @@ services:
dbrepo-logging-service:
condition: service_healthy
environment:
- MARIADB_EXTRA_FLAGS: "${DATA_DB_EXTRA_FLAGS:---max_connections=155 --max-statement-time=60}"
+ MARIADB_GALERA_MARIABACKUP_PASSWORD: "${DATA_DB_BACKUP_PASSWORD:-dbrepo}"
MARIADB_PASSWORD: "${READONLY_PASSWORD:-readonly}"
MARIADB_ROOT_PASSWORD: "${DATA_DB_PASSWORD:-dbrepo}"
+ MARIADB_SKIP_TEST_DB: "yes"
MARIADB_USER: "${READONLY_USERNAME:-readonly}"
healthcheck:
- test: /opt/bitnami/scripts/mariadb/healthcheck.sh --connect --innodb_initialized
+ test: /opt/bitnami/scripts/mariadb-galera/healthcheck.sh --connect --innodb_initialized
<<: *healthcheck-params
deploy:
- <<: *resources-micro-hm
+ <<: *resources-xlarge
+ logging:
+ driver: fluentd
+ options:
+ fluentd-address: 127.0.0.1:24224
+ tag: mariadb.data
+
+ dbrepo-data-db-metrics:
+ restart: "no"
+ container_name: dbrepo-data-db-metrics
+ hostname: data-db-metrics
+ image: docker.io/bitnami/mysqld-exporter:${MYSQLD_EXPORTER_VERSION}
+ command:
+ - --mysqld.address=data-db:3306
+ - --config.my-cnf=/.my.cnf
+ volumes:
+ - ./dbrepo-data-db/metrics.cnf:/.my.cnf
+ healthcheck:
+ test: -h
+ <<: *healthcheck-params
+ deploy:
+ <<: *resources-nano
+ depends_on:
+ dbrepo-data-db:
+ condition: service_healthy
+ dbrepo-logging-service:
+ condition: service_healthy
logging:
driver: fluentd
options:
@@ -182,6 +209,31 @@ services:
fluentd-address: 127.0.0.1:24224
tag: postgres.auth
+ dbrepo-dashboard-db:
+ restart: "no"
+ container_name: dbrepo-dashboard-db
+ hostname: dashboard-db
+ image: docker.io/bitnami/postgresql:${POSTGRES_VERSION}
+ volumes:
+ - dashboard-db-data:/bitnami/postgresql
+ depends_on:
+ dbrepo-logging-service:
+ condition: service_healthy
+ environment:
+ POSTGRESQL_DATABASE: "${DASHBOARD_DB_NAME:-grafana}"
+ POSTGRESQL_USERNAME: "${DASHBOARD_DB_USERNAME:-grafana}"
+ POSTGRESQL_PASSWORD: "${DASHBOARD_DB_PASSWORD:-dbrepo}"
+ healthcheck:
+ test: "psql -U ${DASHBOARD_DB_USERNAME:-grafana} -h 127.0.0.1 -p 5432 -d ${DASHBOARD_DB_NAME:-grafana} -c 'select version();'"
+ <<: *healthcheck-params
+ deploy:
+ <<: *resources-micro
+ logging:
+ driver: fluentd
+ options:
+ fluentd-address: 127.0.0.1:24224
+ tag: postgres.auth
+
dbrepo-auth-service:
restart: "no"
container_name: dbrepo-auth-service
@@ -729,9 +781,18 @@ services:
dbrepo-logging-service:
condition: service_healthy
volumes:
- - dashboard-ui-data:/opt/bitnami/grafana/data
+ - ./config/dashboards:/app/dashboards
+ - ./config/grafana.ini:/opt/bitnami/grafana/conf/grafana.ini
+ - ./config/ldap.toml:/opt/bitnami/grafana/conf/ldap.toml
+ - ./config/provisioning/dashboards/provider.yaml:/opt/bitnami/grafana/conf/provisioning/dashboards/provider.yaml
+ - ./config/provisioning/datasources/infinity.yaml:/opt/bitnami/grafana/conf/provisioning/datasources/infinity.yaml
+ - ./config/provisioning/datasources/prometheus.yaml:/opt/bitnami/grafana/conf/provisioning/datasources/prometheus.yaml
environment:
BASE_URL: "${BASE_URL:-http://localhost}"
+ # do not attempt to set it in the grafana.ini, hours wasted here: 7
+ DASHBOARD_DB_NAME: "${DASHBOARD_DB_NAME:-grafana}"
+ DASHBOARD_DB_USERNAME: "${DASHBOARD_DB_USERNAME:-grafana}"
+ DASHBOARD_DB_PASSWORD: "${DASHBOARD_DB_PASSWORD:-dbrepo}"
GF_SERVER_ROOT_URL: http://dashboard-ui:3000/dashboard/
GF_INSTALL_PLUGINS: "yesoreyeram-infinity-datasource,grafana-opensearch-datasource"
GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION: "true"
@@ -784,7 +845,7 @@ services:
restart: "no"
container_name: dbrepo-logging-service
hostname: logging-service
- image: docker.io/bitnami/fluent-bit:4.0.0
+ image: docker.io/bitnami/fluent-bit:${FLUENTBIT_VERSION}
ports:
- "24224:24224"
volumes:
diff --git a/docker-compose.yml b/docker-compose.yml
index bd5b6dbf9aa21b11063a514e94a06d13eb528e61..bf7dcd1cffb1dcc62de0fd583b1420c3a0082e7a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -562,6 +562,8 @@ services:
healthcheck:
test: wget -O- http://127.0.0.1:8080
<<: *healthcheck-params
+ deploy:
+ <<: *resources-nano
depends_on:
dbrepo-analyse-service:
condition: service_healthy
diff --git a/helm/dbrepo/files/system.json b/helm/dbrepo/files/dbrepo.json
similarity index 91%
rename from helm/dbrepo/files/system.json
rename to helm/dbrepo/files/dbrepo.json
index 7b59f987787fad8009432ca3c6c1549f401c8393..a4fd78436580cb1cea99475bb702fece82bee413 100644
--- a/helm/dbrepo/files/system.json
+++ b/helm/dbrepo/files/dbrepo.json
@@ -18,7 +18,7 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
- "id": 3,
+ "id": 7,
"links": [
{
"asDropdown": false,
@@ -42,6 +42,241 @@
"x": 0,
"y": 0
},
+ "id": 52,
+ "panels": [],
+ "title": "Storage",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dbrepometrics0"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "smooth",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 1
+ },
+ "id": 53,
+ "options": {
+ "alertThreshold": true,
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.5.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dbrepometrics0"
+ },
+ "expr": "sum(SeaweedFS_volumeServer_volumes) by (collection, type)",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 2,
+ "legendFormat": "{{collection}} {{type}}",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dbrepometrics0"
+ },
+ "expr": "sum(SeaweedFS_volumeServer_max_volumes)",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Total",
+ "refId": "B"
+ }
+ ],
+ "title": "Volume Count",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dbrepometrics0"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "smooth",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 1
+ },
+ "id": 54,
+ "options": {
+ "alertThreshold": true,
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.5.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dbrepometrics0"
+ },
+ "expr": "sum(SeaweedFS_volumeServer_total_disk_size) by (collection, type)",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 2,
+ "legendFormat": "{{collection}} {{type}}",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dbrepometrics0"
+ },
+ "expr": "sum(SeaweedFS_volumeServer_total_disk_size)",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Total",
+ "refId": "B"
+ }
+ ],
+ "title": "Used Disk Space by Collection and Type",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 9
+ },
"id": 34,
"panels": [],
"title": "tl;dr",
@@ -92,7 +327,7 @@
"h": 3,
"w": 4,
"x": 0,
- "y": 1
+ "y": 10
},
"id": 9,
"options": {
@@ -162,7 +397,7 @@
"h": 3,
"w": 4,
"x": 4,
- "y": 1
+ "y": 10
},
"id": 28,
"options": {
@@ -230,7 +465,7 @@
"h": 3,
"w": 4,
"x": 8,
- "y": 1
+ "y": 10
},
"id": 4,
"options": {
@@ -298,7 +533,7 @@
"h": 3,
"w": 4,
"x": 12,
- "y": 1
+ "y": 10
},
"id": 39,
"options": {
@@ -414,7 +649,7 @@
"h": 3,
"w": 4,
"x": 16,
- "y": 1
+ "y": 10
},
"id": 8,
"options": {
@@ -482,7 +717,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 4
+ "y": 13
},
"id": 38,
"options": {
@@ -538,15 +773,6 @@
"color": {
"mode": "thresholds"
},
- "custom": {
- "fillOpacity": 70,
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineWidth": 1
- },
"mappings": [
{
"options": {
@@ -582,23 +808,32 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 4
+ "y": 13
},
- "id": 16,
+ "id": 51,
"options": {
- "colWidth": 0.9,
+ "displayMode": "basic",
"legend": {
+ "calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
- "rowHeight": 0.9,
- "showValue": "auto",
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
+ "maxVizHeight": 300,
+ "minVizHeight": 16,
+ "minVizWidth": 8,
+ "namePlacement": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showUnfilled": true,
+ "sizing": "auto",
+ "valueMode": "color"
},
"pluginVersion": "11.5.3",
"targets": [
@@ -611,6 +846,7 @@
"editorMode": "builder",
"expr": "up",
"fullMetaSearch": false,
+ "hide": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "{{instance}}",
@@ -620,7 +856,7 @@
}
],
"title": "Service QoS",
- "type": "status-history"
+ "type": "bargauge"
},
{
"collapsed": false,
@@ -628,7 +864,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 12
+ "y": 21
},
"id": 6,
"panels": [],
@@ -744,7 +980,7 @@
"h": 8,
"w": 24,
"x": 0,
- "y": 13
+ "y": 22
},
"id": 42,
"options": {
@@ -911,7 +1147,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 21
+ "y": 30
},
"id": 43,
"options": {
@@ -991,7 +1227,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 21
+ "y": 30
},
"id": 44,
"options": {
@@ -1063,7 +1299,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 29
+ "y": 38
},
"id": 45,
"panels": [],
@@ -1084,7 +1320,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 30
+ "y": 39
},
"id": 46,
"options": {
@@ -1164,7 +1400,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 30
+ "y": 39
},
"id": 47,
"options": {
@@ -1349,10 +1585,10 @@
]
},
"gridPos": {
- "h": 7,
+ "h": 8,
"w": 12,
"x": 0,
- "y": 38
+ "y": 47
},
"id": 40,
"options": {
@@ -1397,7 +1633,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 45
+ "y": 55
},
"id": 5,
"panels": [],
@@ -1454,7 +1690,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 46
+ "y": 56
},
"id": 7,
"options": {
@@ -1693,7 +1929,7 @@
"h": 9,
"w": 24,
"x": 0,
- "y": 54
+ "y": 64
},
"id": 3,
"options": {
@@ -1854,7 +2090,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 63
+ "y": 73
},
"id": 22,
"panels": [],
@@ -1901,7 +2137,7 @@
"h": 3,
"w": 4,
"x": 0,
- "y": 64
+ "y": 74
},
"id": 17,
"options": {
@@ -1983,7 +2219,7 @@
"h": 3,
"w": 4,
"x": 4,
- "y": 64
+ "y": 74
},
"id": 24,
"options": {
@@ -2040,8 +2276,16 @@
"mode": "absolute",
"steps": [
{
- "color": "blue",
+ "color": "red",
"value": null
+ },
+ {
+ "color": "yellow",
+ "value": 1
+ },
+ {
+ "color": "green",
+ "value": 2
}
]
},
@@ -2053,9 +2297,9 @@
"h": 3,
"w": 4,
"x": 8,
- "y": 64
+ "y": 74
},
- "id": 25,
+ "id": 48,
"options": {
"colorMode": "background",
"graphMode": "none",
@@ -2110,8 +2354,12 @@
"mode": "absolute",
"steps": [
{
- "color": "blue",
+ "color": "red",
"value": null
+ },
+ {
+ "color": "green",
+ "value": 1
}
]
},
@@ -2123,9 +2371,9 @@
"h": 3,
"w": 4,
"x": 12,
- "y": 64
+ "y": 74
},
- "id": 26,
+ "id": 49,
"options": {
"colorMode": "background",
"graphMode": "none",
@@ -2182,7 +2430,7 @@
"mode": "absolute",
"steps": [
{
- "color": "blue",
+ "color": "green",
"value": null
}
]
@@ -2195,9 +2443,9 @@
"h": 3,
"w": 4,
"x": 16,
- "y": 64
+ "y": 74
},
- "id": 27,
+ "id": 50,
"options": {
"colorMode": "background",
"graphMode": "none",
@@ -2277,7 +2525,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 67
+ "y": 77
},
"id": 20,
"options": {
@@ -2391,7 +2639,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 67
+ "y": 77
},
"id": 21,
"options": {
@@ -2435,7 +2683,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 74
+ "y": 84
},
"id": 31,
"panels": [],
@@ -2486,7 +2734,7 @@
"h": 3,
"w": 4,
"x": 0,
- "y": 75
+ "y": 85
},
"id": 32,
"options": {
@@ -2556,7 +2804,7 @@
"h": 3,
"w": 4,
"x": 4,
- "y": 75
+ "y": 85
},
"id": 29,
"options": {
@@ -2626,7 +2874,7 @@
"h": 3,
"w": 4,
"x": 8,
- "y": 75
+ "y": 85
},
"id": 30,
"options": {
@@ -2708,7 +2956,7 @@
"h": 3,
"w": 4,
"x": 12,
- "y": 75
+ "y": 85
},
"id": 35,
"options": {
@@ -2790,7 +3038,7 @@
"h": 3,
"w": 4,
"x": 16,
- "y": 75
+ "y": 85
},
"id": 36,
"options": {
@@ -2872,7 +3120,7 @@
"h": 3,
"w": 4,
"x": 20,
- "y": 75
+ "y": 85
},
"id": 37,
"options": {
@@ -3007,7 +3255,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 78
+ "y": 88
},
"id": 33,
"options": {
@@ -3169,7 +3417,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 78
+ "y": 88
},
"id": 41,
"options": {
@@ -3214,7 +3462,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 85
+ "y": 95
},
"id": 2,
"panels": [],
@@ -3286,7 +3534,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 86
+ "y": 96
},
"id": 23,
"options": {
@@ -3329,6 +3577,7 @@
"refresh": "1m",
"schemaVersion": 40,
"tags": [
+ "provisioned",
"ui",
"dashboard",
"metadata",
@@ -3343,13 +3592,13 @@
"list": []
},
"time": {
- "from": "now-1h",
+ "from": "now-3h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "DBRepo",
"uid": "bdz20owu8zn5se2",
- "version": 45,
+ "version": 7,
"weekStart": ""
}
\ No newline at end of file
diff --git a/make/dev.mk b/make/dev.mk
index 81ce84db6a349e3411a0e3c979e8184a5a4eeeed..e580b178b37af91fb32cedf523478c473db4ba62 100644
--- a/make/dev.mk
+++ b/make/dev.mk
@@ -12,7 +12,7 @@ stop-dev: ## Stop the development deployment and remove all data.
.PHONY: package-config
package-config: ## Package the config files
- mkdir -p ./.docker/config/{dashboards,provisioning}
+ mkdir -p ./.docker/config
cp ./dbrepo-auth-service/dbrepo-realm.json ./.docker/config
cp ./dbrepo-auth-service/import-realms.sh ./.docker/config
cp ./dbrepo-auth-service/master-realm.json ./.docker/config
@@ -24,6 +24,10 @@ package-config: ## Package the config files
cp ./dbrepo-broker-service/definitions.json ./.docker/config
cp ./dbrepo-broker-service/advanced.config ./.docker/config
cp ./dbrepo-gateway-service/dbrepo.conf ./.docker/config
+ cp -r ./dbrepo-dashboard-ui/dashboards ./.docker/config
+ cp -r ./dbrepo-dashboard-ui/provisioning ./.docker/config
+ cp ./dbrepo-dashboard-ui/grafana.ini ./.docker/config/grafana.ini
+ cp ./dbrepo-dashboard-ui/ldap.toml ./.docker/config/ldap.toml
cp ./dbrepo-logging-service/fluentbit.conf ./.docker/config
cp ./dbrepo-logging-service/fluentbit_parser.conf ./.docker/config
cp ./dbrepo-metric-db/prometheus.yml ./.docker/config