Skip to content
Snippets Groups Projects
Commit 9d57ce7e authored by Martin Weise's avatar Martin Weise
Browse files

Merge branch 'dev' into 'master'

Dev

See merge request !325
parents f84ed5a5 fda6016b
Branches
Tags
1 merge request!325Dev
Showing
with 9711 additions and 63 deletions
File deleted
......@@ -309,6 +309,8 @@ services:
condition: service_healthy
dbrepo-broker-service:
condition: service_healthy
dbrepo-dashboard-service:
condition: service_healthy
dbrepo-metadata-service:
condition: service_healthy
dbrepo-search-db:
......@@ -363,7 +365,7 @@ services:
container_name: dbrepo-storage-service
hostname: storage-service
image: docker.io/chrislusf/seaweedfs:3.59
command: [ "server", "-dir=/data", "-s3", "-s3.port=9000", "-s3.config=/app/s3_config.json", "-metricsPort=9091" ]
command: [ "server", "-dir=/data", "-s3", "-s3.port=9000", "-s3.config=/app/s3_config.json", "-metricsPort=9090" ]
volumes:
- ./config/s3_config.json:/app/s3_config.json
- storage-service-data:/data
......@@ -377,6 +379,49 @@ services:
logging:
driver: json-file
dbrepo-metric-db:
restart: "no"
container_name: dbrepo-metric-db
hostname: metric-db
image: bitnami/prometheus:2.54.1-debian-12-r4
volumes:
- ./config/prometheus.yml:/etc/prometheus/prometheus.yml
healthcheck:
test: promtool check healthy
interval: 10s
timeout: 5s
retries: 12
logging:
driver: json-file
dbrepo-dashboard-service:
restart: "no"
container_name: dbrepo-dashboard-service
hostname: dashboard-service
image: docker.io/bitnami/grafana:11.2.0-debian-12-r4
volumes:
- ./config/dashboards:/app/dashboards
- ./config/provisioning:/etc/grafana/provisioning
- ./config/grafana.ini:/etc/grafana/grafana.ini
- ./config/ldap.toml:/etc/grafana/ldap.toml
environment:
GF_SERVER_DOMAIN: "dashboard-service"
GF_SERVER_ROOT_URL: "${BASE_URL:-http://localhost}/dashboard/"
GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION: "true"
LDAP_ADMIN_USERNAME: "${IDENTITY_SERVICE_ADMIN_USERNAME:-admin}"
LDAP_ADMIN_PASSWORD: "${IDENTITY_SERVICE_ADMIN_PASSWORD:-admin}"
LDAP_ROOT: "${IDENTITY_SERVICE_ROOT:-dc=dbrepo,dc=at}"
healthcheck:
test: test -f /opt/bitnami/grafana/tmp/grafana.pid
interval: 10s
timeout: 5s
retries: 12
depends_on:
dbrepo-metric-db:
condition: service_started
logging:
driver: json-file
dbrepo-storage-service-init:
restart: "no"
container_name: dbrepo-storage-service-init
......
---
author: Martin Weise
---
## tl;dr
!!! debug "Debug Information"
Image: [`docker.io/bitnami/grafana:11.2.0-debian-12-r4`](https://hub.docker.com/r/bitnami/grafana)
* Ports: `http://<hostname>/dashboard`
* Prometheus: `http://<hostname>/dashboard/prometheus`
## Overview
The Dashboard Service is visualizing the status of DBRepo with charts. The default dashboard provisioner located in
`/etc/grafana/provisioning/dashboards/provider.yaml` checks for new `JSON` dashboard files in `/app/dashboards` every 10
seconds and makes the available in the Dashboard Service.
<figure markdown>
![DBRepo Dashboard](../images/screenshots/grafana4.png)
<figcaption>Figure 1: DBRepo Dashboard</figcaption>
</figure>
<figure markdown>
![MariaDB Galera Dashboard](../images/screenshots/grafana3.png)
<figcaption>Figure 2: Database Dashboard</figcaption>
</figure>
<figure markdown>
![RabbitMQ Dashboard](../images/screenshots/grafana5.png)
<figcaption>Figure 3: Broker Service Dashboard</figcaption>
</figure>
## Limitations
!!! question "Do you miss functionality? Do these limitations affect you?"
We strongly encourage you to help us implement it as we are welcoming contributors to open-source software and get
in [contact](../../contact) with us, we happily answer requests for collaboration with attached CV and your programming
experience!
## Security
(none)
---
author: Martin Weise
---
## tl;dr
!!! debug "Debug Information"
Image: [`bitnami/prometheus:2.54.1-debian-12-r4`](https://hub.docker.com/r/bitnami/prometheus)
## Overview
The Metric Database is responsible for saving time-series data for the [Dashboard Service](../../api/dashboard-service)
## Limitations
!!! question "Do you miss functionality? Do these limitations affect you?"
We strongly encourage you to help us implement it as we are welcoming contributors to open-source software and get
in [contact](../../contact) with us, we happily answer requests for collaboration with attached CV and your programming
experience!
## Security
(none)
File moved
......@@ -64,7 +64,12 @@ The [Data Database Sidecar](#) metrics are:
## Metadata Service
| **Metric** | **Description** |
|------------------------------------|-----------------------------------|
|------------------------------------|---------------------------------------------------|
| `dbrepo_database_count` | The total number of managed research databases |
| `dbrepo_view_count` | The total number of available view data sources |
| `dbrepo_subset_count` | The total number of available subset data sources |
| `dbrepo_table_count` | The total number of available table data sources |
| `dbrepo_volume_sum` | The total volume of available research data |
| `dbrepo_user_refresh_token` | Refresh user token |
| `dbrepo_identifier_save` | Save identifier |
| `dbrepo_oai_record_get` | Get the record |
......
.docs/images/screenshots/grafana3.png

110 KiB

.docs/images/screenshots/grafana4.png

150 KiB

.docs/images/screenshots/grafana5.png

212 KiB

......@@ -156,6 +156,39 @@ build-helm:
- apk add sed helm curl
- helm package ./helm/dbrepo --destination ./build
lint-docker-compose:
image: docker.io/alpine:3.18
stage: lint
variables:
VERSION: 3.3.0
BINARY: yq_linux_amd64
before_script:
- 'apk --no-cache add bash wget'
- 'wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /usr/bin/yq && chmod +x /usr/bin/yq'
- 'ls -la .scripts'
script:
- "yq compare -P docker-compose.yml .docker/docker-compose.yml 'volumes.*'"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-analyse-service'"
- "bash .scripts/check-service.sh 'dbrepo-auth-db'"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-auth-service'"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-broker-service'"
- "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-dashboard-service'"
- "bash .scripts/check-service.sh 'dbrepo-data-db'"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-data-db-sidecar'"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-data-service'"
- "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-gateway-service'"
- "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-identity-service'"
- "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-metadata-db'"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-metadata-service'"
- "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-metric-db'"
- "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-search-db'"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-search-service'"
- "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-search-service-init'"
- "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-storage-service'"
- "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-storage-service-init'"
- "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-ui'"
- "bash .scripts/check-service.sh 'dbrepo-upload-service'"
verify-install-script:
image: docker.io/docker:24-dind
stage: verify
......@@ -401,12 +434,13 @@ release-docs:
- "pip install -r ./requirements.txt"
- "mkdir -p ./final/${APP_VERSION}/rest"
script:
- "make gen-lib-doc gen-docs-doc"
- "make gen-lib-doc gen-docs-doc package-config"
- "cp -r ./lib/python/docs/build/html ./final/${APP_VERSION}/python" # sphinx
- "cp .docs/.swagger/api.yaml ./final/${APP_VERSION}/rest/api.yaml" # swagger
- "cp .docs/.swagger/swagger-ui.html ./final/${APP_VERSION}/rest/index.html" # swagger
- "cp .docs/.swagger/custom.css ./final/${APP_VERSION}/rest/custom.css" # swagger
- "cp -r ./site/* ./final/${APP_VERSION}" # mkdocs
- "cp .docker/dist.tar.gz ./final/${APP_VERSION}/dist.tar.gz" # dist
- eval $(ssh-agent -s)
- "mkdir -p /root/.ssh"
- echo "$CI_KEY_PRIVATE" > /root/.ssh/id_rsa && chmod 0600 /root/.ssh/id_rsa
......
#!/bin/bash
yq compare -P docker-compose.yml .docker/docker-compose.yml "services.$1.restart"
yq compare -P docker-compose.yml .docker/docker-compose.yml "services.$1.container_name"
yq compare -P docker-compose.yml .docker/docker-compose.yml "services.$1.hostname"
if [ -z "$IGNORE_IMAGE" ]; then
yq compare -P docker-compose.yml .docker/docker-compose.yml "services.$1.image"
fi
if [ -z "$IGNORE_VOLUMES" ]; then
yq compare -P docker-compose.yml .docker/docker-compose.yml "services.$1.volumes"
fi
if [ -z "$IGNORE_PORTS" ]; then
yq compare -P docker-compose.yml .docker/docker-compose.yml "services.$1.ports"
fi
yq compare -P docker-compose.yml .docker/docker-compose.yml "services.$1.environment"
yq compare -P docker-compose.yml .docker/docker-compose.yml "services.$1.healthcheck"
yq compare -P docker-compose.yml .docker/docker-compose.yml "services.$1.logging"
\ No newline at end of file
......@@ -16,7 +16,6 @@ version: ## Get current version.
@echo $(APP_VERSION)
include make/build.mk
include make/dep.mk
include make/dev.mk
include make/gen.mk
include make/rel.mk
......
#!/bin/bash
bash /app/disable-tls.sh &
/opt/keycloak/bin/kc.sh start-dev --import-realm
/opt/keycloak/bin/kc.sh start-dev --import-realm --metrics-enabled=true
FROM docker.io/bitnami/grafana:11.2.0-debian-12-r4 AS runtime
LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at"
WORKDIR /app
COPY ./grafana.ini /etc/grafana/grafana.ini
COPY ldap.toml /etc/grafana/ldap.toml
COPY provisioning/datasources/prometheus.yaml /etc/grafana/provisioning/datasources/datasource.yaml
#COPY --chown=999:999 ./system.yaml /var/lib/grafana/dashboards/system.yaml
This diff is collapsed.
This diff is collapsed.
[server]
http_port = 3000
[security]
disable_initial_admin_creation = true
[auth.ldap]
enabled = true
config_file = /etc/grafana/ldap.toml
[auth.anonymous]
enabled = true
org_role = Viewer
hide_version = true
[dashboards]
enabled = true
path = /app/dashboards
#[log]
#filters = ldap:trace
\ No newline at end of file
[[servers]]
host = "identity-service"
port = 1389
use_ssl = false
# Search user bind dn
bind_dn = "cn=${LDAP_ADMIN_USERNAME},${LDAP_ROOT}"
bind_password = "${LDAP_ADMIN_PASSWORD}"
# Timeout in seconds. Applies to each host specified in the 'host' entry (space separated).
timeout = 10
# User search filter, for example "(cn=%s)" or "(sAMAccountName=%s)" or "(uid=%s)"
# Allow login from email or username, example "(|(sAMAccountName=%s)(userPrincipalName=%s))"
search_filter = "(cn=%s)"
# An array of base dns to search through
search_base_dns = ["${LDAP_ROOT}"]
group_search_base_dns = ["ou=users,${LDAP_ROOT}"]
group_search_filter = "(&(objectClass=groupOfNames)(member=cn=%s,ou=users,${LDAP_ROOT}))"
group_search_filter_user_attribute = "uid"
[servers.attributes]
name = "givenName"
surname = "sn"
username = "cn"
member_of = "member"
email = "email"
[[servers.group_mappings]]
group_dn = "cn=${LDAP_ADMIN_USERNAME},ou=users,${LDAP_ROOT}"
org_role = "Admin"
grafana_admin = true
[[servers.group_mappings]]
group_dn = "*"
org_role = "Viewer"
apiVersion: 1
providers:
# <string> an unique provider name. Required
- name: 'dbrepo'
# <int> Org id. Default to 1
orgId: 1
# <string> name of the dashboard folder.
folder: '/app/dashboards'
# <string> folder UID. will be automatically generated if not specified
folderUid: ''
# <string> provider type. Default to 'file'
type: file
# <bool> disable dashboard deletion
disableDeletion: false
# <int> how often Grafana will scan for changed dashboards
updateIntervalSeconds: 10
# <bool> allow updating provisioned dashboards from the UI
allowUiUpdates: false
options:
# <string, required> path to dashboard files on disk. Required when using the 'file' type
path: /app/dashboards
# <bool> use folder names from filesystem to create folders in Grafana
foldersFromFilesStructure: true
\ No newline at end of file
apiVersion: 1
datasources:
- name: dbrepo-metric-db
type: prometheus
uid: P18F45E9DC7E75912
url: http://metric-db:9090
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment