Skip to content
Snippets Groups Projects
Commit fda6016b authored by Martin Weise's avatar Martin Weise
Browse files

Dev

parent f84ed5a5
Branches
Tags
1 merge request!325Dev
Showing
with 9711 additions and 63 deletions
File deleted
...@@ -309,6 +309,8 @@ services: ...@@ -309,6 +309,8 @@ services:
condition: service_healthy condition: service_healthy
dbrepo-broker-service: dbrepo-broker-service:
condition: service_healthy condition: service_healthy
dbrepo-dashboard-service:
condition: service_healthy
dbrepo-metadata-service: dbrepo-metadata-service:
condition: service_healthy condition: service_healthy
dbrepo-search-db: dbrepo-search-db:
...@@ -363,7 +365,7 @@ services: ...@@ -363,7 +365,7 @@ services:
container_name: dbrepo-storage-service container_name: dbrepo-storage-service
hostname: storage-service hostname: storage-service
image: docker.io/chrislusf/seaweedfs:3.59 image: docker.io/chrislusf/seaweedfs:3.59
command: [ "server", "-dir=/data", "-s3", "-s3.port=9000", "-s3.config=/app/s3_config.json", "-metricsPort=9091" ] command: [ "server", "-dir=/data", "-s3", "-s3.port=9000", "-s3.config=/app/s3_config.json", "-metricsPort=9090" ]
volumes: volumes:
- ./config/s3_config.json:/app/s3_config.json - ./config/s3_config.json:/app/s3_config.json
- storage-service-data:/data - storage-service-data:/data
...@@ -377,6 +379,49 @@ services: ...@@ -377,6 +379,49 @@ services:
logging: logging:
driver: json-file driver: json-file
dbrepo-metric-db:
restart: "no"
container_name: dbrepo-metric-db
hostname: metric-db
image: bitnami/prometheus:2.54.1-debian-12-r4
volumes:
- ./config/prometheus.yml:/etc/prometheus/prometheus.yml
healthcheck:
test: promtool check healthy
interval: 10s
timeout: 5s
retries: 12
logging:
driver: json-file
dbrepo-dashboard-service:
restart: "no"
container_name: dbrepo-dashboard-service
hostname: dashboard-service
image: docker.io/bitnami/grafana:11.2.0-debian-12-r4
volumes:
- ./config/dashboards:/app/dashboards
- ./config/provisioning:/etc/grafana/provisioning
- ./config/grafana.ini:/etc/grafana/grafana.ini
- ./config/ldap.toml:/etc/grafana/ldap.toml
environment:
GF_SERVER_DOMAIN: "dashboard-service"
GF_SERVER_ROOT_URL: "${BASE_URL:-http://localhost}/dashboard/"
GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION: "true"
LDAP_ADMIN_USERNAME: "${IDENTITY_SERVICE_ADMIN_USERNAME:-admin}"
LDAP_ADMIN_PASSWORD: "${IDENTITY_SERVICE_ADMIN_PASSWORD:-admin}"
LDAP_ROOT: "${IDENTITY_SERVICE_ROOT:-dc=dbrepo,dc=at}"
healthcheck:
test: test -f /opt/bitnami/grafana/tmp/grafana.pid
interval: 10s
timeout: 5s
retries: 12
depends_on:
dbrepo-metric-db:
condition: service_started
logging:
driver: json-file
dbrepo-storage-service-init: dbrepo-storage-service-init:
restart: "no" restart: "no"
container_name: dbrepo-storage-service-init container_name: dbrepo-storage-service-init
......
---
author: Martin Weise
---
## tl;dr
!!! debug "Debug Information"
Image: [`docker.io/bitnami/grafana:11.2.0-debian-12-r4`](https://hub.docker.com/r/bitnami/grafana)
* Ports: `http://<hostname>/dashboard`
* Prometheus: `http://<hostname>/dashboard/prometheus`
## Overview
The Dashboard Service is visualizing the status of DBRepo with charts. The default dashboard provisioner located in
`/etc/grafana/provisioning/dashboards/provider.yaml` checks for new `JSON` dashboard files in `/app/dashboards` every 10
seconds and makes the available in the Dashboard Service.
<figure markdown>
![DBRepo Dashboard](../images/screenshots/grafana4.png)
<figcaption>Figure 1: DBRepo Dashboard</figcaption>
</figure>
<figure markdown>
![MariaDB Galera Dashboard](../images/screenshots/grafana3.png)
<figcaption>Figure 2: Database Dashboard</figcaption>
</figure>
<figure markdown>
![RabbitMQ Dashboard](../images/screenshots/grafana5.png)
<figcaption>Figure 3: Broker Service Dashboard</figcaption>
</figure>
## Limitations
!!! question "Do you miss functionality? Do these limitations affect you?"
We strongly encourage you to help us implement it as we are welcoming contributors to open-source software and get
in [contact](../../contact) with us, we happily answer requests for collaboration with attached CV and your programming
experience!
## Security
(none)
---
author: Martin Weise
---
## tl;dr
!!! debug "Debug Information"
Image: [`bitnami/prometheus:2.54.1-debian-12-r4`](https://hub.docker.com/r/bitnami/prometheus)
## Overview
The Metric Database is responsible for saving time-series data for the [Dashboard Service](../../api/dashboard-service)
## Limitations
!!! question "Do you miss functionality? Do these limitations affect you?"
We strongly encourage you to help us implement it as we are welcoming contributors to open-source software and get
in [contact](../../contact) with us, we happily answer requests for collaboration with attached CV and your programming
experience!
## Security
(none)
File moved
...@@ -64,7 +64,12 @@ The [Data Database Sidecar](#) metrics are: ...@@ -64,7 +64,12 @@ The [Data Database Sidecar](#) metrics are:
## Metadata Service ## Metadata Service
| **Metric** | **Description** | | **Metric** | **Description** |
|------------------------------------|-----------------------------------| |------------------------------------|---------------------------------------------------|
| `dbrepo_database_count` | The total number of managed research databases |
| `dbrepo_view_count` | The total number of available view data sources |
| `dbrepo_subset_count` | The total number of available subset data sources |
| `dbrepo_table_count` | The total number of available table data sources |
| `dbrepo_volume_sum` | The total volume of available research data |
| `dbrepo_user_refresh_token` | Refresh user token | | `dbrepo_user_refresh_token` | Refresh user token |
| `dbrepo_identifier_save` | Save identifier | | `dbrepo_identifier_save` | Save identifier |
| `dbrepo_oai_record_get` | Get the record | | `dbrepo_oai_record_get` | Get the record |
......
.docs/images/screenshots/grafana3.png

110 KiB

.docs/images/screenshots/grafana4.png

150 KiB

.docs/images/screenshots/grafana5.png

212 KiB

...@@ -156,6 +156,39 @@ build-helm: ...@@ -156,6 +156,39 @@ build-helm:
- apk add sed helm curl - apk add sed helm curl
- helm package ./helm/dbrepo --destination ./build - helm package ./helm/dbrepo --destination ./build
lint-docker-compose:
image: docker.io/alpine:3.18
stage: lint
variables:
VERSION: 3.3.0
BINARY: yq_linux_amd64
before_script:
- 'apk --no-cache add bash wget'
- 'wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /usr/bin/yq && chmod +x /usr/bin/yq'
- 'ls -la .scripts'
script:
- "yq compare -P docker-compose.yml .docker/docker-compose.yml 'volumes.*'"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-analyse-service'"
- "bash .scripts/check-service.sh 'dbrepo-auth-db'"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-auth-service'"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-broker-service'"
- "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-dashboard-service'"
- "bash .scripts/check-service.sh 'dbrepo-data-db'"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-data-db-sidecar'"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-data-service'"
- "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-gateway-service'"
- "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-identity-service'"
- "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-metadata-db'"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-metadata-service'"
- "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-metric-db'"
- "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-search-db'"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-search-service'"
- "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-search-service-init'"
- "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-storage-service'"
- "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-storage-service-init'"
- "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-ui'"
- "bash .scripts/check-service.sh 'dbrepo-upload-service'"
verify-install-script: verify-install-script:
image: docker.io/docker:24-dind image: docker.io/docker:24-dind
stage: verify stage: verify
...@@ -401,12 +434,13 @@ release-docs: ...@@ -401,12 +434,13 @@ release-docs:
- "pip install -r ./requirements.txt" - "pip install -r ./requirements.txt"
- "mkdir -p ./final/${APP_VERSION}/rest" - "mkdir -p ./final/${APP_VERSION}/rest"
script: script:
- "make gen-lib-doc gen-docs-doc" - "make gen-lib-doc gen-docs-doc package-config"
- "cp -r ./lib/python/docs/build/html ./final/${APP_VERSION}/python" # sphinx - "cp -r ./lib/python/docs/build/html ./final/${APP_VERSION}/python" # sphinx
- "cp .docs/.swagger/api.yaml ./final/${APP_VERSION}/rest/api.yaml" # swagger - "cp .docs/.swagger/api.yaml ./final/${APP_VERSION}/rest/api.yaml" # swagger
- "cp .docs/.swagger/swagger-ui.html ./final/${APP_VERSION}/rest/index.html" # swagger - "cp .docs/.swagger/swagger-ui.html ./final/${APP_VERSION}/rest/index.html" # swagger
- "cp .docs/.swagger/custom.css ./final/${APP_VERSION}/rest/custom.css" # swagger - "cp .docs/.swagger/custom.css ./final/${APP_VERSION}/rest/custom.css" # swagger
- "cp -r ./site/* ./final/${APP_VERSION}" # mkdocs - "cp -r ./site/* ./final/${APP_VERSION}" # mkdocs
- "cp .docker/dist.tar.gz ./final/${APP_VERSION}/dist.tar.gz" # dist
- eval $(ssh-agent -s) - eval $(ssh-agent -s)
- "mkdir -p /root/.ssh" - "mkdir -p /root/.ssh"
- echo "$CI_KEY_PRIVATE" > /root/.ssh/id_rsa && chmod 0600 /root/.ssh/id_rsa - echo "$CI_KEY_PRIVATE" > /root/.ssh/id_rsa && chmod 0600 /root/.ssh/id_rsa
......
#!/bin/bash
yq compare -P docker-compose.yml .docker/docker-compose.yml "services.$1.restart"
yq compare -P docker-compose.yml .docker/docker-compose.yml "services.$1.container_name"
yq compare -P docker-compose.yml .docker/docker-compose.yml "services.$1.hostname"
if [ -z "$IGNORE_IMAGE" ]; then
yq compare -P docker-compose.yml .docker/docker-compose.yml "services.$1.image"
fi
if [ -z "$IGNORE_VOLUMES" ]; then
yq compare -P docker-compose.yml .docker/docker-compose.yml "services.$1.volumes"
fi
if [ -z "$IGNORE_PORTS" ]; then
yq compare -P docker-compose.yml .docker/docker-compose.yml "services.$1.ports"
fi
yq compare -P docker-compose.yml .docker/docker-compose.yml "services.$1.environment"
yq compare -P docker-compose.yml .docker/docker-compose.yml "services.$1.healthcheck"
yq compare -P docker-compose.yml .docker/docker-compose.yml "services.$1.logging"
\ No newline at end of file
...@@ -16,7 +16,6 @@ version: ## Get current version. ...@@ -16,7 +16,6 @@ version: ## Get current version.
@echo $(APP_VERSION) @echo $(APP_VERSION)
include make/build.mk include make/build.mk
include make/dep.mk
include make/dev.mk include make/dev.mk
include make/gen.mk include make/gen.mk
include make/rel.mk include make/rel.mk
......
#!/bin/bash #!/bin/bash
bash /app/disable-tls.sh & bash /app/disable-tls.sh &
/opt/keycloak/bin/kc.sh start-dev --import-realm /opt/keycloak/bin/kc.sh start-dev --import-realm --metrics-enabled=true
FROM docker.io/bitnami/grafana:11.2.0-debian-12-r4 AS runtime
LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at"
WORKDIR /app
COPY ./grafana.ini /etc/grafana/grafana.ini
COPY ldap.toml /etc/grafana/ldap.toml
COPY provisioning/datasources/prometheus.yaml /etc/grafana/provisioning/datasources/datasource.yaml
#COPY --chown=999:999 ./system.yaml /var/lib/grafana/dashboards/system.yaml
This diff is collapsed.
This diff is collapsed.
[server]
http_port = 3000
[security]
disable_initial_admin_creation = true
[auth.ldap]
enabled = true
config_file = /etc/grafana/ldap.toml
[auth.anonymous]
enabled = true
org_role = Viewer
hide_version = true
[dashboards]
enabled = true
path = /app/dashboards
#[log]
#filters = ldap:trace
\ No newline at end of file
[[servers]]
host = "identity-service"
port = 1389
use_ssl = false
# Search user bind dn
bind_dn = "cn=${LDAP_ADMIN_USERNAME},${LDAP_ROOT}"
bind_password = "${LDAP_ADMIN_PASSWORD}"
# Timeout in seconds. Applies to each host specified in the 'host' entry (space separated).
timeout = 10
# User search filter, for example "(cn=%s)" or "(sAMAccountName=%s)" or "(uid=%s)"
# Allow login from email or username, example "(|(sAMAccountName=%s)(userPrincipalName=%s))"
search_filter = "(cn=%s)"
# An array of base dns to search through
search_base_dns = ["${LDAP_ROOT}"]
group_search_base_dns = ["ou=users,${LDAP_ROOT}"]
group_search_filter = "(&(objectClass=groupOfNames)(member=cn=%s,ou=users,${LDAP_ROOT}))"
group_search_filter_user_attribute = "uid"
[servers.attributes]
name = "givenName"
surname = "sn"
username = "cn"
member_of = "member"
email = "email"
[[servers.group_mappings]]
group_dn = "cn=${LDAP_ADMIN_USERNAME},ou=users,${LDAP_ROOT}"
org_role = "Admin"
grafana_admin = true
[[servers.group_mappings]]
group_dn = "*"
org_role = "Viewer"
apiVersion: 1
providers:
# <string> an unique provider name. Required
- name: 'dbrepo'
# <int> Org id. Default to 1
orgId: 1
# <string> name of the dashboard folder.
folder: '/app/dashboards'
# <string> folder UID. will be automatically generated if not specified
folderUid: ''
# <string> provider type. Default to 'file'
type: file
# <bool> disable dashboard deletion
disableDeletion: false
# <int> how often Grafana will scan for changed dashboards
updateIntervalSeconds: 10
# <bool> allow updating provisioned dashboards from the UI
allowUiUpdates: false
options:
# <string, required> path to dashboard files on disk. Required when using the 'file' type
path: /app/dashboards
# <bool> use folder names from filesystem to create folders in Grafana
foldersFromFilesStructure: true
\ No newline at end of file
apiVersion: 1
datasources:
- name: dbrepo-metric-db
type: prometheus
uid: P18F45E9DC7E75912
url: http://metric-db:9090
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment