Skip to content
Snippets Groups Projects

WIP

Merged WIP
524-logging-service into dev
Merged
Martin Weiserequested to merge
524-logging-service into dev
Viewing commit d7675d13
Prev Next
Show latest version
29 files
+ 444
− 1954
Compare changes
  • Side-by-side
  • Inline

Files

.docker/docker-compose.yml
+ 24
− 272
x-resources-nano: &resources-nano
resources:
reservations:
cpus: '0.1'
memory: 128M
limits:
cpus: '0.15'
memory: 192M
x-resources-micro: &resources-micro
resources:
reservations:
cpus: '0.25'
memory: 256M
limits:
cpus: '0.375'
memory: 384M
x-resources-small: &resources-small
resources:
reservations:
cpus: '0.5'
memory: 512M
limits:
cpus: '0.75'
memory: 768M
x-resources-medium: &resources-medium
resources:
reservations:
cpus: '0.5'
memory: 1024M
limits:
cpus: '0.75'
memory: 1536M
x-resources-large: &resources-large
resources:
reservations:
cpus: '1'
memory: 2048M
limits:
cpus: '1.5'
memory: 3072M
x-resources-xlarge: &resources-xlarge
resources:
reservations:
cpus: '1.5'
memory: 3072M
limits:
cpus: '2'
memory: 6144M
x-healthcheck-params: &healthcheck-params
interval: 30s
timeout: 10s
@@ -80,9 +26,6 @@ services:
- ./config/2_setup-data.sql:/docker-entrypoint-initdb.d/2_setup-data.sql
ports:
- "3306:3306"
depends_on:
dbrepo-logging-service:
condition: service_healthy
environment:
MARIADB_GALERA_MARIABACKUP_PASSWORD: "${METADATA_DB_BACKUP_PASSWORD:-dbrepo}"
MARIADB_DATABASE: "${METADATA_DB:-dbrepo}"
@@ -93,13 +36,8 @@ services:
healthcheck:
test: /opt/bitnami/scripts/mariadb-galera/healthcheck.sh --connect --innodb_initialized
<<: *healthcheck-params
deploy:
<<: *resources-xlarge
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: mariadb.metadata
driver: json-file
dbrepo-metadata-db-metrics:
restart: "no"
@@ -114,18 +52,11 @@ services:
healthcheck:
test: -h
<<: *healthcheck-params
deploy:
<<: *resources-nano
depends_on:
dbrepo-metadata-db:
condition: service_healthy
dbrepo-logging-service:
condition: service_healthy
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: exporter.metadata
driver: json-file
dbrepo-data-db:
restart: "no"
@@ -137,9 +68,6 @@ services:
- ./config/1_grant-user.sql:/docker-entrypoint-initdb.d/1_grant-user.sql
ports:
- "3307:3306"
depends_on:
dbrepo-logging-service:
condition: service_healthy
environment:
MARIADB_GALERA_MARIABACKUP_PASSWORD: "${DATA_DB_BACKUP_PASSWORD:-dbrepo}"
MARIADB_PASSWORD: "${READONLY_PASSWORD:-readonly}"
@@ -149,13 +77,8 @@ services:
healthcheck:
test: /opt/bitnami/scripts/mariadb-galera/healthcheck.sh --connect --innodb_initialized
<<: *healthcheck-params
deploy:
<<: *resources-xlarge
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: mariadb.data
driver: json-file
dbrepo-data-db-metrics:
restart: "no"
@@ -170,18 +93,11 @@ services:
healthcheck:
test: -h
<<: *healthcheck-params
deploy:
<<: *resources-nano
depends_on:
dbrepo-data-db:
condition: service_healthy
dbrepo-logging-service:
condition: service_healthy
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: mariadb.data
driver: json-file
dbrepo-auth-db:
restart: "no"
@@ -190,9 +106,6 @@ services:
image: docker.io/bitnami/postgresql:${POSTGRES_VERSION}
volumes:
- auth-db-data:/bitnami/postgresql
depends_on:
dbrepo-logging-service:
condition: service_healthy
environment:
POSTGRESQL_DATABASE: "${AUTH_DB_NAME:-keycloak}"
POSTGRESQL_USERNAME: "${AUTH_DB_USERNAME:-keycloak}"
@@ -201,13 +114,8 @@ services:
healthcheck:
test: "psql -U ${AUTH_DB_USERNAME:-keycloak} -h 127.0.0.1 -p 5432 -d ${AUTH_DB_NAME:-keycloak} -c 'select version();'"
<<: *healthcheck-params
deploy:
<<: *resources-micro
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: postgres.auth
driver: json-file
dbrepo-dashboard-db:
restart: "no"
@@ -216,9 +124,6 @@ services:
image: docker.io/bitnami/postgresql:${POSTGRES_VERSION}
volumes:
- dashboard-db-data:/bitnami/postgresql
depends_on:
dbrepo-logging-service:
condition: service_healthy
environment:
POSTGRESQL_DATABASE: "${DASHBOARD_DB_NAME:-grafana}"
POSTGRESQL_USERNAME: "${DASHBOARD_DB_USERNAME:-grafana}"
@@ -226,13 +131,8 @@ services:
healthcheck:
test: "psql -U ${DASHBOARD_DB_USERNAME:-grafana} -h 127.0.0.1 -p 5432 -d ${DASHBOARD_DB_NAME:-grafana} -c 'select version();'"
<<: *healthcheck-params
deploy:
<<: *resources-micro
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: postgres.auth
driver: json-file
dbrepo-auth-service:
restart: "no"
@@ -265,20 +165,13 @@ services:
healthcheck:
test: curl -fsS localhost:8080/realms/master
<<: *healthcheck-params
deploy:
<<: *resources-medium
depends_on:
dbrepo-identity-service:
condition: service_healthy
dbrepo-auth-db:
condition: service_healthy
dbrepo-logging-service:
condition: service_healthy
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: keycloak.auth
driver: json-file
dbrepo-auth-service-init:
init: true
@@ -293,8 +186,6 @@ services:
READONLY_USERNAME: "${READONLY_USERNAME:-readonly}"
SYSTEM_USERNAME: "${SYSTEM_USERNAME:-admin}"
SYSTEM_PASSWORD: ${SYSTEM_PASSWORD:-admin}
deploy:
<<: *resources-nano
depends_on:
dbrepo-auth-service:
condition: service_healthy
@@ -302,13 +193,8 @@ services:
condition: service_healthy
dbrepo-metadata-db:
condition: service_healthy
dbrepo-logging-service:
condition: service_healthy
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: python.auth
driver: json-file
dbrepo-metadata-service:
restart: "no"
@@ -356,8 +242,6 @@ services:
healthcheck:
test: curl -sSL localhost:8080/actuator/health/liveness | grep 'UP' || exit 1
<<: *healthcheck-params
deploy:
<<: *resources-small
depends_on:
dbrepo-auth-service:
condition: service_healthy
@@ -371,22 +255,14 @@ services:
condition: service_healthy
dbrepo-metadata-db:
condition: service_healthy
dbrepo-logging-service:
condition: service_healthy
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: java.metadata
driver: json-file
dbrepo-analyse-service:
restart: "no"
container_name: dbrepo-analyse-service
hostname: analyse-service
image: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.8.2
depends_on:
dbrepo-logging-service:
condition: service_healthy
environment:
AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client}
AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
@@ -403,13 +279,8 @@ services:
healthcheck:
test: curl -sSL localhost:8080/health | grep 'UP' || exit 1
<<: *healthcheck-params
deploy:
<<: *resources-micro
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: python.analyse
driver: json-file
dbrepo-broker-service:
restart: "no"
@@ -430,18 +301,11 @@ services:
depends_on:
dbrepo-identity-service:
condition: service_healthy
dbrepo-logging-service:
condition: service_healthy
healthcheck:
test: rabbitmq-diagnostics -q is_running | grep 'is fully booted and running'
<<: *healthcheck-params
deploy:
<<: *resources-micro
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: rabbitmq.broker
driver: json-file
dbrepo-search-db:
restart: "no"
@@ -455,8 +319,6 @@ services:
healthcheck:
test: curl -sSL 127.0.0.1:9200
<<: *healthcheck-params
deploy:
<<: *resources-medium
logging:
driver: json-file
@@ -465,9 +327,6 @@ services:
container_name: dbrepo-search-service
hostname: search-service
image: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.8.2
depends_on:
dbrepo-logging-service:
condition: service_healthy
environment:
AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client}
AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT_SECRET:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
@@ -482,13 +341,8 @@ services:
healthcheck:
test: curl -sSL localhost:8080/health | grep 'UP' || exit 1
<<: *healthcheck-params
deploy:
<<: *resources-micro
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: python.search
driver: json-file
dbrepo-ui:
restart: "no"
@@ -511,35 +365,26 @@ services:
healthcheck:
test: curl -fsSL 127.0.0.1:3000 && curl -fsSL 127.0.0.1:3000/health
<<: *healthcheck-params
deploy:
<<: *resources-micro
depends_on:
dbrepo-search-service:
condition: service_healthy
dbrepo-logging-service:
condition: service_healthy
extra_hosts:
- "localhost:host-gateway"
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: nuxt.ui
driver: json-file
dbrepo-gateway-service:
restart: "no"
container_name: dbrepo-gateway-service
hostname: gateway-service
image: docker.io/nginx:${NGINX_VERSION}
image: docker.io/bitnami/nginx:${NGINX_VERSION}
ports:
- "80:8080"
volumes:
- ./config/dbrepo.conf:/etc/nginx/conf.d/default.conf
- ./dbrepo-gateway-service/dbrepo.conf:/opt/bitnami/nginx/conf/server_blocks/dbrepo.conf:ro
healthcheck:
test: wget -O- http://127.0.0.1:8080
<<: *healthcheck-params
deploy:
<<: *resources-nano
depends_on:
dbrepo-analyse-service:
condition: service_healthy
@@ -555,22 +400,14 @@ services:
condition: service_healthy
dbrepo-ui:
condition: service_healthy
dbrepo-logging-service:
condition: service_healthy
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: nginx.gateway
driver: json-file
dbrepo-identity-service:
restart: "no"
container_name: dbrepo-identity-service
hostname: identity-service
image: docker.io/bitnami/openldap:${OPENLDAP_VERSION}
depends_on:
dbrepo-logging-service:
condition: service_healthy
environment:
LDAP_ADMIN_USERNAME: "${IDENTITY_SERVICE_ADMIN_USERNAME:-admin}"
LDAP_ADMIN_PASSWORD: "${IDENTITY_SERVICE_ADMIN_PASSWORD:-admin}"
@@ -584,13 +421,8 @@ services:
healthcheck:
test: "ldapwhoami -H ldap://localhost:1389 -D ${IDENTITY_SERVICE_ADMIN_DN:-cn=admin,dc=dbrepo,dc=at} -w ${IDENTITY_SERVICE_ADMIN_PASSWORD:-admin} || exit 1"
<<: *healthcheck-params
deploy:
<<: *resources-nano
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: openldap.identity
driver: json-file
dbrepo-search-service-init:
restart: "no"
@@ -605,20 +437,13 @@ services:
OPENSEARCH_PORT: ${OPENSEARCH_PORT:-9200}
SYSTEM_USERNAME: "${SYSTEM_USERNAME:-admin}"
SYSTEM_PASSWORD: "${SYSTEM_PASSWORD:-admin}"
deploy:
<<: *resources-nano
depends_on:
dbrepo-search-db:
condition: service_healthy
dbrepo-metadata-service:
condition: service_healthy
dbrepo-logging-service:
condition: service_healthy
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: python.search-init
driver: json-file
dbrepo-dashboard-service-init:
restart: "no"
@@ -632,20 +457,13 @@ services:
METADATA_SERVICE_ENDPOINT: ${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080}
SYSTEM_USERNAME: "${SYSTEM_USERNAME:-admin}"
SYSTEM_PASSWORD: "${SYSTEM_PASSWORD:-admin}"
deploy:
<<: *resources-nano
depends_on:
dbrepo-dashboard-ui:
condition: service_healthy
dbrepo-metadata-service:
condition: service_healthy
dbrepo-logging-service:
condition: service_healthy
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: python.dashboard-init
driver: json-file
dbrepo-storage-service:
restart: "no"
@@ -658,19 +476,11 @@ services:
ports:
- "9000:9000"
- "8888:8888"
depends_on:
dbrepo-logging-service:
condition: service_healthy
healthcheck:
test: echo "cluster.check" | weed shell | grep "checking master.*ok" || exit 1
<<: *healthcheck-params
deploy:
<<: *resources-nano
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: seaweedfs.storage
driver: json-file
dbrepo-metric-db:
restart: "no"
@@ -680,19 +490,11 @@ services:
volumes:
- ./config/prometheus.yml:/etc/prometheus/prometheus.yml
- metric-db-data:/opt/bitnami/prometheus/data
depends_on:
dbrepo-logging-service:
condition: service_healthy
healthcheck:
test: promtool check healthy
<<: *healthcheck-params
deploy:
<<: *resources-nano
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: prometheus.metric
driver: json-file
dbrepo-storage-service-init:
restart: "no"
@@ -705,18 +507,11 @@ services:
S3_BUCKET: "${S3_BUCKET:-dbrepo}"
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}
STORAGE_ENDPOINT: ${STORAGE_ENDPOINT:-http://storage-service:9000}
deploy:
<<: *resources-nano
depends_on:
dbrepo-storage-service:
condition: service_healthy
dbrepo-logging-service:
condition: service_healthy
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: python.storage-init
driver: json-file
dbrepo-data-service:
restart: "no"
@@ -757,18 +552,11 @@ services:
healthcheck:
test: curl -sSL localhost:8080/actuator/health/liveness | grep 'UP' || exit 1
<<: *healthcheck-params
deploy:
<<: *resources-large
depends_on:
dbrepo-data-db:
condition: service_healthy
dbrepo-logging-service:
condition: service_healthy
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: java.data
driver: json-file
dbrepo-dashboard-ui:
restart: "no"
@@ -777,9 +565,6 @@ services:
image: docker.io/bitnami/grafana:${GRAFANA_VERSION}
ports:
- "3000:3000"
depends_on:
dbrepo-logging-service:
condition: service_healthy
volumes:
- ./config/dashboards:/app/dashboards
- ./config/grafana.ini:/opt/bitnami/grafana/conf/grafana.ini
@@ -801,15 +586,10 @@ services:
healthcheck:
test: curl -fsSL --head 127.0.0.1:3000
<<: *healthcheck-params
deploy:
<<: *resources-nano
extra_hosts:
- "localhost:host-gateway"
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: opensearch.ui
driver: json-file
dbrepo-dashboard-service:
restart: "no"
@@ -827,36 +607,8 @@ services:
healthcheck:
test: curl -fsSL --head 127.0.0.1:8080/health
<<: *healthcheck-params
deploy:
<<: *resources-micro
depends_on:
dbrepo-dashboard-ui:
condition: service_healthy
dbrepo-logging-service:
condition: service_healthy
logging:
driver: fluentd
options:
fluentd-address: 127.0.0.1:24224
tag: python.dashboard
dbrepo-logging-service:
restart: "no"
container_name: dbrepo-logging-service
hostname: logging-service
image: docker.io/bitnami/fluent-bit:${FLUENTBIT_VERSION}
ports:
- "24224:24224"
volumes:
- ./config/fluentbit.conf:/opt/bitnami/fluent-bit/conf/fluent-bit.conf
- ./config/fluentbit_parser.conf:/opt/bitnami/fluent-bit/conf/dbrepo_parser.conf
healthcheck:
test: ps -p 1 | grep "fluent-bit"
<<: *healthcheck-params
deploy:
<<: *resources-micro
depends_on:
dbrepo-search-db:
condition: service_healthy
logging:
driver: json-file