Skip to content
Snippets Groups Projects
Verified Commit 99184a81 authored by Martin Weise's avatar Martin Weise
Browse files

WIP 


Signed-off-by: Martin Weise's avatarMartin Weise <martin.weise@tuwien.ac.at>
parent 6d9ae4ce
No related branches found
No related tags found
1 merge request!411WIP
Show whitespace changes
Inline Side-by-side
helm/dbrepo/Chart.lock
+ 5
− 2
View file @ 99184a81
...@@ -29,5 +29,8 @@ dependencies: ...@@ -29,5 +29,8 @@ dependencies:
- name: postgresql - name: postgresql
repository: https://charts.bitnami.com/bitnami repository: https://charts.bitnami.com/bitnami
version: 16.1.0 version: 16.1.0
digest: sha256:fd6fb7a9547e6bb1c7d311b67dae7a3987247e16256ef3b553593ddd90b91340 - name: fluent-bit
generated: "2025-05-03T19:27:21.68619261+02:00" repository: https://charts.bitnami.com/bitnami
version: 3.1.1
digest: sha256:4e9c46db1248aedb0121621d771ded45fdcc230a5cd72781fd36f5c6a58a508a
generated: "2025-05-04T11:12:25.048191538+02:00"
helm/dbrepo/Chart.yaml
+ 6
− 1
View file @ 99184a81
...@@ -67,3 +67,8 @@ dependencies: ...@@ -67,3 +67,8 @@ dependencies:
version: 16.1.0 version: 16.1.0
repository: https://charts.bitnami.com/bitnami repository: https://charts.bitnami.com/bitnami
condition: dashboarddb.enabled condition: dashboarddb.enabled
- name: fluent-bit
alias: loggingservice
version: 3.1.1
repository: https://charts.bitnami.com/bitnami
condition: loggingservice.enabled
\ No newline at end of file
helm/dbrepo/charts/fluent-bit-3.1.1.tgz 0 → 100644
+ 0
− 0
View file @ 99184a81
File added
helm/dbrepo/charts/seaweedfs-4.2.1.tgz
+ 0
− 0
View file @ 99184a81
No preview for this file type
helm/dbrepo/templates/logging-configmap.yaml deleted 100644 → 0
+ 0
− 124
View file @ 6d9ae4ce
{{- if .Values.loggingservice.enabled }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: logging-service-config
namespace: {{ include "common.names.namespace" . | quote }}
data:
dbrepo_parser.conf: |-
[PARSER]
# https://rubular.com/r/78ieBhDKvlzPnW
Name java
Format regex
Regex (?<time>[^ ]* {1,2}[^ ]*) \[(?<thread>.*)\] *(?<level>[^ ]*) (?<logger>.*) - (?<message>.*)$
Time_Key time
Time_Format %Y-%m-%d %H:%M:%S,%L
Time_Keep On
[PARSER]
# https://rubular.com/r/oTnpOflRX0HwaG
Name quarkus
Format regex
Regex ^(?<time>[^ ]* {1,2}[^ ]*) (?<level>[^ ]*).*\[(?<module>.*)\].*\(executor-thread-(?<executor>[0-9]+)\) (?<message>.*)$
Time_Key time
Time_Format %Y-%m-%d %H:%M:%S,%L
Time_Keep On
[PARSER]
# https://rubular.com/r/JqOp74qykhKeBJ
Name gunicorn
Format regex
Regex \[(?<time>[^ ]* {1,2}[^ ]*)\] \[(?<level>[a-zA-Z]+)\] *(?<message>.*)$
Time_Key time
Time_Format %Y-%m-%d %H:%M:%S %z
Time_Keep On
[PARSER]
# https://rubular.com/r/bqLY5HuZfP07p3
Name seaweedfs
Format regex
Regex (?<i>I[0-9]*) (?<time>[^ ]*) (?<golocation>[a-z0-9_\.:]*) (?<message>.*)
Time_Key time
Time_Format %H:%M:%S.%L
Time_Keep On
[PARSER]
# https://rubular.com/r/SMa7y7BTOOgf9z
Name openldap
Format regex
Regex (?<rnd>[^ ]*) (?<register>[^ ]*) conn=(?<conn>[0-9]+) op=(?<op>[^ ]*) (?<message>.*)
Time_Key time
Time_Format %H:%M:%S.%L
Time_Keep On
fluentbit.conf: |-
[SERVICE]
Flush 5
Daemon Off
Log_Level debug
Parsers_File parsers.conf
Parsers_File dbrepo_parser.conf
[INPUT]
Name forward
Listen 0.0.0.0
Port 24224
[FILTER]
Name parser
Parser nginx
Match nginx.*
Key_Name log
Reserve_Data On
Preserve_Key On
[FILTER]
Name parser
Parser java
Match java.*
Key_Name log
Reserve_Data On
Preserve_Key On
[FILTER]
Name parser
Parser gunicorn
Match python.*
Key_Name log
Reserve_Data On
Preserve_Key On
[FILTER]
Name parser
Parser seaweedfs
Match seaweedfs.*
Key_Name log
Reserve_Data On
Preserve_Key On
[FILTER]
Name parser
Parser openldap
Match openldap.*
Key_Name log
Reserve_Data On
Preserve_Key On
[FILTER]
Name parser
Parser quarkus
Match keycloak.*
Key_Name log
Reserve_Data On
Preserve_Key On
[OUTPUT]
Name opensearch
Match *
Host search-db
Port 9200
Index logging
Replace_Dots On
Include_Tag_Key On
Suppress_Type_Name On
{{- end }}
helm/dbrepo/templates/logging-deployment.yaml deleted 100644 → 0
+ 0
− 69
View file @ 6d9ae4ce
{{- if .Values.loggingservice.enabled }}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: logging-service
namespace: {{ include "common.names.namespace" . | quote }}
labels:
app: logging-service
service: logging-service
spec:
replicas: {{ .Values.loggingservice.replicaCount }}
strategy:
type: {{ .Values.strategyType }}
selector:
matchLabels:
app: logging-service
service: logging-service
template:
metadata:
labels:
app: logging-service
service: logging-service
spec:
{{- if .Values.loggingservice.podSecurityContext.enabled }}
securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.loggingservice.podSecurityContext "context" $) | nindent 8 }}
{{- end }}
containers:
- name: logging-service
image: {{ .Values.loggingservice.image.name }}
imagePullPolicy: {{ .Values.loggingservice.image.pullPolicy | default "IfNotPresent" }}
{{- if .Values.loggingservice.containerSecurityContext.enabled }}
securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.loggingservice.containerSecurityContext "context" $) | nindent 12 }}
{{- end }}
ports:
- containerPort: 24224
protocol: TCP
- containerPort: 24224
protocol: UDP
volumeMounts:
- mountPath: /opt/bitnami/fluent-bit/conf/
name: config
readOnly: true
livenessProbe:
exec:
command:
- /bin/bash
- -ec
- "ps -p 1 | grep 'fluent-bit'"
initialDelaySeconds: 150
periodSeconds: 10
readinessProbe:
exec:
command:
- /bin/bash
- -ec
- "ps -p 1 | grep 'fluent-bit'"
initialDelaySeconds: 30
periodSeconds: 10
{{- if .Values.loggingservice.resources }}
resources: {{- toYaml .Values.loggingservice.resources | nindent 12 }}
{{- else if ne .Values.loggingservice.resourcesPreset "none" }}
resources: {{- include "common.resources.preset" (dict "type" .Values.loggingservice.resourcesPreset) | nindent 12 }}
{{- end }}
volumes:
- name: config
configMap:
name: logging-service-config
{{- end }}
helm/dbrepo/values.yaml
+ 136
− 46
View file @ 99184a81
...@@ -1082,6 +1082,7 @@ metricdb: ...@@ -1082,6 +1082,7 @@ metricdb:
- dashboard-db-metrics:9187 - dashboard-db-metrics:9187
- dashboard-service - dashboard-service
- data-db-metrics:9104 - data-db-metrics:9104
- logging-service:2021
- metadata-db-metrics:9104 - metadata-db-metrics:9104
- search-service - search-service
- storage-service-master-metrics:9327 - storage-service-master-metrics:9327
...@@ -1121,53 +1122,142 @@ computeservice: ...@@ -1121,53 +1122,142 @@ computeservice:
loggingservice: loggingservice:
## @param loggingservice.enabled Enable the Logging Service. ## @param loggingservice.enabled Enable the Logging Service.
enabled: true enabled: true
image: ## @skip loggingservice.fullnameOverride
## @skip loggingservice.image.name fullnameOverride: logging-service
name: docker.io/bitnami/fluent-bit:4.0.0 daemonset:
podSecurityContext:
## @param loggingservice.podSecurityContext.enabled Enable pods' Security Context
enabled: true
## @param loggingservice.podSecurityContext.fsGroupChangePolicy Set filesystem group change policy
fsGroupChangePolicy: Always
## @param loggingservice.podSecurityContext.sysctls Set kernel settings using the sysctl interface
sysctls: [ ]
## @param loggingservice.podSecurityContext.supplementalGroups Set filesystem extra groups
supplementalGroups: [ ]
## @param loggingservice.podSecurityContext.fsGroup Set RabbitMQ pod's Security Context fsGroup
fsGroup: 0
containerSecurityContext:
## @param loggingservice.containerSecurityContext.enabled Enable containers' Security Context
enabled: true enabled: true
## @param loggingservice.containerSecurityContext.seLinuxOptions Set SELinux options in container config:
seLinuxOptions: { } ## @skip loggingservice.config.inputs
## @param loggingservice.containerSecurityContext.runAsUser Set RabbitMQ containers' Security Context runAsUser inputs: |-
runAsUser: 1001 [INPUT]
## @param loggingservice.containerSecurityContext.runAsGroup Set RabbitMQ containers' Security Context runAsGroup Name forward
runAsGroup: 1001 Listen 0.0.0.0
## @param loggingservice.containerSecurityContext.runAsNonRoot Set RabbitMQ container's Security Context runAsNonRoot Port 24224
runAsNonRoot: true
## @param loggingservice.containerSecurityContext.allowPrivilegeEscalation Set container's privilege escalation [INPUT]
allowPrivilegeEscalation: false name fluentbit_metrics
## @param loggingservice.containerSecurityContext.readOnlyRootFilesystem Set container's Security Context readOnlyRootFilesystem tag internal_metrics
readOnlyRootFilesystem: false scrape_interval 2
capabilities: ## @skip loggingservice.config.filters
## @param loggingservice.containerSecurityContext.capabilities.drop Set container's Security Context runAsNonRoot filters: |-
drop: [ "ALL" ] [FILTER]
seccompProfile: Name parser
## @param loggingservice.containerSecurityContext.seccompProfile.type Set container's Security Context seccomp profile Parser nginx
type: "RuntimeDefault" Match nginx.*
## @param loggingservice.resourcesPreset The container resource preset Key_Name log
resourcesPreset: "micro" Reserve_Data On
## @param loggingservice.resources Set container requests and limits for different resources like CPU or memory (essential for production workloads) Preserve_Key On
resources: { }
## requests: [FILTER]
## cpu: 250m Name parser
## memory: 64Mi Parser java
## limits: Match java.*
## cpu: 500m Key_Name log
## memory: 1024Mi Reserve_Data On
## @param loggingservice.replicaCount The number of replicas. Preserve_Key On
replicaCount: 2
[FILTER]
Name parser
Parser gunicorn
Match python.*
Key_Name log
Reserve_Data On
Preserve_Key On
[FILTER]
Name parser
Parser seaweedfs
Match seaweedfs.*
Key_Name log
Reserve_Data On
Preserve_Key On
[FILTER]
Name parser
Parser openldap
Match openldap.*
Key_Name log
Reserve_Data On
Preserve_Key On
[FILTER]
Name parser
Parser quarkus
Match keycloak.*
Key_Name log
Reserve_Data On
Preserve_Key On
## @skip loggingservice.config.outputs
outputs: |-
[OUTPUT]
Name opensearch
Match *
Host search-db
Port 9200
Index logging
Replace_Dots On
Include_Tag_Key On
Suppress_Type_Name On
[OUTPUT]
name prometheus_exporter
match internal_metrics
host 0.0.0.0
port 2021
## @skip loggingservice.config.customParsers
customParsers: |-
[PARSER]
# https://rubular.com/r/78ieBhDKvlzPnW
Name java
Format regex
Regex (?<time>[^ ]* {1,2}[^ ]*) \[(?<thread>.*)\] *(?<level>[^ ]*) (?<logger>.*) - (?<message>.*)$
Time_Key time
Time_Format %Y-%m-%d %H:%M:%S,%L
Time_Keep On
[PARSER]
# https://rubular.com/r/oTnpOflRX0HwaG
Name quarkus
Format regex
Regex ^(?<time>[^ ]* {1,2}[^ ]*) (?<level>[^ ]*).*\[(?<module>.*)\].*\(executor-thread-(?<executor>[0-9]+)\) (?<message>.*)$
Time_Key time
Time_Format %Y-%m-%d %H:%M:%S,%L
Time_Keep On
[PARSER]
# https://rubular.com/r/JqOp74qykhKeBJ
Name gunicorn
Format regex
Regex \[(?<time>[^ ]* {1,2}[^ ]*)\] \[(?<level>[a-zA-Z]+)\] *(?<message>.*)$
Time_Key time
Time_Format %Y-%m-%d %H:%M:%S %z
Time_Keep On
[PARSER]
# https://rubular.com/r/bqLY5HuZfP07p3
Name seaweedfs
Format regex
Regex (?<i>I[0-9]*) (?<time>[^ ]*) (?<golocation>[a-z0-9_\.:]*) (?<message>.*)
Time_Key time
Time_Format %H:%M:%S.%L
Time_Keep On
[PARSER]
# https://rubular.com/r/SMa7y7BTOOgf9z
Name openldap
Format regex
Regex (?<rnd>[^ ]*) (?<register>[^ ]*) conn=(?<conn>[0-9]+) op=(?<op>[^ ]*) (?<message>.*)
Time_Key time
Time_Format %H:%M:%S.%L
Time_Keep On
extraContainerPorts:
- name: metrics
containerPort: 2021
service:
extraPorts:
- name: metrics
port: 2021
targetPort: 2021
## @section Ingress ## @section Ingress
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment