diff --git a/helm/dbrepo/Chart.lock b/helm/dbrepo/Chart.lock index 50562acfcba6deb9cc874e003179a6c1458879e7..354e41f3a4d585da81aa4b874b0795a59d860b8b 100644 --- a/helm/dbrepo/Chart.lock +++ b/helm/dbrepo/Chart.lock @@ -29,5 +29,8 @@ dependencies: - name: postgresql repository: https://charts.bitnami.com/bitnami version: 16.1.0 -digest: sha256:fd6fb7a9547e6bb1c7d311b67dae7a3987247e16256ef3b553593ddd90b91340 -generated: "2025-05-03T19:27:21.68619261+02:00" +- name: fluent-bit + repository: https://charts.bitnami.com/bitnami + version: 3.1.1 +digest: sha256:4e9c46db1248aedb0121621d771ded45fdcc230a5cd72781fd36f5c6a58a508a +generated: "2025-05-04T11:12:25.048191538+02:00" diff --git a/helm/dbrepo/Chart.yaml b/helm/dbrepo/Chart.yaml index ee659eb5386b3cdabee8c3f6ab20ea38c5a071db..2f60045e5961cbf0290fe26e0e4a9cb5e5c88734 100644 --- a/helm/dbrepo/Chart.yaml +++ b/helm/dbrepo/Chart.yaml @@ -66,4 +66,9 @@ dependencies: alias: dashboarddb version: 16.1.0 repository: https://charts.bitnami.com/bitnami - condition: dashboarddb.enabled \ No newline at end of file + condition: dashboarddb.enabled + - name: fluent-bit + alias: loggingservice + version: 3.1.1 + repository: https://charts.bitnami.com/bitnami + condition: loggingservice.enabled \ No newline at end of file diff --git a/helm/dbrepo/charts/fluent-bit-3.1.1.tgz b/helm/dbrepo/charts/fluent-bit-3.1.1.tgz new file mode 100644 index 0000000000000000000000000000000000000000..23b16fdd48284203390a057ec1dafaf1f3cfa3af Binary files /dev/null and b/helm/dbrepo/charts/fluent-bit-3.1.1.tgz differ diff --git a/helm/dbrepo/charts/seaweedfs-4.2.1.tgz b/helm/dbrepo/charts/seaweedfs-4.2.1.tgz index 9c5c79a5d2e6d591b59287f2f278c064e1ab9982..6101f07e64e7e9515cba0bf545fde7dc25662c98 100644 Binary files a/helm/dbrepo/charts/seaweedfs-4.2.1.tgz and b/helm/dbrepo/charts/seaweedfs-4.2.1.tgz differ diff --git a/helm/dbrepo/templates/logging-configmap.yaml b/helm/dbrepo/templates/logging-configmap.yaml deleted file mode 100644 index f4c088642367fafebffce22b7d5c110d272cf728..0000000000000000000000000000000000000000 --- a/helm/dbrepo/templates/logging-configmap.yaml +++ /dev/null @@ -1,124 +0,0 @@ -{{- if .Values.loggingservice.enabled }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: logging-service-config - namespace: {{ include "common.names.namespace" . | quote }} -data: - dbrepo_parser.conf: |- - [PARSER] - # https://rubular.com/r/78ieBhDKvlzPnW - Name java - Format regex - Regex (?<time>[^ ]* {1,2}[^ ]*) \[(?<thread>.*)\] *(?<level>[^ ]*) (?<logger>.*) - (?<message>.*)$ - Time_Key time - Time_Format %Y-%m-%d %H:%M:%S,%L - Time_Keep On - - [PARSER] - # https://rubular.com/r/oTnpOflRX0HwaG - Name quarkus - Format regex - Regex ^(?<time>[^ ]* {1,2}[^ ]*) (?<level>[^ ]*).*\[(?<module>.*)\].*\(executor-thread-(?<executor>[0-9]+)\) (?<message>.*)$ - Time_Key time - Time_Format %Y-%m-%d %H:%M:%S,%L - Time_Keep On - - [PARSER] - # https://rubular.com/r/JqOp74qykhKeBJ - Name gunicorn - Format regex - Regex \[(?<time>[^ ]* {1,2}[^ ]*)\] \[(?<level>[a-zA-Z]+)\] *(?<message>.*)$ - Time_Key time - Time_Format %Y-%m-%d %H:%M:%S %z - Time_Keep On - - [PARSER] - # https://rubular.com/r/bqLY5HuZfP07p3 - Name seaweedfs - Format regex - Regex (?<i>I[0-9]*) (?<time>[^ ]*) (?<golocation>[a-z0-9_\.:]*) (?<message>.*) - Time_Key time - Time_Format %H:%M:%S.%L - Time_Keep On - - [PARSER] - # https://rubular.com/r/SMa7y7BTOOgf9z - Name openldap - Format regex - Regex (?<rnd>[^ ]*) (?<register>[^ ]*) conn=(?<conn>[0-9]+) op=(?<op>[^ ]*) (?<message>.*) - Time_Key time - Time_Format %H:%M:%S.%L - Time_Keep On - fluentbit.conf: |- - [SERVICE] - Flush 5 - Daemon Off - Log_Level debug - Parsers_File parsers.conf - Parsers_File dbrepo_parser.conf - - [INPUT] - Name forward - Listen 0.0.0.0 - Port 24224 - - [FILTER] - Name parser - Parser nginx - Match nginx.* - Key_Name log - Reserve_Data On - Preserve_Key On - - [FILTER] - Name parser - Parser java - Match java.* - Key_Name log - Reserve_Data On - Preserve_Key On - - [FILTER] - Name parser - Parser gunicorn - Match python.* - Key_Name log - Reserve_Data On - Preserve_Key On - - [FILTER] - Name parser - Parser seaweedfs - Match seaweedfs.* - Key_Name log - Reserve_Data On - Preserve_Key On - - [FILTER] - Name parser - Parser openldap - Match openldap.* - Key_Name log - Reserve_Data On - Preserve_Key On - - [FILTER] - Name parser - Parser quarkus - Match keycloak.* - Key_Name log - Reserve_Data On - Preserve_Key On - - [OUTPUT] - Name opensearch - Match * - Host search-db - Port 9200 - Index logging - Replace_Dots On - Include_Tag_Key On - Suppress_Type_Name On -{{- end }} diff --git a/helm/dbrepo/templates/logging-deployment.yaml b/helm/dbrepo/templates/logging-deployment.yaml deleted file mode 100644 index 80b93a4053896df409ce0edf9a0897933f08fbc2..0000000000000000000000000000000000000000 --- a/helm/dbrepo/templates/logging-deployment.yaml +++ /dev/null @@ -1,69 +0,0 @@ -{{- if .Values.loggingservice.enabled }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: logging-service - namespace: {{ include "common.names.namespace" . | quote }} - labels: - app: logging-service - service: logging-service -spec: - replicas: {{ .Values.loggingservice.replicaCount }} - strategy: - type: {{ .Values.strategyType }} - selector: - matchLabels: - app: logging-service - service: logging-service - template: - metadata: - labels: - app: logging-service - service: logging-service - spec: - {{- if .Values.loggingservice.podSecurityContext.enabled }} - securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.loggingservice.podSecurityContext "context" $) | nindent 8 }} - {{- end }} - containers: - - name: logging-service - image: {{ .Values.loggingservice.image.name }} - imagePullPolicy: {{ .Values.loggingservice.image.pullPolicy | default "IfNotPresent" }} - {{- if .Values.loggingservice.containerSecurityContext.enabled }} - securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.loggingservice.containerSecurityContext "context" $) | nindent 12 }} - {{- end }} - ports: - - containerPort: 24224 - protocol: TCP - - containerPort: 24224 - protocol: UDP - volumeMounts: - - mountPath: /opt/bitnami/fluent-bit/conf/ - name: config - readOnly: true - livenessProbe: - exec: - command: - - /bin/bash - - -ec - - "ps -p 1 | grep 'fluent-bit'" - initialDelaySeconds: 150 - periodSeconds: 10 - readinessProbe: - exec: - command: - - /bin/bash - - -ec - - "ps -p 1 | grep 'fluent-bit'" - initialDelaySeconds: 30 - periodSeconds: 10 - {{- if .Values.loggingservice.resources }} - resources: {{- toYaml .Values.loggingservice.resources | nindent 12 }} - {{- else if ne .Values.loggingservice.resourcesPreset "none" }} - resources: {{- include "common.resources.preset" (dict "type" .Values.loggingservice.resourcesPreset) | nindent 12 }} - {{- end }} - volumes: - - name: config - configMap: - name: logging-service-config -{{- end }} diff --git a/helm/dbrepo/values.yaml b/helm/dbrepo/values.yaml index e2498a2305fcc0f386cdb7f8743c79a6875f57a5..5de4adfa858f9411d7ba3cee96e1f593c57a187a 100644 --- a/helm/dbrepo/values.yaml +++ b/helm/dbrepo/values.yaml @@ -1082,6 +1082,7 @@ metricdb: - dashboard-db-metrics:9187 - dashboard-service - data-db-metrics:9104 + - logging-service:2021 - metadata-db-metrics:9104 - search-service - storage-service-master-metrics:9327 @@ -1121,54 +1122,143 @@ computeservice: loggingservice: ## @param loggingservice.enabled Enable the Logging Service. enabled: true - image: - ## @skip loggingservice.image.name - name: docker.io/bitnami/fluent-bit:4.0.0 - podSecurityContext: - ## @param loggingservice.podSecurityContext.enabled Enable pods' Security Context + ## @skip loggingservice.fullnameOverride + fullnameOverride: logging-service + daemonset: enabled: true - ## @param loggingservice.podSecurityContext.fsGroupChangePolicy Set filesystem group change policy - fsGroupChangePolicy: Always - ## @param loggingservice.podSecurityContext.sysctls Set kernel settings using the sysctl interface - sysctls: [ ] - ## @param loggingservice.podSecurityContext.supplementalGroups Set filesystem extra groups - supplementalGroups: [ ] - ## @param loggingservice.podSecurityContext.fsGroup Set RabbitMQ pod's Security Context fsGroup - fsGroup: 0 - containerSecurityContext: - ## @param loggingservice.containerSecurityContext.enabled Enable containers' Security Context - enabled: true - ## @param loggingservice.containerSecurityContext.seLinuxOptions Set SELinux options in container - seLinuxOptions: { } - ## @param loggingservice.containerSecurityContext.runAsUser Set RabbitMQ containers' Security Context runAsUser - runAsUser: 1001 - ## @param loggingservice.containerSecurityContext.runAsGroup Set RabbitMQ containers' Security Context runAsGroup - runAsGroup: 1001 - ## @param loggingservice.containerSecurityContext.runAsNonRoot Set RabbitMQ container's Security Context runAsNonRoot - runAsNonRoot: true - ## @param loggingservice.containerSecurityContext.allowPrivilegeEscalation Set container's privilege escalation - allowPrivilegeEscalation: false - ## @param loggingservice.containerSecurityContext.readOnlyRootFilesystem Set container's Security Context readOnlyRootFilesystem - readOnlyRootFilesystem: false - capabilities: - ## @param loggingservice.containerSecurityContext.capabilities.drop Set container's Security Context runAsNonRoot - drop: [ "ALL" ] - seccompProfile: - ## @param loggingservice.containerSecurityContext.seccompProfile.type Set container's Security Context seccomp profile - type: "RuntimeDefault" - ## @param loggingservice.resourcesPreset The container resource preset - resourcesPreset: "micro" - ## @param loggingservice.resources Set container requests and limits for different resources like CPU or memory (essential for production workloads) - resources: { } - ## requests: - ## cpu: 250m - ## memory: 64Mi - ## limits: - ## cpu: 500m - ## memory: 1024Mi - ## @param loggingservice.replicaCount The number of replicas. - replicaCount: 2 + config: + ## @skip loggingservice.config.inputs + inputs: |- + [INPUT] + Name forward + Listen 0.0.0.0 + Port 24224 + + [INPUT] + name fluentbit_metrics + tag internal_metrics + scrape_interval 2 + ## @skip loggingservice.config.filters + filters: |- + [FILTER] + Name parser + Parser nginx + Match nginx.* + Key_Name log + Reserve_Data On + Preserve_Key On + + [FILTER] + Name parser + Parser java + Match java.* + Key_Name log + Reserve_Data On + Preserve_Key On + + [FILTER] + Name parser + Parser gunicorn + Match python.* + Key_Name log + Reserve_Data On + Preserve_Key On + + [FILTER] + Name parser + Parser seaweedfs + Match seaweedfs.* + Key_Name log + Reserve_Data On + Preserve_Key On + + [FILTER] + Name parser + Parser openldap + Match openldap.* + Key_Name log + Reserve_Data On + Preserve_Key On + [FILTER] + Name parser + Parser quarkus + Match keycloak.* + Key_Name log + Reserve_Data On + Preserve_Key On + ## @skip loggingservice.config.outputs + outputs: |- + [OUTPUT] + Name opensearch + Match * + Host search-db + Port 9200 + Index logging + Replace_Dots On + Include_Tag_Key On + Suppress_Type_Name On + + [OUTPUT] + name prometheus_exporter + match internal_metrics + host 0.0.0.0 + port 2021 + ## @skip loggingservice.config.customParsers + customParsers: |- + [PARSER] + # https://rubular.com/r/78ieBhDKvlzPnW + Name java + Format regex + Regex (?<time>[^ ]* {1,2}[^ ]*) \[(?<thread>.*)\] *(?<level>[^ ]*) (?<logger>.*) - (?<message>.*)$ + Time_Key time + Time_Format %Y-%m-%d %H:%M:%S,%L + Time_Keep On + + [PARSER] + # https://rubular.com/r/oTnpOflRX0HwaG + Name quarkus + Format regex + Regex ^(?<time>[^ ]* {1,2}[^ ]*) (?<level>[^ ]*).*\[(?<module>.*)\].*\(executor-thread-(?<executor>[0-9]+)\) (?<message>.*)$ + Time_Key time + Time_Format %Y-%m-%d %H:%M:%S,%L + Time_Keep On + + [PARSER] + # https://rubular.com/r/JqOp74qykhKeBJ + Name gunicorn + Format regex + Regex \[(?<time>[^ ]* {1,2}[^ ]*)\] \[(?<level>[a-zA-Z]+)\] *(?<message>.*)$ + Time_Key time + Time_Format %Y-%m-%d %H:%M:%S %z + Time_Keep On + + [PARSER] + # https://rubular.com/r/bqLY5HuZfP07p3 + Name seaweedfs + Format regex + Regex (?<i>I[0-9]*) (?<time>[^ ]*) (?<golocation>[a-z0-9_\.:]*) (?<message>.*) + Time_Key time + Time_Format %H:%M:%S.%L + Time_Keep On + + [PARSER] + # https://rubular.com/r/SMa7y7BTOOgf9z + Name openldap + Format regex + Regex (?<rnd>[^ ]*) (?<register>[^ ]*) conn=(?<conn>[0-9]+) op=(?<op>[^ ]*) (?<message>.*) + Time_Key time + Time_Format %H:%M:%S.%L + Time_Keep On + extraContainerPorts: + - name: metrics + containerPort: 2021 + service: + extraPorts: + - name: metrics + port: 2021 + targetPort: 2021 + ## @section Ingress