WIP

Signed-off-by: Martin Weise <martin.weise@tuwien.ac.at>

helm/dbrepo/templates/data-deployment.yaml

@@ -18,6 +18,13 @@ spec:
       service: data-service
   template:
     metadata:
+      annotations:
+        {{- if .Values.loggingservice.enabled }}
+        fluentbit.io/parser: java
+        {{- end }}
+        {{- if .Values.dataservice.podAnnotations }}
+        {{- include "common.tplvalues.render" (dict "value" .Values.dataservice.podAnnotations "context" $) | nindent 8 }}
+        {{- end }}
       labels:
         app: data-service
         service: data-service

helm/dbrepo/templates/identity-statefulset.yaml

@@ -16,6 +16,13 @@ spec:
       service: identity-service
   template:
     metadata:
+      annotations:
+        {{- if .Values.loggingservice.enabled }}
+        fluentbit.io/parser: openldap
+        {{- end }}
+        {{- if .Values.identityservice.podAnnotations }}
+        {{- include "common.tplvalues.render" (dict "value" .Values.identityservice.podAnnotations "context" $) | nindent 8 }}
+        {{- end }}
       labels:
         app: identity-service
         service: identity-service

helm/dbrepo/templates/logging-service-configmap.yaml

+{{- if .Values.loggingservice.enabled }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: logging-service-config
+  namespace: {{ include "common.names.namespace" . | quote }}
+data:
+  custom_parsers.conf: |-
+    [PARSER]
+        # https://rubular.com/r/dyenlhs3wJioGo
+        Name        java
+        Format      regex
+        Regex       ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*).*\[(?<thread>.*)\] *(?<level>[^ ]*) (?<logger>.*) - (?<message>.*)$
+        Time_Key    time
+        Time_Format %Y-%m-%d %H:%M:%S,%L
+        Time_Keep   On
+
+    [PARSER]
+        # https://rubular.com/r/N30SNcHJbh30yY
+        Name        quarkus
+        Format      regex
+        Regex       ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*).* (?<level>[^ ]+).*\[(?<module>.*)\] \((?<executor>[^ \)]+)\) (?<message>.*)$
+        Time_Key    time
+        Time_Format %Y-%m-%d %H:%M:%S,%L
+        Time_Keep   On
+
+    [PARSER]
+        # https://rubular.com/r/sUgLA4QAdPtOqW
+        Name        gunicorn
+        Format      regex
+        Regex       ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*).* \[(?<level>[a-zA-Z]+)\] *(?<message>.*)$
+        Time_Key    time
+        Time_Format %Y-%m-%d %H:%M:%S %z
+        Time_Keep   On
+
+    [PARSER]
+        # https://rubular.com/r/bqLY5HuZfP07p3
+        Name        seaweedfs
+        Format      regex
+        Regex       (?<i>I[0-9]*) (?<time>[^ ]*) (?<golocation>[a-z0-9_\.:]*) (?<message>.*)
+        Time_Key    time
+        Time_Format %H:%M:%S.%L
+        Time_Keep   On
+
+    [PARSER]
+        # https://rubular.com/r/SMa7y7BTOOgf9z
+        Name        openldap
+        Format      regex
+        Regex       (?<rnd>[^ ]*) (?<register>[^ ]*) conn=(?<conn>[0-9]+) op=(?<op>[^ ]*) (?<message>.*)
+        Time_Key    time
+        Time_Format %H:%M:%S.%L
+        Time_Keep   On
+
+    [PARSER]
+        # https://rubular.com/r/FqZiF1s7b64rQf
+        Name        prometheus
+        Format      regex
+        Regex       ts=(?<time>[^ ]+).*level=(?<level>[^ ]+).*msg="(?<message>.*)"
+        Time_Key    time
+        Time_Format %H:%M:%S.%L
+        Time_Keep   On
+
+    [PARSER]
+        # https://rubular.com/r/NZVtoH0VAUBp65
+        Name        grafana
+        Format      regex
+        Regex       t=(?<time>[^ ]+).*level=(?<level>[^ ]+).*msg="(?<message>.*)"
+        Time_Key    time
+        Time_Format %H:%M:%S.%L
+        Time_Keep   On
+
+    [PARSER]
+        # https://rubular.com/r/3SSbts0642zCZp
+        Name        rabbitmq
+        Format      regex
+        Regex       (?<time>[^ ]*) \[(?<level>[^\]]+)\] \\u[a-f0-9]+(?<started>[0-9]+\.[0-9]+\.[0-9]+)\\u[a-f0-9]+ (?<message>.*)$
+        Time_Key    time
+        Time_Format %H:%M:%S.%L
+        Time_Keep   On
+
+    [PARSER]
+        Name        k8s-custom
+        Format      regex
+        Regex       (?<namespace_name>[^.]+)\.(?<pod_name>[^.]+)\.(?<container_name>[^.]+)$
+  fluent-bit.conf: |-
+    [SERVICE]
+        Flush                       1
+        Daemon                      Off
+        Log_Level                   info
+        Config_Watch                On
+        HTTP_Server                 On
+        HTTP_Listen                 0.0.0.0
+        HTTP_Port                   2020
+        Parsers_File                /opt/bitnami/fluent-bit/conf/parsers.conf
+        Parsers_File                /opt/bitnami/fluent-bit/conf/custom_parsers.conf
+
+    [INPUT]
+        Name                        tail
+        Path                        /var/log/containers/*_{{ .Release.Namespace }}_*.log
+        Tag                         kube.<namespace_name>.<pod_name>.<container_name>
+        Tag_Regex                   (?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<container_id>[a-z0-9]{64})\.log$
+        multiline.parser            cri
+
+    [INPUT]
+        name                        fluentbit_metrics
+        tag                         internal_metrics
+        scrape_interval             2
+
+    [FILTER]
+        Name                        kubernetes
+        Match                       kube.*
+        Kube_Tag_Prefix             kube.
+        Regex_Parser                k8s-custom
+        K8S-Logging.Parser          On
+        Merge_Log                   On
+
+    [OUTPUT]
+        Name                        opensearch
+        Match                       *
+        Host                        search-db
+        Port                        9200
+        Index                       logging
+        Replace_Dots                On
+        Include_Tag_Key             On
+        Suppress_Type_Name          On
+
+    [OUTPUT]
+        name                        prometheus_exporter
+        match                       internal_metrics
+        host                        0.0.0.0
+        port                        2021
+{{- end }}

helm/dbrepo/templates/metadata-deployment.yaml

@@ -18,6 +18,13 @@ spec:
       service: metadata-service
   template:
     metadata:
+      annotations:
+        {{- if .Values.loggingservice.enabled }}
+        fluentbit.io/parser: java
+        {{- end }}
+        {{- if .Values.metadataservice.podAnnotations }}
+        {{- include "common.tplvalues.render" (dict "value" .Values.metadataservice.podAnnotations "context" $) | nindent 8 }}
+        {{- end }}
       labels:
         app: metadata-service
         service: metadata-service

helm/dbrepo/templates/search-deployment.yaml

@@ -18,6 +18,13 @@ spec:
       service: search-service
   template:
     metadata:
+      annotations:
+        {{- if .Values.loggingservice.enabled }}
+        fluentbit.io/parser: gunicorn
+        {{- end }}
+        {{- if .Values.searchservice.podAnnotations }}
+        {{- include "common.tplvalues.render" (dict "value" .Values.searchservice.podAnnotations "context" $) | nindent 8 }}
+        {{- end }}
       labels:
         app: search-service
         service: search-service

helm/dbrepo/values.yaml

@@ -270,6 +270,9 @@ brokerservice:
   image:
     ## @param brokerservice.image.debug Set the logging level to `trace`. Otherwise, set to `info`.
     debug: true
+  ## @skip brokerservice.podAnnotations
+  podAnnotations:
+    fluentbit.io/parser: rabbitmq
   metrics:
     ## @skip brokerservice.metrics.enabled
     enabled: true
@@ -356,6 +359,9 @@ analyseservice:
   image:
     ## @skip analyseservice.image.name
     name: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.8.2
+  ## @param analyseservice.podAnnotations the pod annotations. Evaluated as a template
+  ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+  podAnnotations: { }
   ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod
   podSecurityContext:
     ## @param analyseservice.podSecurityContext.enabled Enable pods' Security Context
@@ -417,6 +423,9 @@ metadataservice:
   image:
     ## @skip metadataservice.image.name
     name: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.8.2
+  ## @param metadataservice.podAnnotations the pod annotations. Evaluated as a template
+  ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+  podAnnotations: { }
   ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod
   podSecurityContext:
     ## @param metadataservice.podSecurityContext.enabled Enable pods' Security Context
@@ -514,6 +523,9 @@ dataservice:
   image:
     ## @skip dataservice.image.name
     name: registry.datalab.tuwien.ac.at/dbrepo/data-service:1.8.2
+  ## @param dataservice.podAnnotations the pod annotations. Evaluated as a template
+  ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+  podAnnotations: { }
   ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod
   podSecurityContext:
     ## @param dataservice.podSecurityContext.enabled Enable pods' Security Context
@@ -600,6 +612,9 @@ searchservice:
   image:
     ## @skip searchservice.image.name
     name: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.8.2
+  ## @param searchservice.podAnnotations the pod annotations. Evaluated as a template
+  ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+  podAnnotations: { }
   ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod
   podSecurityContext:
     ## @param searchservice.podSecurityContext.enabled Enable pods' Security Context
@@ -739,6 +754,9 @@ identityservice:
     adminPassword: admin
     ## @skip identityservice.global.configUserEnabled
     configUserEnabled: false
+  ## @param identityservice.podAnnotations the pod annotations. Evaluated as a template
+  ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+  podAnnotations: { }
   ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod
   podSecurityContext:
     ## @param identityservice.podSecurityContext.enabled Enable pods' Security Context
@@ -937,6 +955,9 @@ dashboardservice:
     name: registry.datalab.tuwien.ac.at/dbrepo/dashboard-service:1.8.2
   ## @param dashboardservice.endpoint The endpoint for the microservices.
   endpoint: http://dashboard-service
+  ## @param dashboardservice.podAnnotations the pod annotations. Evaluated as a template
+  ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+  podAnnotations: { }
   ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod
   podSecurityContext:
     ## @param dashboardservice.podSecurityContext.enabled Enable pods' Security Context
@@ -1005,6 +1026,9 @@ dashboardui:
   enabled: true
   ## @skip dashboardui.fullnameOverride
   fullnameOverride: dashboard-ui
+  ## @skip dashboardui.podAnnotations
+  podAnnotations:
+    fluentbit.io/parser: grafana
   metrics:
     ## @param dashboardui.metrics.enabled Enable the metrics sidecar.
     enabled: true
@@ -1100,6 +1124,9 @@ gatewayservice:
   enabled: true
   ## @skip gatewayservice.fullnameOverride
   fullnameOverride: gateway-service
+  ## @skip gatewayservice.podAnnotations
+  podAnnotations:
+    fluentbit.io/parser: nginx
   service:
     ## @param gatewayservice.service.type The service type.
     type: ClusterIP
@@ -1126,130 +1153,7 @@ loggingservice:
   fullnameOverride: logging-service
   daemonset:
     enabled: true
-  config:
-    ## @skip loggingservice.config.inputs
-    inputs: |-
-      [INPUT]
-          Name                        forward
-          Listen                      0.0.0.0
-          Port                        24224
-
-      [INPUT]
-          name                        fluentbit_metrics
-          tag                         internal_metrics
-          scrape_interval             2
-    ## @skip loggingservice.config.filters
-    filters: |-
-      [FILTER]
-          Name                        parser
-          Parser                      nginx
-          Match                       nginx.*
-          Key_Name                    log
-          Reserve_Data                On
-          Preserve_Key                On
-
-      [FILTER]
-          Name                        parser
-          Parser                      java
-          Match                       java.*
-          Key_Name                    log
-          Reserve_Data                On
-          Preserve_Key                On
-  
-      [FILTER]
-          Name                        parser
-          Parser                      gunicorn
-          Match                       python.*
-          Key_Name                    log
-          Reserve_Data                On
-          Preserve_Key                On
-  
-      [FILTER]
-          Name                        parser
-          Parser                      seaweedfs
-          Match                       seaweedfs.*
-          Key_Name                    log
-          Reserve_Data                On
-          Preserve_Key                On
-  
-      [FILTER]
-          Name                        parser
-          Parser                      openldap
-          Match                       openldap.*
-          Key_Name                    log
-          Reserve_Data                On
-          Preserve_Key                On
-  
-      [FILTER]
-          Name                        parser
-          Parser                      quarkus
-          Match                       keycloak.*
-          Key_Name                    log
-          Reserve_Data                On
-          Preserve_Key                On
-    ## @skip loggingservice.config.outputs
-    outputs: |-
-      [OUTPUT]
-          Name                        opensearch
-          Match                       *
-          Host                        search-db
-          Port                        9200
-          Index                       logging
-          Replace_Dots                On
-          Include_Tag_Key             On
-          Suppress_Type_Name          On
-
-      [OUTPUT]
-          name                        prometheus_exporter
-          match                       internal_metrics
-          host                        0.0.0.0
-          port                        2021
-    ## @skip loggingservice.config.customParsers
-    customParsers: |-
-      [PARSER]
-          # https://rubular.com/r/78ieBhDKvlzPnW
-          Name        java
-          Format      regex
-          Regex       (?<time>[^ ]* {1,2}[^ ]*) \[(?<thread>.*)\] *(?<level>[^ ]*) (?<logger>.*) - (?<message>.*)$
-          Time_Key    time
-          Time_Format %Y-%m-%d %H:%M:%S,%L
-          Time_Keep   On
-      
-      [PARSER]
-          # https://rubular.com/r/oTnpOflRX0HwaG
-          Name        quarkus
-          Format      regex
-          Regex       ^(?<time>[^ ]* {1,2}[^ ]*) (?<level>[^ ]*).*\[(?<module>.*)\].*\(executor-thread-(?<executor>[0-9]+)\) (?<message>.*)$
-          Time_Key    time
-          Time_Format %Y-%m-%d %H:%M:%S,%L
-          Time_Keep   On
-      
-      [PARSER]
-          # https://rubular.com/r/JqOp74qykhKeBJ
-          Name        gunicorn
-          Format      regex
-          Regex       \[(?<time>[^ ]* {1,2}[^ ]*)\] \[(?<level>[a-zA-Z]+)\] *(?<message>.*)$
-          Time_Key    time
-          Time_Format %Y-%m-%d %H:%M:%S %z
-          Time_Keep   On
-      
-      [PARSER]
-          # https://rubular.com/r/bqLY5HuZfP07p3
-          Name        seaweedfs
-          Format      regex
-          Regex       (?<i>I[0-9]*) (?<time>[^ ]*) (?<golocation>[a-z0-9_\.:]*) (?<message>.*)
-          Time_Key    time
-          Time_Format %H:%M:%S.%L
-          Time_Keep   On
-      
-      [PARSER]
-          # https://rubular.com/r/SMa7y7BTOOgf9z
-          Name        openldap
-          Format      regex
-          Regex       (?<rnd>[^ ]*) (?<register>[^ ]*) conn=(?<conn>[0-9]+) op=(?<op>[^ ]*) (?<message>.*)
-          Time_Key    time
-          Time_Format %H:%M:%S.%L
-          Time_Keep   On
+  existingConfigMap: logging-service-config
   extraContainerPorts:
     - name: metrics
       containerPort: 2021