WIP

Signed-off-by: Martin Weise <martin.weise@tuwien.ac.at>

dbrepo-dashboard-ui/provisioning/datasources/opensearch.yaml

+apiVersion: 1
+
+datasources:
+  - name: dbrepo-opensearch
+    uid: dbrepoopensearch0
+    type: grafana-opensearch-datasource
+    basicAuth: false
+    url: http://search-db:9200
+    jsonData:
+      flavor: "opensearch"
+      logLevelField: "level"
+      logMessageField: "message"
+      timeField: "@timestamp"
+      version: "2.19.0"

helm/dbrepo/templates/dashboard-ui-datasources-secret.yaml

@@ -30,4 +30,18 @@ stringData:
         type: prometheus
         uid: dbrepometrics0
         url: http://metric-db-server
+  opensearch.yaml: |-
+    apiVersion: 1
+    datasources:
+      - name: dbrepo-opensearch
+        uid: dbrepoopensearch0
+        type: grafana-opensearch-datasource
+        basicAuth: false
+        url: http://search-db:9200
+        jsonData:
+          flavor: "opensearch"
+          logLevelField: "level"
+          logMessageField: "message"
+          timeField: "@timestamp"
+          version: "2.19.0"
 {{- end }}

helm/dbrepo/templates/logging-configmap.yaml

+{{- if .Values.loggingservice.enabled }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: logging-service-config
+  namespace: {{ include "common.names.namespace" . | quote }}
+data:
+  dbrepo_parser.conf: |-
+    [PARSER]
+        # https://rubular.com/r/78ieBhDKvlzPnW
+        Name        java
+        Format      regex
+        Regex       (?<time>[^ ]* {1,2}[^ ]*) \[(?<thread>.*)\] *(?<level>[^ ]*) (?<logger>.*) - (?<message>.*)$
+        Time_Key    time
+        Time_Format %Y-%m-%d %H:%M:%S,%L
+        Time_Keep   On
+
+    [PARSER]
+        # https://rubular.com/r/oTnpOflRX0HwaG
+        Name        quarkus
+        Format      regex
+        Regex       ^(?<time>[^ ]* {1,2}[^ ]*) (?<level>[^ ]*).*\[(?<module>.*)\].*\(executor-thread-(?<executor>[0-9]+)\) (?<message>.*)$
+        Time_Key    time
+        Time_Format %Y-%m-%d %H:%M:%S,%L
+        Time_Keep   On
+
+    [PARSER]
+        # https://rubular.com/r/JqOp74qykhKeBJ
+        Name        gunicorn
+        Format      regex
+        Regex       \[(?<time>[^ ]* {1,2}[^ ]*)\] \[(?<level>[a-zA-Z]+)\] *(?<message>.*)$
+        Time_Key    time
+        Time_Format %Y-%m-%d %H:%M:%S %z
+        Time_Keep   On
+
+    [PARSER]
+        # https://rubular.com/r/bqLY5HuZfP07p3
+        Name        seaweedfs
+        Format      regex
+        Regex       (?<i>I[0-9]*) (?<time>[^ ]*) (?<golocation>[a-z0-9_\.:]*) (?<message>.*)
+        Time_Key    time
+        Time_Format %H:%M:%S.%L
+        Time_Keep   On
+
+    [PARSER]
+        # https://rubular.com/r/SMa7y7BTOOgf9z
+        Name        openldap
+        Format      regex
+        Regex       (?<rnd>[^ ]*) (?<register>[^ ]*) conn=(?<conn>[0-9]+) op=(?<op>[^ ]*) (?<message>.*)
+        Time_Key    time
+        Time_Format %H:%M:%S.%L
+        Time_Keep   On
+  fluentbit.conf: |-
+    [SERVICE]
+        Flush                       5
+        Daemon                      Off
+        Log_Level                   debug
+        Parsers_File                parsers.conf
+        Parsers_File                dbrepo_parser.conf
+
+    [INPUT]
+        Name                        forward
+        Listen                      0.0.0.0
+        Port                        24224
+
+    [FILTER]
+        Name                        parser
+        Parser                      nginx
+        Match                       nginx.*
+        Key_Name                    log
+        Reserve_Data                On
+        Preserve_Key                On
+
+    [FILTER]
+        Name                        parser
+        Parser                      java
+        Match                       java.*
+        Key_Name                    log
+        Reserve_Data                On
+        Preserve_Key                On
+
+    [FILTER]
+        Name                        parser
+        Parser                      gunicorn
+        Match                       python.*
+        Key_Name                    log
+        Reserve_Data                On
+        Preserve_Key                On
+
+    [FILTER]
+        Name                        parser
+        Parser                      seaweedfs
+        Match                       seaweedfs.*
+        Key_Name                    log
+        Reserve_Data                On
+        Preserve_Key                On
+
+    [FILTER]
+        Name                        parser
+        Parser                      openldap
+        Match                       openldap.*
+        Key_Name                    log
+        Reserve_Data                On
+        Preserve_Key                On
+
+    [FILTER]
+        Name                        parser
+        Parser                      quarkus
+        Match                       keycloak.*
+        Key_Name                    log
+        Reserve_Data                On
+        Preserve_Key                On
+
+    [OUTPUT]
+        Name                        opensearch
+        Match                       *
+        Host                        search-db
+        Port                        9200
+        Index                       logging
+        Replace_Dots                On
+        Include_Tag_Key             On
+        Suppress_Type_Name          On
+{{- end }}

helm/dbrepo/templates/logging-deployment.yaml

+{{- if .Values.loggingservice.enabled }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: logging-service
+  namespace: {{ include "common.names.namespace" . | quote }}
+  labels:
+    app: logging-service
+    service: logging-service
+spec:
+  replicas: {{ .Values.loggingservice.replicaCount }}
+  strategy:
+    type: {{ .Values.strategyType }}
+  selector:
+    matchLabels:
+      app: logging-service
+      service: logging-service
+  template:
+    metadata:
+      labels:
+        app: logging-service
+        service: logging-service
+    spec:
+      {{- if .Values.loggingservice.podSecurityContext.enabled }}
+      securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.loggingservice.podSecurityContext "context" $) | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: logging-service
+          image: {{ .Values.loggingservice.image.name }}
+          imagePullPolicy: {{ .Values.loggingservice.image.pullPolicy | default "IfNotPresent" }}
+          {{- if .Values.loggingservice.containerSecurityContext.enabled }}
+          securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.loggingservice.containerSecurityContext "context" $) | nindent 12 }}
+          {{- end }}
+          ports:
+            - containerPort: 24224
+              protocol: TCP
+            - containerPort: 24224
+              protocol: UDP
+          volumeMounts:
+            - mountPath: /opt/bitnami/fluent-bit/conf/
+              name: config
+              readOnly: true
+          livenessProbe:
+            exec:
+              command:
+                - /bin/bash
+                - -ec
+                - "ps -p 1 | grep 'fluent-bit'"
+            initialDelaySeconds: 150
+            periodSeconds: 10
+          readinessProbe:
+            exec:
+              command:
+                - /bin/bash
+                - -ec
+                - "ps -p 1 | grep 'fluent-bit'"
+            initialDelaySeconds: 30
+            periodSeconds: 10
+          {{- if .Values.loggingservice.resources }}
+          resources: {{- toYaml .Values.loggingservice.resources | nindent 12 }}
+          {{- else if ne .Values.loggingservice.resourcesPreset "none" }}
+          resources: {{- include "common.resources.preset" (dict "type" .Values.loggingservice.resourcesPreset) | nindent 12 }}
+          {{- end }}
+      volumes:
+        - name: config
+          configMap:
+            name: logging-service-config
+{{- end }}

helm/dbrepo/values.yaml

@@ -1120,7 +1120,7 @@ computeservice:
 
 loggingservice:
   ## @param loggingservice.enabled Enable the Logging Service.
-  enabled: false
+  enabled: true
   image:
     ## @skip loggingservice.image.name
     name: docker.io/bitnami/fluent-bit:4.0.0