diff --git a/dbrepo-dashboard-ui/provisioning/datasources/opensearch.yaml b/dbrepo-dashboard-ui/provisioning/datasources/opensearch.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b26f0a4746712c597b90799f5004aab23d31b51 --- /dev/null +++ b/dbrepo-dashboard-ui/provisioning/datasources/opensearch.yaml @@ -0,0 +1,14 @@ +apiVersion: 1 + +datasources: + - name: dbrepo-opensearch + uid: dbrepoopensearch0 + type: grafana-opensearch-datasource + basicAuth: false + url: http://search-db:9200 + jsonData: + flavor: "opensearch" + logLevelField: "level" + logMessageField: "message" + timeField: "@timestamp" + version: "2.19.0" diff --git a/helm/dbrepo/templates/dashboard-ui-datasources-secret.yaml b/helm/dbrepo/templates/dashboard-ui-datasources-secret.yaml index 5ce6c299e9094bb14500df16ad53e428034a4e33..8a21a20db8dd41dad8f36174cf23ca5420562a60 100644 --- a/helm/dbrepo/templates/dashboard-ui-datasources-secret.yaml +++ b/helm/dbrepo/templates/dashboard-ui-datasources-secret.yaml @@ -30,4 +30,18 @@ stringData: type: prometheus uid: dbrepometrics0 url: http://metric-db-server + opensearch.yaml: |- + apiVersion: 1 + datasources: + - name: dbrepo-opensearch + uid: dbrepoopensearch0 + type: grafana-opensearch-datasource + basicAuth: false + url: http://search-db:9200 + jsonData: + flavor: "opensearch" + logLevelField: "level" + logMessageField: "message" + timeField: "@timestamp" + version: "2.19.0" {{- end }} diff --git a/helm/dbrepo/templates/logging-configmap.yaml b/helm/dbrepo/templates/logging-configmap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4c088642367fafebffce22b7d5c110d272cf728 --- /dev/null +++ b/helm/dbrepo/templates/logging-configmap.yaml @@ -0,0 +1,124 @@ +{{- if .Values.loggingservice.enabled }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: logging-service-config + namespace: {{ include "common.names.namespace" . | quote }} +data: + dbrepo_parser.conf: |- + [PARSER] + # https://rubular.com/r/78ieBhDKvlzPnW + Name java + Format regex + Regex (?<time>[^ ]* {1,2}[^ ]*) \[(?<thread>.*)\] *(?<level>[^ ]*) (?<logger>.*) - (?<message>.*)$ + Time_Key time + Time_Format %Y-%m-%d %H:%M:%S,%L + Time_Keep On + + [PARSER] + # https://rubular.com/r/oTnpOflRX0HwaG + Name quarkus + Format regex + Regex ^(?<time>[^ ]* {1,2}[^ ]*) (?<level>[^ ]*).*\[(?<module>.*)\].*\(executor-thread-(?<executor>[0-9]+)\) (?<message>.*)$ + Time_Key time + Time_Format %Y-%m-%d %H:%M:%S,%L + Time_Keep On + + [PARSER] + # https://rubular.com/r/JqOp74qykhKeBJ + Name gunicorn + Format regex + Regex \[(?<time>[^ ]* {1,2}[^ ]*)\] \[(?<level>[a-zA-Z]+)\] *(?<message>.*)$ + Time_Key time + Time_Format %Y-%m-%d %H:%M:%S %z + Time_Keep On + + [PARSER] + # https://rubular.com/r/bqLY5HuZfP07p3 + Name seaweedfs + Format regex + Regex (?<i>I[0-9]*) (?<time>[^ ]*) (?<golocation>[a-z0-9_\.:]*) (?<message>.*) + Time_Key time + Time_Format %H:%M:%S.%L + Time_Keep On + + [PARSER] + # https://rubular.com/r/SMa7y7BTOOgf9z + Name openldap + Format regex + Regex (?<rnd>[^ ]*) (?<register>[^ ]*) conn=(?<conn>[0-9]+) op=(?<op>[^ ]*) (?<message>.*) + Time_Key time + Time_Format %H:%M:%S.%L + Time_Keep On + fluentbit.conf: |- + [SERVICE] + Flush 5 + Daemon Off + Log_Level debug + Parsers_File parsers.conf + Parsers_File dbrepo_parser.conf + + [INPUT] + Name forward + Listen 0.0.0.0 + Port 24224 + + [FILTER] + Name parser + Parser nginx + Match nginx.* + Key_Name log + Reserve_Data On + Preserve_Key On + + [FILTER] + Name parser + Parser java + Match java.* + Key_Name log + Reserve_Data On + Preserve_Key On + + [FILTER] + Name parser + Parser gunicorn + Match python.* + Key_Name log + Reserve_Data On + Preserve_Key On + + [FILTER] + Name parser + Parser seaweedfs + Match seaweedfs.* + Key_Name log + Reserve_Data On + Preserve_Key On + + [FILTER] + Name parser + Parser openldap + Match openldap.* + Key_Name log + Reserve_Data On + Preserve_Key On + + [FILTER] + Name parser + Parser quarkus + Match keycloak.* + Key_Name log + Reserve_Data On + Preserve_Key On + + [OUTPUT] + Name opensearch + Match * + Host search-db + Port 9200 + Index logging + Replace_Dots On + Include_Tag_Key On + Suppress_Type_Name On +{{- end }} diff --git a/helm/dbrepo/templates/logging-deployment.yaml b/helm/dbrepo/templates/logging-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80b93a4053896df409ce0edf9a0897933f08fbc2 --- /dev/null +++ b/helm/dbrepo/templates/logging-deployment.yaml @@ -0,0 +1,69 @@ +{{- if .Values.loggingservice.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: logging-service + namespace: {{ include "common.names.namespace" . | quote }} + labels: + app: logging-service + service: logging-service +spec: + replicas: {{ .Values.loggingservice.replicaCount }} + strategy: + type: {{ .Values.strategyType }} + selector: + matchLabels: + app: logging-service + service: logging-service + template: + metadata: + labels: + app: logging-service + service: logging-service + spec: + {{- if .Values.loggingservice.podSecurityContext.enabled }} + securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.loggingservice.podSecurityContext "context" $) | nindent 8 }} + {{- end }} + containers: + - name: logging-service + image: {{ .Values.loggingservice.image.name }} + imagePullPolicy: {{ .Values.loggingservice.image.pullPolicy | default "IfNotPresent" }} + {{- if .Values.loggingservice.containerSecurityContext.enabled }} + securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.loggingservice.containerSecurityContext "context" $) | nindent 12 }} + {{- end }} + ports: + - containerPort: 24224 + protocol: TCP + - containerPort: 24224 + protocol: UDP + volumeMounts: + - mountPath: /opt/bitnami/fluent-bit/conf/ + name: config + readOnly: true + livenessProbe: + exec: + command: + - /bin/bash + - -ec + - "ps -p 1 | grep 'fluent-bit'" + initialDelaySeconds: 150 + periodSeconds: 10 + readinessProbe: + exec: + command: + - /bin/bash + - -ec + - "ps -p 1 | grep 'fluent-bit'" + initialDelaySeconds: 30 + periodSeconds: 10 + {{- if .Values.loggingservice.resources }} + resources: {{- toYaml .Values.loggingservice.resources | nindent 12 }} + {{- else if ne .Values.loggingservice.resourcesPreset "none" }} + resources: {{- include "common.resources.preset" (dict "type" .Values.loggingservice.resourcesPreset) | nindent 12 }} + {{- end }} + volumes: + - name: config + configMap: + name: logging-service-config +{{- end }} diff --git a/helm/dbrepo/values.yaml b/helm/dbrepo/values.yaml index c713559d18f7ba74c177fe2690a25a3d1c78414c..e2498a2305fcc0f386cdb7f8743c79a6875f57a5 100644 --- a/helm/dbrepo/values.yaml +++ b/helm/dbrepo/values.yaml @@ -1120,7 +1120,7 @@ computeservice: loggingservice: ## @param loggingservice.enabled Enable the Logging Service. - enabled: false + enabled: true image: ## @skip loggingservice.image.name name: docker.io/bitnami/fluent-bit:4.0.0