Skip to content
Snippets Groups Projects
Verified Commit 1f6fe70c authored by Martin Weise's avatar Martin Weise
Browse files

Added structured logging 


Signed-off-by: Martin Weise's avatarMartin Weise <martin.weise@tuwien.ac.at>
parent b760b744
No related branches found
No related tags found
1 merge request!411WIP
Show whitespace changes
Inline Side-by-side
Showing
with 781 additions and 18 deletions
.env
+ 1
− 1
View file @ 1f6fe70c
...@@ -2,7 +2,7 @@ MARIADB_VERSION=11.3.2 ...@@ -2,7 +2,7 @@ MARIADB_VERSION=11.3.2
POSTGRES_VERSION=17.0.0 POSTGRES_VERSION=17.0.0
KEYCLOAK_VERSION=26.0.4 KEYCLOAK_VERSION=26.0.4
RABBITMQ_VERSION=3.13.7 RABBITMQ_VERSION=3.13.7
OPENSEARCH_VERSION=2.10.0 OPENSEARCH_VERSION=2.19.0
NGINX_VERSION=1.27.3-alpine3.20-slim NGINX_VERSION=1.27.3-alpine3.20-slim
OPENLDAP_VERSION=2.6.8 OPENLDAP_VERSION=2.6.8
SEAWEEDFS_VERSION=3.71.0 SEAWEEDFS_VERSION=3.71.0
......
dbrepo-analyse-service/app.py
+ 1
− 1
View file @ 1f6fe70c
...@@ -28,7 +28,7 @@ dictConfig({ ...@@ -28,7 +28,7 @@ dictConfig({
'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s', 'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
}, },
'simple': { 'simple': {
'format': '[%(asctime)s] %(levelname)s: %(message)s', 'format': '[%(asctime)s] [%(levelname)s] %(message)s',
}, },
}, },
'handlers': {'wsgi': { 'handlers': {'wsgi': {
......
dbrepo-dashboard-service/app.py
+ 1
− 1
View file @ 1f6fe70c
...@@ -29,7 +29,7 @@ dictConfig({ ...@@ -29,7 +29,7 @@ dictConfig({
'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s', 'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
}, },
'simple': { 'simple': {
'format': '[%(asctime)s] %(levelname)s: %(message)s', 'format': '[%(asctime)s] [%(levelname)s] %(message)s',
}, },
}, },
'handlers': {'wsgi': { 'handlers': {'wsgi': {
......
dbrepo-dashboard-service/init.py
+ 1
− 1
View file @ 1f6fe70c
...@@ -18,7 +18,7 @@ dictConfig({ ...@@ -18,7 +18,7 @@ dictConfig({
'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s', 'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
}, },
'simple': { 'simple': {
'format': '[%(asctime)s] %(levelname)s: %(message)s', 'format': '[%(asctime)s] [%(levelname)s] %(message)s',
}, },
}, },
'handlers': {'wsgi': { 'handlers': {'wsgi': {
......
dbrepo-dashboard-ui/dashboards/System/dbrepo.json
+ 3
− 2
View file @ 1f6fe70c
...@@ -30,7 +30,7 @@ ...@@ -30,7 +30,7 @@
"title": "Docs", "title": "Docs",
"tooltip": "", "tooltip": "",
"type": "link", "type": "link",
"url": "https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.7/" "url": "https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.8/"
} }
], ],
"panels": [ "panels": [
...@@ -2428,7 +2428,8 @@ ...@@ -2428,7 +2428,8 @@
"gateway", "gateway",
"analyse", "analyse",
"metrics", "metrics",
"auth" "auth",
"search"
], ],
"templating": { "templating": {
"list": [] "list": []
......
dbrepo-dashboard-ui/dashboards/System/logging.json 0 → 100644
+ 694
− 0
View file @ 1f6fe70c
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 3,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 5,
"panels": [],
"title": "HTTP Gateway",
"type": "row"
},
{
"datasource": {
"type": "grafana-opensearch-datasource",
"uid": "dbrepoopensearch0"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "request.method"
},
"properties": [
{
"id": "custom.width"
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 1
},
"id": 7,
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": []
},
"pluginVersion": "11.5.3",
"targets": [
{
"alias": "",
"bucketAggs": [
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto"
},
"type": "date_histogram"
}
],
"datasource": {
"type": "grafana-opensearch-datasource",
"uid": "dbrepoopensearch0"
},
"format": "table",
"hide": false,
"luceneQueryType": "Logs",
"metrics": [
{
"id": "1",
"type": "logs"
}
],
"query": "container_name: \"/dbrepo-gateway-service\" AND code: >=400",
"queryType": "lucene",
"refId": "A",
"timeField": "@timestamp"
}
],
"title": "HTTP Gateway Errors",
"transformations": [
{
"id": "filterFieldsByName",
"options": {
"include": {
"names": [
"code",
"method",
"path",
"size"
]
}
}
},
{
"id": "organize",
"options": {
"excludeByName": {},
"includeByName": {},
"indexByName": {
"code": 2,
"method": 0,
"path": 1,
"size": 3
},
"renameByName": {
"@timestamp": "",
"code": "response.status",
"method": "request.method",
"path": "request.url",
"size": "body.size"
}
}
},
{
"id": "calculateField",
"options": {
"alias": "Count",
"mode": "reduceRow",
"reduce": {
"include": [
"request.url",
"response.status"
],
"reducer": "distinctCount"
},
"replaceFields": false
}
},
{
"id": "sortBy",
"options": {
"fields": {},
"sort": [
{
"desc": true,
"field": "Count"
}
]
}
}
],
"type": "table"
},
{
"datasource": {
"type": "grafana-opensearch-datasource",
"uid": "dbrepoopensearch0"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "fixed"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "Count",
"axisPlacement": "auto",
"fillOpacity": 80,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineWidth": 1,
"scaleDistribution": {
"type": "linear"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byFrameRefID",
"options": "200"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "green",
"mode": "fixed"
}
},
{
"id": "displayName",
"value": "200s"
}
]
},
{
"matcher": {
"id": "byFrameRefID",
"options": "300"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "purple",
"mode": "fixed"
}
},
{
"id": "displayName",
"value": "300s"
}
]
},
{
"matcher": {
"id": "byFrameRefID",
"options": "400"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "orange",
"mode": "fixed"
}
},
{
"id": "displayName",
"value": "400s"
}
]
},
{
"matcher": {
"id": "byFrameRefID",
"options": "500"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
},
{
"id": "displayName",
"value": "500s"
}
]
}
]
},
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 9
},
"id": 3,
"options": {
"barRadius": 0,
"barWidth": 0.97,
"fullHighlight": false,
"groupWidth": 0.7,
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "right",
"showLegend": true
},
"orientation": "auto",
"showValue": "auto",
"stacking": "normal",
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "none"
},
"xTickLabelRotation": 0,
"xTickLabelSpacing": 100
},
"pluginVersion": "11.5.3",
"targets": [
{
"alias": "",
"bucketAggs": [
{
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": "0",
"trimEdges": "0"
},
"type": "date_histogram"
}
],
"datasource": {
"type": "grafana-opensearch-datasource",
"uid": "dbrepoopensearch0"
},
"format": "table",
"hide": false,
"luceneQueryType": "Metric",
"metrics": [
{
"id": "1",
"type": "count"
}
],
"query": "container_name: \"/dbrepo-gateway-service\" AND code: (>=200 AND <300)",
"queryType": "lucene",
"refId": "200",
"timeField": "@timestamp"
},
{
"alias": "",
"bucketAggs": [
{
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": "0",
"trimEdges": "0"
},
"type": "date_histogram"
}
],
"datasource": {
"type": "grafana-opensearch-datasource",
"uid": "dbrepoopensearch0"
},
"format": "table",
"hide": false,
"luceneQueryType": "Metric",
"metrics": [
{
"id": "1",
"type": "count"
}
],
"query": "container_name: \"/dbrepo-gateway-service\" AND code: (>=300 AND <400)",
"queryType": "lucene",
"refId": "300",
"timeField": "@timestamp"
},
{
"alias": "",
"bucketAggs": [
{
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": "0",
"trimEdges": "0"
},
"type": "date_histogram"
}
],
"datasource": {
"type": "grafana-opensearch-datasource",
"uid": "dbrepoopensearch0"
},
"format": "table",
"hide": false,
"luceneQueryType": "Metric",
"metrics": [
{
"id": "1",
"type": "count"
}
],
"query": "container_name: \"/dbrepo-gateway-service\" AND code: (>=400 AND <500)",
"queryType": "lucene",
"refId": "400",
"timeField": "@timestamp"
},
{
"alias": "",
"bucketAggs": [
{
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": "0",
"trimEdges": "0"
},
"type": "date_histogram"
}
],
"datasource": {
"type": "grafana-opensearch-datasource",
"uid": "dbrepoopensearch0"
},
"format": "table",
"hide": false,
"luceneQueryType": "Metric",
"metrics": [
{
"id": "1",
"type": "count"
}
],
"query": "container_name: \"/dbrepo-gateway-service\" AND code: >=500",
"queryType": "lucene",
"refId": "500",
"timeField": "@timestamp"
}
],
"title": "HTTP Status Codes over Time",
"type": "barchart"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 18
},
"id": 6,
"panels": [],
"title": "Application",
"type": "row"
},
{
"datasource": {
"type": "grafana-opensearch-datasource",
"uid": "dbrepoopensearch0"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "fixed"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"fillOpacity": 80,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineWidth": 1,
"scaleDistribution": {
"type": "linear"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byFrameRefID",
"options": "warn"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "orange",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byFrameRefID",
"options": "error"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 19
},
"id": 1,
"options": {
"barRadius": 0,
"barWidth": 0.97,
"fullHighlight": false,
"groupWidth": 0.7,
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "right",
"showLegend": false
},
"orientation": "auto",
"showValue": "auto",
"stacking": "none",
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
},
"xTickLabelRotation": 0,
"xTickLabelSpacing": 100
},
"pluginVersion": "11.5.3",
"targets": [
{
"alias": "sdfsdf",
"bucketAggs": [
{
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": "0",
"trimEdges": "0"
},
"type": "date_histogram"
}
],
"datasource": {
"type": "grafana-opensearch-datasource",
"uid": "dbrepoopensearch0"
},
"format": "table",
"hide": false,
"luceneQueryType": "Metric",
"metrics": [
{
"hide": false,
"id": "1",
"type": "count"
}
],
"query": "level: WARN",
"queryType": "lucene",
"refId": "warn",
"timeField": "@timestamp"
},
{
"alias": "adadas",
"bucketAggs": [
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto"
},
"type": "date_histogram"
}
],
"datasource": {
"type": "grafana-opensearch-datasource",
"uid": "dbrepoopensearch0"
},
"format": "table",
"hide": false,
"luceneQueryType": "Metric",
"metrics": [
{
"id": "1",
"type": "count"
}
],
"query": "level: ERROR",
"queryType": "lucene",
"refId": "error",
"timeField": "@timestamp"
}
],
"title": "Errors and Warnings over Time",
"type": "barchart"
}
],
"preload": false,
"refresh": "",
"schemaVersion": 40,
"tags": [
"ui",
"dashboard",
"metadata",
"data",
"gateway",
"analyse",
"metrics",
"auth",
"search"
],
"templating": {
"list": []
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Logging",
"uid": "aejhojr0mrpj4c",
"version": 23,
"weekStart": ""
}
\ No newline at end of file
dbrepo-dashboard-ui/provisioning/datasources/opensearch.yaml
+ 1
− 1
View file @ 1f6fe70c
...@@ -10,5 +10,5 @@ datasources: ...@@ -10,5 +10,5 @@ datasources:
flavor: "opensearch" flavor: "opensearch"
logLevelField: "level" logLevelField: "level"
logMessageField: "message" logMessageField: "message"
timeField: "@time" timeField: "@timestamp"
version: "2.10.0" version: "2.10.0"
dbrepo-data-service/lib/at/ac/tuwien/ifs/dbrepo/dbrepo-core/1.8.1/dbrepo-core-1.8.1.jar
+ 0
− 0
View file @ 1f6fe70c
No preview for this file type
dbrepo-data-service/lib/at/ac/tuwien/ifs/dbrepo/dbrepo-core/1.8.1/dbrepo-core-1.8.1.jar.md5
+ 1
− 1
View file @ 1f6fe70c
dd9863e47bad7b44d48640513a8a3d4b 4bb9c10ee8dcc4f05fef0928ec526b5b
\ No newline at end of file \ No newline at end of file
dbrepo-data-service/lib/at/ac/tuwien/ifs/dbrepo/dbrepo-core/1.8.1/dbrepo-core-1.8.1.jar.sha1
+ 1
− 1
View file @ 1f6fe70c
8ade955fe62f8e6cd809f37f3479ddc843882bce f4bc86db3f023f2b9481db399c6d439ba41e5272
\ No newline at end of file \ No newline at end of file
dbrepo-data-service/lib/at/ac/tuwien/ifs/dbrepo/dbrepo-core/maven-metadata.xml
+ 1
− 1
View file @ 1f6fe70c
...@@ -7,6 +7,6 @@ ...@@ -7,6 +7,6 @@
<versions> <versions>
<version>1.8.1</version> <version>1.8.1</version>
</versions> </versions>
<lastUpdated>20250420103842</lastUpdated> <lastUpdated>20250420181800</lastUpdated>
</versioning> </versioning>
</metadata> </metadata>
dbrepo-data-service/lib/at/ac/tuwien/ifs/dbrepo/dbrepo-core/maven-metadata.xml.md5
+ 1
− 1
View file @ 1f6fe70c
779db7a879fe20234417cce3fa298472 c77e7629604adca32895fd22fcf85f37
\ No newline at end of file \ No newline at end of file
dbrepo-data-service/lib/at/ac/tuwien/ifs/dbrepo/dbrepo-core/maven-metadata.xml.sha1
+ 1
− 1
View file @ 1f6fe70c
1caca64fae673f34667cdb4bda7f739ba9144748 33a845a4d64bc82da3fb93f817539603aa7a159d
\ No newline at end of file \ No newline at end of file
dbrepo-logging-service/dbrepo_parser.conf 0 → 100644
+ 35
− 0
View file @ 1f6fe70c
[PARSER]
# https://rubular.com/r/u029eqdrtxeOT9
Name java
Format regex
Regex (?<time>[^ ]* {1,2}[^ ]*) (?<level>[^ ]*) *(?<message>.*)$
Time_Key time
Time_Format %Y-%m-%d %H:%M:%S,%L
Time_Keep On
[PARSER]
# https://rubular.com/r/JqOp74qykhKeBJ
Name gunicorn
Format regex
Regex \[(?<time>[^ ]* {1,2}[^ ]*)\] \[(?<level>[a-zA-Z]+)\] *(?<message>.*)$
Time_Key time
Time_Format %Y-%m-%d %H:%M:%S %z
Time_Keep On
[PARSER]
# https://rubular.com/r/bqLY5HuZfP07p3
Name seaweedfs
Format regex
Regex (?<i>I[0-9]*) (?<time>[^ ]*) (?<golocation>[a-z0-9_\.:]*) (?<message>.*)
Time_Key time
Time_Format %H:%M:%S.%L
Time_Keep On
[PARSER]
# https://rubular.com/r/SMa7y7BTOOgf9z
Name openldap
Format regex
Regex (?<rnd>[^ ]*) (?<register>[^ ]*) conn=(?<conn>[0-9]+) op=(?<op>[^ ]*) (?<message>.*)
Time_Key time
Time_Format %H:%M:%S.%L
Time_Keep On
dbrepo-logging-service/default.conf
+ 35
− 2
View file @ 1f6fe70c
...@@ -3,6 +3,7 @@ ...@@ -3,6 +3,7 @@
Daemon Off Daemon Off
Log_Level debug Log_Level debug
Parsers_File parsers.conf Parsers_File parsers.conf
Parsers_File dbrepo_parser.conf
[INPUT] [INPUT]
Name forward Name forward
...@@ -11,8 +12,40 @@ ...@@ -11,8 +12,40 @@
[FILTER] [FILTER]
Name parser Name parser
Parser docker Parser nginx
Match * Match nginx.*
Key_Name log
Reserve_Data On
Preserve_Key On
[FILTER]
Name parser
Parser java
Match java.*
Key_Name log
Reserve_Data On
Preserve_Key On
[FILTER]
Name parser
Parser gunicorn
Match python.*
Key_Name log
Reserve_Data On
Preserve_Key On
[FILTER]
Name parser
Parser seaweedfs
Match seaweedfs.*
Key_Name log
Reserve_Data On
Preserve_Key On
[FILTER]
Name parser
Parser openldap
Match openldap.*
Key_Name log Key_Name log
Reserve_Data On Reserve_Data On
Preserve_Key On Preserve_Key On
......
dbrepo-metadata-service/lib/at/ac/tuwien/ifs/dbrepo/dbrepo-core/1.8.1/dbrepo-core-1.8.1.jar
+ 0
− 0
View file @ 1f6fe70c
No preview for this file type
dbrepo-metadata-service/lib/at/ac/tuwien/ifs/dbrepo/dbrepo-core/1.8.1/dbrepo-core-1.8.1.jar.md5
+ 1
− 1
View file @ 1f6fe70c
dd9863e47bad7b44d48640513a8a3d4b 4bb9c10ee8dcc4f05fef0928ec526b5b
\ No newline at end of file \ No newline at end of file
dbrepo-metadata-service/lib/at/ac/tuwien/ifs/dbrepo/dbrepo-core/1.8.1/dbrepo-core-1.8.1.jar.sha1
+ 1
− 1
View file @ 1f6fe70c
8ade955fe62f8e6cd809f37f3479ddc843882bce f4bc86db3f023f2b9481db399c6d439ba41e5272
\ No newline at end of file \ No newline at end of file
dbrepo-metadata-service/lib/at/ac/tuwien/ifs/dbrepo/dbrepo-core/maven-metadata.xml
+ 1
− 1
View file @ 1f6fe70c
...@@ -7,6 +7,6 @@ ...@@ -7,6 +7,6 @@
<versions> <versions>
<version>1.8.1</version> <version>1.8.1</version>
</versions> </versions>
<lastUpdated>20250420103843</lastUpdated> <lastUpdated>20250420181801</lastUpdated>
</versioning> </versioning>
</metadata> </metadata>
dbrepo-metadata-service/lib/at/ac/tuwien/ifs/dbrepo/dbrepo-core/maven-metadata.xml.md5
+ 1
− 1
View file @ 1f6fe70c
89409a86b1aa8f975e87a5a57220ef58 9690000f3e5e3480a1e848c01b121660
\ No newline at end of file \ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment