ppetru/alo-cluster
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Send glusterfs logs to loki.

Browse Source
This commit is contained in:
Petru
2023-08-08 14:38:25 +01:00
parent ab80d00632
commit 7a30c0e9f2
1 changed files with 27 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
services/vector.hcl
View File

@@ -17,7 +17,10 @@ job "vector" {
config { config {
image = "timberio/vector:0.28.X-alpine" image = "timberio/vector:0.28.X-alpine"
ports = ["api"] ports = ["api"]
volumes = ["/var/run/docker.sock:/var/run/docker.sock:ro"] volumes = [
"/var/run/docker.sock:/var/run/docker.sock:ro",
"/var/log/glusterfs:/var/log/glusterfs:ro",
]
} }
env { env {
@@ -48,17 +51,19 @@ job "vector" {
left_delimiter = "[[" left_delimiter = "[["
right_delimiter = "]]" right_delimiter = "]]"
data=<<EOH data=<<EOH
data_dir = "alloc/data/vector/" data_dir = "alloc/data/"
[api] [api]
enabled = true enabled = true
address = "0.0.0.0:8686" address = "0.0.0.0:8686"
playground = false playground = false
[sources.logs]
[sources.docker_raw]
type = "docker_logs" type = "docker_logs"
[transforms.transformed] [transforms.docker_transformed]
type = "remap" type = "remap"
inputs = [ "logs" ] inputs = [ "docker_raw" ]
source = ''' source = '''
.node = del(.label."com.hashicorp.nomad.node_name")
.job_name, err = .label."com.hashicorp.nomad.job_name" + "." + .label."com.hashicorp.nomad.task_group_name" + "." + .label."com.hashicorp.nomad.task_name" .job_name, err = .label."com.hashicorp.nomad.job_name" + "." + .label."com.hashicorp.nomad.task_group_name" + "." + .label."com.hashicorp.nomad.task_name"
result = parse_regex_all!(.message, r'^(?P<log>.+) (?P<total_requests>\d+) "(?P<frontend_name>.+)" "(?P<backend_url>.+)" (?P<duration_ms>\d+)ms$') result = parse_regex_all!(.message, r'^(?P<log>.+) (?P<total_requests>\d+) "(?P<frontend_name>.+)" "(?P<backend_url>.+)" (?P<duration_ms>\d+)ms$')
if length(result) > 0 { if length(result) > 0 {
@@ -69,9 +74,24 @@ if length(result) > 0 {
.total_requests = to_int!(result[0].total_requests) .total_requests = to_int!(result[0].total_requests)
} }
''' '''
[sources.glusterfs_raw]
type = "file"
include = [ "/var/log/glusterfs/**/*.log" ]
host_key = "node"
[transforms.glusterfs_transformed]
type = "remap"
inputs = [ "glusterfs_raw" ]
source = '''
.node = "[[env "attr.unique.hostname" ]]"
'''
[sinks.loki] [sinks.loki]
type = "loki" type = "loki"
inputs = ["transformed"] inputs = [
"docker_transformed",
"glusterfs_transformed"
]
endpoint = "http://[[ range service "loki" ]][[ .Address ]]:[[ .Port ]][[ end ]]" endpoint = "http://[[ range service "loki" ]][[ .Address ]]:[[ .Port ]][[ end ]]"
encoding.codec = "json" encoding.codec = "json"
buffer.type = "memory" buffer.type = "memory"
@@ -81,7 +101,7 @@ if length(result) > 0 {
healthcheck.enabled = true healthcheck.enabled = true
[sinks.loki.labels] [sinks.loki.labels]
job_name = "{{job_name}}" job_name = "{{job_name}}"
node = "{{label.\"com.hashicorp.nomad.node_name\" }}" node = "{{node}}"
frontend_name = "{{frontend_name}}" frontend_name = "{{frontend_name}}"
EOH EOH
} }