112 lines
3.2 KiB
HCL
112 lines
3.2 KiB
HCL
job "vector" {
|
|
datacenters = ["alo"]
|
|
type = "system"
|
|
group "vector" {
|
|
network {
|
|
port "api" {
|
|
#host_network = "tailscale"
|
|
to = 8686
|
|
}
|
|
}
|
|
ephemeral_disk {
|
|
size = 500
|
|
sticky = true
|
|
}
|
|
task "vector" {
|
|
driver = "docker"
|
|
config {
|
|
image = "timberio/vector:0.28.X-alpine"
|
|
ports = ["api"]
|
|
volumes = [
|
|
"/var/run/docker.sock:/var/run/docker.sock:ro",
|
|
"/var/log/glusterfs:/var/log/glusterfs:ro",
|
|
]
|
|
}
|
|
|
|
env {
|
|
VECTOR_CONFIG = "local/vector.toml"
|
|
VECTOR_REQUIRE_HEALTHY = "true"
|
|
}
|
|
|
|
service {
|
|
check {
|
|
port = "api"
|
|
type = "http"
|
|
path = "/health"
|
|
interval = "30s"
|
|
timeout = "5s"
|
|
}
|
|
}
|
|
|
|
resources {
|
|
cpu = 500 # 500 MHz
|
|
memory = 256 # 256MB
|
|
}
|
|
|
|
template {
|
|
destination = "local/vector.toml"
|
|
change_mode = "signal"
|
|
change_signal = "SIGHUP"
|
|
# overriding the delimiters to [[ ]] to avoid conflicts with Vector's native templating, which also uses {{ }}
|
|
left_delimiter = "[["
|
|
right_delimiter = "]]"
|
|
data=<<EOH
|
|
data_dir = "alloc/data/"
|
|
[api]
|
|
enabled = true
|
|
address = "0.0.0.0:8686"
|
|
playground = false
|
|
|
|
[sources.docker_raw]
|
|
type = "docker_logs"
|
|
[transforms.docker_transformed]
|
|
type = "remap"
|
|
inputs = [ "docker_raw" ]
|
|
source = '''
|
|
.node = del(.label."com.hashicorp.nomad.node_name")
|
|
.job_name, err = .label."com.hashicorp.nomad.job_name" + "." + .label."com.hashicorp.nomad.task_group_name" + "." + .label."com.hashicorp.nomad.task_name"
|
|
result = parse_regex_all!(.message, r'^(?P<log>.+) (?P<total_requests>\d+) "(?P<frontend_name>.+)" "(?P<backend_url>.+)" (?P<duration_ms>\d+)ms$')
|
|
if length(result) > 0 {
|
|
. |= parse_apache_log!(result[0].log, "combined")
|
|
.backend_url = result[0].backend_url
|
|
.frontend_name = result[0].frontend_name
|
|
.duration_ms = to_int!(result[0].duration_ms)
|
|
.total_requests = to_int!(result[0].total_requests)
|
|
}
|
|
'''
|
|
|
|
[sources.glusterfs_raw]
|
|
type = "file"
|
|
include = [ "/var/log/glusterfs/**/*.log" ]
|
|
host_key = "node"
|
|
[transforms.glusterfs_transformed]
|
|
type = "remap"
|
|
inputs = [ "glusterfs_raw" ]
|
|
source = '''
|
|
.node = "[[env "attr.unique.hostname" ]]"
|
|
'''
|
|
|
|
[sinks.loki]
|
|
type = "loki"
|
|
inputs = [
|
|
"docker_transformed",
|
|
"glusterfs_transformed"
|
|
]
|
|
endpoint = "http://[[ range service "loki" ]][[ .Address ]]:[[ .Port ]][[ end ]]"
|
|
encoding.codec = "json"
|
|
buffer.type = "memory"
|
|
out_of_order_action = "accept"
|
|
request.concurrency = "adaptive"
|
|
remove_label_fields = true
|
|
healthcheck.enabled = true
|
|
[sinks.loki.labels]
|
|
job_name = "{{job_name}}"
|
|
node = "{{node}}"
|
|
frontend_name = "{{frontend_name}}"
|
|
EOH
|
|
}
|
|
kill_timeout = "30s"
|
|
}
|
|
}
|
|
}
|