From 09894bdee5a0dfe2a0ce72d4fcfb16f60fa9f1a6 Mon Sep 17 00:00:00 2001
From: Petru Paler <petru@paler.net>
Date: Tue, 8 Aug 2023 15:52:01 +0100
Subject: [PATCH] Export host and internal metrics for prometheus, and simplify
 hostname setting.

---
 services/vector.hcl | 48 ++++++++++++++++++++++++++++-----------------
 1 file changed, 30 insertions(+), 18 deletions(-)

diff --git a/services/vector.hcl b/services/vector.hcl
index b49c9e4..0b46975 100644
--- a/services/vector.hcl
+++ b/services/vector.hcl
@@ -7,6 +7,9 @@ job "vector" {
         #host_network = "tailscale"
         to = 8686
       }
+      port "metrics" {
+        to = 9598
+      }
     }
     ephemeral_disk {
       size    = 500
@@ -17,13 +20,14 @@ job "vector" {
       config {
         # debian instead of alpine to get journalctl
         image = "timberio/vector:0.31.X-debian"
-        ports = ["api"]
+        ports = ["api", "metrics"]
         volumes = [
           "/var/run/docker.sock:/var/run/docker.sock:ro",
           "/var/log/glusterfs:/var/log/glusterfs:ro",
           "/var/log/journal:/var/log/journal:ro",
           "/etc/machine-id:/etc/machine-id:ro",
         ]
+        hostname = "${node.unique.name}"
       }
 
       env {
@@ -32,6 +36,7 @@ job "vector" {
       }
 
       service {
+        name = "vector"
         check {
           port     = "api"
           type     = "http"
@@ -40,6 +45,11 @@ job "vector" {
           timeout  = "5s"
         }
       }
+      service {
+        name = "vector-metrics"
+        port = "metrics"
+        tags = ["metrics"]
+      }
 
       resources {
         cpu    = 500 # 500 MHz
@@ -62,7 +72,6 @@ job "vector" {
 
           [sources.vector_raw]
             type = "internal_logs"
-            host_key = "node"
 
           [sources.docker_raw]
             type = "docker_logs"
@@ -70,43 +79,33 @@ job "vector" {
             type = "remap"
             inputs = [ "docker_raw" ]
             source = '''
-.node = del(.label."com.hashicorp.nomad.node_name")
 .job_name, err = .label."com.hashicorp.nomad.job_name" + "." + .label."com.hashicorp.nomad.task_group_name" + "." + .label."com.hashicorp.nomad.task_name"
 result = parse_regex_all!(.message, r'^(?P<log>.+) (?P<total_requests>\d+) "(?P<frontend_name>.+)" "(?P<backend_url>.+)" (?P<duration_ms>\d+)ms$')
 if length(result) > 0 {
   . |= parse_apache_log!(result[0].log, "combined")
+  .remote_host = del(.host)
   .backend_url = result[0].backend_url
   .frontend_name = result[0].frontend_name
   .duration_ms = to_int!(result[0].duration_ms)
   .total_requests = to_int!(result[0].total_requests)
 }
+.host = del(.label."com.hashicorp.nomad.node_name")
             '''
 
           [sources.glusterfs_raw]
             type = "file"
             include = [ "/var/log/glusterfs/**/*.log" ]
-            host_key = "node"
 
           [sources.journald_raw]
             type = "journald"
 
-          [transforms.nomadify_hostname]
-            type = "remap"
-            inputs = [
-              "vector_raw",
-              "glusterfs_raw",
-              "journald_raw",
-            ]
-            source = '''
-del(.host)
-.node = "[[env "attr.unique.hostname" ]]"
-            '''
-
           [sinks.loki]
             type = "loki"
             inputs = [
+              "vector_raw",
               "docker_transformed",
-              "nomadify_hostname",
+              "glusterfs_raw",
+              "journald_raw",
             ]
             endpoint = "http://[[ range service "loki" ]][[ .Address ]]:[[ .Port ]][[ end ]]"
             encoding.codec = "json"
@@ -116,7 +115,20 @@ del(.host)
             remove_label_fields = true
             healthcheck.enabled = true
               [sinks.loki.labels]
-              node = "{{node}}"
+              host = "{{host}}"
+
+          [sources.internal_metrics]
+            type = "internal_metrics"
+
+          [sources.host_metrics]
+            type = "host_metrics"
+
+          [sinks.prometheus]
+            type = "prometheus_exporter"
+            inputs = [
+              "internal_metrics",
+              "host_metrics",
+            ]
         EOH
       }
       kill_timeout = "30s"