From 4cb69f1f83ee4d7a0a571a2b3570e57f77c4c448 Mon Sep 17 00:00:00 2001
From: Petru Paler <petru@paler.net>
Date: Tue, 1 Aug 2023 06:33:50 +0100
Subject: [PATCH] Replace promtail with vector for loki.

---
 hosts/common/nomad.nix |  1 -
 services/traefik.hcl   | 66 ------------------------------------------
 services/vector.hcl    | 19 +++++++-----
 3 files changed, 11 insertions(+), 75 deletions(-)

diff --git a/hosts/common/nomad.nix b/hosts/common/nomad.nix
index 816a5c5..1469b5b 100644
--- a/hosts/common/nomad.nix
+++ b/hosts/common/nomad.nix
@@ -47,7 +47,6 @@ in
         "job_name"
         "task_group_name"
         "task_name"
-        "namespace"
         "node_name"
       ];
     };
diff --git a/services/traefik.hcl b/services/traefik.hcl
index 6736c48..282bbd5 100644
--- a/services/traefik.hcl
+++ b/services/traefik.hcl
@@ -13,7 +13,6 @@ job "traefik" {
         #host_network = "tailscale"
         static = "9002"
       }
-      port "promtail_healthcheck" {}
     }
 
     task "traefik" {
@@ -147,70 +146,5 @@ EOH
         cap_add      = ["NET_ADMIN", "NET_BROADCAST", "NET_RAW"]
       }
     }
-
-    task "promtail" {
-      driver = "docker"
-      config {
-        image = "grafana/promtail:2.2.0"
-        args = [
-          "-config.file",
-          "local/config.yaml",
-          "-print-config-stderr",
-        ]
-        # the only port required is for the healthcheck
-        ports = ["promtail_healthcheck"]
-      }
-      # the template with Promtail's YAML configuration file, configuring the files to scrape,
-      # the Loki server to send the logs to (based on a registered Consul service, but it could be a fixed URL),
-      # the regex to parse the Common Log Format (https://en.wikipedia.org/wiki/Common_Log_Format) used for access logs,
-      # and the labels ( HTTP method and status code)
-      template {
-        data = <<EOH
-          server:
-            http_listen_port: {{ env "NOMAD_PORT_promtail_healthcheck" }}
-            grpc_listen_port: 0
-          positions:
-            filename: /alloc/positions.yaml
-          client:
-            url: http://{{ range service "loki" }}{{ .Address }}:{{ .Port }}{{ end }}/loki/api/v1/push
-          scrape_configs:
-          - job_name: local
-            static_configs:
-            - targets:
-                - localhost
-              labels:
-                job: traefik
-                __path__: "/alloc/logs/traefik.std*.0"
-            pipeline_stages:
-              - regex:
-                  expression: '^(?P<remote_addr>[\w\.]+) - (?P<remote_user>[^ ]*) \[(?P<time_local>[^\]]+)\] "(?P<method>[^ ]*) (?P<request>[^ ]*) (?P<protocol>[^ ]*)" (?P<status>[\d]+) (?P<body_bytes_sent>[\d]+) "(?P<http_referer>[^"]*)" "(?P<http_user_agent>[^"]*)"? (?P<total_requests>\d+) "(?P<frontend_name>.+)" "(?P<backend_url>.+)" (?P<duration_ms>\d+)ms'
-              - labels:
-                  method:
-                  status:
-                  frontend_name:
-        EOH
-        destination = "local/config.yaml"
-        }
-        resources {
-          cpu    = 50
-          memory = 256
-        }
-        # poststart, and sidecar=true, so Promtail will start *after* Traefik ( since it has nothing to do before Traefik isup and running),
-        # and run for as long as it does
-        lifecycle {
-          hook    = "poststart"
-          sidecar = true
-        }
-        # a service for a health check to determine the state of Promtail
-        service {
-          check {
-            type     = "http"
-            port     = "promtail_healthcheck"
-            path     = "/ready"
-            interval = "10s"
-            timeout  = "2s"
-          }
-        }
-    }
   }
 }
diff --git a/services/vector.hcl b/services/vector.hcl
index 2d9ea15..5a9786c 100644
--- a/services/vector.hcl
+++ b/services/vector.hcl
@@ -59,8 +59,15 @@ job "vector" {
             type = "remap"
             inputs = [ "logs" ]
             source = '''
-                   .debug = parse_key_value!(.message)
-                   .job_name = split(get!(value: .label, path: ["com.hashicorp.nomad.job_name"]), "/")[0] ?? get!(value: .label, path: ["com.hashicorp.nomad.job_name"])
+.job_name, err = .label."com.hashicorp.nomad.job_name" + .label."com.hashicorp.nomad.task_group_name" + .label."com.hashicorp.nomad.task_name"
+result = parse_regex_all!(.message, r'^(?P<log>.+) (?P<total_requests>\d+) "(?P<frontend_name>.+)" "(?P<backend_url>.+)" (?P<duration_ms>\d+)ms$')
+if length(result) > 0 {
+  . |= parse_apache_log!(result[0].log, "combined")
+  .backend_url = result[0].backend_url
+  .frontend_name = result[0].frontend_name
+  .duration_ms = to_int!(result[0].duration_ms)
+  .total_requests = to_int!(result[0].total_requests)
+}
             '''
           [sinks.loki]
             type = "loki"
@@ -73,13 +80,9 @@ job "vector" {
             remove_label_fields = true
             healthcheck.enabled = true
               [sinks.loki.labels]
-              job = "{{label.\"com.hashicorp.nomad.job_name\" }}"
-              task = "{{label.\"com.hashicorp.nomad.task_name\" }}"
-              group = "{{label.\"com.hashicorp.nomad.task_group_name\" }}"
-              namespace = "{{label.\"com.hashicorp.nomad.namespace\" }}"
+              job_name = "{{job_name}}"
               node = "{{label.\"com.hashicorp.nomad.node_name\" }}"
-              correlation_id = "{{ message.requestId }}"
-              stream = "{{ stream }}"
+              frontend_name = "{{frontend_name}}"
         EOH
       }
       kill_timeout = "30s"