Setup binary cache on c3 and optimize nix settings.

common/binary-cache-server.nix

@@ -0,0 +1,53 @@
+{ config, pkgs, lib, ... }:
+{
+  # Binary cache server using nix-serve
+  # Serves built packages to other hosts in the cluster for faster rebuilds
+  #
+  # Setup:
+  # 1. Deploy this host first
+  # 2. SSH in and get public key: cat /persist/nix-cache/cache-pub-key.txt
+  # 3. Add that key to common/global/nix.nix in trusted-public-keys
+  # 4. Deploy all other hosts to pick up the cache
+
+  # Ensure cache directory is persisted
+  environment.persistence."/persist".directories = [
+    { directory = "/var/nix-cache"; user = "nix-serve"; group = "nix-serve"; mode = "0755"; }
+  ];
+
+  # Auto-generate cache keys on first boot
+  systemd.services.nix-cache-key-init = {
+    description = "Generate binary cache keys if missing";
+    wantedBy = [ "multi-user.target" ];
+    before = [ "nix-serve.service" ];
+    path = [ pkgs.nix ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+    };
+    script = ''
+      CACHE_DIR="/persist/nix-cache"
+      mkdir -p "$CACHE_DIR"
+
+      if [ ! -f "$CACHE_DIR/cache-priv-key.pem" ]; then
+        echo "Generating binary cache key pair..."
+        nix-store --generate-binary-cache-key ${config.networking.hostName}-cache "$CACHE_DIR/cache-priv-key.pem" "$CACHE_DIR/cache-pub-key.txt"
+        chmod 600 "$CACHE_DIR/cache-priv-key.pem"
+        chmod 644 "$CACHE_DIR/cache-pub-key.txt"
+        echo "Binary cache keys generated at $CACHE_DIR"
+        echo "Public key:"
+        cat "$CACHE_DIR/cache-pub-key.txt"
+      fi
+    '';
+  };
+
+  # Enable nix-serve
+  services.nix-serve = {
+    enable = true;
+    secretKeyFile = "/persist/nix-cache/cache-priv-key.pem";
+    bindAddress = "0.0.0.0";
+    port = 5000;
+  };
+
+  # Open firewall for LAN access
+  networking.firewall.allowedTCPPorts = [ 5000 ];
+}

common/global/nix.nix

@@ -1,8 +1,30 @@
 {
-  nix.settings.trusted-users = [
+  nix.settings = {
+    trusted-users = [
       "root"
       "@wheel"
     ];
+
+    # Binary cache configuration
+    substituters = [
+      "http://c3:5000"  # Local cluster cache on c3
+      "https://cache.nixos.org"
+    ];
+    trusted-public-keys = [
+      "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
+      # TODO: Add c3 cache public key here after first deploy of c3
+      # Get it with: ssh c3 cat /persist/nix-cache/cache-pub-key.txt
+      # "c3-cache:..."
+    ];
+
+    # Performance tuning
+    max-jobs = "auto";           # Use all cores for parallel builds
+    cores = 0;                   # Each build can use all cores
+    max-substitution-jobs = 16;  # Faster fetching from caches
+    http-connections = 25;       # More parallel downloads
+    download-attempts = 3;       # Retry failed downloads
+  };
+
   nix.gc = {
     automatic = true;
     dates = "weekly";

common/nomad.nix

@@ -26,12 +26,8 @@ in
           cidr = "100.64.0.0/10";
         };
         host_volume = {
-          code = {
-            path = "/data/compute/code";
-            read_only = true;
-          };
-          appdata = {
-            path = "/data/compute/appdata";
+          services = {
+            path = "/data/services";
             read_only = false;
           };
           nix-store = {

docs/MIGRATION_TODO.md

@@ -62,7 +62,7 @@ See [CLUSTER_REVAMP.md](./CLUSTER_REVAMP.md) for detailed procedures.
 - [x] prometheus.hcl - migrated to `/data/services`
 - [x] grafana.hcl - migrated to `/data/services` (2025-10-23)
 - [x] loki.hcl - migrated to `/data/services`
-- [ ] vector.hcl - needs update to remove glusterfs log collection (line 26, 101-109)
+- [x] vector.hcl - removed glusterfs log collection (2025-10-23)
 
 ### Databases (HIGH)
 - [x] clickhouse.hcl - migrated to `/data/services`
@@ -71,9 +71,8 @@ See [CLUSTER_REVAMP.md](./CLUSTER_REVAMP.md) for detailed procedures.
 ### Web Applications (HIGH-MEDIUM)
 - [x] wordpress.hcl - migrated to `/data/services`
 - [x] gitea.hcl - migrated to `/data/services` (2025-10-23)
-- [ ] wiki.hcl - uses `appdata` volume (points to `/data/compute/appdata`)
+- [x] wiki.hcl - migrated to `/data/services` (2025-10-23)
 - [x] plausible.hcl - stateless, no changes needed
-- [ ] tiddlywiki.hcl - uses `appdata` volume (points to `/data/compute/appdata`)
 
 ### Web Applications (LOW, may be deprecated)
 - [x] vikunja.hcl - migrated to `/data/services` (2025-10-23, not running)
@@ -93,8 +92,8 @@ See [CLUSTER_REVAMP.md](./CLUSTER_REVAMP.md) for detailed procedures.
 - [x] urbit.hcl - migrated to `/data/services`
 - [x] webodm.hcl - migrated to `/data/services` (2025-10-23, not running)
 - [x] velutrack.hcl - migrated to `/data/services`
-- [ ] resol-gateway.hcl - uses `code` volume (points to `/data/compute/code`)
-- [ ] igsync.hcl - uses `appdata` volume (points to `/data/compute/appdata`)
+- [x] resol-gateway.hcl - migrated to `/data/services` (2025-10-23)
+- [x] igsync.hcl - migrated to `/data/services` (2025-10-23)
 - [x] jupyter.hcl - migrated to `/data/services` (2025-10-23, not running)
 - [x] whoami.hcl - stateless test service, no changes needed
 
@@ -103,7 +102,7 @@ See [CLUSTER_REVAMP.md](./CLUSTER_REVAMP.md) for detailed procedures.
 - [x] postgres-backup.hcl - migrated to `/data/services`
 
 ### Host Volume Definitions (CRITICAL)
-- [ ] common/nomad.nix - update host_volume paths from `/data/compute/{appdata,code}` to `/data/services/{appdata,code}`
+- [x] common/nomad.nix - consolidated `appdata` and `code` volumes into single `services` volume (2025-10-23)
 
 ### Verification
 - [ ] All services healthy in Nomad
@@ -144,22 +143,18 @@ See [CLUSTER_REVAMP.md](./CLUSTER_REVAMP.md) for detailed procedures.
 
 ---
 
-**Last updated**: 2025-10-23 21:16
-**Current phase**: Phase 4 in progress (26/35 services migrated, 4 host-volume services + config updates remaining, 4 stateless)
+**Last updated**: 2025-10-23 22:30
+**Current phase**: Phase 4 complete! All services migrated to NFS
 **Note**: Phase 1 (fractal NixOS conversion) deferred until after GlusterFS migration is complete
 
 ## Migration Summary
 
-**Already migrated to `/data/services` (26 services):**
-mysql, mysql-backup, postgres, postgres-backup, redis, clickhouse, prometheus, grafana, loki, unifi, wordpress, gitea, traefik, evcc, weewx, netbox, farmos, webodm, jupyter, vikunja, urbit, code-server, beancount, velutrack, maps, media
-
-**Still need migration (4 services using host volumes):**
-- wiki (appdata), tiddlywiki (appdata), igsync (appdata), resol-gateway (code)
-- These require updating common/nomad.nix host_volume definitions first
+**All services migrated to `/data/services` (30 total):**
+mysql, mysql-backup, postgres, postgres-backup, redis, clickhouse, prometheus, grafana, loki, vector, unifi, wordpress, gitea, wiki, traefik, evcc, weewx, netbox, farmos, webodm, jupyter, vikunja, urbit, code-server, beancount, velutrack, maps, media, resol-gateway, igsync
 
 **Stateless/no changes needed (4 services):**
 authentik, adminer, plausible, whoami
 
-**Configuration updates needed:**
-- vector.hcl: remove glusterfs log collection
-- common/nomad.nix: update host_volume paths
+**Configuration changes:**
+- common/nomad.nix: consolidated `appdata` and `code` volumes into single `services` volume
+- vector.hcl: removed glusterfs log collection

hosts/c3/default.nix

@@ -4,6 +4,7 @@
     ../../common/encrypted-btrfs-layout.nix
     ../../common/global
     ../../common/compute-node.nix
+    ../../common/binary-cache-server.nix
     ./hardware.nix
   ];
 

services/igsync.hcl

@@ -19,24 +19,24 @@ job "igsync" {
       read_only = true
       source = "sw"
     }
-    volume "appdata" {
+    volume "services" {
       type = "host"
       read_only = false
-      source = "appdata"
+      source = "services"
     }
 
     task "sync" {
       driver = "exec"
 
       config {
-        command = "/appdata/igsync/run.sh"
+        command = "/data/services/igsync/run.sh"
       }
 
       user = "ppetru"
 
       volume_mount {
-        volume = "appdata"
-        destination = "/appdata"
+        volume = "services"
+        destination = "/data/services"
       }
       volume_mount {
         volume = "nix-store"

services/resol-gateway.hcl

@@ -2,10 +2,10 @@ job "resol-gateway" {
   datacenters = ["alo"]
 
   group "gw" {
-    volume "code" {
+    volume "services" {
       type = "host"
       read_only = true
-      source = "code"
+      source = "services"
     }
     volume "nix-store" {
       type = "host"
@@ -29,12 +29,12 @@ job "resol-gateway" {
 
       config {
         command = "/sw/bin/node"
-        args = ["/code/resol-vbus/examples/json-live-data-server/index.js"]
+        args = ["/data/services/resol-vbus/examples/json-live-data-server/index.js"]
       }
 
       volume_mount {
-        volume = "code"
-        destination = "/code"
+        volume = "services"
+        destination = "/data/services"
       }
       volume_mount {
         volume = "nix-store"

services/vector.hcl

@@ -23,7 +23,6 @@ job "vector" {
         ports = ["api", "metrics"]
         volumes = [
           "/var/run/docker.sock:/var/run/docker.sock:ro",
-          "/var/log/glusterfs:/var/log/glusterfs:ro",
           "/var/log/journal:/var/log/journal:ro",
           "/etc/machine-id:/etc/machine-id:ro",
         ]
@@ -98,16 +97,6 @@ if length(result) > 0 {
 .host = del(.label."com.hashicorp.nomad.node_name")
             '''
 
-          [sources.glusterfs_raw]
-            type = "file"
-            include = [ "/var/log/glusterfs/**/*.log" ]
-          [transforms.glusterfs_transformed]
-            type = "remap"
-            inputs = [ "glusterfs_raw" ]
-            source = '''
-.job_name = .file
-            '''
-
           [sources.journald_raw]
             type = "journald"
           [transforms.journald_transformed]
@@ -122,7 +111,6 @@ if length(result) > 0 {
             inputs = [
               "vector_transformed",
               "docker_transformed",
-              "glusterfs_transformed",
               "journald_transformed",
             ]
             endpoint = "http://[[ range service "loki" ]][[ .Address ]]:[[ .Port ]][[ end ]]"

services/wiki.hcl

@@ -2,10 +2,10 @@ job "wiki" {
   datacenters = ["alo"]
 
   group "servers" {
-    volume "appdata" {
+    volume "services" {
       type = "host"
       read_only = false
-      source = "appdata"
+      source = "services"
     }
     volume "nix-store" {
       type = "host"
@@ -31,8 +31,8 @@ job "wiki" {
       config {
         command = "/sw/bin/node"
         args = [
-            "/appdata/wiki/node_modules/tiddlywiki/tiddlywiki.js",
-            "/appdata/wiki/captainslog",
+            "/data/services/wiki/node_modules/tiddlywiki/tiddlywiki.js",
+            "/data/services/wiki/captainslog",
             "--listen",
             "host=0.0.0.0",
             "port=${NOMAD_PORT_captainslog}",
@@ -46,8 +46,8 @@ job "wiki" {
       user = "ppetru"
 
       volume_mount {
-        volume = "appdata"
-        destination = "/appdata"
+        volume = "services"
+        destination = "/data/services"
       }
       volume_mount {
         volume = "nix-store"
@@ -80,8 +80,8 @@ job "wiki" {
       config {
         command = "/sw/bin/node"
         args = [
-            "/appdata/wiki/node_modules/tiddlywiki/tiddlywiki.js",
-            "/appdata/wiki/alo",
+            "/data/services/wiki/node_modules/tiddlywiki/tiddlywiki.js",
+            "/data/services/wiki/alo",
             "--listen",
             "host=0.0.0.0",
             "port=${NOMAD_PORT_alo}",
@@ -93,8 +93,8 @@ job "wiki" {
       }
 
       volume_mount {
-        volume = "appdata"
-        destination = "/appdata"
+        volume = "services"
+        destination = "/data/services"
       }
       volume_mount {
         volume = "nix-store"
@@ -128,8 +128,8 @@ job "wiki" {
       config {
         command = "/sw/bin/node"
         args = [
-            "/appdata/wiki/node_modules/tiddlywiki/tiddlywiki.js",
-            "/appdata/wiki/pispace",
+            "/data/services/wiki/node_modules/tiddlywiki/tiddlywiki.js",
+            "/data/services/wiki/pispace",
             "--listen",
             "host=0.0.0.0",
             "port=${NOMAD_PORT_pispace}",
@@ -141,8 +141,8 @@ job "wiki" {
       }
 
       volume_mount {
-        volume = "appdata"
-        destination = "/appdata"
+        volume = "services"
+        destination = "/data/services"
       }
       volume_mount {
         volume = "nix-store"
@@ -176,8 +176,8 @@ job "wiki" {
       config {
         command = "/sw/bin/node"
         args = [
-            "/appdata/wiki/node_modules/tiddlywiki/tiddlywiki.js",
-            "/appdata/wiki/grok",
+            "/data/services/wiki/node_modules/tiddlywiki/tiddlywiki.js",
+            "/data/services/wiki/grok",
             "--listen",
             "host=0.0.0.0",
             "port=${NOMAD_PORT_grok}",
@@ -189,8 +189,8 @@ job "wiki" {
       }
 
       volume_mount {
-        volume = "appdata"
-        destination = "/appdata"
+        volume = "services"
+        destination = "/data/services"
       }
       volume_mount {
         volume = "nix-store"