From 94f71cc62e64aab894cdbac5d1b770e4d994638e Mon Sep 17 00:00:00 2001
From: Petru Paler <petru@paler.net>
Date: Thu, 23 Oct 2025 21:59:08 +0100
Subject: [PATCH] Setup binary cache on c3 and optimize nix settings.

---
 common/binary-cache-server.nix | 53 ++++++++++++++++++++++++++++++++++
 common/global/nix.nix          | 30 ++++++++++++++++---
 hosts/c3/default.nix           |  1 +
 3 files changed, 80 insertions(+), 4 deletions(-)
 create mode 100644 common/binary-cache-server.nix

diff --git a/common/binary-cache-server.nix b/common/binary-cache-server.nix
new file mode 100644
index 0000000..52d83cd
--- /dev/null
+++ b/common/binary-cache-server.nix
@@ -0,0 +1,53 @@
+{ config, pkgs, lib, ... }:
+{
+  # Binary cache server using nix-serve
+  # Serves built packages to other hosts in the cluster for faster rebuilds
+  #
+  # Setup:
+  # 1. Deploy this host first
+  # 2. SSH in and get public key: cat /persist/nix-cache/cache-pub-key.txt
+  # 3. Add that key to common/global/nix.nix in trusted-public-keys
+  # 4. Deploy all other hosts to pick up the cache
+
+  # Ensure cache directory is persisted
+  environment.persistence."/persist".directories = [
+    { directory = "/var/nix-cache"; user = "nix-serve"; group = "nix-serve"; mode = "0755"; }
+  ];
+
+  # Auto-generate cache keys on first boot
+  systemd.services.nix-cache-key-init = {
+    description = "Generate binary cache keys if missing";
+    wantedBy = [ "multi-user.target" ];
+    before = [ "nix-serve.service" ];
+    path = [ pkgs.nix ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+    };
+    script = ''
+      CACHE_DIR="/persist/nix-cache"
+      mkdir -p "$CACHE_DIR"
+
+      if [ ! -f "$CACHE_DIR/cache-priv-key.pem" ]; then
+        echo "Generating binary cache key pair..."
+        nix-store --generate-binary-cache-key ${config.networking.hostName}-cache "$CACHE_DIR/cache-priv-key.pem" "$CACHE_DIR/cache-pub-key.txt"
+        chmod 600 "$CACHE_DIR/cache-priv-key.pem"
+        chmod 644 "$CACHE_DIR/cache-pub-key.txt"
+        echo "Binary cache keys generated at $CACHE_DIR"
+        echo "Public key:"
+        cat "$CACHE_DIR/cache-pub-key.txt"
+      fi
+    '';
+  };
+
+  # Enable nix-serve
+  services.nix-serve = {
+    enable = true;
+    secretKeyFile = "/persist/nix-cache/cache-priv-key.pem";
+    bindAddress = "0.0.0.0";
+    port = 5000;
+  };
+
+  # Open firewall for LAN access
+  networking.firewall.allowedTCPPorts = [ 5000 ];
+}
diff --git a/common/global/nix.nix b/common/global/nix.nix
index e3f9d4b..730f393 100644
--- a/common/global/nix.nix
+++ b/common/global/nix.nix
@@ -1,8 +1,30 @@
 {
-  nix.settings.trusted-users = [
-    "root"
-    "@wheel"
-  ];
+  nix.settings = {
+    trusted-users = [
+      "root"
+      "@wheel"
+    ];
+
+    # Binary cache configuration
+    substituters = [
+      "http://c3:5000"  # Local cluster cache on c3
+      "https://cache.nixos.org"
+    ];
+    trusted-public-keys = [
+      "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
+      # TODO: Add c3 cache public key here after first deploy of c3
+      # Get it with: ssh c3 cat /persist/nix-cache/cache-pub-key.txt
+      # "c3-cache:..."
+    ];
+
+    # Performance tuning
+    max-jobs = "auto";           # Use all cores for parallel builds
+    cores = 0;                   # Each build can use all cores
+    max-substitution-jobs = 16;  # Faster fetching from caches
+    http-connections = 25;       # More parallel downloads
+    download-attempts = 3;       # Retry failed downloads
+  };
+
   nix.gc = {
     automatic = true;
     dates = "weekly";
diff --git a/hosts/c3/default.nix b/hosts/c3/default.nix
index aec9444..35d65d5 100644
--- a/hosts/c3/default.nix
+++ b/hosts/c3/default.nix
@@ -4,6 +4,7 @@
     ../../common/encrypted-btrfs-layout.nix
     ../../common/global
     ../../common/compute-node.nix
+    ../../common/binary-cache-server.nix
     ./hardware.nix
   ];