Compare commits

..

4 Commits

4 changed files with 156 additions and 20 deletions

View File

@@ -42,14 +42,24 @@ in
# Cleanup old snapshots on standby (keep last 4 hours for HA failover) # Cleanup old snapshots on standby (keep last 4 hours for HA failover)
systemd.services.cleanup-services-standby-snapshots = { systemd.services.cleanup-services-standby-snapshots = {
description = "Cleanup old btrfs snapshots in services-standby"; description = "Cleanup old btrfs snapshots in services-standby";
path = [ pkgs.btrfs-progs pkgs.findutils ]; path = [ pkgs.btrfs-progs pkgs.findutils pkgs.coreutils ];
script = '' script = ''
set -euo pipefail set -euo pipefail
# Keep last 4 hours of snapshots (48 snapshots at 5min intervals)
find /persist/services-standby -maxdepth 1 -name 'services@*' -mmin +240 -exec btrfs subvolume delete {} \; || true
'';
# Keep at least 2 hours of snapshots (24 snapshots at 5min intervals)
MIN_KEEP=24
# Count existing snapshots
count=$(find /persist/services-standby -maxdepth 1 -name 'services@*' -type d | wc -l)
# Only delete old snapshots if we have more than the minimum
if [ $count -gt $MIN_KEEP ]; then
# Delete snapshots older than 4 hours
find /persist/services-standby -maxdepth 1 -name 'services@*' -mmin +240 -exec btrfs subvolume delete {} \; || true
else
echo "Only $count snapshots found, keeping all (minimum: $MIN_KEEP)"
fi
'';
serviceConfig = { serviceConfig = {
Type = "oneshot"; Type = "oneshot";
User = "root"; User = "root";

View File

@@ -3,9 +3,10 @@
imports = [ imports = [
../../common/encrypted-btrfs-layout.nix ../../common/encrypted-btrfs-layout.nix
../../common/global ../../common/global
../../common/desktop-node.nix # Hyprland + GUI environment ../../common/cluster-member.nix
../../common/cluster-member.nix # Consul + storage clients ../../common/cluster-tools.nix
../../common/cluster-tools.nix # Nomad CLI (no service) ../../common/nomad-worker.nix
../../common/nfs-services-standby.nix
./hardware.nix ./hardware.nix
]; ];

View File

@@ -6,7 +6,6 @@
../../common/cluster-member.nix # Consul + storage clients ../../common/cluster-member.nix # Consul + storage clients
../../common/nomad-worker.nix # Nomad client (runs jobs) ../../common/nomad-worker.nix # Nomad client (runs jobs)
# NOTE: zippy is NOT a server - no nomad-server.nix import # NOTE: zippy is NOT a server - no nomad-server.nix import
# ../../common/ethereum.nix
../../common/nfs-services-server.nix # NFS server for /data/services ../../common/nfs-services-server.nix # NFS server for /data/services
# To move NFS server role to another host: # To move NFS server role to another host:
# 1. Follow procedure in docs/NFS_FAILOVER.md # 1. Follow procedure in docs/NFS_FAILOVER.md
@@ -24,6 +23,8 @@
networking.hostName = "zippy"; networking.hostName = "zippy";
services.tailscaleAutoconnect.authkey = "tskey-auth-ktKyQ59f2p11CNTRL-ut8E71dLWPXsVtb92hevNX9RTjmk4owBf"; services.tailscaleAutoconnect.authkey = "tskey-auth-ktKyQ59f2p11CNTRL-ut8E71dLWPXsVtb92hevNX9RTjmk4owBf";
# NFS server configuration: replicate to c1 as standby nfsServicesServer.standbys = [
nfsServicesServer.standbys = [ "c1" ]; "c1"
"sparky"
];
} }

View File

@@ -14,6 +14,102 @@ BLUE='\033[0;34m'
YELLOW='\033[0;33m' YELLOW='\033[0;33m'
NC='\033[0m' # No Color NC='\033[0m' # No Color
# Normalize nix store paths by replacing 32-char hashes with placeholder
normalize_nix_paths() {
sed -E 's|/nix/store/[a-z0-9]{32}-|/nix/store/HASH-|g'
}
# Filter diff output to remove hunks where only nix store hashes differ
# Returns: filtered diff (empty if only hash changes), exit code 0 if real changes found
filter_hash_only_diffs() {
local diff_output="$1"
local current_hunk=""
local output=""
local has_real_changes=false
# Process line by line
while IFS= read -r line || [ -n "$line" ]; do
if [[ "$line" =~ ^@@ ]]; then
# New hunk starts - process previous one if it exists
if [ -n "$current_hunk" ]; then
if hunk_has_real_changes "$current_hunk"; then
output+="$current_hunk"$'\n'
has_real_changes=true
fi
fi
# Start new hunk
current_hunk="$line"$'\n'
else
# Add line to current hunk
current_hunk+="$line"$'\n'
fi
done <<< "$diff_output"
# Process last hunk
if [ -n "$current_hunk" ]; then
if hunk_has_real_changes "$current_hunk"; then
output+="$current_hunk"
has_real_changes=true
fi
fi
# Remove trailing newline
output="${output%$'\n'}"
if [ "$has_real_changes" = true ]; then
echo "$output"
return 0
else
return 1
fi
}
# Check if a diff hunk has real changes (not just hash changes)
hunk_has_real_changes() {
local hunk="$1"
# Use temp file to avoid bash here-string issues
local temp_hunk=$(mktemp)
printf '%s' "$hunk" > "$temp_hunk"
local minus_lines=()
local plus_lines=()
# Extract - and + lines (skip @@ and context lines)
while IFS= read -r line || [ -n "$line" ]; do
if [[ "$line" =~ ^- && ! "$line" =~ ^--- ]]; then
minus_lines+=("${line:1}") # Remove the - prefix
elif [[ "$line" =~ ^\+ && ! "$line" =~ ^\+\+\+ ]]; then
plus_lines+=("${line:1}") # Remove the + prefix
fi
done < "$temp_hunk"
rm -f "$temp_hunk"
# If counts don't match, there are structural changes
if [ ${#minus_lines[@]} -ne ${#plus_lines[@]} ]; then
return 0 # Has real changes
fi
# If no changes at all, skip
if [ ${#minus_lines[@]} -eq 0 ]; then
return 1 # No real changes
fi
# Compare each pair of lines after normalization
for i in "${!minus_lines[@]}"; do
local minus_norm=$(echo "${minus_lines[$i]}" | normalize_nix_paths)
local plus_norm=$(echo "${plus_lines[$i]}" | normalize_nix_paths)
if [ "$minus_norm" != "$plus_norm" ]; then
return 0 # Has real changes
fi
done
# All changes are hash-only
return 1
}
# Check for nvd # Check for nvd
if ! command -v nvd &> /dev/null; then if ! command -v nvd &> /dev/null; then
echo "Error: nvd not found in PATH" echo "Error: nvd not found in PATH"
@@ -205,11 +301,17 @@ for host in $hosts; do
echo echo
echo -e " ${BLUE}$basename${NC}" echo -e " ${BLUE}$basename${NC}"
# If it's a directory, diff key files within it # If it's a directory, diff all files within it
if [ -d "$old_path" ] && [ -d "$new_path" ]; then if [ -d "$old_path" ] && [ -d "$new_path" ]; then
# Focus on important files # Count files to avoid processing huge directories
for pattern in "activate" "etc/*" "*.conf" "*.fish" "*.service" "*.nix"; do file_count=$(find "$new_path" -maxdepth 3 -type f 2>/dev/null | wc -l)
while IFS= read -r file; do
# Skip very large directories (e.g., system-path with 900+ files)
if [ "$file_count" -gt 100 ]; then
echo " (skipping directory with $file_count files - too large)"
else
# Diff all files in the directory
for file in $(find "$new_path" -maxdepth 3 -type f 2>/dev/null); do
[ -z "$file" ] && continue [ -z "$file" ] && continue
relpath="${file#$new_path/}" relpath="${file#$new_path/}"
old_file="$old_path/$relpath" old_file="$old_path/$relpath"
@@ -217,16 +319,38 @@ for host in $hosts; do
if [ -f "$old_file" ] && [ -f "$file" ]; then if [ -f "$old_file" ] && [ -f "$file" ]; then
# Check if file is text # Check if file is text
if file "$file" | grep -q "text"; then if file "$file" | grep -q "text"; then
# Get diff output
diff_output=$(diff -u "$old_file" "$file" 2>/dev/null | head -50 | tail -n +3 || true)
# Filter hash-only changes
if [ -n "$diff_output" ]; then
filtered_diff=$(filter_hash_only_diffs "$diff_output" || true)
if [ -n "$filtered_diff" ]; then
echo -e " ${YELLOW}$relpath:${NC}" echo -e " ${YELLOW}$relpath:${NC}"
diff -u "$old_file" "$file" 2>/dev/null | head -50 | tail -n +3 | sed 's/^/ /' || true echo "$filtered_diff" | sed 's/^/ /'
fi
fi
fi fi
fi fi
done < <(find "$new_path" -type f -name "$pattern" 2>/dev/null | head -20)
done done
fi
# If it's a file, diff it directly # If it's a file, diff it directly
elif [ -f "$old_path" ] && [ -f "$new_path" ]; then elif [ -f "$old_path" ] && [ -f "$new_path" ]; then
if file "$new_path" | grep -q "text"; then if file "$new_path" | grep -q "text"; then
diff -u "$old_path" "$new_path" 2>/dev/null | head -50 | tail -n +3 | sed 's/^/ /' || true # Get diff output
diff_output=$(diff -u "$old_path" "$new_path" 2>/dev/null | head -50 | tail -n +3 || true)
# Filter hash-only changes
if [ -n "$diff_output" ]; then
filtered_diff=$(filter_hash_only_diffs "$diff_output" || true)
if [ -n "$filtered_diff" ]; then
echo "$filtered_diff" | sed 's/^/ /'
else
echo " (only hash changes)"
fi
fi
else else
echo " (binary file)" echo " (binary file)"
fi fi