ppetru/alo-cluster
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Only keep 10 snapshots, and push metrics.

Browse Source
This commit is contained in:
Petru
2025-11-04 10:22:11 +00:00
parent 49afc0c084
commit 5ef4d832fb
2 changed files with 40 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
common/nfs-services-server.nix
View File

@@ -103,11 +103,14 @@ in
] ++ (lib.forEach cfg.standbys (standby: {
"replicate-services-to-${standby}" = {
description = "Replicate /persist/services to ${standby}";
path = [ pkgs.btrfs-progs pkgs.openssh pkgs.coreutils pkgs.findutils pkgs.gnugrep ];
path = [ pkgs.btrfs-progs pkgs.openssh pkgs.coreutils pkgs.findutils pkgs.gnugrep pkgs.curl ];
script = ''
set -euo pipefail
START_TIME=$(date +%s)
REPLICATION_SUCCESS=0
SSH_KEY="/persist/root/.ssh/btrfs-replication"
if [ ! -f "$SSH_KEY" ]; then
echo "ERROR: SSH key not found at $SSH_KEY"
@@ -134,11 +137,13 @@ in
ssh -i "$SSH_KEY" -o StrictHostKeyChecking=accept-new root@${standby} \
"btrfs receive /persist/services-standby"; then
echo "Incremental send completed successfully"
REPLICATION_SUCCESS=1
else
echo "Incremental send failed (likely missing parent on receiver), falling back to full send"
btrfs send "$SNAPSHOT_PATH" | \
ssh -i "$SSH_KEY" -o StrictHostKeyChecking=accept-new root@${standby} \
"btrfs receive /persist/services-standby"
REPLICATION_SUCCESS=1
fi
else
# First snapshot, do full send
@@ -146,10 +151,28 @@ in
btrfs send "$SNAPSHOT_PATH" | \
ssh -i "$SSH_KEY" -o StrictHostKeyChecking=accept-new root@${standby} \
"btrfs receive /persist/services-standby"
REPLICATION_SUCCESS=1
fi
# Cleanup old snapshots on sender (keep last 24 hours = 288 snapshots at 5min intervals)
find /persist -maxdepth 1 -name 'services@*' -mmin +1440 -exec btrfs subvolume delete {} \;
# Cleanup old snapshots on sender (keep last 10 snapshots, sorted by name/timestamp)
ls -1d /persist/services@* 2>/dev/null | sort | head -n -10 | xargs -r btrfs subvolume delete
# Calculate metrics
END_TIME=$(date +%s)
DURATION=$((END_TIME - START_TIME))
SNAPSHOT_COUNT=$(ls -1d /persist/services@* 2>/dev/null | wc -l)
# Push metrics to Prometheus pushgateway
cat <<METRICS | curl --data-binary @- http://pushgateway.service.consul:9091/metrics/job/nfs_replication/instance/${standby}
# TYPE nfs_replication_last_success_timestamp gauge
nfs_replication_last_success_timestamp $END_TIME
# TYPE nfs_replication_duration_seconds gauge
nfs_replication_duration_seconds $DURATION
# TYPE nfs_replication_snapshot_count gauge
nfs_replication_snapshot_count $SNAPSHOT_COUNT
# TYPE nfs_replication_success gauge
nfs_replication_success $REPLICATION_SUCCESS
METRICS
'';
serviceConfig = {