Increase redo log capacity.

All done with gluster volume reset compute <flag>

common/compute-node.nix

@@ -10,6 +10,7 @@
     ./sshd.nix
     ./user-ppetru.nix
     ./unattended-encryption.nix
+    ./syncthing-data.nix
     ./systemd-boot.nix
   ];
 }

common/global/nix.nix

@@ -1,3 +1,8 @@
 {
   nix.settings.trusted-users = [ "root" "@wheel" ];
+  nix.gc = {
+    automatic = true;
+    dates = "weekly";
+    options = "--delete-older-than 30d";
+  };
 }

common/syncthing-data.nix

@@ -0,0 +1,44 @@
+{
+  # TODO: when deploying this to a new machine for the first time, first
+  # comment this out to get /data/sync created with the right owner and
+  # permissions. then, do it again with persistence enabled.
+  # This could list the owner user but I'm not sure if it's already created at
+  # the time impermanence setup runs.
+  environment.persistence."/persist".directories = [
+    "/data/sync"
+  ];
+
+  services.syncthing = {
+    enable = true;
+    dataDir = "/data/sync";
+    openDefaultPorts = true;
+    #guiAddress = "0.0.0.0:8384";
+    overrideDevices = true;
+    overrideFolders = true;
+    settings = {
+      devices = {
+        "c1" = { id = "53JGRHQ-VGBYIGH-7IT6Z5S-3IMRY2I-LJZAE3B-QUDH3QF-4F4QKVC-VBWPJQ4"; };
+        "c2" = { id = "Z3D476N-PUV6WAD-DSJWVBO-TWEOD4I-KDDMNRB-QEBOP6T-BYPGYTX-RAAYGAW"; };
+        "c3" = { id = "JUQZO6M-65VXR5Y-ZDWY66F-F2EEMQC-HKMD6EB-33TAGPY-ADCB2OX-QT6IAAQ"; };
+      };
+      folders = {
+        "wordpress" = {
+          path = "/data/sync/wordpress";
+          devices = [
+            "c1"
+            "c2"
+            "c3"
+          ];
+          ignorePerms = false;
+          versioning = {
+            type = "staggered";
+            params = {
+              cleanInterval = "3600";
+              maxAge = "15768000";
+            };
+          };
+        };
+      };
+    };
+  };
+}

common/systemd-boot.nix

@@ -1,5 +1,7 @@
 {
-  # Use the systemd-boot EFI boot loader.
-  boot.loader.systemd-boot.enable = true;
+  boot.loader.systemd-boot = {
+    enable = true;
+    configurationLimit = 5;
+  };
   boot.loader.efi.canTouchEfiVariables = true;
 }

flake.lock

@@ -55,11 +55,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1705183652,
-        "narHash": "sha256-rnfkyUH0x72oHfiSDhuCHDHg3gFgF+lF8zkkg5Zihsw=",
+        "lastModified": 1705916986,
+        "narHash": "sha256-iBpfltu6QvN4xMpen6jGGEb6jOqmmVQKUrXdOJ32u8w=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "428544ae95eec077c7f823b422afae5f174dee4b",
+        "rev": "d7f206b723e42edb09d9d753020a84b3061a79d8",
         "type": "github"
       },
       "original": {
@@ -71,11 +71,11 @@
     },
     "nixpkgs-unstable": {
       "locked": {
-        "lastModified": 1705133751,
-        "narHash": "sha256-rCIsyE80jgiOU78gCWN3A0wE0tR2GI5nH6MlS+HaaSQ=",
+        "lastModified": 1705856552,
+        "narHash": "sha256-JXfnuEf5Yd6bhMs/uvM67/joxYKoysyE3M2k6T3eWbg=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "9b19f5e77dd906cb52dade0b7bd280339d2a1f3d",
+        "rev": "612f97239e2cc474c13c9dafa0df378058c5ad8d",
         "type": "github"
       },
       "original": {

services/adminer.hcl

@@ -16,6 +16,10 @@ job "adminer" {
         ports = ["http"]
       }
 
+      env {
+        ADMINER_DEFAULT_SERVER = "mysql.service.consul"
+      }
+
       service {
         name = "adminer"
         port = "http"

services/beancount.hcl

@@ -16,6 +16,7 @@ job "beancount" {
 
     task "fava" {
       driver = "docker"
+      user = "1000"
 
       config {
         image = "gitea.v.paler.net/ppetru/fava:latest"
@@ -39,6 +40,10 @@ job "beancount" {
           "traefik.http.routers.finances.middlewares=authentik@file",
         ]
       }
+
+      resources {
+        memory = 400
+      }
     }
   }
 }

services/clickhouse.hcl

@@ -12,7 +12,7 @@ job "clickhouse" {
       driver = "docker"
 
       config {
-        image = "clickhouse/clickhouse-server:23.3.7.5-alpine"
+        image = "clickhouse/clickhouse-server:23.8-alpine"
         volumes = [
           "/data/compute/appdata/clickhouse:/var/lib/clickhouse",
           "local/clickhouse-config.xml:/etc/clickhouse-server/config.d/logging.xml:ro",

services/farmos.hcl

@@ -12,7 +12,7 @@ job "farmos" {
       driver = "docker"
 
       config {
-        image = "farmos/farmos:3.0.0"
+        image = "farmos/farmos:3.0.1"
         ports = ["http"]
         volumes = [
           "/data/compute/appdata/farmos/sites:/opt/drupal/web/sites",

services/mysql.hcl

@@ -1,6 +1,10 @@
 job "mysql" {
   datacenters = ["alo"]
 
+  meta {
+    uuid = uuidv4()
+  }
+
   group "db" {
     network {
       port "db" {
@@ -14,7 +18,11 @@ job "mysql" {
       driver = "docker"
 
       config {
-        image = "mysql:8.2"
+        image = "mysql:8.3"
+        args = [
+          # 200M, up from default of 100M to see if it helps with running out and crashing (glusterfs weirdness suspected)
+          "--innodb-redo-log-capacity=209715200",
+        ]
         ports = ["db"]
         volumes = [
           "/data/compute/appdata/mysql:/var/lib/mysql",

services/wordpress-backup.hcl

@@ -0,0 +1,30 @@
+job "wordpress-backup" {
+  datacenters = ["alo"]
+  type = "batch"
+
+  periodic {
+    cron = "*/5 * * * * *"
+    prohibit_overlap = true
+  }
+
+  group "cron" {
+    task "rsync" {
+      driver = "raw_exec"
+
+      config {
+        command = "/bin/sh"
+        args = [ "local/script.sh" ]
+      }
+
+      template {
+        destination = "local/script.sh"
+        data = <<EOH
+          set -e
+          /run/current-system/sw/bin/rsync -a /data/sync/wordpress/ /data/compute/appdata/wordpress/ && \
+          echo "last_success $(date +%s)" | \
+          /run/current-system/sw/bin/curl --data-binary @- http://pushgateway.service.consul:9091/metrics/job/wordpress_backup
+        EOH
+      }
+    }
+  }
+}

services/wordpress.hcl

@@ -14,12 +14,13 @@ job "wordpress" {
 
     task "server" {
       driver = "docker"
+      user = "237"
 
       config {
         image = "wordpress:latest"
         ports = ["http"]
         volumes = [
-          "/data/compute/appdata/wordpress:/var/www/html",
+          "/data/sync/wordpress:/var/www/html",
         ]
       }
 

stateful-commands.txt

@@ -6,23 +6,6 @@ glusterfs setup on c1:
   * gluster volume start compute
   * gluster volume bitrot compute enable
 
-glusterfs tweaking for wordpress performance:
-  * gluster volume set compute nl-cache on
-  * gluster volume set compute features.cache-invalidation on
-  * gluster volume set compute features.cache-invalidation-timeout 600
-  * gluster volume set compute performance.cache-invalidation on
-  * gluster volume set compute performance.qr-cache-timeout 600
-  * gluster volume set compute performance.stat-prefetch on
-  * gluster volume set compute performance.md-cache-timeout 600
-  * gluster volume set compute network.inode-lru-limit 200000
-  * gluster volume set compute performance.nl-cache-timeout 600
-  * gluster volume set compute performance.parallel-readdir on
-  * gluster volume set compute performance.cache-size 1GB
-  * gluster volume set compute client.event-threads 8
-  * gluster volume set compute server.event-threads 8
-  * gluster volume set compute cluster.readdir-optimize on
-  * gluster volume set compute server.outstanding-rpc-limit 256
-
 mysql credentials
   * Put secrets/mysql_root_password into a Nomad var named secrets/mysql.root_password