diff --git a/hosts/common/compute-node.nix b/hosts/common/compute-node.nix index d4a9f64..d399777 100644 --- a/hosts/common/compute-node.nix +++ b/hosts/common/compute-node.nix @@ -2,9 +2,9 @@ { imports = [ ./consul.nix + ./glusterfs.nix ./impermanence.nix ./nomad.nix - ./seaweedfs.nix ./sshd.nix ./user-ppetru.nix ./unattended-encryption.nix diff --git a/hosts/common/glusterfs.nix b/hosts/common/glusterfs.nix new file mode 100644 index 0000000..f7be77b --- /dev/null +++ b/hosts/common/glusterfs.nix @@ -0,0 +1,13 @@ +{ pkgs, config, lib, ... }: +{ + services.glusterfs = { + enable = true; + }; + + environment.persistence."/persist".directories = [ + "/var/lib/glusterd" + ]; + + # TODO: each volume needs its own port starting at 49152 + networking.firewall.allowedTCPPorts = [ 24007 24008 24009 49152 49153 49154 49155 ]; +} diff --git a/hosts/common/seaweedfs.nix b/hosts/common/seaweedfs.nix deleted file mode 100644 index 3ef96a3..0000000 --- a/hosts/common/seaweedfs.nix +++ /dev/null @@ -1,56 +0,0 @@ -{ pkgs, config, lib, ... }: -with builtins; -let - masters = { - c1 = "192.168.1.71:9333"; - c2 = "192.168.1.72:9333"; - c3 = "192.168.1.73:9333"; - }; - master_enabled = masters ? ${config.networking.hostName}; -in -{ - imports = [ ./seaweedfs_lib.nix ]; - - services.seaweedfs.master = { - enable = master_enabled; - peers = attrValues masters; - extraConfig = { - ip = head (split ":" masters.${config.networking.hostName}); - defaultReplication = "001"; - volumeSizeLimitMB = 8000; - }; - }; - services.seaweedfs.filer = { - enable = true; - master = attrValues masters; - }; - services.seaweedfs.volumes = { - default = { - enable = true; - mserver = attrValues masters; - dir = [ "/persist/weed/volume-default" ]; - extraConfig = { - dataCenter = "alo"; - rack = "g"; - max = 32; - }; - }; - }; - - users.users.seaweedfs.group = "seaweedfs"; - users.groups.seaweedfs = {}; - - environment.persistence."/persist".directories = [ - "/var/lib/seaweedfs" - ]; - - networking.firewall.allowedTCPPorts = [ - config.services.seaweedfs.volumes.default.port - 19333 - config.services.seaweedfs.filer.port - 18888 - ] ++ (if master_enabled then [ - config.services.seaweedfs.master.port - 19333 - ] else [ ]); -} diff --git a/hosts/common/seaweedfs_lib.nix b/hosts/common/seaweedfs_lib.nix deleted file mode 100644 index 03eafde..0000000 --- a/hosts/common/seaweedfs_lib.nix +++ /dev/null @@ -1,188 +0,0 @@ -# https://discourse.nixos.org/t/casual-nixpkgs-contributions/9607/11? - -{ config, lib, pkgs, ... }: -with lib; -let - - seaweedfs = pkgs.unstable.seaweedfs; - - user = "seaweedfs"; - group = "seaweedfs"; - cfg = config.services.seaweedfs; - enabledVolumes = filterAttrs (_: v: v.enable) cfg.volumes; - anyEnabled = cfg.master.enable || cfg.filer.enable || cfg.webdav.enable - || enabledVolumes != { }; - - mkCmdLineArguments = mapAttrsToList (option: value: - if isBool value then - "-${option}" - else - "-${option}=${ - if isList value then - builtins.concatStringsSep "," value - else - toString value - }"); - - mkWeedExec = subcmd: options: - (toString ([ "${seaweedfs}/bin/weed" "-logtostderr" subcmd ] ++ mkCmdLineArguments - ((removeAttrs options [ "enable" "extraConfig" ]) - // (if options ? "extraConfig" then - (removeAttrs options.extraConfig (builtins.attrNames options)) - else - { })))); - - mkExtraConfigOption = subcmd: - mkOption { - default = { }; - type = with types; attrs; - description = '' - Additional configuration, see output of 'weed ${subcmd} --help' for attributes. - Do not define settings for flags for which explicit configuration options exist, these will be ignored. - ''; - }; - - mkPortOption = defaultPort: - mkOption { - default = defaultPort; - type = with types; uniq port; - description = "Http listen port"; - }; - - mkServerListOption = subcmd: - mkOption { - default = [ "localhost:${toString cfg.${subcmd}.port}" ]; - type = with types; listOf str; - description = "List of ${subcmd} servers (host/ip:port)"; - }; - - mkVolumeService = id: options: - nameValuePair "seaweedfs-volume-${id}" { - wantedBy = [ "multi-user.target" ]; - after = [ "network-online.target" ] - ++ optional cfg.master.enable "seaweedfs-master.service"; - description = "SeaweedFS volume - ${id}"; - unitConfig.ConditionPathIsDirectory = options.dir; - serviceConfig = { - User = user; - Group = group; - ExecStart = mkWeedExec "volume" options; - KillSignal = "SIGTERM"; - Restart = "on-failure"; - }; - }; - - mkVolumeOptions = { id, ... }: { - options = { - enable = mkEnableOption "SeaweedFS volume server"; - port = mkPortOption 8080; - mserver = mkServerListOption "master"; - extraConfig = mkExtraConfigOption "volume"; - - dir = mkOption { - default = [ "/tmp" ]; - type = with types; listOf path; - description = '' - One or more directories to store data files. - These must exist before the volume server service is started, - and must be owned by ${user}:${group}.''; - }; - }; - }; - -in { - - ###### interface - - options = { - services.seaweedfs = { - - master = { - enable = mkEnableOption "SeaweedFS master server"; - port = mkPortOption 9333; - peers = mkServerListOption "peers"; - extraConfig = mkExtraConfigOption "master"; - - }; - - filer = { - enable = mkEnableOption "SeaweedFS file server"; - port = mkPortOption 8888; - master = mkServerListOption "master"; - extraConfig = mkExtraConfigOption "filer"; - - }; - - volumes = mkOption { - default = { }; - type = with types; attrsOf (submodule mkVolumeOptions); - }; - - webdav = { - enable = mkEnableOption "SeaweedFS webdav server"; - port = mkPortOption 7333; - filer = mkServerListOption "filer"; - extraConfig = mkExtraConfigOption "webdav"; - - }; - }; - }; - - ###### implementation - - config = mkIf anyEnabled { - environment.systemPackages = [ seaweedfs ]; - users.users.${user} = { - description = "SeaweedFS user"; - isSystemUser = true; - uid = 10000; - }; - users.groups.${group}.gid = 10000; - systemd.services = (mapAttrs' mkVolumeService enabledVolumes) // { - seaweedfs-master = mkIf cfg.master.enable { - wantedBy = [ "multi-user.target" ]; - wants = [ "network-online.target" ]; - description = "SeaweedFS master"; - serviceConfig = rec { - User = user; - Group = group; - StateDirectory = "seaweedfs/master"; - ExecStart = (mkWeedExec "master" cfg.master) - + " -mdir=/var/lib/${StateDirectory}"; - KillSignal = "SIGTERM"; - Restart = "on-failure"; - }; - }; - - seaweedfs-filer = mkIf cfg.filer.enable { - wantedBy = [ "multi-user.target" ]; - wants = [ "network-online.target" ] - ++ optional cfg.master.enable "seaweedfs-master.service"; - description = "SeaweedFS filer"; - serviceConfig = rec { - User = user; - Group = group; - WorkingDirectory = "/var/lib/${StateDirectory}"; - StateDirectory = "seaweedfs/filer"; - ExecStart = mkWeedExec "filer" cfg.filer; - KillSignal = "SIGTERM"; - Restart = "on-failure"; - }; - }; - - seaweedfs-webdav = mkIf cfg.webdav.enable { - wantedBy = [ "multi-user.target" ]; - wants = [ "network-online.target" ] - ++ optional cfg.filer.enable "seaweedfs-filer.service"; - description = "SeaweedFS webdav"; - serviceConfig = { - User = user; - Group = group; - ExecStart = mkWeedExec "webdav" cfg.webdav; - KillSignal = "SIGTERM"; - Restart = "on-failure"; - }; - }; - }; - }; -}