diff --git a/flake.lock b/flake.lock index 97182e93..67011d4a 100644 --- a/flake.lock +++ b/flake.lock @@ -402,11 +402,11 @@ }, "nixpkgs-unstable": { "locked": { - "lastModified": 1738009863, - "narHash": "sha256-KxmFlQ2j9PpDhKRXWu85bv3R2wmfkUqdpJhEwz9JN/E=", + "lastModified": 1751786137, + "narHash": "sha256-lIlUKVGCGsh0Q2EA7/6xRtKUZjaQ/ur8uUyY+MynHXQ=", "owner": "nixos", "repo": "nixpkgs", - "rev": "f898cbfddfab52593da301a397a17d0af801bbc3", + "rev": "ceb24d94c6feaa4e8737a8e2bd3cf71c3a7eaaa0", "type": "github" }, "original": { diff --git a/non-critical-infra/flake-module.nix b/non-critical-infra/flake-module.nix index 45c5cd33..45967658 100644 --- a/non-critical-infra/flake-module.nix +++ b/non-critical-infra/flake-module.nix @@ -21,6 +21,7 @@ system = "x86_64-linux"; specialArgs = { inherit inputs; + unstable = inputs.nixpkgs-unstable.legacyPackages."x86_64-linux"; }; modules = [ value @@ -57,6 +58,7 @@ in { packages.encrypt-email = pkgs.callPackage ./packages/encrypt-email { }; + packages.hydra-queue-runner = pkgs.callPackage ./packages/hydra-queue-runner { }; devShells.non-critical-infra = pkgs.mkShellNoCC { packages = [ diff --git a/non-critical-infra/hosts/staging-hydra/default.nix b/non-critical-infra/hosts/staging-hydra/default.nix index d2cf69cf..a8feb018 100644 --- a/non-critical-infra/hosts/staging-hydra/default.nix +++ b/non-critical-infra/hosts/staging-hydra/default.nix @@ -5,6 +5,8 @@ inputs.srvos.nixosModules.server inputs.srvos.nixosModules.hardware-hetzner-cloud-arm ../../modules/common.nix + ../../modules/hydra-queue-runner-v2.nix + ../../modules/hydra-queue-builder-v2.nix ./hydra-proxy.nix ./hydra.nix inputs.hydra.nixosModules.hydra diff --git a/non-critical-infra/hosts/staging-hydra/hydra.nix b/non-critical-infra/hosts/staging-hydra/hydra.nix index 2c939231..6d0f853b 100644 --- a/non-critical-infra/hosts/staging-hydra/hydra.nix +++ b/non-critical-infra/hosts/staging-hydra/hydra.nix @@ -1,7 +1,8 @@ -{ lib -, pkgs -, config -, ... +{ + lib, + pkgs, + config, + ... }: let narCache = "/var/cache/hydra/nar-cache"; @@ -43,62 +44,117 @@ in }; }; - services.hydra-dev = { - enable = true; - package = pkgs.hydra; - buildMachinesFiles = [ - (pkgs.writeText "local" '' - localhost ${lib.concatStringsSep "," localSystems} - 3 1 ${lib.concatStringsSep "," config.nix.settings.system-features} - - - '') - ]; - logo = ../../../build/hydra-logo.png; - hydraURL = "https://hydra.nixos.org"; - notificationSender = "edolstra@gmail.com"; - smtpHost = "localhost"; - useSubstitutes = true; - extraConfig = '' - max_servers 30 + services = { + hydra-dev = { + enable = true; + package = pkgs.hydra; + buildMachinesFiles = [ + (pkgs.writeText "local" '' + localhost ${lib.concatStringsSep "," localSystems} - 3 1 ${lib.concatStringsSep "," config.nix.settings.system-features} - - + '') + ]; + logo = ../../../build/hydra-logo.png; + hydraURL = "https://hydra.nixos.org"; + notificationSender = "edolstra@gmail.com"; + smtpHost = "localhost"; + useSubstitutes = true; + extraConfig = '' + max_servers 30 - store_uri = s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br - server_store_uri = https://cache-staging.nixos.org?local-nar-cache=${narCache} - binary_cache_public_uri = https://cache-staging.nixos.org + store_uri = s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br + server_store_uri = https://cache-staging.nixos.org?local-nar-cache=${narCache} + binary_cache_public_uri = https://cache-staging.nixos.org - - cache_size = 32m - + + cache_size = 32m + - # patchelf:master:3 - xxx-jobset-repeats = nixos:reproducibility:1 + # patchelf:master:3 + xxx-jobset-repeats = nixos:reproducibility:1 - upload_logs_to_binary_cache = true - compress_build_logs = false # conflicts with upload_logs_to_binary_cache + upload_logs_to_binary_cache = true + compress_build_logs = false # conflicts with upload_logs_to_binary_cache - log_prefix = https://cache.nixos.org/ + log_prefix = https://cache.nixos.org/ - evaluator_workers = 1 - evaluator_max_memory_size = 4096 + evaluator_workers = 1 + evaluator_max_memory_size = 4096 - max_concurrent_evals = 1 + max_concurrent_evals = 1 - # increase the number of active compress slots (CPU is 48*2 on mimas) - max_local_worker_threads = 144 + # increase the number of active compress slots (CPU is 48*2 on mimas) + max_local_worker_threads = 144 - max_unsupported_time = 86400 + max_unsupported_time = 86400 - allow_import_from_derivation = false + allow_import_from_derivation = false - max_output_size = 3821225472 # 3 << 30 + 600000000 = 3 GiB + 0.6 GB - max_db_connections = 350 + max_output_size = 3821225472 # 3 << 30 + 600000000 = 3 GiB + 0.6 GB + max_db_connections = 350 - queue_runner_metrics_address = [::]:9198 + queue_runner_metrics_address = [::]:9198 - - - listen_address = 0.0.0.0 - port = 9199 - - - ''; + + + listen_address = 0.0.0.0 + port = 9199 + + + ''; + }; + + hydra-queue-runner-v2 = { + enable = true; + }; + + hydra-queue-builder-v2 = { + enable = true; + queueRunnerAddr = "https://hydra-queue-runner-staging.nixos.org"; + # TODO + # mtls = { + # serverRootCaCertPath = "${../../1systems/helsinki-hydra-builder01/ca.crt}"; + # clientCertPath = "${../../1systems/${config.networking.hostName}/client.crt}"; + # clientKeyPath = "/run/secrets/hydra/builder/client.key"; + # domainName = "hydra-queue-runner-staging.nixos.org"; + # }; + }; + + nginx = { + enable = true; + virtualHosts."hydra-queue-runner-staging.nixos.org" = { + default = true; + # TODO + # extraConfig = '' + # ssl_client_certificate ${./ca.crt}; + # ssl_verify_depth 2; + # ssl_verify_client on; + # ''; + + # sslCertificate = ./server.crt; + # sslCertificateKey = "/run/secrets/hydra/runner/server.key"; + # onlySSL = true; + + locations."/".extraConfig = '' + # This is necessary so that grpc connections do not get closed early + # see https://stackoverflow.com/a/67805465 + client_body_timeout 31536000s; + + grpc_pass grpc://[::1]:50051; + + grpc_read_timeout 31536000s; # 1 year in seconds + grpc_send_timeout 31536000s; # 1 year in seconds + grpc_socket_keepalive on; + + grpc_set_header Host $host; + grpc_set_header X-Real-IP $remote_addr; + grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + grpc_set_header X-Forwarded-Proto $scheme; + + grpc_set_header X-Client-DN $ssl_client_s_dn; + grpc_set_header X-Client-Cert $ssl_client_escaped_cert; + ''; + }; + }; }; sops.secrets.hydra-users = { @@ -116,6 +172,8 @@ in services = { hydra-notify.enable = false; hydra-queue-runner = { + enable = false; + # restarting the scheduler is very expensive restartIfChanged = false; serviceConfig = { diff --git a/non-critical-infra/modules/hydra-queue-builder-v2.nix b/non-critical-infra/modules/hydra-queue-builder-v2.nix new file mode 100644 index 00000000..1c14176a --- /dev/null +++ b/non-critical-infra/modules/hydra-queue-builder-v2.nix @@ -0,0 +1,183 @@ +{ + config, + pkgs, + lib, + unstable, + ... +}: +let + cfg = config.services.hydra-queue-builder-v2; +in +{ + options = { + services.hydra-queue-builder-v2 = { + enable = lib.mkEnableOption "QueueBuilder"; + + queueRunnerAddr = lib.mkOption { + description = "Queue Runner address to the grpc server"; + type = lib.types.singleLineStr; + }; + + pingInterval = lib.mkOption { + description = "Interval in which pings are send to the runner"; + type = lib.types.ints.positive; + default = 30; + }; + + speedFactor = lib.mkOption { + description = "Additional Speed factor for this machine"; + type = lib.types.oneOf [ + lib.types.ints.positive + lib.types.float + ]; + default = 1; + }; + + mtls = lib.mkOption { + description = "mtls options"; + default = null; + type = lib.types.nullOr ( + lib.types.submodule { + options = { + serverRootCaCertPath = lib.mkOption { + description = "Server root ca certificate path"; + type = lib.types.path; + }; + clientCertPath = lib.mkOption { + description = "Client certificate path"; + type = lib.types.path; + }; + clientKeyPath = lib.mkOption { + description = "Client key path"; + type = lib.types.path; + }; + domainName = lib.mkOption { + description = "Domain name for mtls"; + type = lib.types.singleLineStr; + }; + }; + } + ); + }; + + package = lib.mkOption { + type = lib.types.package; + default = pkgs.callPackage ../packages/hydra-queue-runner { inherit (unstable) rustPackages; }; + }; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.hydra-queue-builder-v2 = { + description = "hydra-queue-builder-v2 main service"; + + requires = [ "nix-daemon.socket" ]; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + + environment = { + NIX_REMOTE = "daemon"; + LIBEV_FLAGS = "4"; # go ahead and mandate epoll(2) + RUST_BACKTRACE = "1"; + + # Note: it's important to set this for nix-store, because it wants to use + # $HOME in order to use a temporary cache dir. bizarre failures will occur + # otherwise + HOME = "/run/hydra-queue-builder-v2"; + }; + + serviceConfig = { + Type = "notify"; + Restart = "always"; + RestartSec = "5s"; + + ExecStart = lib.escapeShellArgs ( + [ + "${cfg.package}/bin/builder" + "--gateway-endpoint" + cfg.queueRunnerAddr + "--ping-interval" + cfg.pingInterval + "--speed-factor" + cfg.speedFactor + ] + ++ lib.optionals (cfg.mtls != null) [ + "--server-root-ca-cert-path" + cfg.mtls.serverRootCaCertPath + "--client-cert-path" + cfg.mtls.clientCertPath + "--client-key-path" + cfg.mtls.clientKeyPath + "--domain-name" + cfg.mtls.domainName + ] + ); + + User = "hydra-queue-builder"; + Group = "hydra"; + + ReadWritePaths = [ + "/nix/var/nix/gcroots/" + "/nix/var/nix/daemon-socket/socket" + ]; + ReadOnlyPaths = [ "/nix/" ]; + RuntimeDirectory = "hydra-queue-builder-v2"; + + PrivateNetwork = false; + SystemCallFilter = [ + "@system-service" + "~@privileged" + "~@resources" + ]; + + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectControlGroups = true; + RestrictSUIDSGID = true; + PrivateMounts = true; + RemoveIPC = true; + UMask = "0077"; + + CapabilityBoundingSet = ""; + NoNewPrivileges = true; + + ProtectKernelModules = true; + SystemCallArchitectures = "native"; + ProtectKernelLogs = true; + ProtectClock = true; + + RestrictAddressFamilies = "AF_INET6"; + + LockPersonality = true; + ProtectHostname = true; + RestrictRealtime = true; + MemoryDenyWriteExecute = true; + PrivateUsers = true; + RestrictNamespaces = true; + }; + }; + systemd.tmpfiles.rules = [ + "d /nix/var/nix/gcroots/per-user/hydra-queue-builder 0755 hydra-queue-builder hydra -" + ]; + nix = { + settings = { + allowed-users = [ "hydra-queue-builder" ]; + trusted-users = [ "hydra-queue-builder" ]; + }; + extraOptions = '' + experimental-features = nix-command + ''; + }; + + users = { + groups.hydra = { }; + users.hydra-queue-builder = { + group = "hydra"; + isSystemUser = true; + }; + }; + }; +} diff --git a/non-critical-infra/modules/hydra-queue-runner-v2.nix b/non-critical-infra/modules/hydra-queue-runner-v2.nix new file mode 100644 index 00000000..c8e69317 --- /dev/null +++ b/non-critical-infra/modules/hydra-queue-runner-v2.nix @@ -0,0 +1,260 @@ +{ + config, + pkgs, + lib, + unstable, + ... +}: +let + cfg = config.services.hydra-queue-runner-v2; + + format = pkgs.formats.toml { }; +in +{ + options = { + services.hydra-queue-runner-v2 = { + enable = lib.mkEnableOption "QueueRunner"; + + settings = lib.mkOption { + description = "Reloadable settings for queue runner"; + type = lib.types.submodule { + options = { + hydraLogDir = lib.mkOption { + description = "Hydra log directory"; + type = lib.types.path; + default = "/var/lib/hydra/build-logs"; + }; + dbUrl = lib.mkOption { + description = "Postgresql database url"; + type = lib.types.singleLineStr; + default = "postgres://hydra@%2Frun%2Fpostgresql:5432/hydra"; + }; + maxDbConnections = lib.mkOption { + description = "Postgresql maximum db connections"; + type = lib.types.ints.positive; + default = 128; + }; + machineSortFn = lib.mkOption { + description = "Function name for sorting machines"; + type = lib.types.enum [ + "SpeedFactorOnly" + "CpuCoreCountWithSpeedFactor" + "BogomipsWithSpeedFactor" + ]; + default = "SpeedFactorOnly"; + }; + dispatchTriggerTimerInS = lib.mkOption { + description = "Timer for triggering dispatch in an interval in seconds. Setting this to a value <= 0 will disable this timer and only trigger the dispatcher if queue changes happend."; + type = lib.types.int; + default = 120; + }; + remoteStoreAddr = lib.mkOption { + description = "Remote store address"; + type = lib.types.nullOr lib.types.singleLineStr; + default = null; + }; + signingKeyPath = lib.mkOption { + description = "Signing key path"; + type = lib.types.nullOr lib.types.path; + default = null; + }; + useSubstitutes = lib.mkOption { + description = "Use substitution for paths"; + type = lib.types.bool; + default = false; + }; + rootsDir = lib.mkOption { + description = "Gcroots directory, defaults to /nix/var/nix/gcroots/per-user/$LOGNAME/hydra-roots"; + type = lib.types.nullOr lib.types.path; + default = null; + }; + maxRetries = lib.mkOption { + description = "Number of maximum amount of retries for a build step."; + type = lib.types.ints.positive; + default = 5; + }; + retryInterval = lib.mkOption { + description = "Interval in which retires should be able to be attempted again."; + type = lib.types.ints.positive; + default = 60; + }; + retryBackoff = lib.mkOption { + description = "Additional backoff on top of the retry interval."; + type = lib.types.float; + default = 3.0; + }; + }; + }; + default = { }; + }; + + grpcAddress = lib.mkOption { + type = lib.types.singleLineStr; + default = "[::1]"; + description = "The IP address the grpc listener should bound to"; + }; + + grpcPort = lib.mkOption { + description = "Which grpc port this app should listen on"; + type = lib.types.port; + default = 50051; + }; + + restAddress = lib.mkOption { + type = lib.types.singleLineStr; + default = "[::1]"; + description = "The IP address the rest listener should bound to"; + }; + + restPort = lib.mkOption { + description = "Which rest port this app should listen on"; + type = lib.types.port; + default = 8080; + }; + + mtls = lib.mkOption { + description = "mtls options"; + default = null; + type = lib.types.nullOr ( + lib.types.submodule { + options = { + serverCertPath = lib.mkOption { + description = "Server certificate path"; + type = lib.types.path; + }; + serverKeyPath = lib.mkOption { + description = "Server key path"; + type = lib.types.path; + }; + clientCaCertPath = lib.mkOption { + description = "Client ca certificate path"; + type = lib.types.path; + }; + }; + } + ); + }; + package = lib.mkOption { + type = lib.types.package; + default = pkgs.callPackage ../packages/hydra-queue-runner { inherit (unstable) rustPackages; }; + }; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.hydra-queue-runner-v2 = { + description = "hydra queue-runner-v2 main service"; + + requires = [ "nix-daemon.socket" ]; + after = [ + "network.target" + "postgresql.service" + ]; + wantedBy = [ "multi-user.target" ]; + reloadTriggers = [ config.environment.etc."hydra/queue-runner.toml".source ]; + + environment = { + NIX_REMOTE = "daemon"; + LIBEV_FLAGS = "4"; # go ahead and mandate epoll(2) + RUST_BACKTRACE = "1"; + + # Note: it's important to set this for nix-store, because it wants to use + # $HOME in order to use a temporary cache dir. bizarre failures will occur + # otherwise + HOME = "/run/hydra-queue-runner"; + }; + + serviceConfig = { + Type = "notify"; + Restart = "always"; + RestartSec = "5s"; + + ExecStart = lib.escapeShellArgs ( + [ + "${cfg.package}/bin/queue-runner" + "--rest-bind" + "${cfg.restAddress}:${toString cfg.restPort}" + "--grpc-bind" + "${cfg.grpcAddress}:${toString cfg.grpcPort}" + "--config-path" + "/etc/hydra/queue-runner.toml" + ] + ++ lib.optionals (cfg.mtls != null) [ + "--server-cert-path" + cfg.mtls.serverCertPath + "--server-key-path" + cfg.mtls.serverKeyPath + "--client-ca-cert-path" + cfg.mtls.clientCaCertPath + ] + ); + ExecReload = "${pkgs.util-linux}/bin/kill -HUP $MAINPID"; + + User = "hydra-queue-runner"; + Group = "hydra"; + + StateDirectory = [ "hydra/queue-runner" ]; + StateDirectoryMode = "0700"; + ReadWritePaths = [ + "/nix/var/nix/gcroots/" + "/run/postgresql/.s.PGSQL.${toString config.services.postgresql.port}" + "/nix/var/nix/daemon-socket/socket" + "/var/lib/hydra/build-logs/" + ]; + ReadOnlyPaths = [ "/nix/" ]; + RuntimeDirectory = "hydra-queue-runner-v2"; + + PrivateNetwork = false; + SystemCallFilter = [ + "@system-service" + "~@privileged" + "~@resources" + ]; + + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectControlGroups = true; + RestrictSUIDSGID = true; + PrivateMounts = true; + RemoveIPC = true; + UMask = "0077"; + + CapabilityBoundingSet = ""; + NoNewPrivileges = true; + + ProtectKernelModules = true; + SystemCallArchitectures = "native"; + ProtectKernelLogs = true; + ProtectClock = true; + + RestrictAddressFamilies = "AF_INET6"; + + LockPersonality = true; + ProtectHostname = true; + RestrictRealtime = true; + MemoryDenyWriteExecute = true; + PrivateUsers = true; + RestrictNamespaces = true; + }; + }; + + environment.etc."hydra/queue-runner.toml".source = format.generate "queue-runner.toml" ( + lib.filterAttrsRecursive (_: v: v != null) cfg.settings + ); + systemd.tmpfiles.rules = [ + "d /nix/var/nix/gcroots/per-user/hydra-queue-runner 0755 hydra-queue-runner hydra -" + "d /var/lib/hydra/build-logs/ 0755 hydra-queue-runner hydra -" + ]; + + users = { + groups.hydra = { }; + users.hydra-queue-runner = { + group = "hydra"; + isSystemUser = true; + }; + }; + }; +} diff --git a/non-critical-infra/packages/hydra-queue-runner/default.nix b/non-critical-infra/packages/hydra-queue-runner/default.nix new file mode 100644 index 00000000..7b3c571b --- /dev/null +++ b/non-critical-infra/packages/hydra-queue-runner/default.nix @@ -0,0 +1,55 @@ +{ + rustPackages, + fetchFromGitHub, + pkg-config, + openssl, + zlib, + protobuf, + lib, + makeWrapper, + nix, +}: +rustPackages.rustPlatform.buildRustPackage rec { + name = "hydra-queue-runner"; + version = "unstable-2025-07-07"; + __structuredAttrs = true; + strictDeps = true; + + src = fetchFromGitHub { + owner = "helsinki-systems"; + repo = "hydra-queue-runner"; + rev = "7b1b4872a6b786d0080e6951c6a61e6da21c0401"; + hash = "sha256-C/jePC2yhAD62XuaZ0n3/sXnZwlmLpMMnvkWwYjGe3E="; + }; + + cargoDeps = rustPackages.rustPlatform.fetchCargoVendor { + inherit src; + hash = "sha256-4nE2reDksPV3KyJ7T93hO6zBoeg6+pswe/KWdoBDjdw="; + }; + + nativeBuildInputs = [ + pkg-config + protobuf + makeWrapper + ]; + buildInputs = [ + openssl + zlib + protobuf + ]; + + postInstall = '' + wrapProgram $out/bin/queue-runner \ + --prefix PATH : ${lib.makeBinPath [ nix ]} + wrapProgram $out/bin/builder \ + --prefix PATH : ${lib.makeBinPath [ nix ]} + ''; + + meta = { + description = "Hydra Queue-Runner implemented in rust"; + homepage = "https://github.com/helsinki-systems/queue-runner"; + license = [ lib.licenses.gpl3 ]; + maintainers = [ lib.maintainers.conni2461 ]; + platforms = lib.platforms.all; + }; +}