diff --git a/dev/packages.nix b/dev/packages.nix index 3d1d94632..a513ab1b0 100644 --- a/dev/packages.nix +++ b/dev/packages.nix @@ -9,6 +9,9 @@ nixComponents = final.nixVersions.nixComponents_2_29; rawSrc = inputs.hydra; }; + hydra-queue-runner = final.callPackage (import "${inputs.hydra-queue-runner}/default.nix") { + pkgs = final; + }; rfc39 = final.rustPlatform.buildRustPackage { pname = "rfc39"; version = "0-unstable-2025-05-21"; diff --git a/dnscontrol/dnsconfig.js b/dnscontrol/dnsconfig.js index 4cc1d503f..f2d27f3a7 100644 --- a/dnscontrol/dnsconfig.js +++ b/dnscontrol/dnsconfig.js @@ -66,6 +66,7 @@ var cnames = { "nl.meet": "nixnl.codeberg.page.", "nur-update": "build03", "prometheus": "web02", + "queue-runner.hydra": "hydra", "temp-cache": "build03", // keep-sorted end }; diff --git a/flake.lock b/flake.lock index c50510f8c..0ecf2bcda 100644 --- a/flake.lock +++ b/flake.lock @@ -138,6 +138,23 @@ "type": "github" } }, + "hydra-queue-runner": { + "flake": false, + "locked": { + "lastModified": 1756677376, + "narHash": "sha256-l5ERRgyeXYSOZAo3CvCtAihlhpDkJhVNlaArgDUT79I=", + "owner": "qowoz", + "repo": "hydra-queue-runner", + "rev": "9aaade3cb49f1396d9e1bc10b6dbce62d780b4d4", + "type": "github" + }, + "original": { + "owner": "qowoz", + "ref": "infra", + "repo": "hydra-queue-runner", + "type": "github" + } + }, "lite-config": { "locked": { "lastModified": 1748349708, @@ -432,6 +449,7 @@ "flake-parts": "flake-parts", "hercules-ci-effects": "hercules-ci-effects", "hydra": "hydra", + "hydra-queue-runner": "hydra-queue-runner", "lite-config": "lite-config", "nix-darwin": "nix-darwin", "nix-index-database": "nix-index-database", diff --git a/flake.nix b/flake.nix index 3ec87f956..05e322687 100644 --- a/flake.nix +++ b/flake.nix @@ -26,6 +26,8 @@ hercules-ci-effects.inputs.flake-parts.follows = "flake-parts"; hercules-ci-effects.inputs.nixpkgs.follows = "nixpkgs"; hercules-ci-effects.url = "github:hercules-ci/hercules-ci-effects"; + hydra-queue-runner.flake = false; + hydra-queue-runner.url = "github:qowoz/hydra-queue-runner/infra"; hydra.flake = false; hydra.url = "github:NixOS/hydra"; lite-config.url = "github:yelite/lite-config"; diff --git a/hosts/build03/ca.crt b/hosts/build03/ca.crt new file mode 100644 index 000000000..c011eedb9 --- /dev/null +++ b/hosts/build03/ca.crt @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBkzCCAUWgAwIBAgIUM3IOGPv05wdHhykHyC7J0eYl4qYwBQYDK2VwMD4xHDAa +BgNVBAoME05peCBDb21tdW5pdHkgSW5mcmExHjAcBgNVBAMMFWh5ZHJhLXF1ZXVl +LXJ1bm5lci1jYTAgFw0yNTA4MDQwNDMxMTFaGA8yMDc1MDcyMzA0MzExMVowPjEc +MBoGA1UECgwTTml4IENvbW11bml0eSBJbmZyYTEeMBwGA1UEAwwVaHlkcmEtcXVl +dWUtcnVubmVyLWNhMCowBQYDK2VwAyEApTUfa9PNgjIqQIU8ur4gJ/EAClvVX+oJ +hduaCt0iQ+WjUzBRMB0GA1UdDgQWBBSs13lAhWgE2ji+4Yvm6b5bCI9pYjAfBgNV +HSMEGDAWgBSs13lAhWgE2ji+4Yvm6b5bCI9pYjAPBgNVHRMBAf8EBTADAQH/MAUG +AytlcANBAIsqwp4tW+P5yAdhZy8rWGeKhwVyKmtkf2EjCeWbxDAVqeQvEXWcP1o0 +SFwPhoW5BaccYOgsrSDq3hY7xs2BUgQ= +-----END CERTIFICATE----- diff --git a/hosts/build03/client.crt b/hosts/build03/client.crt new file mode 100644 index 000000000..a3c289898 --- /dev/null +++ b/hosts/build03/client.crt @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBiDCCATqgAwIBAgIULzcgJlY8HPEN7WebeYdUwmvL0dQwBQYDK2VwMD4xHDAa +BgNVBAoME05peCBDb21tdW5pdHkgSW5mcmExHjAcBgNVBAMMFWh5ZHJhLXF1ZXVl +LXJ1bm5lci1jYTAgFw0yNTA4MDQwNDMxMTFaGA8yMDc1MDcyMzA0MzExMVowRDEc +MBoGA1UECgwTTml4IENvbW11bml0eSBJbmZyYTEkMCIGA1UEAwwbaHlkcmEtcXVl +dWUtYnVpbGRlci1idWlsZDAzMCowBQYDK2VwAyEAgT2MFRB7qdN1Hx+ipgFOgYVc +zdDmZtm/jrPS/BOerrCjQjBAMB0GA1UdDgQWBBTR0QWcQtL2JbUeFmoHOO7y8wj7 +5zAfBgNVHSMEGDAWgBSs13lAhWgE2ji+4Yvm6b5bCI9pYjAFBgMrZXADQQCZtrwk +DknHpZPuceVHJPYYv9mYJvXfbHy5XAUkkS8lMWvH78dIQA+tK0atIvt364HkRBN1 +8pQWdhCU+bkKDDcC +-----END CERTIFICATE----- diff --git a/hosts/build03/default.nix b/hosts/build03/default.nix index 78eeb5460..1ac288037 100644 --- a/hosts/build03/default.nix +++ b/hosts/build03/default.nix @@ -5,6 +5,7 @@ ./cache.nix ./landscape.nix ./postgresql.nix + ./queue-runner.nix inputs.self.nixosModules.buildbot inputs.self.nixosModules.cgroups inputs.self.nixosModules.ci-builder diff --git a/hosts/build03/queue-runner.nix b/hosts/build03/queue-runner.nix new file mode 100644 index 000000000..6f7401d1f --- /dev/null +++ b/hosts/build03/queue-runner.nix @@ -0,0 +1,61 @@ +{ + config, + inputs, + lib, + ... +}: +{ + imports = [ "${inputs.self}/modules/queue-runner/hydra-queue-runner-v2.nix" ]; + + sops.secrets.queue-runner-server-key.owner = "nginx"; + + nixCommunity.hydra-queue-runner-v2 = { + enable = true; + settings = { + queueTriggerTimerInS = 300; + useSubstitutes = true; + }; + rest.port = 9090; + }; + + services.hydra = { + extraConfig = lib.mkAfter '' + queue_runner_endpoint = http://localhost:9090 + ''; + }; + + systemd.services.hydra-queue-runner.enable = false; + + services.nginx.virtualHosts."queue-runner.hydra.nix-community.org" = { + # disable defaults + enableACME = false; + forceSSL = false; + + extraConfig = '' + client_max_body_size 5120M; + ssl_client_certificate ${./ca.crt}; + ssl_verify_depth 2; + ssl_verify_client on; + ''; + + sslCertificate = "${./server.crt}"; + sslCertificateKey = config.sops.secrets.queue-runner-server-key.path; + onlySSL = true; + + locations."/".extraConfig = '' + # This is necessary so that grpc connections do not get closed early + # see https://stackoverflow.com/a/67805465 + client_body_timeout 31536000s; + grpc_pass grpc://[::1]:50051; + grpc_read_timeout 31536000s; # 1 year in seconds + grpc_send_timeout 31536000s; # 1 year in seconds + grpc_socket_keepalive on; + grpc_set_header Host $host; + grpc_set_header X-Real-IP $remote_addr; + grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + grpc_set_header X-Forwarded-Proto $scheme; + grpc_set_header X-Client-DN $ssl_client_s_dn; + grpc_set_header X-Client-Cert $ssl_client_escaped_cert; + ''; + }; +} diff --git a/hosts/build03/secrets.yaml b/hosts/build03/secrets.yaml index 7830f6cdc..52cbe927a 100644 --- a/hosts/build03/secrets.yaml +++ b/hosts/build03/secrets.yaml @@ -11,6 +11,9 @@ buildbot-nix-worker-password: ENC[AES256_GCM,data:TaMHVzlzuAHfTBAyqG5JJFwpG2We+w buildbot-effects-nix-community-infra: ENC[AES256_GCM,data:wAYSvEza+1xjT/rIp5MtR3HEQe3OscRBtrWF/f6G/z+ftaiWXMfWhW9X8z53I20lzx9a6BqmO4jcWAGmXRSV4TX6wxKsUP9BnzxTImDS41zWfOtHV9sDifk6jmVJyAAsrY8CrebGTmPvWUXEUMmAZkUorQkv8gfxoL66GCtPfqa7d1OSKX8rAXd1YAJL58aXFeD6X6iZqo2tVd1OhlmUuFj5oum+9uLPjCY54IfilLlERw5DdTPDhMBdKt5owR0dpJDmp9t8i0AV2j4Hm36ZsY8L9ket1iB04MlapsUI24RiXHk+aFJqSy+0onvmqeniiBeaa106YomSQAS1cyOp0U+Rcqwhtnh9MNUZzciVXi2F5nkPzA3OcbQh8MDcJ/66z/VUkSac0iMXvr/47d5V0XCm9JSKXuijOUGTu9LRbdEAAQ+gsZ/Z8i6+9d75CeSoVH5Mtm7eU4CbhloNmpAcz5qyFf+sRgWz5qK8z1MwT795P/P/Oi2YQswtwnyLO7UkQ5tCrE+qcFxM9cAAcJzkzGctB5QdHDaPkkpMGic8vpAoYgx3t7Qmasuef96Fhhj4lk10sTTUrWZlnFbmduO/fDOULvisnW/BTpYsnsIPSKREg8wQAu7Fiu9EO0TZOagAQf03EiFDcxosetxPJIXoNWsZK/1LsX+7SnBBklVDtwxu6urOUCW8vYXD4aGZvkTGc6O6vwiOs+oEHd6J7Mes/+23hTzldnFobodwnzbRNVt/WT2FQjX5qFehyxbARL+EtT4MQC4sWGpXSybUs10=,iv:rdLHfK4NbCaMIIhhQd2MfVf1DdKKF9Sqe4Kxuy57yok=,tag:DPxsDTLIhA0d4KPXwseL9g==,type:str] temp-cache-key: ENC[AES256_GCM,data:weL92egwmo4z32jXmWjgbfHo6h61+mWwHrHVAg0N8cHzBOjgsAL3NTRgpiiaePw/FcZa1rJ/ygBGyUYJbIq036fxhd1I/Vu+dPFXz7PPIphRj/q8wru21qfLXep39rk+bIqsJJB2++070SCKwgRLb4re9cM05ah3,iv:sX78dExpTL+UFkHWfQmYN8nsZcMCFhrgXwvtzvoWdJA=,tag:j3MNmXSdQEuh9oYP00yJAw==,type:str] rfc39-record-ssh-key: ENC[AES256_GCM,data:2jmn3F+y1xFACYfUoxJqp7EAEZVVuDYUGISk7/d8xAfjfpnekAO8OHRVT+RSgLQuVXsV7ffocb3M319+Kgdp5/hRUr7zNlCkBTMdXz9XBHDUmeEjhjtjQCmH0WYRjLgpyDrUY4gk1+EuAIKY0J9MktKJ4JKIxSm0Keei2iByoXxIjQhHDNg48UXQBW0A8ZPqWPmJsHCFtGcQXSVG/rehGqYh2cDC5Q4XpU4Rw1j+ZyE81RakbNxWx/uIVvbQxGuYI3qXyOunllokj5jh8VBQUpz3QdCkIR7dFTuoBy7N79sdZntAqHhNdeZPLpxIVTNwGBzgTvdcA9PCQSIs7TYg7X7l98Z9QQUnSXpg3APz+jlsoOF/icXrcOxJBAPAhBqH/2mBUEjbNiWQR44cZi4Bt6n1LZCA0IniZsBwUFDE9o8a4z70Nc2QGcLLixbWc+g8GtrH1qxtXe2mZ13uWFL1THuIypJzrbGKAPV+NAao9Q60znzWgQiuSkN2eXaGZQLadEXi,iv:wbhy54VM7WqRSgjoyjYKenliXgxjd41lFJdTr+1UH84=,tag:DB7VXqK2RHdadeEHabTwRw==,type:str] +queue-runner-ca-key: ENC[AES256_GCM,data:aNs2Xk3n6vAJIyFeYEj28Lm89gCAMN8op90Yo5U4HrnoxR+aJnwnF5sbwT989iNAjGpKdgu9KH47lo4maDJRSTLprEKb8ytEpUWTKgq6VTujPhlxDtdqlj2QtTeSJxlrXZwxLs/miH4a/qFUNcbmLJJmPLe+e7w=,iv:iEVZqsJjkXAJ7Dqadf9dyDTtTMLq6DX1gjE3GU11SUY=,tag:hFz/4Syc8vDtrPdKeiBe9w==,type:str] +queue-runner-server-key: ENC[AES256_GCM,data:cxTbXFV2ckIk468TbrdHrqxZDvGJ7cCaI+/hbfzRtnYmT2W26f3SzjM4EDg9ZCQcXIcdb6Ii/bfCwJj/Yhp60t/o5QuBvYHoNgyrM9v7bRZeNwY3JE5EM64UEIIgSzT/WlOwvf7c+mSS+4n1MfeMoRPZQM5mbOg=,iv:VoL4YPCusW0/XNEBPAyC1SuHRCfvipmHamLxcPlN+ZM=,tag:Lrov4NonT6CRLWs2P9OKig==,type:str] +queue-runner-client-key: ENC[AES256_GCM,data:CP16NvBDGOC0rOOxZO7EoIzWjdxEhpaPUM35RkkWoSgIlGYoXhT/1k/jaRAnyhfyj7h8rMBQMbgFeXqYAW6B7dTgxtB/HqDL7dZNk1HILhzYyMJlZ5+Px9S+qDutqJjwDhLSh7lyLOWBABjYYWAJEpNQ50Eakmc=,iv:K/FeMftONTpMMVcyfpCJFWhcoULdzwebl/k+aKM9NPw=,tag:VJU1QFjy3HAQQit8AtKPFA==,type:str] sops: age: - recipient: age1qg7tfjwzp6dxwkw9vej6knkhdvqre3fu7ryzsdk5ggvtdx854ycqevlwnq @@ -67,7 +70,7 @@ sops: WUZQSGQyQy9halJsRTIvb1FGV08zZEEKmjlYY6epTuZKRBcVyjPvJI5XKQtP5Yag FMrI+M6hUeyBeCade5C+Y4eGQbt57BWLmsX7u0J1WTlkUSS5j7+wPg== -----END AGE ENCRYPTED FILE----- - lastmodified: "2025-07-01T23:53:43Z" - mac: ENC[AES256_GCM,data:McXvWpN6kIprv2BOTCAqtDHiZv4xF3kmaltSJqGzfV3midOC8eeJVBwRhCU53h1TTJXoZU8Ar0cloNkiT1sqFFy8WUcdFC0NHHRnLAGX4ihSuSh5RA2odWUhua9QY73xfBwSGy876bVKtMzDVNMeecjRjBpZFJ3O/8FpKrMig/g=,iv:8iQPZv+jKJqI694cyJ5r0Je4jmvnmIj7QL06kq1ttwc=,tag:xP82ALLVO/Qq50TKx/1pWw==,type:str] + lastmodified: "2025-08-04T04:34:06Z" + mac: ENC[AES256_GCM,data:qpy+3Ifv9ydOKA51E/EoJjLZVhkEl/CwcqLu+rDCfTw920agoHXUMBeXCsiE3U3WoNPMJQUcQOXUr4MVZ/wtYHnGGqYHDlx3Ap0b1+Kjl0SJxlhpvib3mKpQEXTqh1bB5ylujydaDZasknjncWl96nKJG6KhF6xJE5zDqCaHWrU=,iv:kiLGLOnxIqpu+UpiBqx1pwkBqvncks8DBDqURp1BdGE=,tag:oJp0P+xeBinYhihUsqBWSg==,type:str] unencrypted_suffix: _unencrypted version: 3.10.2 diff --git a/hosts/build03/server.crt b/hosts/build03/server.crt new file mode 100644 index 000000000..c920244f1 --- /dev/null +++ b/hosts/build03/server.crt @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB9DCCAaagAwIBAgIULzcgJlY8HPEN7WebeYdUwmvL0dMwBQYDK2VwMD4xHDAa +BgNVBAoME05peCBDb21tdW5pdHkgSW5mcmExHjAcBgNVBAMMFWh5ZHJhLXF1ZXVl +LXJ1bm5lci1jYTAgFw0yNTA4MDQwNDMxMTFaGA8yMDc1MDcyMzA0MzExMVowTTEc +MBoGA1UECgwTTml4IENvbW11bml0eSBJbmZyYTEtMCsGA1UEAwwkcXVldWUtcnVu +bmVyLmh5ZHJhLm5peC1jb21tdW5pdHkub3JnMCowBQYDK2VwAyEAZVsAufKBynGu +MGDtn7Mryt5zkoxJ+Q3D/camesUKjFKjgaQwgaEwCQYDVR0TBAIwADALBgNVHQ8E +BAMCA+gwFgYDVR0lAQH/BAwwCgYIKwYBBQUHAwEwLwYDVR0RBCgwJoIkcXVldWUt +cnVubmVyLmh5ZHJhLm5peC1jb21tdW5pdHkub3JnMB0GA1UdDgQWBBQNqGtr7msZ ++1Ljn5sXVmxftth3KzAfBgNVHSMEGDAWgBSs13lAhWgE2ji+4Yvm6b5bCI9pYjAF +BgMrZXADQQCYRvZxS6cFMXTWr0Gy8svwctT6VL2Lfsrvg64SkmBFfFQdmlJpCSI1 +LCPSU5Q3NUMj6ILhZXN7J1cclj54iusD +-----END CERTIFICATE----- diff --git a/hosts/build04/client.crt b/hosts/build04/client.crt new file mode 100644 index 000000000..ce7221975 --- /dev/null +++ b/hosts/build04/client.crt @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBiDCCATqgAwIBAgIULzcgJlY8HPEN7WebeYdUwmvL0dUwBQYDK2VwMD4xHDAa +BgNVBAoME05peCBDb21tdW5pdHkgSW5mcmExHjAcBgNVBAMMFWh5ZHJhLXF1ZXVl +LXJ1bm5lci1jYTAgFw0yNTA4MDQwNDMxMTFaGA8yMDc1MDcyMzA0MzExMVowRDEc +MBoGA1UECgwTTml4IENvbW11bml0eSBJbmZyYTEkMCIGA1UEAwwbaHlkcmEtcXVl +dWUtYnVpbGRlci1idWlsZDA0MCowBQYDK2VwAyEAHWeEmssQwnUKflXcxOHQ/C+l +ocBon2fcXLTGHKqEG7WjQjBAMB0GA1UdDgQWBBT0YR68z7ajrcNtOduOKbQpWUpf +yjAfBgNVHSMEGDAWgBSs13lAhWgE2ji+4Yvm6b5bCI9pYjAFBgMrZXADQQC5Le1B +C6DawMayrg2mrwYcnlYdI9WZVLPDmtZTzSs74rPHHhSE+qxTueDVU6IQ/3v9extQ +7cToX7Ts0bMxqeYJ +-----END CERTIFICATE----- diff --git a/hosts/build04/secrets.yaml b/hosts/build04/secrets.yaml new file mode 100644 index 000000000..307738f25 --- /dev/null +++ b/hosts/build04/secrets.yaml @@ -0,0 +1,61 @@ +queue-runner-client-key: ENC[AES256_GCM,data:MONqxLGUOaVBbn8G432CphX3TkfdYSqX3SzGpQfQ60xgZXBbug2VrJKS0i7JxFgRWxeWwZUqz3o5ZXTR9etUqGhVxnlEFNFYpH7tnKlyeJb+b7Vg9GJJA7DgxJH7C/5YHCEEbqEiLDCFjp9D1ImwdGK1RgjPORc=,iv:ZiooKszkmu6UqjFka6kO5BHTXYotsnj3/nkKuEll2tw=,tag:RsaXGwZAdj9p68bF3I9XNg==,type:str] +sops: + age: + - recipient: age1r464z5e2shvnh9ekzapgghevr9wy7spd4d7pt5a89ucdk6kr6yhqzv5gkj + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4Qm4xS2xtblV6SkdHSlhj + NFRoZkMvd2pRY09MZmt2d2l1OVFMU1IrS2hvCjRFYkpWbGlsdTNwRUxndjJhYzdx + L21FOWVHV0tNeVZvaGJPYlNuL0pXdmsKLS0tIE96YTVTaFQ4YnM3MzNoUVRCbXg4 + cWtRcENweEhnb0x2cWtJOUdOcElzeFUKvmP1L/hBMt1/rekkZVAuW7l+c9YjUor6 + vIpJlOPYKgWOoiWmCAyI9oDd8ieWYAVlw2DCpq9/SmnXn0yXNGTTJg== + -----END AGE ENCRYPTED FILE----- + - recipient: age1dzvjjum2p240qtdt2qcxpm7pl2s5w36mh4fs3q9dhhq0uezvdqaq9vrgfy + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAranZjYWw3LzdtMTAxWWdk + cXo0aVNKS1Y2ZERkQUtVUk0rbE9zaXByK0M0CmVwaXpqUThKSEcxV2ZhU0VkRGov + MU1jNTIwNUxLTExUVXZVKzBjcGRkTkEKLS0tIE0wSzRhT09KWmdqNmk4aklKbEp5 + NXFiV1JIaEQyRHd6SzgvVStnb3pkaTQKBvOkeDmhas+oj+Aw2ZZLz2UanJcRjyeH + d5vllyoUIqg1+S9GbNIYHGbvhytLaSJlljvLRmYEx7rwRzv95DIP1Q== + -----END AGE ENCRYPTED FILE----- + - recipient: age17n64ahe3wesh8l8lj0zylf4nljdmqn28hvqns2g7hgm9mdkhlsvsjuvkxz + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvaDZTSG9hYzlyYzVHQnlF + dmo4cFlSYThCT0hQUUdTVzN2RGxINDRBY1VNCjZVVTZ0NDJPenVzYmM3cGRleDhH + SDA4TmZNZjlESUplemZiNE83V045TUkKLS0tIEpWaUluSXMxRFNwTEJLMFU5QXpP + S3hFckJGNit1aVFCc0ZIT0N4OWlGaXMKVwYgsIAUDvGJVB/7t1zFIx2V7wMztpLb + 4MsfIxQqhn8SnAwiEpu0wCQHLjQncXPVZhFnXy1Zqd5q0dtG5yAsRg== + -----END AGE ENCRYPTED FILE----- + - recipient: age1d87z3zqlv6ullnzyng8l722xzxwqr677csacf3zf3l28dau7avfs6pc7ay + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBEQ05VVHpqelY1ZkVTTnlu + TU9LUnNoYmJoZmU4VW1GSVYrZ3VFakpBQkZrClBReUR5cjV5ZXp1RnhCdTZDaVN5 + Z2RyWlJRYTNRVVNJSHFzWXVaTVdQMjAKLS0tIGtGVkJuY3BhbFIyR3dKMEJRQzBt + Wkw4ZVo4NUhZTkNNNUxUbWw3SU9HbkUKW2orZiOVtl54drQB13JDjN/mtsKmpmWb + 6TmvEt7qNyxlu1HpkeywuzT7iDbUqyh3t23Cp1UxsYnyS2djXEFDwA== + -----END AGE ENCRYPTED FILE----- + - recipient: age1jrh8yyq3swjru09s75s4mspu0mphh7h6z54z946raa9wx3pcdegq0x8t4h + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSNWkxLzFwTC90UGV0UnJu + aXJERFNHMFBaMHdnSk1RQ3d0OWowcGc0YjN3CmlNQnpzbUZJWUo5ZTF2UUNZbVBq + eUhocjczaXZVMm03c2x2dzJVV2ZhU1kKLS0tIHFXbVUvKy9UT1MzY2NQV1hwck1x + VkhuYUpmcTVOdFhHNzB1WjIvY1RzNTAKqQ4iuiG2PqIFXu7T8/CVsRtC49Ydqzsh + mMDcYwHc/7/6by23CudFgqJ6RwrMPKR1laSokhIYNbhUpU2uJjvJvA== + -----END AGE ENCRYPTED FILE----- + - recipient: age1m7xhem3qll35d539f364pm6txexvnp6k0tk34d8jxu4ry3pptv7smm0k5n + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3Sm4wQUVXdm1NdjBqMHFB + Wk0yV3VRejU1cy9QK05hcDJqbThFeG5LampJCjdlU2V5MWlTSkhIVnpHcGkwU1l0 + WW5teE04eXpPckF3bmt6a1JoMnZ4bTQKLS0tIEtud0tXaGFVaXEzbXVFaVlTT0k2 + VStSb0NaZG5yczdaaXdsUmU3U2RTUTAKJ2IVAccM023SboKeWDaO1B4zzCrMdEPS + pdCMhZXIq0DJL415+081IEcx7/UxYx4vfxcJa2Kyu8aRfMXTwTg6kQ== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2025-08-04T04:34:19Z" + mac: ENC[AES256_GCM,data:6UWlFNdHDst6El8o1PRxNSvLyGWxX7AtTmTRq3TfZxLY3ROD4coxwXnGpf+xolCV/EZZYLXS98wmts3YhUv7LuOPEY8jroh8T5c7x9sBeA8EuQUXDkxDZgD+DtjeqwSAA+5op8SdbVPhtBtH0bKoGAdNp/Ep800aCpSwRqyJlEw=,iv:4hotQiNPfLJ6GXQldtbzKoB+sfZRvX5ctVy3c+D1z38=,tag:edOe2Ef03YRQ1lKxfSo1Cw==,type:str] + unencrypted_suffix: _unencrypted + version: 3.10.2 diff --git a/modules/nixos/ci-builder.nix b/modules/nixos/ci-builder.nix index 2538f806a..756d2b88d 100644 --- a/modules/nixos/ci-builder.nix +++ b/modules/nixos/ci-builder.nix @@ -1,3 +1,24 @@ { - imports = [ ../shared/ci-builder.nix ]; + config, + inputs, + ... +}: +{ + imports = [ + ../shared/ci-builder.nix + "${inputs.self}/modules/queue-runner/hydra-queue-builder-v2.nix" + ]; + + sops.secrets.queue-runner-client-key.owner = "hydra-queue-builder"; + + nixCommunity.hydra-queue-builder-v2 = { + enable = true; + queueRunnerAddr = "https://queue-runner.hydra.nix-community.org"; + mtls = { + serverRootCaCertPath = "${../../hosts/build03/ca.crt}"; + clientCertPath = "${../../hosts/${config.networking.hostName}/client.crt}"; + clientKeyPath = config.sops.secrets.queue-runner-client-key.path; + domainName = "queue-runner.hydra.nix-community.org"; + }; + }; } diff --git a/modules/nixos/hydra.nix b/modules/nixos/hydra.nix index 65c29bc33..965ffcba8 100644 --- a/modules/nixos/hydra.nix +++ b/modules/nixos/hydra.nix @@ -4,13 +4,6 @@ lib, ... }: -let - inherit (lib) concatStringsSep; - localSystems = [ - "builtin" - pkgs.stdenv.hostPlatform.system - ]; -in { sops.secrets.hydra-admin-password.owner = "hydra"; sops.secrets.hydra-users.owner = "hydra"; @@ -18,6 +11,10 @@ in # hydra-queue-runner needs to read this key for remote building sops.secrets.id_buildfarm.owner = "hydra-queue-runner"; + nix.settings.extra-allowed-users = [ + "hydra-www" + "hydra" + ]; nix.settings.keep-outputs = lib.mkForce false; nix.settings.allowed-uris = [ @@ -42,22 +39,15 @@ in hydra-send-stats.enable = false; }; - environment.etc."nix/hydra/localhost".text = '' - localhost ${concatStringsSep "," localSystems} - 3 1 ${concatStringsSep "," config.nix.settings.system-features} - - - ''; environment.etc."nix/hydra/machines".source = pkgs.runCommand "machines" { machines = config.environment.etc."nix/machines".text; } '' - printf "$machines" | grep -e bsd -e linux > $out - substituteInPlace $out --replace-fail 'ssh-ng://' 'ssh://' - substituteInPlace $out --replace-fail ' 80 ' ' 3 ' + printf "$machines" | grep -e bsd > $out ''; services.hydra = { enable = true; - # remote builders set in /etc/nix/machines + localhost buildMachinesFiles = [ - "/etc/nix/hydra/localhost" "/etc/nix/hydra/machines" ]; hydraURL = "https://hydra.nix-community.org"; diff --git a/modules/queue-runner/hydra-queue-builder-v2.nix b/modules/queue-runner/hydra-queue-builder-v2.nix new file mode 100644 index 000000000..dbf314781 --- /dev/null +++ b/modules/queue-runner/hydra-queue-builder-v2.nix @@ -0,0 +1,276 @@ +# https://github.com/NixOS/infra/blob/b15878728090c9d22608546a75a3eb8c36beba11/non-critical-infra/modules/hydra-queue-builder-v2.nix +{ + config, + pkgs, + lib, + ... +}: +let + cfg = config.nixCommunity.hydra-queue-builder-v2; +in +{ + options = { + nixCommunity.hydra-queue-builder-v2 = { + enable = lib.mkEnableOption "QueueBuilder"; + + queueRunnerAddr = lib.mkOption { + description = "Queue Runner address to the grpc server"; + type = lib.types.singleLineStr; + }; + + pingInterval = lib.mkOption { + description = "Interval in which pings are send to the runner"; + type = lib.types.ints.positive; + default = 10; + }; + + speedFactor = lib.mkOption { + description = "Additional Speed factor for this machine"; + type = lib.types.oneOf [ + lib.types.ints.positive + lib.types.float + ]; + default = 1; + }; + + maxJobs = lib.mkOption { + description = "Maximum allowed of jobs. This only is used if the queue runner uses this metrics for determining free machines."; + type = lib.types.ints.positive; + default = 4; + }; + + tmpAvailThreshold = lib.mkOption { + description = "Threshold in percent free for /tmp before jobs are no longer scheduled on the machine"; + type = lib.types.float; + default = 10.0; + }; + + storeAvailThreshold = lib.mkOption { + description = "Threshold in percent free for /nix/store before jobs are no longer scheduled on the machine"; + type = lib.types.float; + default = 10.0; + }; + + load1Threshold = lib.mkOption { + description = "Maximum Load1 threshold before we stop scheduling jobs on that node. Only used if PSI is not available."; + type = lib.types.float; + default = 8.0; + }; + + cpuPsiThreshold = lib.mkOption { + description = "Maximum CPU PSI in the last 10s before we stop scheduling jobs on that node"; + type = lib.types.float; + default = 75.0; + }; + + memPsiThreshold = lib.mkOption { + description = "Maximum Memory PSI in the last 10s before we stop scheduling jobs on that node"; + type = lib.types.float; + default = 80.0; + }; + + ioPsiThreshold = lib.mkOption { + description = "Maximum IO PSI in the last 10s before we stop scheduling jobs on that node. If null then this pressure check is disabled."; + type = lib.types.nullOr lib.types.float; + default = null; + }; + + systems = lib.mkOption { + description = "List of supported systems. If none are passed, system and extra-platforms are read from nix."; + type = lib.types.listOf lib.types.singleLineStr; + default = [ ]; + }; + + supportedFeatures = lib.mkOption { + description = "Pass supported features to the builder. If none are passed, system features will be used."; + type = lib.types.listOf lib.types.singleLineStr; + default = [ ]; + }; + + mandatoryFeatures = lib.mkOption { + description = "Pass mandatory features to the builder."; + type = lib.types.listOf lib.types.singleLineStr; + default = [ ]; + }; + + useSubstitutes = lib.mkOption { + description = "Use substitution for paths"; + type = lib.types.bool; + default = true; + }; + + mtls = lib.mkOption { + description = "mtls options"; + default = null; + type = lib.types.nullOr ( + lib.types.submodule { + options = { + serverRootCaCertPath = lib.mkOption { + description = "Server root ca certificate path"; + type = lib.types.path; + }; + clientCertPath = lib.mkOption { + description = "Client certificate path"; + type = lib.types.path; + }; + clientKeyPath = lib.mkOption { + description = "Client key path"; + type = lib.types.path; + }; + domainName = lib.mkOption { + description = "Domain name for mtls"; + type = lib.types.singleLineStr; + }; + }; + } + ); + }; + + package = lib.mkOption { + type = lib.types.package; + default = pkgs.hydra-queue-runner; + }; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.hydra-queue-builder-v2 = { + description = "hydra-queue-builder-v2 main service"; + + requires = [ "nix-daemon.socket" ]; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + + environment = { + NIX_REMOTE = "daemon"; + LIBEV_FLAGS = "4"; # go ahead and mandate epoll(2) + RUST_BACKTRACE = "1"; + + # Note: it's important to set this for nix-store, because it wants to use + # $HOME in order to use a temporary cache dir. bizarre failures will occur + # otherwise + HOME = "/run/hydra-queue-builder-v2"; + }; + + serviceConfig = { + Type = "notify"; + Restart = "always"; + RestartSec = "5s"; + + ExecStart = lib.escapeShellArgs ( + [ + (lib.getExe' cfg.package "builder") + "--gateway-endpoint" + cfg.queueRunnerAddr + "--ping-interval" + cfg.pingInterval + "--speed-factor" + cfg.speedFactor + "--max-jobs" + cfg.maxJobs + "--tmp-avail-threshold" + cfg.tmpAvailThreshold + "--store-avail-threshold" + cfg.storeAvailThreshold + "--load1-threshold" + cfg.load1Threshold + "--cpu-psi-threshold" + cfg.cpuPsiThreshold + "--mem-psi-threshold" + cfg.memPsiThreshold + ] + ++ lib.optionals (cfg.ioPsiThreshold != null) [ + "--io-psi-threshold" + cfg.ioPsiThreshold + ] + ++ (builtins.concatMap (v: [ + "--systems" + v + ]) cfg.systems) + ++ (builtins.concatMap (v: [ + "--supported-features" + v + ]) cfg.supportedFeatures) + ++ (builtins.concatMap (v: [ + "--mandatory-features" + v + ]) cfg.mandatoryFeatures) + ++ lib.optionals (cfg.useSubstitutes != null) [ + "--use-substitutes" + ] + ++ lib.optionals (cfg.mtls != null) [ + "--server-root-ca-cert-path" + cfg.mtls.serverRootCaCertPath + "--client-cert-path" + cfg.mtls.clientCertPath + "--client-key-path" + cfg.mtls.clientKeyPath + "--domain-name" + cfg.mtls.domainName + ] + ); + + User = "hydra-queue-builder"; + Group = "hydra"; + + ReadWritePaths = [ + "/nix/var/nix/gcroots/" + "/nix/var/nix/daemon-socket/socket" + ]; + ReadOnlyPaths = [ "/nix/" ]; + RuntimeDirectory = "hydra-queue-builder-v2"; + + PrivateNetwork = false; + SystemCallFilter = [ + "@system-service" + "~@privileged" + "~@resources" + ]; + + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectControlGroups = true; + RestrictSUIDSGID = true; + PrivateMounts = true; + RemoveIPC = true; + UMask = "0077"; + + CapabilityBoundingSet = ""; + NoNewPrivileges = true; + + ProtectKernelModules = true; + SystemCallArchitectures = "native"; + ProtectKernelLogs = true; + ProtectClock = true; + + RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6"; + + LockPersonality = true; + ProtectHostname = true; + RestrictRealtime = true; + MemoryDenyWriteExecute = true; + PrivateUsers = true; + RestrictNamespaces = true; + }; + }; + systemd.tmpfiles.rules = [ + "d /nix/var/nix/gcroots/per-user/hydra-queue-builder 0755 hydra-queue-builder hydra -" + ]; + nix.settings = { + extra-allowed-users = [ "hydra-queue-builder" ]; + experimental-features = [ "nix-command" ]; + trusted-users = [ "hydra-queue-builder" ]; + }; + + users = { + groups.hydra = { }; + users.hydra-queue-builder = { + group = "hydra"; + isSystemUser = true; + }; + }; + }; +} diff --git a/modules/queue-runner/hydra-queue-runner-v2.nix b/modules/queue-runner/hydra-queue-runner-v2.nix new file mode 100644 index 000000000..78afa4551 --- /dev/null +++ b/modules/queue-runner/hydra-queue-runner-v2.nix @@ -0,0 +1,314 @@ +# https://github.com/NixOS/infra/blob/b15878728090c9d22608546a75a3eb8c36beba11/non-critical-infra/modules/hydra-queue-runner-v2.nix +{ + config, + pkgs, + lib, + ... +}: +let + cfg = config.nixCommunity.hydra-queue-runner-v2; + + format = pkgs.formats.toml { }; +in +{ + options = { + nixCommunity.hydra-queue-runner-v2 = { + enable = lib.mkEnableOption "QueueRunner"; + + settings = lib.mkOption { + description = "Reloadable settings for queue runner"; + type = lib.types.submodule { + options = { + hydraDataDir = lib.mkOption { + description = "Hydra data directory"; + type = lib.types.path; + default = "/var/lib/hydra"; + }; + dbUrl = lib.mkOption { + description = "Postgresql database url"; + type = lib.types.singleLineStr; + default = "postgres://hydra@%2Frun%2Fpostgresql:5432/hydra"; + }; + maxDbConnections = lib.mkOption { + description = "Postgresql maximum db connections"; + type = lib.types.ints.positive; + default = 128; + }; + machineSortFn = lib.mkOption { + description = "Function name for sorting machines"; + type = lib.types.enum [ + "SpeedFactorOnly" + "CpuCoreCountWithSpeedFactor" + "BogomipsWithSpeedFactor" + ]; + default = "SpeedFactorOnly"; + }; + machineFreeFn = lib.mkOption { + description = "Function name for determining \"idle\" machines"; + type = lib.types.enum [ + "Dynamic" + "DynamicWithMaxJobLimit" + "Static" + ]; + default = "Static"; + }; + dispatchTriggerTimerInS = lib.mkOption { + description = "Timer for triggering dispatch in an interval in seconds. Setting this to a value <= 0 will disable this timer and only trigger the dispatcher if queue changes happend."; + type = lib.types.int; + default = 120; + }; + queueTriggerTimerInS = lib.mkOption { + description = "Timer for triggering queue in an interval in seconds. Setting this to a value <= 0 will disable this timer and only trigger via pg notifications."; + type = lib.types.int; + default = -1; + }; + remoteStoreAddr = lib.mkOption { + description = "Remote store address"; + type = lib.types.listOf lib.types.singleLineStr; + default = [ ]; + }; + useSubstitutes = lib.mkOption { + description = "Use substitution for paths"; + type = lib.types.bool; + default = false; + }; + rootsDir = lib.mkOption { + description = "Gcroots directory, defaults to /nix/var/nix/gcroots/per-user/$LOGNAME/hydra-roots"; + type = lib.types.nullOr lib.types.path; + default = null; + }; + maxRetries = lib.mkOption { + description = "Number of maximum amount of retries for a build step."; + type = lib.types.ints.positive; + default = 5; + }; + retryInterval = lib.mkOption { + description = "Interval in which retires should be able to be attempted again."; + type = lib.types.ints.positive; + default = 60; + }; + retryBackoff = lib.mkOption { + description = "Additional backoff on top of the retry interval."; + type = lib.types.float; + default = 3.0; + }; + maxUnsupportedTimeInS = lib.mkOption { + description = "Time until unsupported steps are aborted."; + type = lib.types.ints.unsigned; + default = 120; + }; + stopQueueRunAfterInS = lib.mkOption { + description = "Seconds after which the queue run should be interupted early. Setting this to a value <= 0 will disable this feature and the queue run will never exit early."; + type = lib.types.int; + default = 60; + }; + maxConcurrentDownloads = lib.mkOption { + description = "Max count of concurrent downloads per build. Increasing this will increase memory usage of the queue runner."; + type = lib.types.ints.positive; + default = 5; + }; + concurrentUploadLimit = lib.mkOption { + description = "Concurrent limit for uploading to s3."; + type = lib.types.ints.positive; + default = 5; + }; + }; + }; + default = { }; + }; + + grpc = lib.mkOption { + description = "grpc options"; + default = { }; + type = lib.types.submodule { + options = { + address = lib.mkOption { + type = lib.types.singleLineStr; + default = "[::1]"; + description = "The IP address the grpc listener should bound to"; + }; + + port = lib.mkOption { + description = "Which grpc port this app should listen on"; + type = lib.types.port; + default = 50051; + }; + }; + }; + }; + + rest = lib.mkOption { + description = "rest options"; + default = { }; + type = lib.types.submodule { + options = { + address = lib.mkOption { + type = lib.types.singleLineStr; + default = "[::1]"; + description = "The IP address the rest listener should bound to"; + }; + + port = lib.mkOption { + description = "Which rest port this app should listen on"; + type = lib.types.port; + default = 8080; + }; + }; + }; + }; + + mtls = lib.mkOption { + description = "mtls options"; + default = null; + type = lib.types.nullOr ( + lib.types.submodule { + options = { + serverCertPath = lib.mkOption { + description = "Server certificate path"; + type = lib.types.path; + }; + serverKeyPath = lib.mkOption { + description = "Server key path"; + type = lib.types.path; + }; + clientCaCertPath = lib.mkOption { + description = "Client ca certificate path"; + type = lib.types.path; + }; + }; + } + ); + }; + package = lib.mkOption { + type = lib.types.package; + default = pkgs.hydra-queue-runner; + }; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.hydra-queue-runner-v2 = { + description = "hydra queue-runner-v2 main service"; + + requires = [ "nix-daemon.socket" ]; + after = [ + "network.target" + "postgresql-setup.service" + ]; + wantedBy = [ "multi-user.target" ]; + reloadTriggers = [ config.environment.etc."hydra/queue-runner.toml".source ]; + + environment = { + NIX_REMOTE = "daemon"; + LIBEV_FLAGS = "4"; # go ahead and mandate epoll(2) + RUST_BACKTRACE = "1"; + + # Note: it's important to set this for nix-store, because it wants to use + # $HOME in order to use a temporary cache dir. bizarre failures will occur + # otherwise + HOME = "/var/lib/hydra/queue-runner"; + }; + + serviceConfig = { + Type = "notify"; + Restart = "always"; + RestartSec = "5s"; + + ExecStart = lib.escapeShellArgs ( + [ + (lib.getExe' cfg.package "queue-runner") + "--rest-bind" + "${cfg.rest.address}:${toString cfg.rest.port}" + "--grpc-bind" + "${cfg.grpc.address}:${toString cfg.grpc.port}" + "--config-path" + "/etc/hydra/queue-runner.toml" + ] + ++ lib.optionals (cfg.mtls != null) [ + "--server-cert-path" + cfg.mtls.serverCertPath + "--server-key-path" + cfg.mtls.serverKeyPath + "--client-ca-cert-path" + cfg.mtls.clientCaCertPath + ] + ); + ExecReload = "${pkgs.util-linux}/bin/kill -HUP $MAINPID"; + + User = "hydra-queue-runner"; + Group = "hydra"; + + StateDirectory = [ "hydra/queue-runner" ]; + StateDirectoryMode = "0700"; + ReadWritePaths = [ + "/nix/var/nix/gcroots/" + "/run/postgresql/.s.PGSQL.${toString config.services.postgresql.settings.port}" + "/nix/var/nix/daemon-socket/socket" + "/var/lib/hydra/build-logs/" + ]; + ReadOnlyPaths = [ "/nix/" ]; + WorkingDirectory = "/var/lib/hydra/queue-runner"; + + PrivateNetwork = false; + SystemCallFilter = [ + "@system-service" + "~@privileged" + "~@resources" + ]; + + ManagedOOMPreference = "avoid"; + LimitNOFILE = 65536; + + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectControlGroups = true; + RestrictSUIDSGID = true; + PrivateMounts = true; + RemoveIPC = true; + UMask = "0022"; + + CapabilityBoundingSet = ""; + NoNewPrivileges = true; + + ProtectKernelModules = true; + SystemCallArchitectures = "native"; + ProtectKernelLogs = true; + ProtectClock = true; + + RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6"; + + LockPersonality = true; + ProtectHostname = true; + RestrictRealtime = true; + MemoryDenyWriteExecute = true; + PrivateUsers = true; + RestrictNamespaces = true; + }; + }; + + environment.etc."hydra/queue-runner.toml".source = format.generate "queue-runner.toml" ( + lib.filterAttrsRecursive (_: v: v != null) cfg.settings + ); + systemd.tmpfiles.rules = [ + "d /nix/var/nix/gcroots/per-user/hydra-queue-runner 0755 hydra-queue-runner hydra -" + "d /var/lib/hydra/build-logs/ 0755 hydra-queue-runner hydra -" + ]; + + nix.settings = { + extra-allowed-users = [ "hydra-queue-runner" ]; + experimental-features = [ "nix-command" ]; + trusted-users = [ "hydra-queue-runner" ]; + }; + + users = { + groups.hydra = { }; + users.hydra-queue-runner = { + group = "hydra"; + isSystemUser = true; + }; + }; + }; +}