From ccb345a69a0e26db5f43fff98824240a58d1db5d Mon Sep 17 00:00:00 2001 From: Mike Wilson Date: Mon, 12 Jun 2023 09:30:05 -0400 Subject: [PATCH 1/3] Make devbox sandbox friendly, add devbox CI. The devbox is now sandbox friendly, and devbox CI has been added to ensure that any changes to Devbox are properly validated. --- .github/workflows/check-devbox-bypass.yaml | 26 +++++ .github/workflows/check-devbox.yaml | 34 ++++++ build.assets/flake/flake.lock | 71 +++++++++++- build.assets/flake/flake.nix | 129 ++++++++++----------- devbox.json | 5 +- devbox.lock | 1 - 6 files changed, 193 insertions(+), 73 deletions(-) create mode 100644 .github/workflows/check-devbox-bypass.yaml create mode 100644 .github/workflows/check-devbox.yaml diff --git a/.github/workflows/check-devbox-bypass.yaml b/.github/workflows/check-devbox-bypass.yaml new file mode 100644 index 0000000000000..24db1964cc9cf --- /dev/null +++ b/.github/workflows/check-devbox-bypass.yaml @@ -0,0 +1,26 @@ +name: Check Devbox +run-name: Check Devbox - ${{ github.run_id }} - @${{ github.actor }} + +on: + pull_request: + paths-ignore: + - 'devbox.json' + - 'devbox.lock' + - 'build.assets/flake/**' + merge_group: + paths-ignore: + - 'devbox.json' + - 'devbox.lock' + - 'build.assets/flake/**' + +jobs: + check-devbox: + if: ${{ !startsWith(github.head_ref, 'dependabot/') }} + name: Check Devbox + runs-on: ubuntu-latest + + permissions: + contents: none + + steps: + - run: 'echo "No changes to verify"' diff --git a/.github/workflows/check-devbox.yaml b/.github/workflows/check-devbox.yaml new file mode 100644 index 0000000000000..88e6cb43496a3 --- /dev/null +++ b/.github/workflows/check-devbox.yaml @@ -0,0 +1,34 @@ +name: Check Devbox +run-name: Check Devbox - ${{ github.run_id }} - @${{ github.actor }} + +on: + pull_request: + paths: + - 'devbox.json' + - 'devbox.lock' + - 'build.assets/flake/**' + merge_group: + paths: + - 'devbox.json' + - 'devbox.lock' + - 'build.assets/flake/**' + +jobs: + check-devbox: + if: ${{ !startsWith(github.head_ref, 'dependabot/') }} + name: Check Devbox + runs-on: ubuntu-latest + + permissions: + contents: read + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install devbox + run: curl -fsSL https://get.jetpack.io/devbox | FORCE=1 bash + + - name: Install devbox dependencies + run: | + devbox install \ No newline at end of file diff --git a/build.assets/flake/flake.lock b/build.assets/flake/flake.lock index 8f0c18d3bfb69..bf85aebea267d 100644 --- a/build.assets/flake/flake.lock +++ b/build.assets/flake/flake.lock @@ -34,6 +34,24 @@ "type": "github" } }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1681202837, + "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "cfacdce06f30d2b68473a46042957675eebb3401", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, "helmPkgs": { "locked": { "lastModified": 1678571061, @@ -82,13 +100,49 @@ "type": "github" } }, + "nixpkgs_2": { + "locked": { + "lastModified": 1681358109, + "narHash": "sha256-eKyxW4OohHQx9Urxi7TQlFBTDWII+F+x2hklDOQPB50=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "96ba1c52e54e74c3197f4d43026b3f3d92e83ff9", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, "root": { "inputs": { "batsPkgs": "batsPkgs", "flake-utils": "flake-utils", "helmPkgs": "helmPkgs", "libbpfPkgs": "libbpfPkgs", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay" + } + }, + "rust-overlay": { + "inputs": { + "flake-utils": "flake-utils_2", + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "lastModified": 1686537156, + "narHash": "sha256-mJD80brS6h6P4jzwdKID0S9RvfyiruxgJbXvPPIDqF0=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "e75da5cfc7da874401decaa88f4ccb3b4d64d20d", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" } }, "systems": { @@ -105,6 +159,21 @@ "repo": "default", "type": "github" } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } } }, "root": "root", diff --git a/build.assets/flake/flake.nix b/build.assets/flake/flake.nix index 2f2fc6b786cce..7027f11a1046a 100644 --- a/build.assets/flake/flake.nix +++ b/build.assets/flake/flake.nix @@ -23,6 +23,8 @@ inputs = { flake-utils.url = "github:numtide/flake-utils"; nixpkgs.url = "github:nixos/nixpkgs/8ad5e8132c5dcf977e308e7bf5517cc6cc0bf7d8"; # general packages + rust-overlay.url = "github:oxalica/rust-overlay"; + # Linting dependencies helmPkgs.url = "github:nixos/nixpkgs/8ad5e8132c5dcf977e308e7bf5517cc6cc0bf7d8"; # helm 3.11.1 @@ -37,6 +39,7 @@ outputs = { self, flake-utils, nixpkgs, + rust-overlay, helmPkgs, libbpfPkgs, @@ -46,15 +49,16 @@ (system: let # These versions are not available from nixpkgs - golangciLintVersion = "v1.53.2"; - rustVersion = "1.68.0"; gogoVersion = "v1.3.2"; helmUnittestVersion = "v1.0.16"; - nodeProtocTsVersion = "5.0.1"; + nodeProtocTsVersion = "v5.0.1"; grpcToolsVersion = "1.12.4"; libpcscliteVersion = "1.9.9-teleport"; + rustVersion = "1.68.0"; yarnVersion = "1.22.19"; + overlays = [ (import rust-overlay) ]; + # Package aliases to make reusing these packages easier. # The individual package names here have been determined by using # https://lazamar.co.uk/nix-versions/ @@ -63,7 +67,9 @@ # pkgs is an alias for the nixpkgs at the system level. This will be used # for general utilities. - pkgs = nixpkgs.legacyPackages.${system}; + pkgs = import nixpkgs { + inherit system overlays; + }; # The helm unittest plugin. helm-unittest = pkgs.buildGoModule rec { @@ -90,19 +96,6 @@ # Wrap helm with the unittest plugin. helm = (pkgs.wrapHelm helmPkgs.legacyPackages.${system}.kubernetes-helm {plugins = [helm-unittest];}); - # Install golangci-lint - golangci-lint = pkgs.stdenv.mkDerivation { - name = "golangci-lint"; - buildInputs = [ - pkgs.cacert - pkgs.curl - ]; - dontUnpack = true; - buildPhase = '' - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $out/bin ${golangciLintVersion} - ''; - }; - libpcscliteAdditionalNativeBuildInputs = if pkgs.stdenv.isDarwin then [pkgs.darwin.IOKit] else []; libpcscliteAdditionalBuildInputs = if pkgs.stdenv.isLinux then @@ -126,30 +119,29 @@ pkgs.gcc pkgs.pkg-config ] ++ libpcscliteAdditionalBuildInputs; - autoreconfPhase = '' - ./bootstrap - ''; configurePhase = '' - ./configure --enable-static --with-pic --disable-libsystemd --prefix="$out" - ''; - shellHook = '' - export CFLAGS="-std=gnu99" + ./bootstrap + ./configure --enable-static --with-pic --disable-libsystemd --with-systemdsystemunitdir=$out --exec-prefix=$out --prefix=$out ''; + makeFlags = [ + "CFLAGS=\"-std=gnu99\"" + ]; }; # Compile protoc-gen-gogo for golang protobuf compilation. - protoc-gen-gogo = pkgs.stdenv.mkDerivation { + protoc-gen-gogo = pkgs.buildGoModule { name = "protoc-gen-gogo"; + version = gogoVersion; + src = pkgs.fetchFromGitHub { owner = "gogo"; repo = "protobuf"; rev = gogoVersion; sha256 = "sha256-CoUqgLFnLNCS9OxKFS7XwjE17SlH6iL1Kgv+0uEK2zU="; }; - buildInputs = [ - pkgs.cacert - pkgs.go - ]; + + vendorSha256 = "sha256-nOL2Ulo9VlOHAqJgZuHl7fGjz/WFAaWPdemplbQWcak="; + buildPhase = '' export GOBIN="$out/bin" export GOCACHE="$(mktemp -d)" @@ -158,58 +150,57 @@ ''; }; - # Compile grpc-tools for nodejs protobuf compilation. - grpc-tools = pkgs.stdenv.mkDerivation { - name = "grpc-tools"; - dontUnpack = true; - buildInputs = [ - pkgs.nodejs-16_x - ]; - buildPhase = '' - export HOME="$(mktemp -d)" - export TEMPDIR="$(mktemp -d)" - npm install --prefix "$TEMPDIR" grpc_tools_node_protoc_ts@${nodeProtocTsVersion} grpc-tools@${grpcToolsVersion} - mv "$TEMPDIR" "$out" - mkdir "$out/bin" - cd "$out/bin" - ln -s ../node_modules/.bin/* "$out/bin/" - ''; + node-protoc-ts = pkgs.buildNpmPackage { + name = "grpc_tools_node_protoc_ts"; + version = nodeProtocTsVersion; + + src = pkgs.fetchFromGitHub { + owner = "agreatfool"; + repo = "grpc_tools_node_protoc_ts"; + rev = nodeProtocTsVersion; + sha256 = "sha256-kDrflQVENjOY7ei3+D3Znx4eUDPoja8UGG2Phv1eptA="; + }; + + npmDepsHash = "sha256-fxOyItDkkv5OAmtScD9ykq26Meh6qyZSDmWegeh+GRY="; }; - # Rust and cargo binaries. - rust = pkgs.stdenv.mkDerivation { - name = "rust"; - dontUnpack = true; - buildInputs = [ - pkgs.cacert - pkgs.curl - ]; - buildPhase = '' - export RUSTUP_HOME="$out" - export CARGO_HOME="$out" - curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain "${rustVersion}" + grpc-tools = pkgs.stdenv.mkDerivation rec { + pname = "grpc-tools"; + version = grpcToolsVersion; + + src = pkgs.fetchFromGitHub { + owner = "grpc"; + repo = "grpc-node"; + rev = "grpc-tools@${grpcToolsVersion}"; + fetchSubmodules = true; + sha256 = "sha256-708lBIGW5+vvSTrZHl/kc+ck7JKNXElrghIGDrMSyx8="; + }; + + sourceRoot = "source/packages/grpc-tools"; + + nativeBuildInputs = [ pkgs.cmake ]; + + installPhase = '' + install -Dm755 -t $out/bin grpc_node_plugin + install -Dm755 -t $out/bin deps/protobuf/protoc ''; }; + rust = pkgs.rust-bin.stable.${rustVersion}.default; + # Yarn binary. yarn = pkgs.stdenv.mkDerivation { name = "yarn"; - dontUnpack = true; + src = fetchTarball { + url = "https://yarnpkg.com/downloads/${yarnVersion}/yarn-v${yarnVersion}.tar.gz"; + sha256 = "sha256:0jl77rl2sidsj3ym637w7g35wnv190l96n050aqlm4pyc6wi8v6p"; + }; buildInputs = [ - pkgs.cacert - pkgs.curl pkgs.nodejs-16_x ]; buildPhase = '' mkdir "$out" - export HOME="$out" - export PROFILE="$HOME/.bashrc" - touch "$PROFILE" - curl -o- -L https://yarnpkg.com/install.sh | bash -s -- --version ${yarnVersion} - cd "$out/.yarn" - mv * .. - cd "$out" - rm -rf .yarn + cp -R * "$out" ''; }; @@ -226,7 +217,7 @@ { packages = { conditional = conditional; - golangci-lint = golangci-lint; + node-protoc-ts = node-protoc-ts; grpc-tools = grpc-tools; helm = helm; libpcsclite = libpcsclite; diff --git a/devbox.json b/devbox.json index 80bf0cd129118..7410a81462079 100644 --- a/devbox.json +++ b/devbox.json @@ -8,6 +8,7 @@ "gci@0.9.1", "git", "github:nixos/nixpkgs/757a0d107c238d031652a8c09d1f6bf1b6f523a3#go", + "github:nixos/nixpkgs/3a785fc61f9d2960970bdce4fa86eb634c86b9d6#golangci-lint", "libiconvReal@1.16", "libfido2@1.13.0", "nodejs@16.18.1", @@ -18,10 +19,10 @@ "shellcheck@0.9.0", "yamllint@1.28.0", "path:build.assets/flake#conditional", - "path:build.assets/flake#helm", - "path:build.assets/flake#golangci-lint", "path:build.assets/flake#grpc-tools", + "path:build.assets/flake#helm", "path:build.assets/flake#libpcsclite", + "path:build.assets/flake#node-protoc-ts", "path:build.assets/flake#protoc-gen-gogo", "path:build.assets/flake#rust", "path:build.assets/flake#yarn" diff --git a/devbox.lock b/devbox.lock index b23046c397317..88f230466bc84 100644 --- a/devbox.lock +++ b/devbox.lock @@ -54,7 +54,6 @@ }, "python@3.11.2": { "last_modified": "2023-03-31T22:52:29Z", - "plugin_version": "0.0.1", "resolved": "github:NixOS/nixpkgs/242246ee1e58f54d2322227fc5eef53b4a616a31#python311", "version": "3.11.2" }, From c7d3ae3db006a9a92a8fc3672e9367e30cf4390d Mon Sep 17 00:00:00 2001 From: Mike Wilson Date: Tue, 13 Jun 2023 09:47:40 -0400 Subject: [PATCH 2/3] Install correct binary name for node protoc plugin. --- build.assets/flake/flake.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build.assets/flake/flake.nix b/build.assets/flake/flake.nix index 7027f11a1046a..31f197773d150 100644 --- a/build.assets/flake/flake.nix +++ b/build.assets/flake/flake.nix @@ -182,6 +182,10 @@ installPhase = '' install -Dm755 -t $out/bin grpc_node_plugin + + cp grpc_node_plugin grpc_tools_node_protoc_plugin + install -Dm755 -t $out/bin grpc_tools_node_protoc_plugin + install -Dm755 -t $out/bin deps/protobuf/protoc ''; }; From 10400b23773cb3d1f0867708698ded9fa1c6252b Mon Sep 17 00:00:00 2001 From: Mike Wilson Date: Tue, 13 Jun 2023 15:44:54 -0400 Subject: [PATCH 3/3] Add in zlib to support LIBFIDO compilation. --- devbox.json | 1 + devbox.lock | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/devbox.json b/devbox.json index 7410a81462079..8f32deff6a328 100644 --- a/devbox.json +++ b/devbox.json @@ -18,6 +18,7 @@ "python@3.11.2", "shellcheck@0.9.0", "yamllint@1.28.0", + "zlib@1.2.13", "path:build.assets/flake#conditional", "path:build.assets/flake#grpc-tools", "path:build.assets/flake#helm", diff --git a/devbox.lock b/devbox.lock index 88f230466bc84..b8656588fcd3a 100644 --- a/devbox.lock +++ b/devbox.lock @@ -66,6 +66,11 @@ "last_modified": "2023-02-28T22:11:13Z", "resolved": "github:NixOS/nixpkgs/995edc972ad3a1e291ac22d74b9610821357175f#yamllint", "version": "1.28.0" + }, + "zlib@1.2.13": { + "last_modified": "2023-05-01T16:53:22Z", + "resolved": "github:NixOS/nixpkgs/8670e496ffd093b60e74e7fa53526aa5920d09eb#zlib", + "version": "1.2.13" } } } \ No newline at end of file