Skip to content

Commit 8a4e70e

Browse files
committed
Build toolchain-examples
1 parent d85e491 commit 8a4e70e

File tree

11 files changed

+116
-36
lines changed

11 files changed

+116
-36
lines changed

.github/workflows/nix.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ jobs:
9797
name: ${{ matrix.test-name }}
9898
strategy:
9999
matrix:
100-
test-name: [buildstream, mongo]
100+
test-name: [buildstream, mongo, rbe-toolchain]
101101
runs-on: ubuntu-24.04
102102
timeout-minutes: 45
103103
steps:

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,3 +23,4 @@ rust-project.json
2323
darwin.bazelrc
2424
nativelink.bazelrc
2525
integration_tests/**/*.log
26+
toolchain-examples/*.log

MODULE.bazel

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ module(
44
compatibility_level = 0,
55
)
66

7-
bazel_dep(name = "rules_cc", version = "0.1.1")
8-
bazel_dep(name = "platforms", version = "0.0.11")
7+
bazel_dep(name = "rules_cc", version = "0.1.5")
8+
bazel_dep(name = "platforms", version = "1.0.0")
99
bazel_dep(name = "bazel_skylib", version = "1.7.1")
1010
bazel_dep(name = "rules_python", version = "1.3.0") # TODO(palfrey): Bump.
1111
bazel_dep(name = "rules_shell", version = "0.4.1")

flake.nix

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -379,6 +379,9 @@
379379
mongo-with-nativelink-test = pkgs.callPackage integration_tests/mongo/mongo-with-nativelink-test.nix {
380380
inherit nativelink mongodb wait4x bazelisk;
381381
};
382+
rbe-toolchain-with-nativelink-test = pkgs.callPackage toolchain-examples/rbe-toolchain-test.nix {
383+
inherit nativelink bazelisk;
384+
};
382385
}
383386
// (
384387
# It's not possible to crosscompile to darwin, not even between

local-remote-execution/MODULE.bazel

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@ module(
77
compatibility_level = 0,
88
)
99

10-
bazel_dep(name = "platforms", version = "0.0.11")
10+
bazel_dep(name = "platforms", version = "1.0.0")
1111

1212
# Use the starlark implementation of C++ rules instead of the builtin ones.
13-
bazel_dep(name = "rules_cc", version = "0.1.1")
13+
bazel_dep(name = "rules_cc", version = "0.1.5")
1414

1515
# Use the starlark implementation of Java rules instead of the builtin ones.
1616
bazel_dep(name = "rules_java", version = "8.11.0")

nativelink-util/src/telemetry.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ fn otlp_filter() -> EnvFilter {
7171

7272
// Create a tracing layer intended for stdout printing.
7373
//
74-
// The output of this layer is configurable via the `NL_LOG_FMT` environment
74+
// The output of this layer is configurable via the `NL_LOG` environment
7575
// variable.
7676
fn tracing_stdout_layer() -> impl Layer<Registry> {
7777
let nl_log_fmt = env::var("NL_LOG").unwrap_or_else(|_| "pretty".to_string());

toolchain-examples/.bazelrc

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,6 @@
1-
# Don't use the host's default PATH and LD_LIBRARY_PATH.
2-
build --incompatible_strict_action_env
3-
41
# Use rules_python's builtin script to emulate a bootstrap python.
52
build --@rules_python//python/config_settings:bootstrap_impl=script
63

7-
# Toolchain to verify remote execution with zig-cc.
8-
build:zig-cc --platforms @zig_sdk//platform:linux_amd64
9-
build:zig-cc --platforms @zig_sdk//platform:linux_amd64
10-
114
# These toolchains map out everything in
125
# https://github.com/uber/hermetic_cc_toolchain/blob/bfc407599e503a44928a3cefad27421c9341eff0/MODULE.bazel#L44
136
#
@@ -75,7 +68,6 @@ build:zig-cc --extra_toolchains @zig_sdk-macos-amd64//libc_aware/toolchain:linux
7568
build:zig-cc --extra_toolchains @zig_sdk-macos-amd64//toolchain:wasip1_wasm
7669

7770
# Toolchain to verify remote execution with contrib/toolchains_llvm.
78-
build:llvm --platforms=@toolchains_llvm//platforms:linux-x86_64
7971
build:llvm --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux
8072

8173
# Java runtime to ensure hermeticity on the remote.

toolchain-examples/MODULE.bazel

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,10 @@ module(
44
compatibility_level = 0,
55
)
66

7-
bazel_dep(name = "platforms", version = "0.0.11")
7+
bazel_dep(name = "platforms", version = "1.0.0")
88

99
# C++
10-
bazel_dep(name = "rules_cc", version = "0.1.1")
10+
bazel_dep(name = "rules_cc", version = "0.1.5")
1111

1212
# Java
1313
bazel_dep(name = "rules_java", version = "8.11.0")
@@ -34,7 +34,16 @@ python.toolchain(
3434
use_repo(pip, "pip")
3535

3636
# Go
37-
bazel_dep(name = "rules_go", version = "0.53.0")
37+
bazel_dep(name = "rules_go", version = "0.57.0")
38+
39+
# Adds https://github.com/bazel-contrib/rules_go/commit/74199c92e20399b6ef46684b2c6fdd94b50a7892
40+
# to fix bash issues with Nix
41+
archive_override(
42+
module_name = "rules_go",
43+
integrity = "sha256-ukyyC80j4VhRCD7DOaenkk41Vvnmsp7uAfHr4lxdXtQ=",
44+
strip_prefix = "rules_go-74199c92e20399b6ef46684b2c6fdd94b50a7892",
45+
urls = ["https://github.com/bazel-contrib/rules_go/archive/74199c92e20399b6ef46684b2c6fdd94b50a7892.zip"],
46+
)
3847

3948
# Rust
4049
bazel_dep(name = "rules_rust", version = "0.61.0")
@@ -89,8 +98,5 @@ bazel_dep(name = "abseil-cpp", version = "20250127.0")
8998
# Abseil for python
9099
bazel_dep(name = "abseil-py", version = "2.1.0")
91100

92-
# GRPC
93-
bazel_dep(name = "grpc", version = "1.71.0")
94-
95101
# Circl (Go, C++)
96102
bazel_dep(name = "circl", version = "1.3.8")

toolchain-examples/nativelink-config.json5

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,17 @@
11
{
2-
stores: {
3-
AC_MAIN_STORE: {
2+
stores: [
3+
{
4+
name: "AC_MAIN_STORE",
45
filesystem: {
56
content_path: "/tmp/nativelink/data-worker-test/content_path-ac",
67
temp_path: "/tmp/nativelink/data-worker-test/tmp_path-ac",
78
eviction_policy: {
8-
// 1gb.
9-
max_bytes: 1000000000,
9+
max_bytes: "1gb",
1010
},
1111
},
1212
},
13-
WORKER_FAST_SLOW_STORE: {
13+
{
14+
name: "WORKER_FAST_SLOW_STORE",
1415
fast_slow: {
1516
// "fast" must be a "filesystem" store because the worker uses it to make
1617
// hardlinks on disk to a directory where the jobs are running.
@@ -19,8 +20,7 @@
1920
content_path: "/tmp/nativelink/data-worker-test/content_path-cas",
2021
temp_path: "/tmp/nativelink/data-worker-test/tmp_path-cas",
2122
eviction_policy: {
22-
// 10gb.
23-
max_bytes: 10000000000,
23+
max_bytes: "10gb",
2424
},
2525
},
2626
},
@@ -34,9 +34,10 @@
3434
},
3535
},
3636
},
37-
},
38-
schedulers: {
39-
MAIN_SCHEDULER: {
37+
],
38+
schedulers: [
39+
{
40+
name: "MAIN_SCHEDULER",
4041
simple: {
4142
supported_platform_properties: {
4243
cpu_count: "minimum",
@@ -48,7 +49,7 @@
4849
},
4950
},
5051
},
51-
},
52+
],
5253
workers: [
5354
{
5455
local: {
@@ -127,11 +128,11 @@
127128
},
128129
},
129130
],
130-
bytestream: {
131-
cas_stores: {
132-
"": "WORKER_FAST_SLOW_STORE",
131+
bytestream: [
132+
{
133+
cas_store: "WORKER_FAST_SLOW_STORE",
133134
},
134-
},
135+
],
135136
},
136137
},
137138
{
Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
{
2+
nativelink,
3+
writeShellScriptBin,
4+
bazelisk,
5+
}:
6+
writeShellScriptBin "rbe-toolchain-test" ''
7+
set -uo pipefail
8+
9+
cleanup() {
10+
local pids=$(jobs -pr)
11+
[ -n "$pids" ] && kill $pids
12+
}
13+
trap "cleanup" INT QUIT TERM EXIT
14+
15+
NO_COLOR=true ${nativelink}/bin/nativelink -- toolchain-examples/nativelink-config.json5 | tee -i toolchain-examples/nativelink.log &
16+
17+
CORE_BAZEL_ARGS="--check_direct_dependencies=error --remote_cache=grpc://localhost:50051 --remote_executor=grpc://localhost:50051"
18+
19+
CPU_TYPE=$(uname -m)
20+
21+
if [[ "$CPU_TYPE" == 'x86_64' ]]; then
22+
PLATFORM='amd64'
23+
else
24+
PLATFORM='arm64'
25+
fi
26+
27+
LLVM_PLATFORM="--platforms=@toolchains_llvm//platforms:linux-''${CPU_TYPE}"
28+
ZIG_PLATFORM="--platforms @zig_sdk//platform:linux_''${PLATFORM}"
29+
30+
# As per https://nativelink.com/docs/rbe/remote-execution-examples#minimal-example-targets
31+
COMMANDS=("test //cpp $ZIG_PLATFORM"
32+
"test //cpp --config=llvm $LLVM_PLATFORM"
33+
"test //python"
34+
"test //go $ZIG_PLATFORM"
35+
# "test //rust $ZIG_PLATFORM" # rules_rust isn't RBE-compatible
36+
"test //java:HelloWorld --config=java"
37+
"build @curl//... $ZIG_PLATFORM"
38+
"build @zstd//... $ZIG_PLATFORM"
39+
# "test @abseil-cpp//... $ZIG_PLATFORM" # Buggy build due to google_benchmark errors
40+
"test @abseil-py//..."
41+
"test @circl//... $ZIG_PLATFORM"
42+
)
43+
44+
echo "" > toolchain-examples/cmd.log
45+
for cmd in "''${COMMANDS[@]}"
46+
do
47+
FULL_CMD="${bazelisk}/bin/bazelisk $cmd $CORE_BAZEL_ARGS"
48+
echo $FULL_CMD
49+
echo -e \\n$FULL_CMD\\n >> toolchain-examples/cmd.log
50+
cmd_output=$(cd toolchain-examples && eval "$FULL_CMD" 2>&1 | tee -ai cmd.log)
51+
cmd_exit_code=$?
52+
case $cmd_exit_code in
53+
0 )
54+
echo "Saw a successful $cmd build"
55+
;;
56+
*)
57+
echo "Failed $cmd build:"
58+
echo $cmd_output
59+
exit 1
60+
;;
61+
esac
62+
done
63+
64+
nativelink_output=$(cat toolchain-examples/nativelink.log)
65+
66+
case $nativelink_output in
67+
*"ERROR "* )
68+
echo "Error in nativelink build"
69+
exit 1
70+
;;
71+
*)
72+
echo 'Successful nativelink build'
73+
;;
74+
esac
75+
''

0 commit comments

Comments
 (0)