Implement untaring in rust

WORKSPACE

@@ -32,7 +32,7 @@ new_git_repository(
     name = "libc",
     build_file = "@io_bazel_rules_rust//:libc.BUILD",
     remote = "https://github.com/rust-lang/libc",
-    tag = "0.2.20",
+    tag = "0.2.71",
 )
 
 load("@io_bazel_rules_rust//proto:repositories.bzl", "rust_proto_repositories")
@@ -102,3 +102,6 @@ bazel_version(name = "bazel_version")
 
 load("@examples//hello_sys:workspace.bzl", "remote_deps")
 remote_deps()
+
+load("//tar_wrapper/remote_crates:remote_crates.bzl", "fetch_remote_crates")
+fetch_remote_crates()

cargo/cargo_build_script_runner/BUILD

@@ -14,5 +14,8 @@ rust_binary(
     name = "cargo_build_script_runner",
     srcs = ["bin.rs"],
     visibility = ["//visibility:public"],
-    deps = [":cargo_build_script_output_parser"],
+    deps = [
+        ":cargo_build_script_output_parser",
+        "//tar_wrapper",
+    ],
 )

cargo/cargo_build_script_runner/bin.rs

@@ -15,6 +15,7 @@
 // A simple wrapper around a build_script execution to generate file to reuse
 // by rust_library/rust_binary.
 extern crate cargo_build_script_output_parser;
+extern crate tar_wrapper;
 
 use cargo_build_script_output_parser::{BuildScriptOutput, CompileAndLinkFlags};
 use std::env;

libc.BUILD

@@ -4,4 +4,5 @@ rust_library(
     name = "libc",
     srcs = glob(["src/**/*.rs"]),
     visibility = ["//visibility:public"],
+    untar = "@io_bazel_rules_rust//tar_wrapper:false",
 )

rust/private/clippy.bzl

@@ -60,7 +60,7 @@ def _clippy_aspect_impl(target, ctx):
         toolchain,
     )
 
-    compile_inputs, prep_commands, dynamic_env, dynamic_build_flags = collect_inputs(
+    compile_inputs, tool_inputs, prep_commands, dynamic_env, dynamic_build_flags = collect_inputs(
         ctx,
         ctx.rule.file,
         ctx.rule.files,
@@ -111,7 +111,7 @@ def _clippy_aspect_impl(target, ctx):
         inputs = compile_inputs,
         outputs = [clippy_marker],
         env = env,
-        tools = [toolchain.clippy_driver],
+        tools = tool_inputs + [toolchain.clippy_driver],
         arguments = [args],
         mnemonic = "Clippy",
     )

rust/private/rust.bzl

@@ -360,6 +360,7 @@ _rust_common_attrs = {
         ],
     ),
     "_cc_toolchain": attr.label(default = "@bazel_tools//tools/cpp:current_cc_toolchain"),
+    "untar": attr.label(default = "@io_bazel_rules_rust//tar_wrapper:untar", cfg = "exec", allow_single_file = True, executable = True),
 }
 
 _rust_library_attrs = {

rust/private/rustc.bzl

@@ -249,10 +249,10 @@ def _process_build_scripts(
         build_info,
         dep_info,
         compile_inputs):
-    extra_inputs, prep_commands, dynamic_env, dynamic_build_flags = _create_out_dir_action(ctx, file, build_info, dep_info)
+    extra_inputs, tool_inputs, prep_commands, dynamic_env, dynamic_build_flags = _create_out_dir_action(ctx, file, build_info, dep_info)
     if extra_inputs:
         compile_inputs = depset(extra_inputs, transitive = [compile_inputs])
-    return compile_inputs, prep_commands, dynamic_env, dynamic_build_flags
+    return compile_inputs, tool_inputs, prep_commands, dynamic_env, dynamic_build_flags
 
 def collect_inputs(
         ctx,
@@ -441,7 +441,7 @@ def rustc_compile_action(
         toolchain,
     )
 
-    compile_inputs, prep_commands, dynamic_env, dynamic_build_flags = collect_inputs(
+    compile_inputs, tool_inputs, prep_commands, dynamic_env, dynamic_build_flags = collect_inputs(
         ctx,
         ctx.file,
         ctx.files,
@@ -482,6 +482,7 @@ def rustc_compile_action(
     ctx.actions.run_shell(
         command = command,
         inputs = compile_inputs,
+        tools = tool_inputs,
         outputs = [crate_info.output],
         env = env,
         arguments = [args],
@@ -524,12 +525,12 @@ def _create_out_dir_action(ctx, file, build_info, dep_info):
 
     prep_commands = []
     input_files = []
+    tool_inputs = []
     # Env vars and build flags which need to be set in the action's command line, rather than on the action's env,
     # because they rely on other env vars or commands.
     dynamic_env = {}
     dynamic_build_flags = []
 
-    # TODO: Remove system tar usage
     if build_info:
         prep_commands.append("export $(cat %s)" % build_info.rustc_env.path)
         # out_dir will be added as input by the transitive_build_infos loop below.
@@ -538,8 +539,9 @@ def _create_out_dir_action(ctx, file, build_info, dep_info):
     elif tar_file_attr:
         out_dir = ".out-dir"
         prep_commands.append("mkdir -p $OUT_DIR")
-        prep_commands.append("tar -xzf {tar} -C $OUT_DIR".format(tar=tar_file_attr.path))
+        prep_commands.append("{untar} {tar} $OUT_DIR".format(untar = file.untar.path, tar=tar_file_attr.path))
         input_files.append(tar_file_attr)
+        tool_inputs.append(file.untar)
         dynamic_env["OUT_DIR"] = "${{EXEC_ROOT}}/{}".format(out_dir)
 
     # This should probably only actually be exposed to actions which link.
@@ -548,7 +550,7 @@ def _create_out_dir_action(ctx, file, build_info, dep_info):
         dynamic_build_flags.append("$(cat '{}' | sed -e \"s#\${{EXEC_ROOT}}#${{EXEC_ROOT}}#g\")".format(dep_build_info.link_flags.path))
         input_files.append(dep_build_info.link_flags)
 
-    return input_files, prep_commands, dynamic_env, dynamic_build_flags
+    return input_files, tool_inputs, prep_commands, dynamic_env, dynamic_build_flags
 
 def _compute_rpaths(toolchain, output_dir, dep_info):
     """

tar_wrapper/BUILD

@@ -0,0 +1,32 @@
+load("//rust:rust.bzl", "rust_binary", "rust_library", "rust_test")
+
+rust_library(
+    name = "tar_wrapper",
+    srcs = ["src/lib.rs"],
+    deps = [
+        "@flate2//:flate2",
+        "@tar//:tar",
+    ],
+    edition = "2018",
+    visibility = ["//visibility:public"],
+    untar = ":false",
+)
+
+rust_binary(
+    name = "untar",
+    srcs = ["src/untar.rs"],
+    deps = [":tar_wrapper"],
+    edition = "2018",
+    untar = ":false",
+    visibility = ["//visibility:public"],
+)
+
+rust_test(
+    name = "test",
+    crate = ":tar_wrapper",
+    data = ["//test:example.tar.gz"],
+)
+
+# Hacky script we use as the value of the `untar` attribute for rust_libraries used to build untar itself, which don't
+# actually need untar, to avoid dependency cycles.
+exports_files(["false"])

tar_wrapper/false

@@ -0,0 +1,3 @@
+#!/bin/bash -eu
+echo >&2 "Tried to run bogus untar"
+exit 1

tar_wrapper/remote_crates/adler32-1.1.0.BUILD

@@ -0,0 +1,22 @@
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_library",
+)
+
+rust_library(
+    name = "adler32",
+    crate_type = "lib",
+    srcs = glob(["**/*.rs"]),
+    crate_root = "src/lib.rs",
+    edition = "2015",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    version = "1.1.0",
+    crate_features = [
+        "default",
+        "std",
+    ],
+    visibility = ["//visibility:public"],
+    untar = "@io_bazel_rules_rust//tar_wrapper:false",
+)

tar_wrapper/remote_crates/cfg-if-0.1.10.BUILD

@@ -0,0 +1,18 @@
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_library",
+)
+
+rust_library(
+    name = "cfg_if",
+    crate_type = "lib",
+    srcs = glob(["**/*.rs"]),
+    crate_root = "src/lib.rs",
+    edition = "2018",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    version = "0.1.10",
+    visibility = ["//visibility:public"],
+    untar = "@io_bazel_rules_rust//tar_wrapper:false",
+)

tar_wrapper/remote_crates/crc32fast-1.2.0.BUILD

@@ -0,0 +1,32 @@
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_library",
+)
+
+load(
+    "@io_bazel_rules_rust//cargo:cargo_build_script.bzl",
+    "cargo_build_script",
+)
+
+rust_library(
+    name = "crc32fast",
+    crate_type = "lib",
+    deps = [
+        #":crc32fast_build_script",
+        "@cfg_if//:cfg_if",
+    ],
+    srcs = glob(["**/*.rs"]),
+    crate_root = "src/lib.rs",
+    edition = "2015",
+    rustc_flags = [
+        "--cap-lints=allow",
+        "--cfg=rc32fast_stdarchx86",
+    ],
+    version = "1.2.0",
+    crate_features = [
+        "default",
+        "std",
+    ],
+    visibility = ["//visibility:public"],
+    untar = "@io_bazel_rules_rust//tar_wrapper:false",
+)

tar_wrapper/remote_crates/filetime-0.2.10.BUILD

@@ -0,0 +1,22 @@
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_library",
+)
+
+rust_library(
+    name = "filetime",
+    crate_type = "lib",
+    deps = [
+        "@cfg_if//:cfg_if",
+        "@libc//:libc",
+    ],
+    srcs = glob(["**/*.rs"]),
+    crate_root = "src/lib.rs",
+    edition = "2018",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    version = "0.2.10",
+    visibility = ["//visibility:public"],
+    untar = "@io_bazel_rules_rust//tar_wrapper:false",
+)

tar_wrapper/remote_crates/flate2-1.0.14.BUILD

@@ -0,0 +1,27 @@
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_library",
+)
+
+rust_library(
+    name = "flate2",
+    crate_type = "lib",
+    deps = [
+        "@cfg_if//:cfg_if",
+        "@crc32fast//:crc32fast",
+        "@libc//:libc",
+        "@miniz_oxide//:miniz_oxide",
+    ],
+    srcs = glob(["**/*.rs"]),
+    crate_root = "src/lib.rs",
+    edition = "2018",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    version = "1.0.14",
+    crate_features = [
+        "miniz_oxide",
+    ],
+    visibility = ["//visibility:public"],
+    untar = "@io_bazel_rules_rust//tar_wrapper:false",
+)

tar_wrapper/remote_crates/miniz_oxide-0.3.7.BUILD

@@ -0,0 +1,21 @@
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_library",
+)
+
+rust_library(
+    name = "miniz_oxide",
+    crate_type = "lib",
+    deps = [
+        "@adler32//:adler32",
+    ],
+    srcs = glob(["**/*.rs"]),
+    crate_root = "src/lib.rs",
+    edition = "2018",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    version = "0.3.7",
+    visibility = ["//visibility:public"],
+    untar = "@io_bazel_rules_rust//tar_wrapper:false",
+)

tar_wrapper/remote_crates/remote_crates.bzl

@@ -0,0 +1,74 @@
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+def fetch_remote_crates():
+    http_archive(
+        name = "adler32",
+        url = "https://crates-io.s3-us-west-1.amazonaws.com/crates/adler32/adler32-1.1.0.crate",
+        sha256 = "567b077b825e468cc974f0020d4082ee6e03132512f207ef1a02fd5d00d1f32d",
+        type = "tar.gz",
+        strip_prefix = "adler32-1.1.0",
+        build_file = Label("//tar_wrapper/remote_crates:adler32-1.1.0.BUILD"),
+    )
+
+    http_archive(
+        name = "cfg_if",
+        url = "https://crates-io.s3-us-west-1.amazonaws.com/crates/cfg-if/cfg-if-0.1.10.crate",
+        sha256 = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822",
+        type = "tar.gz",
+        strip_prefix = "cfg-if-0.1.10",
+        build_file = Label("//tar_wrapper/remote_crates:cfg-if-0.1.10.BUILD"),
+    )
+
+    http_archive(
+        name = "crc32fast",
+        url = "https://crates-io.s3-us-west-1.amazonaws.com/crates/crc32fast/crc32fast-1.2.0.crate",
+        sha256 = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1",
+        type = "tar.gz",
+        strip_prefix = "crc32fast-1.2.0",
+        build_file = Label("//tar_wrapper/remote_crates:crc32fast-1.2.0.BUILD"),
+    )
+
+    http_archive(
+        name = "filetime",
+        url = "https://crates-io.s3-us-west-1.amazonaws.com/crates/filetime/filetime-0.2.10.crate",
+        sha256 = "affc17579b132fc2461adf7c575cc6e8b134ebca52c51f5411388965227dc695",
+        type = "tar.gz",
+        strip_prefix = "filetime-0.2.10",
+        build_file = Label("//tar_wrapper/remote_crates:filetime-0.2.10.BUILD"),
+    )
+
+    http_archive(
+        name = "flate2",
+        url = "https://crates-io.s3-us-west-1.amazonaws.com/crates/flate2/flate2-1.0.14.crate",
+        sha256 = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42",
+        type = "tar.gz",
+        strip_prefix = "flate2-1.0.14",
+        build_file = Label("//tar_wrapper/remote_crates:flate2-1.0.14.BUILD"),
+    )
+
+    http_archive(
+        name = "miniz_oxide",
+        url = "https://crates-io.s3-us-west-1.amazonaws.com/crates/miniz_oxide/miniz_oxide-0.3.7.crate",
+        sha256 = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435",
+        type = "tar.gz",
+        strip_prefix = "miniz_oxide-0.3.7",
+        build_file = Label("//tar_wrapper/remote_crates:miniz_oxide-0.3.7.BUILD"),
+    )
+
+    http_archive(
+        name = "tar",
+        url = "https://crates-io.s3-us-west-1.amazonaws.com/crates/tar/tar-0.4.29.crate",
+        sha256 = "c8a4c1d0bee3230179544336c15eefb563cf0302955d962e456542323e8c2e8a",
+        type = "tar.gz",
+        strip_prefix = "tar-0.4.29",
+        build_file = Label("//tar_wrapper/remote_crates:tar-0.4.29.BUILD"),
+    )
+
+    http_archive(
+        name = "xattr",
+        url = "https://crates-io.s3-us-west-1.amazonaws.com/crates/xattr/xattr-0.2.2.crate",
+        sha256 = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c",
+        type = "tar.gz",
+        strip_prefix = "xattr-0.2.2",
+        build_file = Label("//tar_wrapper/remote_crates:xattr-0.2.2.BUILD"),
+    )