From 9c98120f33579b72561e02826d9fccf222eccb3c Mon Sep 17 00:00:00 2001 From: Kevin Lubick Date: Tue, 5 Apr 2022 06:56:26 -0700 Subject: [PATCH] Add support for .ar archives (and .deb files) This implements #[15130](https://github.com/bazelbuild/bazel/issues/15130). As I was updating the docs for .ar and .deb formats, I also addressed some previous formats that had been added but not propagated through to all the documentation places. Closes #15132. PiperOrigin-RevId: 439569440 --- .../lib/bazel/repository/ArFunction.java | 81 +++++++++++++++++ .../bazel/repository/DecompressorValue.java | 4 +- .../starlark/StarlarkRepositoryContext.java | 3 +- .../lib/bazel/repository/ArFunctionTest.java | 88 +++++++++++++++++++ .../devtools/build/lib/bazel/repository/BUILD | 1 + .../repository/DecompressorValueTest.java | 4 + .../build/lib/bazel/repository/test_files.ar | 9 ++ tools/build_defs/repo/http.bzl | 7 +- 8 files changed, 192 insertions(+), 5 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java create mode 100644 src/test/java/com/google/devtools/build/lib/bazel/repository/ArFunctionTest.java create mode 100644 src/test/java/com/google/devtools/build/lib/bazel/repository/test_files.ar diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java new file mode 100644 index 00000000000000..dc4ac44f96f8ea --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java @@ -0,0 +1,81 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository; + +import com.google.common.io.ByteStreams; +import com.google.devtools.build.lib.bazel.repository.DecompressorValue.Decompressor; +import com.google.devtools.build.lib.vfs.Path; +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import org.apache.commons.compress.archivers.ar.ArArchiveEntry; +import org.apache.commons.compress.archivers.ar.ArArchiveInputStream; + +/** + * Opens a .ar archive file. It ignores the prefix setting because these archives cannot contain + * directories. + */ +public class ArFunction implements Decompressor { + + public static final Decompressor INSTANCE = new ArFunction(); + + // This is the same value as picked for .tar files, which appears to have worked well. + private static final int BUFFER_SIZE = 32 * 1024; + + private InputStream getDecompressorStream(DecompressorDescriptor descriptor) throws IOException { + return new BufferedInputStream( + new FileInputStream(descriptor.archivePath().getPathFile()), BUFFER_SIZE); + } + ; + + @Override + public Path decompress(DecompressorDescriptor descriptor) + throws InterruptedException, IOException { + if (Thread.interrupted()) { + throw new InterruptedException(); + } + + try (InputStream decompressorStream = getDecompressorStream(descriptor)) { + ArArchiveInputStream arStream = new ArArchiveInputStream(decompressorStream); + ArArchiveEntry entry; + while ((entry = arStream.getNextArEntry()) != null) { + Path filePath = descriptor.repositoryPath().getRelative(entry.getName()); + filePath.getParentDirectory().createDirectoryAndParents(); + if (entry.isDirectory()) { + // ar archives don't contain any directory information, so this should never + // happen + continue; + } else { + // We do not have to worry about symlinks in .ar files - it's not supported + // by the .ar file format. + try (OutputStream out = filePath.getOutputStream()) { + ByteStreams.copy(arStream, out); + } + filePath.chmod(entry.getMode()); + // entry.getLastModified() appears to be in seconds, so we need to convert + // it into milliseconds for setLastModifiedTime + filePath.setLastModifiedTime(entry.getLastModified() * 1000L); + } + if (Thread.interrupted()) { + throw new InterruptedException(); + } + } + } + + return descriptor.repositoryPath(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java index 49ff20b73b3498..a651c2cd921325 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java @@ -107,11 +107,13 @@ static Decompressor getDecompressor(Path archivePath) return TarZstFunction.INSTANCE; } else if (baseName.endsWith(".tar.bz2")) { return TarBz2Function.INSTANCE; + } else if (baseName.endsWith(".ar") || baseName.endsWith(".deb")) { + return ArFunction.INSTANCE; } else { throw new RepositoryFunctionException( Starlark.errorf( "Expected a file with a .zip, .jar, .war, .aar, .tar, .tar.gz, .tgz, .tar.xz, .txz," - + " .tar.zst, .tzst, or .tar.bz2 suffix (got %s)", + + " .tar.zst, .tzst, .tar.bz2, .ar or .deb suffix (got %s)", archivePath), Transience.PERSISTENT); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java index 217f8b0c6661a9..1ef1d77b31fdae 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java @@ -825,7 +825,8 @@ public void extract(Object archive, Object output, String stripPrefix, StarlarkT + " By default, the archive type is determined from the file extension of" + " the URL." + " If the file has no extension, you can explicitly specify either \"zip\"," - + " \"jar\", \"war\", \"aar\", \"tar.gz\", \"tgz\", \"tar.bz2\", or \"tar.xz\"" + + " \"jar\", \"war\", \"aar\", \"tar\", \"tar.gz\", \"tgz\", \"tar.xz\"," + + " \"txz\", \".tar.zst\", \".tzst\", \"tar.bz2\", \".ar\", or \".deb\"" + " here."), @Param( name = "stripPrefix", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/ArFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/ArFunctionTest.java new file mode 100644 index 00000000000000..96077f04a0be9b --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/ArFunctionTest.java @@ -0,0 +1,88 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.devtools.build.lib.testutil.TestConstants; +import com.google.devtools.build.lib.testutil.TestUtils; +import com.google.devtools.build.lib.unix.UnixFileSystem; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.vfs.DigestHashFunction; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.JavaIoFileSystem; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.runfiles.Runfiles; +import java.io.File; +import java.io.IOException; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests decompressing archives. */ +@RunWith(JUnit4.class) +public class ArFunctionTest { + /* + * .ar archive created with ar cr test_files.ar archived_first.txt archived_second.md + * The files contain short UTF-8 encoded strings. + */ + private static final String ARCHIVE_NAME = "test_files.ar"; + private static final String PATH_TO_TEST_ARCHIVE = + "/com/google/devtools/build/lib/bazel/repository/"; + private static final String FIRST_FILE_NAME = "archived_first.txt"; + private static final String SECOND_FILE_NAME = "archived_second.md"; + + @Test + public void testDecompress() throws Exception { + Path outputDir = decompress(createDescriptorBuilder()); + + assertThat(outputDir.exists()).isTrue(); + Path firstFile = outputDir.getRelative(FIRST_FILE_NAME); + assertThat(firstFile.exists()).isTrue(); + // There are 20 bytes in the content "this is test file 1" + assertThat(firstFile.getFileSize()).isEqualTo(20); + assertThat(firstFile.isSymbolicLink()).isFalse(); + + Path secondFile = outputDir.getRelative(SECOND_FILE_NAME); + assertThat(secondFile.exists()).isTrue(); + // There are 20 bytes in the content "this is the second test file" + assertThat(secondFile.getFileSize()).isEqualTo(29); + assertThat(secondFile.isSymbolicLink()).isFalse(); + } + + private Path decompress(DecompressorDescriptor.Builder descriptorBuilder) throws Exception { + descriptorBuilder.setDecompressor(ArFunction.INSTANCE); + return new ArFunction().decompress(descriptorBuilder.build()); + } + + private DecompressorDescriptor.Builder createDescriptorBuilder() throws IOException { + // This was cribbed from TestArchiveDescriptor + FileSystem testFS = + OS.getCurrent() == OS.WINDOWS + ? new JavaIoFileSystem(DigestHashFunction.SHA256) + : new UnixFileSystem(DigestHashFunction.SHA256, /*hashAttributeName=*/ ""); + + // do not rely on TestConstants.JAVATESTS_ROOT end with slash, but ensure separators + // are not duplicated + String path = + (TestConstants.JAVATESTS_ROOT + PATH_TO_TEST_ARCHIVE + ARCHIVE_NAME).replace("//", "/"); + Path tarballPath = testFS.getPath(Runfiles.create().rlocation(path)); + + Path workingDir = testFS.getPath(new File(TestUtils.tmpDir()).getCanonicalPath()); + Path outDir = workingDir.getRelative("out"); + + return DecompressorDescriptor.builder().setRepositoryPath(outDir).setArchivePath(tarballPath); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD index 2397b27e4bdfba..f9cf05bb792511 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD @@ -27,6 +27,7 @@ java_library( data = [ "test_decompress_archive.tar.gz", "test_decompress_archive.zip", + "test_files.ar", ], deps = [ "//src/main/java/com/google/devtools/build/lib/bazel/repository", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java index 032da6fa321f39..f457f2b61161d8 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java @@ -56,6 +56,10 @@ public void testKnownFileExtensionsDoNotThrow() throws Exception { DecompressorDescriptor.builder().setArchivePath(path).build(); path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.tar.bz2"); DecompressorDescriptor.builder().setArchivePath(path).build(); + path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.ar"); + DecompressorDescriptor.builder().setArchivePath(path).build(); + path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.deb"); + DecompressorDescriptor.builder().setArchivePath(path).build(); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/test_files.ar b/src/test/java/com/google/devtools/build/lib/bazel/repository/test_files.ar new file mode 100644 index 00000000000000..8aaddbe34b89de --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/test_files.ar @@ -0,0 +1,9 @@ +! +// 40 ` +archived_first.txt/ +archived_second.md/ +/0 0 0 0 644 20 ` +this is test file 1 +/20 0 0 0 644 29 ` +this is the second test file + diff --git a/tools/build_defs/repo/http.bzl b/tools/build_defs/repo/http.bzl index 1bbd2f981a3aed..fd40a5b22afadc 100644 --- a/tools/build_defs/repo/http.bzl +++ b/tools/build_defs/repo/http.bzl @@ -269,7 +269,7 @@ match a directory in the archive, Bazel will return an error.""", By default, the archive type is determined from the file extension of the URL. If the file has no extension, you can explicitly specify one of the following: `"zip"`, `"jar"`, `"war"`, `"aar"`, `"tar"`, `"tar.gz"`, `"tgz"`, -`"tar.xz"`, or `tar.bz2`.""", +`"tar.xz"`, `"txz"`, `"tar.zst"`, `"tzst"`, `tar.bz2`, `"ar"`, or `"deb"`.""", ), "patches": attr.label_list( default = [], @@ -357,8 +357,9 @@ http_archive = repository_rule( """Downloads a Bazel repository as a compressed archive file, decompresses it, and makes its targets available for binding. -It supports the following file extensions: `"zip"`, `"jar"`, `"war"`, `"aar"`, -`"tar"`, `"tar.gz"`, `"tgz"`, `"tar.xz"`, and `tar.bz2`. +It supports the following file extensions: `"zip"`, `"jar"`, `"war"`, `"aar"`, `"tar"`, +`"tar.gz"`, `"tgz"`, `"tar.xz"`, `"txz"`, `"tar.zst"`, `"tzst"`, `tar.bz2`, `"ar"`, +or `"deb"`. Examples: Suppose the current repository contains the source code for a chat program,