Skip to content

Commit

Permalink
Add support for .ar archives (and .deb files)
Browse files Browse the repository at this point in the history
This implements #[15130](bazelbuild#15130).

As I was updating the docs for .ar and .deb formats, I also addressed some previous formats that had been added but not propagated through to all the documentation places.

Closes bazelbuild#15132.

PiperOrigin-RevId: 439569440
  • Loading branch information
kjlubick authored and copybara-github committed Apr 5, 2022
1 parent 4a1e5e4 commit 9c98120
Show file tree
Hide file tree
Showing 8 changed files with 192 additions and 5 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
// Copyright 2022 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.bazel.repository;

import com.google.common.io.ByteStreams;
import com.google.devtools.build.lib.bazel.repository.DecompressorValue.Decompressor;
import com.google.devtools.build.lib.vfs.Path;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import org.apache.commons.compress.archivers.ar.ArArchiveEntry;
import org.apache.commons.compress.archivers.ar.ArArchiveInputStream;

/**
* Opens a .ar archive file. It ignores the prefix setting because these archives cannot contain
* directories.
*/
public class ArFunction implements Decompressor {

public static final Decompressor INSTANCE = new ArFunction();

// This is the same value as picked for .tar files, which appears to have worked well.
private static final int BUFFER_SIZE = 32 * 1024;

private InputStream getDecompressorStream(DecompressorDescriptor descriptor) throws IOException {
return new BufferedInputStream(
new FileInputStream(descriptor.archivePath().getPathFile()), BUFFER_SIZE);
}
;

@Override
public Path decompress(DecompressorDescriptor descriptor)
throws InterruptedException, IOException {
if (Thread.interrupted()) {
throw new InterruptedException();
}

try (InputStream decompressorStream = getDecompressorStream(descriptor)) {
ArArchiveInputStream arStream = new ArArchiveInputStream(decompressorStream);
ArArchiveEntry entry;
while ((entry = arStream.getNextArEntry()) != null) {
Path filePath = descriptor.repositoryPath().getRelative(entry.getName());
filePath.getParentDirectory().createDirectoryAndParents();
if (entry.isDirectory()) {
// ar archives don't contain any directory information, so this should never
// happen
continue;
} else {
// We do not have to worry about symlinks in .ar files - it's not supported
// by the .ar file format.
try (OutputStream out = filePath.getOutputStream()) {
ByteStreams.copy(arStream, out);
}
filePath.chmod(entry.getMode());
// entry.getLastModified() appears to be in seconds, so we need to convert
// it into milliseconds for setLastModifiedTime
filePath.setLastModifiedTime(entry.getLastModified() * 1000L);
}
if (Thread.interrupted()) {
throw new InterruptedException();
}
}
}

return descriptor.repositoryPath();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -107,11 +107,13 @@ static Decompressor getDecompressor(Path archivePath)
return TarZstFunction.INSTANCE;
} else if (baseName.endsWith(".tar.bz2")) {
return TarBz2Function.INSTANCE;
} else if (baseName.endsWith(".ar") || baseName.endsWith(".deb")) {
return ArFunction.INSTANCE;
} else {
throw new RepositoryFunctionException(
Starlark.errorf(
"Expected a file with a .zip, .jar, .war, .aar, .tar, .tar.gz, .tgz, .tar.xz, .txz,"
+ " .tar.zst, .tzst, or .tar.bz2 suffix (got %s)",
+ " .tar.zst, .tzst, .tar.bz2, .ar or .deb suffix (got %s)",
archivePath),
Transience.PERSISTENT);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -825,7 +825,8 @@ public void extract(Object archive, Object output, String stripPrefix, StarlarkT
+ " By default, the archive type is determined from the file extension of"
+ " the URL."
+ " If the file has no extension, you can explicitly specify either \"zip\","
+ " \"jar\", \"war\", \"aar\", \"tar.gz\", \"tgz\", \"tar.bz2\", or \"tar.xz\""
+ " \"jar\", \"war\", \"aar\", \"tar\", \"tar.gz\", \"tgz\", \"tar.xz\","
+ " \"txz\", \".tar.zst\", \".tzst\", \"tar.bz2\", \".ar\", or \".deb\""
+ " here."),
@Param(
name = "stripPrefix",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
// Copyright 2022 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.bazel.repository;

import static com.google.common.truth.Truth.assertThat;

import com.google.devtools.build.lib.testutil.TestConstants;
import com.google.devtools.build.lib.testutil.TestUtils;
import com.google.devtools.build.lib.unix.UnixFileSystem;
import com.google.devtools.build.lib.util.OS;
import com.google.devtools.build.lib.vfs.DigestHashFunction;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.JavaIoFileSystem;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.runfiles.Runfiles;
import java.io.File;
import java.io.IOException;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;

/** Tests decompressing archives. */
@RunWith(JUnit4.class)
public class ArFunctionTest {
/*
* .ar archive created with ar cr test_files.ar archived_first.txt archived_second.md
* The files contain short UTF-8 encoded strings.
*/
private static final String ARCHIVE_NAME = "test_files.ar";
private static final String PATH_TO_TEST_ARCHIVE =
"/com/google/devtools/build/lib/bazel/repository/";
private static final String FIRST_FILE_NAME = "archived_first.txt";
private static final String SECOND_FILE_NAME = "archived_second.md";

@Test
public void testDecompress() throws Exception {
Path outputDir = decompress(createDescriptorBuilder());

assertThat(outputDir.exists()).isTrue();
Path firstFile = outputDir.getRelative(FIRST_FILE_NAME);
assertThat(firstFile.exists()).isTrue();
// There are 20 bytes in the content "this is test file 1"
assertThat(firstFile.getFileSize()).isEqualTo(20);
assertThat(firstFile.isSymbolicLink()).isFalse();

Path secondFile = outputDir.getRelative(SECOND_FILE_NAME);
assertThat(secondFile.exists()).isTrue();
// There are 20 bytes in the content "this is the second test file"
assertThat(secondFile.getFileSize()).isEqualTo(29);
assertThat(secondFile.isSymbolicLink()).isFalse();
}

private Path decompress(DecompressorDescriptor.Builder descriptorBuilder) throws Exception {
descriptorBuilder.setDecompressor(ArFunction.INSTANCE);
return new ArFunction().decompress(descriptorBuilder.build());
}

private DecompressorDescriptor.Builder createDescriptorBuilder() throws IOException {
// This was cribbed from TestArchiveDescriptor
FileSystem testFS =
OS.getCurrent() == OS.WINDOWS
? new JavaIoFileSystem(DigestHashFunction.SHA256)
: new UnixFileSystem(DigestHashFunction.SHA256, /*hashAttributeName=*/ "");

// do not rely on TestConstants.JAVATESTS_ROOT end with slash, but ensure separators
// are not duplicated
String path =
(TestConstants.JAVATESTS_ROOT + PATH_TO_TEST_ARCHIVE + ARCHIVE_NAME).replace("//", "/");
Path tarballPath = testFS.getPath(Runfiles.create().rlocation(path));

Path workingDir = testFS.getPath(new File(TestUtils.tmpDir()).getCanonicalPath());
Path outDir = workingDir.getRelative("out");

return DecompressorDescriptor.builder().setRepositoryPath(outDir).setArchivePath(tarballPath);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ java_library(
data = [
"test_decompress_archive.tar.gz",
"test_decompress_archive.zip",
"test_files.ar",
],
deps = [
"//src/main/java/com/google/devtools/build/lib/bazel/repository",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@ public void testKnownFileExtensionsDoNotThrow() throws Exception {
DecompressorDescriptor.builder().setArchivePath(path).build();
path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.tar.bz2");
DecompressorDescriptor.builder().setArchivePath(path).build();
path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.ar");
DecompressorDescriptor.builder().setArchivePath(path).build();
path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.deb");
DecompressorDescriptor.builder().setArchivePath(path).build();
}

@Test
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
!<arch>
// 40 `
archived_first.txt/
archived_second.md/
/0 0 0 0 644 20 `
this is test file 1
/20 0 0 0 644 29 `
this is the second test file

7 changes: 4 additions & 3 deletions tools/build_defs/repo/http.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ match a directory in the archive, Bazel will return an error.""",
By default, the archive type is determined from the file extension of the
URL. If the file has no extension, you can explicitly specify one of the
following: `"zip"`, `"jar"`, `"war"`, `"aar"`, `"tar"`, `"tar.gz"`, `"tgz"`,
`"tar.xz"`, or `tar.bz2`.""",
`"tar.xz"`, `"txz"`, `"tar.zst"`, `"tzst"`, `tar.bz2`, `"ar"`, or `"deb"`.""",
),
"patches": attr.label_list(
default = [],
Expand Down Expand Up @@ -357,8 +357,9 @@ http_archive = repository_rule(
"""Downloads a Bazel repository as a compressed archive file, decompresses it,
and makes its targets available for binding.
It supports the following file extensions: `"zip"`, `"jar"`, `"war"`, `"aar"`,
`"tar"`, `"tar.gz"`, `"tgz"`, `"tar.xz"`, and `tar.bz2`.
It supports the following file extensions: `"zip"`, `"jar"`, `"war"`, `"aar"`, `"tar"`,
`"tar.gz"`, `"tgz"`, `"tar.xz"`, `"txz"`, `"tar.zst"`, `"tzst"`, `tar.bz2`, `"ar"`,
or `"deb"`.
Examples:
Suppose the current repository contains the source code for a chat program,
Expand Down

0 comments on commit 9c98120

Please sign in to comment.