From 6714c30507edc70ec84f8c97d47cffc497356c0b Mon Sep 17 00:00:00 2001 From: Yannic Bonenberger Date: Fri, 15 Jul 2022 02:52:53 -0700 Subject: [PATCH] [credentialhelper] Implement invoking credential helper as subprocess Progress on https://github.com/bazelbuild/proposals/blob/main/designs/2022-06-07-bazel-credential-helpers.md Progress on #15856 Closes #15861. PiperOrigin-RevId: 461159351 Change-Id: I28eb4817ced8db8f095a1f35092fdefba28e0ede --- .../lib/authandtls/credentialhelper/BUILD | 2 + .../credentialhelper/CredentialHelper.java | 116 +++++++++++++- .../CredentialHelperEnvironment.java | 75 +++++++++ .../GetCredentialsResponse.java | 2 +- .../lib/authandtls/credentialhelper/BUILD | 11 ++ .../CredentialHelperTest.java | 143 ++++++++++++++++++ .../test_credential_helper.py | 92 +++++++++++ 7 files changed, 439 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperEnvironment.java create mode 100644 src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperTest.java create mode 100644 src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/test_credential_helper.py diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD index 745e6eaebae2b7..8b11bce467a160 100644 --- a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD +++ b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD @@ -14,6 +14,8 @@ java_library( name = "credentialhelper", srcs = glob(["*.java"]), deps = [ + "//src/main/java/com/google/devtools/build/lib/events", + "//src/main/java/com/google/devtools/build/lib/shell", "//src/main/java/com/google/devtools/build/lib/vfs", "//third_party:auto_value", "//third_party:error_prone_annotations", diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelper.java b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelper.java index c6b60b1fb1a0b4..c82417b0034383 100644 --- a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelper.java +++ b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelper.java @@ -14,14 +14,32 @@ package com.google.devtools.build.lib.authandtls.credentialhelper; +import static java.nio.charset.StandardCharsets.UTF_8; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.io.CharStreams; +import com.google.devtools.build.lib.shell.Subprocess; +import com.google.devtools.build.lib.shell.SubprocessBuilder; import com.google.devtools.build.lib.vfs.Path; import com.google.errorprone.annotations.Immutable; +import com.google.gson.Gson; +import com.google.gson.JsonSyntaxException; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.Reader; +import java.io.Writer; +import java.net.URI; +import java.util.Locale; +import java.util.Objects; /** Wraps an external tool used to obtain credentials. */ @Immutable public final class CredentialHelper { + private static final Gson GSON = new Gson(); + // `Path` is immutable, but not annotated. @SuppressWarnings("Immutable") private final Path path; @@ -35,5 +53,101 @@ Path getPath() { return path; } - // TODO(yannic): Implement running the helper subprocess. + /** + * Fetches credentials for the specified {@link URI} by invoking the credential helper as + * subprocess according to the credential + * helper protocol. + * + * @param environment The environment to run the subprocess in. + * @param uri The {@link URI} to fetch credentials for. + * @return The response from the subprocess. + */ + public GetCredentialsResponse getCredentials(CredentialHelperEnvironment environment, URI uri) + throws InterruptedException, IOException { + Preconditions.checkNotNull(environment); + Preconditions.checkNotNull(uri); + + Subprocess process = spawnSubprocess(environment, "get"); + try (Reader stdout = new InputStreamReader(process.getInputStream(), UTF_8); + Reader stderr = new InputStreamReader(process.getErrorStream(), UTF_8)) { + try (Writer stdin = new OutputStreamWriter(process.getOutputStream(), UTF_8)) { + GSON.toJson(GetCredentialsRequest.newBuilder().setUri(uri).build(), stdin); + } + + process.waitFor(); + if (process.timedout()) { + throw new IOException( + String.format( + Locale.US, + "Failed to get credentials for '%s' from helper '%s': process timed out", + uri, + path)); + } + if (process.exitValue() != 0) { + throw new IOException( + String.format( + Locale.US, + "Failed to get credentials for '%s' from helper '%s': process exited with code %d." + + " stderr: %s", + uri, + path, + process.exitValue(), + CharStreams.toString(stderr))); + } + + try { + GetCredentialsResponse response = GSON.fromJson(stdout, GetCredentialsResponse.class); + if (response == null) { + throw new IOException( + String.format( + Locale.US, + "Failed to get credentials for '%s' from helper '%s': process exited without" + + " output. stderr: %s", + uri, + path, + CharStreams.toString(stderr))); + } + return response; + } catch (JsonSyntaxException e) { + throw new IOException( + String.format( + Locale.US, + "Failed to get credentials for '%s' from helper '%s': error parsing output. stderr:" + + " %s", + uri, + path, + CharStreams.toString(stderr)), + e); + } + } + } + + private Subprocess spawnSubprocess(CredentialHelperEnvironment environment, String... args) + throws IOException { + Preconditions.checkNotNull(environment); + Preconditions.checkNotNull(args); + + return new SubprocessBuilder() + .setArgv(ImmutableList.builder().add(path.getPathString()).add(args).build()) + .setWorkingDirectory(environment.getWorkspacePath().getPathFile()) + .setEnv(environment.getClientEnvironment()) + .setTimeoutMillis(environment.getHelperExecutionTimeout().toMillis()) + .start(); + } + + @Override + public boolean equals(Object o) { + if (o instanceof CredentialHelper) { + CredentialHelper that = (CredentialHelper) o; + return Objects.equals(this.getPath(), that.getPath()); + } + + return false; + } + + @Override + public int hashCode() { + return Objects.hashCode(getPath()); + } } diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperEnvironment.java b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperEnvironment.java new file mode 100644 index 00000000000000..e2ae01c190b3da --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperEnvironment.java @@ -0,0 +1,75 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.authandtls.credentialhelper; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.vfs.Path; +import java.time.Duration; + +/** Environment for running {@link CredentialHelper}s in. */ +@AutoValue +public abstract class CredentialHelperEnvironment { + /** Returns the reporter for reporting events related to {@link CredentialHelper}s. */ + public abstract Reporter getEventReporter(); + + /** + * Returns the (absolute) path to the workspace. + * + *

Used as working directory when invoking the subprocess. + */ + public abstract Path getWorkspacePath(); + + /** + * Returns the environment from the Bazel client. + * + *

Passed as environment variables to the subprocess. + */ + public abstract ImmutableMap getClientEnvironment(); + + /** Returns the execution timeout for the helper subprocess. */ + public abstract Duration getHelperExecutionTimeout(); + + /** Returns a new builder for {@link CredentialHelperEnvironment}. */ + public static CredentialHelperEnvironment.Builder newBuilder() { + return new AutoValue_CredentialHelperEnvironment.Builder(); + } + + /** Builder for {@link CredentialHelperEnvironment}. */ + @AutoValue.Builder + public abstract static class Builder { + /** Sets the reporter for reporting events related to {@link CredentialHelper}s. */ + public abstract Builder setEventReporter(Reporter reporter); + + /** + * Sets the (absolute) path to the workspace to use as working directory when invoking the + * subprocess. + */ + public abstract Builder setWorkspacePath(Path path); + + /** + * Sets the environment from the Bazel client to pass as environment variables to the + * subprocess. + */ + public abstract Builder setClientEnvironment(ImmutableMap environment); + + /** Sets the execution timeout for the helper subprocess. */ + public abstract Builder setHelperExecutionTimeout(Duration timeout); + + /** Returns the newly constructed {@link CredentialHelperEnvironment}. */ + public abstract CredentialHelperEnvironment build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/GetCredentialsResponse.java b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/GetCredentialsResponse.java index 72f25cf4ddf196..6d450e001fb682 100644 --- a/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/GetCredentialsResponse.java +++ b/src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/GetCredentialsResponse.java @@ -50,7 +50,7 @@ public static Builder newBuilder() { /** Builder for {@link GetCredentialsResponse}. */ @AutoValue.Builder public abstract static class Builder { - protected abstract ImmutableMap.Builder> headersBuilder(); + public abstract ImmutableMap.Builder> headersBuilder(); /** Returns the newly constructed {@link GetCredentialsResponse}. */ public abstract GetCredentialsResponse build(); diff --git a/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD b/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD index 95b3ee4483cdc9..024763f7c1dd63 100644 --- a/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD +++ b/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD @@ -17,12 +17,17 @@ filegroup( java_test( name = "credentialhelper", srcs = glob(["*.java"]), + data = [ + ":test_credential_helper", + ], test_class = "com.google.devtools.build.lib.AllTests", runtime_deps = [ "//src/test/java/com/google/devtools/build/lib:test_runner", ], deps = [ "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper", + "//src/main/java/com/google/devtools/build/lib/events", + "//src/main/java/com/google/devtools/build/lib/util:os", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs", @@ -30,5 +35,11 @@ java_test( "//third_party:guava", "//third_party:junit4", "//third_party:truth", + "@bazel_tools//tools/java/runfiles", ], ) + +py_binary( + name = "test_credential_helper", + srcs = ["test_credential_helper.py"], +) diff --git a/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperTest.java b/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperTest.java new file mode 100644 index 00000000000000..4aeb4595b084d0 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelperTest.java @@ -0,0 +1,143 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.authandtls.credentialhelper; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.vfs.DigestHashFunction; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; +import com.google.devtools.build.runfiles.Runfiles; +import java.io.IOException; +import java.net.URI; +import java.time.Duration; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class CredentialHelperTest { + private static final PathFragment TEST_WORKSPACE_PATH = + PathFragment.create(System.getenv("TEST_TMPDIR")); + private static final PathFragment TEST_CREDENTIAL_HELPER_PATH = + PathFragment.create( + "io_bazel/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/test_credential_helper" + + (OS.getCurrent() == OS.WINDOWS ? ".exe" : "")); + + private static final Reporter reporter = new Reporter(new EventBus()); + + private GetCredentialsResponse getCredentialsFromHelper( + String uri, ImmutableMap env) throws Exception { + Preconditions.checkNotNull(uri); + Preconditions.checkNotNull(env); + + FileSystem fs = new InMemoryFileSystem(DigestHashFunction.SHA256); + + CredentialHelper credentialHelper = + new CredentialHelper( + fs.getPath( + Runfiles.create().rlocation(TEST_CREDENTIAL_HELPER_PATH.getSafePathString()))); + return credentialHelper.getCredentials( + CredentialHelperEnvironment.newBuilder() + .setEventReporter(reporter) + .setWorkspacePath(fs.getPath(TEST_WORKSPACE_PATH)) + .setClientEnvironment(env) + .setHelperExecutionTimeout(Duration.ofSeconds(5)) + .build(), + URI.create(uri)); + } + + private GetCredentialsResponse getCredentialsFromHelper(String uri) throws Exception { + Preconditions.checkNotNull(uri); + + return getCredentialsFromHelper(uri, ImmutableMap.of()); + } + + @Test + public void knownUriWithSingleHeader() throws Exception { + GetCredentialsResponse response = getCredentialsFromHelper("https://singleheader.example.com"); + assertThat(response.getHeaders()).containsExactly("header1", ImmutableList.of("value1")); + } + + @Test + public void knownUriWithMultipleHeaders() throws Exception { + GetCredentialsResponse response = + getCredentialsFromHelper("https://multipleheaders.example.com"); + assertThat(response.getHeaders()) + .containsExactly( + "header1", + ImmutableList.of("value1"), + "header2", + ImmutableList.of("value1", "value2"), + "header3", + ImmutableList.of("value1", "value2", "value3")); + } + + @Test + public void unknownUri() { + IOException ioException = + assertThrows( + IOException.class, () -> getCredentialsFromHelper("https://unknown.example.com")); + assertThat(ioException).hasMessageThat().contains("Unknown uri 'https://unknown.example.com'"); + } + + @Test + public void credentialHelperOutputsNothing() throws Exception { + IOException ioException = + assertThrows( + IOException.class, () -> getCredentialsFromHelper("https://printnothing.example.com")); + assertThat(ioException).hasMessageThat().contains("exited without output"); + } + + @Test + public void credentialHelperOutputsExtraFields() throws Exception { + GetCredentialsResponse response = getCredentialsFromHelper("https://extrafields.example.com"); + assertThat(response.getHeaders()).containsExactly("header1", ImmutableList.of("value1")); + } + + @Test + public void helperRunsInWorkspace() throws Exception { + GetCredentialsResponse response = getCredentialsFromHelper("https://cwd.example.com"); + ImmutableMap> headers = response.getHeaders(); + assertThat(PathFragment.create(headers.get("cwd").get(0))).isEqualTo(TEST_WORKSPACE_PATH); + } + + @Test + public void helperGetEnvironment() throws Exception { + GetCredentialsResponse response = + getCredentialsFromHelper( + "https://env.example.com", ImmutableMap.of("FOO", "BAR!", "BAR", "123")); + assertThat(response.getHeaders()) + .containsExactly( + "foo", ImmutableList.of("BAR!"), + "bar", ImmutableList.of("123")); + } + + @Test + public void helperTimeout() throws Exception { + IOException ioException = + assertThrows( + IOException.class, () -> getCredentialsFromHelper("https://timeout.example.com")); + assertThat(ioException).hasMessageThat().contains("process timed out"); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/test_credential_helper.py b/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/test_credential_helper.py new file mode 100644 index 00000000000000..c21fd7b22524e6 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/authandtls/credentialhelper/test_credential_helper.py @@ -0,0 +1,92 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Credential helper for testing.""" + +import json +import os +import sys +import time + + +def eprint(*args, **kargs): + print(*args, file=sys.stderr, **kargs) + + +def main(argv): + if len(argv) != 2: + eprint("Usage: test_credential_helper ") + return 1 + + if argv[1] != "get": + eprint("Unknown command '{}'".format(argv[1])) + return 1 + + request = json.load(sys.stdin) + if request["uri"] == "https://singleheader.example.com": + response = { + "headers": { + "header1": ["value1"], + }, + } + elif request["uri"] == "https://multipleheaders.example.com": + response = { + "headers": { + "header1": ["value1"], + "header2": ["value1", "value2"], + "header3": ["value1", "value2", "value3"], + }, + } + elif request["uri"] == "https://extrafields.example.com": + response = { + "foo": "YES", + "headers": { + "header1": ["value1"], + }, + "umlaut": [ + "ß", + "å", + ], + } + elif request["uri"] == "https://printnothing.example.com": + return 0 + elif request["uri"] == "https://cwd.example.com": + response = { + "headers": { + "cwd": [os.getcwd()], + }, + } + elif request["uri"] == "https://env.example.com": + response = { + "headers": { + "foo": [os.getenv("FOO")], + "bar": [os.getenv("BAR")], + }, + } + elif request["uri"] == "https://timeout.example.com": + # We expect the subprocess to be killed after 5s. + time.sleep(10) + response = { + "headers": { + "header1": ["value1"], + }, + } + else: + eprint("Unknown uri '{}'".format(request["uri"])) + return 1 + json.dump(response, sys.stdout) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv))