Skip to content

Commit

Permalink
Credential helpers (#729)
Browse files Browse the repository at this point in the history
See also #647 for previous discussions.
  • Loading branch information
TheIndifferent authored and rnorth committed Jun 16, 2018
1 parent 6a83e85 commit 92266b2
Show file tree
Hide file tree
Showing 16 changed files with 375 additions and 3 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ node_modules/

.gradle/
build/
out/
*.class

# Eclipse IDE files
**/.project
Expand Down
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,10 @@ All notable changes to this project will be documented in this file.

## [1.7.2] - 2018-04-30

- Add support for private repositories using docker credential stores/helpers (fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567))

### Fixed
- Add support for private repositories using docker credential stores/helpers (fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567))
- Retry any exceptions (not just `DockerClientException`) on image pull ([\#662](https://github.com/testcontainers/testcontainers-java/issues/662))
- Fixed handling of the paths with `+` in them ([\#664](https://github.com/testcontainers/testcontainers-java/issues/664))

Expand All @@ -57,6 +60,7 @@ All notable changes to this project will be documented in this file.
- Fixed `HostPortWaitStrategy` throws `NumberFormatException` when port is exposed but not mapped ([\#640](https://github.com/testcontainers/testcontainers-java/issues/640))
- Fixed log processing: multibyte unicode, linebreaks and ASCII color codes. Color codes can be turned on with `withRemoveAnsiCodes(false)` ([\#643](https://github.com/testcontainers/testcontainers-java/pull/643))
- Fixed Docker host IP detection within docker container (detect only if not explicitly set) ([\#648](https://github.com/testcontainers/testcontainers-java/pull/648))
- Add support for private repositories using docker credential stores/helpers ([PR \#647](https://github.com/testcontainers/testcontainers-java/pull/647), fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567))

### Changed
- Support multiple HTTP status codes for HttpWaitStrategy ([\#630](https://github.com/testcontainers/testcontainers-java/issues/630))
Expand Down
8 changes: 8 additions & 0 deletions circle.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ jobs:
when: always
- store_test_results:
path: ~/junit
- store_artifacts:
path: ~/junit
okhttp:
steps:
- checkout
Expand Down Expand Up @@ -45,6 +47,8 @@ jobs:
when: always
- store_test_results:
path: ~/junit
- store_artifacts:
path: ~/junit
modules-jdbc-test:
steps:
- checkout
Expand All @@ -61,6 +65,8 @@ jobs:
when: always
- store_test_results:
path: ~/junit
- store_artifacts:
path: ~/junit
selenium:
steps:
- checkout
Expand All @@ -74,6 +80,8 @@ jobs:
when: always
- store_test_results:
path: ~/junit
- store_artifacts:
path: ~/junit

workflows:
version: 2
Expand Down
2 changes: 1 addition & 1 deletion core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ dependencies {
shaded 'com.squareup.okhttp3:okhttp:3.10.0'

shaded 'javax.ws.rs:javax.ws.rs-api:2.0.1'
shaded 'org.zeroturnaround:zt-exec:1.8'
shaded 'org.zeroturnaround:zt-exec:1.10'
shaded 'commons-lang:commons-lang:2.6'
shaded 'commons-io:commons-io:2.5'
shaded 'commons-codec:commons-codec:1.11'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.ListImagesCmd;
import com.github.dockerjava.api.exception.DockerClientException;
import com.github.dockerjava.api.model.AuthConfig;
import com.github.dockerjava.api.model.Image;
import com.github.dockerjava.core.command.PullImageResultCallback;
import lombok.NonNull;
Expand All @@ -14,6 +15,7 @@
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.DockerLoggerFactory;
import org.testcontainers.utility.LazyFuture;
import org.testcontainers.utility.RegistryAuthLocator;

import java.util.HashSet;
import java.util.List;
Expand Down Expand Up @@ -93,10 +95,14 @@ protected final String resolve() {

// The image is not available locally - pull it
try {
final RegistryAuthLocator authLocator = new RegistryAuthLocator(dockerClient.authConfig());
final AuthConfig effectiveAuthConfig = authLocator.lookupAuthConfig(imageName);

final PullImageResultCallback callback = new PullImageResultCallback();
dockerClient
.pullImageCmd(imageName.getUnversionedPart())
.withTag(imageName.getVersionPart())
.withAuthConfig(effectiveAuthConfig)
.exec(callback);
callback.awaitCompletion();
AVAILABLE_IMAGE_NAME_CACHE.add(imageName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,11 @@ public String getVersionPart() {

@Override
public String toString() {
return getUnversionedPart() + versioning.getSeparator() + versioning.toString();
if (versioning == null) {
return getUnversionedPart();
} else {
return getUnversionedPart() + versioning.getSeparator() + versioning.toString();
}
}

/**
Expand All @@ -116,6 +120,10 @@ public void assertValid() {
}
}

public String getRegistry() {
return registry;
}

private interface Versioning {
boolean isValid();
String getSeparator();
Expand Down
184 changes: 184 additions & 0 deletions core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
package org.testcontainers.utility;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.dockerjava.api.model.AuthConfig;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang.SystemUtils;
import org.slf4j.Logger;
import org.zeroturnaround.exec.ProcessExecutor;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.TimeUnit;

import static org.slf4j.LoggerFactory.getLogger;

/**
* Utility to look up registry authentication information for an image.
*/
public class RegistryAuthLocator {

private static final Logger log = getLogger(RegistryAuthLocator.class);

private final AuthConfig defaultAuthConfig;
private final File configFile;
private final String commandPathPrefix;
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();

@VisibleForTesting
RegistryAuthLocator(AuthConfig defaultAuthConfig, File configFile, String commandPathPrefix) {
this.defaultAuthConfig = defaultAuthConfig;
this.configFile = configFile;
this.commandPathPrefix = commandPathPrefix;
}

/**
* @param defaultAuthConfig an AuthConfig object that should be returned if there is no overriding authentication
* available for images that are looked up
*/
public RegistryAuthLocator(AuthConfig defaultAuthConfig) {
this.defaultAuthConfig = defaultAuthConfig;
final String dockerConfigLocation = System.getenv().getOrDefault("DOCKER_CONFIG",
System.getProperty("user.home") + "/.docker");
this.configFile = new File(dockerConfigLocation + "/config.json");
this.commandPathPrefix = "";
}

/**
* Looks up an AuthConfig for a given image name.
* <p>
* Lookup is performed in following order:
* <ol>
* <li>{@code auths} is checked for existing credentials for the specified registry.</li>
* <li>if no existing auth is found, {@code credHelpers} are checked for helper for the specified registry.</li>
* <li>if no suitable {@code credHelpers} found, {@code credsStore} is used.</li>
* <li>if no {@code credsStore} is found then the default configuration is returned.</li>
* </ol>
*
* @param dockerImageName image name to be looked up (potentially including a registry URL part)
* @return an AuthConfig that is applicable to this specific image OR the defaultAuthConfig that has been set for
* this {@link RegistryAuthLocator}.
*/
public AuthConfig lookupAuthConfig(DockerImageName dockerImageName) {

if (SystemUtils.IS_OS_WINDOWS) {
log.debug("RegistryAuthLocator is not supported on Windows. Please help test or improve it and update " +
"https://github.com/testcontainers/testcontainers-java/issues/756");
return defaultAuthConfig;
}

log.debug("Looking up auth config for image: {}", dockerImageName);

log.debug("RegistryAuthLocator has configFile: {} ({}) and commandPathPrefix: {}",
configFile,
configFile.exists() ? "exists" : "does not exist",
commandPathPrefix);

try {
final JsonNode config = OBJECT_MAPPER.readTree(configFile);
final String reposName = dockerImageName.getRegistry();

final AuthConfig existingAuthConfig = findExistingAuthConfig(config, reposName);
if (existingAuthConfig != null) {
return existingAuthConfig;
}
// auths is empty, using helper:
final AuthConfig helperAuthConfig = authConfigUsingHelper(config, reposName);
if (helperAuthConfig != null) {
return helperAuthConfig;
}
// no credsHelper to use, using credsStore:
final AuthConfig storeAuthConfig = authConfigUsingStore(config, reposName);
if (storeAuthConfig != null) {
return storeAuthConfig;
}
// otherwise, defaultAuthConfig should already contain any credentials available
} catch (Exception e) {
log.error("Failure when attempting to lookup auth config (dockerImageName: {}, configFile: {}. " +
"Falling back to docker-java default behaviour",
dockerImageName,
configFile,
e);
}
return defaultAuthConfig;
}

private AuthConfig findExistingAuthConfig(final JsonNode config, final String reposName) throws Exception {
final Map.Entry<String, JsonNode> entry = findAuthNode(config, reposName);
if (entry != null && entry.getValue() != null && entry.getValue().size() > 0) {
return OBJECT_MAPPER
.treeToValue(entry.getValue(), AuthConfig.class)
.withRegistryAddress(entry.getKey());
}
return null;
}

private AuthConfig authConfigUsingHelper(final JsonNode config, final String reposName) throws Exception {
final JsonNode credHelpers = config.get("credHelpers");
if (credHelpers != null && credHelpers.size() > 0) {
final JsonNode helperNode = credHelpers.get(reposName);
if (helperNode != null && helperNode.isTextual()) {
final String helper = helperNode.asText();
return runCredentialProvider(reposName, helper);
}
}
return null;
}

private AuthConfig authConfigUsingStore(final JsonNode config, final String reposName) throws Exception {
final JsonNode credsStoreNode = config.get("credsStore");
if (credsStoreNode != null && !credsStoreNode.isMissingNode() && credsStoreNode.isTextual()) {
final String credsStore = credsStoreNode.asText();
return runCredentialProvider(reposName, credsStore);
}
return null;
}

private Map.Entry<String, JsonNode> findAuthNode(final JsonNode config, final String reposName) throws Exception {
final JsonNode auths = config.get("auths");
if (auths != null && auths.size() > 0) {
final Iterator<Map.Entry<String, JsonNode>> fields = auths.fields();
while (fields.hasNext()) {
final Map.Entry<String, JsonNode> entry = fields.next();
if (entry.getKey().endsWith("://" + reposName)) {
return entry;
}
}
}
return null;
}

private AuthConfig runCredentialProvider(String hostName, String credHelper) throws Exception {
final String credentialHelperName = commandPathPrefix + "docker-credential-" + credHelper;
String data;

log.debug("Executing docker credential helper: {} to locate auth config for: {}",
credentialHelperName, hostName);

try {
data = new ProcessExecutor()
.command(credentialHelperName, "get")
.redirectInput(new ByteArrayInputStream(hostName.getBytes()))
.readOutput(true)
.exitValueNormal()
.timeout(30, TimeUnit.SECONDS)
.execute()
.outputUTF8()
.trim();
} catch (Exception e) {
log.error("Failure running docker credential helper ({})", credentialHelperName);
throw e;
}

final JsonNode helperResponse = OBJECT_MAPPER.readTree(data);
log.debug("Credential helper provided auth config for: {}", hostName);

return new AuthConfig()
.withRegistryAddress(helperResponse.at("/ServerURL").asText())
.withUsername(helperResponse.at("/Username").asText())
.withPassword(helperResponse.at("/Secret").asText());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ public static String[] parameters() {
"gliderlabs/alpine@sha256:a19aa4a17a525c97e5a90a0c53a9f3329d2dc61b0a14df5447757a865671c085",
"quay.io/testcontainers/ryuk:latest",
"quay.io/testcontainers/ryuk:0.2.2",
"quay.io/testcontainers/ryuk@sha256:4b606e54c4bba1af4fd814019d342e4664d51e28d3ba2d18d24406edbefd66da"
"quay.io/testcontainers/ryuk@sha256:4b606e54c4bba1af4fd814019d342e4664d51e28d3ba2d18d24406edbefd66da",
};
}

Expand Down
Loading

0 comments on commit 92266b2

Please sign in to comment.