Adds support for using job-specific policies

README.md

@@ -21,6 +21,24 @@ When registering the approle backend you can set a couple of different parameter
 * many more
 
 This is just a short introduction, please refer to [Hashicorp itself](https://www.vaultproject.io/docs/auth/approle.html) to get detailed information.
+
+### Isolating policies for different jobs
+It may be desirable to have jobs or folders with separate Vault policies allocated. This may be done
+with the optional `policies` configuration option combined with authentication such as the AppRole
+credential. The process is the following:
+* The Jenkins job attempts to retrieve a secret from Vault
+* The AppRole authentication is used to retrieve a new token (if the old one has not expired yet)
+* The Vault plugin then uses the `policies` configuration value with job info to come up with a list of policies
+* If this list is not empty, the AppRole token is used to retrieve a new token that only has the specified policies applied
+* This token is then used for all Vault plugin operations in the job
+
+The policies list may be templatized with values that can come from each job in order to customize
+policies per job or folder. See the `policies` configuration help for more information on available
+tokens to use in the configuration. The `Limit Token Policies` option must also be enabled on the
+auth credential. Please note that the AppRole (or other authentication method) should have all policies
+configured as `token_policies` and not `identity_policies`, as job-specific tokens inherit all
+`identity_policies` automatically.  
+
 ### What about other backends?
 Hashicorp explicitly recommends the AppRole Backend for machine-to-machine authentication. Token based auth is mainly supported for backward compatibility.
 Other backends that might make sense are the AWS EC2 backend, the Azure backend, and the Kubernetes backend. But we do not support these yet. Feel free to contribute!

src/main/java/com/datapipe/jenkins/vault/VaultAccessor.java

@@ -27,6 +27,7 @@
 import java.io.PrintStream;
 import java.io.Serializable;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -42,6 +43,7 @@
 
     private VaultConfig config;
     private VaultCredential credential;
+    private List<String> policies;
     private int maxRetries = 0;
     private int retryIntervalMilliseconds = 1000;
 
@@ -63,7 +65,7 @@
             if (credential == null) {
                 vault = new Vault(config);
             } else {
-                vault = credential.authorizeWithVault(config);
+                vault = credential.authorizeWithVault(config, policies);
             }
 
             vault.withRetries(maxRetries, retryIntervalMilliseconds);
@@ -89,6 +91,14 @@
         this.credential = credential;
     }
 
+    public List<String> getPolicies() {
+        return policies;
+    }
+
+    public void setPolicies(List<String> policies) {
+        this.policies = policies;
+    }
+
     public int getMaxRetries() {
         return maxRetries;
     }
@@ -130,6 +140,38 @@
         }
     }
 
+    public static String replacePolicyTokens(String policy, EnvVars envVars) {
+        if (!policy.contains("{")) {
+            return policy;
+        }
+        String jobName = envVars.get("JOB_NAME");
+        String jobBaseName = envVars.get("JOB_BASE_NAME");
+        String folder = "";
+        if (!jobName.equals(jobBaseName) && jobName.contains("/")) {
+            String[] jobElements = jobName.split("/");
+            folder = Arrays.stream(jobElements)
+                .limit(jobElements.length - 1)
+                .collect(Collectors.joining("/"));
+        }
+        return policy
+            .replaceAll("\\{job_base_name}", jobBaseName)
+            .replaceAll("\\{job_name}", jobName)
+            .replaceAll("\\{job_name_us}", jobName.replaceAll("/", "_"))
+            .replaceAll("\\{job_folder}", folder)
+            .replaceAll("\\{job_folder_us}", folder.replaceAll("/", "_"))
+            .replaceAll("\\{node_name}", envVars.get("NODE_NAME"));
+    }
+
+    public static List<String> generatePolicies(String policies, EnvVars envVars) {
+        if (StringUtils.isBlank(policies)) {
+            return null;
+        }
+        return Arrays.stream(policies.split("\n"))
+            .filter(StringUtils::isNotBlank)
+            .map(policy -> replacePolicyTokens(policy.trim(), envVars))
+            .collect(Collectors.toList());
+    }
+
     public static Map<String, String> retrieveVaultSecrets(Run<?,?> run, PrintStream logger, EnvVars envVars, VaultAccessor vaultAccessor, VaultConfiguration initialConfiguration, List<VaultSecret> vaultSecrets) {
         Map<String, String> overrides = new HashMap<>();
 
@@ -156,6 +198,7 @@
         }
         vaultAccessor.setConfig(vaultConfig);
         vaultAccessor.setCredential(credential);
+        vaultAccessor.setPolicies(generatePolicies(config.getPolicies(), envVars));
         vaultAccessor.setMaxRetries(config.getMaxRetries());
         vaultAccessor.setRetryIntervalMilliseconds(config.getRetryIntervalMilliseconds());
         vaultAccessor.init();

src/main/java/com/datapipe/jenkins/vault/configuration/VaultConfiguration.java

@@ -50,6 +50,8 @@
 
     private String prefixPath;
 
+    private String policies;
+
     private Integer timeout = DEFAULT_TIMEOUT;
 
     @DataBoundConstructor
@@ -73,6 +75,7 @@
         this.engineVersion = toCopy.engineVersion;
         this.vaultNamespace = toCopy.vaultNamespace;
         this.prefixPath = toCopy.prefixPath;
+        this.policies = toCopy.policies;
         this.timeout = toCopy.timeout;
     }
 
@@ -99,6 +102,9 @@
         if (StringUtils.isBlank(result.getPrefixPath())) {
             result.setPrefixPath(parent.getPrefixPath());
         }
+        if (StringUtils.isBlank(result.getPolicies())) {
+            result.setPolicies(parent.getPolicies());
+        }
         if (result.timeout == null) {
             result.setTimeout(parent.getTimeout());
         }
@@ -183,6 +189,15 @@
         this.prefixPath = fixEmptyAndTrim(prefixPath);
     }
 
+    public String getPolicies() {
+        return policies;
+    }
+
+    @DataBoundSetter
+    public void setPolicies(String policies) {
+        this.policies = fixEmptyAndTrim(policies);;
+    }
+
     public Integer getTimeout() {
         return timeout;
     }

src/main/java/com/datapipe/jenkins/vault/credentials/AbstractAuthenticatingVaultTokenCredential.java

@@ -46,7 +46,7 @@ public void setNamespace(String namespace) {
     }
 
     @Override
-    protected final String getToken(Vault vault) {
+    protected Auth getVaultAuth(@NonNull Vault vault) {
         // set authentication namespace if configured for this credential.
         // importantly, this will not effect the underlying VaultConfig namespace.
         Auth auth = vault.auth();
@@ -57,7 +57,12 @@ protected final String getToken(Vault vault) {
                 auth.withNameSpace(null);
             }
         }
-        return getToken(auth);
+        return auth;
+    }
+
+    @Override
+    protected final String getToken(Vault vault) {
+        return getToken(getVaultAuth(vault));
     }
 
     /**

src/main/java/com/datapipe/jenkins/vault/credentials/AbstractVaultTokenCredential.java

@@ -4,6 +4,7 @@
 import com.bettercloud.vault.VaultConfig;
 import com.cloudbees.plugins.credentials.CredentialsScope;
 import com.cloudbees.plugins.credentials.impl.BaseStandardCredentials;
+import java.util.List;
 
 public abstract class AbstractVaultTokenCredential
     extends BaseStandardCredentials implements VaultCredential {
@@ -15,7 +16,7 @@ protected AbstractVaultTokenCredential(CredentialsScope scope, String id, String
     protected abstract String getToken(Vault vault);
 
     @Override
-    public Vault authorizeWithVault(VaultConfig config) {
+    public Vault authorizeWithVault(VaultConfig config, List<String> policies) {
         Vault vault = new Vault(config);
         return new Vault(config.token(getToken(vault)));
     }

src/main/java/com/datapipe/jenkins/vault/credentials/AbstractVaultTokenCredentialWithExpiration.java

@@ -3,36 +3,130 @@
 import com.bettercloud.vault.Vault;
 import com.bettercloud.vault.VaultConfig;
 import com.bettercloud.vault.VaultException;
+import com.bettercloud.vault.api.Auth;
+import com.bettercloud.vault.api.Auth.TokenRequest;
 import com.cloudbees.plugins.credentials.CredentialsScope;
+import com.datapipe.jenkins.vault.exception.VaultPluginException;
+import edu.umd.cs.findbugs.annotations.CheckForNull;
+import edu.umd.cs.findbugs.annotations.NonNull;
 import java.util.Calendar;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import org.kohsuke.stapler.DataBoundSetter;
 
 public abstract class AbstractVaultTokenCredentialWithExpiration
     extends AbstractVaultTokenCredential {
 
-    private final static Logger LOGGER = Logger
+    protected final static Logger LOGGER = Logger
         .getLogger(AbstractVaultTokenCredentialWithExpiration.class.getName());
 
-    private Calendar tokenExpiry;
-    private String currentClientToken;
+    @CheckForNull
+    private Boolean usePolicies;
+
+    /**
+     * Get if the configured policies should be used or not.
+     * @return true if the policies should be used, false or null otherwise
+     */
+    @CheckForNull
+    public Boolean getUsePolicies() {
+        return usePolicies;
+    }
+
+    /**
+     * Set if the configured policies are used or not.
+     * @param usePolicies true if policies should be used, false otherwise
+     */
+    @DataBoundSetter
+    public void setUsePolicies(Boolean usePolicies) {
+        this.usePolicies = usePolicies;
+    }
+
+    private Map<String, Calendar> tokenExpiry;
+    private Map<String, String> tokenCache;
 
     protected AbstractVaultTokenCredentialWithExpiration(CredentialsScope scope, String id,
         String description) {
         super(scope, id, description);
+        tokenExpiry = new HashMap<>();
+        tokenCache = new HashMap<>();
     }
 
     protected abstract String getToken(Vault vault);
 
+    /**
+     * Retrieve the Vault auth client. May be overridden in subclasses.
+     * @param vault the Vault instance
+     * @return the Vault auth client
+     */
+    protected Auth getVaultAuth(@NonNull Vault vault) {
+        return vault.auth();
+    }
+
+    /**
+     * Retrieves a new child token with specific policies if this credential is configured to use
+     * policies and a list of requested policies is provided.
+     * @param vault the vault instance
+     * @param policies the policies list
+     * @return the new token or null if it cannot be provisioned
+     */
+    protected String getChildToken(Vault vault, List<String> policies) {
+        if (usePolicies == null || !usePolicies || policies == null || policies.isEmpty()) {
+            return null;
+        }
+        Auth auth = getVaultAuth(vault);
+        try {
+            String ttl = String.format("%ds", getTokenTTL(vault));
+            TokenRequest tokenRequest = (new TokenRequest())
+                .polices(policies)
+                // Set the TTL to the parent token TTL
+                .ttl(ttl);
+            LOGGER.log(Level.FINE, "Requesting child token with policies {0} and TTL {1}",
+                new Object[] {policies, ttl});
+            return auth.createToken(tokenRequest).getAuthClientToken();
+        } catch (VaultException e) {
+            throw new VaultPluginException("Could not retrieve token with policies from Vault", e);
+        }
+    }
+
+    /**
+     * Retrieves a key to be used for the token cache based on a list of policies.
+     * @param policies the list of policies
+     * @return the key to use for the map, either an empty string or a comma-separated list of policies
+     */
+    private String getCacheKey(List<String> policies) {
+        if (policies == null || policies.isEmpty()) {
+            return "";
+        }
+        return String.join(",", policies);
+    }
+
     @Override
-    public Vault authorizeWithVault(VaultConfig config) {
+    public Vault authorizeWithVault(VaultConfig config, List<String> policies) {
+        // Upgraded instances can have these not initialized in the constructor (serialized jobs possibly)
+        if (tokenCache == null) {
+            tokenCache = new HashMap<>();
+            tokenExpiry = new HashMap<>();
+        }
+
+        String cacheKey = getCacheKey(policies);
         Vault vault = getVault(config);
-        if (tokenExpired()) {
-            currentClientToken = getToken(vault);
-            config.token(currentClientToken);
-            setTokenExpiry(vault);
+        if (tokenExpired(cacheKey)) {
+            tokenCache.put(cacheKey, getToken(vault));
+            config.token(tokenCache.get(cacheKey));
+
+            // After current token is configured, try to retrieve a new child token with limited policies
+            String childToken = getChildToken(vault, policies);
+            if (childToken != null) {
+                // A new token was generated, put it in the cache and configure vault
+                tokenCache.put(cacheKey, childToken);
+                config.token(childToken);
+            }
+            setTokenExpiry(vault, cacheKey);
         } else {
-            config.token(currentClientToken);
+            config.token(tokenCache.get(cacheKey));
         }
         return vault;
     }
@@ -41,33 +135,41 @@
         return new Vault(config);
     }
 
-    private void setTokenExpiry(Vault vault) {
+    private long getTokenTTL(Vault vault) throws VaultException {
+        return getVaultAuth(vault).lookupSelf().getTTL();
+    }
+
+    private void setTokenExpiry(Vault vault, String cacheKey) {
         int tokenTTL = 0;
         try {
-            tokenTTL = (int) vault.auth().lookupSelf().getTTL();
+            tokenTTL = (int) getTokenTTL(vault);
         } catch (VaultException e) {
-            LOGGER.log(Level.WARNING, "Could not determine token expiration. " +
-                "Check if token is allowed to access auth/token/lookup-self. " +
+            LOGGER.log(Level.WARNING, "Could not determine token expiration for policies '" +
+                cacheKey + "'. Check if token is allowed to access auth/token/lookup-self. " +
                 "Assuming token TTL expired.", e);
         }
-        tokenExpiry = Calendar.getInstance();
-        tokenExpiry.add(Calendar.SECOND, tokenTTL);
+        Calendar expiry = Calendar.getInstance();
+        expiry.add(Calendar.SECOND, tokenTTL);
+        tokenExpiry.put(cacheKey, expiry);
     }
 
-    private boolean tokenExpired() {
-        if (tokenExpiry == null) {
+    private boolean tokenExpired(String cacheKey) {
+        Calendar expiry = tokenExpiry.get(cacheKey);
+        if (expiry == null) {
             return true;
         }
 
         boolean result = true;
         Calendar now = Calendar.getInstance();
-        long timeDiffInMillis = now.getTimeInMillis() - tokenExpiry.getTimeInMillis();
+        long timeDiffInMillis = now.getTimeInMillis() - expiry.getTimeInMillis();
+        LOGGER.log(Level.FINE, "Expiration for " + cacheKey + " is " + expiry + ", diff: " + timeDiffInMillis);
         if (timeDiffInMillis < -10000L) {
             // token will be valid for at least another 10s
             result = false;
-            LOGGER.log(Level.FINE, "Auth token is still valid");
+            LOGGER.log(Level.FINE, "Auth token is still valid for policies '" + cacheKey + "'");
         } else {
-            LOGGER.log(Level.FINE, "Auth token has to be re-issued" + timeDiffInMillis);
+            LOGGER.log(Level.FINE,"Auth token has to be re-issued for policies '" + cacheKey +
+                    "' (" + timeDiffInMillis + "ms difference)");
         }
 
         return result;