Add SAS Token Authentication Support to Azure Repo Plugin (#43841)

* Add SAS Token Authentication Support to Azure Repo Plugin (#42982) (#43618)

* Added setting for SAS token
* Added support for the token in tests
* Relates #42117

Author: original-brownbear

docs/plugins/repository-azure.asciidoc

@@ -19,7 +19,11 @@ bin/elasticsearch-keystore add azure.client.default.account
 bin/elasticsearch-keystore add azure.client.default.key
 ----------------------------------------------------------------
 
-Where `account` is the azure account name and `key` the azure secret key.
+Where `account` is the azure account name and `key` the azure secret key. Instead of an azure secret key under `key`, you can alternatively
+define a shared access signatures (SAS) token under `sas_token` to use for authentication instead. When using an SAS token instead of an
+account key, the SAS token must have read (r), write (w), list (l), and delete (d) permissions for the repository base path and
+all its contents. These permissions need to be granted for the blob service (b) and apply to resource types service (s), container (c), and
+object (o).
 These settings are used by the repository's internal azure client.
 
 Note that you can also define more than one account:
@@ -29,14 +33,14 @@ Note that you can also define more than one account:
 bin/elasticsearch-keystore add azure.client.default.account
 bin/elasticsearch-keystore add azure.client.default.key
 bin/elasticsearch-keystore add azure.client.secondary.account
-bin/elasticsearch-keystore add azure.client.secondary.key
+bin/elasticsearch-keystore add azure.client.secondary.sas_token
 ----------------------------------------------------------------
 
 `default` is the default account name which will be used by a repository,
 unless you set an explicit one in the
 <<repository-azure-repository-settings, repository settings>>.
 
-Both `account` and `key` storage settings are
+The `account`, `key`, and `sas_token` storage settings are
 {ref}/secure-settings.html#reloadable-secure-settings[reloadable]. After you
 reload the settings, the internal azure clients, which are used to transfer the
 snapshot, will utilize the latest settings from the keystore.

plugins/repository-azure/qa/microsoft-azure-storage/build.gradle

@@ -33,12 +33,14 @@ String azureAccount = System.getenv("azure_storage_account")
 String azureKey = System.getenv("azure_storage_key")
 String azureContainer = System.getenv("azure_storage_container")
 String azureBasePath = System.getenv("azure_storage_base_path")
+String azureSasToken = System.getenv("azure_storage_sas_token")
 
-if (!azureAccount && !azureKey && !azureContainer && !azureBasePath) {
+if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) {
     azureAccount = 'azure_integration_test_account'
     azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64
     azureContainer = 'container_test'
     azureBasePath = 'integration_test'
+    azureSasToken = ''
     useFixture = true
 }
 
@@ -62,8 +64,15 @@ processTestResources {
 
 integTestCluster {
     keystoreSetting 'azure.client.integration_test.account', azureAccount
-    keystoreSetting 'azure.client.integration_test.key', azureKey
 
+    if (azureKey != null && azureKey.isEmpty() == false) {
+        println "Using access key in external service tests."
+        keystoreSetting 'azure.client.integration_test.key', azureKey
+    }
+    if (azureSasToken != null && azureSasToken.isEmpty() == false) {
+        println "Using SAS token in external service tests."
+        keystoreSetting 'azure.client.integration_test.sas_token', azureSasToken
+    }
     if (useFixture) {
         dependsOn azureStorageFixture
         // Use a closure on the string to delay evaluation until tests are executed. The endpoint_suffix is used

plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java

@@ -59,6 +59,7 @@ public List<Setting<?>> getSettings() {
             AzureStorageSettings.Storage.STORAGE_ACCOUNTS,
             AzureStorageSettings.ACCOUNT_SETTING,
             AzureStorageSettings.KEY_SETTING,
+            AzureStorageSettings.SAS_TOKEN_SETTING,
             AzureStorageSettings.ENDPOINT_SUFFIX_SETTING,
             AzureStorageSettings.TIMEOUT_SETTING,
             AzureStorageSettings.MAX_RETRIES_SETTING,

plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java

@@ -112,8 +112,8 @@ protected CloudBlobClient buildClient(AzureStorageSettings azureStorageSettings)
         return client;
     }
 
-    protected CloudBlobClient createClient(AzureStorageSettings azureStorageSettings) throws InvalidKeyException, URISyntaxException {
-        final String connectionString = azureStorageSettings.buildConnectionString();
+    private static CloudBlobClient createClient(AzureStorageSettings azureStorageSettings) throws InvalidKeyException, URISyntaxException {
+        final String connectionString = azureStorageSettings.getConnectString();
         return CloudStorageAccount.parse(connectionString).createCloudBlobClient();
     }
 

plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java

@@ -21,6 +21,7 @@
 
 import com.microsoft.azure.storage.LocationMode;
 import com.microsoft.azure.storage.RetryPolicy;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.collect.Tuple;
@@ -57,6 +58,10 @@ public final class AzureStorageSettings {
     public static final AffixSetting<SecureString> KEY_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "key",
         key -> SecureSetting.secureString(key, null));
 
+    /** Azure SAS token */
+    public static final AffixSetting<SecureString> SAS_TOKEN_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "sas_token",
+        key -> SecureSetting.secureString(key, null));
+
     /** max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT). */
     public static final Setting<Integer> MAX_RETRIES_SETTING =
         Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "max_retries",
@@ -118,7 +123,7 @@ public interface Storage {
     @Deprecated
     private final String name;
     private final String account;
-    private final String key;
+    private final String connectString;
     private final String endpointSuffix;
     private final TimeValue timeout;
     @Deprecated
@@ -128,11 +133,11 @@ public interface Storage {
     private final LocationMode locationMode;
 
     // copy-constructor
-    private AzureStorageSettings(String name, String account, String key, String endpointSuffix, TimeValue timeout, boolean activeByDefault,
-            int maxRetries, Proxy proxy, LocationMode locationMode) {
+    private AzureStorageSettings(String name, String account, String connectString, String endpointSuffix, TimeValue timeout,
+                                 boolean activeByDefault, int maxRetries, Proxy proxy, LocationMode locationMode) {
         this.name = name;
         this.account = account;
-        this.key = key;
+        this.connectString = connectString;
         this.endpointSuffix = endpointSuffix;
         this.timeout = timeout;
         this.activeByDefault = activeByDefault;
@@ -145,7 +150,7 @@ private AzureStorageSettings(String name, String account, String key, String end
     public AzureStorageSettings(String name, String account, String key, TimeValue timeout, boolean activeByDefault, int maxRetries) {
         this.name = name;
         this.account = account;
-        this.key = key;
+        this.connectString = buildConnectString(account, key, null, null);
         this.endpointSuffix = null;
         this.timeout = timeout;
         this.activeByDefault = activeByDefault;
@@ -154,11 +159,11 @@ public AzureStorageSettings(String name, String account, String key, TimeValue t
         this.locationMode = LocationMode.PRIMARY_ONLY;
     }
 
-    AzureStorageSettings(String account, String key, String endpointSuffix, TimeValue timeout, int maxRetries,
-                                Proxy.Type proxyType, String proxyHost, Integer proxyPort) {
+    private AzureStorageSettings(String account, String key, String sasToken, String endpointSuffix, TimeValue timeout, int maxRetries,
+                                 Proxy.Type proxyType, String proxyHost, Integer proxyPort) {
         this.name = null;
         this.account = account;
-        this.key = key;
+        this.connectString = buildConnectString(account, key, sasToken, endpointSuffix);
         this.endpointSuffix = endpointSuffix;
         this.timeout = timeout;
         this.activeByDefault = false;
@@ -189,10 +194,6 @@ public String getName() {
         return name;
     }
 
-    public String getKey() {
-        return key;
-    }
-
     public String getAccount() {
         return account;
     }
@@ -218,13 +219,26 @@ public Proxy getProxy() {
         return proxy;
     }
 
-    public String buildConnectionString() {
+    public String getConnectString() {
+        return connectString;
+    }
+
+    private static String buildConnectString(String account, @Nullable String key, @Nullable String sasToken, String endpointSuffix) {
+        final boolean hasSasToken = Strings.hasText(sasToken);
+        final boolean hasKey = Strings.hasText(key);
+        if (hasSasToken == false && hasKey == false) {
+            throw new SettingsException("Neither a secret key nor a shared access token was set.");
+        }
+        if (hasSasToken && hasKey) {
+            throw new SettingsException("Both a secret as well as a shared access token were set.");
+        }
         final StringBuilder connectionStringBuilder = new StringBuilder();
-        connectionStringBuilder.append("DefaultEndpointsProtocol=https")
-                .append(";AccountName=")
-                .append(account)
-                .append(";AccountKey=")
-                .append(key);
+        connectionStringBuilder.append("DefaultEndpointsProtocol=https").append(";AccountName=").append(account);
+        if (hasKey) {
+            connectionStringBuilder.append(";AccountKey=").append(key);
+        } else {
+            connectionStringBuilder.append(";SharedAccessSignature=").append(sasToken);
+        }
         if (Strings.hasText(endpointSuffix)) {
             connectionStringBuilder.append(";EndpointSuffix=").append(endpointSuffix);
         }
@@ -239,7 +253,6 @@ public LocationMode getLocationMode() {
     public String toString() {
         final StringBuilder sb = new StringBuilder("AzureStorageSettings{");
         sb.append("account='").append(account).append('\'');
-        sb.append(", key='").append(key).append('\'');
         sb.append(", activeByDefault='").append(activeByDefault).append('\'');
         sb.append(", timeout=").append(timeout);
         sb.append(", endpointSuffix='").append(endpointSuffix).append('\'');
@@ -309,8 +322,9 @@ static Map<String, AzureStorageSettings> loadRegular(Settings settings) {
     /** Parse settings for a single client. */
     static AzureStorageSettings getClientSettings(Settings settings, String clientName) {
         try (SecureString account = getConfigValue(settings, clientName, ACCOUNT_SETTING);
-             SecureString key = getConfigValue(settings, clientName, KEY_SETTING)) {
-            return new AzureStorageSettings(account.toString(), key.toString(),
+             SecureString key = getConfigValue(settings, clientName, KEY_SETTING);
+             SecureString sasToken = getConfigValue(settings, clientName, SAS_TOKEN_SETTING)) {
+            return new AzureStorageSettings(account.toString(), key.toString(), sasToken.toString(),
                 getValue(settings, clientName, ENDPOINT_SUFFIX_SETTING),
                 getValue(settings, clientName, TIMEOUT_SETTING),
                 getValue(settings, clientName, MAX_RETRIES_SETTING),
@@ -358,8 +372,8 @@ private static AzureStorageSettings getPrimary(List<AzureStorageSettings> settin
         } else if (settings.size() == 1) {
             // the only storage settings belong (implicitly) to the default primary storage
             AzureStorageSettings storage = settings.get(0);
-            return new AzureStorageSettings(storage.getName(), storage.getAccount(), storage.getKey(), storage.getTimeout(), true,
-                storage.getMaxRetries());
+            return new AzureStorageSettings(storage.getName(), storage.getAccount(), storage.connectString, null, storage.getTimeout(),
+                true, storage.getMaxRetries(), null, LocationMode.PRIMARY_ONLY);
         } else {
             AzureStorageSettings primary = null;
             for (AzureStorageSettings setting : settings) {
@@ -398,8 +412,8 @@ public static Map<String, AzureStorageSettings> overrideLocationMode(Map<String,
         final MapBuilder<String, AzureStorageSettings> mapBuilder = new MapBuilder<>();
         for (final Map.Entry<String, AzureStorageSettings> entry : clientsSettings.entrySet()) {
             final AzureStorageSettings azureSettings = new AzureStorageSettings(entry.getValue().name, entry.getValue().account,
-                    entry.getValue().key, entry.getValue().endpointSuffix, entry.getValue().timeout, entry.getValue().activeByDefault,
-                    entry.getValue().maxRetries, entry.getValue().proxy, locationMode);
+                entry.getValue().connectString, entry.getValue().endpointSuffix, entry.getValue().timeout,
+                entry.getValue().activeByDefault, entry.getValue().maxRetries, entry.getValue().proxy, locationMode);
             mapBuilder.put(entry.getKey(), azureSettings);
         }
         return mapBuilder.immutableMap();

plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTests.java

@@ -49,11 +49,9 @@ public void testParseTwoSettingsExplicitDefault() {
         Tuple<AzureStorageSettings, Map<String, AzureStorageSettings>> tuple = AzureStorageSettings.loadLegacy(settings);
         assertThat(tuple.v1(), notNullValue());
         assertThat(tuple.v1().getAccount(), is("myaccount1"));
-        assertThat(tuple.v1().getKey(), is("mykey1"));
         assertThat(tuple.v2().keySet(), hasSize(1));
         assertThat(tuple.v2().get("azure2"), notNullValue());
         assertThat(tuple.v2().get("azure2").getAccount(), is("myaccount2"));
-        assertThat(tuple.v2().get("azure2").getKey(), is("mykey2"));
         assertSettingDeprecationsAndWarnings(new Setting<?>[]{
             getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure1"),
             getConcreteSetting(DEPRECATED_KEY_SETTING, "azure1"),
@@ -72,7 +70,6 @@ public void testParseUniqueSettings() {
         Tuple<AzureStorageSettings, Map<String, AzureStorageSettings>> tuple = AzureStorageSettings.loadLegacy(settings);
         assertThat(tuple.v1(), notNullValue());
         assertThat(tuple.v1().getAccount(), is("myaccount1"));
-        assertThat(tuple.v1().getKey(), is("mykey1"));
         assertThat(tuple.v2().keySet(), hasSize(0));
         assertSettingDeprecationsAndWarnings(new Setting<?>[]{
             getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure1"),

plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java

@@ -176,15 +176,21 @@ public void testReinitClientWrongSettings() throws IOException {
         secureSettings2.setString("azure.client.azure1.account", "myaccount1");
         // missing key
         final Settings settings2 = Settings.builder().setSecureSettings(secureSettings2).build();
+        final MockSecureSettings secureSettings3 = new MockSecureSettings();
+        secureSettings3.setString("azure.client.azure1.account", "myaccount3");
+        secureSettings3.setString("azure.client.azure1.key", encodeKey("mykey33"));
+        secureSettings3.setString("azure.client.azure1.sas_token", encodeKey("mysasToken33"));
+        final Settings settings3 = Settings.builder().setSecureSettings(secureSettings3).build();
         try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings1)) {
             final AzureStorageService azureStorageService = plugin.azureStoreService;
             final CloudBlobClient client11 = azureStorageService.client("azure1").v1();
             assertThat(client11.getEndpoint().toString(), equalTo("https://myaccount1.blob.core.windows.net"));
-            plugin.reload(settings2);
+            final SettingsException e1 = expectThrows(SettingsException.class, () -> plugin.reload(settings2));
+            assertThat(e1.getMessage(), is("Neither a secret key nor a shared access token was set."));
+            final SettingsException e2 = expectThrows(SettingsException.class, () -> plugin.reload(settings3));
+            assertThat(e2.getMessage(), is("Both a secret as well as a shared access token were set."));
             // existing client untouched
             assertThat(client11.getEndpoint().toString(), equalTo("https://myaccount1.blob.core.windows.net"));
-            final SettingsException e = expectThrows(SettingsException.class, () -> azureStorageService.client("azure1"));
-            assertThat(e.getMessage(), is("Invalid azure client settings with name [azure1]"));
         }
     }