Skip to content

Add azure storage endpoint suffix #26432 #26568

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Sep 21, 2017
Merged

Add azure storage endpoint suffix #26432 #26568

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7269c84
Allow specify azure storage endpoint suffix explicitly
liketic Sep 10, 2017
46b5da0
Update documentation for Azure storage endpoint suffix
liketic Sep 10, 2017
61fe372
Make endpoint_suffix to be a simple string setting for Azure plugin
liketic Sep 14, 2017
78ec96b
Do not use secure string for endpoint suffix
liketic Sep 14, 2017
cc6ffc5
Merge master
liketic Sep 15, 2017
c6ce182
Convert string to bytes with encoding
liketic Sep 16, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions docs/plugins/repository-azure.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,15 @@ bin/elasticsearch-keystore add azure.client.default.key

Where `account` is the azure account name and `key` the azure secret key.

You can specify an `endpoint_suffix` for storage account explicitly. Defaults to `core.windows.net`.

[source,sh]
-------------------------------------------------------------------
bin/elasticsearch-keystore add azure.client.default.account
bin/elasticsearch-keystore add azure.client.default.key
bin/elasticsearch-keystore add azure.client.default.endpoint_suffix
-------------------------------------------------------------------

Note that you can also define more than one account:

[source,sh]
Expand Down
8 changes: 6 additions & 2 deletions ...ry-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,14 +102,18 @@ public AzureStorageServiceImpl(Settings settings, Map<String, AzureStorageSettin

void createClient(AzureStorageSettings azureStorageSettings) {
try {
logger.trace("creating new Azure storage client using account [{}], key [{}]",
azureStorageSettings.getAccount(), azureStorageSettings.getKey());
logger.trace("creating new Azure storage client using account [{}], key [{}], endpoint suffix [{}]",
azureStorageSettings.getAccount(), azureStorageSettings.getKey(), azureStorageSettings.getEndpointSuffix());

String storageConnectionString =
"DefaultEndpointsProtocol=https;"
+ "AccountName=" + azureStorageSettings.getAccount() + ";"
+ "AccountKey=" + azureStorageSettings.getKey();

String endpointSuffix = azureStorageSettings.getEndpointSuffix();
if (endpointSuffix != null && !endpointSuffix.isEmpty()) {
storageConnectionString += ";EndpointSuffix=" + endpointSuffix;
}
// Retrieve storage account from connection-string.
CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);

Expand Down
20 changes: 18 additions & 2 deletions ...itory-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,12 @@ public final class AzureStorageSettings {
public static final AffixSetting<SecureString> ACCOUNT_SETTING = Setting.affixKeySetting(PREFIX, "account",
key -> SecureSetting.secureString(key, null));

/**
* Azure endpoint suffix. Default to core.windows.net (CloudStorageAccount.DEFAULT_DNS).
*/
public static final AffixSetting<SecureString> ENDPOINT_SUFFIX_SETTING = Setting.affixKeySetting(PREFIX, "endpoint_suffix",
Copy link
Member

@jasontedor jasontedor Sep 14, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why does this need to be a secure setting?

Copy link
Author

@liketic liketic Sep 14, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks! I think a normal string is OK. Shall I merge with master for review?

key -> SecureSetting.secureString(key, null));

/**
* max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT).
*/
Expand Down Expand Up @@ -85,15 +91,17 @@ public final class AzureStorageSettings {
private final String name;
private final String account;
private final String key;
private final String endpointSuffix;
private final TimeValue timeout;
@Deprecated
private final boolean activeByDefault;
private final int maxRetries;

public AzureStorageSettings(String account, String key, TimeValue timeout, int maxRetries) {
public AzureStorageSettings(String account, String key, String endpointSuffix, TimeValue timeout, int maxRetries) {
this.name = null;
this.account = account;
this.key = key;
this.endpointSuffix = endpointSuffix;
this.timeout = timeout;
this.activeByDefault = false;
this.maxRetries = maxRetries;
Expand All @@ -104,6 +112,7 @@ public AzureStorageSettings(String name, String account, String key, TimeValue t
this.name = name;
this.account = account;
this.key = key;
this.endpointSuffix = null;
this.timeout = timeout;
this.activeByDefault = activeByDefault;
this.maxRetries = maxRetries;
Expand All @@ -118,6 +127,10 @@ public String getKey() {
return key;
}

public String getEndpointSuffix() {
return endpointSuffix;
}

public String getAccount() {
return account;
}
Expand All @@ -141,6 +154,7 @@ public String toString() {
sb.append("name='").append(name).append('\'');
sb.append(", account='").append(account).append('\'');
sb.append(", key='").append(key).append('\'');
sb.append(", endpointSuffix='").append(endpointSuffix).append('\'');
sb.append(", activeByDefault='").append(activeByDefault).append('\'');
sb.append(", timeout=").append(timeout);
sb.append(", maxRetries=").append(maxRetries);
Expand Down Expand Up @@ -185,8 +199,10 @@ public static Map<String, AzureStorageSettings> load(Settings settings) {
/** Parse settings for a single client. */
static AzureStorageSettings getClientSettings(Settings settings, String clientName) {
try (SecureString account = getConfigValue(settings, clientName, ACCOUNT_SETTING);
SecureString key = getConfigValue(settings, clientName, KEY_SETTING)) {
SecureString key = getConfigValue(settings, clientName, KEY_SETTING);
SecureString endpointSuffix = getConfigValue(settings, clientName, ENDPOINT_SUFFIX_SETTING)) {
return new AzureStorageSettings(account.toString(), key.toString(),
endpointSuffix.toString(),
getValue(settings, clientName, TIMEOUT_SETTING),
getValue(settings, clientName, MAX_RETRIES_SETTING));
}
Expand Down
1 change: 1 addition & 0 deletions ...-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ public List<Setting<?>> getSettings() {
AzureStorageService.Storage.STORAGE_ACCOUNTS,
AzureStorageSettings.ACCOUNT_SETTING,
AzureStorageSettings.KEY_SETTING,
AzureStorageSettings.ENDPOINT_SUFFIX_SETTING,
AzureStorageSettings.TIMEOUT_SETTING
);
}
Expand Down
25 changes: 25 additions & 0 deletions ...y-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
import com.microsoft.azure.storage.LocationMode;
import com.microsoft.azure.storage.RetryExponentialRetry;
import com.microsoft.azure.storage.blob.CloudBlobClient;
import com.microsoft.azure.storage.core.Base64;

import org.elasticsearch.common.settings.MockSecureSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
Expand All @@ -38,8 +40,10 @@
import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.DEPRECATED_TIMEOUT_SETTING;
import static org.elasticsearch.repositories.azure.AzureSettingsParserTests.getConcreteSetting;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.isEmptyString;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;

Expand Down Expand Up @@ -82,10 +86,31 @@ public void testReadSecuredSettings() {
secureSettings.setString("azure.client.azure2.key", "mykey2");
secureSettings.setString("azure.client.azure3.account", "myaccount3");
secureSettings.setString("azure.client.azure3.key", "mykey3");
secureSettings.setString("azure.client.azure3.endpoint_suffix", "my_endpoint_suffix");
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();

Map<String, AzureStorageSettings> loadedSettings = AzureStorageSettings.load(settings);
assertThat(loadedSettings.keySet(), containsInAnyOrder("azure1","azure2","azure3","default"));

assertThat(loadedSettings.get("azure1").getEndpointSuffix(), isEmptyString());
assertThat(loadedSettings.get("azure2").getEndpointSuffix(), isEmptyString());
assertThat(loadedSettings.get("azure3").getEndpointSuffix(), equalTo("my_endpoint_suffix"));
}

public void testCreateClientWithEndpointSuffix() {
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString("azure.client.azure1.account", "myaccount1");
secureSettings.setString("azure.client.azure1.key", Base64.encode("mykey1".getBytes()));
secureSettings.setString("azure.client.azure1.endpoint_suffix", "my_endpoint_suffix");
secureSettings.setString("azure.client.azure2.account", "myaccount2");
secureSettings.setString("azure.client.azure2.key", Base64.encode("mykey2".getBytes()));
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
AzureStorageServiceImpl azureStorageService = new AzureStorageServiceImpl(settings, AzureStorageSettings.load(settings));
CloudBlobClient client1 = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY);
assertThat(client1.getEndpoint().toString(), equalTo("https://myaccount1.blob.my_endpoint_suffix"));

CloudBlobClient client2 = azureStorageService.getSelectedClient("azure2", LocationMode.PRIMARY_ONLY);
assertThat(client2.getEndpoint().toString(), equalTo("https://myaccount2.blob.core.windows.net"));
}

public void testGetSelectedClientWithNoPrimaryAndSecondary() {
Expand Down