Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Misc aks followup #507

Merged
merged 6 commits into from
Oct 16, 2022
Merged

Misc aks followup #507

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f4a9c5d
Remove storage mounts for TES, only needed for cromwell
jsaun Oct 6, 2022
64baf75
No need to validate helm for manual deployment
jsaun Oct 6, 2022
ae98843
Default ProvisionPostgreSqlOnAzure to true is UseAKS is true
jsaun Oct 6, 2022
adb7c27
Upload values file to storage account
jsaun Oct 8, 2022
12b41f5
Use aksValues.yaml to store settings.
jsaun Oct 11, 2022
146be65
Merge branch 'develop' into jsaun/misc-aks-followup
jsaun Oct 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 31 additions & 29 deletions src/deploy-cromwell-on-azure/Deployer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ public async Task<int> DeployAsync()
storageAccount = storageAccounts.First();
}

accountNames = Utility.DelimitedTextToDictionary(await DownloadTextFromStorageAccountAsync(storageAccount, ConfigurationContainerName, PersonalizedSettingsFileName, cts), SettingsDelimiter);
accountNames = await kubernetesManager.GetAKSSettings(storageAccount);
}
else
{
Expand Down Expand Up @@ -308,7 +308,8 @@ public async Task<int> DeployAsync()
{
if (accountNames.TryGetValue("CrossSubscriptionAKSDeployment", out var crossSubscriptionAKSDeployment))
{
configuration.CrossSubscriptionAKSDeployment = bool.Parse(crossSubscriptionAKSDeployment);
bool.TryParse(crossSubscriptionAKSDeployment, out var parsed);
configuration.CrossSubscriptionAKSDeployment = parsed;
}

if (accountNames.TryGetValue("KeyVaultName", out var keyVaultName))
Expand All @@ -326,18 +327,17 @@ public async Task<int> DeployAsync()
?? throw new ValidationException($"Managed Identity {managedIdentityClientId} does not exist in region {configuration.RegionName} or is not accessible to the current user.");

// Override any configuration that is used by the update.
var systemSettings = Utility.DelimitedTextToDictionary(await DownloadTextFromStorageAccountAsync(storageAccount, ConfigurationContainerName, NonpersonalizedSettingsFileName, cts), SettingsDelimiter);
var versionString = systemSettings["CromwellOnAzureVersion"];
var aksValues = await kubernetesManager.GetAKSSettings(storageAccount);
var versionString = aksValues["CromwellOnAzureVersion"];
var installedVersion = !string.IsNullOrEmpty(versionString) && Version.TryParse(versionString, out var version) ? version : null;
var settings = ConfigureSettings(managedIdentity, aksValues, installedVersion);

await kubernetesManager.UpgradeAKSDeployment(
GetPersonalizedSettings(managedIdentity, accountNames, installedVersion).Union(GetSystemSettings(systemSettings)).ToDictionary(kv => kv.Key, kv => kv.Value),
settings,
resourceGroup,
storageAccount,
managedIdentity,
keyVaultUri);

await UploadTextToStorageAccountAsync(storageAccount, ConfigurationContainerName, NonpersonalizedSettingsFileName, Utility.DictionaryToDelimitedText(systemSettings, SettingsDelimiter));
}
else
{
Expand Down Expand Up @@ -506,15 +506,14 @@ await Task.Run(async () =>
ConsoleEx.WriteLine();
compute = Task.Run(async () =>
{
var personalizedSettings = GetPersonalizedSettings(managedIdentity, default, default);
var systemSettings = GetSystemSettings(default);
var settings = ConfigureSettings(managedIdentity, default, default);

if (aksCluster == null && !configuration.ManualHelmDeployment)
{
await ProvisionManagedCluster(resourceGroup, managedIdentity, logAnalyticsWorkspace, vnetAndSubnet?.virtualNetwork, vnetAndSubnet?.vmSubnet.Name, configuration.PrivateNetworking.GetValueOrDefault());
}

await kubernetesManager.UpdateHelmValuesAsync(storageAccount.Name, keyVaultUri, resourceGroup.Name, personalizedSettings.Union(systemSettings).ToDictionary(kv => kv.Key, kv => kv.Value), managedIdentity);
await kubernetesManager.UpdateHelmValuesAsync(storageAccount, keyVaultUri, resourceGroup.Name, settings, managedIdentity);

if (configuration.ManualHelmDeployment)
{
Expand All @@ -528,9 +527,6 @@ await Task.Run(async () =>
await kubernetesManager.DeployCoADependencies();
await kubernetesManager.DeployHelmChartToClusterAsync();
}

await UploadTextToStorageAccountAsync(storageAccount, ConfigurationContainerName, PersonalizedSettingsFileName, Utility.DictionaryToDelimitedText(personalizedSettings, SettingsDelimiter));
await UploadTextToStorageAccountAsync(storageAccount, ConfigurationContainerName, NonpersonalizedSettingsFileName, Utility.DictionaryToDelimitedText(systemSettings, SettingsDelimiter));
});
}
else
Expand Down Expand Up @@ -965,10 +961,10 @@ private static Dictionary<string, string> GetDefaultValues(string[] files)
return settings;
}

private Dictionary<string, string> GetSystemSettings(Dictionary<string, string> settings)
private Dictionary<string, string> ConfigureSettings(IIdentity managedIdentity, Dictionary<string, string> settings, Version installedVersion)
{
settings ??= new();
var defaults = GetDefaultValues(new[] { "env-00-coa-version.txt", "env-02-internal-images.txt", "env-03-external-images.txt", });
var defaults = GetDefaultValues(new [] { "env-00-coa-version.txt", "env-01-account-names.txt", "env-02-internal-images.txt", "env-03-external-images.txt", "env-04-settings.txt" });

// We always overwrite the CoA version
UpdateSetting(settings, defaults, "CromwellOnAzureVersion", default(string), ignoreDefaults: false);
Expand All @@ -984,14 +980,6 @@ private Dictionary<string, string> GetSystemSettings(Dictionary<string, string>
UpdateSetting(settings, defaults, "BlobxferImageName", configuration.BlobxferImageName);
UpdateSetting(settings, defaults, "DisableBatchNodesPublicIpAddress", configuration.DisableBatchNodesPublicIpAddress, b => b.GetValueOrDefault().ToString(), configuration.DisableBatchNodesPublicIpAddress.GetValueOrDefault().ToString());

BackFillSettings(settings, defaults);
return settings;
}

private Dictionary<string, string> GetPersonalizedSettings(IIdentity managedIdentity, Dictionary<string, string> settings, Version installedVersion)
{
settings ??= new();
var defaults = GetDefaultValues(new [] { "env-01-account-names.txt", "env-04-settings.txt" });

if (installedVersion is null)
{
Expand All @@ -1005,7 +993,6 @@ private Dictionary<string, string> GetPersonalizedSettings(IIdentity managedIden
UpdateSetting(settings, defaults, "AksCoANamespace", configuration.AksCoANamespace, ignoreDefaults: true);

var provisionPostgreSqlOnAzure = configuration.ProvisionPostgreSqlOnAzure.GetValueOrDefault();
UpdateSetting(settings, defaults, "ProvisionPostgreSqlOnAzure", configuration.ProvisionPostgreSqlOnAzure);
UpdateSetting(settings, defaults, "CrossSubscriptionAKSDeployment", configuration.CrossSubscriptionAKSDeployment);
UpdateSetting(settings, defaults, "PostgreSqlServerName", provisionPostgreSqlOnAzure ? configuration.PostgreSqlServerName : string.Empty, ignoreDefaults: true);
UpdateSetting(settings, defaults, "PostgreSqlDatabaseName", provisionPostgreSqlOnAzure ? configuration.PostgreSqlCromwellDatabaseName : string.Empty, ignoreDefaults: true);
Expand Down Expand Up @@ -2773,19 +2760,31 @@ void ValidateHelmInstall(string helmPath, string featureName)

if (!configuration.Update)
{
ValidateDependantFeature(configuration.UseAks, nameof(configuration.UseAks), configuration.ProvisionPostgreSqlOnAzure.GetValueOrDefault(), nameof(configuration.ProvisionPostgreSqlOnAzure));
ValidateDependantFeature(configuration.CrossSubscriptionAKSDeployment.GetValueOrDefault(), nameof(configuration.CrossSubscriptionAKSDeployment), configuration.UseAks, nameof(configuration.UseAks));
}

ThrowIfBothProvided(configuration.UseAks, nameof(configuration.UseAks), configuration.CustomTesImagePath != null, nameof(configuration.CustomTesImagePath));
ThrowIfBothProvided(configuration.UseAks, nameof(configuration.UseAks), configuration.CustomTriggerServiceImagePath != null, nameof(configuration.CustomTriggerServiceImagePath));
ThrowIfBothProvided(configuration.UseAks, nameof(configuration.UseAks), configuration.CustomCromwellImagePath != null, nameof(configuration.CustomCromwellImagePath));
ValidateDependantFeature(configuration.UseAks, nameof(configuration.UseAks), !string.IsNullOrWhiteSpace(configuration.HelmBinaryPath), nameof(configuration.HelmBinaryPath));


if (configuration.UseAks)
{
ThrowIfNotProvidedForUpdate(configuration.AksClusterName, nameof(configuration.AksClusterName));
ValidateHelmInstall(configuration.HelmBinaryPath, nameof(configuration.HelmBinaryPath));

if (!configuration.ManualHelmDeployment)
{
ValidateDependantFeature(configuration.UseAks, nameof(configuration.UseAks), !string.IsNullOrWhiteSpace(configuration.HelmBinaryPath), nameof(configuration.HelmBinaryPath));
ValidateHelmInstall(configuration.HelmBinaryPath, nameof(configuration.HelmBinaryPath));
}

if (configuration.ProvisionPostgreSqlOnAzure is null)
{
configuration.ProvisionPostgreSqlOnAzure = true;
}
else if (!configuration.ProvisionPostgreSqlOnAzure.GetValueOrDefault())
{
ValidateDependantFeature(configuration.UseAks, nameof(configuration.UseAks), configuration.ProvisionPostgreSqlOnAzure.GetValueOrDefault(), nameof(configuration.ProvisionPostgreSqlOnAzure));
}
}
}

Expand Down Expand Up @@ -3047,12 +3046,15 @@ public static async Task<string> DownloadTextFromStorageAccountAsync(IStorageAcc
}

private async Task UploadTextToStorageAccountAsync(IStorageAccount storageAccount, string containerName, string blobName, string content)
=> await UploadTextToStorageAccountAsync(storageAccount, containerName, blobName, content, cts.Token);

public static async Task UploadTextToStorageAccountAsync(IStorageAccount storageAccount, string containerName, string blobName, string content, CancellationToken token)
{
var blobClient = await GetBlobClientAsync(storageAccount);
var container = blobClient.GetBlobContainerClient(containerName);

await container.CreateIfNotExistsAsync();
await container.GetBlobClient(blobName).UploadAsync(BinaryData.FromString(content), true, cts.Token);
await container.GetBlobClient(blobName).UploadAsync(BinaryData.FromString(content), true, token);
}

private static string GetLinuxParentPath(string path)
Expand Down
Loading