diff --git a/src/azure-cli/azure/cli/command_modules/rdbms/_flexible_server_util.py b/src/azure-cli/azure/cli/command_modules/rdbms/_flexible_server_util.py index 88d74c91906..49dde216e0f 100644 --- a/src/azure-cli/azure/cli/command_modules/rdbms/_flexible_server_util.py +++ b/src/azure-cli/azure/cli/command_modules/rdbms/_flexible_server_util.py @@ -6,21 +6,30 @@ # pylint: disable=unused-argument, line-too-long, import-outside-toplevel, raise-missing-from import datetime as dt from datetime import datetime +import os import random +import subprocess import secrets import string +import yaml from knack.log import get_logger from azure.core.paging import ItemPaged +from azure.cli.core.commands.client_factory import get_subscription_id from azure.cli.core.commands import LongRunningOperation, _is_poller from azure.cli.core.azclierror import RequiredArgumentMissingError, InvalidArgumentValueError +from azure.cli.command_modules.role.custom import create_service_principal_for_rbac from azure.mgmt.resource.resources.models import ResourceGroup from msrestazure.tools import parse_resource_id from ._client_factory import resource_client_factory, cf_mysql_flexible_location_capabilities, cf_postgres_flexible_location_capabilities -from .flexible_server_custom_common import firewall_rule_create_func + logger = get_logger(__name__) DEFAULT_LOCATION_PG = 'eastus' # For testing: 'eastus2euap' DEFAULT_LOCATION_MySQL = 'westus2' +AZURE_CREDENTIALS = 'AZURE_CREDENTIALS' +AZURE_POSTGRESQL_CONNECTION_STRING = 'AZURE_POSTGRESQL_CONNECTION_STRING' +AZURE_MYSQL_CONNECTION_STRING = 'AZURE_MYSQL_CONNECTION_STRING' +GITHUB_ACTION_PATH = '/.github/workflows/' def resolve_poller(result, cli_ctx, name): @@ -78,40 +87,6 @@ def generate_password(administrator_login_password): return administrator_login_password -def create_firewall_rule(db_context, cmd, resource_group_name, server_name, start_ip, end_ip): - # allow access to azure ip addresses - cf_firewall, logging_name = db_context.cf_firewall, db_context.logging_name # NOQA pylint: disable=unused-variable - now = datetime.now() - firewall_name = 'FirewallIPAddress_{}-{}-{}_{}-{}-{}'.format(now.year, now.month, now.day, now.hour, now.minute, - now.second) - if start_ip == '0.0.0.0' and end_ip == '0.0.0.0': - logger.warning('Configuring server firewall rule, \'azure-access\', to accept connections from all ' - 'Azure resources...') - firewall_name = 'AllowAllAzureServicesAndResourcesWithinAzureIps_{}-{}-{}_{}-{}-{}'.format(now.year, now.month, - now.day, now.hour, - now.minute, - now.second) - elif start_ip == end_ip: - logger.warning('Configuring server firewall rule to accept connections from \'%s\'...', start_ip) - else: - if start_ip == '0.0.0.0' and end_ip == '255.255.255.255': - firewall_name = 'AllowAll_{}-{}-{}_{}-{}-{}'.format(now.year, now.month, now.day, now.hour, now.minute, - now.second) - logger.warning('Configuring server firewall rule to accept connections from \'%s\' to \'%s\'...', start_ip, - end_ip) - firewall_client = cf_firewall(cmd.cli_ctx, None) - - # Commenting out until firewall_id is properly returned from RP - # return resolve_poller( - # firewall_client.create_or_update(resource_group_name, server_name, firewall_name , start_ip, end_ip), - # cmd.cli_ctx, '{} Firewall Rule Create/Update'.format(logging_name)) - - firewall = firewall_rule_create_func(firewall_client, resource_group_name, server_name, firewall_rule_name=firewall_name, - start_ip_address=start_ip, end_ip_address=end_ip) - - return firewall.result().name - - # pylint: disable=inconsistent-return-statements def parse_public_access_input(public_access): # pylint: disable=no-else-return @@ -339,3 +314,92 @@ def _resolve_api_version(client, provider_namespace, resource_type, parent_path) raise RequiredArgumentMissingError( 'API version is required and could not be resolved for resource {}' .format(resource_type)) + + +def run_subprocess(command, stdout_show=None): + if stdout_show: + process = subprocess.Popen(command, shell=True) + else: + process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process.wait() + if process.returncode: + logger.warning(process.stderr.read().strip().decode('UTF-8')) + + +def run_subprocess_get_output(command): + commands = command.split() + process = subprocess.Popen(commands, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process.wait() + return process + + +def register_credential_secrets(cmd, server, repository): + resource_group = parse_resource_id(server.id)["resource_group"] + scope = "/subscriptions/{}/resourceGroups/{}".format(get_subscription_id(cmd.cli_ctx), resource_group) + + app = create_service_principal_for_rbac(cmd, name=server.name, role='contributor', scopes=[scope]) + app['clientId'], app['clientSecret'], app['tenantId'] = app.pop('appId'), app.pop('password'), app.pop('tenant') + app['subscriptionId'] = get_subscription_id(cmd.cli_ctx) + app.pop('displayName') + app.pop('name') + + app_key_val = [] + for key, val in app.items(): + app_key_val.append('"{}": "{}"'.format(key, val)) + + app_json = ',\n '.join(app_key_val) + app_json = '{\n ' + app_json + '\n}' + credential_file = "./temp_app_credential.txt" + with open(credential_file, "w") as f: + f.write(app_json) + run_subprocess('gh secret set {} --repo {} < {}'.format(AZURE_CREDENTIALS, repository, credential_file)) + os.remove(credential_file) + + +def register_connection_secrets(cmd, database_engine, server, database_name, administrator_login, administrator_login_password, repository): + if database_engine == 'postgresql': + connection_string = "host={} port=5432 dbname={} user={} password={} sslmode=require".format(server.fully_qualified_domain_name, database_name, administrator_login, administrator_login_password) + run_subprocess('gh secret set {} --repo {} -b"{}"'.format(AZURE_POSTGRESQL_CONNECTION_STRING, repository, connection_string)) + elif database_engine == 'mysql': + connection_string = "Server={}; Port=3306; Database={}; Uid={}; Pwd={}; SslMode=Preferred;".format(server.fully_qualified_domain_name, database_name, administrator_login, administrator_login_password) + run_subprocess('gh secret set {} --repo {} -b"{}"'.format(AZURE_MYSQL_CONNECTION_STRING, repository, connection_string)) + + +def fill_action_template(cmd, database_engine, server, database_name, administrator_login, administrator_login_password, file_name, action_name, repository): + + action_dir = get_git_root_dir() + GITHUB_ACTION_PATH + if not os.path.exists(action_dir): + os.makedirs(action_dir) + + process = run_subprocess_get_output("gh secret list --repo {}".format(repository)) + github_secrets = process.stdout.read().strip().decode('UTF-8') + connection_string = AZURE_POSTGRESQL_CONNECTION_STRING if database_engine == 'postgresql' else AZURE_MYSQL_CONNECTION_STRING + + if AZURE_CREDENTIALS not in github_secrets: + register_credential_secrets(cmd, + server=server, + repository=repository) + + if connection_string not in github_secrets: + register_connection_secrets(cmd, + database_engine=database_engine, + server=server, + database_name=database_name, + administrator_login=administrator_login, + administrator_login_password=administrator_login_password, + repository=repository) + + current_location = os.path.dirname(__file__) + + with open(current_location + "/templates/" + database_engine + "_githubaction_template.yaml", "r") as template_file: + template = yaml.safe_load(template_file) + template['jobs']['build']['steps'][2]['with']['server-name'] = server.fully_qualified_domain_name + template['jobs']['build']['steps'][2]['with']['sql-file'] = file_name + with open(action_dir + action_name + '.yml', 'w', encoding='utf8') as yml_file: + yml_file.write("on: [workflow_dispatch]\n") + yml_file.write(yaml.dump(template)) + + +def get_git_root_dir(): + process = run_subprocess_get_output("git rev-parse --show-toplevel") + return process.stdout.read().strip().decode('UTF-8') diff --git a/src/azure-cli/azure/cli/command_modules/rdbms/_helptext_mysql.py b/src/azure-cli/azure/cli/command_modules/rdbms/_helptext_mysql.py index 0203e822820..36dab8ba23e 100644 --- a/src/azure-cli/azure/cli/command_modules/rdbms/_helptext_mysql.py +++ b/src/azure-cli/azure/cli/command_modules/rdbms/_helptext_mysql.py @@ -318,3 +318,26 @@ - name: Show connection strings for cmd and programming languages. text: az mysql flexible-server show-connection-string -s testServer -u username -p password -d databasename """ + +helps['mysql flexible-server deploy'] = """ +type: group +short-summary: Enable and run github action workflow for MySQL server +""" + +helps['mysql flexible-server deploy setup'] = """ +type: command +short-summary: Create github action workflow file for MySQL server. +examples: + - name: Create github action workflow file for MySQL server. + text: az mysql flexible-server deploy setup -s testServer -g testGroup -u username -p password --sql-file test.sql --repo username/userRepo -d flexibleserverdb --action-name testAction + - name: Create github action workflow file for MySQL server and push it to the remote repository + text: az mysql flexible-server deploy setup -s testServer -g testGroup -u username -p password --sql-file test.sql --repo username/userRepo -d flexibleserverdb --action-name testAction --branch userBranch --allow-push +""" + +helps['mysql flexible-server deploy run'] = """ +type: command +short-summary: Run an existing workflow in your github repository +examples: + - name: Run an existing workflow in your github repository + text: az mysql flexible-server deploy run --action-name testAction --branch userBranch +""" diff --git a/src/azure-cli/azure/cli/command_modules/rdbms/_helptext_pg.py b/src/azure-cli/azure/cli/command_modules/rdbms/_helptext_pg.py index 7ade40808b1..cd5331bad7e 100644 --- a/src/azure-cli/azure/cli/command_modules/rdbms/_helptext_pg.py +++ b/src/azure-cli/azure/cli/command_modules/rdbms/_helptext_pg.py @@ -44,7 +44,7 @@ - name: Create a PostgreSQL flexible server using already existing virtual network and subnet. If provided virtual network and subnet does not exists then virtual network and subnet with default address prefix will be created. text: | az postgres flexible-server create --vnet myVnet --subnet mySubnet - - name: Create a PostgreSQL flexible server using already existing virtual network, subnet, and using the subnet ID. The provided subnet should not have any other resource deployed in it and this subnet will be delegated to Microsoft.DBforMySQL/flexibleServers, if not already delegated. + - name: Create a PostgreSQL flexible server using already existing virtual network, subnet, and using the subnet ID. The provided subnet should not have any other resource deployed in it and this subnet will be delegated to Microsoft.DBforPostgreSQL/flexibleServers, if not already delegated. text: | az postgres flexible-server create --subnet /subscriptions/{SubID}/resourceGroups/{ResourceGroup}/providers/Microsoft.Network/virtualNetworks/{VNetName}/subnets/{SubnetName} - name: Create a PostgreSQL flexible server using new virtual network, subnet with non-default address prefix. @@ -299,3 +299,26 @@ - name: Show connection strings for cmd and programming languages. text: az postgres flexible-server show-connection-string -s testServer -u username -p password -d databasename """ + +helps['postgres flexible-server deploy'] = """ +type: group +short-summary: Enable and run github action workflow for PostgreSQL server +""" + +helps['postgres flexible-server deploy setup'] = """ +type: command +short-summary: Create github action workflow file for PostgreSQL server. +examples: + - name: Create github action workflow file for PostgreSQL server. + text: az postgres flexible-server deploy setup -s testServer -g testGroup -u username -p password --sql-file test.sql --repo username/userRepo -d flexibleserverdb --action-name testAction + - name: Create github action workflow file for PostgreSQL server and push it to the remote repository + text: az postgres flexible-server deploy setup -s testServer -g testGroup -u username -p password --sql-file test.sql --repo username/userRepo -d flexibleserverdb --action-name testAction --branch userBranch --allow-push +""" + +helps['postgres flexible-server deploy run'] = """ +type: command +short-summary: Run an existing workflow in your github repository +examples: + - name: Run an existing workflow in your github repository + text: az postgres flexible-server deploy run --action-name testAction --branch userBranch +""" diff --git a/src/azure-cli/azure/cli/command_modules/rdbms/_params.py b/src/azure-cli/azure/cli/command_modules/rdbms/_params.py index f0f5ee01088..a5e885b4186 100644 --- a/src/azure-cli/azure/cli/command_modules/rdbms/_params.py +++ b/src/azure-cli/azure/cli/command_modules/rdbms/_params.py @@ -465,5 +465,22 @@ def _flexible_server_params(command_group): with self.argument_context('{} flexible-server replica stop-replication'.format(command_group)) as c: c.argument('server_name', options_list=['--name', '-n'], help='Name of the replica server.') + with self.argument_context('{} flexible-server deploy setup'.format(command_group)) as c: + c.argument('resource_group_name', arg_type=resource_group_name_type) + c.argument('server_name', id_part='name', options_list=['--server-name', '-s'], arg_type=server_name_arg_type) + c.argument('database_name', options_list=['--database-name', '-d'], help='The name of the database') + c.argument('administrator_login', options_list=['--admin-user', '-u'], arg_group='Authentication', arg_type=administrator_login_arg_type, + help='Administrator username for the server.') + c.argument('administrator_login_password', options_list=['--admin-password', '-p'], arg_group='Authentication', help='The password of the administrator.') + c.argument('sql_file_path', options_list=['--sql-file'], help='The path of the sql file. The sql file should be already in the repository') + c.argument('action_name', options_list=['--action-name'], help='The name of the github action') + c.argument('repository', options_list=['--repo'], help='The name of your github username and repository e.g., Azure/azure-cli ') + c.argument('branch', options_list=['--branch'], help='The name of the branch you want upload github action file. The default will be your current branch.') + c.argument('allow_push', default=False, options_list=['--allow-push'], arg_type=get_three_state_flag(), help='Push the action yml file to the remote repository. The changes will be pushed to origin repository, speicified branch or current branch if not specified.') + + with self.argument_context('{} flexible-server deploy run'.format(command_group)) as c: + c.argument('action_name', options_list=['--action-name'], help='The name of the github action') + c.argument('branch', options_list=['--branch'], help='The name of the branch you want upload github action file. The default will be your current branch.') + _flexible_server_params('postgres') _flexible_server_params('mysql') diff --git a/src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_commands.py b/src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_commands.py index e755dac9245..a9254100b88 100644 --- a/src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_commands.py +++ b/src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_commands.py @@ -152,6 +152,13 @@ def load_flexibleserver_command_table(self, _): g.show_command('show', 'get') g.command('list', 'list_by_server') + with self.command_group('postgres flexible-server deploy', postgres_flexible_servers_sdk, + custom_command_type=flexible_server_custom_common, + client_factory=cf_postgres_flexible_servers, + is_preview=True) as g: + g.custom_command('setup', 'github_actions_setup') + g.custom_command('run', 'github_actions_run') + # MySQL commands with self.command_group('mysql flexible-server', mysql_flexible_servers_sdk, custom_command_type=flexible_servers_custom_mysql, @@ -221,3 +228,10 @@ def load_flexibleserver_command_table(self, _): is_preview=True) as g: g.custom_command('create', 'flexible_replica_create', supports_no_wait=True) g.custom_command('stop-replication', 'flexible_replica_stop', confirmation=True) + + with self.command_group('mysql flexible-server deploy', mysql_flexible_servers_sdk, + custom_command_type=flexible_server_custom_common, + client_factory=cf_mysql_flexible_servers, + is_preview=True) as g: + g.custom_command('setup', 'github_actions_setup') + g.custom_command('run', 'github_actions_run') diff --git a/src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_custom_common.py b/src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_custom_common.py index f3c3e272a6d..df7bc56ea07 100644 --- a/src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_custom_common.py +++ b/src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_custom_common.py @@ -8,6 +8,10 @@ from knack.log import get_logger from knack.util import CLIError from azure.cli.core.util import user_confirmation +from azure.cli.core.azclierror import ClientRequestError, RequiredArgumentMissingError +from azure.mgmt.rdbms.mysql_flexibleservers.operations._servers_operations import ServersOperations as MySqlServersOperations +from ._flexible_server_util import run_subprocess, run_subprocess_get_output, fill_action_template, get_git_root_dir, \ + GITHUB_ACTION_PATH logger = get_logger(__name__) # pylint: disable=raise-missing-from @@ -121,3 +125,93 @@ def database_delete_func(client, resource_group_name=None, server_name=None, dat except Exception as ex: # pylint: disable=broad-except logger.error(ex) return result + + +def create_firewall_rule(db_context, cmd, resource_group_name, server_name, start_ip, end_ip): + # allow access to azure ip addresses + cf_firewall, logging_name = db_context.cf_firewall, db_context.logging_name # NOQA pylint: disable=unused-variable + now = datetime.now() + firewall_name = 'FirewallIPAddress_{}-{}-{}_{}-{}-{}'.format(now.year, now.month, now.day, now.hour, now.minute, + now.second) + if start_ip == '0.0.0.0' and end_ip == '0.0.0.0': + logger.warning('Configuring server firewall rule, \'azure-access\', to accept connections from all ' + 'Azure resources...') + firewall_name = 'AllowAllAzureServicesAndResourcesWithinAzureIps_{}-{}-{}_{}-{}-{}'.format(now.year, now.month, + now.day, now.hour, + now.minute, + now.second) + elif start_ip == end_ip: + logger.warning('Configuring server firewall rule to accept connections from \'%s\'...', start_ip) + else: + if start_ip == '0.0.0.0' and end_ip == '255.255.255.255': + firewall_name = 'AllowAll_{}-{}-{}_{}-{}-{}'.format(now.year, now.month, now.day, now.hour, now.minute, + now.second) + logger.warning('Configuring server firewall rule to accept connections from \'%s\' to \'%s\'...', start_ip, + end_ip) + firewall_client = cf_firewall(cmd.cli_ctx, None) + + # Commenting out until firewall_id is properly returned from RP + # return resolve_poller( + # firewall_client.create_or_update(resource_group_name, server_name, firewall_name , start_ip, end_ip), + # cmd.cli_ctx, '{} Firewall Rule Create/Update'.format(logging_name)) + + firewall = firewall_rule_create_func(firewall_client, resource_group_name, server_name, firewall_rule_name=firewall_name, + start_ip_address=start_ip, end_ip_address=end_ip) + + return firewall.result().name + + +def github_actions_setup(cmd, client, resource_group_name, server_name, database_name, administrator_login, administrator_login_password, sql_file_path, repository, action_name=None, branch=None, allow_push=None): + + server = client.get(resource_group_name, server_name) + if server.public_network_access == 'Disabled': + raise ClientRequestError("This command only works with public access enabled server.") + if allow_push and not branch: + raise RequiredArgumentMissingError("Provide remote branch name to allow pushing the action file to your remote branch.") + if action_name is None: + action_name = server.name + '_' + database_name + "_deploy" + gitcli_check_and_login() + + if isinstance(client, MySqlServersOperations): + database_engine = 'mysql' + else: + database_engine = 'postgresql' + + fill_action_template(cmd, + database_engine=database_engine, + server=server, + database_name=database_name, + administrator_login=administrator_login, + administrator_login_password=administrator_login_password, + file_name=sql_file_path, + repository=repository, + action_name=action_name) + + action_path = get_git_root_dir() + GITHUB_ACTION_PATH + action_name + '.yml' + logger.warning("Making git commit for file %s", action_path) + run_subprocess("git add {}".format(action_path)) + run_subprocess("git commit -m \"Add github action file\"") + + if allow_push: + logger.warning("Pushing the created action file to origin %s branch", branch) + run_subprocess("git push origin {}".format(branch)) + github_actions_run(action_name, branch) + else: + logger.warning('You did not set --allow-push parameter. Please push the prepared file %s to your remote repo and run "deploy run" command to activate the workflow.', action_path) + + +def github_actions_run(action_name, branch): + + gitcli_check_and_login() + logger.warning("Created event for %s.yml in branch %s", action_name, branch) + run_subprocess("gh workflow run {}.yml --ref {}".format(action_name, branch)) + + +def gitcli_check_and_login(): + output = run_subprocess_get_output("gh") + if output.returncode: + raise ClientRequestError('Please install "Github CLI" to run this command.') + + output = run_subprocess_get_output("gh auth status") + if output.returncode: + run_subprocess("gh auth login", stdout_show=True) diff --git a/src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_custom_mysql.py b/src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_custom_mysql.py index fc416e9a5a0..b45e23421ff 100644 --- a/src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_custom_mysql.py +++ b/src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_custom_mysql.py @@ -16,9 +16,9 @@ from azure.mgmt.rdbms import mysql_flexibleservers from ._client_factory import get_mysql_flexible_management_client, cf_mysql_flexible_firewall_rules, \ cf_mysql_flexible_db, cf_mysql_check_resource_availability -from ._flexible_server_util import resolve_poller, generate_missing_parameters, create_firewall_rule, \ - parse_public_access_input, generate_password, parse_maintenance_window, get_mysql_list_skus_info, \ - DEFAULT_LOCATION_MySQL +from ._flexible_server_util import resolve_poller, generate_missing_parameters, parse_public_access_input, \ + DEFAULT_LOCATION_MySQL, generate_password, parse_maintenance_window, get_mysql_list_skus_info +from .flexible_server_custom_common import user_confirmation, create_firewall_rule from .flexible_server_virtual_network import prepare_private_network from .validators import mysql_arguments_validator, validate_server_name diff --git a/src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_custom_postgres.py b/src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_custom_postgres.py index d7348d1e28f..ffeec6bd129 100644 --- a/src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_custom_postgres.py +++ b/src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_custom_postgres.py @@ -15,9 +15,9 @@ from azure.cli.core.azclierror import RequiredArgumentMissingError, ArgumentUsageError from azure.mgmt.rdbms import postgresql_flexibleservers from ._client_factory import cf_postgres_flexible_firewall_rules, get_postgresql_flexible_management_client, cf_postgres_flexible_db, cf_postgres_check_resource_availability -from ._flexible_server_util import generate_missing_parameters, resolve_poller, create_firewall_rule, \ - parse_public_access_input, generate_password, parse_maintenance_window, get_postgres_list_skus_info, \ - DEFAULT_LOCATION_PG +from .flexible_server_custom_common import user_confirmation, create_firewall_rule +from ._flexible_server_util import generate_missing_parameters, resolve_poller, parse_public_access_input, \ + generate_password, parse_maintenance_window, get_postgres_list_skus_info, DEFAULT_LOCATION_PG from .flexible_server_virtual_network import prepare_private_network, prepare_private_dns_zone from .validators import pg_arguments_validator, validate_server_name