diff --git a/web/README.md b/web/README.md new file mode 100644 index 000000000..6509d6c76 --- /dev/null +++ b/web/README.md @@ -0,0 +1,36 @@ +# Prototype for web-based checklist + +This is a Minimum Viable Product (MVP) for an architecture for web-based checklist reviews. It consists of the following elements: + +- A MySQL database +- An Azure Container Instance that will launch 3 containers: + - filldb (init container): creates the required database and tables in the MySQL server, and fills in the data imported from the latest checklist + - fillgraphdb (init container): executes any Azure Resource Graph queries stored in the checklist, and stores the results in the MySQL database + - flask (main container): a flask-based web frontend that allows inspecting the MysQL checklist table, as well as updating the status and comments of each individual checklist item + +The `fillgraphdb` container needs to authenticate to Azure to send the Azure Resource Graph queries. There are two options: + +- Working today: With Service Principal credentials +- Roadmap: With a [User-Managed Identity](https://docs.microsoft.com/azure/active-directory/managed-identities-azure-resources/overview#how-can-i-use-managed-identities-for-azure-resources) with read access to the subscription(s). The `identityId` parameter of the ARM template needs to be provided. Initial tests have shown that the User-Managed Identity is not available in the init containers. + +The [Azure CLI deployment script for Service Principals](./arm/deploy_sp.azcli) shows how to create the Service Principal, assign the reader role for the whole subscription, and launch the ARM template to create the MySQL server and the Azure Container Instance (it doesn't store the Service Principal secret in an Azure Key Vault, that would be highly advisable). If you already have the Service Principal, you can deploy the ARM template graphically as well using the button below: + +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Freview-checklists%2Fweb_jose%2Fweb%2Farm%2Ftemplate.json) + +The web interface will be available in the public IP address of the ACI container group, on TCP port 5000. + +## Further improvements + +Since this is only a prototype, there are some aspects not being addressed for the sake of simplicity: + +- Figure out why the user-managed identities seem not be reachable from the init containers +- No HTTPS (it could be easily achieved with an nginx sidecar in the ACI container group) +- No authentication (an authentication proxy such as Ambassador could be leveraged for this) +- The network firewall of the MySQL server is fully open (it could be closed down to the ACI egress IP address) +- The UI of the flask container is rather rudimentary, but it shows the basic principles and does live updates to the MySQL database without having to press any "Submit" button +- SSL Enforcement is disabled in the MySQL Server due to `flask-mysql` not using encryption +- Decouple the containers, so that they can be launched independently: + - It should be possible to launch the `fillgraphdb` container at any time, to refresh the Graph results + - It should be possible to restart the `flask` container (web) without having t + +Contributions highly appreciated! diff --git a/web/arm/deploy_mi.azcli b/web/arm/deploy_mi.azcli new file mode 100644 index 000000000..3b97f3a27 --- /dev/null +++ b/web/arm/deploy_mi.azcli @@ -0,0 +1,62 @@ +############################################## +# 1. Retrieve Manged Identity, or create one # +############################################## + +id_name=checklistid +id_location=westeurope +id_rg=checklistid +id_id=$(az identity show -n $id_name -g $id_rg --query id -o tsv) +if [[ -z "$id_id" ]]; then + echo "Creating user identity in resource group ${id_rg}..." + az group create -n $id_rg -l $id_location -o none + az identity create -n $id_name -g $id_rg -o none + id_id=$(az identity show -n $id_name -g $id_rg --query id -o tsv) + id_client_id=$(az identity show -n $id_name -g $id_rg --query clientId -o tsv) + subscription_id=$(az account show --query id -o tsv) + scope="/subscriptions/${subscription_id}" + az role assignment create --scope $scope --assignee $id_client_id --role 'Reader' -o none + echo "User-managed identity has been created: $id_id" +else + echo "User-managed identity has been located: $id_id" +fi + +#################################################### +# 2. Create RG and deploy checklist infrastructure # +#################################################### + +# Variables +suffix=$RANDOM +rg="checklist${suffix}" +location=westeurope +aci_name="checklistaci${suffix}" +mysql_server_name="mysql${suffix}" +mysql_server_user=$(whoami) +mysql_server_password=$(echo $(tr -dc a-zA-Z0-9 /dev/null| head -c 12)) + +# Create RG +echo "Creating resource group ${rg} in ${location}..." +az group create -n $rg -l $location -o none + +# Deploy ARM template +echo "Deploying template to resource group ${rg}..." +az deployment group create -n checklist$RANDOM -g $rg --template-file ./template.json --parameters \ + "checklistTech=aks" \ + "identityId=${id_id}" \ + "aciName=${aci_name}" \ + "serverName=${mysql_server_name}" \ + "administratorLogin=${mysql_server_user}" \ + "administratorLoginPassword=${mysql_server_password}" + +# Optional: re-run fillgraphdb container to refresh the Graph query results +graph_aci_name="graph${suffix}" +graph_image="erjosito/checklist-fillgraphdb:1.0" +mysql_server_fqdn=$(az mysql server show -n $mysql_server_name -g $rg --query 'fullyQualifiedDomainName' -o tsv) +echo "Creating ACI ${graph_aci_name}..." +az container create -n $graph_aci_name -g $rg --assign-identity "$id_id" --image "$graph_image" --ip-address public --restart-policy OnFailure -o none -e \ + "MYSQL_PASSWORD=${mysql_server_password}" \ + "MYSQL_USER=${mysql_server_user}" \ + "MYSQL_FQDN=${mysql_server_fqdn}" \ + "WAIT_INTERVALS=6" + +# Cleanup +# az group delete $rg -y --no-wait \ No newline at end of file diff --git a/web/arm/deploy_sp.azcli b/web/arm/deploy_sp.azcli new file mode 100644 index 000000000..bb231ee30 --- /dev/null +++ b/web/arm/deploy_sp.azcli @@ -0,0 +1,100 @@ +################################################################## +# 1. Retrieve SP credentials from Azure Key Vault, or create one # +################################################################## + +# Variables +keyvault_name=erjositoKeyvault +keyvault_rg=keyvaults # Only required if new AKV is to be created +keyvault_loc=westeurope # Only required if new AKV is to be created +purpose=checklists + +# Day zero: create Azure Key Vault if required +echo "Verifying if AKV ${keyvault_name} exists..." +keyvault_rg_found=$(az keyvault list -o tsv --query "[?name=='$keyvault_name'].resourceGroup") +if [[ -n ${keyvault_rg_found} ]] +then + echo "AKV ${keyvault_name} found in resource group $keyvault_rg_found" + keyvault_rg="$keyvault_rg_found" +else + echo "Creating AKV ${keyvault_name} in RG ${keyvault_rg}..." + az group create -n $keyvault_rg -l $keyvault_loc -o none + az keyvault create -n $keyvault_name -g $keyvault_rg -l $keyvault_loc -o none + user_name=$(az account show --query 'user.name' -o tsv) + echo "Setting policies for user ${user_name}..." + az keyvault set-policy -n $keyvault_name -g $keyvault_rg --upn $user_name -o none \ + --certificate-permissions backup create delete deleteissuers get getissuers import list listissuers managecontacts manageissuers purge recover restore setissuers update \ + --key-permissions backup create decrypt delete encrypt get import list purge recover restore sign unwrapKey update verify wrapKey \ + --secret-permissions backup delete get list purge recover restore set \ + --storage-permissions backup delete deletesas get getsas list listsas purge recover regeneratekey restore set setsas update +fi + +# Try to get SP details from AKV +echo "Trying to get secrets from AKV ${keyvault_name}..." +keyvault_appid_secret_name=$purpose-sp-appid +keyvault_password_secret_name=$purpose-sp-secret +sp_app_id=$(az keyvault secret show --vault-name $keyvault_name -n $keyvault_appid_secret_name --query 'value' -o tsv) +sp_app_secret=$(az keyvault secret show --vault-name $keyvault_name -n $keyvault_password_secret_name --query 'value' -o tsv) + +# If either variable is blank, create new SP with the required name +if [[ -z "$sp_app_id" ]] || [[ -z "$sp_app_secret" ]] +then + # Create new SP + echo "Service Principal secrets for ${purpose} not found, creating new Service Principal..." + sp_name=$purpose + sp_output=$(az ad sp create-for-rbac --name $sp_name --skip-assignment 2>/dev/null) + sp_app_id=$(echo $sp_output | jq -r '.appId') + sp_app_secret=$(echo $sp_output | jq -r '.password') + az keyvault secret set --vault-name $keyvault_name --name $keyvault_appid_secret_name --value $sp_app_id -o none + az keyvault secret set --vault-name $keyvault_name --name $keyvault_password_secret_name --value $sp_app_secret -o none + # Optionally, assign Azure RBAC roles (example a RG) or AKV policy (example certificate/secret get if the SP should be able to retrieve certs) + echo "Adding Reader role to new Service Principal..." + subscription_id=$(az account show --query id -o tsv) + scope="/subscriptions/${subscription_id}" + az role assignment create --scope $scope --assignee $sp_app_id --role 'Reader' -o none +fi + +# Verify whether SP has expired, and if so, renew it +sp_end_date=$(az ad app show --id $sp_app_id --query 'passwordCredentials[0].endDate' -o tsv) +sp_end_date=$(date --date="$sp_end_date" +%s) +now=$(date +%s) +if [[ $sp_end_date < $now ]] +then + echo "SP expired, extending one year" + new_password=$(az ad app credential reset --id $sp_app_id --years 1 --query password -o tsv) + az keyvault secret set --vault-name $keyvault_name --name $keyvault_password_secret_name --value $new_password -o none + sp_app_secret=$new_password +else + echo "SP not expired, not required to renew" +fi + +# Optional: test the SP credentials +# tenant_id=$(az account show --query tenantId -o tsv) +# az login --service-principal -u $sp_app_id -p $sp_app_secret --tenant $tenant_id + +#################################################### +# 2. Create RG and deploy checklist infrastructure # +#################################################### + +suffix=$RANDOM +rg="checklist${suffix}" +location=westeurope +aci_name="checklistaci${suffix}" +mysql_server_name="mysql${suffix}" +mysql_server_user=$(whoami) +mysql_server_password=$(echo $(tr -dc a-zA-Z0-9 /dev/null| head -c 12)) +echo "Creating resource group ${rg} in ${location}..." +az group create -n $rg -l $location -o none +echo "Deploying template to resource group ${rg}..." +tenant_id=$(az account show --query tenantId -o tsv) +az deployment group create -n checklist$RANDOM -g $rg --template-file ./template.json --parameters \ + "checklistTech=aks" \ + "tenantId=${tenant_id}" \ + "clientId=${sp_app_id}" \ + "clientSecret=${sp_app_secret}" \ + "aciName=${aci_name}" \ + "serverName=${mysql_server_name}" \ + "administratorLogin=${mysql_server_user}" \ + "administratorLoginPassword=${mysql_server_password}" + +# Cleanup +# az group delete $rg -y --no-wait \ No newline at end of file diff --git a/web/arm/parameters.json b/web/arm/parameters.json new file mode 100644 index 000000000..c198e4596 --- /dev/null +++ b/web/arm/parameters.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "identityName": { + "value": "checklistid" + }, + "serverName": { + "value": "checklistsql1138" + }, + "administratorLogin": { + "value": "jose" + }, + "administratorLoginPassword": { + "reference": { + "keyVault": { + "id": "/subscriptions/e7da9914-9b05-4891-893c-546cb7b0422e/resourceGroups/myKeyvault/providers/Microsoft.KeyVault/vaults/erjositoKeyvault" + }, + "secretName": "defaultPassword" + } + } + } +} \ No newline at end of file diff --git a/web/arm/template.json b/web/arm/template.json new file mode 100644 index 000000000..4e99e831c --- /dev/null +++ b/web/arm/template.json @@ -0,0 +1,290 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": {}, + "parameters": { + "checklistTech": { + "type": "string", + "defaultValue": "lz", + "allowedValues": [ + "lz", + "aks", + "avd", + "security" + ], + "metadata": { + "description": "Options available for the checklist" + } + }, + "identityId": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "ID for user-assigned identity that will be assigned to the ACI (not required if specifying Tenant ID, Client ID and Client secret)" + } + }, + "tenantId": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Tenant ID for SP-based authentication (not required if specifying Identity Id)" + } + }, + "clientId": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Client ID for SP-based authentication (not required if specifying Identity Id)" + } + }, + "clientSecret": { + "type": "securestring", + "defaultValue": "", + "metadata": { + "description": "Client secret for SP-based authentication (not required if specifying Identity Id)" + } + }, + "aciName": { + "type": "string", + "metadata": { + "description": "ACI name for container group" + } + }, + "serverName": { + "type": "string", + "metadata": { + "description": "Server Name for Azure database for MySQL" + } + }, + "administratorLogin": { + "type": "string", + "minLength": 1, + "metadata": { + "description": "MySQL server administrator login name" + } + }, + "administratorLoginPassword": { + "type": "securestring", + "minLength": 8, + "metadata": { + "description": "MySQL server administrator password" + } + }, + "waitIntervals": { + "type": "int", + "defaultValue": "[if(equals(parameters('identityId'),''), 0, 3)]", + "metadata": { + "description": "MySQL server administrator password" + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Location for all resources." + } + } + }, + "functions": [], + "variables": { + "mysql_api_version": "2017-12-01", + "identity_api_version": "2018-11-30", + "roleassignment_api_version": "2020-04-01-preview", + "aci_api_version": "2019-12-01", + "skuName": "B_Gen5_1", + "skuFamily": "Gen5", + "skuCapacity": 1, + "skuTier": "Basic", + "skuSizeMB": 5120, + "mySqlVersion": "5.7", + "backupRetentionDays": 7, + "geoRedundantBackup": "Disabled", + "sslEnforcement": "Disabled", + "firewallrules": [ + { + "Name": "permitAll", + "StartIpAddress": "0.0.0.0", + "EndIpAddress": "255.255.255.255" + } + ], + "webImage": "erjosito/checklist-flask:1.0", + "filldbImage": "erjosito/checklist-filldb:1.0", + "fillgraphdbImage": "erjosito/checklist-fillgraphdb:1.0", + "aciPort": 5000, + "idJson": "[concat('{\"type\": \"UserAssigned\", \"userAssignedIdentities\": {\"', parameters('identityId'), '\": {} } }')]" + }, + "resources": [ + { + "type": "Microsoft.DBforMySQL/servers", + "apiVersion": "[variables('mysql_api_version')]", + "name": "[parameters('serverName')]", + "location": "[parameters('location')]", + "sku": { + "name": "[variables('skuName')]", + "tier": "[variables('skuTier')]", + "capacity": "[variables('skuCapacity')]", + "size": "[format('{0}', variables('skuSizeMB'))]", + "family": "[variables('skuFamily')]" + }, + "properties": { + "createMode": "Default", + "version": "[variables('mysqlVersion')]", + "administratorLogin": "[parameters('administratorLogin')]", + "administratorLoginPassword": "[parameters('administratorLoginPassword')]", + "sslEnforcement": "[variables('sslEnforcement')]", + "storageProfile": { + "storageMB": "[variables('skuSizeMB')]", + "backupRetentionDays": "[variables('backupRetentionDays')]", + "geoRedundantBackup": "[variables('geoRedundantBackup')]" + } + } + }, + { + "copy": { + "name": "firewallRules", + "count": "[length(variables('firewallrules'))]", + "mode": "serial", + "batchSize": 1 + }, + "type": "Microsoft.DBforMySQL/servers/firewallRules", + "apiVersion": "[variables('mysql_api_version')]", + "name": "[format('{0}/{1}', parameters('serverName'), variables('firewallrules')[copyIndex()].Name)]", + "properties": { + "startIpAddress": "[variables('firewallrules')[copyIndex()].StartIpAddress]", + "endIpAddress": "[variables('firewallrules')[copyIndex()].EndIpAddress]" + }, + "dependsOn": [ + "[resourceId('Microsoft.DBforMySQL/servers', parameters('serverName'))]" + ] + }, + { + "type": "Microsoft.ContainerInstance/containerGroups", + "apiVersion": "[variables('aci_api_version')]", + "name": "[parameters('aciName')]", + "location": "[parameters('location')]", + "dependsOn": [ + "[resourceId('Microsoft.DBforMySQL/servers', parameters('serverName'))]" + ], + "identity": "[if(equals(parameters('identityId'), ''), json('null'), json(variables('idJson')))]", + "properties": { + "initContainers": [ + { + "name": "filldb", + "properties": { + "image": "[variables('filldbImage')]", + "environmentVariables": [ + { + "name": "CHECKLIST_TECHNOLOGY", + "value": "[parameters('checklistTech')]" + }, + { + "name": "MYSQL_USER", + "secureValue": "[parameters('administratorLogin')]" + }, + { + "name": "MYSQL_PASSWORD", + "secureValue": "[parameters('administratorLoginPassword')]" + }, + { + "name": "MYSQL_FQDN", + "value": "[reference(resourceId('Microsoft.DBforMySQL/servers', parameters('serverName'))).fullyQualifiedDomainName]" + } + ] + } + }, + { + "name": "fillgraphdb", + "properties": { + "image": "[variables('fillgraphdbImage')]", + "environmentVariables": [ + { + "name": "MYSQL_USER", + "secureValue": "[parameters('administratorLogin')]" + }, + { + "name": "MYSQL_PASSWORD", + "secureValue": "[parameters('administratorLoginPassword')]" + }, + { + "name": "MYSQL_FQDN", + "value": "[reference(resourceId('Microsoft.DBforMySQL/servers', parameters('serverName'))).fullyQualifiedDomainName]" + }, + { + "name": "AZURE_TENANT_ID", + "secureValue": "[parameters('tenantId')]" + }, + { + "name": "AZURE_CLIENT_ID", + "secureValue": "[parameters('clientId')]" + }, + { + "name": "AZURE_CLIENT_SECRET", + "secureValue": "[parameters('clientSecret')]" + }, + { + "name": "WAIT_INTERVALS", + "secureValue": "[parameters('waitIntervals')]" + } + ] + } + } + ], + "containers": [ + { + "name": "flask", + "properties": { + "image": "[variables('webImage')]", + "environmentVariables": [ + { + "name": "MYSQL_USER", + "secureValue": "[parameters('administratorLogin')]" + }, + { + "name": "MYSQL_PASSWORD", + "secureValue": "[parameters('administratorLoginPassword')]" + }, + { + "name": "MYSQL_FQDN", + "value": "[reference(resourceId('Microsoft.DBforMySQL/servers', parameters('serverName'))).fullyQualifiedDomainName]" + } + ], + "ports": [ + { + "port": "[variables('aciPort')]", + "protocol": "TCP" + } + ], + "resources": { + "requests": { + "cpu": 1, + "memoryInGB": 1 + } + } + } + } + ], + "osType": "Linux", + "restartPolicy": "Always", + "ipAddress": { + "type": "Public", + "ports": [ + { + "port": "[variables('aciPort')]", + "protocol": "TCP" + } + ] + } + } + } + ], + "outputs": { + "containerIPv4Address": { + "type": "string", + "value": "[reference(resourceId('Microsoft.ContainerInstance/containerGroups', parameters('aciName'))).ipAddress.ip]" + }, + "waitIntervals": { + "type": "int", + "value": "[parameters('waitIntervals')]" + } + } +} diff --git a/web/createdb/create_mysql.azcli b/web/createdb/create_mysql.azcli new file mode 100644 index 000000000..a35e2e541 --- /dev/null +++ b/web/createdb/create_mysql.azcli @@ -0,0 +1,38 @@ +# Variables +rg=checklistweb +location=westeurope +suffix=$RANDOM +mysql_server_name="checklist${suffix}" +mysql_server_username=checklist +mysql_server_password=$(echo $(tr -dc a-zA-Z0-9 /dev/null| head -c 12)) +akv_name="checklist${suffix}" +akv_secret_usr="mysqluser" +akv_secret_pwd="mysqlpassword" + +# Create RG and mysql server +echo "Creating resource group $rg in $location..." +az group create -n $rg -l $location -o none +echo "Creating MySQL server $sql_server_name..." +az mysql server create -l $location -g $rg -n $mysql_server_name \ + -u $mysql_server_username -p $mysql_server_password \ + --sku-name B_Gen5_1 --ssl-enforcement Enabled --minimal-tls-version TLS1_1 \ + --public-network-access Enabled -o none + +# Add this IP address to MySQL server firewall +myip=$(curl -s4 ifconfig.co) +echo "Adding IP address $myip to MySQL server firewall..." +az mysql server firewall-rule create -g $rg -s $mysql_server_name -n allow$RANDOM --start-ip-address $myip --end-ip-address $myip -o none + +# Create key vault and store username and password +echo "Creating Azure Key Vault $akv_name..." +az keyvault create -n $akv_name -g $rg -l $location -o none +echo "Storing MySQL username and password in AKV $akv_name..." +az keyvault secret set -n $akv_secret_usr --vault-name $akv_name --value "$mysql_server_username" -o none +az keyvault secret set -n $akv_secret_pwd --vault-name $akv_name --value "$mysql_server_password" -o none + +# Test retrieval and export as environment variables +mysql_server_name=$(az mysql server list -g $rg --query '[0].name' -o tsv) +akv_name=$(az keyvault list -g $rg --query '[0].name' -o tsv) +export MYSQL_FQDN=$(az mysql server show -n $mysql_server_name -g $rg --query 'fullyQualifiedDomainName' -o tsv) +export MYSQL_USER="$(az keyvault secret show -n $akv_secret_usr --vault-name $akv_name --query value -o tsv)" +export MYSQL_PASSWORD="$(az keyvault secret show -n $akv_secret_pwd --vault-name $akv_name --query value -o tsv)" diff --git a/web/filldb/Dockerfile b/web/filldb/Dockerfile new file mode 100644 index 000000000..0eeef1806 --- /dev/null +++ b/web/filldb/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:18.04 +MAINTAINER Jose Moreno "jose.moreno@microsoft.com" + +RUN apt-get update -y && apt-get install -y python3-pip python3-dev build-essential curl libssl1.0.0 libssl-dev libpq-dev +COPY . /app +WORKDIR /app +RUN pip3 install -r requirements.txt +CMD ["sh", "-c", "python3 fill_db.py"] diff --git a/web/filldb/check_db.py b/web/filldb/check_db.py new file mode 100644 index 000000000..e9b846257 --- /dev/null +++ b/web/filldb/check_db.py @@ -0,0 +1,48 @@ +import os +import sys +import pymysql + +# Database and table name +mysql_db_name = "checklist" +mysql_db_table = "items" +use_ssl = "yes" + +# Format a string to be included in a SQL query as value +def escape_quotes(this_value): + return str(this_value).replace("'", "\\'") + +# Get database credentials from environment variables +mysql_server_fqdn = os.environ.get("MYSQL_FQDN") +if mysql_server_fqdn == None: + print("Please define an environment variable 'MYSQL_FQDN' with the FQDN of the MySQL server") + sys.exit(1) +mysql_server_name = mysql_server_fqdn.split('.')[0] +mysql_server_username = os.environ.get("MYSQL_USER") +if mysql_server_username == None: + print("Please define an environment variable 'MYSQL_USER' with the FQDN of the MySQL username") + sys.exit(1) +if not mysql_server_username.__contains__('@'): + mysql_server_username += '@' + mysql_server_name +mysql_server_password = os.environ.get("MYSQL_PASSWORD") +if mysql_server_password == None: + print("Please define an environment variable 'MYSQL_PASSWORD' with the FQDN of the MySQL password") + sys.exit(1) + +# Create connection to MySQL server and number of records +print ("Connecting to {0} with username {1}...".format(mysql_server_fqdn, mysql_server_username)) +if use_ssl == 'yes': + db = pymysql.connect(host=mysql_server_fqdn, user = mysql_server_username, database = mysql_db_name, passwd = mysql_server_password, ssl = {'ssl':{'ca': 'BaltimoreCyberTrustRoot.crt.pem'}}) +else: + db = pymysql.connect(host=mysql_server_fqdn, user = mysql_server_username, database = mysql_db_name, passwd = mysql_server_password) +sql_query = "SELECT COUNT(*) FROM {0};".format (mysql_db_table) +cursor = db.cursor() +cursor.execute(sql_query) +rows = cursor.fetchall() +if len(rows) > 0: + row_count = rows[0][0] +else: + row_count = 0 +print ("Table {0} in database {1} contains {2} records".format(mysql_db_table, mysql_db_name, str(row_count))) + +# Bye +db.close() \ No newline at end of file diff --git a/web/filldb/fill_db.py b/web/filldb/fill_db.py new file mode 100644 index 000000000..8368d4ed5 --- /dev/null +++ b/web/filldb/fill_db.py @@ -0,0 +1,137 @@ +import requests +import json +import os +import sys +import pymysql + +# Database and table name +mysql_db_name = "checklist" +mysql_db_table = "items" +use_ssl = "yes" + +# Format a string to be included in a SQL query as value +def escape_quotes(this_value): + return str(this_value).replace("'", "\\'") + +# Get database credentials from environment variables +mysql_server_fqdn = os.environ.get("MYSQL_FQDN") +if mysql_server_fqdn == None: + print("ERROR: Please define an environment variable 'MYSQL_FQDN' with the FQDN of the MySQL server") + sys.exit(1) +mysql_server_name = mysql_server_fqdn.split('.')[0] +mysql_server_username = os.environ.get("MYSQL_USER") +if mysql_server_username == None: + print("ERROR: Please define an environment variable 'MYSQL_USER' with the FQDN of the MySQL username") + sys.exit(1) +if not mysql_server_username.__contains__('@'): + mysql_server_username += '@' + mysql_server_name +mysql_server_password = os.environ.get("MYSQL_PASSWORD") +if mysql_server_password == None: + print("ERROR: Please define an environment variable 'MYSQL_PASSWORD' with the FQDN of the MySQL password") + sys.exit(1) + +# Create connection to MySQL server and get version +print ("INFO: Connecting to {0} with username {1}...".format(mysql_server_fqdn, mysql_server_username)) +if use_ssl == 'yes': + db = pymysql.connect(host=mysql_server_fqdn, user = mysql_server_username, passwd = mysql_server_password, ssl = {'ssl':{'ca': 'BaltimoreCyberTrustRoot.crt.pem'}}) +else: + db = pymysql.connect(host=mysql_server_fqdn, user = mysql_server_username, passwd = mysql_server_password) +sql_query = "SELECT VERSION();" +cursor = db.cursor() +cursor.execute(sql_query) +rows = cursor.fetchall() +data = "" +if len(rows) > 0: + for row in rows: + if len(data) > 0: + data += ', ' + data += str(''.join(row)) +print ("INFO: Connected to MySQL server {0} with version {1}".format(mysql_server_fqdn, data)) + +# Delete db if existed +sql_query = "DROP DATABASE IF EXISTS {0};".format(mysql_db_name) +# print ("Sending query: {0}".format(sql_query)) +cursor.execute(sql_query) +db.commit() + +# Create database +sql_query = "CREATE DATABASE IF NOT EXISTS {0};".format(mysql_db_name) +# print ("Sending query: {0}".format(sql_query)) +cursor.execute(sql_query) +db.commit() +sql_query = "USE {0}".format(mysql_db_name) +# print ("Sending query: {0}".format(sql_query)) +cursor.execute(sql_query) +db.commit() + +# Create table +sql_query = """CREATE TABLE {0} ( + guid varchar(40), + text varchar(1024), + description varchar(1024), + link varchar(255), + training varchar(255), + comments varchar(1024), + severity varchar(10), + status varchar(15), + category varchar(255), + subcategory varchar(255), + graph_query_success varchar(1024), + graph_query_failure varchar(1024), + graph_query_result varchar(4096) +);""".format(mysql_db_table) +# print ("DEBUG: Sending query: {0}".format(sql_query)) +cursor.execute(sql_query) +db.commit() + +# Download checklist +technology = os.environ.get("CHECKLIST_TECHNOLOGY") +if technology: + checklist_url = "https://raw.githubusercontent.com/Azure/review-checklists/main/checklists/" + technology + "_checklist.en.json" +else: + checklist_url = "https://raw.githubusercontent.com/Azure/review-checklists/main/checklists/lz_checklist.en.json" +response = requests.get(checklist_url) + +# If download was successful +if response.status_code == 200: + print ("INFO: File {0} downloaded successfully".format(checklist_url)) + try: + # Deserialize JSON to object variable + checklist_object = json.loads(response.text) + except Exception as e: + print("Error deserializing JSON content: {0}".format(str(e))) + sys.exit(1) + # Get default status from the JSON, default to "Not verified" + try: + status_list = checklist_object.get("status") + default_status = status_list[0].get("name") + except: + default_status = "Not verified" + pass + # For each checklist item, add a row to mysql DB + row_counter = 0 + for item in checklist_object.get("items"): + guid = item.get("guid") + category = item.get("category") + subcategory = item.get("subcategory") + text = escape_quotes(item.get("text")) + description = escape_quotes(item.get("description")) + severity = item.get("severity") + link = item.get("link") + training = item.get("training") + status = default_status + graph_query_success = escape_quotes(item.get("graph_success")) + graph_query_failure = escape_quotes(item.get("graph_failure")) + # print("DEBUG: Adding to table {0}: '{1}', '{2}', '{3}', '{4}', '{5}', '{6}', '{7}', '{8}', '{9}', '{10}'".format(mysql_db_table, category, subcategory, text, description, severity, link, training, graph_query_success, graph_query_failure, guid)) + sql_query = """INSERT INTO {0} (category,subcategory,text,description,severity,link,training,graph_query_success,graph_query_failure,guid,status) + VALUES ('{1}','{2}','{3}','{4}','{5}', '{6}','{7}','{8}','{9}','{10}', '{11}');""".format(mysql_db_table, category, subcategory, text, description, severity, link, training, graph_query_success, graph_query_failure, guid, status) + # print ("DEBUG: Sending query: {0}".format(sql_query)) + cursor.execute(sql_query) + db.commit() + row_counter += 1 +else: + print ("Error downloading {0}".format(checklist_url)) + +# Bye +print("INFO: {0} rows added to database.".format(str(row_counter))) +db.close() \ No newline at end of file diff --git a/web/filldb/requirements.txt b/web/filldb/requirements.txt new file mode 100644 index 000000000..bf5147f69 --- /dev/null +++ b/web/filldb/requirements.txt @@ -0,0 +1,2 @@ +requests +pymysql \ No newline at end of file diff --git a/web/fillgraphdb/Dockerfile b/web/fillgraphdb/Dockerfile new file mode 100644 index 000000000..3a1c7c483 --- /dev/null +++ b/web/fillgraphdb/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:18.04 +MAINTAINER Jose Moreno "jose.moreno@microsoft.com" + +RUN apt-get update -y && apt-get install -y python3-pip python3-dev build-essential curl libssl1.0.0 libssl-dev libpq-dev gir1.2-secret-1 +COPY . /app +WORKDIR /app +RUN pip3 install -r requirements.txt +CMD ["sh", "-c", "python3 graph_db.py"] diff --git a/web/fillgraphdb/graph_db.py b/web/fillgraphdb/graph_db.py new file mode 100644 index 000000000..9f99a6c5d --- /dev/null +++ b/web/fillgraphdb/graph_db.py @@ -0,0 +1,190 @@ +import os +import sys +import pymysql +import json +import time +import requests +import azure.mgmt.resourcegraph as arg +from datetime import datetime +from azure.mgmt.resource import SubscriptionClient +from azure.identity import AzureCliCredential +from azure.identity import DefaultAzureCredential +from azure.identity import ClientSecretCredential + +# Database and table name +mysql_db_name = "checklist" +mysql_db_table = "items" +use_ssl = "yes" + +# Format a string to be included in a SQL query as value +def escape_quotes (this_value): + return str(this_value).replace("'", "\\'") + +# Function to send an Azure Resource Graph query +def get_resources (graph_query, argClient, subsList, argQueryOptions): + # TO DO: Authentication should probably happen outside of this function + try: + # Create query + argQuery = arg.models.QueryRequest(subscriptions=subsList, query=graph_query, options=argQueryOptions) + # Run query and return results + argResults = argClient.resources(argQuery) + print("DEBUG: query results: {0}".format(str(argResults))) + return argResults + except Exception as e: + print("ERROR: Error sending Azure Resource Graph query to Azure: {0}".format(str(e))) + # sys.exit(0) # Debugging.... Probably this should be exit(1) + return '' + +# Wait for IMDS endpoint to be available +try: + wait_max_intervals = int(os.environ.get("WAIT_INTERVALS")) + print ("DEBUG: WAIT_INTERVALS read from environment variable: {0}".format(str(wait_max_intervals))) +except: + wait_max_intervals = 5 + print ("DEBUG: WAIT_INTERVALS set to default value: {0}".format(str(wait_max_intervals))) +wait_interval = 10.0 +imds_url = 'http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://management.azure.com/' +imds_headers = { + "Metadata" : "true" +} +imds_tries = 0 +break_loop = False +print ('DEBUG: Going into while loop...') +while not break_loop: + imds_tries += 1 + print ("DEBUG: We are in the loop, pass {0}/{1} ({2}). Trying the IMDS endpoint...".format(str(imds_tries), str(wait_max_intervals), str(datetime.now()))) + if imds_tries > wait_max_intervals: + print("ERROR: max wait intervals exceeded when waiting for IMDS to answer, hopefully you specified some SP credentials as SP variables...") + break_loop = True + else: + print ("DEBUG: Sending GET request to {0}...".format(imds_url)) + try: + imds_response = requests.get(imds_url, headers=imds_headers, timeout=1) + if imds_response.status_code >= 200 and imds_response.status_code <= 299: + print ("DEBUG: IMDS endpoint seems to be working, received status code {0} and answer {1}".format(str(imds_response.status_code), str(imds_response.text))) + break_loop = True + else: + print ("DEBUG: IMDS endpoint doesnt seem to be working, received status code {0} and answer {1}".format(str(imds_response.status_code), str(imds_response.text))) + except Exception as e: + print("DEBUG: Error sending request to IMDS endpoint: {0}".format(str(e))) + pass + if not break_loop: + print("DEBUG: Going to sleep {0} seconds before next try...".format(str(wait_interval))) + time.sleep (wait_interval) + +# Authenticate to Azure, either with Managed Identity or SP +print('DEBUG: Authenticating to Azure...') +try: + print('DEBUG: Getting environment variables...') + # credential = AzureCliCredential() # Get your credentials from Azure CLI (development only!) and get your subscription list + tenant_id = os.environ.get("AZURE_TENANT_ID") + client_id = os.environ.get("AZURE_CLIENT_ID") + client_secret = os.environ.get("AZURE_CLIENT_SECRET") +except Exception as e: + print("ERROR: Error getting environment variables: {0}".format(str(e))) + tenant_id = None + client_id = None + client_secret = None + pass +try: + if tenant_id and client_id and client_secret: + print("DEBUG: Service principal credentials (client ID {0}, tenant ID {1}) retrieved from environment variables, trying SP-based authentication now...".format(str(client_id), str(tenant_id))) + credential = ClientSecretCredential(tenant_id=tenant_id, client_id=client_id, client_secret=client_secret) + else: + print('DEBUG: Service principal credentials could not be retrieved from environment variables, trying default authentication methods...') + credential = DefaultAzureCredential() # Managed identity + print('DEBUG: Getting subscriptions...') + subsClient = SubscriptionClient(credential) + subsRaw = [] + for sub in subsClient.subscriptions.list(): + subsRaw.append(sub.as_dict()) + subsList = [] + for sub in subsRaw: + subsList.append(sub.get('subscription_id')) + print ("DEBUG: provided credentials give access to {0} subscription(s)".format(str(len(subsList)))) + # Create Azure Resource Graph client and set options + print('DEBUG: Creating client object...') + argClient = arg.ResourceGraphClient(credential) + argQueryOptions = arg.models.QueryRequestOptions(result_format="objectArray") +except Exception as e: + print("ERROR: Error during Azure Authentication: {0}".format(str(e))) + sys.exit(1) + +# Get database credentials from environment variables +mysql_server_fqdn = os.environ.get("MYSQL_FQDN") +if mysql_server_fqdn == None: + print("ERROR: Please define an environment variable 'MYSQL_FQDN' with the FQDN of the MySQL server") + sys.exit(1) +else: + print("DEBUG: mysql FQDN retrieved from environment variables: '{0}'".format(mysql_server_fqdn)) +mysql_server_name = mysql_server_fqdn.split('.')[0] +mysql_server_username = os.environ.get("MYSQL_USER") +if mysql_server_username == None: + print("ERROR: Please define an environment variable 'MYSQL_USER' with the FQDN of the MySQL username") + sys.exit(1) +else: + print("DEBUG: mysql authentication username retrieved from environment variables: '{0}'".format(mysql_server_username)) +if not mysql_server_username.__contains__('@'): + mysql_server_username += '@' + mysql_server_name +mysql_server_password = os.environ.get("MYSQL_PASSWORD") +if mysql_server_password == None: + print("ERROR: Please define an environment variable 'MYSQL_PASSWORD' with the FQDN of the MySQL password") + sys.exit(1) +else: + print("DEBUG: mysql authentication password retrieved from environment variables: {0}".format("********")) + +# Create connection to MySQL server and number of records +print ("DEBUG: Connecting to '{0}' with username \{1}'...".format(mysql_server_fqdn, mysql_server_username)) +if use_ssl == 'yes': + db = pymysql.connect(host=mysql_server_fqdn, user = mysql_server_username, database = mysql_db_name, passwd = mysql_server_password, ssl = {'ssl':{'ca': 'BaltimoreCyberTrustRoot.crt.pem'}}) +else: + db = pymysql.connect(host=mysql_server_fqdn, user = mysql_server_username, database = mysql_db_name, passwd = mysql_server_password) +sql_query = "SELECT * FROM {0} WHERE graph_query_success IS NOT null AND graph_query_failure IS NOT null AND graph_query_success != 'None' AND graph_query_failure != 'None';".format (mysql_db_table) +cursor = db.cursor() +cursor.execute(sql_query) +rows = cursor.fetchall() +row_cnt = 0 +if len(rows) > 0: + for row in rows: + row_cnt += 1 + result_text = '' + item_guid = row[0] + item_success_query = row[10] + item_failure_query = row[11] + # print ("DEBUG {0}: '{1}', '{2}'".format(item_guid, item_success_query, item_failure_query)) + success_resources = str(get_resources(item_success_query, argClient, subsList, argQueryOptions)).replace("'", '"') + success_resources = success_resources.replace(': None', ': "None"') + # print ("DEBUG: SUCCESS QUERY: {0}".format(success_resources)) + if success_resources: + try: + success_resources_object = json.loads(success_resources) + except: + print("ERROR: JSON returned from Azure Graph Query not valid: {0}".format(success_resources)) + for resource in success_resources_object['data']: + if result_text: result_text += '\n' + result_text += "SUCCESS: {0}".format(resource["id"]) + failure_resources = str(get_resources(item_failure_query, argClient, subsList, argQueryOptions)).replace("'", '"') + failure_resources = failure_resources.replace(': None', ': "None"') + # print ("DEBUG: FAILURE QUERY: {0}".format(failure_resources)) + if failure_resources: + try: + failure_resources_object = json.loads(failure_resources) + except: + print("ERROR: JSON returned from Azure Graph Query not valid: {0}".format(failure_resources)) + for resource in failure_resources_object['data']: + if result_text: result_text += '\n' + result_text += "FAILURE: {0}".format(resource["id"]) + # print ("DEBUG: Result summary: \n{0}".format(result_text)) + if result_text: + update_query = "UPDATE items SET graph_query_result = '{0}' WHERE guid = '{1}';".format(result_text, item_guid) + print ("DEBUG: sending SQL query '{0}'".format(update_query)) + cursor.execute(update_query) + db.commit() + else: + print("DEBUG: No results could be retrieved for the success and failure queries of checklist item {0}".format(item_guid)) +else: + row_count = 0 +print ("INFO: Processed table {0} in database {1} with {2} records with graph queries. Happy review!".format(mysql_db_table, mysql_db_name, str(row_cnt))) + +# Bye +db.close() \ No newline at end of file diff --git a/web/fillgraphdb/requirements.txt b/web/fillgraphdb/requirements.txt new file mode 100644 index 000000000..9995ac41e --- /dev/null +++ b/web/fillgraphdb/requirements.txt @@ -0,0 +1,6 @@ +pymysql +azure-mgmt-resourcegraph +azure-mgmt-resource +azure-cli-core +azure.identity +requests \ No newline at end of file diff --git a/web/flaskmysql/Dockerfile b/web/flaskmysql/Dockerfile new file mode 100644 index 000000000..71cf4c455 --- /dev/null +++ b/web/flaskmysql/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:18.04 +MAINTAINER Jose Moreno "jose.moreno@microsoft.com" + +RUN apt-get update -y && apt-get install -y python3-pip python3-dev build-essential curl libssl1.0.0 libssl-dev libpq-dev +COPY . /app +WORKDIR /app +RUN pip3 install -r requirements.txt +CMD ["sh", "-c", "python3 ./app.py"] + +EXPOSE 5000 \ No newline at end of file diff --git a/web/flaskmysql/app.py b/web/flaskmysql/app.py new file mode 100644 index 000000000..77579e166 --- /dev/null +++ b/web/flaskmysql/app.py @@ -0,0 +1,134 @@ +#app.py +from flask import Flask, request, render_template, jsonify +from flaskext.mysql import MySQL #pip install flask-mysql +import pymysql +import os + +app = Flask(__name__) + +# Get database credentials from environment variables +mysql_server_fqdn = os.environ.get("MYSQL_FQDN") +if mysql_server_fqdn == None: + print("ERROR: Please define an environment variable 'MYSQL_FQDN' with the FQDN of the MySQL server") + sys.exit(1) +mysql_server_name = mysql_server_fqdn.split('.')[0] +mysql_server_username = os.environ.get("MYSQL_USER") +if mysql_server_username == None: + print("ERROR: Please define an environment variable 'MYSQL_USER' with the FQDN of the MySQL username") + sys.exit(1) +if not mysql_server_username.__contains__('@'): + mysql_server_username += '@' + mysql_server_name +mysql_server_password = os.environ.get("MYSQL_PASSWORD") +if mysql_server_password == None: + print("ERROR: Please define an environment variable 'MYSQL_PASSWORD' with the FQDN of the MySQL password") + sys.exit(1) + +# Open connection +mysql = MySQL() +app.config['MYSQL_DATABASE_USER'] = mysql_server_username +app.config['MYSQL_DATABASE_PASSWORD'] = mysql_server_password +app.config['MYSQL_DATABASE_DB'] = 'checklist' +app.config['MYSQL_DATABASE_HOST'] = mysql_server_fqdn +mysql.init_app(app) + +@app.route('/') +def home(): + app.logger.info("DEBUG: Connecting to database...") + try: + category_filter = request.args.get('category', None) + status_filter = request.args.get('status', None) + severity_filter = request.args.get('severity', None) + except Exception as e: + app.logger.info("ERROR reading query parameters for filters: {0}".format(str(e))) + pass + try: + conn = mysql.connect() + cursor = conn.cursor(pymysql.cursors.DictCursor) + except Exception as e: + app.logger.info("ERROR opening cursor to DB connection: {0}".format(str(e))) + return jsonify(str(e)) + try: + sqlquery = "SELECT * from items" + filter1_added = False + # category filter + if category_filter: + if filter1_added: + sqlquery += " AND " + else: + sqlquery += " WHERE " + filter1_added = True + sqlquery += "category = '{0}'".format(category_filter) + # status filter + if status_filter: + if filter1_added: + sqlquery += " AND " + else: + sqlquery += " WHERE " + filter1_added = True + sqlquery += "status = '{0}'".format(status_filter) + # severity filter + if severity_filter: + if filter1_added: + sqlquery += " AND " + else: + sqlquery += " WHERE " + filter1_added = True + sqlquery += "severity = '{0}'".format(severity_filter) + # send queries + app.logger.info ("Retrieving checklist items with query '{0}'".format(sqlquery)) + cursor.execute(sqlquery) + itemslist = cursor.fetchall() + cursor.execute("SELECT DISTINCT category FROM items") + categorylist = cursor.fetchall() + cursor.execute("SELECT DISTINCT severity FROM items") + severitylist = cursor.fetchall() + cursor.execute("SELECT DISTINCT status FROM items") + statuslist = cursor.fetchall() + return render_template('index.html', itemslist=itemslist, categorylist=categorylist, severitylist=severitylist, statuslist=statuslist) + except Exception as e: + app.logger.info("ERROR sending query: {0}".format(str(e))) + return jsonify(str(e)) + +@app.route("/update",methods=["POST","GET"]) +def update(): + app.logger.info("Processing {0} with request.form {1}".format(str(request.method), str(request.form))) + try: + conn = mysql.connect() + cursor = conn.cursor(pymysql.cursors.DictCursor) + if request.method == 'POST': + field = request.form['field'] + value = request.form['value'] + editid = request.form['id'] + app.logger.info("Processing POST for field '{0}', editid '{1}' and value '{2}'".format(field, value, editid)) + + if field == 'comment' and value != '': + sql = "UPDATE items SET comments=%s WHERE guid=%s" + data = (value, editid) + conn = mysql.connect() + cursor = conn.cursor() + app.logger.info ("Sending SQL query '{0}' with data '{1}'".format(sql, str(data))) + cursor.execute(sql, data) + conn.commit() + elif field == 'status' and value != '': + sql = "UPDATE items SET status=%s WHERE guid=%s" + data = (value, editid) + conn = mysql.connect() + cursor = conn.cursor() + app.logger.info ("Sending SQL query '{0}' with data '{1}'".format(sql, str(data))) + cursor.execute(sql, data) + conn.commit() + else: + app.logger.info ("Field is '{0}', value is '{1}': not doing anything".format(field, value)) + success = 1 + return jsonify(success) + except Exception as e: + app.logger.info("Oh oh, there is an error: {0}".format(str(e))) + success = 0 + return jsonify(success) + finally: + cursor.close() + conn.close() + +if __name__ == "__main__": + app.run(host='0.0.0.0', debug=True) + diff --git a/web/flaskmysql/requirements.txt b/web/flaskmysql/requirements.txt new file mode 100644 index 000000000..e7b4eba9e --- /dev/null +++ b/web/flaskmysql/requirements.txt @@ -0,0 +1,3 @@ +flask +pymysql +flask-mysql \ No newline at end of file diff --git a/web/flaskmysql/templates/index.html b/web/flaskmysql/templates/index.html new file mode 100644 index 000000000..62ef0bbcd --- /dev/null +++ b/web/flaskmysql/templates/index.html @@ -0,0 +1,346 @@ + + + +Azure Review + + + + + + +
+
+

Azure Review

+ + + + + + + + + + + + + + + + + + + {% for row in itemslist %} + + + + + + + + + + + + {% endfor %} +
GUIDCategorySubcategorySeverityTextStatusComment
{{row.guid}}{{row.category}}{{row.subcategory}}{{row.severity}} + {{row.text}} + {% if row.link != 'None' %} + (More info) + {% endif %} + {% if row.training != 'None' %} + (Training) + {% endif %} + {% if row.graph_query_result %} + + + {% endif %} + +
{{row.status}}
+ + + + + + + + 		+
{{row.comments}}
+ +
+
+
+ + + + + + + +