[AppService] Fix #12251 app settings race condition during zip deploy

src/azure-cli/HISTORY.rst

@@ -3,6 +3,10 @@
 Release History
 ===============
 
+**AppService**
+
+* functionapp: Fix an issue when using `az functionapp deployment source config-zip` to deploy on Linux app service / premium plan causing 5xx error
+
 2.1.0
 ++++++
 

src/azure-cli/azure/cli/command_modules/appservice/custom.py

@@ -52,7 +52,7 @@
 from ._params import AUTH_TYPES, MULTI_CONTAINER_TYPES, LINUX_RUNTIMES, WINDOWS_RUNTIMES
 from ._client_factory import web_client_factory, ex_handler_factory
 from ._appservice_utils import _generic_site_operation
-from .utils import _normalize_sku, get_sku_name
+from .utils import _normalize_sku, get_sku_name, retryable_method
 from ._create_util import (zip_contents_from_dir, get_runtime_version_details, create_resource_group, get_app_details,
                            should_create_new_rg, set_location, does_app_already_exist, get_profile_username,
                            get_plan_to_use, get_lang_from_content, get_rg_to_use, get_sku_to_use,
@@ -353,7 +353,8 @@ def enable_zip_deploy(cmd, resource_group_name, name, src, timeout=None, slot=No
     import urllib3
     authorization = urllib3.util.make_headers(basic_auth='{0}:{1}'.format(user_name, password))
     headers = authorization
-    headers['content-type'] = 'application/octet-stream'
+    headers['Content-Type'] = 'application/octet-stream'
+    headers['Cache-Control'] = 'no-cache'
     headers['User-Agent'] = UA_AGENT
 
     import requests
@@ -380,54 +381,92 @@ def enable_zip_deploy(cmd, resource_group_name, name, src, timeout=None, slot=No
 
 def add_remote_build_app_settings(cmd, resource_group_name, name, slot):
     settings = get_app_settings(cmd, resource_group_name, name, slot)
-    enable_oryx_build = None
     scm_do_build_during_deployment = None
     website_run_from_package = None
+    enable_oryx_build = None
+
+    app_settings_should_not_have = []
+    app_settings_should_contains = {}
+
     for keyval in settings:
         value = keyval['value'].lower()
-        if keyval['name'] == 'ENABLE_ORYX_BUILD':
-            enable_oryx_build = value in ('true', '1')
         if keyval['name'] == 'SCM_DO_BUILD_DURING_DEPLOYMENT':
             scm_do_build_during_deployment = value in ('true', '1')
         if keyval['name'] == 'WEBSITE_RUN_FROM_PACKAGE':
             website_run_from_package = value
+        if keyval['name'] == 'ENABLE_ORYX_BUILD':
+            enable_oryx_build = value
 
-    if not ((enable_oryx_build is True) and (scm_do_build_during_deployment is True)):
-        logger.warning("Setting ENABLE_ORYX_BUILD to true")
+    if scm_do_build_during_deployment is not True:
         logger.warning("Setting SCM_DO_BUILD_DURING_DEPLOYMENT to true")
         update_app_settings(cmd, resource_group_name, name, [
-            "ENABLE_ORYX_BUILD=true",
             "SCM_DO_BUILD_DURING_DEPLOYMENT=true"
         ], slot)
-        time.sleep(5)
+        app_settings_should_contains['SCM_DO_BUILD_DURING_DEPLOYMENT'] = 'true'
 
-    if website_run_from_package is not None:
+    if website_run_from_package:
         logger.warning("Removing WEBSITE_RUN_FROM_PACKAGE app setting")
         delete_app_settings(cmd, resource_group_name, name, [
             "WEBSITE_RUN_FROM_PACKAGE"
         ], slot)
-        time.sleep(5)
+        app_settings_should_not_have.append('WEBSITE_RUN_FROM_PACKAGE')
+
+    if enable_oryx_build:
+        logger.warning("Removing ENABLE_ORYX_BUILD app setting")
+        delete_app_settings(cmd, resource_group_name, name, [
+            "ENABLE_ORYX_BUILD"
+        ], slot)
+        app_settings_should_not_have.append('ENABLE_ORYX_BUILD')
+
+    # Wait for scm site to get the latest app settings
+    if app_settings_should_not_have or app_settings_should_contains:
+        logger.warning("Waiting SCM site to be updated with the latest app settings")
+        scm_is_up_to_date = False
+        retries = 10
+        while not scm_is_up_to_date and retries >= 0:
+            scm_is_up_to_date = validate_app_settings_in_scm(
+                cmd, resource_group_name, name, slot,
+                should_contain=app_settings_should_contains,
+                should_not_have=app_settings_should_not_have)
+            retries -= 1
+            time.sleep(5)
+
+        if retries < 0:
+            logger.warning("App settings may not be propagated to the SCM site.")
 
 
 def remove_remote_build_app_settings(cmd, resource_group_name, name, slot):
     settings = get_app_settings(cmd, resource_group_name, name, slot)
-    enable_oryx_build = None
     scm_do_build_during_deployment = None
+
+    app_settings_should_contains = {}
+
     for keyval in settings:
         value = keyval['value'].lower()
-        if keyval['name'] == 'ENABLE_ORYX_BUILD':
-            enable_oryx_build = value in ('true', '1')
         if keyval['name'] == 'SCM_DO_BUILD_DURING_DEPLOYMENT':
             scm_do_build_during_deployment = value in ('true', '1')
 
-    if not ((enable_oryx_build is False) and (scm_do_build_during_deployment is False)):
-        logger.warning("Setting ENABLE_ORYX_BUILD to false")
+    if scm_do_build_during_deployment is not False:
         logger.warning("Setting SCM_DO_BUILD_DURING_DEPLOYMENT to false")
         update_app_settings(cmd, resource_group_name, name, [
-            "ENABLE_ORYX_BUILD=false",
             "SCM_DO_BUILD_DURING_DEPLOYMENT=false"
         ], slot)
-        time.sleep(5)
+        app_settings_should_contains['SCM_DO_BUILD_DURING_DEPLOYMENT'] = 'false'
+
+    # Wait for scm site to get the latest app settings
+    if app_settings_should_contains:
+        logger.warning("Waiting SCM site to be updated with the latest app settings")
+        scm_is_up_to_date = False
+        retries = 10
+        while not scm_is_up_to_date and retries >= 0:
+            scm_is_up_to_date = validate_app_settings_in_scm(
+                cmd, resource_group_name, name, slot,
+                should_contain=app_settings_should_contains)
+            retries -= 1
+            time.sleep(5)
+
+        if retries < 0:
+            logger.warning("App settings may not be propagated to the SCM site")
 
 
 def upload_zip_to_storage(cmd, resource_group_name, name, src, slot=None):
@@ -753,6 +792,47 @@ def get_app_settings(cmd, resource_group_name, name, slot=None):
     return _build_app_settings_output(result.properties, slot_app_setting_names)
 
 
+# Check if the app setting is propagated to the Kudu site correctly by calling api/settings endpoint
+# should_have [] is a list of app settings which are expected to be set
+# should_not_have [] is a list of app settings which are expected to be absent
+# should_contain {} is a dictionary of app settings which are expected to be set with precise values
+# Return True if validation succeeded
+def validate_app_settings_in_scm(cmd, resource_group_name, name, slot=None,
+                                 should_have=None, should_not_have=None, should_contain=None):
+    scm_settings = _get_app_settings_from_scm(cmd, resource_group_name, name, slot)
+    scm_setting_keys = set(scm_settings.keys())
+
+    if should_have and not set(should_have).issubset(scm_setting_keys):
+        return False
+
+    if should_not_have and set(should_not_have).intersection(scm_setting_keys):
+        return False
+
+    temp_setting = scm_settings.copy()
+    temp_setting.update(should_contain or {})
+    if temp_setting != scm_settings:
+        return False
+
+    return True
+
+
+@retryable_method(3, 5)
+def _get_app_settings_from_scm(cmd, resource_group_name, name, slot=None):
+    scm_url = _get_scm_url(cmd, resource_group_name, name, slot)
+    settings_url = '{}/api/settings'.format(scm_url)
+    username, password = _get_site_credential(cmd.cli_ctx, resource_group_name, name, slot)
+    headers = {
+        'Content-Type': 'application/octet-stream',
+        'Cache-Control': 'no-cache',
+        'User-Agent': UA_AGENT
+    }
+
+    import requests
+    response = requests.get(settings_url, headers=headers, auth=(username, password), timeout=3)
+
+    return response.json() or {}
+
+
 def get_connection_strings(cmd, resource_group_name, name, slot=None):
     result = _generic_site_operation(cmd.cli_ctx, resource_group_name, name, 'list_connection_strings', slot)
     client = web_client_factory(cmd.cli_ctx)