{CI} Fix extension release pipeline: Migrate from storage sdk to azure cli commands.

scripts/ci/sync_extensions.py

@@ -8,10 +8,9 @@
 import os
 import re
 import json
-from subprocess import check_output
+import subprocess
 
 DEFAULT_TARGET_INDEX_URL = os.getenv('AZURE_EXTENSION_TARGET_INDEX_URL')
-STORAGE_ACCOUNT_KEY = os.getenv('AZURE_EXTENSION_TARGET_STORAGE_ACCOUNT_KEY')
 STORAGE_ACCOUNT = os.getenv('AZURE_EXTENSION_TARGET_STORAGE_ACCOUNT')
 STORAGE_CONTAINER = os.getenv('AZURE_EXTENSION_TARGET_STORAGE_CONTAINER')
 COMMIT_NUM = os.getenv('AZURE_EXTENSION_COMMIT_NUM') or 1
@@ -20,7 +19,7 @@
 
 def _get_updated_extension_filenames():
     cmd = 'git --no-pager diff --diff-filter=ACMRT HEAD~{} -- src/index.json'.format(COMMIT_NUM)
-    updated_content = check_output(cmd.split()).decode('utf-8')
+    updated_content = subprocess.check_output(cmd.split()).decode('utf-8')
     FILENAME_REGEX = r'"filename":\s+"(.*?)"'
     added_ext_filenames = {re.findall(FILENAME_REGEX, line)[0] for line in updated_content.splitlines() if line.startswith('+') and not line.startswith('+++') and 'filename' in line}
     deleted_ext_filenames = {re.findall(FILENAME_REGEX, line)[0] for line in updated_content.splitlines() if line.startswith('-') and not line.startswith('---') and 'filename' in line}
@@ -50,7 +49,7 @@ def download_file(url, file_path):
                 f.write(chunk)
 
 
-def _sync_wheel(ext, updated_indexes, failed_urls, client, overwrite, temp_dir):
+def _sync_wheel(ext, updated_indexes, failed_urls, overwrite, temp_dir):
     download_url = ext['downloadUrl']
     whl_file = download_url.split('/')[-1]
     whl_path = os.path.join(temp_dir, whl_file)
@@ -61,13 +60,21 @@ def _sync_wheel(ext, updated_indexes, failed_urls, client, overwrite, temp_dir):
         failed_urls.append(download_url)
         return
     if not overwrite:
-        exists = client.exists(container_name=STORAGE_CONTAINER, blob_name=blob_name)
-        if exists:
+        cmd = ['az', 'storage', 'blob', 'exists', '--container-name', f'{STORAGE_CONTAINER}', '--account-name',
+               f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--auth-mode', 'login']
+        result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        if json.loads(result.stdout)['exists']:
             print("Skipping '{}' as it already exists...".format(whl_file))
             return
-    client.create_blob_from_path(container_name=STORAGE_CONTAINER, blob_name=blob_name,
-                                 file_path=os.path.abspath(whl_path))
-    url = client.make_blob_url(container_name=STORAGE_CONTAINER, blob_name=blob_name)
+
+    cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name',
+               f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--file', 'file_path', '--auth-mode', 'login']
+    result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    print(json.loads(result.stdout))
+    cmd = ['az', 'storage', 'blob', 'url', '--container-name', f'{STORAGE_CONTAINER}', '--account-name',
+           f'{STORAGE_ACCOUNT}', '--name',  f'{blob_name}', '--auth-mode', 'login']
+    result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    url = json.loads(result.stdout)
     updated_index = ext
     updated_index['downloadUrl'] = url
     updated_indexes.append(updated_index)
@@ -104,7 +111,6 @@ def _update_target_extension_index(updated_indexes, deleted_ext_filenames, targe
 def main():
     import shutil
     import tempfile
-    from azure.storage.blob import BlockBlobService
 
     net_added_ext_filenames = []
     net_deleted_ext_filenames = []
@@ -134,22 +140,23 @@ def main():
             open(target_index_path, 'w').write(json.dumps(initial_index, indent=4, sort_keys=True))
         else:
             raise
-    client = BlockBlobService(account_name=STORAGE_ACCOUNT, account_key=STORAGE_ACCOUNT_KEY)
     updated_indexes = []
     failed_urls = []
     if sync_all:
         print('Syncing all extensions...\n')
         # backup the old index.json
         backup_index_name = f'{BLOB_PREFIX}/index.json.sav' if BLOB_PREFIX else 'index.json.sav'
-        client.create_blob_from_path(container_name=STORAGE_CONTAINER, blob_name=backup_index_name,
-                                     file_path=os.path.abspath(target_index_path))
+        cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name',
+               f'{STORAGE_ACCOUNT}', '--name', f'{backup_index_name}', '--file', f'{os.path.abspath(target_index_path)}', '--auth-mode', 'login']
+        result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        print(json.loads(result.stdout))
         # start with an empty index.json to sync all extensions
         initial_index = {"extensions": {}, "formatVersion": "1"}
         open(target_index_path, 'w').write(json.dumps(initial_index, indent=4, sort_keys=True))
         for extension_name in current_extensions.keys():
             for ext in current_extensions[extension_name]:
                 print('Uploading {}'.format(ext['filename']))
-                _sync_wheel(ext, updated_indexes, failed_urls, client, True, temp_dir)
+                _sync_wheel(ext, updated_indexes, failed_urls, True, temp_dir)
     else:
         NAME_REGEX = r'^(.*?)-\d+.\d+.\d+'
         for filename in net_added_ext_filenames:
@@ -159,13 +166,16 @@ def main():
             if ext['filename'] != filename:
                 ext = next((ext for ext in current_extensions[extension_name] if ext['filename'] == filename), None)
             if ext is not None:
-                _sync_wheel(ext, updated_indexes, failed_urls, client, True, temp_dir)
+                _sync_wheel(ext, updated_indexes, failed_urls, True, temp_dir)
 
     print("")
     _update_target_extension_index(updated_indexes, net_deleted_ext_filenames, target_index_path)
     index_name = f'{BLOB_PREFIX}/index.json' if BLOB_PREFIX else 'index.json'
-    client.create_blob_from_path(container_name=STORAGE_CONTAINER, blob_name=index_name,
-                                 file_path=os.path.abspath(target_index_path))
+    cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name',
+           f'{STORAGE_ACCOUNT}', '--name', f'{index_name}', '--file', f'{os.path.abspath(target_index_path)}',
+           '--auth-mode', 'login']
+    result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    print(json.loads(result.stdout))
     print("\nSync finished.")
     if updated_indexes:
         print("New extensions available in:")

scripts/ci/sync_extensions.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -ev
+pip install requests
+
+echo $(pwd)
+
+python scripts/ci/sync_extensions.py

scripts/ci/update_ext_cmd_tree.py

@@ -4,16 +4,16 @@
 # --------------------------------------------------------------------------------------------
 
 import filecmp
+import json
 import os
+import subprocess
 import sys
 from azure.cli.core import get_default_cli
 from azure.cli.core._session import Session
 from azure.cli.core.commands import _load_extension_command_loader
 from azure.cli.core.extension import get_extension_modname, get_extension_path
-from azure.storage.blob import BlockBlobService
 from sync_extensions import download_file
 
-STORAGE_ACCOUNT_KEY = os.getenv('AZURE_EXTENSION_CMD_TREE_STORAGE_ACCOUNT_KEY')
 STORAGE_ACCOUNT = os.getenv('AZURE_EXTENSION_CMD_TREE_STORAGE_ACCOUNT')
 STORAGE_CONTAINER = os.getenv('AZURE_EXTENSION_CMD_TREE_STORAGE_CONTAINER')
 BLOB_PREFIX = os.getenv('AZURE_EXTENSION_CMD_TREE_BLOB_PREFIX')
@@ -80,11 +80,15 @@ def upload_cmd_tree():
     downloaded_file_name = 'extCmdTreeDownloaded.json'
     file_path = os.path.expanduser(os.path.join('~', '.azure', file_name))
 
-    client = BlockBlobService(account_name=STORAGE_ACCOUNT, account_key=STORAGE_ACCOUNT_KEY)
-    client.create_blob_from_path(container_name=STORAGE_CONTAINER, blob_name=blob_file_name,
-                                 file_path=file_path)
+    cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name',
+           f'{STORAGE_ACCOUNT}', '--name', f'{blob_file_name}', '--file', f'{file_path}', '--auth-mode', 'login']
+    result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    print(json.loads(result.stdout))
 
-    url = client.make_blob_url(container_name=STORAGE_CONTAINER, blob_name=blob_file_name)
+    cmd = ['az', 'storage', 'blob', 'url', '--container-name', f'{STORAGE_CONTAINER}', '--account-name',
+           f'{STORAGE_ACCOUNT}', '--name', f'{blob_file_name}', '--auth-mode', 'login']
+    result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    url = json.loads(result.stdout)
 
     download_file_path = os.path.expanduser(os.path.join('~', '.azure', downloaded_file_name))
     download_file(url, download_file_path)