{Packaging/RPM} Remove unused *.pyo and *.pyc in RPM

.gitattributes

@@ -13,3 +13,5 @@
 *.bat eol=crlf
 # The az script for Git Bash/Cygwin should be LF
 build_scripts/windows/scripts/az eol=lf
+# sh scripts should be LF
+*.sh eol=lf

azure-pipelines.yml

@@ -639,10 +639,10 @@ jobs:
 
        echo "== Test yum package on CentOS =="
 
-       docker pull centos:centos7
-       docker run --rm -v $SYSTEM_ARTIFACTSDIRECTORY/yum:/mnt/yum centos:centos7 /bin/bash -c "yum --nogpgcheck localinstall /mnt/yum/$YUM_NAME -y && time az self-test && time az --version && sleep 5"
+       docker pull centos:centos8
+       docker run --rm -e YUM_NAME=$YUM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/yum:/mnt/yum -v $(pwd):/azure-cli centos:centos8 /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh"
 
-    displayName: 'Bash Script'
+    displayName: 'Test Yum Package'
 
 - job: BuildUbuntuPackages
   displayName: Build Ubuntu Packages

build_scripts/windows/scripts/build.cmd

@@ -12,7 +12,7 @@ if "%CLI_VERSION%"=="" (
 set PYTHON_VERSION=3.6.6
 
 set WIX_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/msi/wix310-binaries-mirror.zip"
-set PYTHON_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/Python366-32.zip"
+set PYTHON_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/Python368-64.zip"
 set PROPAGATE_ENV_CHANGE_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/propagate_env_change.zip"
 
 :: Set up the output directory and temp. directories
@@ -26,7 +26,7 @@ mkdir %ARTIFACTS_DIR%
 set TEMP_SCRATCH_FOLDER=%ARTIFACTS_DIR%\cli_scratch
 set BUILDING_DIR=%ARTIFACTS_DIR%\cli
 set WIX_DIR=%ARTIFACTS_DIR%\wix
-set PYTHON_DIR=%ARTIFACTS_DIR%\Python366-32
+set PYTHON_DIR=%ARTIFACTS_DIR%\Python368-64
 set PROPAGATE_ENV_CHANGE_DIR=%~dp0..\propagate_env_change
 
 set REPO_ROOT=%~dp0..\..\..
@@ -75,10 +75,10 @@ if not exist %PYTHON_DIR% (
     mkdir %PYTHON_DIR%
     pushd %PYTHON_DIR%
     echo Downloading Python.
-    curl -o Python366-32.zip %PYTHON_DOWNLOAD_URL% -k
-    unzip -q Python366-32.zip
+    curl -o Python368-64.zip %PYTHON_DOWNLOAD_URL% -k
+    unzip -q Python368-64.zip
     if %errorlevel% neq 0 goto ERROR
-    del Python366-32.zip
+    del Python368-64.zip
     echo Python downloaded and extracted successfully.
     popd
 )

build_scripts/windows/scripts/setup_msi_test.ps1

@@ -0,0 +1,21 @@
+# Prerequisites:
+# 1. Install the MSI built with current branch
+# 2. Run bash azure-cli\scripts\ci\build.sh with Git Bash first to generate artifacts under azure-cli\artifacts\build so we can use the testsdk and fulltest wheels.
+
+# Elevate to Admin
+If (-NOT ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator"))
+{
+    $arguments = "& '" + $myinvocation.mycommand.definition + "'"
+    Start-Process powershell -Verb runAs -ArgumentList $arguments
+}
+
+& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install pytest
+& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install pytest-xdist
+
+$testsdk = Get-ChildItem -Path $PSScriptRoot\..\..\..\artifacts\build\azure_cli_testsdk*.whl | Select-Object Name
+& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install $testsdk.Name
+
+$fulltest = Get-ChildItem -Path $PSScriptRoot\..\..\..\artifacts\build\azure_cli_fulltest*.whl | Select-Object Name
+& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install $fulltest.Name
+
+& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' $PSScriptRoot\test_msi_package.py

build_scripts/windows/scripts/test_msi_package.py

@@ -0,0 +1,25 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+# Invoke this script in Powershell with:
+# & 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' test_msi_package.py
+
+import os 
+import sys
+import subprocess 
+
+root_dir = 'C:\\Program Files (x86)\\Microsoft SDKs\\Azure\\CLI2\\Lib\\site-packages\\azure\\cli\\command_modules'
+
+mod_list = [mod for mod in sorted(os.listdir(root_dir)) if os.path.isdir(os.path.join(root_dir, mod)) and mod != '__pycache__']
+
+for mod_name in mod_list:
+    exit_code = subprocess.call(['python', '-m', 'pytest', '-x', '-v', '-p', 'no:warnings', '--log-level', 'WARN', 
+        '--junit-xml', '{}\\azure_cli_test_result\\{}.xml'.format(os.path.expanduser('~'), mod_name), '-n', 'auto', '--pyargs', 'azure.cli.command_modules.{}'.format(mod_name)])
+    if exit_code != 0 and exit_code != 5:  # exit code is 5 when there is no tests collected in the module
+        sys.exit(exit_code)
+
+exit_code = subprocess.call(['python', '-m', 'pytest', '-x', '-v', '-p', 'no:warnings', '--log-level', 'WARN', 
+    '--junit-xml', '{}\\azure_cli_test_result\\azure-cli-core.xml'.format(os.path.expanduser('~')), '-n', 'auto', '--pyargs', 'azure.cli.core', '--import-mode=append'])
+sys.exit(exit_code)

scripts/release/rpm/azure-cli.spec

@@ -1,6 +1,10 @@
 # RPM spec file for Azure CLI
 # Definition of macros used - https://fedoraproject.org/wiki/Packaging:RPMMacros?rd=Packaging/RPMMacros
 
+%global __python %{__python3}
+# Turn off python byte compilation
+%global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g')
+
 # .el7.centos -> .el7
 %if 0%{?rhel}
   %define dist .el%{?rhel}

scripts/release/rpm/test_rpm_in_docker.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# This script should be run in a centos8 docker.
+set -exv
+
+yum --nogpgcheck localinstall /mnt/yum/$YUM_NAME -y
+
+yum install git -y
+
+ln -s /usr/bin/python3 /usr/bin/python
+ln -s /usr/bin/pip3 /usr/bin/pip
+time az self-test
+time az --version
+
+cd /azure-cli/
+pip3 install wheel
+./scripts/ci/build.sh
+pip3 install pytest --prefix /usr/lib64/az
+pip3 install pytest-xdist --prefix /usr/lib64/az
+
+find /azure-cli/artifacts/build -name "azure_cli_testsdk*" | xargs pip3 install --prefix /usr/lib64/az --upgrade --ignore-installed
+find /azure-cli/artifacts/build -name "azure_cli_fulltest*" | xargs pip3 install --prefix /usr/lib64/az --upgrade --ignore-installed --no-deps
+
+python3 /azure-cli/scripts/release/rpm/test_rpm_package.py

scripts/release/rpm/test_rpm_package.py

@@ -0,0 +1,25 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import os
+import sys
+import subprocess
+
+root_dir = '/usr/lib64/az/lib/python3.6/site-packages/azure/cli/command_modules'
+mod_list = [mod for mod in sorted(os.listdir(root_dir)) if os.path.isdir(os.path.join(root_dir, mod)) and mod != '__pycache__']
+
+pytest_base_cmd = 'PYTHONPATH=/usr/lib64/az/lib/python3.6/site-packages python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN'
+pytest_parallel_cmd = '{} -n auto'.format(pytest_base_cmd)
+
+for mod_name in mod_list:
+    if mod_name in ['botservice', 'network']:
+        exit_code = subprocess.call(['{} --junit-xml /azure_cli_test_result/{}.xml --pyargs azure.cli.command_modules.{}'.format(pytest_base_cmd, mod_name, mod_name)], shell=True)
+    else:
+        exit_code = subprocess.call(['{} --junit-xml /azure_cli_test_result/{}.xml --pyargs azure.cli.command_modules.{}'.format(pytest_parallel_cmd, mod_name, mod_name)], shell=True)
+    if exit_code != 0 and exit_code != 5:  # exit code is 5 when there is no tests collected in the module
+        sys.exit(exit_code)
+
+exit_code = subprocess.call(['{} --junit-xml /azure_cli_test_result/azure-cli-core.xml --pyargs azure.cli.core'.format(pytest_parallel_cmd)], shell=True)
+sys.exit(exit_code)