From c03d7c91553d71f1d741085f1dee8ed12e6d9fee Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Tue, 30 Jun 2020 17:56:08 +0800
Subject: [PATCH 01/10] init benchmark command

---
 azdev/commands.py               |   1 +
 azdev/help.py                   |   3 +
 azdev/operations/performance.py | 117 ++++++++++++++++++++++++++++++--
 azdev/params.py                 |   2 +
 4 files changed, 118 insertions(+), 5 deletions(-)

diff --git a/azdev/commands.py b/azdev/commands.py
index c3f191998..bd8e4be7a 100644
--- a/azdev/commands.py
+++ b/azdev/commands.py
@@ -43,6 +43,7 @@ def operation_group(name):
 
     with CommandGroup(self, 'perf', operation_group('performance')) as g:
         g.command('load-times', 'check_load_time')
+        g.command('benchmark', 'benchmark', is_preview=True)
 
     with CommandGroup(self, 'extension', operation_group('extensions')) as g:
         g.command('add', 'add_extension')
diff --git a/azdev/help.py b/azdev/help.py
index c102d44d6..7af98f604 100644
--- a/azdev/help.py
+++ b/azdev/help.py
@@ -144,6 +144,9 @@
     short-summary: Verify that all modules load within an acceptable timeframe.
 """
 
+helps['perf benchmark'] = """
+    short-summary: Display benchmark staticstic of Azure CLI (Extensions) commands
+"""
 
 helps['extension'] = """
     short-summary: Control which CLI extensions are visible in the development environment.
diff --git a/azdev/operations/performance.py b/azdev/operations/performance.py
index 5cc7eec23..d700211d8 100644
--- a/azdev/operations/performance.py
+++ b/azdev/operations/performance.py
@@ -5,6 +5,7 @@
 # -----------------------------------------------------------------------------
 
 import re
+import timeit
 
 from knack.log import get_logger
 from knack.util import CLIError
@@ -104,19 +105,23 @@ def _claim_higher_threshold(val):
 FAILED: Some modules failed. If values are close to the threshold, rerun. If values
 are large, check that you do not have top-level imports like azure.mgmt or msrestazure
 in any modified files.
-""")
+"""
+        )
 
-    display('== PASSED MODULES ==')
+    display("== PASSED MODULES ==")
     display_table(passed_mods)
-    display('\nPASSED: Average load time all modules: {} ms'.format(
-        int(passed_mods[TOTAL]['average'])))
+    display(
+        "\nPASSED: Average load time all modules: {} ms".format(
+            int(passed_mods[TOTAL]["average"])
+        )
+    )
 
 
 def mean(data):
     """Return the sample arithmetic mean of data."""
     n = len(data)
     if n < 1:
-        raise ValueError('len < 1')
+        raise ValueError("len < 1")
     return sum(data) / float(n)
 
 
@@ -140,3 +145,105 @@ def display_table(data):
     for key, val in data.items():
         display('{:<20} {:>12.0f} {:>12.0f} {:>12.0f} {:>25}'.format(
             key, val['average'], val['threshold'], val['stdev'], str(val['values'])))
+
+
+# require azdev setup
+def benchmark(command_prefixes=None, top=20, runs=20):
+    if runs <= 0:
+        raise CLIError("Number of runs must be greater than 0.")
+
+    import multiprocessing
+    from azure.cli.core import get_default_cli
+    from azure.cli.core.file_util import create_invoker_and_load_cmds_and_args
+
+    def _process_pool_init():
+        import signal
+        def sigint_dummay_pass():
+            pass
+        signal.signal(signal.SIGINT, sigint_dummay_pass)
+
+    # load command table
+    az_cli = get_default_cli()
+    create_invoker_and_load_cmds_and_args(az_cli)
+    command_table = az_cli.invocation.commands_loader.command_table
+
+    line_head = "| {cmd:<35s} | {min:10s} | {max:10s} | {avg:10s} | {mid:10s} | {std:10s} | {runs:10s} |".format(
+        cmd="Command",
+        min="Min",
+        max="Max",
+        avg="Mean",
+        mid="Median",
+        std="Std",
+        runs="Runs",
+    )
+    line_tmpl = "| {cmd:<35s} | {min:10s} | {max:10s} | {avg:10s} | {mid:10s} | {std:10s} | {runs:10s} |"
+
+    logger.warning(line_head)
+    logger.warning("-" * 120)
+
+    # Measure every wanted commands
+    for raw_command in command_table:
+        cmd_tpl = "az {} -h --verbose".format(raw_command)
+
+        logger.info("Measuring %s...", raw_command)
+
+        pool = multiprocessing.Pool(multiprocessing.cpu_count(), _process_pool_init)
+        try:
+            time_series = pool.map_async(_benchmark_cmd_timer, [cmd_tpl] * runs).get(1000)
+        except multiprocessing.TimeoutError:
+            pool.terminate()
+            break
+        else:
+            pool.close()
+        pool.join()
+
+        staticstic = _benchmark_cmd_staticstic(time_series)
+
+        line_body = line_tmpl.format(
+            cmd=raw_command,
+            min=str(staticstic["min"]),
+            max=str(staticstic["max"]),
+            avg=str(staticstic["avg"]),
+            mid=str(staticstic["media"]),
+            std=str(staticstic["std"]),
+            runs=str(runs),
+        )
+        logger.warning(line_body)
+
+    logger.warning("-" * 100)
+
+
+def _benchmark_cmd_timer(cmd_tpl):
+    s = timeit.default_timer()
+    cmd(cmd_tpl)
+    e = timeit.default_timer()
+    return round(e - s, 4)
+
+
+def _benchmark_cmd_staticstic(time_series: list):
+    from math import sqrt
+
+    time_series.sort()
+
+    size = len(time_series)
+
+    if size % 2 == 0:
+        mid_time = (time_series[size // 2 - 1] + time_series[size // 2]) / 2
+    else:
+        mid_time = time_series[(size - 1) // 2]
+
+    min_time = time_series[0]
+    max_time = time_series[-1]
+    avg_time = sum(time_series) / size
+
+    std_deviation = sqrt(
+        sum([(t - avg_time) * (t - avg_time) for t in time_series]) / size
+    )
+
+    return {
+        "min": round(min_time, 4),
+        "max": round(max_time, 4),
+        "media": round(mid_time, 4),
+        "avg": round(avg_time, 4),
+        "std": round(std_deviation, 4),
+    }
diff --git a/azdev/params.py b/azdev/params.py
index 39982be85..a15ee784b 100644
--- a/azdev/params.py
+++ b/azdev/params.py
@@ -100,6 +100,8 @@ def load_arguments(self, _):
 
     with ArgumentsContext(self, 'perf') as c:
         c.argument('runs', type=int, help='Number of runs to average performance over.')
+        c.argument('commands', help="")
+        c.argument('top', type=int, help='Show N slowest commands. 0 for all.')
 
     with ArgumentsContext(self, 'extension') as c:
         c.argument('dist_dir', help='Name of a directory in which to save the resulting WHL files.')

From cd9576f8cfbc8dabaf35c4760659e0e3062c0adb Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Tue, 30 Jun 2020 18:06:30 +0800
Subject: [PATCH 02/10] fix wrong sigunature of singal callback

---
 azdev/operations/performance.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azdev/operations/performance.py b/azdev/operations/performance.py
index d700211d8..e2a662d7f 100644
--- a/azdev/operations/performance.py
+++ b/azdev/operations/performance.py
@@ -158,7 +158,7 @@ def benchmark(command_prefixes=None, top=20, runs=20):
 
     def _process_pool_init():
         import signal
-        def sigint_dummay_pass():
+        def sigint_dummay_pass(signal_num, frame):
             pass
         signal.signal(signal.SIGINT, sigint_dummay_pass)
 

From 7987160188c985069de3f08d14337131ce1c59ba Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Tue, 30 Jun 2020 18:07:59 +0800
Subject: [PATCH 03/10] refine help

---
 azdev/help.py                   | 2 +-
 azdev/operations/performance.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/azdev/help.py b/azdev/help.py
index 7af98f604..4e6558a66 100644
--- a/azdev/help.py
+++ b/azdev/help.py
@@ -145,7 +145,7 @@
 """
 
 helps['perf benchmark'] = """
-    short-summary: Display benchmark staticstic of Azure CLI (Extensions) commands
+    short-summary: Display benchmark staticstic of Azure CLI (Extensions) commands via execute it with -h.
 """
 
 helps['extension'] = """
diff --git a/azdev/operations/performance.py b/azdev/operations/performance.py
index e2a662d7f..eb2e68656 100644
--- a/azdev/operations/performance.py
+++ b/azdev/operations/performance.py
@@ -183,7 +183,7 @@ def sigint_dummay_pass(signal_num, frame):
 
     # Measure every wanted commands
     for raw_command in command_table:
-        cmd_tpl = "az {} -h --verbose".format(raw_command)
+        cmd_tpl = "az {} -h".format(raw_command)
 
         logger.info("Measuring %s...", raw_command)
 

From bc05eab887d3e2cba747a149be209313c792468a Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Tue, 30 Jun 2020 18:11:07 +0800
Subject: [PATCH 04/10] pylint: disable=unused-argument

---
 azdev/operations/performance.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azdev/operations/performance.py b/azdev/operations/performance.py
index eb2e68656..16504c407 100644
--- a/azdev/operations/performance.py
+++ b/azdev/operations/performance.py
@@ -158,7 +158,7 @@ def benchmark(command_prefixes=None, top=20, runs=20):
 
     def _process_pool_init():
         import signal
-        def sigint_dummay_pass(signal_num, frame):
+        def sigint_dummay_pass(signal_num, frame):  # pylint: disable=unused-argument
             pass
         signal.signal(signal.SIGINT, sigint_dummay_pass)
 

From 03020e9701fb8c2cb70cf22cca7c2b1987a1a098 Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Tue, 30 Jun 2020 19:03:46 +0800
Subject: [PATCH 05/10] add --prefix action and len auto-adjust

---
 azdev/help.py                   |  5 ++++-
 azdev/operations/actions.py     | 14 ++++++++++++++
 azdev/operations/performance.py | 26 ++++++++++++++++++++------
 azdev/params.py                 |  7 ++++++-
 4 files changed, 44 insertions(+), 8 deletions(-)
 create mode 100644 azdev/operations/actions.py

diff --git a/azdev/help.py b/azdev/help.py
index 4e6558a66..31149c867 100644
--- a/azdev/help.py
+++ b/azdev/help.py
@@ -145,7 +145,10 @@
 """
 
 helps['perf benchmark'] = """
-    short-summary: Display benchmark staticstic of Azure CLI (Extensions) commands via execute it with -h.
+    short-summary: Display benchmark staticstic of Azure CLI (Extensions) commands via execute it with -h in a separate process.
+    examples:
+        - name: Run benchmark on "network application-gateway" and "storage account"
+          text: azdev perf benchmark --prefix "network application-gateway" --prefix "storage account"
 """
 
 helps['extension'] = """
diff --git a/azdev/operations/actions.py b/azdev/operations/actions.py
new file mode 100644
index 000000000..937584556
--- /dev/null
+++ b/azdev/operations/actions.py
@@ -0,0 +1,14 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+
+from argparse import Action
+
+class PerfBenchmarkCommandPrefixAction(Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        if not namespace.command_prefixes:
+            namespace.command_prefixes = []
+
+        namespace.command_prefixes.append(' '.join(values))
diff --git a/azdev/operations/performance.py b/azdev/operations/performance.py
index 16504c407..065cc0df6 100644
--- a/azdev/operations/performance.py
+++ b/azdev/operations/performance.py
@@ -148,7 +148,7 @@ def display_table(data):
 
 
 # require azdev setup
-def benchmark(command_prefixes=None, top=20, runs=20):
+def benchmark(command_prefixes=None, runs=20):
     if runs <= 0:
         raise CLIError("Number of runs must be greater than 0.")
 
@@ -158,16 +158,31 @@ def benchmark(command_prefixes=None, top=20, runs=20):
 
     def _process_pool_init():
         import signal
+
         def sigint_dummay_pass(signal_num, frame):  # pylint: disable=unused-argument
             pass
+
         signal.signal(signal.SIGINT, sigint_dummay_pass)
 
     # load command table
     az_cli = get_default_cli()
     create_invoker_and_load_cmds_and_args(az_cli)
-    command_table = az_cli.invocation.commands_loader.command_table
+    raw_command_table = az_cli.invocation.commands_loader.command_table
+
+    command_table = []
+    if command_prefixes:
+        for k in raw_command_table:
+            if any(prefix for prefix in command_prefixes if k.startswith(prefix)):
+                command_table.append(k)
+    else:
+        command_table = list(raw_command_table.keys())
 
-    line_head = "| {cmd:<35s} | {min:10s} | {max:10s} | {avg:10s} | {mid:10s} | {std:10s} | {runs:10s} |".format(
+    max_len_cmd = max(command_table, key=len)
+
+    line_tmpl = "| {" + "cmd:" + "<" + str(len(max_len_cmd)) + "s} |"
+    line_tmpl = line_tmpl + " {min:10s} | {max:10s} | {avg:10s} | {mid:10s} | {std:10s} | {runs:10s} |"
+
+    line_head = line_tmpl.format(
         cmd="Command",
         min="Min",
         max="Max",
@@ -176,10 +191,9 @@ def sigint_dummay_pass(signal_num, frame):  # pylint: disable=unused-argument
         std="Std",
         runs="Runs",
     )
-    line_tmpl = "| {cmd:<35s} | {min:10s} | {max:10s} | {avg:10s} | {mid:10s} | {std:10s} | {runs:10s} |"
 
     logger.warning(line_head)
-    logger.warning("-" * 120)
+    logger.warning("-" * (80 + len(max_len_cmd)))
 
     # Measure every wanted commands
     for raw_command in command_table:
@@ -210,7 +224,7 @@ def sigint_dummay_pass(signal_num, frame):  # pylint: disable=unused-argument
         )
         logger.warning(line_body)
 
-    logger.warning("-" * 100)
+    logger.warning("-" * 120)
 
 
 def _benchmark_cmd_timer(cmd_tpl):
diff --git a/azdev/params.py b/azdev/params.py
index a15ee784b..243a5c81d 100644
--- a/azdev/params.py
+++ b/azdev/params.py
@@ -11,6 +11,7 @@
 
 from azdev.completer import get_test_completion
 from azdev.operations.linter import linter_severity_choices
+from azdev.operations.actions import PerfBenchmarkCommandPrefixAction
 
 
 class Flag(object):
@@ -100,7 +101,11 @@ def load_arguments(self, _):
 
     with ArgumentsContext(self, 'perf') as c:
         c.argument('runs', type=int, help='Number of runs to average performance over.')
-        c.argument('commands', help="")
+        c.argument('command_prefixes',
+                   nargs="+",
+                   action=PerfBenchmarkCommandPrefixAction,
+                   options_list="--prefix",
+                   help="Command prefix to run benchmark")
         c.argument('top', type=int, help='Show N slowest commands. 0 for all.')
 
     with ArgumentsContext(self, 'extension') as c:

From caea59b8332075b0496e31bbc21c7f7011a01fa5 Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Tue, 30 Jun 2020 19:06:23 +0800
Subject: [PATCH 06/10] refine a bit

---
 azdev/operations/performance.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/azdev/operations/performance.py b/azdev/operations/performance.py
index 065cc0df6..f2f169c8f 100644
--- a/azdev/operations/performance.py
+++ b/azdev/operations/performance.py
@@ -193,7 +193,7 @@ def sigint_dummay_pass(signal_num, frame):  # pylint: disable=unused-argument
     )
 
     logger.warning(line_head)
-    logger.warning("-" * (80 + len(max_len_cmd)))
+    logger.warning("-" * (85 + len(max_len_cmd)))
 
     # Measure every wanted commands
     for raw_command in command_table:
@@ -224,7 +224,7 @@ def sigint_dummay_pass(signal_num, frame):  # pylint: disable=unused-argument
         )
         logger.warning(line_body)
 
-    logger.warning("-" * 120)
+    logger.warning("-" * (85 + len(max_len_cmd)))
 
 
 def _benchmark_cmd_timer(cmd_tpl):

From d46988589cb87cc13313490589d92ea629552a3a Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Wed, 1 Jul 2020 12:49:24 +0800
Subject: [PATCH 07/10] fix pylint error

---
 azdev/operations/performance.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/azdev/operations/performance.py b/azdev/operations/performance.py
index f2f169c8f..f9f5fb691 100644
--- a/azdev/operations/performance.py
+++ b/azdev/operations/performance.py
@@ -105,8 +105,7 @@ def _claim_higher_threshold(val):
 FAILED: Some modules failed. If values are close to the threshold, rerun. If values
 are large, check that you do not have top-level imports like azure.mgmt or msrestazure
 in any modified files.
-"""
-        )
+""")
 
     display("== PASSED MODULES ==")
     display_table(passed_mods)

From 77f1923e524a8658dfe1218071cb0eaa004d6d64 Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Wed, 1 Jul 2020 13:12:23 +0800
Subject: [PATCH 08/10] fix flake8 error

---
 azdev/operations/actions.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/azdev/operations/actions.py b/azdev/operations/actions.py
index 937584556..f202df400 100644
--- a/azdev/operations/actions.py
+++ b/azdev/operations/actions.py
@@ -6,6 +6,7 @@
 
 from argparse import Action
 
+
 class PerfBenchmarkCommandPrefixAction(Action):
     def __call__(self, parser, namespace, values, option_string=None):
         if not namespace.command_prefixes:

From bf61b6651fb877791a3b475fa1ff2c46c072a85c Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Wed, 1 Jul 2020 13:35:21 +0800
Subject: [PATCH 09/10] add PerformanceCheck

---
 azure-pipelines.yml | 41 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 40 insertions(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index ab41a87cb..b9c920c9c 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -41,7 +41,7 @@ jobs:
     inputs:
       targetType: 'filePath'
       filePath: scripts/ci/run_tox.sh
-      
+
 - job: Tox38
   displayName: 'Tox: Python 3.8'
   condition: succeeded()
@@ -267,3 +267,42 @@ jobs:
       # verify azdev style works
       azdev style redis
     displayName: 'Test azdev style'
+
+- job: PerformanceCheck
+  displayName: "PerformanceCheck"
+  pool:
+    vmImage: 'ubuntu-16.04'
+  strategy:
+    matrix:
+      Python36:
+        python.version: '3.6'
+      Python38:
+        python.version: '3.8'
+  steps:
+  - task: UsePythonVersion@0
+    displayName: 'Use Python $(python.version)'
+    inputs:
+      versionSpec: '$(python.version)'
+  - bash: |
+      python -m venv env
+      chmod +x env/bin/activate
+      . env/bin/activate
+
+      pip install -e .
+      azdev --version
+
+      git clone https://github.com/Azure/azure-cli.git
+      git clone https://github.com/Azure/azure-cli-extensions.git
+
+      azdev setup -c ./azure-cli -r ./azure-cli-extensions
+    displayName: 'Azdev Setup'
+  - bash: |
+      set -ev
+      . env/bin/activate
+      azdev perf load-times
+    displayName: "Load Performance"
+  - bash: |
+      set -ev
+      . env/bin/activate
+      azdev perf benchmark --prefix "version" --prefix "network vnet"
+    displayName: "Execution Performance"

From b45a8b07a5bd2cd3b621f9168adc7a570fe8a96a Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Wed, 1 Jul 2020 13:49:20 +0800
Subject: [PATCH 10/10] fix typo

---
 azure-pipelines.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index b9c920c9c..f6ecbdbf3 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -304,5 +304,5 @@ jobs:
   - bash: |
       set -ev
       . env/bin/activate
-      azdev perf benchmark --prefix "version" --prefix "network vnet"
+      azdev perf benchmark --prefix "version" --prefix "network vnet "
     displayName: "Execution Performance"