Faster command module loading & tab completion performance

doc/authoring_command_modules/README.md

@@ -45,7 +45,7 @@ thrown whilst attempting to load your module.
 <a name="heading_author_command_mod"></a>Authoring command modules
 ------
 Currently, all command modules should start with `azure-cli-`.  
-When the CLI loads, it search for packages installed via pip that start with that prefix.
+When the CLI loads, it search for packages installed that start with that prefix.
 
 The `example_module_template` directory gives a basic command module with 1 command.
 
@@ -65,6 +65,20 @@ Command modules should have the following structure:
 `-- setup.py
 ```
 
+**Create an \_\_init__.py for your module**
+
+In the \_\_init__ file, two methods need to be defined:
+  - `load_commands` - Uses the file in the 'Writing a Command' section below to load the commands.
+  - `load_params` - Uses the file in the 'Customizing Arguments' section below to load parameter customizations.
+
+```Python
+def load_params(command):
+    import azure.cli.command_modules.<module_name>._params
+
+def load_commands():
+    import azure.cli.command_modules.<module_name>.commands
+```
+
 ```python
 from azure.cli.commands import cli_command
 

doc/authoring_command_modules/authoring_commands.md

@@ -7,10 +7,11 @@ The document provides instructions and guidelines on how to author individual co
 
 The basic process of adding commands is presented below, and elaborated upon later in this document.
 
-1. Write your command as a standard Python function.
-2. Register your command using the `cli_command` (or similar) function.
-3. Write up your command's help entry.
-4. Use the `register_cli_argument` function to add the following enhancements to your arguments, as needed:
+1. Create an \_\_init__.py file for your command module.
+2. Write your command as a standard Python function.
+3. Register your command using the `cli_command` (or similar) function.
+4. Write up your command's help entry.
+5. Use the `register_cli_argument` function to add the following enhancements to your arguments, as needed:
     - option names, including short names
     - validators, actions or types
     - choice lists
@@ -34,11 +35,13 @@ from azure.cli.commands import cli_command
 
 The signature of this method is 
 ```Python
-def cli_command(name, operation, client_factory=None, transform=None, table_transformer=None):
+def cli_command(module_name, name, operation, client_factory=None, transform=None, table_transformer=None):
 ```
 You will generally only specify `name`, `operation` and possibly `table_transformer`.
+  - `module_name` - The name of the module that is registering the command (e.g. `azure.cli.command_modules.vm.commands`). Typically this will be `__name__`.
   - `name` - String uniquely naming your command and placing it within the command hierachy. It will be the string that you would type at the command line, omitting `az` (ex: access your command at `az mypackage mycommand` using a name of `mypackage mycommand`).
-  - `operation` - Your function's name.
+  - `operation` - The handler that will be executed. Format is `<module_to_import>#<attribute_list>`
+      - For example if `operation='azure.mgmt.compute.operations.virtual_machines_operations#VirtualMachinesOperations.get'`, the CLI will import `azure.mgmt.compute.operations.virtual_machines_operations`, get the `VirtualMachinesOperations` attribute and then the `get` attribute of `VirtualMachinesOperations`.
   - `table_transformer` (optional) - Supply a callable that takes, transforms and returns a result for table output.
 
 At this point, you should be able to access your command using `az [name]` and access the built-in help with `az [name] -h/--help`. Your command will automatically be 'wired up' with the global parameters.
@@ -99,7 +102,7 @@ The update commands within the CLI expose a set of generic update arguments: `--
 def cli_generic_update_command(name, getter, setter, factory=None, setter_arg_name='parameters',
                                table_transformer=None, child_collection_prop_name=None,
                                child_collection_key='name', child_arg_name='item_name',
-                               custom_function=None):
+                               custom_function_op=None):
 ```
 For many commands will only specify `name`, `getter`, `setter` and `factory`.
   - `name` - Same as registering a command with `cli_command(...)`.

src/azure-cli-core/azure/cli/core/application.py

@@ -8,7 +8,7 @@
 import os
 import uuid
 import argparse
-from azure.cli.core.parser import AzCliCommandParser
+from azure.cli.core.parser import AzCliCommandParser, enable_autocomplete
 from azure.cli.core._output import CommandResultItem
 import azure.cli.core.extensions
 import azure.cli.core._help as _help
@@ -29,15 +29,13 @@ def __init__(self, argv):
         self.argv = argv or sys.argv[1:]
         self.output_format = None
 
-    def get_command_table(self):
+    def get_command_table(self): # pylint: disable=no-self-use
         import azure.cli.core.commands as commands
-        # Find the first noun on the command line and only load commands from that
-        # module to improve startup time.
-        for a in self.argv:
-            if not a.startswith('-'):
-                return commands.get_command_table(a)
-        # No noun found, so load all commands.
-        return  commands.get_command_table()
+        return commands.get_command_table()
+
+    def load_params(self, command): # pylint: disable=no-self-use
+        import azure.cli.core.commands as commands
+        commands.load_params(command)
 
 class Application(object):
 
@@ -47,6 +45,7 @@ class Application(object):
     COMMAND_PARSER_LOADED = 'CommandParser.Loaded'
     COMMAND_PARSER_PARSED = 'CommandParser.Parsed'
     COMMAND_TABLE_LOADED = 'CommandTable.Loaded'
+    COMMAND_TABLE_PARAMS_LOADED = 'CommandTableParams.Loaded'
 
     def __init__(self, config=None):
         self._event_handlers = defaultdict(lambda: [])
@@ -85,6 +84,7 @@ def execute(self, unexpanded_argv):
         self.raise_event(self.COMMAND_PARSER_LOADED, parser=self.parser)
 
         if len(argv) == 0:
+            enable_autocomplete(self.parser)
             az_subparser = self.parser.subparsers[tuple()]
             _help.show_welcome(az_subparser)
             log_telemetry('welcome')
@@ -93,7 +93,24 @@ def execute(self, unexpanded_argv):
         if argv[0].lower() == 'help':
             argv[0] = '--help'
 
+        # Rudimentary parsing to get the command
+        nouns = []
+        for noun in argv:
+            if noun[0] == '-':
+                break
+            nouns.append(noun)
+        command = ' '.join(nouns)
+
+        if argv[-1] in ('--help', '-h') or command in command_table:
+            self.configuration.load_params(command)
+            self.raise_event(self.COMMAND_TABLE_PARAMS_LOADED, command_table=command_table)
+            self.parser.load_command_table(command_table)
+
+        if self.session['completer_active']:
+            enable_autocomplete(self.parser)
+
         args = self.parser.parse_args(argv)
+
         self.raise_event(self.COMMAND_PARSER_PARSED, command=args.command, args=args)
         results = []
         for expanded_arg in _explode_list_args(args):

src/azure-cli-core/azure/cli/core/commands/__init__.py

@@ -9,12 +9,15 @@
 import time
 import traceback
 import pkgutil
+import timeit
 from importlib import import_module
 from collections import OrderedDict, defaultdict
+from six import string_types
 
 from azure.cli.core._util import CLIError
 import azure.cli.core._logging as _logging
 from azure.cli.core.telemetry import log_telemetry
+from azure.cli.core.application import APPLICATION
 
 from ._introspection import (extract_args_from_signature,
                              extract_full_summary_from_signature)
@@ -142,14 +145,26 @@ def wrapped(func):
 
 class CliCommand(object):
 
-    def __init__(self, name, handler, description=None, table_transformer=None):
+    def __init__(self, name, handler, description=None, table_transformer=None,
+                 arguments_loader=None, description_loader=None):
         self.name = name
         self.handler = handler
-        self.description = description
         self.help = None
+        self.description = description_loader() \
+                           if description_loader and CliCommand._should_load_description() \
+                           else description
         self.arguments = {}
+        self.arguments_loader = arguments_loader
         self.table_transformer = table_transformer
 
+    @staticmethod
+    def _should_load_description():
+        return not APPLICATION.session['completer_active']
+
+    def load_arguments(self):
+        if self.arguments_loader:
+            self.arguments.update(self.arguments_loader())
+
     def add_argument(self, param_name, *option_strings, **kwargs):
         dest = kwargs.pop('dest', None)
         argument = CliCommandArgument(
@@ -165,42 +180,50 @@ def execute(self, **kwargs):
 
 command_table = CommandTable()
 
-def get_command_table(module_name=None):
+# Map to determine what module a command was registered in
+command_module_map = {}
+
+def load_params(command):
+    try:
+        command_table[command].load_arguments()
+    except KeyError:
+        return
+    command_module = command_module_map.get(command, None)
+    if not command_module:
+        logger.debug("Unable to load commands for '%s'. No module in command module map found.", command) #pylint: disable=line-too-long
+        return
+    module_to_load = command_module[:command_module.rfind('.')]
+    import_module(module_to_load).load_params(command)
+    _update_command_definitions(command_table)
+
+def get_command_table():
     '''Loads command table(s)
-    When `module_name` is specified, only commands from that module will be loaded.
-    If the module is not found, all commands are loaded.
     '''
-    loaded = False
-    # TODO Remove check for acs module. Issue #1110
-    if module_name and module_name != 'acs':
+    installed_command_modules = []
+    try:
+        mods_ns_pkg = import_module('azure.cli.command_modules')
+        installed_command_modules = [modname for _, modname, _ in \
+                                    pkgutil.iter_modules(mods_ns_pkg.__path__)]
+    except ImportError:
+        pass
+    logger.info('Installed command modules %s', installed_command_modules)
+    cumulative_elapsed_time = 0
+    for mod in installed_command_modules:
         try:
-            import_module('azure.cli.command_modules.' + module_name)
-            logger.info("Successfully loaded command table from module '%s'.", module_name)
-            loaded = True
-        except ImportError:
-            logger.info("Loading all installed modules as module with name '%s' not found.", module_name) #pylint: disable=line-too-long
+            start_time = timeit.default_timer()
+            import_module('azure.cli.command_modules.' + mod).load_commands()
+            elapsed_time = timeit.default_timer() - start_time
+            logger.debug("Loaded module '%s' in %.3f seconds.", mod, elapsed_time)
+            cumulative_elapsed_time += elapsed_time
         except Exception: #pylint: disable=broad-except
-            pass
-    if not loaded:
-        installed_command_modules = []
-        try:
-            mods_ns_pkg = import_module('azure.cli.command_modules')
-            installed_command_modules = [modname for _, modname, _ in \
-                                        pkgutil.iter_modules(mods_ns_pkg.__path__)]
-        except ImportError:
-            pass
-        logger.info('Installed command modules %s', installed_command_modules)
-        logger.info('Loading command tables from all installed modules.')
-        for mod in installed_command_modules:
-            try:
-                import_module('azure.cli.command_modules.' + mod)
-            except Exception: #pylint: disable=broad-except
-                # Changing this error message requires updating CI script that checks for failed
-                # module loading.
-                logger.error("Error loading command module '%s'", mod)
-                log_telemetry('Error loading module', module=mod)
-                logger.debug(traceback.format_exc())
-
+            # Changing this error message requires updating CI script that checks for failed
+            # module loading.
+            logger.error("Error loading command module '%s'", mod)
+            log_telemetry('Error loading module', module=mod)
+            logger.debug(traceback.format_exc())
+    logger.debug("Loaded all modules in %.3f seconds. "\
+                 "(note: there's always an overhead with the first module loaded)",
+                 cumulative_elapsed_time)
     _update_command_definitions(command_table)
     ordered_commands = OrderedDict(command_table)
     return ordered_commands
@@ -216,12 +239,28 @@ def register_extra_cli_argument(command, dest, **kwargs):
     '''
     _cli_extra_argument_registry[command][dest] = CliCommandArgument(dest, **kwargs)
 
-def cli_command(name, operation, client_factory=None, transform=None, table_transformer=None):
+def cli_command(module_name, name, operation,
+                client_factory=None, transform=None, table_transformer=None):
     """ Registers a default Azure CLI command. These commands require no special parameters. """
-    command_table[name] = create_command(name, operation, transform, table_transformer,
+    command_table[name] = create_command(module_name, name, operation, transform, table_transformer,
                                          client_factory)
 
-def create_command(name, operation, transform_result, table_transformer, client_factory):
+def get_op_handler(operation):
+    """ Import and load the operation handler """
+    try:
+        mod_to_import, attr_path = operation.split('#')
+        op = import_module(mod_to_import)
+        for part in attr_path.split('.'):
+            op = getattr(op, part)
+        return op
+    except (ValueError, AttributeError):
+        raise ValueError("The operation '{}' is invalid.".format(operation))
+
+def create_command(module_name, name, operation,
+                   transform_result, table_transformer, client_factory):
+
+    if not isinstance(operation, string_types):
+        raise ValueError("Operation must be a string. Got '{}'".format(operation))
 
     def _execute_command(kwargs):
         from msrest.paging import Paged
@@ -231,7 +270,8 @@ def _execute_command(kwargs):
 
         client = client_factory(kwargs) if client_factory else None
         try:
-            result = operation(client, **kwargs) if client else operation(**kwargs)
+            op = get_op_handler(operation)
+            result = op(client, **kwargs) if client else op(**kwargs)
             # apply results transform if specified
             if transform_result:
                 return transform_result(result)
@@ -255,13 +295,14 @@ def _execute_command(kwargs):
             log_telemetry('value exception', log_type='trace')
             raise CLIError(value_error)
 
+    command_module_map[name] = module_name
     name = ' '.join(name.split())
-    cmd = CliCommand(name, _execute_command, table_transformer=table_transformer)
-    cmd.description = extract_full_summary_from_signature(operation)
-    cmd.arguments.update(extract_args_from_signature(operation))
+    arguments_loader = lambda: extract_args_from_signature(get_op_handler(operation))
+    description_loader = lambda: extract_full_summary_from_signature(get_op_handler(operation))
+    cmd = CliCommand(name, _execute_command, table_transformer=table_transformer,
+                     arguments_loader=arguments_loader, description_loader=description_loader)
     return cmd
 
-
 def _get_cli_argument(command, argname):
     return _cli_argument_registry.get_cli_argument(command, argname)