YANG XPath Fixes (v2 for significant change) (#55)

* Use latest xpath parsing logic

* Fix YANG XPath usage

* Simplify jinja method

* Ignore empty xpath split

* Keep empty element in xpath builder

* Don't fully qualify by default

* v1 to v2 migration script

Author: remingtonc

etl/src/yang/__init__.py

@@ -161,17 +161,14 @@ def add_version_modules(db, os_key, version, modules):
 def add_data_paths_to_dm(db, dm_node, module, dp_parent=None):
     """Add the parsed DataPaths from the corresponding DataModels."""
     for _, path_data in module.items():
-        path_key = '%s%s' % (
-            path_data['base_module'],
-            path_data['xpath']
-        )
+        path_key = path_data['machine_id']
         path_node = None
         if path_key in dp_cache.keys():
             path_node = dp_cache[path_key]
         else:
             path_node = db['DataPath'].createDocument({
                 'machine_id': path_key,
-                'human_id': path_data['cisco_xpath'],
+                'human_id': path_data['xpath'],
                 'description': path_data['description'],
                 'is_leaf': False if path_data['children'] else True,
                 'is_variable': False,

etl/src/yang/yang_base.py

@@ -61,15 +61,14 @@ def parse_versions(self):
             for module_key, module_revision in modules.items():
                 module_data = version_data[module_key] = {}
                 for revision_key, module in module_revision.items():
-                    module_data[revision_key] = self.parse_module_oper_attrs(module, module)
+                    module_data[revision_key] = self.parse_module_oper_attrs(module)
                 logging.debug('Parsed %d revision(s) for %s.', len(module_data.keys()), module_key)
             logging.debug('Parsed %d module(s) for %s.', len(version_data.keys()), version)
         logging.debug('Parsed %d version(s).', len(version_module_map.keys()))
         return version_module_map
 
-    def parse_module_oper_attrs(self, module, base_module):
+    def parse_module_oper_attrs(self, module):
         """Parse out the readable DataPaths from parsed data models.
-        TODO: Validate the i_config filtering with pyang.
         """
         if not hasattr(module, 'i_children'):
             return {}
@@ -80,19 +79,14 @@ def parse_module_oper_attrs(self, module, base_module):
         parsed_modules = {}
         for child in module_children:
             attr_dict = {
-                'base_module': base_module.arg,
-                'xpath': yang_parser.get_xpath(child, with_prefixes=True),
-                'cisco_xpath': yang_parser.get_cisco_xpath(child, base_module),
+                'machine_id': '/%s' % ('/'.join(map(lambda x: ':'.join(x), yang_parser.mk_path_list(child)))),
+                'qualified_xpath': yang_parser.get_xpath(child, qualified=True, prefix_to_module=True),
+                'xpath': yang_parser.get_xpath(child, prefix_to_module=True),
                 'type': yang_parser.get_qualified_type(child),
                 'primitive_type': yang_parser.get_primitive_type(child),
                 'rw': True if getattr(child, 'i_config', False) else False,
                 'description': yang_parser.get_description(child),
-                'children': self.parse_module_oper_attrs(child, base_module)
+                'children': self.parse_module_oper_attrs(child)
             }
-            # Qualify based on module and xpath, not cisco_xpath.
-            # Would not account for derivation/augmentations.
-            parsed_modules[str((
-                attr_dict['base_module'],
-                attr_dict['xpath']
-            ))] = attr_dict
+            parsed_modules[attr_dict['machine_id']] = attr_dict
         return parsed_modules

etl/src/yang/yang_parser.py

@@ -93,31 +93,64 @@ def get_filtered_modules(context, filter_pattern):
         filtered_modules[module_name] = revs
     return filtered_modules
 
-def get_cisco_xpath(module, base_module):
-    """Generate the Cisco XPath representation.
-    Cleaner but less absolute.
+def mk_path_list(stmt):
+    """Derives a list of tuples containing
+    (module name, prefix, xpath)
+    per node in the statement.
     """
-    module_name = base_module.arg
-    no_prefix_xpath = get_xpath(module, with_prefixes=False)
-    cisco_xpath = '%s:%s' % (module_name, no_prefix_xpath[1:])
-    return cisco_xpath
-
-def mk_path_str(stmt, with_prefixes=False):
-    """Returns the XPath path of the node"""
-    if stmt.keyword in ['choice', 'case']:
-        return mk_path_str(stmt.parent, with_prefixes)
-    def name(stmt):
-        if with_prefixes:
-            return '%s:%s' % (stmt.i_module.i_prefix, stmt.arg)
+    resolved_names = []
+    def resolve_stmt(stmt, resolved_names):
+        if stmt.keyword in ['choice', 'case']:
+            resolve_stmt(stmt.parent, resolved_names)
+            return
+        def qualified_name_elements(stmt):
+            """(module name, prefix, name)"""
+            return (stmt.i_module.arg, stmt.i_module.i_prefix, stmt.arg)
+        if stmt.parent.keyword in ['module', 'submodule']:
+            resolved_names.append(qualified_name_elements(stmt))
+            return
         else:
-            return stmt.arg
-    if stmt.parent.keyword in ['module', 'submodule']:
-        return '/%s' % name(stmt)
-    else:
-        xpath = mk_path_str(stmt.parent, with_prefixes)
-        return '%s/%s' % (xpath, name(stmt))
+            resolve_stmt(stmt.parent, resolved_names)
+            resolved_names.append(qualified_name_elements(stmt))
+            return
+    resolve_stmt(stmt, resolved_names)
+    return resolved_names
 
-get_xpath = mk_path_str
+def mk_path_str(stmt, with_prefixes=False, prefix_onchange=False, prefix_to_module=False, resolve_top_prefix_to_module=False):
+    """Returns the XPath path of the node.
+    with_prefixes indicates whether or not to prefix every node.
+    prefix_onchange modifies the behavior of with_prefixes and only adds prefixes when the prefix changes mid-XPath.
+    prefix_to_module replaces prefixes with the module name of the prefix.
+    resolve_top_prefix_to_module resolves the module-level prefix to the module name.
+    Prefixes may be included in the path if the prefix changes mid-path.
+    """
+    resolved_names = mk_path_list(stmt)
+    xpath_elements = []
+    last_prefix = None
+    for index, resolved_name in enumerate(resolved_names):
+        module_name, prefix, node_name = resolved_name
+        xpath_element = node_name
+        if with_prefixes or (prefix_onchange and prefix != last_prefix):
+            new_prefix = prefix
+            if prefix_to_module or (index == 0 and resolve_top_prefix_to_module):
+                new_prefix = module_name
+            xpath_element = '%s:%s' % (new_prefix, node_name)
+        xpath_elements.append(xpath_element)
+        last_prefix = prefix
+    return '/%s' % '/'.join(xpath_elements)
+
+def get_xpath(stmt, qualified=False, prefix_to_module=False):
+    """Gets the XPath of the statement.
+    Unless qualified=True, does not include prefixes unless the prefix changes mid-XPath.
+    qualified will add a prefix to each node.
+    prefix_to_module will resolve prefixes to module names instead.
+    For RFC 8040, set prefix_to_module=True.
+    /prefix:root/node/prefix:node/...
+    qualified=True: /prefix:root/prefix:node/prefix:node/...
+    qualified=True, prefix_to_module=True: /module:root/module:node/module:node/...
+    prefix_to_module=True: /module:root/node/module:node/...
+    """
+    return mk_path_str(stmt, with_prefixes=qualified, prefix_onchange=True, prefix_to_module=prefix_to_module)
 
 def get_type(stmt):
     """Gets the immediate, top-level type of the node.

migrate/1_to_2/.gitignore

@@ -0,0 +1,2 @@
+yang/
+*.json

migrate/1_to_2/Pipfile

@@ -0,0 +1,12 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+pyang = "==1.7.5"
+
+[requires]
+python_version = "3.7"

migrate/1_to_2/README.md

@@ -0,0 +1,2 @@
+# v1.x to 2.0
+This migration is purely for migrating mappings within TDM from v1.x to v2.0 format. v2.0 expresses both modules and prefixes in the `machine_id` and breaks v1.x mapping portability. This script will translate v1.x mapping `machine_id` fields to the v2.x format.

migrate/1_to_2/mappings.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+"""Copyright 2019 Cisco Systems
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+"""Extremely ugly migration."""
+import logging
+import json
+import yang_parser
+
+def parse_repository(repo_path):
+    modules = yang_parser.parse_repository(repo_path)
+    parsed_modules = {}
+    for module_key, module_revision in modules.items():
+        revision_with_data = set()
+        for revision_key, module in module_revision.items():
+            revision_data = None
+            try:
+                revision_data = __parse_node_attrs(module, module)
+            except Exception:
+                logging.exception("Failure while parsing %s!", module_key)
+            if not revision_data:
+                logging.debug("%s@%s is empty.", module_key, revision_key)
+                continue
+            revision_with_data.add("%s@%s" % (module_key, revision_key))
+            if module_key in parsed_modules.keys() and parsed_modules[module_key]:
+                logging.warn(
+                    "%s being replaced with %s@%s! Only one revision should be used.",
+                    module_key,
+                    module_key,
+                    revision_key,
+                )
+            parsed_modules[module_key] = revision_data
+        if revision_with_data:
+            logging.debug("%s have data.", ", ".join(revision_with_data))
+        else:
+            logging.debug("%s has no revisions with data.", module_key)
+    return parsed_modules
+
+
+def __parse_node_attrs(node, base_module):
+    if not hasattr(node, "i_children"):
+        return {}
+    children = (
+        child
+        for child in node.i_children
+        if child.keyword in yang_parser.statements.data_definition_keywords
+    )
+    parsed_children = {}
+    multi_map = {}
+    for child in children:
+        qualified_xpath = yang_parser.get_xpath(
+            child, qualified=True, prefix_to_module=True
+        )
+        if qualified_xpath in parsed_children.keys():
+            logging.debug("%s encountered more than once! Muxing." % qualified_xpath)
+            if qualified_xpath not in multi_map.keys():
+                multi_map[qualified_xpath] = 0
+            multi_map[qualified_xpath] += 1
+            qualified_xpath += "_%i" % multi_map[qualified_xpath]
+        attr_dict = {
+            '1_machine_id': '%s%s' % (base_module.arg, yang_parser.old_get_xpath(child, with_prefixes=True)),
+            '2_machine_id': '/%s' % ('/'.join(map(lambda x: ':'.join(x), yang_parser.mk_path_list(child)))),
+            'children': __parse_node_attrs(child, base_module)
+        }
+        parsed_children[qualified_xpath] = attr_dict
+    return parsed_children
+
+logging.basicConfig(level=logging.INFO)
+
+upgrade_map = {}
+
+# Any paths come out missing, go to instance of TDM
+# Datapath Direct the path and add path to version folder
+repository_paths = [
+    'yang/vendor/cisco/nx/9.3-1',
+    'yang/vendor/cisco/xe/16111',
+    'yang/vendor/cisco/xr/602',
+    'yang/vendor/cisco/xr/631',
+    'yang/vendor/cisco/xr/662'
+]
+
+found = 0
+missing = 0
+
+def upgrade(key):
+    if key in upgrade_map.keys():
+        global found
+        found += 1
+        return upgrade_map[key]
+    else:
+        global missing
+        if not key.startswith('1.'):
+            missing += 1
+            logging.warning('%s not in upgrade map!', key)
+        return key
+
+for repo in repository_paths:
+    logging.info('Parsing %s ...', repo)
+    modules = parse_repository(repo)
+    def recurse_nodes(node_tree):
+        for node_element in node_tree.values():
+            upgrade_map[node_element['1_machine_id']] = node_element['2_machine_id']
+            recurse_nodes(node_element['children'])
+    module_top_children = {}
+    for top_children in modules.values():
+        module_top_children.update(top_children)
+    recurse_nodes(module_top_children)
+tdm_mappings = None
+with open('tdm_mappings.json', 'r') as mappings_fd:
+    tdm_mappings = json.load(mappings_fd)
+logging.info('Upgrading DataPathMatch ...')
+for match in tdm_mappings['DataPathMatch']:
+    match['_from'] = upgrade(match['_from'])
+    match['_to'] = upgrade(match['_to'])
+logging.info('%i found, %i missing.', found, missing)
+found = 0
+missing = 0
+logging.info('Upgrading Calculation ...')
+for calculation in tdm_mappings['Calculation']:
+    new_in_calculation = []
+    for path in calculation['InCalculation']:
+        new_in_calculation.append(upgrade(path))
+    calculation['InCalculation'] = new_in_calculation
+    new_result = []
+    for path in calculation['CalculationResult']:
+        new_result.append(upgrade(path))
+    calculation['CalculationResult'] = new_result
+logging.info('%i found, %i missing.', found, missing)
+with open('upgraded_tdm_mappings.json', 'w') as upgraded_fd:
+    json.dump(tdm_mappings, upgraded_fd, indent=4, sort_keys=True)

migrate/1_to_2/run.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+pipenv run python mappings.py

migrate/1_to_2/setup.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+git clone https://github.com/YangModels/yang
+pipenv --three install