From e24eca57bafe82726df23862e7eb58c866604451 Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Mon, 29 Jun 2020 17:34:04 -0700 Subject: [PATCH] =?UTF-8?q?[=5Fsonic=5Fyang=5Fext.py]:=20Using=20python=20?= =?UTF-8?q?mixin=20to=20extend=20sonic=5Fyang=20class=20i=E2=80=A6=20(#65)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [_sonic_yang_ext.py]: Using python mixin to extend sonic_yang class into 2 files. Changes: 1.) Moved al functions of _sonic_yang_ext.py file in sonic_yang_ext_mixin class. 2.) Use this class in sonic_yang to extend funtionality. Note: Python mixin are different from parent class since it can access all variables of the class which is using it. Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com * [sonic-yang-mgmt]: Resolving LGTM. Changes: To address 'import *' related LGTM alerts 1.) Moved all functions of _sonic_yang_ext.py file in sonic_yang_ext_mixin class. 2.) Use this class in sonic_yang to extend funtionality, i.e. python mixin. Note: Python mixin are different from parent class since it can access all variables of the class which is using it. Other LGTM issues 3.) Address Other LGTM minor issues found in below link. https://lgtm.com/projects/g/Azure/sonic-buildimage/rev/pr-0f82616403c02577e1155347eb5b693a51c76a9e Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com --- src/sonic-yang-mgmt/_sonic_yang_ext.py | 1244 ++++++++--------- src/sonic-yang-mgmt/setup.py | 17 +- src/sonic-yang-mgmt/sonic_yang.py | 24 +- .../libyang-python-tests/test_sonic_yang.py | 13 +- 4 files changed, 643 insertions(+), 655 deletions(-) diff --git a/src/sonic-yang-mgmt/_sonic_yang_ext.py b/src/sonic-yang-mgmt/_sonic_yang_ext.py index c0e04c217406..8c69f56faf6b 100644 --- a/src/sonic-yang-mgmt/_sonic_yang_ext.py +++ b/src/sonic-yang-mgmt/_sonic_yang_ext.py @@ -3,658 +3,656 @@ import yang as ly import re -import pprint import syslog -from json import dump, load, dumps, loads +from json import dump, dumps, loads from xmltodict import parse -from os import listdir, walk, path -from os.path import isfile, join, splitext from glob import glob -# class sonic_yang methods - -""" -load all YANG models, create JSON of yang models -""" -def loadYangModel(self): - - try: - # get all files - self.yangFiles = glob(self.yang_dir +"/*.yang") - # load yang modules - for file in self.yangFiles: - m = self.load_schema_module(file) - if m is not None: - self.sysLog(msg="module: {} is loaded successfully".format(m.name())) +# class sonic_yang methods, use mixin to extend sonic_yang +class sonic_yang_ext_mixin: + + """ + load all YANG models, create JSON of yang models + """ + def loadYangModel(self): + + try: + # get all files + self.yangFiles = glob(self.yang_dir +"/*.yang") + # load yang modules + for file in self.yangFiles: + m = self.load_schema_module(file) + if m is not None: + self.sysLog(msg="module: {} is loaded successfully".format(m.name())) + else: + raise(Exception("Could not load module {}".format(file))) + + # keep only modules name in self.yangFiles + self.yangFiles = [f.split('/')[-1] for f in self.yangFiles] + self.yangFiles = [f.split('.')[0] for f in self.yangFiles] + print('Loaded below Yang Models') + print(self.yangFiles) + + # load json for each yang model + self.loadJsonYangModel() + # create a map from config DB table to yang container + self.createDBTableToModuleMap() + + except Exception as e: + print("Yang Models Load failed") + raise e + + return True + + """ + load JSON schema format from yang models + """ + def loadJsonYangModel(self): + + try: + for f in self.yangFiles: + m = self.ctx.get_module(f) + if m is not None: + xml = m.print_mem(ly.LYD_JSON, ly.LYP_FORMAT) + self.yJson.append(parse(xml)) + self.sysLog(msg="Parsed Json for {}".format(m.name())) + except Exception as e: + print('JSON conversion for yang models failed') + raise e + + return + + """ + Create a map from config DB tables to container in yang model + This module name and topLevelContainer are fetched considering YANG models are + written using below Guidelines: + https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md. + """ + def createDBTableToModuleMap(self): + + for j in self.yJson: + # get module name + moduleName = j['module']['@name'] + # topLevelContainer does not exist in sonic-head and sonic-extension. + if "sonic-head" in moduleName or "sonic-extension" in moduleName: + continue; + # get all top level container + topLevelContainer = j['module']['container'] + if topLevelContainer is None: + raise Exception("topLevelContainer not found") + + assert topLevelContainer['@name'] == moduleName + + container = topLevelContainer['container'] + # container is a list + if isinstance(container, list): + for c in container: + self.confDbYangMap[c['@name']] = { + "module" : moduleName, + "topLevelContainer": topLevelContainer['@name'], + "container": c + } + # container is a dict else: - raise(Exception("Could not load module {}".format(file))) - - # keep only modules name in self.yangFiles - self.yangFiles = [f.split('/')[-1] for f in self.yangFiles] - self.yangFiles = [f.split('.')[0] for f in self.yangFiles] - print('Loaded below Yang Models') - print(self.yangFiles) - - # load json for each yang model - self.loadJsonYangModel() - # create a map from config DB table to yang container - self.createDBTableToModuleMap() - - except Exception as e: - print("Yang Models Load failed") - raise e - - return True - -""" -load JSON schema format from yang models -""" -def loadJsonYangModel(self): - - try: - for f in self.yangFiles: - m = self.ctx.get_module(f) - if m is not None: - xml = m.print_mem(ly.LYD_JSON, ly.LYP_FORMAT) - self.yJson.append(parse(xml)) - self.sysLog(msg="Parsed Json for {}".format(m.name())) - except Exception as e: - print('JSON conversion for yang models failed') - raise e - - return - -""" -Create a map from config DB tables to container in yang model -This module name and topLevelContainer are fetched considering YANG models are -written using below Guidelines: -https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md. -""" -def createDBTableToModuleMap(self): - - for j in self.yJson: - # get module name - moduleName = j['module']['@name'] - # topLevelContainer does not exist in sonic-head and sonic-extension. - if "sonic-head" in moduleName or "sonic-extension" in moduleName: - continue; - # get all top level container - topLevelContainer = j['module']['container'] - if topLevelContainer is None: - raise Exception("topLevelContainer not found") - - assert topLevelContainer['@name'] == moduleName - - container = topLevelContainer['container'] - # container is a list - if isinstance(container, list): - for c in container: - self.confDbYangMap[c['@name']] = { + self.confDbYangMap[container['@name']] = { "module" : moduleName, "topLevelContainer": topLevelContainer['@name'], - "container": c + "container": container } - # container is a dict + return + + """ + Get module, topLevelContainer(TLC) and json container for a config DB table + """ + def get_module_TLC_container(self, table): + cmap = self.confDbYangMap + m = cmap[table]['module'] + t = cmap[table]['topLevelContainer'] + c = cmap[table]['container'] + return m, t, c + + """ + Crop config as per yang models, + This Function crops from config only those TABLEs, for which yang models is + provided. + """ + def cropConfigDB(self, croppedFile=None, allowExtraTables=True): + + for table in self.jIn.keys(): + if table not in self.confDbYangMap: + if allowExtraTables: + del self.jIn[table] + else: + raise(Exception("No Yang Model Exist for {}".format(table))) + + if croppedFile: + with open(croppedFile, 'w') as f: + dump(self.jIn, f, indent=4) + + return + + """ + Extract keys from table entry in Config DB and return in a dict + + Input: + tableKey: Config DB Primary Key, Example tableKey = "Vlan111|2a04:5555:45:6709::1/64" + keys: key string from YANG list, i.e. 'vlan_name ip-prefix'. + regex: A regex to extract keys from tableKeys, good to have it as accurate as possible. + + Return: + KeyDict = {"vlan_name": "Vlan111", "ip-prefix": "2a04:5555:45:6709::1/64"} + """ + def extractKey(self, tableKey, keys, regex): + + keyList = keys.split() + # get the value groups + value = re.match(regex, tableKey) + # create the keyDict + i = 1 + keyDict = dict() + for k in keyList: + if value.group(i): + keyDict[k] = value.group(i) + else: + raise Exception("Value not found for {} in {}".format(k, tableKey)) + i = i + 1 + + return keyDict + + """ + Fill the dict based on leaf as a list or dict @model yang model object + """ + def fillLeafDict(self, leafs, leafDict, isleafList=False): + + if leafs is None: + return + + # fill default values + def fillSteps(leaf): + leaf['__isleafList'] = isleafList + leafDict[leaf['@name']] = leaf + return + + if isinstance(leafs, list): + for leaf in leafs: + #print("{}:{}".format(leaf['@name'], leaf)) + fillSteps(leaf) else: - self.confDbYangMap[container['@name']] = { - "module" : moduleName, - "topLevelContainer": topLevelContainer['@name'], - "container": container - } - return - -""" -Get module, topLevelContainer(TLC) and json container for a config DB table -""" -def get_module_TLC_container(self, table): - cmap = self.confDbYangMap - m = cmap[table]['module'] - t = cmap[table]['topLevelContainer'] - c = cmap[table]['container'] - return m, t, c - -""" -Crop config as per yang models, -This Function crops from config only those TABLEs, for which yang models is -provided. -""" -def cropConfigDB(self, croppedFile=None, allowExtraTables=True): - - for table in self.jIn.keys(): - if table not in self.confDbYangMap: - if allowExtraTables: - del self.jIn[table] + #print("{}:{}".format(leaf['@name'], leaf)) + fillSteps(leafs) + + return + + """ + create a dict to map each key under primary key with a dict yang model. + This is done to improve performance of mapping from values of TABLEs in + config DB to leaf in YANG LIST. + """ + def createLeafDict(self, model): + + leafDict = dict() + #Iterate over leaf, choices and leaf-list. + self.fillLeafDict(model.get('leaf'), leafDict) + + #choices, this is tricky, since leafs are under cases in tree. + choices = model.get('choice') + if choices: + for choice in choices: + cases = choice['case'] + for case in cases: + self.fillLeafDict(case.get('leaf'), leafDict) + + # leaf-lists + self.fillLeafDict(model.get('leaf-list'), leafDict, True) + + return leafDict + + """ + Convert a string from Config DB value to Yang Value based on type of the + key in Yang model. + @model : A List of Leafs in Yang model list + """ + def findYangTypedValue(self, key, value, leafDict): + + # convert config DB string to yang Type + def yangConvert(val): + # Convert everything to string + val = str(val) + # find type of this key from yang leaf + type = leafDict[key]['type']['@name'] + + if 'uint' in type: + vValue = int(val, 10) + # TODO: find type of leafref from schema node + elif 'leafref' in type: + vValue = val + #TODO: find type in sonic-head, as of now, all are enumeration + elif 'head:' in type: + vValue = val else: - raise(Exception("No Yang Model Exist for {}".format(table))) - - if croppedFile: - with open(croppedFile, 'w') as f: - dump(self.jIn, f, indent=4) - - return - -""" -Extract keys from table entry in Config DB and return in a dict - -Input: -tableKey: Config DB Primary Key, Example tableKey = "Vlan111|2a04:5555:45:6709::1/64" -keys: key string from YANG list, i.e. 'vlan_name ip-prefix'. -regex: A regex to extract keys from tableKeys, good to have it as accurate as possible. - -Return: -KeyDict = {"vlan_name": "Vlan111", "ip-prefix": "2a04:5555:45:6709::1/64"} -""" -def extractKey(self, tableKey, keys, regex): - - keyList = keys.split() - # get the value groups - value = re.match(regex, tableKey) - # create the keyDict - i = 1 - keyDict = dict() - for k in keyList: - if value.group(i): - keyDict[k] = value.group(i) + vValue = val + return vValue + + # if it is a leaf-list do it for each element + if leafDict[key]['__isleafList']: + vValue = list() + for v in value: + vValue.append(yangConvert(v)) else: - raise Exception("Value not found for {} in {}".format(k, tableKey)) - i = i + 1 + vValue = yangConvert(value) - return keyDict + return vValue -""" -Fill the dict based on leaf as a list or dict @model yang model object -""" -def fillLeafDict(self, leafs, leafDict, isleafList=False): + """ + Xlate a list + This function will xlate from a dict in config DB to a Yang JSON list + using yang model. Output will be go in self.xlateJson + """ + def xlateList(self, model, yang, config, table): + + #create a dict to map each key under primary key with a dict yang model. + #This is done to improve performance of mapping from values of TABLEs in + #config DB to leaf in YANG LIST. + leafDict = self.createLeafDict(model) + + # fetch regex from YANG models. + keyRegEx = model['ext:key-regex-configdb-to-yang']['@value'] + # seperator `|` has special meaning in regex, so change it appropriately. + keyRegEx = re.sub('\|', '\\|', keyRegEx) + # get keys from YANG model list itself + listKeys = model['key']['@value'] + self.sysLog(msg="xlateList regex:{} keyList:{}".\ + format(keyRegEx, listKeys)) + + for pkey in config.keys(): + try: + vKey = None + self.sysLog(syslog.LOG_DEBUG, "xlateList Extract pkey:{}".\ + format(pkey)) + # Find and extracts key from each dict in config + keyDict = self.extractKey(pkey, listKeys, keyRegEx) + # fill rest of the values in keyDict + for vKey in config[pkey]: + self.sysLog(syslog.LOG_DEBUG, "xlateList vkey {}".format(vKey)) + keyDict[vKey] = self.findYangTypedValue(vKey, \ + config[pkey][vKey], leafDict) + yang.append(keyDict) + # delete pkey from config, done to match one key with one list + del config[pkey] + + except Exception as e: + # log debug, because this exception may occur with multilists + self.sysLog(syslog.LOG_DEBUG, "xlateList Exception {}".format(e)) + # with multilist, we continue matching other keys. + continue - if leafs == None: return - # fill default values - def fillSteps(leaf): - leaf['__isleafList'] = isleafList - leafDict[leaf['@name']] = leaf + """ + Xlate a container + This function will xlate from a dict in config DB to a Yang JSON container + using yang model. Output will be stored in self.xlateJson + """ + def xlateContainer(self, model, yang, config, table): + + # To Handle multiple List, Make a copy of config, because we delete keys + # from config after each match. This is done to match one pkey with one list. + configC = config.copy() + + clist = model.get('list') + # If single list exists in container, + if clist and isinstance(clist, dict) and \ + clist['@name'] == model['@name']+"_LIST" and bool(configC): + #print(clist['@name']) + yang[clist['@name']] = list() + self.sysLog(msg="xlateContainer listD {}".format(clist['@name'])) + self.xlateList(clist, yang[clist['@name']], \ + configC, table) + # clean empty lists + if len(yang[clist['@name']]) == 0: + del yang[clist['@name']] + #print(yang[clist['@name']]) + + # If multi-list exists in container, + elif clist and isinstance(clist, list) and bool(configC): + for modelList in clist: + yang[modelList['@name']] = list() + self.sysLog(msg="xlateContainer listL {}".format(modelList['@name'])) + self.xlateList(modelList, yang[modelList['@name']], configC, table) + # clean empty lists + if len(yang[modelList['@name']]) == 0: + del yang[modelList['@name']] + + if len(configC): + self.sysLog(syslog.LOG_ERR, "Alert: Remaining keys in Config") + raise(Exception("All Keys are not parsed in {}".format(table))) + return - if isinstance(leafs, list): - for leaf in leafs: - #print("{}:{}".format(leaf['@name'], leaf)) - fillSteps(leaf) - else: - #print("{}:{}".format(leaf['@name'], leaf)) - fillSteps(leafs) - - return - -""" -create a dict to map each key under primary key with a dict yang model. -This is done to improve performance of mapping from values of TABLEs in -config DB to leaf in YANG LIST. -""" -def createLeafDict(self, model): - - leafDict = dict() - #Iterate over leaf, choices and leaf-list. - self.fillLeafDict(model.get('leaf'), leafDict) - - #choices, this is tricky, since leafs are under cases in tree. - choices = model.get('choice') - if choices: - for choice in choices: - cases = choice['case'] - for case in cases: - self.fillLeafDict(case.get('leaf'), leafDict) - - # leaf-lists - self.fillLeafDict(model.get('leaf-list'), leafDict, True) - - return leafDict - -""" -Convert a string from Config DB value to Yang Value based on type of the -key in Yang model. -@model : A List of Leafs in Yang model list -""" -def findYangTypedValue(self, key, value, leafDict): - - # convert config DB string to yang Type - def yangConvert(val): - # Convert everything to string - val = str(val) - # find type of this key from yang leaf - type = leafDict[key]['type']['@name'] - - if 'uint' in type: - vValue = int(val, 10) - # TODO: find type of leafref from schema node - elif 'leafref' in type: - vValue = val - #TODO: find type in sonic-head, as of now, all are enumeration - elif 'head:' in type: - vValue = val + """ + xlate ConfigDB json to Yang json + """ + def xlateConfigDBtoYang(self, jIn, yangJ): + + # find top level container for each table, and run the xlate_container. + for table in jIn.keys(): + cmap = self.confDbYangMap[table] + # create top level containers + key = cmap['module']+":"+cmap['topLevelContainer'] + subkey = cmap['topLevelContainer']+":"+cmap['container']['@name'] + # Add new top level container for first table in this container + yangJ[key] = dict() if yangJ.get(key) is None else yangJ[key] + yangJ[key][subkey] = dict() + self.sysLog(msg="xlateConfigDBtoYang {}:{}".format(key, subkey)) + self.xlateContainer(cmap['container'], yangJ[key][subkey], \ + jIn[table], table) + + return + + """ + Read config file and crop it as per yang models + """ + def xlateConfigDB(self, xlateFile=None): + + jIn= self.jIn + yangJ = self.xlateJson + # xlation is written in self.xlateJson + self.xlateConfigDBtoYang(jIn, yangJ) + + if xlateFile: + with open(xlateFile, 'w') as f: + dump(self.xlateJson, f, indent=4) + + return + + """ + create config DB table key from entry in yang JSON + """ + def createKey(self, entry, regex): + + keyDict = dict() + keyV = regex + # get the keys from regex of key extractor + keyList = re.findall(r'<(.*?)>', regex) + for key in keyList: + val = entry.get(key) + if val: + #print("pair: {} {}".format(key, val)) + keyDict[key] = sval = str(val) + keyV = re.sub(r'<'+key+'>', sval, keyV) + #print("VAL: {} {}".format(regex, keyV)) + else: + raise Exception("key {} not found in entry".format(key)) + #print("kDict {}".format(keyDict)) + return keyV, keyDict + + """ + Convert a string from Config DB value to Yang Value based on type of the + key in Yang model. + @model : A List of Leafs in Yang model list + """ + def revFindYangTypedValue(self, key, value, leafDict): + + # convert yang Type to config DB string + def revYangConvert(val): + # config DB has only strings, thank god for that :), wait not yet!!! + return str(val) + + # if it is a leaf-list do it for each element + if leafDict[key]['__isleafList']: + vValue = list() + for v in value: + vValue.append(revYangConvert(v)) else: - vValue = val + vValue = revYangConvert(value) + return vValue - # if it is a leaf-list do it for each element - if leafDict[key]['__isleafList']: - vValue = list() - for v in value: - vValue.append(yangConvert(v)) - else: - vValue = yangConvert(value) - - return vValue - -""" -Xlate a list -This function will xlate from a dict in config DB to a Yang JSON list -using yang model. Output will be go in self.xlateJson -""" -def xlateList(self, model, yang, config, table): - - #create a dict to map each key under primary key with a dict yang model. - #This is done to improve performance of mapping from values of TABLEs in - #config DB to leaf in YANG LIST. - leafDict = self.createLeafDict(model) - - # fetch regex from YANG models. - keyRegEx = model['ext:key-regex-configdb-to-yang']['@value'] - # seperator `|` has special meaning in regex, so change it appropriately. - keyRegEx = re.sub('\|', '\\|', keyRegEx) - # get keys from YANG model list itself - listKeys = model['key']['@value'] - self.sysLog(msg="xlateList regex:{} keyList:{}".\ - format(keyRegEx, listKeys)) - - for pkey in config.keys(): - try: - vKey = None - self.sysLog(syslog.LOG_DEBUG, "xlateList Extract pkey:{}".\ - format(pkey)) - # Find and extracts key from each dict in config - keyDict = self.extractKey(pkey, listKeys, keyRegEx) - # fill rest of the values in keyDict - for vKey in config[pkey]: - self.sysLog(syslog.LOG_DEBUG, "xlateList vkey {}".format(vKey)) - keyDict[vKey] = self.findYangTypedValue(vKey, \ - config[pkey][vKey], leafDict) - yang.append(keyDict) - # delete pkey from config, done to match one key with one list - del config[pkey] + """ + Rev xlate from _LIST to table in config DB + """ + def revXlateList(self, model, yang, config, table): + + # fetch regex from YANG models + keyRegEx = model['ext:key-regex-yang-to-configdb']['@value'] + self.sysLog(msg="revXlateList regex:{}".format(keyRegEx)) + + # create a dict to map each key under primary key with a dict yang model. + # This is done to improve performance of mapping from values of TABLEs in + # config DB to leaf in YANG LIST. + leafDict = self.createLeafDict(model) + + # list with name _LIST should be removed, + if "_LIST" in model['@name']: + for entry in yang: + # create key of config DB table + pkey, pkeydict = self.createKey(entry, keyRegEx) + self.sysLog(syslog.LOG_DEBUG, "revXlateList pkey:{}".format(pkey)) + config[pkey]= dict() + # fill rest of the entries + for key in entry: + if key not in pkeydict: + config[pkey][key] = self.revFindYangTypedValue(key, \ + entry[key], leafDict) - except Exception as e: - # log debug, because this exception may occur with multilists - self.sysLog(syslog.LOG_DEBUG, "xlateList Exception {}".format(e)) - # with multilist, we continue matching other keys. - continue - - return - -""" -Xlate a container -This function will xlate from a dict in config DB to a Yang JSON container -using yang model. Output will be stored in self.xlateJson -""" -def xlateContainer(self, model, yang, config, table): - - # To Handle multiple List, Make a copy of config, because we delete keys - # from config after each match. This is done to match one pkey with one list. - configC = config.copy() - - clist = model.get('list') - # If single list exists in container, - if clist and isinstance(clist, dict) and \ - clist['@name'] == model['@name']+"_LIST" and bool(configC): - #print(clist['@name']) - yang[clist['@name']] = list() - self.sysLog(msg="xlateContainer listD {}".format(clist['@name'])) - self.xlateList(clist, yang[clist['@name']], \ - configC, table) - # clean empty lists - if len(yang[clist['@name']]) == 0: - del yang[clist['@name']] - #print(yang[clist['@name']]) - - # If multi-list exists in container, - elif clist and isinstance(clist, list) and bool(configC): - for modelList in clist: - yang[modelList['@name']] = list() - self.sysLog(msg="xlateContainer listL {}".format(modelList['@name'])) - self.xlateList(modelList, yang[modelList['@name']], configC, table) - # clean empty lists - if len(yang[modelList['@name']]) == 0: - del yang[modelList['@name']] - - if len(configC): - self.sysLog(syslog.LOG_ERR, "Alert: Remaining keys in Config") - raise(Exception("All Keys are not parsed in {}".format(table))) - - return - -""" -xlate ConfigDB json to Yang json -""" -def xlateConfigDBtoYang(self, jIn, yangJ): - - # find top level container for each table, and run the xlate_container. - for table in jIn.keys(): - cmap = self.confDbYangMap[table] - # create top level containers - key = cmap['module']+":"+cmap['topLevelContainer'] - subkey = cmap['topLevelContainer']+":"+cmap['container']['@name'] - # Add new top level container for first table in this container - yangJ[key] = dict() if yangJ.get(key) is None else yangJ[key] - yangJ[key][subkey] = dict() - self.sysLog(msg="xlateConfigDBtoYang {}:{}".format(key, subkey)) - self.xlateContainer(cmap['container'], yangJ[key][subkey], \ - jIn[table], table) - - return - -""" -Read config file and crop it as per yang models -""" -def xlateConfigDB(self, xlateFile=None): - - jIn= self.jIn - yangJ = self.xlateJson - # xlation is written in self.xlateJson - self.xlateConfigDBtoYang(jIn, yangJ) - - if xlateFile: - with open(xlateFile, 'w') as f: - dump(self.xlateJson, f, indent=4) - - return - -""" -create config DB table key from entry in yang JSON -""" -def createKey(self, entry, regex): - - keyDict = dict() - keyV = regex - # get the keys from regex of key extractor - keyList = re.findall(r'<(.*?)>', regex) - for key in keyList: - val = entry.get(key) - if val: - #print("pair: {} {}".format(key, val)) - keyDict[key] = sval = str(val) - keyV = re.sub(r'<'+key+'>', sval, keyV) - #print("VAL: {} {}".format(regex, keyV)) - else: - raise Exception("key {} not found in entry".format(key)) - #print("kDict {}".format(keyDict)) - return keyV, keyDict - -""" -Convert a string from Config DB value to Yang Value based on type of the -key in Yang model. -@model : A List of Leafs in Yang model list -""" -def revFindYangTypedValue(self, key, value, leafDict): - - # convert yang Type to config DB string - def revYangConvert(val): - # config DB has only strings, thank god for that :), wait not yet!!! - return str(val) - - # if it is a leaf-list do it for each element - if leafDict[key]['__isleafList']: - vValue = list() - for v in value: - vValue.append(revYangConvert(v)) - else: - vValue = revYangConvert(value) - - return vValue - -""" -Rev xlate from
_LIST to table in config DB -""" -def revXlateList(self, model, yang, config, table): - - # fetch regex from YANG models - keyRegEx = model['ext:key-regex-yang-to-configdb']['@value'] - self.sysLog(msg="revXlateList regex:{}".format(keyRegEx)) - - # create a dict to map each key under primary key with a dict yang model. - # This is done to improve performance of mapping from values of TABLEs in - # config DB to leaf in YANG LIST. - leafDict = self.createLeafDict(model) - - # list with name _LIST should be removed, - if "_LIST" in model['@name']: - for entry in yang: - # create key of config DB table - pkey, pkeydict = self.createKey(entry, keyRegEx) - self.sysLog(syslog.LOG_DEBUG, "revXlateList pkey:{}".format(pkey)) - config[pkey]= dict() - # fill rest of the entries - for key in entry: - if key not in pkeydict: - config[pkey][key] = self.revFindYangTypedValue(key, \ - entry[key], leafDict) - - return - -""" -Rev xlate from yang container to table in config DB -""" -def revXlateContainer(self, model, yang, config, table): - - # Note: right now containers has only LISTs. - # IF container has only one list - if isinstance(model['list'], dict): - modelList = model['list'] - # Pass matching list from Yang Json - self.sysLog(msg="revXlateContainer {}".format(modelList['@name'])) - self.revXlateList(modelList, yang[modelList['@name']], config, table) - - elif isinstance(model['list'], list): - for modelList in model['list']: + return + + """ + Rev xlate from yang container to table in config DB + """ + def revXlateContainer(self, model, yang, config, table): + + # Note: right now containers has only LISTs. + # IF container has only one list + if isinstance(model['list'], dict): + modelList = model['list'] + # Pass matching list from Yang Json self.sysLog(msg="revXlateContainer {}".format(modelList['@name'])) self.revXlateList(modelList, yang[modelList['@name']], config, table) - return + elif isinstance(model['list'], list): + for modelList in model['list']: + self.sysLog(msg="revXlateContainer {}".format(modelList['@name'])) + self.revXlateList(modelList, yang[modelList['@name']], config, table) -""" -rev xlate ConfigDB json to Yang json -""" -def revXlateYangtoConfigDB(self, yangJ, cDbJson): + return - yangJ = self.xlateJson - cDbJson = self.revXlateJson + """ + rev xlate ConfigDB json to Yang json + """ + def revXlateYangtoConfigDB(self, yangJ, cDbJson): + + yangJ = self.xlateJson + cDbJson = self.revXlateJson + + # find table in config DB, use name as a KEY + for module_top in yangJ.keys(): + # module _top will be of from module:top + for container in yangJ[module_top].keys(): + #table = container.split(':')[1] + table = container + #print("revXlate " + table) + cmap = self.confDbYangMap[table] + cDbJson[table] = dict() + #print(key + "--" + subkey) + self.sysLog(msg="revXlateYangtoConfigDB {}".format(table)) + self.revXlateContainer(cmap['container'], yangJ[module_top][container], \ + cDbJson[table], table) - # find table in config DB, use name as a KEY - for module_top in yangJ.keys(): - # module _top will be of from module:top - for container in yangJ[module_top].keys(): - #table = container.split(':')[1] - table = container - #print("revXlate " + table) - cmap = self.confDbYangMap[table] - cDbJson[table] = dict() - #print(key + "--" + subkey) - self.sysLog(msg="revXlateYangtoConfigDB {}".format(table)) - self.revXlateContainer(cmap['container'], yangJ[module_top][container], \ - cDbJson[table], table) - - return - -""" -Reverse Translate tp config DB -""" -def revXlateConfigDB(self, revXlateFile=None): - - yangJ = self.xlateJson - cDbJson = self.revXlateJson - # xlation is written in self.xlateJson - self.revXlateYangtoConfigDB(yangJ, cDbJson) - - if revXlateFile: - with open(revXlateFile, 'w') as f: - dump(self.revXlateJson, f, indent=4) - - return - -""" -Find a list in YANG Container -c = container -l = list name -return: list if found else None -""" -def findYangList(self, container, listName): - - if isinstance(container['list'], dict): - clist = container['list'] - if clist['@name'] == listName: - return clist - - elif isinstance(container['list'], list): - clist = [l for l in container['list'] if l['@name'] == listName] - return clist[0] - - return None - -""" -Find xpath of the PORT Leaf in PORT container/list. Xpath of Leaf is needed, -because only leaf can have leafrefs depend on them. -""" -def findXpathPortLeaf(self, portName): - - try: - table = "PORT" - xpath = self.findXpathPort(portName) - module, topc, container = self.get_module_TLC_container(table) - list = self.findYangList(container, table+"_LIST") - xpath = xpath + "/" + list['key']['@value'].split()[0] - except Exception as e: - print("find xpath of port Leaf failed") - raise e - - return xpath - - -""" -Find xpath of PORT -""" -def findXpathPort(self, portName): - - try: - table = "PORT" - module, topc, container = self.get_module_TLC_container(table) - xpath = "/" + module + ":" + topc + "/" + table - - list = self.findYangList(container, table+"_LIST") - xpath = self.findXpathList(xpath, list, [portName]) - except Exception as e: - print("find xpath of port failed") - raise e - - return xpath - -""" -Find xpath of a YANG LIST from keys, -xpath: xpath till list -list: YANG List -keys: list of keys in YANG LIST -""" -def findXpathList(self, xpath, list, keys): - - try: - # add list name in xpath - xpath = xpath + "/" + list['@name'] - listKeys = list['key']['@value'].split() - i = 0; - for listKey in listKeys: - xpath = xpath + '['+listKey+'=\''+keys[i]+'\']' - i = i + 1 - except Exception as e: - print("find xpath of list failed") - raise e - - return xpath - -""" -load_data: load Config DB, crop, xlate and create data tree from it. -input: data -returns: True - success False - failed -""" -def load_data(self, configdbJson, allowExtraTables=True): - - try: - self.jIn = configdbJson - # reset xlate - self.xlateJson = dict() - # self.jIn will be cropped - self.cropConfigDB(allowExtraTables=allowExtraTables) - # xlated result will be in self.xlateJson - self.xlateConfigDB() - #print(self.xlateJson) - self.sysLog(msg="Try to load Data in the tree") - self.root = self.ctx.parse_data_mem(dumps(self.xlateJson), \ - ly.LYD_JSON, ly.LYD_OPT_CONFIG|ly.LYD_OPT_STRICT) - - except Exception as e: - self.root = None - print("Data Loading Failed") - raise e - - return True - -""" -Get data from Data tree, data tree will be assigned in self.xlateJson -""" -def get_data(self): - - try: - self.xlateJson = loads(self.print_data_mem('JSON')) - # reset reverse xlate - self.revXlateJson = dict() - # result will be stored self.revXlateJson - self.revXlateConfigDB() - - except Exception as e: - print("Get Data Tree Failed") - raise e - - return self.revXlateJson - -""" -Delete a node from data tree, if this is LEAF and KEY Delete the Parent -""" -def delete_node(self, xpath): - - # These MACROS used only here, can we get it from Libyang Header ? - try: - LYS_LEAF = 4 - node = self.find_data_node(xpath) - if node is None: - raise('Node {} not found'.format(xpath)) - - snode = node.schema() - # check for a leaf if it is a key. If yes delete the parent - if (snode.nodetype() == LYS_LEAF): - leaf = ly.Schema_Node_Leaf(snode) - if leaf.is_key(): - # try to delete parent - nodeP = self.find_parent_data_node(xpath) - xpathP = nodeP.path() - if self._delete_node(xpath=xpathP, node=nodeP) == False: - raise('_delete_node failed') - else: - return True + return + + """ + Reverse Translate tp config DB + """ + def revXlateConfigDB(self, revXlateFile=None): + + yangJ = self.xlateJson + cDbJson = self.revXlateJson + # xlation is written in self.xlateJson + self.revXlateYangtoConfigDB(yangJ, cDbJson) + + if revXlateFile: + with open(revXlateFile, 'w') as f: + dump(self.revXlateJson, f, indent=4) + + return + + """ + Find a list in YANG Container + c = container + l = list name + return: list if found else None + """ + def findYangList(self, container, listName): + + if isinstance(container['list'], dict): + clist = container['list'] + if clist['@name'] == listName: + return clist + + elif isinstance(container['list'], list): + clist = [l for l in container['list'] if l['@name'] == listName] + return clist[0] + + return None + + """ + Find xpath of the PORT Leaf in PORT container/list. Xpath of Leaf is needed, + because only leaf can have leafrefs depend on them. + """ + def findXpathPortLeaf(self, portName): - # delete non key element - if self._delete_node(xpath=xpath, node=node) == False: - raise('_delete_node failed') - except Exception as e: - print(e) - raise('Failed to delete node {}'.format(xpath)) + try: + table = "PORT" + xpath = self.findXpathPort(portName) + module, topc, container = self.get_module_TLC_container(table) + list = self.findYangList(container, table+"_LIST") + xpath = xpath + "/" + list['key']['@value'].split()[0] + except Exception as e: + print("find xpath of port Leaf failed") + raise e + + return xpath + + + """ + Find xpath of PORT + """ + def findXpathPort(self, portName): + + try: + table = "PORT" + module, topc, container = self.get_module_TLC_container(table) + xpath = "/" + module + ":" + topc + "/" + table + + list = self.findYangList(container, table+"_LIST") + xpath = self.findXpathList(xpath, list, [portName]) + except Exception as e: + print("find xpath of port failed") + raise e + + return xpath + + """ + Find xpath of a YANG LIST from keys, + xpath: xpath till list + list: YANG List + keys: list of keys in YANG LIST + """ + def findXpathList(self, xpath, list, keys): + + try: + # add list name in xpath + xpath = xpath + "/" + list['@name'] + listKeys = list['key']['@value'].split() + i = 0; + for listKey in listKeys: + xpath = xpath + '['+listKey+'=\''+keys[i]+'\']' + i = i + 1 + except Exception as e: + print("find xpath of list failed") + raise e + + return xpath + + """ + load_data: load Config DB, crop, xlate and create data tree from it. + input: data + returns: True - success False - failed + """ + def load_data(self, configdbJson, allowExtraTables=True): + + try: + self.jIn = configdbJson + # reset xlate + self.xlateJson = dict() + # self.jIn will be cropped + self.cropConfigDB(allowExtraTables=allowExtraTables) + # xlated result will be in self.xlateJson + self.xlateConfigDB() + #print(self.xlateJson) + self.sysLog(msg="Try to load Data in the tree") + self.root = self.ctx.parse_data_mem(dumps(self.xlateJson), \ + ly.LYD_JSON, ly.LYD_OPT_CONFIG|ly.LYD_OPT_STRICT) + + except Exception as e: + self.root = None + print("Data Loading Failed") + raise e + + return True + + """ + Get data from Data tree, data tree will be assigned in self.xlateJson + """ + def get_data(self): + + try: + self.xlateJson = loads(self.print_data_mem('JSON')) + # reset reverse xlate + self.revXlateJson = dict() + # result will be stored self.revXlateJson + self.revXlateConfigDB() + + except Exception as e: + print("Get Data Tree Failed") + raise e + + return self.revXlateJson + + """ + Delete a node from data tree, if this is LEAF and KEY Delete the Parent + """ + def delete_node(self, xpath): + + # These MACROS used only here, can we get it from Libyang Header ? + try: + LYS_LEAF = 4 + node = self.find_data_node(xpath) + if node is None: + raise('Node {} not found'.format(xpath)) + + snode = node.schema() + # check for a leaf if it is a key. If yes delete the parent + if (snode.nodetype() == LYS_LEAF): + leaf = ly.Schema_Node_Leaf(snode) + if leaf.is_key(): + # try to delete parent + nodeP = self.find_parent_node(xpath) + xpathP = nodeP.path() + if self._delete_node(xpath=xpathP, node=nodeP) == False: + raise Exception('_delete_node failed') + else: + return True + + # delete non key element + if self._delete_node(xpath=xpath, node=node) == False: + raise Exception('_delete_node failed') + except Exception as e: + print(e) + raise Exception('Failed to delete node {}'.format(xpath)) - return True + return True -# End of class sonic_yang + # End of class sonic_yang diff --git a/src/sonic-yang-mgmt/setup.py b/src/sonic-yang-mgmt/setup.py index 24a09cf958c7..15c8243aad7f 100644 --- a/src/sonic-yang-mgmt/setup.py +++ b/src/sonic-yang-mgmt/setup.py @@ -5,22 +5,21 @@ from setuptools import setup, find_packages from setuptools.command.build_py import build_py -from os import system +from os import system, environ from sys import exit import pytest -import os # find path of pkgs from os environment vars -prefix = '/sonic'; debs = os.environ["STRETCH_DEBS_PATH"] -wheels = os.environ["PYTHON_WHEELS_PATH"] +prefix = '/sonic'; debs = environ["STRETCH_DEBS_PATH"] +wheels = environ["PYTHON_WHEELS_PATH"] wheels_path = '{}/{}'.format(prefix, wheels) deps_path = '{}/{}'.format(prefix, debs) # dependencies -libyang = '{}/{}'.format(deps_path, os.environ["LIBYANG"]) -libyangCpp = '{}/{}'.format(deps_path, os.environ["LIBYANG_CPP"]) -libyangPy2 = '{}/{}'.format(deps_path, os.environ["LIBYANG_PY2"]) -libyangPy3 = '{}/{}'.format(deps_path, os.environ["LIBYANG_PY3"]) -sonicYangModels = '{}/{}'.format(wheels_path, os.environ["SONIC_YANG_MODELS_PY3"]) +libyang = '{}/{}'.format(deps_path, environ["LIBYANG"]) +libyangCpp = '{}/{}'.format(deps_path, environ["LIBYANG_CPP"]) +libyangPy2 = '{}/{}'.format(deps_path, environ["LIBYANG_PY2"]) +libyangPy3 = '{}/{}'.format(deps_path, environ["LIBYANG_PY3"]) +sonicYangModels = '{}/{}'.format(wheels_path, environ["SONIC_YANG_MODELS_PY3"]) # important reuirements parameters build_requirements = [libyang, libyangCpp, libyangPy2, libyangPy3, sonicYangModels,] diff --git a/src/sonic-yang-mgmt/sonic_yang.py b/src/sonic-yang-mgmt/sonic_yang.py index c1f2ddd1f573..6bc330c9ed6a 100644 --- a/src/sonic-yang-mgmt/sonic_yang.py +++ b/src/sonic-yang-mgmt/sonic_yang.py @@ -3,12 +3,14 @@ from json import dump from glob import glob -from datetime import datetime +from _sonic_yang_ext import sonic_yang_ext_mixin """ Yang schema and data tree python APIs based on libyang python +Here, sonic_yang_ext_mixin extends funtionality of sonic_yang, +i.e. it is mixin not parent class. """ -class sonic_yang: +class sonic_yang(sonic_yang_ext_mixin): def __init__(self, yang_dir, debug=False): self.yang_dir = yang_dir @@ -58,11 +60,6 @@ def fail(self, e): print(e) raise e - """ - import all function from extension file - """ - from _sonic_yang_ext import * - """ load_schema_module(): load a Yang model file input: yang_file - full path of a Yang model file @@ -240,7 +237,7 @@ def validate_data (self, node=None, ctx=None): ctx = self.ctx try: - rc = node.validate(ly.LYD_OPT_CONFIG, ctx) + node.validate(ly.LYD_OPT_CONFIG, ctx) except Exception as e: self.fail(e) @@ -377,7 +374,7 @@ def find_data_node_schema_xpath(self, data_xpath): """ def add_data_node(self, data_xpath, value): try: - data_node = self.new_data_node(data_xpath, value) + self.new_node(xpath, value) #check if the node added to the data tree self.find_data_node(data_xpath) except Exception as e: @@ -455,7 +452,7 @@ def find_data_node_value(self, data_xpath): """ def set_data_node_value(self, data_xpath, value): try: - data_node = self.root.new_path(self.ctx, data_xpath, str(value), ly.LYD_ANYDATA_STRING, ly.LYD_PATH_OPT_UPDATE) + self.root.new_path(self.ctx, data_xpath, str(value), ly.LYD_ANYDATA_STRING, ly.LYD_PATH_OPT_UPDATE) except Exception as e: print("set data node value failed for xpath: " + str(data_xpath)) self.fail(e) @@ -477,7 +474,7 @@ def find_data_nodes(self, data_xpath): raise Exception('data node not found') for data_set in node_set.data(): - schema = data_set.schema() + data_set.schema() list.append(data_set.path()) return list @@ -489,7 +486,6 @@ def find_data_nodes(self, data_xpath): """ def find_schema_dependencies (self, schema_xpath): ref_list = [] - node = self.root try: schema_node = self.find_schema_node(schema_xpath) except Exception as e: @@ -530,7 +526,7 @@ def find_data_dependencies (self, data_xpath): for link in backlinks.schema(): node_set = node.find_path(link.path()) for data_set in node_set.data(): - schema = data_set.schema() + data_set.schema() casted = data_set.subtype() if value == casted.value_str(): ref_list.append(data_set.path()) @@ -651,7 +647,7 @@ def get_leafref_type_schema (self, schema_xpath): if subtype.type().base() != ly.LY_TYPE_LEAFREF: return None else: - leafref_path = subtype.type().info().lref().path() + subtype.type().info().lref().path() target = subtype.type().info().lref().target() target_path = target.path() target_type = self.get_data_type(target_path) diff --git a/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py b/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py index aa98bdf149c8..6002b4386068 100644 --- a/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py +++ b/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py @@ -1,11 +1,8 @@ import sys import os import pytest -import yang as ly import sonic_yang as sy import json -import getopt -import subprocess import glob import logging from ijson import items as ijson_itmes @@ -31,7 +28,6 @@ def data(self): @pytest.fixture(autouse=True, scope='class') def yang_s(self, data): yang_dir = str(data['yang_dir']) - data_file = str(data['data_file']) yang_s = sy.sonic_yang(yang_dir) return yang_s @@ -62,7 +58,7 @@ def readIjsonInput(self, yang_test_file, test): raise(e) return jInput - def setup_class(cls): + def setup_class(self): pass def load_yang_model_file(self, yang_s, yang_dir, yang_file, module_name): @@ -140,14 +136,14 @@ def test_find_node(self, data, yang_s): assert dnode is not None assert dnode.path() == xpath else: - assert dnode == None + assert dnode is None #test add node def test_add_node(self, data, yang_s): for node in data['new_nodes']: xpath = str(node['xpath']) value = node['value'] - status = yang_s.add_data_node(xpath, str(value)) + yang_s.add_node(xpath, str(value)) data_node = yang_s.find_data_node(xpath) assert data_node is not None @@ -165,7 +161,6 @@ def test_find_data_node_value(self, data, yang_s): #test delete data node def test_delete_node(self, data, yang_s): for node in data['delete_nodes']: - expected = node['valid'] xpath = str(node['xpath']) yang_s._delete_node(xpath) @@ -291,5 +286,5 @@ def test_xlate_rev_xlate(self): return - def teardown_class(cls): + def teardown_class(self): pass