From f189986386af2051301a8bd22ffca28676238a1a Mon Sep 17 00:00:00 2001 From: svshah-intel <102195908+svshah-intel@users.noreply.github.com> Date: Wed, 30 Nov 2022 18:12:41 -0800 Subject: [PATCH 01/69] [submodule update] sairedis refpoint to include support for json sai attr value sairedis commits: b1e9c91 2022-11-29 | validation support for SAI_ATTR_VALUE_TYPE_JSON (sonic-net/sonic-sairedis#1152) --- src/sonic-sairedis | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-sairedis b/src/sonic-sairedis index c7ffef8b25e9..b1e9c9109f88 160000 --- a/src/sonic-sairedis +++ b/src/sonic-sairedis @@ -1 +1 @@ -Subproject commit c7ffef8b25e9d577438f5c6893d07b76751c2cb6 +Subproject commit b1e9c9109f8891902a46201bc4bee53d23db72bd From ffa974c7f4e404f270acecb13a05440bbbb175fa Mon Sep 17 00:00:00 2001 From: Junchao-Mellanox <57339448+Junchao-Mellanox@users.noreply.github.com> Date: Thu, 1 Dec 2022 10:38:50 +0800 Subject: [PATCH 02/69] [system-health] Led color shall be controlled by configuration when system is booting (#12487) * [system-health] Led color shall be controlled by configuration when system is booting * Fix unit test issue --- src/system-health/health_checker/config.py | 2 +- src/system-health/health_checker/manager.py | 23 +++++++++++++------ src/system-health/tests/test_system_health.py | 16 ++++++------- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/src/system-health/health_checker/config.py b/src/system-health/health_checker/config.py index 3d287d778377..63af22db4ad0 100644 --- a/src/system-health/health_checker/config.py +++ b/src/system-health/health_checker/config.py @@ -20,7 +20,7 @@ class Config(object): DEFAULT_LED_CONFIG = { 'fault': 'red', 'normal': 'green', - 'booting': 'orange_blink' + 'booting': 'red' } # System health configuration file name diff --git a/src/system-health/health_checker/manager.py b/src/system-health/health_checker/manager.py index b1c6dd15bec3..927086eaea02 100644 --- a/src/system-health/health_checker/manager.py +++ b/src/system-health/health_checker/manager.py @@ -3,14 +3,13 @@ from .service_checker import ServiceChecker from .hardware_checker import HardwareChecker from .user_defined_checker import UserDefinedChecker +from . import utils class HealthCheckerManager(object): """ Manage all system health checkers and system health configuration. """ - boot_timeout = None - def __init__(self): self._checkers = [] self.config = Config() @@ -42,9 +41,7 @@ def check(self, chassis): checker = UserDefinedChecker(udc) self._do_check(checker, stats) - led_status = 'normal' if HealthChecker.summary == HealthChecker.STATUS_OK else 'fault' - self._set_system_led(chassis, self.config, led_status) - + self._set_system_led(chassis) return stats def _do_check(self, checker, stats): @@ -75,10 +72,22 @@ def _do_check(self, checker, stats): else: stats['Internal'].update(entry) - def _set_system_led(self, chassis, config, status): + def _set_system_led(self, chassis): try: - chassis.set_status_led(config.get_led_color(status)) + chassis.set_status_led(self._get_led_target_color()) except NotImplementedError: print('chassis.set_status_led is not implemented') except Exception as e: print('Failed to set system led due to - {}'.format(repr(e))) + + def _get_led_target_color(self): + """Get target LED color according to health status and system uptime + + Returns: + str: LED color + """ + if HealthChecker.summary == HealthChecker.STATUS_OK: + return self.config.get_led_color('normal') + else: + uptime = utils.get_uptime() + return self.config.get_led_color('booting') if uptime < self.config.get_bootup_timeout() else self.config.get_led_color('fault') diff --git a/src/system-health/tests/test_system_health.py b/src/system-health/tests/test_system_health.py index 73beb99cae83..0196efc177a8 100644 --- a/src/system-health/tests/test_system_health.py +++ b/src/system-health/tests/test_system_health.py @@ -457,7 +457,7 @@ def test_config(): assert config.get_led_color('fault') == 'red' assert config.get_led_color('normal') == 'green' - assert config.get_led_color('booting') == 'orange_blink' + assert config.get_led_color('booting') == 'red' config._last_mtime = 1 config._config_file = 'notExistFile' @@ -530,10 +530,10 @@ def test_manager(mock_hw_info, mock_service_info, mock_udc_info): assert stat['Internal']['UserDefinedChecker - some check']['status'] == 'Not OK' chassis.set_status_led.side_effect = NotImplementedError() - manager._set_system_led(chassis, manager.config, 'normal') + manager._set_system_led(chassis) chassis.set_status_led.side_effect = RuntimeError() - manager._set_system_led(chassis, manager.config, 'normal') + manager._set_system_led(chassis) def test_utils(): output = utils.run_command(['some', 'invalid', 'command']) @@ -600,7 +600,7 @@ def test_get_app_ready_status(mock_config_db, mock_run, mock_docker_client): 'has_global_scope': 'True', 'has_per_asic_scope': 'False', 'check_up_status': 'True' - }, + }, 'snmp': { 'state': 'enabled', 'has_global_scope': 'True', @@ -691,10 +691,10 @@ def test_get_all_system_status_not_ok(): result = sysmon.get_all_system_status() print("result:{}".format(result)) assert result == 'DOWN' - + def test_post_unit_status(): sysmon = Sysmonitor() - sysmon.post_unit_status("mock_bgp", 'OK', 'Down', 'mock reason', '-') + sysmon.post_unit_status("mock_bgp", 'OK', 'Down', 'mock reason', '-') result = swsscommon.SonicV2Connector.get_all(MockConnector, 0, 'ALL_SERVICE_STATUS|mock_bgp') print(result) assert result['service_status'] == 'OK' @@ -703,7 +703,7 @@ def test_post_unit_status(): def test_post_system_status(): sysmon = Sysmonitor() - sysmon.post_system_status("UP") + sysmon.post_system_status("UP") result = swsscommon.SonicV2Connector.get(MockConnector, 0, "SYSTEM_READY|SYSTEM_STATE", 'Status') print("post system status result:{}".format(result)) assert result == "UP" @@ -721,7 +721,7 @@ def test_publish_system_status(): @patch('health_checker.sysmonitor.Sysmonitor.publish_system_status', test_publish_system_status()) def test_update_system_status(): sysmon = Sysmonitor() - sysmon.update_system_status() + sysmon.update_system_status() result = swsscommon.SonicV2Connector.get(MockConnector, 0, "SYSTEM_READY|SYSTEM_STATE", 'Status') assert result == "UP" From 11d579ccb13a165d6afe8b0d910891e0a9192976 Mon Sep 17 00:00:00 2001 From: vdahiya12 <67608553+vdahiya12@users.noreply.github.com> Date: Wed, 30 Nov 2022 19:34:36 -0800 Subject: [PATCH 03/69] [sonic-platform-daemons] submodule update (#12841) Signed-off-by: vaibhav-dahiya vdahiya@microsoft.com e474335 (HEAD -> master, origin/master, origin/HEAD) [ycabled] fix minor appl_db retrieving logic for update (#319) 9b84b58 Use github code scanning instead of LGTM (#316) f784ad7 Pass grid parameter while calling set_laser_freq (#317) ed818f8 [PSU daemon] Support PSU power threshold checking (#288) 707a720 (origin/202211) [chassisd] update chassisd to write fabric and lc asics on sep erate table (#311) e8c5657 [ycabled] fix exception-handling logic for ycabled (#306) 905874d [ycabled] move swsscommon API's from subroutines to call them exactly once per task_worker/thread (#303) 510d330 Fix typo in xcvrd (#313) 9ae551f [ycabled] add support for detach mode in 'active-active' topology (#309) The above commits are added to sonic-platform-daemons --- src/sonic-platform-daemons | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-platform-daemons b/src/sonic-platform-daemons index 82fc7a6f858f..e474335f9efd 160000 --- a/src/sonic-platform-daemons +++ b/src/sonic-platform-daemons @@ -1 +1 @@ -Subproject commit 82fc7a6f858f2c0bb8abf0263c8002d77e91cf6a +Subproject commit e474335f9efd0cfaa20ee5832c116264547c18c0 From b774ebfdc200afe16adca9219bcbf8a77710d9a8 Mon Sep 17 00:00:00 2001 From: zitingguo-ms Date: Thu, 1 Dec 2022 14:26:54 +0800 Subject: [PATCH 04/69] [SAI-PTF] Publish docker saiserverv2 in master branch (#12842) Why I did it Publish docker saiserverv2 in the build pipeline. How I did it Add docker saiserverv2 target in the build template. How to verify it Test this by running this pipeline: https://dev.azure.com/mssonic/build/_build/results?buildId=182134&view=results --- .azure-pipelines/azure-pipelines-build.yml | 11 +++++++++++ .azure-pipelines/build-template.yml | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/.azure-pipelines/azure-pipelines-build.yml b/.azure-pipelines/azure-pipelines-build.yml index c5f270665873..d21a6066a01b 100644 --- a/.azure-pipelines/azure-pipelines-build.yml +++ b/.azure-pipelines/azure-pipelines-build.yml @@ -139,9 +139,20 @@ jobs: make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).raw fi if [ $(docker_syncd_rpc_image) == yes ]; then + # workaround for issue in rules/sairedis.dep, git ls-files will list un-exist files for cache make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-rpc.gz + pushd ./src/sonic-sairedis/SAI + git stash + popd if [ $(GROUP_NAME) == broadcom ]; then make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-dnx-rpc.gz + pushd ./src/sonic-sairedis/SAI + git stash + popd + make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y target/docker-saiserverv2-brcm.gz + pushd ./src/sonic-sairedis/SAI + git stash + popd fi fi if [ $(syncd_rpc_image) == yes ]; then diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index e1188924a18a..52bff4aa9e51 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -106,6 +106,16 @@ jobs: fi if [ ${{ parameters.sync_rpc_image }} == true ]; then make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y target/docker-syncd-${{ parameters.platform_short }}-rpc.gz + # workaround for issue in rules/sairedis.dep, git ls-files will list un-exist files for cache + pushd ./src/sonic-sairedis/SAI + git stash + popd + if [ ${{ parameters.platform }} == broadcom ]; then + make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y target/docker-saiserverv2-brcm.gz + pushd ./src/sonic-sairedis/SAI + git stash + popd + fi fi make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/sonic-${{ parameters.platform }}.bin From ec809bd7a11c18db39d1fd75c14b485d6e55bbf5 Mon Sep 17 00:00:00 2001 From: Stephen Sun <5379172+stephenxs@users.noreply.github.com> Date: Thu, 1 Dec 2022 21:35:44 +0800 Subject: [PATCH 05/69] [Submodule] Advance sonic-host-services pointer (#12902) 4a2ef99 Avoid printing message in error level when DEVICE_METADATA|localhost updates (sonic-net/sonic-host-services#25) 6c131c4 Use github code scanning instead of LGTM(sonic-net/sonic-host-services#26) c55f5d1 Use github code scanning instead of LGTM Signed-off-by: Stephen Sun --- src/sonic-host-services | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-host-services b/src/sonic-host-services index d3a4cf1f6c4d..4a2ef99630ee 160000 --- a/src/sonic-host-services +++ b/src/sonic-host-services @@ -1 +1 @@ -Subproject commit d3a4cf1f6c4dd633cfe9b07dfc26331238d79f37 +Subproject commit 4a2ef99630ee02e104643bf356cafd6a1228b882 From 2b3e884209d5028c1763b0f83cfc42b5249b4efb Mon Sep 17 00:00:00 2001 From: Mai Bui Date: Thu, 1 Dec 2022 12:12:50 -0500 Subject: [PATCH 06/69] [nokia] Replace os.system and remove subprocess with shell=True (#12100) Signed-off-by: maipbui Dependency: [https://github.com/sonic-net/sonic-buildimage/pull/12065](https://github.com/sonic-net/sonic-buildimage/pull/12065) #### Why I did it `subprocess.Popen()` and `subprocess.run()` is used with `shell=True`, which is very dangerous for shell injection. `os` - not secure against maliciously constructed input and dangerous if used to evaluate dynamic content `getstatusoutput` is dangerous because it contains `shell=True` in the implementation #### How I did it Replace `os` by `subprocess`, use with `shell=False` Remove unused functions --- .../plugins/eeprom.py | 4 +- .../7215/sonic_platform/chassis.py | 12 ++---- .../7215/sonic_platform/component.py | 43 ++++++++----------- .../7215/sonic_platform/psu.py | 15 +++---- .../7215/sonic_platform/sfp.py | 21 ++++----- .../7215/sonic_platform/sfp_event.py | 9 +--- 6 files changed, 40 insertions(+), 64 deletions(-) diff --git a/device/nokia/armhf-nokia_ixs7215_52x-r0/plugins/eeprom.py b/device/nokia/armhf-nokia_ixs7215_52x-r0/plugins/eeprom.py index cbd48237cc10..12a1bbe5d2aa 100644 --- a/device/nokia/armhf-nokia_ixs7215_52x-r0/plugins/eeprom.py +++ b/device/nokia/armhf-nokia_ixs7215_52x-r0/plugins/eeprom.py @@ -10,5 +10,7 @@ class board(eeprom_tlvinfo.TlvInfoDecoder): def __init__(self, name, path, cpld_root, ro): self.eeprom_path = "/sys/class/i2c-adapter/i2c-0/0-0053/eeprom" if not os.path.exists(self.eeprom_path): - os.system("echo 24c02 0x53 > /sys/class/i2c-adapter/i2c-0/new_device") + file = "/sys/class/i2c-adapter/i2c-0/new_device" + with open(file, 'w') as f: + f.write('24c02 0x53\n') super(board, self).__init__(self.eeprom_path, 0, '', True) diff --git a/platform/marvell-armhf/sonic-platform-nokia/7215/sonic_platform/chassis.py b/platform/marvell-armhf/sonic-platform-nokia/7215/sonic_platform/chassis.py index eb507a49ef17..008d5ba08b77 100755 --- a/platform/marvell-armhf/sonic-platform-nokia/7215/sonic_platform/chassis.py +++ b/platform/marvell-armhf/sonic-platform-nokia/7215/sonic_platform/chassis.py @@ -18,6 +18,7 @@ from sonic_platform.thermal import Thermal from sonic_platform.component import Component from sonic_py_common import logger + from sonic_py_common.general import getstatusoutput_noshell except ImportError as e: raise ImportError(str(e) + "- required module not found") @@ -27,11 +28,6 @@ except ImportError as e: smbus_present = 0 -if sys.version_info[0] < 3: - import commands as cmd -else: - import subprocess as cmd - MAX_SELECT_DELAY = 3600 COPPER_PORT_START = 1 COPPER_PORT_END = 48 @@ -209,7 +205,7 @@ def get_revision(self): string: Revision value of chassis """ if smbus_present == 0: # called from host - cmdstatus, value = cmd.getstatusoutput('sudo i2cget -y 0 0x41 0x0') + cmdstatus, value = getstatusoutput_noshell(['sudo', 'i2cget', '-y', '0', '0x41', '0x0']) else: bus = smbus.SMBus(0) DEVICE_ADDRESS = 0x41 @@ -331,7 +327,7 @@ def set_status_led(self, color): # Write sys led if smbus_present == 0: # called from host (e.g. 'show system-health') - cmdstatus, value = cmd.getstatusoutput('sudo i2cset -y 0 0x41 0x7 %d' % value) + cmdstatus, value = getstatusoutput_noshell(['sudo', 'i2cset', '-y', '0', '0x41', '0x7', str(value)]) if cmdstatus: sonic_logger.log_warning(" System LED set %s failed" % value) return False @@ -353,7 +349,7 @@ def get_status_led(self): """ # Read sys led if smbus_present == 0: # called from host - cmdstatus, value = cmd.getstatusoutput('sudo i2cget -y 0 0x41 0x7') + cmdstatus, value = getstatusoutput_noshell(['sudo', 'i2cget', '-y', '0', '0x41', '0x7']) value = int(value, 16) else: bus = smbus.SMBus(0) diff --git a/platform/marvell-armhf/sonic-platform-nokia/7215/sonic_platform/component.py b/platform/marvell-armhf/sonic-platform-nokia/7215/sonic_platform/component.py index 134856ea4213..4e8863e64fde 100644 --- a/platform/marvell-armhf/sonic-platform-nokia/7215/sonic_platform/component.py +++ b/platform/marvell-armhf/sonic-platform-nokia/7215/sonic_platform/component.py @@ -9,10 +9,10 @@ try: import os - import sys import subprocess import ntpath from sonic_platform_base.component_base import ComponentBase + from sonic_py_common.general import getstatusoutput_noshell, getstatusoutput_noshell_pipe except ImportError as e: raise ImportError(str(e) + "- required module not found") @@ -22,11 +22,6 @@ except ImportError as e: smbus_present = 0 -if sys.version_info[0] < 3: - import commands as cmd -else: - import subprocess as cmd - class Component(ComponentBase): """Nokia platform-specific Component class""" @@ -35,29 +30,20 @@ class Component(ComponentBase): ["System-CPLD", "Used for managing SFPs, LEDs, PSUs and FANs "], ["U-Boot", "Performs initialization during booting"], ] - CPLD_UPDATE_COMMAND = 'cp /usr/sbin/vme /tmp; cp {} /tmp; cd /tmp; ./vme {};' + CPLD_UPDATE_COMMAND1 = ['cp', '/usr/sbin/vme', '/tmp'] + CPLD_UPDATE_COMMAND2 = ['cp', '', '/tmp'] + CPLD_UPDATE_COMMAND3 = ['cd', '/tmp'] + CPLD_UPDATE_COMMAND4 = ['./vme', ''] def __init__(self, component_index): self.index = component_index self.name = self.CHASSIS_COMPONENTS[self.index][0] self.description = self.CHASSIS_COMPONENTS[self.index][1] - def _get_command_result(self, cmdline): - try: - proc = subprocess.Popen(cmdline.split(), stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - stdout = proc.communicate()[0] - proc.wait() - result = stdout.rstrip('\n') - except OSError: - result = None - - return result - def _get_cpld_version(self, cpld_number): if smbus_present == 0: - cmdstatus, cpld_version = cmd.getstatusoutput('sudo i2cget -y 0 0x41 0x2') + cmdstatus, cpld_version = getstatusoutput_noshell(['sudo', 'i2cget', '-y', '0', '0x41', '0x2']) else: bus = smbus.SMBus(0) DEVICE_ADDRESS = 0x41 @@ -144,7 +130,10 @@ def get_firmware_version(self): return self._get_cpld_version(self.index) if self.index == 1: - cmdstatus, uboot_version = cmd.getstatusoutput('grep --null-data U-Boot /dev/mtd0ro|head -1 | cut -d" " -f2-4') + cmd1 = ['grep', '--null-data', 'U-Boot', '/dev/mtd0ro'] + cmd2 = ['head', '-1'] + cmd3 = ['cut', '-d', ' ', '-f2-4'] + cmdstatus, uboot_version = getstatusoutput_noshell_pipe(cmd1, cmd2, cmd3) return uboot_version def install_firmware(self, image_path): @@ -165,12 +154,16 @@ def install_firmware(self, image_path): print("ERROR: the cpld image {} doesn't exist ".format(image_path)) return False - cmdline = self.CPLD_UPDATE_COMMAND.format(image_path, image_name) + self.CPLD_UPDATE_COMMAND2[1] = image_path + self.CPLD_UPDATE_COMMAND4[1] = image_name success_flag = False - - try: - subprocess.check_call(cmdline, stderr=subprocess.STDOUT, shell=True) + + try: + subprocess.check_call(self.CPLD_UPDATE_COMMAND1, stderr=subprocess.STDOUT) + subprocess.check_call(self.CPLD_UPDATE_COMMAND2, stderr=subprocess.STDOUT) + subprocess.check_call(self.CPLD_UPDATE_COMMAND3, stderr=subprocess.STDOUT) + subprocess.check_call(self.CPLD_UPDATE_COMMAND4, stderr=subprocess.STDOUT) success_flag = True except subprocess.CalledProcessError as e: print("ERROR: Failed to upgrade CPLD: rc={}".format(e.returncode)) diff --git a/platform/marvell-armhf/sonic-platform-nokia/7215/sonic_platform/psu.py b/platform/marvell-armhf/sonic-platform-nokia/7215/sonic_platform/psu.py index 383eb2481f68..243b807edb74 100644 --- a/platform/marvell-armhf/sonic-platform-nokia/7215/sonic_platform/psu.py +++ b/platform/marvell-armhf/sonic-platform-nokia/7215/sonic_platform/psu.py @@ -8,18 +8,13 @@ try: import os - import sys from sonic_platform_base.psu_base import PsuBase from sonic_py_common import logger from sonic_platform.eeprom import Eeprom + from sonic_py_common.general import getstatusoutput_noshell except ImportError as e: raise ImportError(str(e) + "- required module not found") -if sys.version_info[0] < 3: - import commands as cmd -else: - import subprocess as cmd - smbus_present = 1 try: import smbus @@ -86,7 +81,7 @@ def get_presence(self): """ if smbus_present == 0: # if called from psuutil outside of pmon - cmdstatus, psustatus = cmd.getstatusoutput('sudo i2cget -y 0 0x41 0xa') + cmdstatus, psustatus = getstatusoutput_noshell(['sudo', 'i2cget', '-y', '0', '0x41', '0xa']) psustatus = int(psustatus, 16) else: bus = smbus.SMBus(0) @@ -150,7 +145,7 @@ def get_status(self): """ if smbus_present == 0: - cmdstatus, psustatus = cmd.getstatusoutput('sudo i2cget -y 0 0x41 0xa') + cmdstatus, psustatus = getstatusoutput_noshell(['sudo', 'i2cget', '-y', '0', '0x41', '0xa']) psustatus = int(psustatus, 16) sonic_logger.log_warning("PMON psu-smbus - presence = 0 ") else: @@ -179,7 +174,7 @@ def get_voltage(self): e.g. 12.1 """ if smbus_present == 0: - cmdstatus, psustatus = cmd.getstatusoutput('sudo i2cget -y 0 0x41 0xa') + cmdstatus, psustatus = getstatusoutput_noshell(['sudo', 'i2cget', '-y', '0', '0x41', '0xa']) psustatus = int(psustatus, 16) else: bus = smbus.SMBus(0) @@ -226,7 +221,7 @@ def get_powergood_status(self): """ if smbus_present == 0: - cmdstatus, psustatus = cmd.getstatusoutput('sudo i2cget -y 0 0x41 0xa') + cmdstatus, psustatus = getstatusoutput_noshell(['sudo', 'i2cget', '-y', '0', '0x41', '0xa']) psustatus = int(psustatus, 16) else: bus = smbus.SMBus(0) diff --git a/platform/marvell-armhf/sonic-platform-nokia/7215/sonic_platform/sfp.py b/platform/marvell-armhf/sonic-platform-nokia/7215/sonic_platform/sfp.py index d95e7a522da8..9d8e7e739033 100644 --- a/platform/marvell-armhf/sonic-platform-nokia/7215/sonic_platform/sfp.py +++ b/platform/marvell-armhf/sonic-platform-nokia/7215/sonic_platform/sfp.py @@ -3,8 +3,7 @@ # ############################################################################# -import os -import sys +import subprocess try: from sonic_platform_base.sfp_base import SfpBase @@ -12,14 +11,10 @@ from sonic_platform_base.sonic_sfp.sff8472 import sff8472Dom from sonic_platform_base.sonic_sfp.sfputilhelper import SfpUtilHelper from sonic_py_common import logger + from sonic_py_common.general import getstatusoutput_noshell except ImportError as e: raise ImportError(str(e) + "- required module not found") -if sys.version_info[0] < 3: - import commands as cmd -else: - import subprocess as cmd - smbus_present = 1 try: @@ -118,7 +113,7 @@ class Sfp(SfpBase): # Paths PLATFORM_ROOT_PATH = "/usr/share/sonic/device" PMON_HWSKU_PATH = "/usr/share/sonic/hwsku" - HOST_CHK_CMD = "docker > /dev/null 2>&1" + HOST_CHK_CMD = ["docker"] PLATFORM = "armhf-nokia_ixs7215_52x-r0" HWSKU = "Nokia-7215" @@ -186,7 +181,7 @@ def __convert_string_to_num(self, value_str): return 'N/A' def __is_host(self): - return os.system(self.HOST_CHK_CMD) == 0 + return subprocess.call(self.HOST_CHK_CMD) == 0 def __get_path_to_port_config_file(self): platform_path = "/".join([self.PLATFORM_ROOT_PATH, self.PLATFORM]) @@ -811,7 +806,7 @@ def tx_disable(self, tx_disable): return False if smbus_present == 0: # if called from sfputil outside of pmon - cmdstatus, register = cmd.getstatusoutput('sudo i2cget -y 0 0x41 0x5') + cmdstatus, register = getstatusoutput_noshell(['sudo', 'i2cget', '-y', '0', '0x41', '0x5']) if cmdstatus: sonic_logger.log_warning("sfp cmdstatus i2c get failed %s" % register ) return False @@ -824,13 +819,13 @@ def tx_disable(self, tx_disable): pos = [1, 2, 4, 8] mask = pos[self.index-SFP_PORT_START] - if tx_disable == True: + if tx_disable is True: setbits = register | mask else: setbits = register & ~mask if smbus_present == 0: # if called from sfputil outside of pmon - cmdstatus, output = cmd.getstatusoutput('sudo i2cset -y -m 0x0f 0 0x41 0x5 %d' % setbits) + cmdstatus, output = getstatusoutput_noshell(['sudo', 'i2cset', '-y', '-m', '0x0f', '0', '0x41', '0x5', str(setbits)]) if cmdstatus: sonic_logger.log_warning("sfp cmdstatus i2c write failed %s" % output ) return False @@ -912,7 +907,7 @@ def get_presence(self): return False if smbus_present == 0: # if called from sfputil outside of pmon - cmdstatus, sfpstatus = cmd.getstatusoutput('sudo i2cget -y 0 0x41 0x3') + cmdstatus, sfpstatus = getstatusoutput_noshell(['sudo', 'i2cget', '-y', '0', '0x41', '0x3']) sfpstatus = int(sfpstatus, 16) else: bus = smbus.SMBus(0) diff --git a/platform/marvell-armhf/sonic-platform-nokia/7215/sonic_platform/sfp_event.py b/platform/marvell-armhf/sonic-platform-nokia/7215/sonic_platform/sfp_event.py index fd494ca674f5..34fc2bc6ae5d 100644 --- a/platform/marvell-armhf/sonic-platform-nokia/7215/sonic_platform/sfp_event.py +++ b/platform/marvell-armhf/sonic-platform-nokia/7215/sonic_platform/sfp_event.py @@ -1,9 +1,9 @@ ''' listen for the SFP change event and return to chassis. ''' -import sys import time from sonic_py_common import logger +from sonic_py_common.general import getstatusoutput_noshell smbus_present = 1 @@ -12,11 +12,6 @@ except ImportError as e: smbus_present = 0 -if sys.version_info[0] < 3: - import commands as cmd -else: - import subprocess as cmd - # system level event/error EVENT_ON_ALL_SFP = '-1' SYSTEM_NOT_READY = 'system_not_ready' @@ -51,7 +46,7 @@ def deinitialize(self): def _get_transceiver_status(self): if smbus_present == 0: sonic_logger.log_info(" PMON - smbus ERROR - DEBUG sfp_event ") - cmdstatus, sfpstatus = cmd.getstatusoutput('sudo i2cget -y 0 0x41 0x3') + cmdstatus, sfpstatus = getstatusoutput_noshell(['sudo', 'i2cget', '-y', '0', '0x41', '0x3']) sfpstatus = int(sfpstatus, 16) else: bus = smbus.SMBus(0) From 3226c40581a23590fddc2277dffa51583d1ec738 Mon Sep 17 00:00:00 2001 From: Saikrishna Arcot Date: Thu, 1 Dec 2022 09:26:46 -0800 Subject: [PATCH 07/69] [build]: Disable stretch slave container (#12868) The only platforms that currently need the stretch slave container are innovium and nephos, and both are not building with the current code due to other issues. All other platforms only need buster and bullseye slave containers. Signed-off-by: Saikrishna Arcot --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ccfefc6c183c..cd9c91a7bf16 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # SONiC make file NOJESSIE ?= 1 -NOSTRETCH ?= 0 +NOSTRETCH ?= 1 NOBUSTER ?= 0 NOBULLSEYE ?= 0 From 15fc527d30353b090ddbe12845190d309a43b048 Mon Sep 17 00:00:00 2001 From: Sudharsan Dhamal Gopalarathnam Date: Thu, 1 Dec 2022 09:30:32 -0800 Subject: [PATCH 08/69] [yang] Add collector_vrf to sflow yang model (#12897) - Why I did it Fixed sflow yang model to include collector_vrf field. - How I did it Added leaf for collector_vrf under sflow_collector. Additionally aligned the configuration guide - How to verify it Added UT to verify. --- src/sonic-yang-models/doc/Configuration.md | 145 +++++++++++------- .../tests/files/sample_config_db.json | 3 +- .../tests/yang_model_tests/tests/sflow.json | 15 ++ .../yang_model_tests/tests_config/sflow.json | 63 ++++++++ .../yang-models/sonic-sflow.yang | 23 +++ 5 files changed, 191 insertions(+), 58 deletions(-) diff --git a/src/sonic-yang-models/doc/Configuration.md b/src/sonic-yang-models/doc/Configuration.md index 7060b352bc24..d05eca0d70ef 100644 --- a/src/sonic-yang-models/doc/Configuration.md +++ b/src/sonic-yang-models/doc/Configuration.md @@ -3,70 +3,71 @@ Table of Contents ================= - * [Introduction](#introduction) - * [Configuration](#configuration) - * [Config Load and Save](#config-load-and-save) - * [Incremental Configuration](#incremental-configuration) - * [Redis and Json Schema](#redis-and-json-schema) - * [ACL and Mirroring](#acl-and-mirroring) - * [BGP Device Global](#bgp-device-global) - * [BGP Sessions](#bgp-sessions) - * [BUFFER_PG](#buffer_pg) - * [Buffer pool](#buffer-pool) - * [Buffer profile](#buffer-profile) - * [Buffer queue](#buffer-queue) - * [Buffer port ingress profile list](#buffer-port-ingress-profile-list) - * [Buffer port egress profile list](#buffer-port-egress-profile-list) - * [Cable length](#cable-length) - * [COPP_TABLE](#copp_table) - * [Console](#console) - * [CRM](#crm) - * [Data Plane L3 Interfaces](#data-plane-l3-interfaces) - * [DEFAULT_LOSSLESS_BUFFER_PARAMETER](#DEFAULT_LOSSLESS_BUFFER_PARAMETER) - * [Device Metadata](#device-metadata) - * [Device neighbor metada](#device-neighbor-metada) - * [DSCP_TO_TC_MAP](#dscp_to_tc_map) - * [FLEX_COUNTER_TABLE](#flex_counter_table) - * [KDUMP](#kdump) - * [Kubernetes Master](#kubernetes-master) - * [L2 Neighbors](#l2-neighbors) - * [Loopback Interface](#loopback-interface) - * [LOSSLESS_TRAFFIC_PATTERN](#LOSSLESS_TRAFFIC_PATTERN) - * [Management Interface](#management-interface) - * [Management port](#management-port) - * [Management VRF](#management-vrf) - * [MAP_PFC_PRIORITY_TO_QUEUE](#map_pfc_priority_to_queue) - * [MUX_CABLE](#muxcable) - * [NTP Global Configuration](#ntp-global-configuration) - * [NTP and SYSLOG servers](#ntp-and-syslog-servers) - * [Peer Switch](#peer-switch) - * [Policer](#policer) - * [Port](#port) - * [Port Channel](#port-channel) - * [Portchannel member](#portchannel-member) - * [Scheduler](#scheduler) - * [Port QoS Map](#port-qos-map) - * [Queue](#queue) - * [Restapi](#restapi) - * [SYSLOG Rate Limit](#syslog-rate-limit) - * [Tacplus Server](#tacplus-server) - * [TC to Priority group map](#tc-to-priority-group-map) - * [TC to Queue map](#tc-to-queue-map) - * [Telemetry](#telemetry) - * [Versions](#versions) - * [VLAN](#vlan) - * [VLAN_MEMBER](#vlan_member) - * [VOQ Inband Interface](#voq-inband-interface) - * [VXLAN](#vxlan) + * [Introduction](#introduction) + * [Configuration](#configuration) + * [Config Load and Save](#config-load-and-save) + * [Incremental Configuration](#incremental-configuration) + * [Redis and Json Schema](#redis-and-json-schema) + * [ACL and Mirroring](#acl-and-mirroring) + * [BGP Device Global](#bgp-device-global) + * [BGP Sessions](#bgp-sessions) + * [BUFFER_PG](#buffer_pg) + * [Buffer pool](#buffer-pool) + * [Buffer profile](#buffer-profile) + * [Buffer queue](#buffer-queue) + * [Buffer port ingress profile list](#buffer-port-ingress-profile-list) + * [Buffer port egress profile list](#buffer-port-egress-profile-list) + * [Cable length](#cable-length) + * [COPP_TABLE](#copp_table) + * [Console](#console) + * [CRM](#crm) + * [Data Plane L3 Interfaces](#data-plane-l3-interfaces) + * [DEFAULT_LOSSLESS_BUFFER_PARAMETER](#DEFAULT_LOSSLESS_BUFFER_PARAMETER) + * [Device Metadata](#device-metadata) + * [Device neighbor metada](#device-neighbor-metada) + * [DSCP_TO_TC_MAP](#dscp_to_tc_map) + * [FLEX_COUNTER_TABLE](#flex_counter_table) + * [KDUMP](#kdump) + * [Kubernetes Master](#kubernetes-master) + * [L2 Neighbors](#l2-neighbors) + * [Loopback Interface](#loopback-interface) + * [LOSSLESS_TRAFFIC_PATTERN](#LOSSLESS_TRAFFIC_PATTERN) + * [Management Interface](#management-interface) + * [Management port](#management-port) + * [Management VRF](#management-vrf) + * [MAP_PFC_PRIORITY_TO_QUEUE](#map_pfc_priority_to_queue) + * [MUX_CABLE](#muxcable) + * [NTP Global Configuration](#ntp-global-configuration) + * [NTP and SYSLOG servers](#ntp-and-syslog-servers) + * [Peer Switch](#peer-switch) + * [Policer](#policer) + * [Port](#port) + * [Port Channel](#port-channel) + * [Portchannel member](#portchannel-member) + * [Scheduler](#scheduler) + * [Port QoS Map](#port-qos-map) + * [Queue](#queue) + * [Syslog Rate Limit](#syslog-rate-limit) + * [Sflow](#sflow) + * [Restapi](#restapi) + * [Tacplus Server](#tacplus-server) + * [TC to Priority group map](#tc-to-priority-group-map) + * [TC to Queue map](#tc-to-queue-map) + * [Telemetry](#telemetry) + * [Versions](#versions) + * [VLAN](#vlan) + * [VLAN_MEMBER](#vlan_member) + * [VOQ Inband Interface](#voq-inband-interface) + * [VXLAN](#vxlan) * [Virtual router](#virtual-router) * [LOGGER](#logger) * [WRED_PROFILE](#wred_profile) * [PASSWORD_HARDENING](#password_hardening) * [SYSTEM_DEFAULTS table](#systemdefaults-table) - * [For Developers](#for-developers) + * [For Developers](#for-developers) * [Generating Application Config by Jinja2 Template](#generating-application-config-by-jinja2-template) * [Incremental Configuration by Subscribing to ConfigDB](#incremental-configuration-by-subscribing-to-configdb) - + # Introduction @@ -1455,6 +1456,36 @@ name as object key and member list as attribute. } } ``` +### Sflow + +The below are the tables and their schema for SFLOW feature + +SFLOW + +| Field | Description | Mandatory | Default | Reference | +|------------------|-----------------------------------------------------------------------------------------|-------------|-----------|-------------------------------------------| +| admin_state | Global sflow admin state | | down | | +| polling_interval | The interval within which sFlow data is collected and sent to the configured collectors | | 20 | | +| agent_id | Interface name | | | PORT:name,PORTCHANNEL:name,MGMT_PORT:name, VLAN:name | + +SFLOW_SESSION + +key - port +| Field | Description | Mandatory | Default | Reference | +|-------------|-------------------------------------------------------------------------------------------------------------------------|-------------|-----------|-------------| +| port | Sets sflow session table attributes for either all interfaces or a specific Ethernet interface. | | | PORT:name | +| admin_state | Per port sflow admin state | | up | | +| sample_rate | Sets the packet sampling rate. The rate is expressed as an integer N, where the intended sampling rate is 1/N packets. | | | | + +SFLOW_COLLECTOR + +key - name +| Field | Description | Mandatory | Default | Reference | +|----------------|-----------------------------------------------------------------------------------------|-------------|-----------|-------------| +| name | Name of the Sflow collector | | | | +| collector_ip | IPv4/IPv6 address of the Sflow collector | true | | | +| collector_port | Destination L4 port of the Sflow collector | | 6343 | | +| collector_vrf | Specify the Collector VRF. In this revision, it is either default VRF or Management VRF.| | | | ### Syslog Rate Limit diff --git a/src/sonic-yang-models/tests/files/sample_config_db.json b/src/sonic-yang-models/tests/files/sample_config_db.json index e7b1a90b0185..99c94e96cfae 100644 --- a/src/sonic-yang-models/tests/files/sample_config_db.json +++ b/src/sonic-yang-models/tests/files/sample_config_db.json @@ -1273,7 +1273,8 @@ "collector_port": "6343" }, "collector2": { - "collector_ip": "10.144.1.2" + "collector_ip": "10.144.1.2", + "collector_vrf": "mgmt" } }, diff --git a/src/sonic-yang-models/tests/yang_model_tests/tests/sflow.json b/src/sonic-yang-models/tests/yang_model_tests/tests/sflow.json index 42d09a773c0d..c631bf6979ae 100644 --- a/src/sonic-yang-models/tests/yang_model_tests/tests/sflow.json +++ b/src/sonic-yang-models/tests/yang_model_tests/tests/sflow.json @@ -14,6 +14,21 @@ "desc": "Configure collectors above the specified limit in SFLOW_COLLECTOR table.", "eStr": ["Too many \"SFLOW_COLLECTOR_LIST\" elements"] }, + "SFLOW_TEST_WITH_COLLECTOR_DEFAULT_VRF": { + "desc": "Configure a collector in SFLOW_COLLECTOR table with default VRF." + }, + "SFLOW_TEST_WITH_COLLECTOR_MGMT_VRF": { + "desc": "Configure a collector in SFLOW_COLLECTOR table with mgmt VRF." + }, + "SFLOW_TEST_WITH_COLLECTOR_NON_EXISTING_MGMT_VRF": { + "desc": "Configure a collector in SFLOW_COLLECTOR table with mgmt VRF when MGMT VRF is not configured", + "eStrKey": "Must" + }, + "SFLOW_TEST_WITH_COLLECTOR_INVALID_VRF": { + "desc": "Configure a collector in SFLOW_COLLECTOR table with invalid vrf", + "eStrKey": "Pattern", + "eStr": ["mgmt|default"] + }, "SFLOW_SESSION_TEST": { "desc": "Configure a sflow session in SFLOW_SESSION table." }, diff --git a/src/sonic-yang-models/tests/yang_model_tests/tests_config/sflow.json b/src/sonic-yang-models/tests/yang_model_tests/tests_config/sflow.json index 5d1a561a9f66..07ef40b242a8 100644 --- a/src/sonic-yang-models/tests/yang_model_tests/tests_config/sflow.json +++ b/src/sonic-yang-models/tests/yang_model_tests/tests_config/sflow.json @@ -59,6 +59,69 @@ } }, + "SFLOW_TEST_WITH_COLLECTOR_DEFAULT_VRF": { + "sonic-sflow:sonic-sflow": { + "sonic-sflow:SFLOW_COLLECTOR": { + "SFLOW_COLLECTOR_LIST": [ + { + "name": "collector1", + "collector_ip": "2001:1:2::23", + "collector_vrf": "default" + } + ] + } + } + }, + + "SFLOW_TEST_WITH_COLLECTOR_MGMT_VRF": { + "sonic-sflow:sonic-sflow": { + "sonic-sflow:SFLOW_COLLECTOR": { + "SFLOW_COLLECTOR_LIST": [ + { + "name": "collector1", + "collector_ip": "10.100.12.13", + "collector_vrf": "mgmt" + } + ] + } + }, + "sonic-mgmt_vrf:sonic-mgmt_vrf": { + "sonic-mgmt_vrf:MGMT_VRF_CONFIG": { + "sonic-mgmt_vrf:vrf_global": { + "mgmtVrfEnabled": "true" + } + } + } + }, + + "SFLOW_TEST_WITH_COLLECTOR_NON_EXISTING_MGMT_VRF": { + "sonic-sflow:sonic-sflow": { + "sonic-sflow:SFLOW_COLLECTOR": { + "SFLOW_COLLECTOR_LIST": [ + { + "name": "collector1", + "collector_ip": "10.100.12.13", + "collector_vrf": "mgmt" + } + ] + } + } + }, + + "SFLOW_TEST_WITH_COLLECTOR_INVALID_VRF": { + "sonic-sflow:sonic-sflow": { + "sonic-sflow:SFLOW_COLLECTOR": { + "SFLOW_COLLECTOR_LIST": [ + { + "name": "collector1", + "collector_ip": "10.100.12.13", + "collector_vrf": "Vrf1" + } + ] + } + } + }, + "SFLOW_SESSION_TEST": { "sonic-port:sonic-port": { "sonic-port:PORT": { diff --git a/src/sonic-yang-models/yang-models/sonic-sflow.yang b/src/sonic-yang-models/yang-models/sonic-sflow.yang index adbeda80c4da..2865e05f34b9 100644 --- a/src/sonic-yang-models/yang-models/sonic-sflow.yang +++ b/src/sonic-yang-models/yang-models/sonic-sflow.yang @@ -25,6 +25,10 @@ module sonic-sflow{ import sonic-mgmt_port { prefix mgmt-port; } + import sonic-mgmt_vrf { + prefix mvrf; + } + description "SFLOW yang Module for SONiC OS"; @@ -43,17 +47,34 @@ module sonic-sflow{ type string { length 1..64; } + description "Name of the Sflow collector"; } leaf collector_ip { mandatory true; type inet:ip-address; + description "IPv4/IPv6 address of the Sflow collector"; } leaf collector_port { type inet:port-number; default 6343; + description "Destination L4 port of the Sflow collector"; + } + + leaf collector_vrf { + must "(current() != 'mgmt') or (/mvrf:sonic-mgmt_vrf/mvrf:MGMT_VRF_CONFIG/mvrf:vrf_global/mvrf:mgmtVrfEnabled = 'true')" { + error-message "Must condition not satisfied. Try enable Management VRF."; + } + + type string { + pattern "mgmt|default"; + } + + description + "Specify the Collector VRF. In this revision, it is either + default VRF or Management VRF."; } } /* end of list SFLOW_COLLECTOR_LIST */ @@ -78,6 +99,7 @@ module sonic-sflow{ leaf admin_state { type stypes:admin_status; default up; + description "Per port sflow admin state"; } leaf sample_rate { @@ -99,6 +121,7 @@ module sonic-sflow{ leaf admin_state { type stypes:admin_status; default down; + description "Global sflow admin state"; } leaf polling_interval { From 7a0152ad15a3f5f06d29e6e72f7c0e0d425f0ebd Mon Sep 17 00:00:00 2001 From: "Robert J. Halstead" Date: Thu, 1 Dec 2022 10:05:47 -0800 Subject: [PATCH 09/69] [sonic-pins] update submodule ptr (#12644) Update submodule for sonic-pins to be aligned to following swss PRs *New P4Orch development. sonic-swss#2425 *Upstream new development on p4orch sonic-swss#2237 --- src/sonic-p4rt/sonic-pins | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-p4rt/sonic-pins b/src/sonic-p4rt/sonic-pins index 4168fdce1c11..7cf4a2aca08b 160000 --- a/src/sonic-p4rt/sonic-pins +++ b/src/sonic-p4rt/sonic-pins @@ -1 +1 @@ -Subproject commit 4168fdce1c1131b923a02b8fffa6089a17f14a11 +Subproject commit 7cf4a2aca08b7378aa7a275c6a5778d7afb3eaa6 From aaeafa8411829160dca80e244c591ce268c151a8 Mon Sep 17 00:00:00 2001 From: Kalimuthu-Velappan <53821802+Kalimuthu-Velappan@users.noreply.github.com> Date: Fri, 2 Dec 2022 05:58:45 +0530 Subject: [PATCH 10/69] 02.Version cache - docker cache build framework (#12001) During docker build, host files can be passed to the docker build through docker context files. But there is no straightforward way to transfer the files from docker build to host. This feature provides a tricky way to pass the cache contents from docker build to host. It tar's the cached content and encodes them as base64 format and passes it through a log file with a special tag as 'VCSTART and VCENT'. Slave.mk in the host, it extracts the cache contents from the log and stores them in the cache folder. Cache contents are encoded as base64 format for easy passing. #### Why I did it #### How I did it #### How to verify it --- Makefile.work | 13 +++++++++---- files/build_templates/build_docker_cache.j2 | 11 +++++++++++ scripts/collect_build_version_files.sh | 2 +- scripts/collect_docker_version_files.sh | 17 +++++++++++++++-- scripts/collect_host_image_version_files.sh | 4 +++- scripts/prepare_docker_buildinfo.sh | 13 +++++++++++-- slave.mk | 15 ++++++++++++--- sonic-slave-buster/Dockerfile.j2 | 2 +- src/sonic-build-hooks/scripts/buildinfo_base.sh | 9 +++++++++ .../scripts/post_run_buildinfo | 6 ++++-- src/sonic-build-hooks/scripts/pre_run_buildinfo | 6 ++++++ 11 files changed, 82 insertions(+), 16 deletions(-) create mode 100644 files/build_templates/build_docker_cache.j2 diff --git a/Makefile.work b/Makefile.work index adc40f381613..1aeb87568640 100644 --- a/Makefile.work +++ b/Makefile.work @@ -228,8 +228,12 @@ SLAVE_TAG = $(shell \ COLLECT_DOCKER=DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \ scripts/collect_docker_version_files.sh \ - $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) \ - target + $(SLAVE_BASE_IMAGE) \ + target \ + $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) \ + $(SLAVE_DIR) \ + $(SLAVE_DIR)/Dockerfile + OVERLAY_MODULE_CHECK := \ lsmod | grep -q "^overlay " &>/dev/null || \ zgrep -q 'CONFIG_OVERLAY_FS=y' /proc/config.gz &>/dev/null || \ @@ -376,7 +380,8 @@ DOCKER_SLAVE_BASE_BUILD = docker build --no-cache \ --build-arg http_proxy=$(http_proxy) \ --build-arg https_proxy=$(https_proxy) \ --build-arg no_proxy=$(no_proxy) \ - $(SLAVE_DIR) $(SPLIT_LOG) $(DOCKER_BASE_LOG) + $(SLAVE_DIR) \ + $(SPLIT_LOG) $(DOCKER_BASE_LOG) DOCKER_BASE_PULL = docker pull \ $(REGISTRY_SERVER):$(REGISTRY_PORT)/$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) @@ -507,7 +512,7 @@ SONIC_BUILD_INSTRUCTION := $(MAKE) \ ifeq ($(filter clean,$(MAKECMDGOALS)),) -COLLECT_BUILD_VERSION = { DBGOPT='$(DBGOPT)' scripts/collect_build_version_files.sh \$$?; } +COLLECT_BUILD_VERSION = { scripts/collect_build_version_files.sh \$$?; } endif ifdef SOURCE_FOLDER diff --git a/files/build_templates/build_docker_cache.j2 b/files/build_templates/build_docker_cache.j2 new file mode 100644 index 000000000000..9f1d0eebf77f --- /dev/null +++ b/files/build_templates/build_docker_cache.j2 @@ -0,0 +1,11 @@ +# Base docker build +FROM {{IMAGENAME}} + +# Copy the cache data to host +From scratch as output +COPY --from={{IMAGENAME}} /cache.tgz cache.tgz + +# Clean up the cache data +FROM {{IMAGENAME}} as final +RUN rm /cache.tgz + diff --git a/scripts/collect_build_version_files.sh b/scripts/collect_build_version_files.sh index b650e421996b..e35ca0590334 100755 --- a/scripts/collect_build_version_files.sh +++ b/scripts/collect_build_version_files.sh @@ -23,6 +23,6 @@ mkdir -p $VERSION_SLAVE_PATH scripts/versions_manager.py merge -t $VERSION_SLAVE_PATH -b $LOG_VERSION_PATH -e $POST_VERSION_PATH -rm -rf $BUILD_VERSION_PATH/* +[ -d $BUILD_VERSION_PATH ] && rm -rf $BUILD_VERSION_PATH/* exit $RET diff --git a/scripts/collect_docker_version_files.sh b/scripts/collect_docker_version_files.sh index 546d8458f259..1f5cc544972c 100755 --- a/scripts/collect_docker_version_files.sh +++ b/scripts/collect_docker_version_files.sh @@ -1,12 +1,17 @@ #!/bin/bash +set -x DOCKER_IMAGE=$1 TARGET_PATH=$2 +DOCKER_IMAGE_TAG=$3 +DOCKER_PATH=$4 +DOCKER_FILE=$5 [ -z "$TARGET_PATH" ] && TARGET_PATH=./target DOCKER_IMAGE_NAME=$(echo $DOCKER_IMAGE | cut -d: -f1 | sed "s/-$DOCKER_USERNAME\$//") -DOCKER_CONTAINER=$DOCKER_IMAGE_NAME +#Create the container specific to the user tag and slave tag +DOCKER_CONTAINER=${DOCKER_IMAGE_TAG/:/-} TARGET_VERSIONS_PATH=$TARGET_PATH/versions/dockers/$DOCKER_IMAGE_NAME [ -d $TARGET_VERSIONS_PATH ] && rm -rf $TARGET_VERSIONS_PATH @@ -18,8 +23,16 @@ export DOCKER_CLI_EXPERIMENTAL=enabled if docker container inspect $DOCKER_IMAGE > /dev/null 2>&1; then docker container rm $DOCKER_IMAGE > /dev/null fi -docker create --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE +docker create --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE_TAG docker cp -L $DOCKER_CONTAINER:/etc/os-release $TARGET_VERSIONS_PATH/ docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/pre-versions $TARGET_VERSIONS_PATH/ docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/post-versions $TARGET_VERSIONS_PATH/ + +# Save the cache contents from docker build +IMAGENAME=${DOCKER_IMAGE_TAG} j2 files/build_templates/build_docker_cache.j2 > ${DOCKER_FILE}.cleanup +docker tag ${DOCKER_IMAGE_TAG} tmp-${DOCKER_IMAGE_TAG} +DOCKER_BUILDKIT=1 docker build -f ${DOCKER_PATH}/Dockerfile.cleanup --target output -o target/vcache/${DOCKER_IMAGE_NAME} ${DOCKER_PATH} +DOCKER_BUILDKIT=1 docker build -f ${DOCKER_PATH}/Dockerfile.cleanup --no-cache --target final --tag ${DOCKER_IMAGE_TAG} ${DOCKER_PATH} +docker rmi tmp-${DOCKER_IMAGE_TAG} + docker container rm $DOCKER_CONTAINER diff --git a/scripts/collect_host_image_version_files.sh b/scripts/collect_host_image_version_files.sh index 2cabc049d9c7..d0d7aabcf9d2 100755 --- a/scripts/collect_host_image_version_files.sh +++ b/scripts/collect_host_image_version_files.sh @@ -3,11 +3,13 @@ TARGET=$1 FILESYSTEM_ROOT=$2 VERSIONS_PATH=$TARGET/versions/host-image +IMAGENAME="host-image" [ -d $VERSIONS_PATH ] && sudo rm -rf $VERSIONS_PATH mkdir -p $VERSIONS_PATH -sudo LANG=C chroot $FILESYSTEM_ROOT post_run_buildinfo +mkdir -p target/vcache/${IMAGENAME} +sudo LANG=C chroot $FILESYSTEM_ROOT post_run_buildinfo ${IMAGENAME} cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/pre-versions $VERSIONS_PATH/ cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/post-versions $VERSIONS_PATH/ diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh index 3c1104eddc53..92fc78c9e9d3 100755 --- a/scripts/prepare_docker_buildinfo.sh +++ b/scripts/prepare_docker_buildinfo.sh @@ -11,6 +11,7 @@ DISTRO=$5 DOCKERFILE_PATH=$(dirname "$DOCKERFILE_TARGE") BUILDINFO_PATH="${DOCKERFILE_PATH}/buildinfo" BUILDINFO_VERSION_PATH="${BUILDINFO_PATH}/versions" +DOCKER_PATH=$(dirname $DOCKERFILE) [ -d $BUILDINFO_PATH ] && rm -rf $BUILDINFO_PATH mkdir -p $BUILDINFO_VERSION_PATH @@ -31,8 +32,11 @@ scripts/docker_version_control.sh $@ DOCKERFILE_PRE_SCRIPT='# Auto-Generated for buildinfo COPY ["buildinfo", "/usr/local/share/buildinfo"] +COPY vcache/ /sonic/target/vcache/'${IMAGENAME}' RUN dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb -RUN pre_run_buildinfo' +ENV IMAGENAME='${IMAGENAME}' +RUN pre_run_buildinfo '${IMAGENAME}' +' # Add the auto-generate code if it is not added in the target Dockerfile if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKERFILE_TARGE; then @@ -42,7 +46,7 @@ if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKE awk -v text="${DOCKERFILE_PRE_SCRIPT}" -v linenumber=$LINE_NUMBER 'NR==linenumber{print text}1' $DOCKERFILE > $TEMP_FILE # Append the docker build script at the end of the docker file - echo -e "\nRUN post_run_buildinfo" >> $TEMP_FILE + echo -e "\nRUN post_run_buildinfo ${IMAGENAME} " >> $TEMP_FILE cat $TEMP_FILE > $DOCKERFILE_TARGE rm -f $TEMP_FILE @@ -55,3 +59,8 @@ cp -rf src/sonic-build-hooks/buildinfo/* $BUILDINFO_PATH scripts/versions_manager.py generate -t "$BUILDINFO_VERSION_PATH" -n "$IMAGENAME" -d "$DISTRO" -a "$ARCH" touch $BUILDINFO_VERSION_PATH/versions-deb + +# Create the cache directories +LOCAL_CACHE_DIR=target/vcache/${IMAGENAME} +mkdir -p ${LOCAL_CACHE_DIR} ${DOCKER_PATH}/vcache/ +chmod -f 777 ${LOCAL_CACHE_DIR} ${DOCKER_PATH}/vcache/ diff --git a/slave.mk b/slave.mk index ae3c6b4dcc3d..8f6ec448f710 100644 --- a/slave.mk +++ b/slave.mk @@ -111,6 +111,7 @@ configure : $(Q)mkdir -p $(PYTHON_DEBS_PATH) $(Q)mkdir -p $(PYTHON_WHEELS_PATH) $(Q)mkdir -p $(DPKG_ADMINDIR_PATH) + $(Q)mkdir -p $(TARGET_PATH)/vcache $(Q)echo $(PLATFORM) > .platform $(Q)echo $(PLATFORM_ARCH) > .arch @@ -914,8 +915,10 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_SIMPLE_DOCKER_IMAGES)) : $(TARGET_PATH)/%.g --label Tag=$(SONIC_IMAGE_VERSION) \ -f $(TARGET_DOCKERFILE)/Dockerfile.buildinfo \ -t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG) + if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi - scripts/collect_docker_version_files.sh $(DOCKER_IMAGE_REF) $(TARGET_PATH) + scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG) + $(call docker-image-save,$*,$@) # Clean up if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi @@ -993,6 +996,7 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform mkdir -p $($*.gz_PATH)/files $(LOG) mkdir -p $($*.gz_PATH)/python-debs $(LOG) mkdir -p $($*.gz_PATH)/python-wheels $(LOG) + mkdir -p $(TARGET_PATH)/vcache/$* $($*.gz_PATH)/vcache $(LOG) sudo mount --bind $($*.gz_DEBS_PATH) $($*.gz_PATH)/debs $(LOG) sudo mount --bind $($*.gz_FILES_PATH) $($*.gz_PATH)/files $(LOG) sudo mount --bind $(PYTHON_DEBS_PATH) $($*.gz_PATH)/python-debs $(LOG) @@ -1034,9 +1038,11 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform --label Tag=$(SONIC_IMAGE_VERSION) \ $($(subst -,_,$(notdir $($*.gz_PATH)))_labels) \ -t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG) + if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi - scripts/collect_docker_version_files.sh $(DOCKER_IMAGE_REF) $(TARGET_PATH) + scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $($*.gz_PATH)/Dockerfile $(LOG) if [ ! -z $(filter $*.gz,$(SONIC_PACKAGES_LOCAL)) ]; then docker tag $(DOCKER_IMAGE_REF) $*:$(SONIC_IMAGE_VERSION); fi + $(call docker-image-save,$*,$@) # Clean up if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi @@ -1065,6 +1071,7 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG mkdir -p $($*.gz_PATH)/debs $(LOG) sudo mount --bind $($*.gz_DEBS_PATH) $($*.gz_PATH)/debs $(LOG) + mkdir -p $(TARGET_PATH)/vcache/$*-dbg $($*.gz_PATH)/vcache $(LOG) # Export variables for j2. Use path for unique variable names, e.g. docker_orchagent_debs $(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_dbg_debs=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_DBG_DEPENDS),RDEPENDS))\n" | awk '!a[$$0]++')) $(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_image_dbgs=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_DBG_IMAGE_PACKAGES)))\n" | awk '!a[$$0]++')) @@ -1088,9 +1095,11 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG --label Tag=$(SONIC_IMAGE_VERSION) \ --file $($*.gz_PATH)/Dockerfile-dbg \ -t $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH) $(LOG) + if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi - scripts/collect_docker_version_files.sh $(DOCKER_DBG_IMAGE_REF) $(TARGET_PATH) + scripts/collect_docker_version_files.sh $*-dbg $(TARGET_PATH) $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH) $($*.gz_PATH)/Dockerfile-dbg $(LOG) if [ ! -z $(filter $*.gz,$(SONIC_PACKAGES_LOCAL)) ]; then docker tag $(DOCKER_IMAGE_REF) $*:$(SONIC_IMAGE_VERSION); fi + $(call docker-image-save,$*-$(DBG_IMAGE_MARK),$@) # Clean up docker rmi -f $(DOCKER_IMAGE_REF) &> /dev/null || true diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2 index 73e9febda14b..0c2ce405f2ad 100644 --- a/sonic-slave-buster/Dockerfile.j2 +++ b/sonic-slave-buster/Dockerfile.j2 @@ -599,7 +599,7 @@ RUN add-apt-repository \ $(lsb_release -cs) \ stable" RUN apt-get update -RUN apt-get install -y docker-ce=5:18.09.5~3-0~debian-buster docker-ce-cli=5:18.09.5~3-0~debian-buster +RUN apt-get install -y docker-ce=5:20.10.21~3-0~debian-buster docker-ce-cli=5:20.10.21~3-0~debian-buster RUN echo "DOCKER_OPTS=\"--experimental --storage-driver=vfs {{ DOCKER_EXTRA_OPTS }}\"" >> /etc/default/docker RUN update-alternatives --set iptables /usr/sbin/iptables-legacy diff --git a/src/sonic-build-hooks/scripts/buildinfo_base.sh b/src/sonic-build-hooks/scripts/buildinfo_base.sh index ff249dfdfb91..1494993448b8 100755 --- a/src/sonic-build-hooks/scripts/buildinfo_base.sh +++ b/src/sonic-build-hooks/scripts/buildinfo_base.sh @@ -15,6 +15,15 @@ REPR_MIRROR_URL_PATTERN='http:\/\/packages.trafficmanager.net\/debian' DPKG_INSTALLTION_LOCK_FILE=/tmp/.dpkg_installation.lock . $BUILDINFO_PATH/config/buildinfo.config +if [ -e /vcache ]; then + PKG_CACHE_PATH=/vcache/${IMAGENAME} +else + PKG_CACHE_PATH=/sonic/target/vcache/${IMAGENAME} +fi +PKG_CACHE_FILE_NAME=${PKG_CACHE_PATH}/cache.tgz +mkdir -p ${PKG_CACHE_PATH} + + URL_PREFIX=$(echo "${PACKAGE_URL_PREFIX}" | sed -E "s#(//[^/]*/).*#\1#") diff --git a/src/sonic-build-hooks/scripts/post_run_buildinfo b/src/sonic-build-hooks/scripts/post_run_buildinfo index 97f47f7efcf1..00f2b0722dab 100755 --- a/src/sonic-build-hooks/scripts/post_run_buildinfo +++ b/src/sonic-build-hooks/scripts/post_run_buildinfo @@ -1,12 +1,14 @@ #!/bin/bash +IMAGENAME=$1 + . /usr/local/share/buildinfo/scripts/buildinfo_base.sh # Collect the version files collect_version_files $POST_VERSION_PATH -[ -d $BUILD_VERSION_PATH ] && [ ! -z "$(ls -A $BUILD_VERSION_PATH)" ] && cp -rf $BUILD_VERSION_PATH/* $POST_VERSION_PATH -rm -rf $BUILD_VERSION_PATH/* +#Save the cache file for exporting it to host. +tar -C ${PKG_CACHE_PATH} --exclude=cache.tgz -zcvf /cache.tgz . # Disable the build hooks symlink_build_hooks -d diff --git a/src/sonic-build-hooks/scripts/pre_run_buildinfo b/src/sonic-build-hooks/scripts/pre_run_buildinfo index 5a8f00b55ecb..eb4d04225ec5 100755 --- a/src/sonic-build-hooks/scripts/pre_run_buildinfo +++ b/src/sonic-build-hooks/scripts/pre_run_buildinfo @@ -1,5 +1,7 @@ #!/bin/bash +IMAGENAME=$1 + . /usr/local/share/buildinfo/scripts/buildinfo_base.sh [ -d $DIFF_VERSION_PATH ] && rm -rf $DIFF_VERSION_PATH @@ -15,10 +17,14 @@ update_version_files symlink_build_hooks set_reproducible_mirrors +mkdir -p /var/cache/apt/archives/ + chmod -R a+rw $BUILDINFO_PATH if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ] && [ -f $VERSION_DEB_PREFERENCE ]; then cp -f $VERSION_DEB_PREFERENCE /etc/apt/preferences.d/ fi +DISTRO=${DISTRO} apt-get update && apt-get install -y rsync + exit 0 From f10f79b7546c6efe48be5a0c44510c4f43e46293 Mon Sep 17 00:00:00 2001 From: Santhosh Kumar T <53558409+santhosh-kt@users.noreply.github.com> Date: Fri, 2 Dec 2022 06:46:10 +0530 Subject: [PATCH 11/69] [DellEMC] Master: S6100: SSD upgrade status: Moving from smartctl to iSMART (#12784) Why I did it smartctl tool is available only in PMON docker. Hence, the tool may be not accessible incase PMON docker goes down. Using iSMART_64 tool to fetch the SSD firmware version and device model information. How I did it Replacing smartctl with iSMART_64. --- .../s6100/scripts/s6100_ssd_upgrade_status.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/platform/broadcom/sonic-platform-modules-dell/s6100/scripts/s6100_ssd_upgrade_status.sh b/platform/broadcom/sonic-platform-modules-dell/s6100/scripts/s6100_ssd_upgrade_status.sh index 2c074fd3319c..b06de8184ca0 100755 --- a/platform/broadcom/sonic-platform-modules-dell/s6100/scripts/s6100_ssd_upgrade_status.sh +++ b/platform/broadcom/sonic-platform-modules-dell/s6100/scripts/s6100_ssd_upgrade_status.sh @@ -15,11 +15,14 @@ fi SSD_UPGRADE_LOG="$SSD_FW_UPGRADE/upgrade.log" +iSMART="/usr/local/bin/iSMART_64" +iSMART_OPTIONS="-d /dev/sda" +iSMART_CMD=`$iSMART $iSMART_OPTIONS` SMART_CMD=`smartctl -a /dev/sda` -SSD_FW_VERSION=$(echo "$SMART_CMD" | grep "Firmware Version" | awk '{print $NF}') +SSD_FW_VERSION=$(echo "$iSMART_CMD" | grep "FW Version" | awk '{print $NF}') SSD_FW_VERSION=${SSD_FW_VERSION,,} -SSD_MODEL=$(echo "$SMART_CMD" | grep "Device Model" | awk '{print $NF}') +SSD_MODEL=$(echo "$iSMART_CMD" | grep "Model" | awk '{print $NF}') if [ -e $SSD_FW_UPGRADE/GPIO7_pending_upgrade ]; then if [ $SSD_MODEL == "3IE" ] && [ $SSD_FW_VERSION == "s141002c" ]; then @@ -34,10 +37,6 @@ fi echo "$0 `date` SSD FW upgrade logs post reboot." >> $SSD_UPGRADE_LOG -iSMART="/usr/local/bin/iSMART_64" -iSMART_OPTIONS="-d /dev/sda" -iSMART_CMD=`$iSMART $iSMART_OPTIONS` - SSD_UPGRADE_STATUS1=`io_rd_wr.py --set --val 06 --offset 210; io_rd_wr.py --set --val 09 --offset 211; io_rd_wr.py --get --offset 212` SSD_UPGRADE_STATUS1=$(echo "$SSD_UPGRADE_STATUS1" | awk '{print $NF}') @@ -69,11 +68,10 @@ elif [ $SSD_FW_VERSION == "s210506g" ] || [ $SSD_FW_VERSION == "s16425cg" ]; the echo "$0 `date` soft-/fast-/warm-reboot is allowed." >> $SSD_UPGRADE_LOG else + rm -rf $SSD_FW_UPGRADE/GPIO7_* + touch $SSD_FW_UPGRADE/GPIO7_high + systemctl start --no-block s6100-ssd-monitor.timer if [ $SSD_UPGRADE_STATUS1 == "0" ]; then - rm -rf $SSD_FW_UPGRADE/GPIO7_* - touch $SSD_FW_UPGRADE/GPIO7_high - systemctl start --no-block s6100-ssd-monitor.timer - if [ $SSD_MODEL == "3IE" ];then echo "$0 `date` SSD FW upgraded from S141002C to S210506G in first mp_64." >> $SSD_UPGRADE_LOG else From c7c921166cab1d18b5f1219e156864000e385c55 Mon Sep 17 00:00:00 2001 From: StormLiangMS <89824293+StormLiangMS@users.noreply.github.com> Date: Fri, 2 Dec 2022 17:43:56 +0800 Subject: [PATCH 12/69] Add 202211 backport option for the PR review template (#12884) Why I did it Add 202211 backport option How I did it Add option in .github/pull_request_template.md How to verify it --- .github/pull_request_template.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 85645d8ff7ee..08f764e82c9d 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -32,6 +32,7 @@ - [ ] 202106 - [ ] 202111 - [ ] 202205 +- [ ] 202211 #### Description for the changelog