Merge branch 'master' into dx010-xcvrd-event-support

.gitignore

@@ -1,4 +1,8 @@
+# OS-generated files
+.DS_Store
+
 # Build system related
+.arch
 .platform
 .screen
 
@@ -9,45 +13,98 @@ target/
 *.deb
 *.changes
 *.buildinfo
+*.tar
+*.xz
+*.gz
+*-dbg
+*dbg.j2
+*.img
 
 # Subdirectories in src
+src/bash/*
+!src/bash/Makefile
+src/ixgbe/*
+!src/ixgbe/Makefile
+src/isc-dhcp/*
+!src/isc-dhcp/Makefile
+!src/isc-dhcp/patch
+src/socat/*
+!src/socat/Makefile
+!src/socat/*.patch
+src/tacacs/*
+src/tacacs/nsm/*
+src/tacacs/pam/*
+!src/tacacs/nsm/Makefile
+!src/tacacs/nsm/*.patch
+!src/tacacs/pam/Makefile
+!src/tacacs/pam/*.patch
 src/hiredis/*
 !src/hiredis/Makefile
 src/igb/*
 !src/igb/Makefile
 src/initramfs-tools/*
 !src/initramfs-tools/Makefile
+src/iproute2/*
+!src/iproute2/Makefile
 src/isc-dhcp/*
 !src/isc-dhcp/Makefile
 !src/isc-dhcp/patch/
 src/libnl3/*
 !src/libnl3/Makefile
 src/libteam/*
 !src/libteam/Makefile
+src/libyang/*
+!src/libyang/Makefile
 src/lldpd/*
 !src/lldpd/Makefile
 !src/lldpd/patch/
+src/lm-sensors/*
+!src/lm-sensors/Makefile
 src/mpdecimal/*
 !src/mpdecimal/Makefile
 src/python-click/*
 !src/python-click/Makefile
 src/python3/*
 !src/python3/Makefile
+src/radvd/*
+!src/radvd/Makefile
+!src/radvd/patch/
 src/redis/*
 !src/redis/Makefile
+src/smartmontools/*
+!src/smartmontools/Makefile
 src/snmpd/*
 !src/snmpd/Makefile
 src/sonic-device-data/src/device/
 src/sonic-device-data/src/debian/
 src/supervisor/*
 !src/supervisor/Makefile
 !src/supervisor/patch/
+src/swig/*
+!src/swig/Makefile
+src/systemd-sonic-generator/systemd-sonic-generator
+src/systemd-sonic-generator/debian/*
+!src/systemd-sonic-generator/debian/changelog
+!src/systemd-sonic-generator/debian/compat
+!src/systemd-sonic-generator/debian/control
+!src/systemd-sonic-generator/debian/rules
+src/telemetry/debian/*
+!src/telemetry/debian/changelog
+!src/telemetry/debian/compat
+!src/telemetry/debian/control
+!src/telemetry/debian/rules
+!src/telemetry/debian/telemetry.init.d
 src/thrift/*
+!src/thrift/patch/
 !src/thrift/Makefile
 
 # Autogenerated Dockerfiles
+sonic-slave/Dockerfile
+sonic-slave-stretch/Dockerfile
 dockers/docker-base/Dockerfile
+dockers/docker-base-stretch/Dockerfile
 dockers/docker-config-engine/Dockerfile
+dockers/docker-config-engine-stretch/Dockerfile
 dockers/docker-database/Dockerfile
 dockers/docker-dhcp-relay/Dockerfile
 dockers/docker-fpm-frr/Dockerfile
@@ -73,3 +130,10 @@ installer/x86_64/platforms/
 src/sonic-config-engine/**/*.pyc
 src/sonic-config-engine/build
 src/sonic-config-engine/sonic_config_engine.egg-info
+src/sonic-daemon-base/**/*.pyc
+src/sonic-daemon-base/build
+src/sonic-daemon-base/sonic_daemon_base.egg-info
+
+# Misc. files
+files/initramfs-tools/arista-convertfs
+files/initramfs-tools/union-mount

.gitmodules

@@ -47,6 +47,7 @@
 [submodule "src/sonic-frr/frr"]
 	path = src/sonic-frr/frr
 	url = https://github.com/Azure/sonic-frr.git
+	branch = frr/7.0
 [submodule "platform/p4/p4-hlir/p4-hlir-v1.1"]
 	path = platform/p4/p4-hlir/p4-hlir-v1.1
 	url = https://github.com/p4lang/p4-hlir.git
@@ -62,3 +63,6 @@
 [submodule "src/redis-dump-load"]
 	path = src/redis-dump-load
 	url = https://github.com/p/redis-dump-load.git
+[submodule "platform/mellanox/mlnx-sai/SAI-Implementation"]
+	path = platform/mellanox/mlnx-sai/SAI-Implementation
+	url = https://github.com/Mellanox/SAI-Implementation

Makefile

@@ -1,20 +1,20 @@
 # SONiC make file
 
-NOSTRETCH ?= 0
+NOJESSIE ?= 0
 
 %::
 	@echo "+++ --- Making $@ --- +++"
-ifeq ($(NOSTRETCH), 0)
-	BLDENV=stretch make -f Makefile.work stretch
+ifeq ($(NOJESSIE), 0)
+	EXTRA_JESSIE_TARGETS=$(notdir $@) make -f Makefile.work jessie
 endif
-	make -f Makefile.work $@
+	BLDENV=stretch make -f Makefile.work $@
 
-stretch:
+jessie:
 	@echo "+++ Making $@ +++"
-ifeq ($(NOSTRETCH), 0)
-	BLDENV=stretch make -f Makefile.work stretch
+ifeq ($(NOJESSIE), 0)
+	make -f Makefile.work jessie
 endif
 
 clean reset init configure showtag sonic-slave-build sonic-slave-bash :
 	@echo "+++ Making $@ +++"
-	make -f Makefile.work $@
+	BLDENV=stretch make -f Makefile.work $@

Makefile.work

@@ -20,13 +20,15 @@
 #  *                Please note that with current Stretch build structure,
 #  *                user of KEEP_SLAVE_ON feature will have to be conscious
 #  *                about which docker to stay inside after build is done.
-#  *                - If user desires to stay inside Stretch docker, please issue
-#  *                  make KEEP_SLAVE_ON=yes stretch
 #  *                - If user desires to stay inside Jessie docker, please issue
-#  *                  (a successful "make stretch" may be needed before the following command)
-#  *                  make NOSTRETCH=1 KEEP_SLAVE_ON=yes <any jessie target>
+#  *                  make KEEP_SLAVE_ON=yes jessie
+#  *                - If user desires to stay inside Stretch docker, please issue
+#  *                  make NOJESSIE=1 KEEP_SLAVE_ON=yes <any target>
 #  * SOURCE_FOLDER: host path to be mount as /var/$(USER)/src, only effective when KEEP_SLAVE_ON=yes
 #  * SONIC_BUILD_JOBS: Specifying number of concurrent build job(s) to run
+#  * VS_PREPARE_MEM: Prepare memory in VS build (drop cache and compact).
+#  *                 Default: yes
+#  *                 Values:  yes, no
 #  * KERNEL_PROCURE_METHOD: Specifying method of obtaining kernel Debian package: download or build
 #
 ###############################################################################
@@ -40,26 +42,50 @@ ifeq ($(USER), root)
 $(error Add your user account to docker group and use your user account to make. root or sudo are not supported!)
 endif
 
+# Check for j2cli availability
+J2_VER := $(shell j2 --version 2>&1 | grep j2cli | awk '{printf $$2}')
+ifeq ($(J2_VER),)
+$(error Install j2cli)
+endif
+
+# Check for minimum Docker version on build host
+# Note: Using the greater of CE (17.05.0) and EE (17.06.1) versions that support ARG before FROM
+docker_min := 17.06.1
+docker_min_ver := $(shell echo "$(docker_min)" | awk -F. '{printf("%d%03d%03d\n",$$1,$$2,$$3);}' 2>/dev/null)
+docker_ver := $(shell docker info 2>/dev/null | grep -i "server version" | rev | cut -d' ' -f1 | rev | awk -F. '{printf("%d%03d%03d\n",$$1,$$2,$$3);}' 2>/dev/null)
+docker_is_valid := $(shell if [[ "$(docker_ver)" -lt $(docker_min_ver) ]] ; then echo "0"; else echo "1"; fi)
+ifeq (0,$(docker_is_valid))
+$(error SONiC requires Docker version $(docker_min) or later)
+endif
+
 # Remove lock file in case previous run was forcefully stopped
 $(shell rm -f .screen)
 
 MAKEFLAGS += -B
 
+CONFIGURED_ARCH := $(shell [ -f .arch ] && cat .arch || echo $(PLATFORM_ARCH))
+ifeq ($(CONFIGURED_ARCH),)
+    override CONFIGURED_ARCH = amd64
+endif
+ifeq ($(PLATFORM_ARCH),)
+	override PLATFORM_ARCH = $(CONFIGURED_ARCH)
+endif
+
 ifeq ($(BLDENV), stretch)
-SLAVE_BASE_TAG = $(shell sha1sum sonic-slave-stretch/Dockerfile | awk '{print substr($$1,0,11);}')
-SLAVE_TAG = $(shell cat sonic-slave-stretch/Dockerfile.user sonic-slave-stretch/Dockerfile | sha1sum | awk '{print substr($$1,0,11);}')
-SLAVE_BASE_IMAGE = sonic-slave-stretch-base
-SLAVE_IMAGE = sonic-slave-stretch-$(USER)
 SLAVE_DIR = sonic-slave-stretch
 else
-SLAVE_BASE_TAG = $(shell sha1sum sonic-slave/Dockerfile | awk '{print substr($$1,0,11);}')
-SLAVE_TAG = $(shell cat sonic-slave/Dockerfile.user sonic-slave/Dockerfile | sha1sum | awk '{print substr($$1,0,11);}')
-SLAVE_BASE_IMAGE = sonic-slave-base
-SLAVE_IMAGE = sonic-slave-$(USER)
 SLAVE_DIR = sonic-slave
 endif
+SLAVE_BASE_TAG = $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile && sha1sum $(SLAVE_DIR)/Dockerfile | awk '{print substr($$1,0,11);}')
+SLAVE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile.user $(SLAVE_DIR)/Dockerfile | sha1sum | awk '{print substr($$1,0,11);}')
+SLAVE_BASE_IMAGE = $(SLAVE_DIR)-base
+SLAVE_IMAGE = $(SLAVE_BASE_IMAGE)-$(USER)
 
-OVERLAY_MODULE_CHECK := lsmod | grep "^overlay " > /dev/null 2>&1 || (echo "ERROR: Module 'overlay' not loaded. Try running 'sudo modprobe overlay'."; exit 1)
+OVERLAY_MODULE_CHECK := \
+    lsmod | grep -q "^overlay " &>/dev/null || \
+    zgrep -q 'CONFIG_OVERLAY_FS=y' /proc/config.gz &>/dev/null || \
+    grep -q 'CONFIG_OVERLAY_FS=y' /boot/config-$(shell uname -r) &>/dev/null || \
+    (echo "ERROR: Module 'overlay' not loaded. Try running 'sudo modprobe overlay'."; exit 1)
 
 BUILD_TIMESTAMP := $(shell date +%Y%m%d\.%H%M%S)
 
@@ -83,27 +109,55 @@ include rules/config
 ifeq ($(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD), y)
     DOCKER_RUN += -v /var/run/docker.sock:/var/run/docker.sock
 endif
+ifneq (,$(filter $(CONFIGURED_ARCH), armhf arm64))
+    # Multiarch docker cannot start dockerd service due to iptables cannot run over different arch kernel
+    SONIC_SERVICE_DOCKERD_FOR_MULTIARCH=y
+    SONIC_NATIVE_DOCKERD_FOR_MUTLIARCH := dockerd  --experimental=true --storage-driver=vfs \
+	--data-root=/var/lib/march/docker/ --exec-root=/var/run/march/docker/ \
+	-H unix:///var/run/march/docker.sock -p /var/run/march/docker.pid
+    DOCKER_RUN += -v /var/run/march/docker.sock:/var/run/docker.sock
+    DOCKER_RUN += -v /var/run/march/docker.pid:/var/run/docker.pid
+    DOCKER_RUN += -v /var/run/march/docker:/var/run/docker
+    DOCKER_RUN += -v /var/lib/march/docker:/var/lib/docker
+    SONIC_USERFACL_DOCKERD_FOR_MUTLIARCH := setfacl -m user:$(USER):rw /var/run/march/docker.sock
+
+    #Override Native config to prevent docker service
+    SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD=y
+
+    DOCKER_MULTIARCH_CHECK := docker inspect --type image multiarch/qemu-user-static:register &> /dev/null || (echo "multiarch docker not found ..."; docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes)
+
+    DOCKER_SERVICE_MULTIARCH_CHECK := docker -H unix:///var/run/march/docker.sock info &> /dev/null || (echo "Docker march service not running..."; sudo rm -fr /var/run/march/docker*; (sudo $(SONIC_NATIVE_DOCKERD_FOR_MUTLIARCH) &) &>/dev/null ; sleep 1; sudo $(SONIC_USERFACL_DOCKERD_FOR_MUTLIARCH);)
+
+   # Docker service to load the compiled dockers-*.gz 
+    SONIC_NATIVE_DOCKERD_FOR_DOCKERFS := rm -fr $(PWD)/dockerfs/; mkdir -p $(PWD)/dockerfs/; sudo dockerd --storage-driver=overlay2 --iptables=false \
+	--data-root $(PWD)/dockerfs/var/lib/docker/ --exec-root=$(PWD)/dockerfs/var/run/docker/ \
+	-H unix://$(PWD)/dockerfs/var/run/docker.sock -p $(PWD)/dockerfs/var/run/docker.pid &
+    SONIC_USERFACL_DOCKERD_FOR_DOCKERFS := setfacl -m user:$(USER):rw $(PWD)/dockerfs/var/run/docker.sock
+    DOCKER_SERVICE_DOCKERFS_CHECK := (sudo docker -H unix://$(PWD)/dockerfs/var/run/docker.sock info &> /dev/null && sudo kill -9 `sudo cat $(PWD)/dockerfs/var/run/docker.pid`  && false) || (echo "Starting docker build service..."; (sudo $(SONIC_NATIVE_DOCKERD_FOR_DOCKERFS) ) &>/dev/null ; sleep 1; sudo $(SONIC_USERFACL_DOCKERD_FOR_DOCKERFS);)
+
+endif
 
 DOCKER_BASE_BUILD = docker build --no-cache \
-		    -t $(SLAVE_BASE_IMAGE) \
+		    -t $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) \
 		    --build-arg http_proxy=$(http_proxy) \
 		    --build-arg https_proxy=$(https_proxy) \
-		    $(SLAVE_DIR) && \
-		    docker tag $(SLAVE_BASE_IMAGE):latest $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG)
+		    $(SLAVE_DIR)
 
 DOCKER_BUILD = docker build --no-cache \
 	       --build-arg user=$(USER) \
 	       --build-arg uid=$(shell id -u) \
 	       --build-arg guid=$(shell id -g) \
 	       --build-arg hostname=$(shell echo $$HOSTNAME) \
-	       -t $(SLAVE_IMAGE) \
+               --build-arg slave_base_tag_ref=$(SLAVE_BASE_TAG) \
+	       -t $(SLAVE_IMAGE):$(SLAVE_TAG) \
 	       -f $(SLAVE_DIR)/Dockerfile.user \
-	       $(SLAVE_DIR) && \
-	       docker tag $(SLAVE_IMAGE):latest $(SLAVE_IMAGE):$(SLAVE_TAG)
+	       $(SLAVE_DIR)
 
 SONIC_BUILD_INSTRUCTION :=  make \
                            -f slave.mk \
+                           BLDENV=$(BLDENV) \
                            PLATFORM=$(PLATFORM) \
+                           PLATFORM_ARCH=$(PLATFORM_ARCH) \
                            BUILD_NUMBER=$(BUILD_NUMBER) \
                            BUILD_TIMESTAMP=$(BUILD_TIMESTAMP) \
                            ENABLE_DHCP_GRAPH_SERVICE=$(ENABLE_DHCP_GRAPH_SERVICE) \
@@ -114,17 +168,27 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            PASSWORD=$(PASSWORD) \
                            USERNAME=$(USERNAME) \
                            SONIC_BUILD_JOBS=$(SONIC_BUILD_JOBS) \
+                           SONIC_USE_DOCKER_BUILDKIT=$(SONIC_USE_DOCKER_BUILDKIT) \
+                           VS_PREPARE_MEM=$(VS_PREPARE_MEM) \
                            KERNEL_PROCURE_METHOD=$(KERNEL_PROCURE_METHOD) \
                            HTTP_PROXY=$(http_proxy) \
                            HTTPS_PROXY=$(https_proxy) \
-                           SONIC_ENABLE_SYSTEM_TELEMETRY=$(ENABLE_SYSTEM_TELEMETRY)
+                           SONIC_ENABLE_SYSTEM_TELEMETRY=$(ENABLE_SYSTEM_TELEMETRY) \
+                           EXTRA_JESSIE_TARGETS=$(EXTRA_JESSIE_TARGETS) \
+                           $(SONIC_OVERRIDE_BUILD_VARS)
 
 .PHONY: sonic-slave-build sonic-slave-bash init reset
 
 .DEFAULT_GOAL :=  all
 
 %::
+ifneq (,$(filter $(CONFIGURED_ARCH), armhf arm64))
+	@$(DOCKER_MULTIARCH_CHECK)
+	@$(DOCKER_SERVICE_MULTIARCH_CHECK)
+	@$(DOCKER_SERVICE_DOCKERFS_CHECK)
+endif
 	@$(OVERLAY_MODULE_CHECK)
+
 	@docker inspect --type image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) &> /dev/null || \
 	    { echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Building... ; \
 	    $(DOCKER_BASE_BUILD) ; }
@@ -166,12 +230,15 @@ init :
 reset :
 	@echo && echo -n "Warning! All local changes will be lost. Proceed? [y/N]: "
 	@read ans && \
-	 if [ $$ans == y ]; then \
-	     git clean -xfdf; \
-	     git reset --hard; \
-	     git submodule foreach --recursive git clean -xfdf; \
-	     git submodule foreach --recursive git reset --hard; \
-	     git submodule update --init --recursive;\
-	 else \
-	     echo "Reset aborted"; \
-	 fi
+	    if [ $$ans == y ]; then \
+	        echo "Resetting local repository. Please wait..."; \
+	        $(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) sudo rm -rf fsroot; \
+	        git clean -xfdf; \
+	        git reset --hard; \
+	        git submodule foreach --recursive git clean -xfdf; \
+	        git submodule foreach --recursive git reset --hard; \
+	        git submodule update --init --recursive; \
+			echo "Reset complete!"; \
+	    else \
+	        echo "Reset aborted"; \
+	    fi