Squash merge latest code to github branch

sonic-net · Jul 26, 2016 · a799669 · a799669
1 parent a79b519
commit a799669
Show file tree

Hide file tree

Showing 29 changed files with 461 additions and 136 deletions.
diff --git a/build_debian.sh b/build_debian.sh
@@ -32,6 +32,9 @@ PASSWORD_ENCRYPTED=$2
 ## Enable debug output for script
 set -x -e
 
+## docker engine version (with platform)
+DOCKER_VERSION=1.11.1-0~jessie_amd64
+
 ## Working directory to prepare the file system
 FILESYSTEM_ROOT=./fsroot
 ## Hostname for the linux image
@@ -127,13 +130,21 @@ sudo chroot $FILESYSTEM_ROOT update-initramfs -u
 
 ## Install docker
 echo '[INFO] Install docker'
-curl -sSL https://get.docker.com/ | sudo LANG=C chroot $FILESYSTEM_ROOT sh
-## Remove garbage left by docker installation script
-sudo rm $FILESYSTEM_ROOT/etc/apt/sources.list.d/docker.list
+## Install apparmor utils since they're missing and apparmor is enabled in the kernel
+## Otherwise Docker will fail to start
+sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install apparmor
+docker_deb_url=https://apt.dockerproject.org/repo/pool/main/d/docker-engine/docker-engine_${DOCKER_VERSION}.deb
+docker_deb_temp=`mktemp`
+trap_push "rm -f $docker_deb_temp"
+wget $docker_deb_url -qO $docker_deb_temp && {                                                  \
+    sudo dpkg --root=$FILESYSTEM_ROOT -i $docker_deb_temp ||                                    \
+    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f;   \
+}
+sudo chroot $FILESYSTEM_ROOT docker version
 sudo chroot $FILESYSTEM_ROOT service docker stop
 ## Add docker config drop-in to select aufs, otherwise it may other storage driver
-## Note: $_ means last argument of last command
 sudo mkdir -p $FILESYSTEM_ROOT/etc/systemd/system/docker.service.d/
+## Note: $_ means last argument of last command
 sudo cp files/docker/docker.service.conf $_
 
 ## Create default user
@@ -149,6 +160,7 @@ sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install      \
 ## Pre-install the fundamental packages
 ## Note: gdisk is needed for sgdisk in install.sh
 ## Note: parted is needed for partprobe in install.sh
+## Note: ca-certificates is needed for easy_install
 sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install      \
     file                    \
     ifupdown                \
@@ -167,8 +179,34 @@ sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install      \
     traceroute              \
     iputils-ping            \
     net-tools               \
+    bsdmainutils            \
+    ca-certificates         \
+    i2c-tools               \
     efibootmgr
 
+## Remove sshd host keys, and will regenerate on first sshd start
+sudo rm -f $FILESYSTEM_ROOT/etc/ssh/ssh_host_*_key*
+sudo cp files/sshd/host-ssh-keygen.sh $FILESYSTEM_ROOT/usr/local/bin/
+sudo cp -f files/sshd/sshd.service $FILESYSTEM_ROOT/lib/systemd/system/ssh.service
+## Config sshd
+sudo augtool --autosave "set /files/etc/ssh/sshd_config/UseDNS no" -r $FILESYSTEM_ROOT
+
+## Config sysctl
+sudo mkdir -p $FILESYSTEM_ROOT/var/core
+sudo augtool --autosave "
+set /files/etc/sysctl.conf/kernel.core_pattern '|/usr/bin/coredump-compress %e %p'
+set /files/etc/sysctl.conf/net.ipv4.conf.default.arp_accept 0
+set /files/etc/sysctl.conf/net.ipv4.conf.default.arp_announce 0
+set /files/etc/sysctl.conf/net.ipv4.conf.default.arp_filter 0
+set /files/etc/sysctl.conf/net.ipv4.conf.default.arp_notify 0
+set /files/etc/sysctl.conf/net.ipv4.conf.default.arp_ignore 0
+set /files/etc/sysctl.conf/net.ipv4.conf.all.arp_accept 0
+set /files/etc/sysctl.conf/net.ipv4.conf.all.arp_announce 1
+set /files/etc/sysctl.conf/net.ipv4.conf.all.arp_filter 0
+set /files/etc/sysctl.conf/net.ipv4.conf.all.arp_notify 1
+set /files/etc/sysctl.conf/net.ipv4.conf.all.arp_ignore 2
+" -r $FILESYSTEM_ROOT
+
 ## docker-py is needed by Ansible docker module
 sudo LANG=C chroot $FILESYSTEM_ROOT easy_install pip
 sudo LANG=C chroot $FILESYSTEM_ROOT pip install 'docker-py==1.6.0'

diff --git a/build_docker.sh b/build_docker.sh
@@ -1,10 +1,45 @@
 #!/bin/bash
 ## This script is to automate the preparation for docker images for ACS.
 ## If registry server and port provided, the images will be pushed there.
-## Usage:
-##   sudo ./build_docker.sh DOCKER_BUILD_DIR [REGISTRY_SERVER REGISTRY_PORT]
 
-set -x -e
+set -e
+
+. ./functions.sh
+
+usage() {
+    cat >&2 <<EOF
+Usage:
+  sudo ./build_docker.sh -i=DOCKER_IMAGE_NAME DOCKER_BUILD_DIR [REGISTRY_SERVER REGISTRY_PORT]
+  
+Description:
+  -i DOCKER_IMAGE_NAME
+       Specifi the docker images name, by default it is DOCKER_BUILD_DIR
+  DOCKER_BUILD_DIR
+       The directory containing Dockerfile
+  REGISTRY_SERVER
+       The server name of the docker registry
+  REGISTRY_PORT
+       The port of the docker registry
+       
+Example:
+  ./build_docker.sh -i docker-orchagent-mlnx docker-orchagent
+EOF
+}
+
+docker_image_name=''
+while getopts ":i:" opt; do
+  case $opt in
+    i)
+      docker_image_name=$OPTARG
+      ;;
+    \?)
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      exit 1
+      ;;
+  esac
+done
+shift "$((OPTIND - 1))"
 
 ## Dockerfile directory
 DOCKER_BUILD_DIR=$1
@@ -18,36 +53,39 @@ REGISTRY_PASSWD=$5
     exit 1
 }
 
-## Docker image label, so no need to remember its hash
-docker_image_name=$DOCKER_BUILD_DIR
-remote_image_name=$REGISTRY_SERVER:$REGISTRY_PORT/$docker_image_name
-
-## File name for docker image
-docker_image_gz=$docker_image_name.gz
-
-[ -n "$docker_image_gz" ] || {
-    echo "Error: Output docker image filename is empty"
-    exit 1
+[ -n "$docker_image_name" ] || {
+    docker_image_name=$DOCKER_BUILD_DIR
 }
 
-function cleanup {
-    rm -rf $DOCKER_BUILD_DIR/files
-    rm -rf $DOCKER_BUILD_DIR/deps
-    docker rmi $remote_image_name || true
+[ ${BUILD_NUMBER} ] || {
+    echo "No BUILD_NUMBER found, setting to 0."
+    BUILD_NUMBER="0"
 }
-trap cleanup exit
+
+remote_image_name=$REGISTRY_SERVER:$REGISTRY_PORT/$docker_image_name:latest
+timestamp="$(date -u +%Y%m%d)"
+build_version="${timestamp}.${BUILD_NUMBER}"
+build_remote_image_name=$REGISTRY_SERVER:$REGISTRY_PORT/$docker_image_name:$build_version
 
 ## Copy dependencies
 ## Note: Dockerfile ADD doesn't support reference files outside the folder, so copy it locally
 if ls deps/* 1>/dev/null 2>&1; then
+    trap_push "rm -rf $DOCKER_BUILD_DIR/deps"
     mkdir -p $DOCKER_BUILD_DIR/deps
     cp -r deps/* $DOCKER_BUILD_DIR/deps
 fi
 
 ## Copy the suggested Debian sources
 ## ref: https://wiki.debian.org/SourcesList
+trap_push "rm -rf $DOCKER_BUILD_DIR/deps"
 cp -r files $DOCKER_BUILD_DIR/files
+docker_try_rmi $docker_image_name
+
+## Build the docker image
 docker build --no-cache -t $docker_image_name $DOCKER_BUILD_DIR
+## Get the ID of the built image
+## Note: inspect output has quotation characters, so sed to remove it as an argument
+image_id=$(docker inspect --format="{{json .Id}}" $docker_image_name | sed -e 's/^"//' -e 's/"$//')
 
 ## Flatten the image by importing an exported container on this image
 ## Note: it will squash the image with only one layer and lost all metadata such as ENTRYPOINT,
@@ -57,18 +95,30 @@ docker build --no-cache -t $docker_image_name $DOCKER_BUILD_DIR
 if [ "$docker_image_name" = "docker-base" ]; then
     tmp_container=$(docker run -d ${docker_image_name} /bin/bash)
     docker export $tmp_container | docker import - ${docker_image_name}
-    docker rm -f $tmp_container || true
+    trap_push "docker rmi $image_id"
+    trap_push "docker rm -f $tmp_container || true"
 fi
 
+image_sha=''
 if [ -n "$REGISTRY_SERVER" ] && [ -n "$REGISTRY_PORT" ]; then
     ## Add registry information as tag, so will push as latest
+    ## Add additional tag with build information
     ## Temporarily add -f option to prevent error message of Docker engine version < 1.10.0
-    docker tag -f $docker_image_name $remote_image_name
+    docker tag $docker_image_name $remote_image_name
+    docker tag $docker_image_name $build_remote_image_name
 
     ## Login the docker image registry server
-    ## Note: user name and password are passed from command line, use fake email address to bypass login check
-    docker login -u $REGISTRY_USERNAME -p "$REGISTRY_PASSWD" -e "@" $REGISTRY_SERVER:$REGISTRY_PORT
-    docker push $remote_image_name
+    ## Note: user name and password are passed from command line
+    docker login -u $REGISTRY_USERNAME -p "$REGISTRY_PASSWD" $REGISTRY_SERVER:$REGISTRY_PORT
+
+    ## Push image to registry server
+    ## And get the image digest SHA256
+    trap_push "docker rmi $remote_image_name"
+    trap_push "docker rmi $build_remote_image_name"
+    image_sha=$(docker push $remote_image_name | sed -n "s/.*: digest: sha256:\([0-9a-f]*\).*/\\1/p")
+    docker push $build_remote_image_name
 fi
 
-docker save $docker_image_name | gzip -c > $docker_image_gz
+mkdir -p target
+rm -f target/$docker_image_name.*.gz
+docker save $docker_image_name | gzip -c > target/$docker_image_name.$image_sha.gz
diff --git a/build_image.sh b/build_image.sh
@@ -23,10 +23,6 @@ sudo rm -f $OUTPUT_ONIE_IMAGE
 if [ "$TARGET_MACHINE" = "generic" ]; then
     ## Generate an ONIE installer image
     ## Note: Don't leave blank between lines. It is single line command.
-    CONSOLE_SPEED=9600 \
-    CONSOLE_DEV=0 \
-    CONSOLE_FLAG=0 \
-    CONSOLE_PORT=0x3f8 \
     ./onie-mk-demo.sh $TARGET_PLATFORM $TARGET_MACHINE $TARGET_PLATFORM-$TARGET_MACHINE-$ONIEIMAGE_VERSION \
           installer $TARGET_MACHINE/platform.conf $OUTPUT_ONIE_IMAGE OS $GIT_REVISION $ONIE_IMAGE_PART_SIZE \
           $ONIE_INSTALLER_PAYLOAD

diff --git a/docker-base b/docker-base
diff --git a/docker-bgp/Dockerfile b/docker-bgp/Dockerfile
@@ -1,11 +1,12 @@
 FROM docker-base
 
-COPY deps /deps
-RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; }; dpkg_apt /deps/quagga_*.deb
+COPY deps/quagga_*.deb /deps/
+RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; } &&        \
+    dpkg_apt /deps/quagga_*.deb &&                                                              \
+    apt-get clean -y && apt-get autoclean -y && apt-get autoremove -y &&                        \
+    rm -rf /deps
 
-## Clean up
-RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
-RUN rm -rf /deps
+COPY daemons /etc/quagga/
 
 ENTRYPOINT service rsyslog start    \
     && service quagga start         \

diff --git a/docker-bgp/daemons b/docker-bgp/daemons
@@ -0,0 +1,31 @@
+# This file tells the quagga package which daemons to start.
+#
+# Entries are in the format: <daemon>=(yes|no|priority)
+#   0, "no"  = disabled
+#   1, "yes" = highest priority
+#   2 .. 10  = lower priorities
+# Read /usr/share/doc/quagga/README.Debian for details.
+#
+# Sample configurations for these daemons can be found in
+# /usr/share/doc/quagga/examples/.
+#
+# ATTENTION: 
+#
+# When activation a daemon at the first time, a config file, even if it is
+# empty, has to be present *and* be owned by the user and group "quagga", else
+# the daemon will not be started by /etc/init.d/quagga. The permissions should
+# be u=rw,g=r,o=.
+# When using "vtysh" such a config file is also needed. It should be owned by
+# group "quaggavty" and set to ug=rw,o= though. Check /etc/pam.d/quagga, too.
+#
+# The watchquagga daemon is always started. Per default in monitoring-only but
+# that can be changed via /etc/quagga/debian.conf.
+#
+zebra=yes
+bgpd=yes
+ospfd=no
+ospf6d=no
+ripd=no
+ripngd=no
+isisd=no
+babeld=no
diff --git a/docker-database/Dockerfile b/docker-database/Dockerfile
@@ -1,14 +1,14 @@
 FROM docker-base
 
 ## Pre-install the fundamental packages
-RUN apt-get update && apt-get -y install    \
-    redis-server
+## Clean up
+RUN apt-get -y install                                                              \
+        redis-server                                                                \
+        &&                                                                          \
+    apt-get clean -y && apt-get autoclean -y && apt-get autoremove -y
 
 RUN sed -ri 's/^daemonize yes$/daemonize no/' /etc/redis/redis.conf                 \
  && sed -ri 's/^logfile .*$/logfile ""/' /etc/redis/redis.conf                      \
  && sed -ri 's/^# syslog-enabled no$/syslog-enabled no/' /etc/redis/redis.conf
 
-## Clean up
-RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
-
 ENTRYPOINT service redis-server start
diff --git a/docker-fpm/Dockerfile b/docker-fpm/Dockerfile
@@ -2,20 +2,21 @@ FROM docker-base
 
 RUN apt-get update
 
-COPY deps /deps
+COPY ["deps/libhiredis0.13*.deb", "deps/libswsscommon_*.deb", "deps/quagga_*", "/deps/"]
+
 ## Get fpmsyncd
 RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; }; dpkg_apt /deps/libhiredis0.13*.deb
 RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; }; dpkg_apt /deps/libswsscommon_*.deb
-COPY /deps/fpmsyncd /usr/local/bin/fpmsyncd
+COPY deps/fpmsyncd /usr/local/bin/
 
 ## Get Quagga
 RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; }; dpkg_apt /deps/quagga_*.deb
 
+COPY start.sh /usr/bin/start.sh
+
 ## Clean up
 RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
 RUN rm -rf /deps
 
-ENTRYPOINT service rsyslog start    \
-    && service quagga start         \
-    && (fpmsyncd &)                 \
-    && /bin/bash
+ENTRYPOINT /usr/bin/start.sh    \
+        && /bin/bash
diff --git a/docker-fpm/start.sh b/docker-fpm/start.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+service rsyslog start
+service quagga start
+fpmsyncd &
diff --git a/docker-lldp/Dockerfile b/docker-lldp/Dockerfile
@@ -1,21 +1,18 @@
 FROM docker-base
 
-## Pre-install the fundamental packages
-RUN apt-get update && apt-get -y install    \
-    lldpd
-
-COPY deps /deps
+COPY deps/*py2*.whl deps/python-sswsdk_*.deb deps/lldpsyncd_*.deb deps/lldpd_*.deb /deps/
 
+## Pre-install the fundamental packages
 ## Install Python SSWSDK (lldpsyncd dependancy)
-## Note: dpkg_apt function has the benefit to detect missing .deb file
-RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; }; dpkg_apt /deps/python-sswsdk_*.deb
 ## Install LLDP Sync Daemon
 ## Note: dpkg_apt function has the benefit to detect missing .deb file
-RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; }; dpkg_apt /deps/lldpsyncd_*.deb
-
 ## Clean up
-RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
-RUN rm -rf /deps
+RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; } &&        \
+    dpkg_apt /deps/lldpd_*.deb &&                                                       \
+    dpkg_apt /deps/lldpsyncd_*.deb &&                                                           \
+    apt-get clean -y && apt-get autoclean -y && apt-get autoremove -y &&                        \
+    pip install --no-cache-dir /deps/*.whl && \
+    rm -rf /deps
 
 ## There is a known bug: agetty processes at 100% cpu
 ## When:

diff --git a/docker-orchagent/Dockerfile b/docker-orchagent/Dockerfile
@@ -2,21 +2,22 @@ FROM docker-base
 
 RUN apt-get update
 
-COPY deps /deps
+COPY ["deps/libhiredis0.13*.deb", "deps/libswsscommon_*.deb", "deps/libsairedis_*.deb", "/deps/"]
 
 RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; }; dpkg_apt /deps/libhiredis0.13*.deb
 RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; }; dpkg_apt /deps/libswsscommon_*.deb
 RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; }; dpkg_apt /deps/libsairedis_*.deb
 
-## TODO: add ifupdown into Depends
-RUN apt-get install -f -y ifupdown
+RUN apt-get install -f -y ifupdown bridge-utils
 
 ## Copy executable binaries
-COPY ["/deps/orchagent","/deps/swssconfig","/deps/portsyncd","/deps/intfsyncd","/deps/neighsyncd","/usr/local/bin/"]
+COPY ["deps/orchagent","deps/swssconfig","deps/portsyncd","deps/intfsyncd","deps/neighsyncd","/usr/local/bin/"]
+
+COPY start.sh /usr/bin/start.sh
 
 ## Clean up
 RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
 RUN rm -rf /deps
 
-ENTRYPOINT service rsyslog start    \
-    && /bin/bash
+ENTRYPOINT /usr/bin/start.sh    \
+        && /bin/bash