[docker-ptf]: Changes to build Python 3 docker-ptf (sonic-net#18909)

Why I did it As Python 2 has reached EOL it is important to migrate all PTF tests to Python 3. The current docker-ptf image contains both Python 2 and Python 3. This PR adds configuration option to build docker-ptf image with its current contents of both Python 2 and 3 by setting SONIC_PTF_ENV_PY_VER to mixed (default value). Or building a Python 3 only image by setting SONIC_PTF_ENV_PY_VER to py3. Once all the tests are fully migrated to Python 3 the older docker-ptf container with Python 2 will only be used for older releases and newer releases can use Python 3 only PTF docker image. The main branch's setting can be updated to use py3 image. How I did it Add SONIC_PTF_ENV_PY_VER configuration variable with mixed and py3 values. platform/vs/docker-ptf.mk and slave.mk uses the configuration value to set dependencies and build the image make target/docker-ptf.gz builds a Python 2 and 3 or Python 3 only image based on the configuration value. How to verify it make USERNAME=admin SONIC_BUILD_JOBS=4 BUILD_MULTIASIC_KVM=y INCLUDE_DHCP_SERVER=y target/docker-ptf.gz Which release branch to backport (provide reason below if selected) Not applicable Tested branch (Please provide the tested image version) None. Individual PTF scripts that have been moved to Python 3 (from the main branch) will be tested. Description for the changelog [docker-ptf]: Changes to build Python 3 docker-ptf Adds SONIC_PTF_ENV_PY_VER to build current docker-ptf with Python 2 and 3 or just Python 3 Update dockers/docker-ptf/Dockerfile.j2 to install Python 3 only and required packages
Junchao-Mellanox · Sep 18, 2024 · e76686d · e76686d
1 parent f0ded9d
commit e76686d
Show file tree

Hide file tree

Showing 5 changed files with 53 additions and 4 deletions.
diff --git a/Makefile.work b/Makefile.work
@@ -61,7 +61,9 @@
 #  *             value to anything will enable it
 #  *             Default: unset
 #  *             Value: y
-#
+#  * SONIC_PTF_ENV_PY_VER: Python version for PTF image
+#  *                       Default: mixed
+#  *                       Values: mixed,py3
 ###############################################################################
 
 SHELL = /bin/bash
@@ -592,6 +594,7 @@ SONIC_BUILD_INSTRUCTION :=  $(MAKE) \
                            SONIC_OS_VERSION=$(SONIC_OS_VERSION) \
                            PIP_HTTP_TIMEOUT=$(PIP_HTTP_TIMEOUT) \
                            LEGACY_SONIC_MGMT_DOCKER=$(LEGACY_SONIC_MGMT_DOCKER) \
+                           SONIC_PTF_ENV_PY_VER=$(SONIC_PTF_ENV_PY_VER) \
                            $(SONIC_OVERRIDE_BUILD_VARS)
 
 .PHONY: sonic-slave-build sonic-slave-bash init reset

diff --git a/dockers/docker-ptf/Dockerfile.j2 b/dockers/docker-ptf/Dockerfile.j2
@@ -3,16 +3,18 @@
 FROM {{ prefix }}multiarch/debian-debootstrap:armhf-stretch
 {% elif CONFIGURED_ARCH == "arm64" and MULTIARCH_QEMU_ENVIRON == "y" %}
 FROM {{ prefix }}multiarch/debian-debootstrap:arm64-stretch
+{% elif PTF_ENV_PY_VER == "mixed" %}
+FROM {{ prefix}}debian:buster
 {% else %}
-FROM {{ prefix }}debian:buster
+FROM {{ prefix }}debian:bullseye
 {% endif %}
 
 {% from "dockers/dockerfile-macros.j2" import install_python_wheels, copy_files %}
 
 USER root
 WORKDIR /root
 
-MAINTAINER Pavel Shirshov
+LABEL maintainer="Pavel Shirshov"
 
 COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
 COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"]
@@ -45,26 +47,35 @@ RUN apt-get update          \
         cmake               \
         libqt5core5a        \
         libqt5network5      \
+{% if PTF_ENV_PY_VER == "mixed" %}
         libboost-atomic1.71.0 \
+{% else %}
+        libboost-atomic1.74.0 \
+{% endif %}
         less                \
         git                 \
         iputils-ping        \
         hping3              \
         curl                \
         tmux                \
+{% if PTF_ENV_PY_VER == "mixed" %}
         python              \
         python-dev          \
         python-libpcap      \
         python-scapy        \
         python-six          \
+{% endif %}
         python3             \
         python3-venv        \
         python3-pip         \
         python3-dev         \
         python3-scapy       \
         python3-six         \
         libpcap-dev         \
+# TODO check if tacacs+ is required by tests
+{% if PTF_ENV_PY_VER == "mixed" %}
         tacacs+             \
+{% endif %}
         rsyslog             \
         ntp                 \
         ntpstat             \
@@ -77,12 +88,20 @@ RUN apt-get update          \
         iproute2            \
         wireshark-common
 
+{% if PTF_ENV_PY_VER == "py3" %}
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
+    && update-alternatives --install /usr/bin/pdb pdb /usr/bin/pdb3 1 \
+    && update-alternatives --install /usr/bin/pydoc pydoc /usr/bin/pydoc3 1 \
+    && update-alternatives --install /usr/bin/pygettext pygettext /usr/bin/pygettext3 1
+{% endif %}
+
 # Install all python modules from pypi. python-scapy is exception, ptf debian package requires python-scapy
 # TODO: Clean up this step
 RUN rm -rf /debs \
     && apt-get -y autoclean \
     && apt-get -y autoremove \
     && rm -rf /var/lib/apt/lists/* \
+{% if PTF_ENV_PY_VER == "mixed" %}
     && wget --https-only https://bootstrap.pypa.io/pip/2.7/get-pip.py \
     && python get-pip.py \
     && rm -f get-pip.py \
@@ -94,6 +113,7 @@ RUN rm -rf /debs \
     && python setup.py install \
     && cd .. \
     && rm -fr scapy-vxlan \
+{% endif %}
     && git clone https://github.com/sflow/sflowtool \
     && cd sflowtool \
     && ./boot.sh \
@@ -121,6 +141,7 @@ RUN rm -rf /debs \
     && cd ../..            \
     && rm -fr nanomsg-1.0.0 \
     && rm -f 1.0.0.tar.gz  \
+{% if PTF_ENV_PY_VER == "mixed" %}
     && pip install cffi    \
     && pip install nnpy    \
     && pip install dpkt    \
@@ -134,6 +155,7 @@ RUN rm -rf /debs \
     && pip install unittest-xml-reporting \
     && pip install pyrasite \
     && pip install retrying \
+{% endif %}
     && mkdir -p /opt       \
     && cd /opt             \
     && wget https://raw.githubusercontent.com/p4lang/ptf/master/ptf_nn/ptf_nn_agent.py
@@ -147,10 +169,14 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
 ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 PYTHONIOENCODING=UTF-8
 
+{% if PTF_ENV_PY_VER == "mixed" %}
 RUN python3 -m pip install --upgrade --ignore-installed pip
+{% else %}
+RUN pip install --upgrade --ignore-installed pip
+{% endif %}
 
 # Install all python modules from pypi. python3-scapy is exception, ptf debian package requires python3-scapy
-RUN python3 -m pip install setuptools \
+RUN pip3 install setuptools \
     && pip3 install supervisor \
     && pip3 install ipython==5.4.1 \
     && pip3 install Cython \
@@ -200,9 +226,11 @@ COPY supervisord.conf /etc/supervisor/
 COPY conf.d/ /etc/supervisor/conf.d/
 COPY ptf_tgen.sh /ptf_tgen/
 
+{% if PTF_ENV_PY_VER == "mixed" %}
 # Move tcpdump into /usr/bin Otherwise it's impossible to run tcpdump due to a docker bug
 RUN mv /usr/sbin/tcpdump /usr/bin/tcpdump
 RUN ln -s /usr/bin/tcpdump /usr/sbin/tcpdump
+{% endif %}
 
 RUN mkdir -p /var/log/supervisor
 
@@ -211,7 +239,12 @@ RUN git clone https://github.com/lguohan/gnxi.git \
     && cd gnxi \
     && git checkout 3adf8b9 \
     && cd gnmi_cli_py \
+{% if PTF_ENV_PY_VER == "mixed" %}
     && pip install -r requirements.txt
+{% else %}
+    && cat requirements.txt | grep -v futures > /tmp/requirements.txt \
+    && pip3 install -r /tmp/requirements.txt
+{% endif %}
 
 COPY \
 {% for deb in docker_ptf_debs.split(' ') -%}

diff --git a/platform/vs/docker-ptf.mk b/platform/vs/docker-ptf.mk
@@ -3,6 +3,10 @@
 DOCKER_PTF = docker-ptf.gz
 $(DOCKER_PTF)_PYTHON_WHEELS += $(PTF_PY3)
 $(DOCKER_PTF)_PATH = $(DOCKERS_PATH)/docker-ptf
+ifeq ($(SONIC_PTF_ENV_PY_VER), mixed)
 $(DOCKER_PTF)_DEPENDS += $(LIBTHRIFT) $(PYTHON_THRIFT) $(PTF) $(PYTHON_SAITHRIFT)
+else
+$(DOCKER_PTF)_DEPENDS += $(LIBTHRIFT_0_14_1) $(PYTHON3_THRIFT_0_14_1) $(PYTHON_SAITHRIFT)
+endif
 SONIC_DOCKER_IMAGES += $(DOCKER_PTF)
 SONIC_BUSTER_DOCKERS += $(DOCKER_PTF)
diff --git a/rules/config b/rules/config
@@ -319,3 +319,7 @@ PIP_HTTP_TIMEOUT ?= 60
 
 # BUILD_REDUCE_IMAGE_SIZE - reduce the image size as much as possbible
 BUILD_REDUCE_IMAGE_SIZE = n
+
+# SONIC_PTF_ENV_PY_VER - SONiC PTF test Python version. Set to 'mixed' to build the 
+# image with both Python 2 and 3. Set to 'py3' to build a Python 3 only image
+SONIC_PTF_ENV_PY_VER = mixed
diff --git a/slave.mk b/slave.mk
@@ -75,6 +75,9 @@ else
 ENABLE_PY2_MODULES = y
 endif
 
+# Python version for PTF image
+PTF_ENV_PY_VER = $(if $(SONIC_PTF_ENV_PY_VER),$(SONIC_PTF_ENV_PY_VER),mixed)
+
 export BUILD_NUMBER
 export BUILD_TIMESTAMP
 export SONIC_IMAGE_VERSION
@@ -94,6 +97,7 @@ export MIRROR_SNAPSHOT
 export SONIC_OS_VERSION
 export FILES_PATH
 export PROJECT_ROOT
+export PTF_ENV_PY_VER
 
 ###############################################################################
 ## Utility rules
@@ -461,6 +465,7 @@ $(info "CROSS_BUILD_ENVIRON"             : "$(CROSS_BUILD_ENVIRON)")
 $(info "GZ_COMPRESS_PROGRAM"             : "$(GZ_COMPRESS_PROGRAM)")
 $(info "LEGACY_SONIC_MGMT_DOCKER"        : "$(LEGACY_SONIC_MGMT_DOCKER)")
 $(info "INCLUDE_EXTERNAL_PATCHES"        : "$(INCLUDE_EXTERNAL_PATCHES)")
+$(info "PTF_ENV_PY_VER"                  : "$(PTF_ENV_PY_VER)")
 $(info )
 else
 $(info SONiC Build System for $(CONFIGURED_PLATFORM):$(CONFIGURED_ARCH))