From cc30771f6b97234a6dd19d8f97d5dfd44551cf20 Mon Sep 17 00:00:00 2001 From: Zhaohui Sun <94606222+ZhaohuiS@users.noreply.github.com> Date: Tue, 26 Apr 2022 09:13:26 +0800 Subject: [PATCH] Add python3 virtual environment for docker-ptf (#10599) Why I did it Migrate ptftests script to python3, in order to do an incremental migration, add python virtual environment firstly, install all required python packages in virtual env as well. Then migrate ptftests scripts from python2 to python3 one by one avoid impacting non-changed scripts. Signed-off-by: Zhaohui Sun zhaohuisun@microsoft.com How I did it Add python3 virtual environment for docker-ptf. Add submodule ptf-py3 and install patched ptf 0.9.3 into virtual environment as well, two ptf issues were reported here: p4lang/ptf#173 p4lang/ptf#174 Signed-off-by: Zhaohui Sun --- .gitmodules | 3 + dockers/docker-ptf/Dockerfile.j2 | 67 ++++++++++++++++++- platform/vs/docker-ptf.mk | 1 + rules/ptf-py3.dep | 11 +++ rules/ptf-py3.mk | 7 ++ src/ptf-py3 | 1 + ...ve-ord-in-get_mac-to-avoid-TypeError.patch | 26 +++++++ ...ted-client-mac-address-in-DHCP-Disco.patch | 36 ++++++++++ src/ptf-py3.patch/series | 2 + 9 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 rules/ptf-py3.dep create mode 100644 rules/ptf-py3.mk create mode 160000 src/ptf-py3 create mode 100644 src/ptf-py3.patch/0001-Remove-ord-in-get_mac-to-avoid-TypeError.patch create mode 100644 src/ptf-py3.patch/0002-Fill-byte-formatted-client-mac-address-in-DHCP-Disco.patch create mode 100644 src/ptf-py3.patch/series diff --git a/.gitmodules b/.gitmodules index 9194c8f4b7a2..59cddea4a8f2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -103,3 +103,6 @@ [submodule "src/sonic-p4rt/sonic-pins"] path = src/sonic-p4rt/sonic-pins url = https://github.com/Azure/sonic-pins.git +[submodule "src/ptf-py3"] + path = src/ptf-py3 + url = https://github.com/p4lang/ptf.git diff --git a/dockers/docker-ptf/Dockerfile.j2 b/dockers/docker-ptf/Dockerfile.j2 index d999d01e0026..bf22ba97ea06 100644 --- a/dockers/docker-ptf/Dockerfile.j2 +++ b/dockers/docker-ptf/Dockerfile.j2 @@ -7,6 +7,11 @@ FROM {{ prefix }}multiarch/debian-debootstrap:arm64-stretch FROM {{ prefix }}debian:buster {% endif %} +{% from "dockers/dockerfile-macros.j2" import install_python_wheels, copy_files %} + +USER root +WORKDIR /root + MAINTAINER Pavel Shirshov RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian buster-backports main" >> /etc/apt/sources.list @@ -51,6 +56,13 @@ RUN sed --in-place 's/httpredir.debian.org/debian-archive.trafficmanager.net/' / python-libpcap \ python-scapy \ python-six \ + python3 \ + python3-venv \ + python3-pip \ + python3-dev \ + python3-scapy \ + python3-six \ + libpcap-dev \ tacacs+ \ rsyslog \ ntp \ @@ -59,7 +71,9 @@ RUN sed --in-place 's/httpredir.debian.org/debian-archive.trafficmanager.net/' / arping \ bridge-utils \ libteam-utils \ - gdb + gdb \ + automake \ + iproute2 # Install all python modules from pypi. python-scapy is exception, ptf debian package requires python-scapy # TODO: Clean up this step @@ -117,10 +131,61 @@ RUN rm -rf /debs \ && pip install pybrctl pyro4 rpyc yabgp \ && pip install unittest-xml-reporting \ && pip install pyrasite \ + && pip install retrying \ && mkdir -p /opt \ && cd /opt \ && wget https://raw.githubusercontent.com/p4lang/ptf/master/ptf_nn/ptf_nn_agent.py +RUN python3 -m venv env-python3 + +# Activating a virtualenv. The virtualenv automatically works for RUN, ENV and CMD. +ENV VIRTUAL_ENV=/root/env-python3 +ARG BACKUP_OF_PATH="$PATH" +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 PYTHONIOENCODING=UTF-8 + +RUN python3 -m pip install --upgrade --ignore-installed pip + +# Install all python modules from pypi. python3-scapy is exception, ptf debian package requires python3-scapy +RUN python3 -m pip install setuptools \ + && pip3 install supervisor \ + && pip3 install ipython==5.4.1 \ + && pip3 install Cython \ + && pip3 install cffi \ + && pip3 install nnpy \ + && pip3 install dpkt \ + && pip3 install ipaddress \ + && pip3 install pysubnettree \ + && pip3 install paramiko \ + && pip3 install Flask \ + && pip3 install exabgp \ + && pip3 install pyaml \ + && pip3 install pybrctl pyro4 rpyc yabgp \ + && pip3 install unittest-xml-reporting \ + && pip3 install pyrasite \ + && pip3 install python-libpcap \ + && pip3 install enum34 \ + && pip3 install grpcio \ + && pip3 install grpcio-tools \ + && pip3 install protobuf \ + && pip3 install six==1.16.0 \ + && pip3 install itsdangerous \ + && pip3 install retrying \ + && pip3 install jinja2 \ + && pip3 install scapy==2.4.5 + +{% if docker_ptf_whls.strip() -%} +# Copy locally-built Python wheel dependencies +{{ copy_files("python-wheels/", docker_ptf_whls.split(' '), "/python-wheels/") }} + +# Install locally-built Python wheel dependencies +{{ install_python_wheels(docker_ptf_whls.split(' ')) }} +{% endif %} + +# Deactivating a virtualenv. +ENV PATH="$BACKUP_OF_PATH" + ## Adjust sshd settings RUN mkdir /var/run/sshd \ && echo 'root:root' | chpasswd \ diff --git a/platform/vs/docker-ptf.mk b/platform/vs/docker-ptf.mk index 445fc8daf4d3..ef102be967f4 100644 --- a/platform/vs/docker-ptf.mk +++ b/platform/vs/docker-ptf.mk @@ -1,6 +1,7 @@ # docker image for docker-ptf DOCKER_PTF = docker-ptf.gz +$(DOCKER_PTF)_PYTHON_WHEELS += $(PTF_PY3) $(DOCKER_PTF)_PATH = $(DOCKERS_PATH)/docker-ptf $(DOCKER_PTF)_DEPENDS += $(LIBTHRIFT) $(PYTHON_THRIFT) $(PTF) $(PYTHON_SAITHRIFT) SONIC_DOCKER_IMAGES += $(DOCKER_PTF) diff --git a/rules/ptf-py3.dep b/rules/ptf-py3.dep new file mode 100644 index 000000000000..eadb32aabdf4 --- /dev/null +++ b/rules/ptf-py3.dep @@ -0,0 +1,11 @@ + +SPATH := $($(PTF_PY3)_SRC_PATH) +DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/ptf-py3.mk rules/ptf-py3.dep +DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST) +SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files)) + +$(PTF_PY3)_CACHE_MODE := GIT_CONTENT_SHA +$(PTF_PY3)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) +$(PTF_PY3)_DEP_FILES := $(DEP_FILES) +$(PTF_PY3)_SMDEP_FILES := $(SMDEP_FILES) +$(PTF_PY3)_SMDEP_PATHS := $(SPATH) diff --git a/rules/ptf-py3.mk b/rules/ptf-py3.mk new file mode 100644 index 000000000000..bf583b57b622 --- /dev/null +++ b/rules/ptf-py3.mk @@ -0,0 +1,7 @@ +# ptf package + +PTF_PY3 = ptf-0.9.3-py3-none-any.whl +$(PTF_PY3)_SRC_PATH = $(SRC_PATH)/ptf-py3 +$(PTF_PY3)_PYTHON_VERSION = 3 +$(PTF_PY3)_TEST = n +SONIC_PYTHON_WHEELS += $(PTF_PY3) diff --git a/src/ptf-py3 b/src/ptf-py3 new file mode 160000 index 000000000000..405513bcad2e --- /dev/null +++ b/src/ptf-py3 @@ -0,0 +1 @@ +Subproject commit 405513bcad2eae3092b0ac4ceb31e8dec5e32311 diff --git a/src/ptf-py3.patch/0001-Remove-ord-in-get_mac-to-avoid-TypeError.patch b/src/ptf-py3.patch/0001-Remove-ord-in-get_mac-to-avoid-TypeError.patch new file mode 100644 index 000000000000..cb2b992548a7 --- /dev/null +++ b/src/ptf-py3.patch/0001-Remove-ord-in-get_mac-to-avoid-TypeError.patch @@ -0,0 +1,26 @@ +From a8b13b9fbaa16ddd305ba2df2238ef606ef222a7 Mon Sep 17 00:00:00 2001 +From: Zhaohui Sun +Date: Wed, 13 Apr 2022 09:24:46 +0000 +Subject: [PATCH 1/2] Remove ord in get_mac() to avoid TypeError + +Signed-off-by: Zhaohui Sun +--- + src/ptf/netutils.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/ptf/netutils.py b/src/ptf/netutils.py +index 752e76c..6aabe79 100644 +--- a/src/ptf/netutils.py ++++ b/src/ptf/netutils.py +@@ -54,7 +54,7 @@ def get_if_index(iff): + + + def get_mac(iff): +- return ":".join(["%02x" % ord(char) for char in get_if(iff, SIOCGIFHWADDR)[18:24]]) ++ return ":".join(["%02x" % char for char in get_if(iff, SIOCGIFHWADDR)[18:24]]) + + + def set_promisc(s, iff, val=1): +-- +2.25.1 + diff --git a/src/ptf-py3.patch/0002-Fill-byte-formatted-client-mac-address-in-DHCP-Disco.patch b/src/ptf-py3.patch/0002-Fill-byte-formatted-client-mac-address-in-DHCP-Disco.patch new file mode 100644 index 000000000000..a0582994539e --- /dev/null +++ b/src/ptf-py3.patch/0002-Fill-byte-formatted-client-mac-address-in-DHCP-Disco.patch @@ -0,0 +1,36 @@ +From 6e570e00ea05882d2db1e480ed041ea631bf37da Mon Sep 17 00:00:00 2001 +From: Zhaohui Sun +Date: Wed, 13 Apr 2022 09:25:28 +0000 +Subject: [PATCH 2/2] Fill byte formatted client mac address in DHCP Discover + packet + +Signed-off-by: Zhaohui Sun +--- + src/ptf/testutils.py | 9 +++++---- + 1 file changed, 5 insertions(+), 4 deletions(-) + +diff --git a/src/ptf/testutils.py b/src/ptf/testutils.py +index ab67cea..83a9075 100755 +--- a/src/ptf/testutils.py ++++ b/src/ptf/testutils.py +@@ -2712,12 +2712,13 @@ def __dhcp_mac_to_chaddr(mac_addr="00:01:02:03:04:05"): + """ + Private helper function to convert a 6-byte MAC address of form: + '00:01:02:03:04:05' +- into a 16-byte chaddr byte string of form: +- '\x00\x01\x02\x03\x04\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' ++ into a 16-byte chaddr byte of form: ++ b'\x00\x01\x02\x03\x04\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + + """ +- chaddr = "".join([chr(int(octet, 16)) for octet in mac_addr.split(":")]) +- chaddr += "\x00" * 10 ++ import binascii ++ chaddr = binascii.unhexlify(mac_addr.replace(':', '')) ++ chaddr += b'\x00\x00\x00\x00\x00\x00' + return chaddr + + +-- +2.25.1 + diff --git a/src/ptf-py3.patch/series b/src/ptf-py3.patch/series new file mode 100644 index 000000000000..d18987dd5913 --- /dev/null +++ b/src/ptf-py3.patch/series @@ -0,0 +1,2 @@ +0001-Remove-ord-in-get_mac-to-avoid-TypeError.patch +0002-Fill-byte-formatted-client-mac-address-in-DHCP-Disco.patch \ No newline at end of file