-
Notifications
You must be signed in to change notification settings - Fork 468
Azure with netvsc DPDK driver (v2.89)
Malcolm Bumgardner edited this page Mar 24, 2021
·
7 revisions
az vm create --resource-group rgXYZ --name TrexUbuntuAN --image Canonical:UbuntuServer:18_04-lts-gen2:18.04.202010140 --size Standard_D16ds_v4 --admin-username azureuser --admin-password trexTesting --nics ANLinux_eth0_NIC ANLinux_eth1_NIC ANLinux_eth2_NIC
|
Note
|
Do not add AN on eth0/management to cut down on mapping confusion with MLX But we may want to add it in the future to cut down interrupts from eth0 that may end up on the TREX cores running at 100% |
-
The Ubuntu Azure kernel provides the best network performance on Azure
sudo add-apt-repository ppa:canonical-server/dpdk-azure -y
sudo apt-get update
sudo apt-get upgrade -y
sudo apt-get dist-upgrade
sudo apt-get install -y librdmacm-dev librdmacm1 build-essential libnuma-dev libmnl-dev
sudo apt install ibverbs-utils
lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description: Ubuntu 18.04.4 LTS
Release: 18.04
Codename: bionicsudo vi /etc/default/grub
# default_hugepagesz=1GB hugepagesz=1G hugepages=8 transparent_hugepage=never
# GRUB_CMDLINE_LINUX=" default_hugepagesz=1GB hugepagesz=1G hugepages=8 transparent_hugepage=never "
cat /etc/default/grub
# If you change this file, run 'update-grub' afterwards to update
# /boot/grub/grub.cfg.
# For full documentation of the options in this file, see:
# info -f grub -n 'Simple configuration'
GRUB_DEFAULT=0
GRUB_TIMEOUT_STYLE=hidden
GRUB_TIMEOUT=0
GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian`
GRUB_CMDLINE_LINUX_DEFAULT="quiet splash"
GRUB_CMDLINE_LINUX=" default_hugepagesz=1GB hugepagesz=1G hugepages=8 transparent_hugepage=never"
# Uncomment to enable BadRAM filtering, modify to suit your needs
# This works with Linux (no patch required) and with any kernel that obtains
# the memory map information from GRUB (GNU Mach, kernel of FreeBSD ...)
#GRUB_BADRAM="0x01234567,0xfefefefe,0x89abcdef,0xefefefef"
# Uncomment to disable graphical terminal (grub-pc only)
#GRUB_TERMINAL=console
# The resolution used on graphical terminal
# note that you can use only modes which your graphic card supports via VBE
# you can see them in real GRUB with the command `vbeinfo'
#GRUB_GFXMODE=640x480
# Uncomment if you don't want GRUB to pass "root=UUID=xxx" parameter to Linux
#GRUB_DISABLE_LINUX_UUID=true
# Uncomment to disable generation of recovery mode menu entries
#GRUB_DISABLE_RECOVERY="true"
# Uncomment to get a beep at grub start
#GRUB_INIT_TUNE="480 440 1"
sudo update-grub
sudo vi /etc/fstab
# nodev /mnt/huge hugetlbfs defaults 0 0
cat /etc/fstab
# CLOUD_IMG: This file was created/modified by the Cloud Image build process
UUID=8c0a4742-2f51-40b4-b659-357cfb0bb2a3 / ext4 defaults,discard 0 0
UUID=5BCE-FF6A /boot/efi vfat defaults,discard 0 0
nodev /mnt/huge hugetlbfs defaults 0 0
/dev/disk/cloud/azure_resource-part1 /mnt auto defaults,nofail,x-systemd.requires=cloud-init.service,comment=cloudconfig 0 2sudo vi /etc/modules-load.d/modules.conf
# ib_uverbs
# mlx4_ib
# mlx5_ib
cat /etc/modules-load.d/modules.conf
# /etc/modules: kernel modules to load at boot time.
#
# This file contains the names of kernel modules that should be loaded
# at boot time, one per line. Lines beginning with "#" are ignored.
ib_uverbs
mlx4_ib
mlx5_ibValidate huge pages and Infiniband drivers are loaded
cat /proc/meminfo | grep Huge
lsmod | grep ib_uverbs|
Note
|
MLX4/CX-3 in latest TREX with DPDK21.02 was disabled (Seems like it can be re-enabled as still supported driver in DPDK 21.02). |
For details on NETVSC see (https://doc.dpdk.org/guides-21.02/nics/netvsc.html)
Need to load uio_hv_generic and unbind eth1/eth2 interfaces from the kernel so TREX can bind to eth1/eth2
Example script added below to assist with this process.
cd ~
sudo apt-get install -y python3-distutils
sudo apt install zlib1g-devgit clone https://github.com/cisco-system-traffic-generator/trex-core.git
cd trex-core
cd linux_dpdk/
./b configure --no-ofed-check
./b build cd ..
cd scripts/
cat ./azure_trex_setup.sh
#!/bin/bash
sudo modprobe uio_hv_generic
NET_UUID="f8615163-df3e-46c5-913f-f2d2f965ed0e"
echo $NET_UUID | sudo tee /sys/bus/vmbus/drivers/uio_hv_generic/new_id
DEV_UUID=$(basename $(readlink /sys/class/net/eth1/device))
echo $DEV_UUID | sudo tee /sys/bus/vmbus/drivers/hv_netvsc/unbind
echo $DEV_UUID | sudo tee /sys/bus/vmbus/drivers/uio_hv_generic/bind
DEV_UUID=$(basename $(readlink /sys/class/net/eth2/device))
echo $DEV_UUID | sudo tee /sys/bus/vmbus/drivers/hv_netvsc/unbind
echo $DEV_UUID | sudo tee /sys/bus/vmbus/drivers/uio_hv_generic/bindcat ./azure_trex_setup.sh./azure_trex_setup.sh
f8615163-df3e-46c5-913f-f2d2f965ed0e
000d3a9b-73fd-000d-3a9b-73fd000d3a9b
000d3a9b-73fd-000d-3a9b-73fd000d3a9b
000d3a9b-7b26-000d-3a9b-7b26000d3a9b
000d3a9b-7b26-000d-3a9b-7b26000d3a9bNeed to update trex_cfg.yaml based on MLX PCI address and UUIDs for hv_netvsc interfaces. Also IP addresses and GWs are specific to VM setup and only examples.
lspci
2180:00:02.0 Ethernet controller: Mellanox Technologies MT27710 Family [ConnectX-4 Lx Virtual Function] (rev 80)
c7e1:00:02.0 Ethernet controller: Mellanox Technologies MT27710 Family [ConnectX-4 Lx Virtual Function] (rev 80)cat /etc/trex_cfg.yaml
- version: 2
interfaces: ['2180:00:02.0', 'c7e1:00:02.0']
ext_dpdk_opt: ['--vdev=net_vdev_netvsc0,ignore=0', '--vdev=net_vdev_netvsc1,ignore=0']
interfaces_vdevs : ['000d3a9b-73fd-000d-3a9b-73fd000d3a9b','000d3a9b-7b26-000d-3a9b-7b26000d3a9b']
rx_desc : 1024
tx_desc : 1024
port_bandwidth_gb : 10
port_speed : 10000
port_info:
- ip: 10.90.23.101
default_gw: 10.90.23.202
- ip: 10.90.130.101
default_gw: 10.90.130.202
platform:
master_thread_id: 0
latency_thread_id: 2
dual_if:
- socket: 0
threads: [4, 6, 8, 10]Turn off TSO for MLX devices
sudo ethtool -K enP51169s2 tso off gro off gso off
sudo ethtool -K enP8576s3 tso off gro off gso offWith NETVSC not limited to “-c 1”
cd ~/trex-core/scripts
sudo ./t-rex-64 -i -c 2 -v 7 --no-ofed-checkstty cols 111 rows 45
cd ~/trex-core/scripts
./trex-console
trex> tui
tui> start -f stl/bench.py -m 800kpps --port 0 1 --force -t size=1514TUI output
Global Statistics
connection : localhost, Port 4501 total_tx_L2 : 19.43 Gbps
version : STL @ v2.89 total_tx_L1 : 19.69 Gbps
cpu_util. : 27.76% @ 2 cores (2 per dual port) total_rx : 19.43 Gbps
rx_cpu_util. : 0.0% / 0 pps total_pps : 1.6 Mpps
async_util. : 0% / 161.14 bps drop_rate : 0 bps
total_cps. : 0 cps queue_full : 0 pkts
Port Statistics
port | 0 | 1 | total
-----------+-------------------+-------------------+------------------
owner | azureuser | azureuser |
link | UP | UP |
state | TRANSMITTING | TRANSMITTING |
speed | 10 Gb/s | 10 Gb/s |
CPU util. | 27.76% | 27.76% |
-- | | |
Tx bps L2 | 9.71 Gbps | 9.71 Gbps | 19.43 Gbps
Tx bps L1 | 9.84 Gbps | 9.84 Gbps | 19.69 Gbps
Tx pps | 802.08 Kpps | 802.08 Kpps | 1.6 Mpps
Line Util. | 98.43 % | 98.43 % |
--- | | |
Rx bps | 9.71 Gbps | 9.71 Gbps | 19.43 Gbps
Rx pps | 802.07 Kpps | 802.07 Kpps | 1.6 Mpps
---- | | |
opackets | 13079888 | 13079888 | 26159776
ipackets | 13079887 | 13079901 | 26159788
obytes | 19802950432 | 19802950432 | 39605900864
ibytes | 19802948918 | 19802970114 | 39605919032
tx-pkts | 13.08 Mpkts | 13.08 Mpkts | 26.16 Mpkts
rx-pkts | 13.08 Mpkts | 13.08 Mpkts | 26.16 Mpkts
tx-bytes | 19.8 GB | 19.8 GB | 39.61 GB
rx-bytes | 19.8 GB | 19.8 GB | 39.61 GB
----- | | |
oerrors | 0 | 0 | 0
ierrors | 0 | 0 | 0