DPVS-1：编译安装DPVS （ubuntu22.04）

操作系统

root@ubuntu22:~# lsb_release  -a
No LSB modules are available.
Distributor ID: Ubuntu
Description:    Ubuntu 22.04.3 LTS
Release:        22.04
Codename:       jammy
root@ubuntu22:~# 

前置软件准备

apt install git
apt install meson
apt install gcc
apt install pkg-config
apt install libnuma-dev
apt install libssl-devapt install automake autoconf
apt install libtool
apt install libpopt-dev

DPDK编译安装

下载DPVS

git clone https://github.com/iqiyi/dpvs.git
cd dpvs

下载DPDK20.11

在dpvs目录下下载dpdk，方便后续操作

root@ubuntu22:~/dpvs# wget https://fast.dpdk.org/rel/dpdk-20.11.10.tar.xz 
--2025-02-20 15:16:23--  https://fast.dpdk.org/rel/dpdk-20.11.10.tar.xz
Resolving fast.dpdk.org (fast.dpdk.org)... 151.101.130.49, 151.101.66.49, 151.101.2.49, ...
Connecting to fast.dpdk.org (fast.dpdk.org)|151.101.130.49|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 14047916 (13M) [application/octet-stream]
Saving to: ‘dpdk-20.11.10.tar.xz’dpdk-20.11.10.tar.xz              100%[============================================================>]  13.40M  6.96MB/s    in 1.9s    2025-02-20 15:16:26 (6.96 MB/s) - ‘dpdk-20.11.10.tar.xz’ saved [14047916/14047916]root@ubuntu22:~/dpvs# 
root@ubuntu22:~/dpvs# tar xf dpdk-20.11.10.tar.xz 

打PATCH

root@ubuntu22:~/dpvs# cp patch/dpdk-stable-20.11.10/* dpdk-stable-20.11.10/
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# patch -p1 < 0001-kni-use-netlink-event-for-multicast-driver-part.patch 
patching file kernel/linux/kni/kni_net.c
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# patch -p1 < 0002-pdump-change-dpdk-pdump-tool-for-dpvs.patch 
patching file app/pdump/main.c
patching file lib/librte_pdump/rte_pdump.c
patching file lib/librte_pdump/rte_pdump.h
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# patch -p1 < 0003-debug-enable-dpdk-eal-memory-debug.patch 
patching file lib/librte_eal/common/rte_malloc.c
patching file lib/librte_eal/include/rte_malloc.h
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# patch -p1 < 0004-ixgbe_flow-patch-ixgbe-fdir-rte_flow-for-dpvs.patch 
patching file drivers/net/ixgbe/ixgbe_flow.c
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# patch -p1 < 0005
-bash: 0005: No such file or directory
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# patch -p1 < 0005-bonding-allow-slaves-from-different-numa-nodes.patch 
patching file drivers/net/bonding/rte_eth_bond_pmd.c
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# patch -p1 < 0006-bonding-fix-problem-in-mode-4-dropping-multicast-pac.patch 
patching file drivers/net/bonding/rte_eth_bond_pmd.c
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# patch -p1 < 0007-bonding-device-sends-packets-with-user-specified-sal.patch 
patching file drivers/net/bonding/rte_eth_bond_pmd.c
patching file lib/librte_mbuf/rte_mbuf.h

编译

在dpdkbuild中编译，安装在dpdklib中，后续指定 pkg-config目录到

root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# mkdir dpdklib 
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# mkdir dpdkbuild
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# meson -Denable_kmods=true -Dprefix=/root/dpvs/dpdk-stable-20.11.10/dpdklib dpdkbuildroot@ubuntu22:~/dpvs/dpdk-stable-20.11.10# ninja -C dpdkbuild/
ninja: Entering directory `dpdkbuild/'
[2419/2421] Generating kernel/linux/kni/rte_kni with a custom command
make: Entering directory '/usr/src/linux-headers-6.2.0-26-generic'
warning: the compiler differs from the one used to build the kernelThe kernel was built by: x86_64-linux-gnu-gcc-11 (Ubuntu 11.3.0-1ubuntu1~22.04.1) 11.3.0You are using:           gcc-11 (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0CC [M]  /root/dpvs/dpdk-stable-20.11.10/dpdkbuild/kernel/linux/kni/kni_misc.oCC [M]  /root/dpvs/dpdk-stable-20.11.10/dpdkbuild/kernel/linux/kni/kni_net.oLD [M]  /root/dpvs/dpdk-stable-20.11.10/dpdkbuild/kernel/linux/kni/rte_kni.oMODPOST /root/dpvs/dpdk-stable-20.11.10/dpdkbuild/kernel/linux/kni/Module.symversCC [M]  /root/dpvs/dpdk-stable-20.11.10/dpdkbuild/kernel/linux/kni/rte_kni.mod.oLD [M]  /root/dpvs/dpdk-stable-20.11.10/dpdkbuild/kernel/linux/kni/rte_kni.koBTF [M] /root/dpvs/dpdk-stable-20.11.10/dpdkbuild/kernel/linux/kni/rte_kni.ko
Skipping BTF generation for /root/dpvs/dpdk-stable-20.11.10/dpdkbuild/kernel/linux/kni/rte_kni.ko due to unavailability of vmlinux
make: Leaving directory '/usr/src/linux-headers-6.2.0-26-generic'
[2421/2421] Linking target app/test/dpdk-testroot@ubuntu22:~/dpvs/dpdk-stable-20.11.10# cd dpdkbuild
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10/dpdkbuild# ninja install
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# 

设置pkg-cofnig

这里的pkgconfig目录与centos(/lib64/pkgconfig)不同，需要根据实际路径设定

root@ubuntu22:~/dpvs/dpdk-stable-20.11.10/dpdklib# export PKG_CONFIG_PATH=/root/dpvs/dpdk-stable-20.11.10/dpdklib/lib/x86_64-linux-gnu/pkgconfig
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10/dpdklib# 

验证libdpdk

root@ubuntu22:~/dpvs/dpdk-stable-20.11.10/dpdklib# pkg-config --libs libdpdk
-L/root/dpvs/dpdk-stable-20.11.10/dpdklib/lib/x86_64-linux-gnu -Wl,--as-needed -lrte_node -lrte_graph -lrte_bpf -lrte_flow_classify -lrte_pipeline -lrte_table -lrte_port -lrte_fib -lrte_ipsec -lrte_vhost -lrte_stack -lrte_security -lrte_sched -lrte_reorder -lrte_rib -lrte_regexdev -lrte_rawdev -lrte_pdump -lrte_power -lrte_member -lrte_lpm -lrte_latencystats -lrte_kni -lrte_jobstats -lrte_ip_frag -lrte_gso -lrte_gro -lrte_eventdev -lrte_efd -lrte_distributor -lrte_cryptodev -lrte_compressdev -lrte_cfgfile -lrte_bitratestats -lrte_bbdev -lrte_acl -lrte_timer -lrte_hash -lrte_metrics -lrte_cmdline -lrte_pci -lrte_ethdev -lrte_meter -lrte_net -lrte_mbuf -lrte_mempool -lrte_rcu -lrte_ring -lrte_eal -lrte_telemetry -lrte_kvargs

设置大页

echo "vm.nr_hugepages=1024" >> /etc/sysctl.conf
sysctl -proot@ubuntu22:/etc/apt# cat /proc/meminfo | grep Huge
AnonHugePages:         0 kB
ShmemHugePages:        0 kB
FileHugePages:         0 kB
HugePages_Total:    1024
HugePages_Free:     1024
HugePages_Rsvd:        0
HugePages_Surp:        0
Hugepagesize:       2048 kB
Hugetlb:         2097152 kB

安装golang

heathcheck dpvs-agent 需要golang 1.20版本

wget https://golang.google.cn/dl/go1.20.linux-amd64.tar.gz
rm -rf /usr/local/go && tar -C /usr/local -xzf go1.20.linux-amd64.tar.gz 
export PATH=$PATH:/usr/local/go/bingo env -w GOPROXY=https://goproxy.cn,direct

编译DPVS

修改编译选项

开启编译healthcheck ipvs-agent，config.mk中修改,

export CONFIG_DPVS_AGENT=y

编译安装

root@ubuntu22:~# make
root@ubuntu22:~# make install
root@ubuntu22:~# cd dpvs
root@ubuntu22:~/dpvs# ls bin
dpip  dpvs  dpvs-agent  healthcheck  ipvsadm  keepalived

报错处理

报错

/root/dpvs/src/ipvs/ip_vs_synproxy.c: In function ‘cookie_hash’:
/root/dpvs/src/ipvs/ip_vs_synproxy.c:216:5: error: ‘MD5’ is deprecated: Since OpenSSL 3.0 [-Werror=deprecated-declarations]216 |     MD5((unsigned char *)data, sizeof(data), hash);|     ^~~

libssl3.0对 MD5有一个废弃告警，这里会把告警变成报错。

src/Makefile中，去掉-Werror

50 #CFLAGS += -Wall -Werror -Wstrict-prototypes -Wmissing-prototypes -mcmodel=medium51 CFLAGS += -Wall -Wstrict-prototypes -Wmissing-prototypes -mcmodel=medium52 

报错

lldp.c: In function ‘lldp_do_cmd’:
lldp.c:101:27: error: format not a string literal and no format arguments [-Werror=format-security]101 |             printf(message->message);

tools/dpip/lldp.c中修改101行

//printf(message->message);
printf("%s", message->message);

运行DPVS

加载驱动

insmod ./dpdk-stable-20.11.10/dpdkbuild/kernel/linux/kni/rte_kni.ko
modprobe uio_pci_generic

修改网卡驱动

root@ubuntu22:~# lspci | grep Eth
03:00.0 Ethernet controller: Intel Corporation 82574L Gigabit Network Connection
0b:00.0 Ethernet controller: VMware VMXNET3 Ethernet Controller (rev 01)
13:00.0 Ethernet controller: VMware VMXNET3 Ethernet Controller (rev 01)dpdk-devbind.py -u 0000:0b:00.0 
dpdk-devbind.py -b uio_pci_generic 0000:0b:00.0

修改配置文件

cp conf/dpvs.conf.single-nic.sample /etc/dpvs.confcat /etc/dpvs.confnetif_defs {<init> pktpool_size     524287<init> pktpool_cache    256<init> device dpdk0 {rx {queue_number        1  # 虚拟机网卡，改为1个RX队列descriptor_number   1024rss                 all}tx {queue_number        1  # 虚拟机网卡，改为1个TX队列descriptor_number   1024}! mtu                   1500! promisc_mode! allmulticastkni_name                dpdk0.kni}
}对应的worker写改为1个rx worker 1个tx worker
worker_defs {<init> worker cpu0 {type    mastercpu_id  0}<init> worker cpu1 {type    slavecpu_id  1port    dpdk0 {rx_queue_ids     0tx_queue_ids     0! isol_rx_cpu_ids  9! isol_rxq_ring_sz 1048576}}<init> worker cpu2 {type    slavecpu_id  2port    dpdk0 {rx_queue_ids     1tx_queue_ids     1! isol_rx_cpu_ids  10! isol_rxq_ring_sz 1048576}}
}
...sa_pool {pool_hash_size  16flow_enable     off  # 虚拟机VMXNET3网卡不支持flow, 所以off掉
}

运行报错

dpvs – -a 0000:0b:00.0 -l 0-2 , 0 master 1 ，2 worker

root@ubuntu22:~/dpvs/bin# dpvs -- -a 0000:0b:00.0 -l 0-2  
Command 'dpvs' not found, did you mean:command 'pvs' from deb lvm2 (2.03.11-2.1ubuntu4)
Try: apt install <deb name>
root@ubuntu22:~/dpvs/bin# ./dpvs -- -a 0000:0b:00.0 -l 0-2
current thread affinity is set to 3F
EAL: Detected 6 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: Detected static linkage of DPDK
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'PA'
EAL: No available hugepages reported in hugepages-1048576kB
EAL: Probing VFIO support...
EAL: Probe PCI driver: net_vmxnet3 (15ad:07b0) device: 0000:0b:00.0 (socket 0)
EAL: No legacy callbacks, legacy socket not created
DPVS: dpvs version: 1.9-8, build on 2025.02.21.01:02:16
DPVS: dpvs-conf-file: /etc/dpvs.conf
DPVS: dpvs-pid-file: /var/run/dpvs.pid
DPVS: dpvs-ipc-file: /var/run/dpvs.ipc
CFG_FILE: Opening configuration file '/etc/dpvs.conf'.
CFG_FILE: log_level = WARNING
NETIF: dpdk0:rx_queue_number = 1
NETIF: worker cpu1:dpdk0 rx_queue_id += 0
NETIF: worker cpu1:dpdk0 tx_queue_id += 0
NETIF: worker cpu2:dpdk0 rx_queue_id += 1
NETIF: worker cpu2:dpdk0 tx_queue_id += 1
NETIF: dpdk0: rte_eth_dev_set_mc_addr_list failed -- Operation not supported,enable all multicast
Segmentation fault (core dumped)

直接段错误，重新以DEBUG模式编译DPDK 和 DPVS ，GDB跟踪

Thread 1 "dpvs" received signal SIGSEGV, Segmentation fault.
0x000055555601d5f4 in vmxnet3_dev_set_rxmode (set=1, feature=8, hw=0x1003ad940) at ../drivers/net/vmxnet3/vmxnet3_ethdev.c:1293
1293                    rxConf->rxMode = rxConf->rxMode | feature;
(gdb) bt
#0  0x000055555601d5f4 in vmxnet3_dev_set_rxmode (set=1, feature=8, hw=0x1003ad940) at ../drivers/net/vmxnet3/vmxnet3_ethdev.c:1293
#1  vmxnet3_dev_allmulticast_enable (dev=0x555556b6c980 <rte_eth_devices>) at ../drivers/net/vmxnet3/vmxnet3_ethdev.c:1341
#2  0x000055555626f3e2 in rte_eth_allmulticast_enable (port_id=0) at ../lib/librte_ethdev/rte_ethdev.c:2595
#3  0x0000555555c51033 in dpdk_set_mc_list (dev=0x10af20ec0) at /root/dpvs/src/netif.c:3356
#4  0x0000555555c1bc8c in __netif_set_mc_list (dev=0x10af20ec0) at /root/dpvs/src/netif_addr.c:189
#5  0x0000555555c1bd3b in netif_mc_add (dev=0x10af20ec0, addr=0x5555562d3500 <LLDP_ETHER_ADDR_DST>) at /root/dpvs/src/netif_addr.c:210
#6  0x0000555555be0150 in lldp_ether_addr_filter (add=true) at /root/dpvs/src/lldp.c:1512
#7  0x0000555555be0241 in lldp_xmit_start () at /root/dpvs/src/lldp.c:1530
#8  0x0000555555be2198 in dpvs_lldp_init () at /root/dpvs/src/lldp.c:1857
#9  0x0000555555aa0e8e in inet_init () at /root/dpvs/src/inet.c:103
#10 0x0000555555bef475 in modules_init () at /root/dpvs/src/main.c:138
#11 0x0000555555befe07 in main (argc=5, argv=0x7fffffffe2f0) at /root/dpvs/src/main.c:339
(gdb) f 0
#0  0x000055555601d5f4 in vmxnet3_dev_set_rxmode (set=1, feature=8, hw=0x1003ad940) at ../drivers/net/vmxnet3/vmxnet3_ethdev.c:1293
1293                    rxConf->rxMode = rxConf->rxMode | feature;
(gdb) p rxConf
$1 = (struct Vmxnet3_RxFilterConf *) 0x78
(gdb) p hw->shared->devRead
Cannot access memory at address 0x8
(gdb) p hw->shared
$2 = (Vmxnet3_DriverShared *) 0x0
(gdb) p *hw
$3 = {hw_addr0 = 0x1100800000 "", hw_addr1 = 0x1100801000 "", back = 0x0, device_id = 1968, vendor_id = 5549, subsystem_device_id = 0, subsystem_vendor_id = 0, adapter_stopped = false, perm_addr = "\000\f)\264\277<", num_tx_queues = 1 '\001', num_rx_queues = 1 '\001', bufs_per_pkt = 1 '\001', version = 1 '\001', txdata_desc_size = 128, rxdata_desc_size = 0, num_intrs = 0 '\000', tqd_start = 0x0, rqd_start = 0x0, shared = 0x0, sharedPA = 0

参考代码

static int
vmxnet3_dev_allmulticast_enable(struct rte_eth_dev *dev)
{struct vmxnet3_hw *hw = dev->data->dev_private;vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_ALL_MULTI, 1);return 0;
}/* Updating rxmode through Vmxnet3_DriverShared structure in adapter */
static void
vmxnet3_dev_set_rxmode(struct vmxnet3_hw *hw, uint32_t feature, int set)
{struct Vmxnet3_RxFilterConf *rxConf = &hw->shared->devRead.rxFilterConf;if (set)rxConf->rxMode = rxConf->rxMode | feature;elserxConf->rxMode = rxConf->rxMode & (~feature);VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_UPDATE_RX_MODE);
}

hw下shared为0 ，rxConf则更是一个野指针，导致coredump.

猜测当前VMware虚拟机的vmxnet3网卡或驱动的问题

物理机测试

将上述编译安装流程都走一遍，配置文件不变, 运行成功

root@r750-132:~/dpvs/bin# ./dpvs -- -a 98:00.0 -l 0-9
current thread affinity is set to FFFFFFFF
EAL: Detected 32 lcore(s)
EAL: Detected 2 NUMA nodes
EAL: Detected static linkage of DPDK
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'PA'
EAL: No available hugepages reported in hugepages-1048576kB
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL: Probe PCI driver: mlx5_pci (15b3:1017) device: 0000:98:00.0 (socket 1)
EAL: No legacy callbacks, legacy socket not created
DPVS: dpvs version: 1.9-8, build on 2025.02.20.15:37:58
DPVS: dpvs-conf-file: /etc/dpvs.conf
DPVS: dpvs-pid-file: /var/run/dpvs.pid
DPVS: dpvs-ipc-file: /var/run/dpvs.ipc
CFG_FILE: Opening configuration file '/etc/dpvs.conf'.
CFG_FILE: log_level = WARNING
NETIF: dpdk0:rx_queue_number = 8
NETIF: worker cpu1:dpdk0 rx_queue_id += 0
NETIF: worker cpu1:dpdk0 tx_queue_id += 0
NETIF: worker cpu2:dpdk0 rx_queue_id += 1
NETIF: worker cpu2:dpdk0 tx_queue_id += 1
NETIF: worker cpu3:dpdk0 rx_queue_id += 2
NETIF: worker cpu3:dpdk0 tx_queue_id += 2
NETIF: worker cpu4:dpdk0 rx_queue_id += 3
NETIF: worker cpu4:dpdk0 tx_queue_id += 3
NETIF: worker cpu5:dpdk0 rx_queue_id += 4
NETIF: worker cpu5:dpdk0 tx_queue_id += 4
NETIF: worker cpu6:dpdk0 rx_queue_id += 5
NETIF: worker cpu6:dpdk0 tx_queue_id += 5
NETIF: worker cpu7:dpdk0 rx_queue_id += 6
NETIF: worker cpu7:dpdk0 tx_queue_id += 6
NETIF: worker cpu8:dpdk0 rx_queue_id += 7
NETIF: worker cpu8:dpdk0 tx_queue_id += 7
SAPOOL: sapool_filter_enable = on
IPVS: dp_vs_conn_init: lcore 9: nothing to do.
NETIF: Ethdev port_id=0 invalid tx_offload: 0x1000e, valid value: 0xc96af

配置接口/查看接口

root@r750-132:~/dpvs/bin# ./dpip link show
1: dpdk0: socket 1 mtu 1500 rx-queue 8 tx-queue 8UP 100000 Mbps full-duplex auto-nego lldp addr E8:EB:D3:A3:83:76 OF_RX_IP_CSUM OF_TX_IP_CSUM OF_TX_TCP_CSUM OF_TX_UDP_CSUM 
root@r750-132:~/dpvs/bin# ./dpip addr show
inet 10.1.1.132/24 scope global dpdk0valid_lft forever preferred_lft forever
inet6 fe80::eaeb:d3ff:fea3:8376/64 scope link dpdk0valid_lft forever preferred_lft forever