DPDK(F-Stack) 实现UDP通信

因刚开始学习DPDK，在学习过程中了解到需使用用户态协议栈，在网上找到F-Stack的相关介绍，但是缺乏DPDK的相关知识，导致使用F-Stack 时UDP数据无法收到

• 一文了解dpdk rte_ring无锁队列
• F-Stack实现UDP服务端、客户端，并进行吞吐量测试的实现
• github F-Stack

环境

在一台机器上，系统为Ubuntu 22.4

硬件上是一张100G网卡 两个网口，光纤直连两个网口

UDPServer 的port0.ini配置文件中设置IP如下并使用网卡的PORT0端口

[port0]
addr=192.168.2.15
netmask=255.255.255.0
broadcast=192.168.2.255
gateway=192.168.2.1

UDPClient 的port1.ini配置文件中设置IP如下并使用网卡的PORT1端口

[port1]
addr=192.168.2.16
netmask=255.255.255.0
broadcast=192.168.2.255
gateway=192.168.2.1

问题

服务端可以接收到客户端的arp请求报文并且应答了arp报文，然后就卡住了，ff_envent()函数无法检测到sockfd有数据
UDPServer配置文件中的 lcore_mask 参数我配置的是f0

lcore_mask=f0

本意是想启动一个UDPServer程序使用4个逻辑核心，经过测试启动一个程序只能使用配置的第一个逻辑核心，此时去使用UDPClient发送数据到UDPServer时就无法收到数据，不知道是我个人有这个问题，还是在同一电脑上的同一张网卡配置多个逻辑核心时都无法收到数据，就是这么设计的吗

解决

后续又买了一块相同的100G网卡，放在另一台电脑，光纤直连，然后发送数据，此时竟然可以收到数据。那就证明代码收发是正确的

后续查看源码时在源码文件 ff_dpdk_if.c 的 main_loop() 函数中有调用 process_dispatch_ring() 函数，函数内调用了 rte_ring_dequeue_burst()，大概了解了一下，我猜测因为我配置了4个核心，在初始化时创建了4个ring队列

从打印的信息来看也是这样的：

create ring:dispatch_ring_p0_q0 success, 2047 ring entries are now free!
create ring:dispatch_ring_p0_q1 success, 2047 ring entries are now free!
create ring:dispatch_ring_p0_q2 success, 2047 ring entries are now free!
create ring:dispatch_ring_p0_q3 success, 2047 ring entries are now free!

我个人猜测，很有可能是因为接收UDP数据被分发给了其它的队列上，我只启动一个程序也就是只是用了第一个ring队列，那么就不可能收到数据，然后我相应的启动了4个程序，终于收到了数据，老天爷，对于一个刚学习DPDK的人简直是折磨。

$ ./UDPServer -c port0.ini -p 0 &
$ ./UDPServer -c port0.ini -p 1 &
$ ./UDPServer -c port0.ini -p 2 &
$ ./UDPServer -c port0.ini -p 3 &

如果不想启动多个进程的话，把配置文件中的核心数配置为1个，启动一个程序就可以收发数据了。

代码

UDPserver

#include <stdio.h>
#include <sys/ioctl.h>
#include <string.h>
#include <cerrno>
#include <stdlib.h>
#include <rte_ethdev.h>// F-stack
#include <ff_api.h>
#include <ff_config.h>#define MAX_EVENTS 512
#define MAXLINE 512int kq;
int sockfd;
/* kevent set */
struct kevent kevSet;
/* events */
struct kevent events[MAX_EVENTS];int loop(void *arg){char buf[MAXLINE] = {"0"};struct sockaddr_in cliAddr;socklen_t recvAddrLen = sizeof(cliAddr);int nevents = ff_kevent(kq, NULL, 0, events, MAX_EVENTS, NULL);if (nevents < 0) {printf("ff_kevent failed:%d, %s\n", errno, strerror(errno));return -1;}for (int i = 0; i < nevents; ++i) {struct kevent event = events[i];int clientfd = (int)event.ident;printf("event.data = %d \n", (int)event.data);printf("listen event %d\n", nevents);printf("clientfd %d\n", clientfd);printf("sockfd %d\n", sockfd);printf("event.flags = %d \n", event.flags);if (clientfd == sockfd) {int n = ff_recvfrom(sockfd, buf, MAXLINE, 0,  (struct linux_sockaddr *)&cliAddr, &recvAddrLen);if(n < 0){printf("ff_recvfrom failed, errno:%d, %s\n", errno, strerror(errno));}else{// 接收数据 并打印printf("Server recv   %s\n", buf);printf("Client Family %d\n", cliAddr.sin_family);printf("Client Addr   %s\n", inet_ntoa(cliAddr.sin_addr));printf("Client Port   %u\n", ntohs(cliAddr.sin_port));strcpy(buf, "hello client, i am server");int num = ff_sendto(sockfd, buf, sizeof(buf) / sizeof(buf[0]), 0, (struct linux_sockaddr *)&cliAddr, recvAddrLen);if(num < 0){printf("ff_sendto failed, errno:%d, %s\n", errno, strerror(errno));ff_stop_run();}else{printf("Server send %d bytes\n", num);}}}}return 1;
}int main(int argc, char **argv){int on = 1;struct sockaddr_in my_addr;ff_init(argc, argv);kq = ff_kqueue();if (kq < 0) {printf("ff_kqueue failed, errno:%d, %s\n", errno, strerror(errno));exit(1);}sockfd = ff_socket(AF_INET, SOCK_DGRAM, 0);ff_ioctl(sockfd, FIONBIO, &on);bzero(&my_addr, sizeof(my_addr));my_addr.sin_family = AF_INET;my_addr.sin_port = htons(34824);my_addr.sin_addr.s_addr =  htonl(INADDR_ANY);int ret = ff_bind(sockfd, (struct linux_sockaddr *)&my_addr, sizeof(my_addr));if (ret < 0) {printf("ff_bind failed, sockfd:%d, errno:%d, %s\n", sockfd, errno, strerror(errno));exit(1);}EV_SET(&kevSet, sockfd, EVFILT_READ, EV_ADD, 0, MAX_EVENTS, NULL);/* Update kqueue */ff_kevent(kq, &kevSet, 1, NULL, 0, NULL);ff_run(loop, NULL);if(rte_eth_dev_stop(0) < 0)rte_exit(EXIT_FAILURE, "Cannot close eth %" PRIu16 "\n", 0);rte_eal_cleanup();return 1;
}

UDPClient

#include <stdio.h>#include <sys/ioctl.h>
#include <string.h>
#include <cerrno>
#include <stdlib.h>
#include <rte_ethdev.h>// F-stack
#include <ff_api.h>
#include <ff_config.h>#define MAX_EVENTS 512
#define MAXLINE 512
int kq;
int sockfd;
/* kevent set */
struct kevent kevSet;
/* events */
struct kevent events[MAX_EVENTS];
struct sockaddr_in serverAddr;int loop(void *arg){int num;char buf[MAXLINE] = {"0"};struct sockaddr_in recvAddr;socklen_t recvAddrLen = sizeof(recvAddr);strcpy(buf,"hello, i am client");num = ff_sendto(sockfd, buf, sizeof(buf)/sizeof(buf[0]), 0, (struct linux_sockaddr *)&serverAddr, sizeof(serverAddr));if(num < 0){printf("ff_kevent failed:%d, %s\n", errno, strerror(errno));ff_stop_run();}elseprintf("sendto num:%d\n", num);int nevents = ff_kevent(kq, NULL, 0, events, MAX_EVENTS, NULL);if (nevents < 0) {printf("ff_kevent failed:%d, %s\n", errno, strerror(errno));ff_stop_run();return -1;}for (int i = 0; i < nevents; ++i) {printf("listen nevents !!! \n");struct kevent event = events[i];int serverfd = (int)event.ident;printf("event.data = %d \n", (int)event.data);printf("event.flags = %d \n", event.flags);printf("serverfd = %d \n", serverfd);printf("sockfd = %d \n", sockfd);if (serverfd == sockfd) {int n = ff_recvfrom(sockfd, buf, MAXLINE, 0,  (struct linux_sockaddr *)&recvAddr, &recvAddrLen);if(n < 0){printf("ff_recvfrom failed, errno:%d, %s\n", errno, strerror(errno));}else{// 接收数据 并打印printf("Client recv   %s\n", buf);printf("Server Family %d\n", recvAddr.sin_family);printf("Server Addr   %s\n", inet_ntoa(recvAddr.sin_addr));printf("Server Port   %u\n", ntohs(recvAddr.sin_port));ff_stop_run();}}}return 1;
}int main(int argc, char **argv){struct sockaddr_in my_addr;int on = 1;ff_init(argc, argv);kq = ff_kqueue();if (kq < 0) {printf("ff_kqueue failed, errno:%d, %s\n", errno, strerror(errno));exit(1);}sockfd = ff_socket(AF_INET, SOCK_DGRAM, 0);ff_ioctl(sockfd, FIONBIO, &on);bzero(&my_addr, sizeof(my_addr));my_addr.sin_family = AF_INET;my_addr.sin_port = htons(34825);my_addr.sin_addr.s_addr = inet_addr( "192.168.2.16" );ff_bind(sockfd, (struct linux_sockaddr *)&my_addr, sizeof(my_addr));// 指定发送的地址和端口号bzero(&serverAddr, sizeof(serverAddr));serverAddr.sin_family = AF_INET;serverAddr.sin_port = htons(34824);serverAddr.sin_addr.s_addr = inet_addr( "192.168.2.15" );EV_SET(&kevSet, sockfd, EVFILT_READ, EV_ADD, 0, MAX_EVENTS, NULL);/* Update kqueue */ff_kevent(kq, &kevSet, 1, NULL, 0, NULL);ff_run(loop, NULL);ff_close(sockfd);return 1;
}