rdma-core debug

export MLX5_DEBUG_MASK=0xff
export MLX5_DEBUG_FILE=/tmp/mlx5.txt

git clone https://github.com/linux-rdma/rdma-core.git

cd rdma-core

./build.sh

修改build/CMakeCache.txt

MLX5_DEBUG:BOOL=TRUE

function install_rdma_core
{local dir=/swgwork/cmi/rdma-core/build/libcd /lib64//bin/cp $dir/librdmacm.so.1.3.56.0      ./bin/cp $dir/libibverbs.so.1.14.56.0    ./bin/cp $dir/libmlx5.so.1.25.56.0       ./bin/rm librdmacm.so.1/bin/rm librdmacm.so/bin/rm libibverbs.so.1/bin/rm libibverbs.so/bin/rm libibverbs/libmlx5-rdmav34.so/bin/rm libmlx5.so.1ln -s librdmacm.so.1.3.56.0 librdmacm.soln -s librdmacm.so.1.3.56.0 librdmacm.so.1ln -s libibverbs.so.1.14.56.0 libibverbs.soln -s libibverbs.so.1.14.56.0 libibverbs.so.1ln -s libmlx5.so.1.25.56.0 libmlx5.so.1cd libibverbsln -s ../libmlx5.so.1.25.56.0 libmlx5-rdmav34.so
}

运行ib_send_bw:

# ip netns exec n11 /swgwork/cmi/perftest/ib_send_bw -d mlx5_3  -z
# ip netns exec n11 /swgwork/cmi/perftest/ib_send_bw -d mlx5_3 1.1.1.1 -D 10000 -z

server端收包的时候得到下面的log:

# tail -f /tmp/mlx5.txt

mlx5_get_next_cqe:564: dump cqe for cqn 0x1a9e:
00000000 00000000 00000000 00000000
00000000 00000000 00000000 00000000
00000000 00000000 00000000 00010000
0001d00e 98d17383 0000024d 03e3f720
mlx5_get_next_cqe:564: dump cqe for cqn 0x1a9e:
00000000 00000000 00000000 00000000
00000000 00000000 00000000 00000000
00000000 00000000 00000000 00010000
0001d00e 98d17f13 0000024d 03e46b20
mlx5_get_next_cqe:564: dump cqe for cqn 0x1a9e:
00000000 00000000 00000000 00000000
00000000 00000000 00000000 00000000
00000000 00000000 00000000 00010000
0001d00e 98d18aa3 0000024d 03e52e20

上面的输出对应下面的数据结构:

struct mlx5_cqe64 {union {struct {uint8_t         rsvd0[2];__be16          wqe_id;uint8_t         rsvd4[13];uint8_t         ml_path;uint8_t         rsvd20[4];__be16          slid;__be32          flags_rqpn;uint8_t         hds_ip_ext;uint8_t         l4_hdr_type_etc;__be16          vlan_info;};struct mlx5_tm_cqe tm_cqe;/* TMH is scattered to CQE upon match */struct ibv_tmh tmh;};__be32          srqn_uidx;__be32          imm_inval_pkey;uint8_t         app;uint8_t         app_op;__be16          app_info;__be32          byte_cnt;__be64          timestamp;__be32          sop_drop_qpn;__be16          wqe_counter;uint8_t         signature;uint8_t         op_own;
};

比如上面的例子byte_cnt就是0x10000(65536)，wqe_counter分别是03e4，03e5，timestamp分别是0001d00e 98d17f13，0001d00e 98d18aa3。 

crash> ps | grep ib_send_bw
>  685270    2581  23  ffff90c5b0c28000  RU   0.0     6716     4832  ib_send_bw
crash> files ffff90c5b0c28000
PID: 685270   TASK: ffff90c5b0c28000  CPU: 23   COMMAND: "ib_send_bw"
ROOT: /    CWD: /labhome/cmi/mi/drgn/ib
 FD       FILE            DENTRY           INODE       TYPE PATH
  0 ffff90c3cda4eb80 ffff90c3cc6399a0 ffff90c3d1325a40 CHR  /dev/pts/9
  1 ffff90c3cda4eb80 ffff90c3cc6399a0 ffff90c3d1325a40 CHR  /dev/pts/9
  2 ffff90c3cda4eb80 ffff90c3cc6399a0 ffff90c3d1325a40 CHR  /dev/pts/9
  3 ffff90c4de695200 ffff90c5504eccd0 ffff90c3cf2dd978 CHR  /dev/infiniband/uverbs3
  4 ffff90c4de695980 ffff90c4c814c148 ffff90c4f9ca6380 REG  /tmp/mlx5.txt
  5 ffff90c4de695f80 ffff90c4c81c8000 ffff90c3c05ed580 UNKN [infinibandevent]
  6 ffff90c4de694c00 ffff90c47ffcf710 ffff90c3e222a0f0 CHR  /dev/infiniband/rdma_cm
  7 ffff90c4de697780 ffff90c5504eccd0 ffff90c3cf2dd978 CHR  /dev/infiniband/uverbs3
  8 ffff90c4de695b00 ffff90c4c814c148 ffff90c4f9ca6380 REG  /tmp/mlx5.txt
  9 ffff90c4de696e80 ffff90c4c81c83d8 ffff90c3c05ed580 UNKN [infinibandevent]