Linux C语言实践eBPF

手动编译了解过程

通过对关键步骤make M=samples/bpf的实践，我们已经可以编译出内核源码中提供的ebpf样例。但这还不够我们充分地理解这个编译过程，我们将这编译过程拆解一下，拆解成可以一步步执行的那种，首先是环境准备：

$ sudo apt-get install linux-source
$ cd /usr/src/
$ ls
linux-source-5.15.0.tar.bz2 linux-headers-5.15.0-43-generic linux-source-5.15.0
$ sudo tar -xvf linux-source-5.15.0.tar.bz2 -C /tmp/
$ cd /tmp/linux-source-5.15.0/

内核源码树环境准备

$ sudo make oldconfig
$ sudo make init 
$ sudo make headers_install

进入tools目录编译依赖库

$ sudo cd tools/lib/bpf/

编译依赖库为object中间件

$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o libbpf.o libbpf.c$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o bpf.o bpf.c$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o btf.o btf.c$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o nlattr.o nlattr.c$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o strset.o strset.c$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o str_error.o str_error.c$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o hashmap.o hashmap.c$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o libbpf_probes.o libbpf_probes.c$ sudo gcc -I. -I/tmp/linux-source-5.15.0/samples/bpf/../..//tools/include -I/tmp/linux-source-5.15.0/samples/bpf/../..//tools/include/uapi -fvisibility=hidden -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -D"BUILD_STR(s)=#s" -c -o /tmp/linux-source-5.15.0/samples/bpf/libbpf/staticobjs/gen_loader.o gen_loader.c$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o relo_core.o relo_core.c$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o netlink.o netlink.c$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o linker.o linker.c$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o xsk.o xsk.c$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o ringbuf.o ringbuf.c$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o btf_dump.o btf_dump.c

将编译好的object文件链接为libbpf.a静态库

$ cd /tmp/linux-source-5.15.0/tools/lib/bpf
$ sudo ar rcs libbpf.a bpf.o btf.o libbpf.o nlattr.o str_error.o strset.o gen_loader.o libbpf_probes.o relo_core.o ringbuf.o linker.o netlink.o xsk.o btf_dump.o

编译用户态测试程序

$ cd /tmp/linux-source-5.15.0
$ sudo gcc -I./usr/include -I./tools/testing/selftests/bpf/ -I/tmp/linux-source-5.15.0/samples/bpf/libbpf/include -I./tools/include -I./tools/perf -DHAVE_ATTR_TEST=0  -c -o samples/bpf/trace_output_user.o samples/bpf/trace_output_user.c
$ sudo gcc -I./usr/include -I./tools/testing/selftests/bpf/ -I/tmp/linux-source-5.15.0/samples/bpf/libbpf/include -I./tools/include -I./tools/perf -DHAVE_ATTR_TEST=0 -o samples/bpf/trace_output samples/bpf/trace_output_user.o /tmp/linux-source-5.15.0/samples/bpf/libbpf/libbpf.a -lelf -lz -lrt

编译示例程序的第二部分，即需要给内核虚拟机运行的，由bpf_load()加载的字节码

$ cd /tmp/linux-source-5.15.0
$ sudo clang -g -nostdinc -isystem /usr/lib/gcc/x86_64-redhat-linux/8/include -I./arch/x86/include -I./arch/x86/include/generated -I./include -I./arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I./include/uapi -I./include/generated/uapi -include ./include/linux/kconfig.h -Isamples/bpf -Isamples/bpf/libbpf/include/ -Isamples/bpf/libbpf/ -I./tools/testing/selftests/bpf/ -D__KERNEL__ -D__BPF_TRACING__ -D__TARGET_ARCH_x86 -O2 -emit-llvm -c samples/bpf/trace_output_kern.c -o - | sudo llc -march=bpf -filetype=obj -o samples/bpf/trace_output_kern.o
$ sudo llvm-strip -g /tmp/linux-source-5.15.0/samples/bpf/trace_output_kern.o

测试 

$ cd /tmp/linux-source-5.15.0/samples/bpf
$ sudo ./trace_output
recv 236472 events per sec
100013+0 records in
100013+0 records out
51206656 bytes (51 MB, 49 MiB) copied, 0.420891 s, 122 MB/s

解析源代码

BPF程序结构图：

由上图可知BPF程序要由两部分组成用户态执行程序和BPF bytecode。

trace_output示例的量部分如下，其中trace_output_user.c为用户态执行程序，trace_output_kern.c为BPF bytecode，所以trace_output_user.c使用gcc编译，trace_output_kern.c使用clang编译。

$ ls samples/bpf/ | grep trace_output
trace_output_kern.c
trace_output_user.c

trace_output_user.c里的主要内容

int main(int argc, char **argv)
{struct perf_buffer_opts pb_opts = {};struct bpf_link *link = NULL;struct bpf_program *prog;struct perf_buffer *pb;struct bpf_object *obj;int map_fd, ret = 0;char filename[256];FILE *f;snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);obj = bpf_object__open_file(filename, NULL);if (libbpf_get_error(obj)) {fprintf(stderr, "ERROR: opening BPF object file failed\n");return 0;}/* load BPF program */if (bpf_object__load(obj)) {fprintf(stderr, "ERROR: loading BPF object file failed\n");goto cleanup;}map_fd = bpf_object__find_map_fd_by_name(obj, "my_map");if (map_fd < 0) {fprintf(stderr, "ERROR: finding a map in obj file failed\n");goto cleanup;}prog = bpf_object__find_program_by_name(obj, "bpf_prog1");if (libbpf_get_error(prog)) {fprintf(stderr, "ERROR: finding a prog in obj file failed\n");goto cleanup;}link = bpf_program__attach(prog);if (libbpf_get_error(link)) {fprintf(stderr, "ERROR: bpf_program__attach failed\n");link = NULL;goto cleanup;}pb_opts.sample_cb = print_bpf_output;pb = perf_buffer__new(map_fd, 8, &pb_opts);ret = libbpf_get_error(pb);if (ret) {printf("failed to setup perf_buffer: %d\n", ret);return 1;}f = popen("taskset 1 dd if=/dev/zero of=/dev/null", "r");(void) f;start_time = time_get_ns();while ((ret = perf_buffer__poll(pb, 1000)) >= 0 && cnt < MAX_CNT) {}kill(0, SIGINT);cleanup:bpf_link__destroy(link);bpf_object__close(obj);return ret;
}

其中bpf_object__open_file打开的字节码文件trace_output_kern.c里的内容如下：

struct {__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);__uint(key_size, sizeof(int));__uint(value_size, sizeof(u32));__uint(max_entries, 2);
} my_map SEC(".maps");SEC("kprobe/" SYSCALL(sys_write))
int bpf_prog1(struct pt_regs *ctx)
{struct S {u64 pid;u64 cookie;} data;data.pid = bpf_get_current_pid_tgid();data.cookie = 0x12345678;bpf_perf_event_output(ctx, &my_map, 0, &data, sizeof(data));return 0;
}char _license[] SEC("license") = "GPL";
u32 _version SEC("version") = LINUX_VERSION_CODE;

参考链接