diff --git a/MAINTAINERS b/MAINTAINERS index 4be087b88e23c..4aafdfce2c9e5 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3173,6 +3173,14 @@ S: Maintained F: hw/semihosting/ F: include/hw/semihosting/ +EBPF: +M: Jason Wang +R: Andrew Melnychenko +R: Yuri Benditovich +S: Maintained +F: ebpf/* +F: tools/ebpf/* + Build and test automation ------------------------- Build and test automation diff --git a/configure b/configure index 5860bdb77ba82..9d18e941f5994 100755 --- a/configure +++ b/configure @@ -342,6 +342,7 @@ vhost_vsock="$default_feature" vhost_user="no" vhost_user_blk_server="auto" vhost_user_fs="$default_feature" +bpf="" kvm="auto" hax="auto" hvf="auto" @@ -1236,6 +1237,10 @@ for opt do ;; --enable-membarrier) membarrier="yes" ;; + --disable-bpf) bpf="no" + ;; + --enable-bpf) bpf="yes" + ;; --disable-blobs) blobs="false" ;; --with-pkgversion=*) pkgversion="$optarg" @@ -1845,6 +1850,7 @@ disabled with --disable-FEATURE, default is enabled if available vhost-user vhost-user backend support vhost-user-blk-server vhost-user-blk server support vhost-vdpa vhost-vdpa kernel backend support + bpf BPF kernel support spice spice rbd rados block device (rbd) libiscsi iscsi support @@ -5057,6 +5063,30 @@ else membarrier=no fi +########################################## +# check for usable bpf system call +if test "$bpf" = ""; then + have_bpf=no + if test "$linux" = "yes" -a "$bigendian" != "yes"; then + cat > $TMPC << EOF + #include + #include + int main(void) { + struct bpf_object *obj = NULL; + bpf_object__load(obj); + exit(0); + } +EOF + if compile_prog "" "-lbpf" ; then + have_bpf=yes + bpf=yes + fi + fi + if test "$have_bpf" = "no"; then + feature_not_found "bpf" "the libbpf is not available" + fi +fi + ########################################## # check if rtnetlink.h exists and is useful have_rtnetlink=no @@ -5905,6 +5935,9 @@ fi if test "$membarrier" = "yes" ; then echo "CONFIG_MEMBARRIER=y" >> $config_host_mak fi +if test "$bpf" = "yes" -a "$bigendian" != "yes" -a "$linux" = "yes" ; then + echo "CONFIG_EBPF=y" >> $config_host_mak +fi if test "$signalfd" = "yes" ; then echo "CONFIG_SIGNALFD=y" >> $config_host_mak fi diff --git a/docs/ebpf_rss.rst b/docs/ebpf_rss.rst new file mode 100644 index 0000000000000..5beecfe33f03a --- /dev/null +++ b/docs/ebpf_rss.rst @@ -0,0 +1,125 @@ +=========================== +eBPF RSS virtio-net support +=========================== + +RSS(Receive Side Scaling) is used to distribute network packets to guest virtqueues +by calculating packet hash. Usually every queue is processed then by a specific guest CPU core. + +For now there are 2 RSS implementations in qemu: +- 'in-qemu' RSS (functions if qemu receives network packets, i.e. vhost=off) +- eBPF RSS (can function with also with vhost=on) + +eBPF support (CONFIG_EBPF) is enabled by 'configure' script. +To enable eBPF RSS support use './configure --enable-bpf'. + +If steering BPF is not set for kernel's TUN module, the TUN uses automatic selection +of rx virtqueue based on lookup table built according to calculated symmetric hash +of transmitted packets. +If steering BPF is set for TUN the BPF code calculates the hash of packet header and +returns the virtqueue number to place the packet to. + +Simplified decision formula: + +.. code:: C + + queue_index = indirection_table[hash()%] + + +Not for all packets, the hash can/should be calculated. + +Note: currently, eBPF RSS does not support hash reporting. + +eBPF RSS turned on by different combinations of vhost-net, vitrio-net and tap configurations: + +- eBPF is used: + + tap,vhost=off & virtio-net-pci,rss=on,hash=off + +- eBPF is used: + + tap,vhost=on & virtio-net-pci,rss=on,hash=off + +- 'in-qemu' RSS is used: + + tap,vhost=off & virtio-net-pci,rss=on,hash=on + +- eBPF is used, hash population feature is not reported to the guest: + + tap,vhost=on & virtio-net-pci,rss=on,hash=on + +If CONFIG_EBPF is not set then only 'in-qemu' RSS is supported. +Also 'in-qemu' RSS, as a fallback, is used if the eBPF program failed to load or set to TUN. + +RSS eBPF program +---------------- + +RSS program located in ebpf/rss.bpf.skeleton.h generated by bpftool. +So the program is part of the qemu binary. +Initially, the eBPF program was compiled by clang and source code located at tools/ebpf/rss.bpf.c. +Prerequisites to recompile the eBPF program (regenerate ebpf/rss.bpf.skeleton.h): + + llvm, clang, kernel source tree, bpftool + Adjust 'linuxhdrs' in Makefile.ebpf to reflect the location of the kernel source tree + + $ cd ebpf + $ make -f Makefile.ebpf + +Current eBPF RSS implementation uses 'bounded loops' with 'backward jump instructions' which present in the last kernels. +Overall eBPF RSS works on kernels 5.8+. + +eBPF RSS implementation +----------------------- + +eBPF RSS loading functionality located in ebpf/ebpf_rss.c and ebpf/ebpf_rss.h. + +The `struct EBPFRSSContext` structure that holds 4 file descriptors: + +- ctx - pointer of the libbpf context. +- program_fd - file descriptor of the eBPF RSS program. +- map_configuration - file descriptor of the 'configuration' map. This map contains one element of 'struct EBPFRSSConfig'. This configuration determines eBPF program behavior. +- map_toeplitz_key - file descriptor of the 'Toeplitz key' map. One element of the 40byte key prepared for the hashing algorithm. +- map_indirections_table - 128 elements of queue indexes. + +`struct EBPFRSSConfig` fields: + +- redirect - "boolean" value, should the hash be calculated, on false - `default_queue` would be used as the final decision. +- populate_hash - for now, not used. eBPF RSS doesn't support hash reporting. +- hash_types - binary mask of different hash types. See `VIRTIO_NET_RSS_HASH_TYPE_*` defines. If for packet hash should not be calculated - `default_queue` would be used. +- indirections_len - length of the indirections table, maximum 128. +- default_queue - the queue index that used for packet that shouldn't be hashed. For some packets, the hash can't be calculated(g.e ARP). + +Functions: + +- `ebpf_rss_init()` - sets ctx to NULL, which indicates that EBPFRSSContext is not loaded. +- `ebpf_rss_load()` - creates 3 maps and loads eBPF program from tun_rss_steering.h. Returns 'true' on success. After that, program_fd can be used to set steering for TAP. +- `ebpf_rss_set_all()` - sets values for eBPF maps. `indirections_table` length is in EBPFRSSConfig. `toeplitz_key` is VIRTIO_NET_RSS_MAX_KEY_SIZE aka 40 bytes array. +- `ebpf_rss_unload()` - close all file descriptors and set ctx to NULL. + +Simplified eBPF RSS workflow: + +.. code:: C + + struct EBPFRSSConfig config; + config.redirect = 1; + config.hash_types = VIRTIO_NET_RSS_HASH_TYPE_UDPv4 | VIRTIO_NET_RSS_HASH_TYPE_TCPv4; + config.indirections_len = VIRTIO_NET_RSS_MAX_TABLE_LEN; + config.default_queue = 0; + + uint16_t table[VIRTIO_NET_RSS_MAX_TABLE_LEN] = {...}; + uint8_t key[VIRTIO_NET_RSS_MAX_KEY_SIZE] = {...}; + + struct EBPFRSSContext ctx; + ebpf_rss_init(&ctx); + ebpf_rss_load(&ctx); + ebpf_rss_set_all(&ctx, &config, table, key); + if (net_client->info->set_steering_ebpf != NULL) { + net_client->info->set_steering_ebpf(net_client, ctx->program_fd); + } + ... + ebpf_unload(&ctx); + + +NetClientState SetSteeringEBPF() +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For now, `set_steering_ebpf()` method supported by Linux TAP NetClientState. The method requires an eBPF program file descriptor as an argument. diff --git a/ebpf/ebpf_rss-stub.c b/ebpf/ebpf_rss-stub.c new file mode 100644 index 0000000000000..e71e229190de9 --- /dev/null +++ b/ebpf/ebpf_rss-stub.c @@ -0,0 +1,40 @@ +/* + * eBPF RSS stub file + * + * Developed by Daynix Computing LTD (http://www.daynix.com) + * + * Authors: + * Yuri Benditovich + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include "ebpf/ebpf_rss.h" + +void ebpf_rss_init(struct EBPFRSSContext *ctx) +{ + +} + +bool ebpf_rss_is_loaded(struct EBPFRSSContext *ctx) +{ + return false; +} + +bool ebpf_rss_load(struct EBPFRSSContext *ctx) +{ + return false; +} + +bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config, + uint16_t *indirections_table, uint8_t *toeplitz_key) +{ + return false; +} + +void ebpf_rss_unload(struct EBPFRSSContext *ctx) +{ + +} diff --git a/ebpf/ebpf_rss.c b/ebpf/ebpf_rss.c new file mode 100644 index 0000000000000..f7f7102da5239 --- /dev/null +++ b/ebpf/ebpf_rss.c @@ -0,0 +1,165 @@ +/* + * eBPF RSS loader + * + * Developed by Daynix Computing LTD (http://www.daynix.com) + * + * Authors: + * Andrew Melnychenko + * Yuri Benditovich + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include "qemu/error-report.h" + +#include +#include + +#include "hw/virtio/virtio-net.h" /* VIRTIO_NET_RSS_MAX_TABLE_LEN */ + +#include "ebpf/ebpf_rss.h" +#include "ebpf/rss.bpf.skeleton.h" +#include "trace.h" + +void ebpf_rss_init(struct EBPFRSSContext *ctx) +{ + if (ctx != NULL) { + ctx->obj = NULL; + } +} + +bool ebpf_rss_is_loaded(struct EBPFRSSContext *ctx) +{ + return ctx != NULL && ctx->obj != NULL; +} + +bool ebpf_rss_load(struct EBPFRSSContext *ctx) +{ + struct rss_bpf *rss_bpf_ctx; + + if (ctx == NULL) { + return false; + } + + rss_bpf_ctx = rss_bpf__open(); + if (rss_bpf_ctx == NULL) { + trace_ebpf_error("eBPF RSS", "can not open eBPF RSS object"); + goto l_issue; + } + + bpf_program__set_socket_filter(rss_bpf_ctx->progs.tun_rss_steering_prog); + + if (rss_bpf__load(rss_bpf_ctx)) { + trace_ebpf_error("eBPF RSS", "can not load RSS program"); + goto l_issue; + } + + ctx->obj = rss_bpf_ctx; + ctx->program_fd = bpf_program__fd( + rss_bpf_ctx->progs.tun_rss_steering_prog); + ctx->map_configuration = bpf_map__fd( + rss_bpf_ctx->maps.tap_rss_map_configurations); + ctx->map_indirections_table = bpf_map__fd( + rss_bpf_ctx->maps.tap_rss_map_indirection_table); + ctx->map_toeplitz_key = bpf_map__fd( + rss_bpf_ctx->maps.tap_rss_map_toeplitz_key); + + return true; +l_issue: + rss_bpf__destroy(rss_bpf_ctx); + ctx->obj = NULL; + + return false; +} + +static bool ebpf_rss_set_config(struct EBPFRSSContext *ctx, + struct EBPFRSSConfig *config) +{ + uint32_t map_key = 0; + + if (!ebpf_rss_is_loaded(ctx)) { + return false; + } + if (bpf_map_update_elem(ctx->map_configuration, + &map_key, config, 0) < 0) { + return false; + } + return true; +} + +static bool ebpf_rss_set_indirections_table(struct EBPFRSSContext *ctx, + uint16_t *indirections_table, + size_t len) +{ + uint32_t i = 0; + + if (!ebpf_rss_is_loaded(ctx) || indirections_table == NULL || + len > VIRTIO_NET_RSS_MAX_TABLE_LEN) { + return false; + } + + for (; i < len; ++i) { + if (bpf_map_update_elem(ctx->map_indirections_table, &i, + indirections_table + i, 0) < 0) { + return false; + } + } + return true; +} + +static bool ebpf_rss_set_toepliz_key(struct EBPFRSSContext *ctx, + uint8_t *toeplitz_key) +{ + uint32_t map_key = 0; + + /* prepare toeplitz key */ + uint8_t toe[VIRTIO_NET_RSS_MAX_KEY_SIZE] = {}; + + if (!ebpf_rss_is_loaded(ctx) || toeplitz_key == NULL) { + return false; + } + memcpy(toe, toeplitz_key, VIRTIO_NET_RSS_MAX_KEY_SIZE); + *(uint32_t *)toe = ntohl(*(uint32_t *)toe); + + if (bpf_map_update_elem(ctx->map_toeplitz_key, &map_key, toe, + 0) < 0) { + return false; + } + return true; +} + +bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config, + uint16_t *indirections_table, uint8_t *toeplitz_key) +{ + if (!ebpf_rss_is_loaded(ctx) || config == NULL || + indirections_table == NULL || toeplitz_key == NULL) { + return false; + } + + if (!ebpf_rss_set_config(ctx, config)) { + return false; + } + + if (!ebpf_rss_set_indirections_table(ctx, indirections_table, + config->indirections_len)) { + return false; + } + + if (!ebpf_rss_set_toepliz_key(ctx, toeplitz_key)) { + return false; + } + + return true; +} + +void ebpf_rss_unload(struct EBPFRSSContext *ctx) +{ + if (!ebpf_rss_is_loaded(ctx)) { + return; + } + + rss_bpf__destroy(ctx->obj); + ctx->obj = NULL; +} diff --git a/ebpf/ebpf_rss.h b/ebpf/ebpf_rss.h new file mode 100644 index 0000000000000..0fcb4e010fe33 --- /dev/null +++ b/ebpf/ebpf_rss.h @@ -0,0 +1,44 @@ +/* + * eBPF RSS header + * + * Developed by Daynix Computing LTD (http://www.daynix.com) + * + * Authors: + * Andrew Melnychenko + * Yuri Benditovich + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + */ + +#ifndef QEMU_EBPF_RSS_H +#define QEMU_EBPF_RSS_H + +struct EBPFRSSContext { + void *obj; + int program_fd; + int map_configuration; + int map_toeplitz_key; + int map_indirections_table; +}; + +struct EBPFRSSConfig { + uint8_t redirect; + uint8_t populate_hash; + uint32_t hash_types; + uint16_t indirections_len; + uint16_t default_queue; +}; + +void ebpf_rss_init(struct EBPFRSSContext *ctx); + +bool ebpf_rss_is_loaded(struct EBPFRSSContext *ctx); + +bool ebpf_rss_load(struct EBPFRSSContext *ctx); + +bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config, + uint16_t *indirections_table, uint8_t *toeplitz_key); + +void ebpf_rss_unload(struct EBPFRSSContext *ctx); + +#endif /* QEMU_EBPF_RSS_H */ diff --git a/ebpf/meson.build b/ebpf/meson.build new file mode 100644 index 0000000000000..f5bd5a0f01d5c --- /dev/null +++ b/ebpf/meson.build @@ -0,0 +1 @@ +common_ss.add(when: 'CONFIG_EBPF', if_true: files('ebpf_rss.c'), if_false: files('ebpf_rss-stub.c')) diff --git a/ebpf/rss.bpf.skeleton.h b/ebpf/rss.bpf.skeleton.h new file mode 100644 index 0000000000000..76616a5ca601a --- /dev/null +++ b/ebpf/rss.bpf.skeleton.h @@ -0,0 +1,397 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ + +/* THIS FILE IS AUTOGENERATED! */ +#ifndef __RSS_BPF_SKEL_H__ +#define __RSS_BPF_SKEL_H__ + +#include +#include + +struct rss_bpf { + struct bpf_object_skeleton *skeleton; + struct bpf_object *obj; + struct { + struct bpf_map *tap_rss_map_configurations; + struct bpf_map *tap_rss_map_indirection_table; + struct bpf_map *tap_rss_map_toeplitz_key; + } maps; + struct { + struct bpf_program *tun_rss_steering_prog; + } progs; + struct { + struct bpf_link *tun_rss_steering_prog; + } links; +}; + +static void +rss_bpf__destroy(struct rss_bpf *obj) +{ + if (!obj) + return; + if (obj->skeleton) + bpf_object__destroy_skeleton(obj->skeleton); + free(obj); +} + +static inline int +rss_bpf__create_skeleton(struct rss_bpf *obj); + +static inline struct rss_bpf * +rss_bpf__open_opts(const struct bpf_object_open_opts *opts) +{ + struct rss_bpf *obj; + + obj = (struct rss_bpf *)calloc(1, sizeof(*obj)); + if (!obj) + return NULL; + if (rss_bpf__create_skeleton(obj)) + goto err; + if (bpf_object__open_skeleton(obj->skeleton, opts)) + goto err; + + return obj; +err: + rss_bpf__destroy(obj); + return NULL; +} + +static inline struct rss_bpf * +rss_bpf__open(void) +{ + return rss_bpf__open_opts(NULL); +} + +static inline int +rss_bpf__load(struct rss_bpf *obj) +{ + return bpf_object__load_skeleton(obj->skeleton); +} + +static inline struct rss_bpf * +rss_bpf__open_and_load(void) +{ + struct rss_bpf *obj; + + obj = rss_bpf__open(); + if (!obj) + return NULL; + if (rss_bpf__load(obj)) { + rss_bpf__destroy(obj); + return NULL; + } + return obj; +} + +static inline int +rss_bpf__attach(struct rss_bpf *obj) +{ + return bpf_object__attach_skeleton(obj->skeleton); +} + +static inline void +rss_bpf__detach(struct rss_bpf *obj) +{ + return bpf_object__detach_skeleton(obj->skeleton); +} + +static inline int +rss_bpf__create_skeleton(struct rss_bpf *obj) +{ + struct bpf_object_skeleton *s; + + s = (struct bpf_object_skeleton *)calloc(1, sizeof(*s)); + if (!s) + return -1; + obj->skeleton = s; + + s->sz = sizeof(*s); + s->name = "rss_bpf"; + s->obj = &obj->obj; + + /* maps */ + s->map_cnt = 3; + s->map_skel_sz = sizeof(*s->maps); + s->maps = (struct bpf_map_skeleton *)calloc(s->map_cnt, s->map_skel_sz); + if (!s->maps) + goto err; + + s->maps[0].name = "tap_rss_map_configurations"; + s->maps[0].map = &obj->maps.tap_rss_map_configurations; + + s->maps[1].name = "tap_rss_map_indirection_table"; + s->maps[1].map = &obj->maps.tap_rss_map_indirection_table; + + s->maps[2].name = "tap_rss_map_toeplitz_key"; + s->maps[2].map = &obj->maps.tap_rss_map_toeplitz_key; + + /* programs */ + s->prog_cnt = 1; + s->prog_skel_sz = sizeof(*s->progs); + s->progs = (struct bpf_prog_skeleton *)calloc(s->prog_cnt, s->prog_skel_sz); + if (!s->progs) + goto err; + + s->progs[0].name = "tun_rss_steering_prog"; + s->progs[0].prog = &obj->progs.tun_rss_steering_prog; + s->progs[0].link = &obj->links.tun_rss_steering_prog; + + s->data_sz = 7088; + s->data = (void *)"\ +\x7f\x45\x4c\x46\x02\x01\x01\0\0\0\0\0\0\0\0\0\x01\0\xf7\0\x01\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\x30\x19\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0a\0\ +\x01\0\x7b\x1a\x38\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\x4c\xff\0\0\0\0\xbf\ +\xa6\0\0\0\0\0\0\x07\x06\0\0\x4c\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\xbf\x62\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x07\0\0\0\0\0\0\x18\x01\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\xbf\x62\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x08\0\0\0\0\0\ +\0\x18\0\0\0\xff\xff\xff\xff\0\0\0\0\0\0\0\0\x15\x07\x0f\x02\0\0\0\0\xbf\x89\0\ +\0\0\0\0\0\x15\x09\x0d\x02\0\0\0\0\x71\x71\0\0\0\0\0\0\x55\x01\x01\0\0\0\0\0\ +\x05\0\x09\x02\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xc0\xff\0\0\0\0\x7b\x1a\xb8\ +\xff\0\0\0\0\x7b\x1a\xb0\xff\0\0\0\0\x7b\x1a\xa8\xff\0\0\0\0\x7b\x1a\xa0\xff\0\ +\0\0\0\x63\x1a\x98\xff\0\0\0\0\x7b\x1a\x90\xff\0\0\0\0\x7b\x1a\x88\xff\0\0\0\0\ +\x7b\x1a\x80\xff\0\0\0\0\x7b\x1a\x78\xff\0\0\0\0\x7b\x1a\x70\xff\0\0\0\0\x7b\ +\x1a\x68\xff\0\0\0\0\x7b\x1a\x60\xff\0\0\0\0\x7b\x1a\x58\xff\0\0\0\0\x7b\x1a\ +\x50\xff\0\0\0\0\x79\xa6\x38\xff\0\0\0\0\x15\x06\x85\0\0\0\0\0\x6b\x1a\xd0\xff\ +\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\xbf\x61\0\0\0\0\0\0\ +\xb7\x02\0\0\x0c\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\0\0\0\0\x85\0\0\0\x44\ +\0\0\0\xb7\x02\0\0\x10\0\0\0\x69\xa1\xd0\xff\0\0\0\0\xbf\x13\0\0\0\0\0\0\xdc\ +\x03\0\0\x10\0\0\0\x15\x03\x02\0\0\x81\0\0\x55\x03\x08\0\xa8\x88\0\0\xb7\x02\0\ +\0\x14\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\xbf\x61\0\0\0\0\0\ +\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\0\0\0\0\x85\0\0\0\x44\0\0\0\x69\xa1\xd0\ +\xff\0\0\0\0\xdc\x01\0\0\x10\0\0\0\x15\x01\x26\0\xdd\x86\0\0\x55\x01\x6c\0\0\ +\x08\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x50\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\ +\x1a\xe0\xff\0\0\0\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\ +\0\0\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\xbf\x61\0\0\0\0\0\0\xb7\x02\0\0\0\0\0\0\ +\xb7\x04\0\0\x14\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x61\xa1\xdc\ +\xff\0\0\0\0\x63\x1a\x5c\xff\0\0\0\0\x61\xa1\xe0\xff\0\0\0\0\x63\x1a\x60\xff\0\ +\0\0\0\x71\xa6\xd9\xff\0\0\0\0\x71\xa1\xd0\xff\0\0\0\0\x67\x01\0\0\x02\0\0\0\ +\x57\x01\0\0\x3c\0\0\0\x7b\x1a\x40\xff\0\0\0\0\x57\x06\0\0\xff\0\0\0\x15\x06\ +\x45\0\x11\0\0\0\x79\xa1\x38\xff\0\0\0\0\x55\x06\x52\0\x06\0\0\0\xb7\x02\0\0\ +\x01\0\0\0\x73\x2a\x53\xff\0\0\0\0\xb7\x02\0\0\0\0\0\0\x63\x2a\xe0\xff\0\0\0\0\ +\x7b\x2a\xd8\xff\0\0\0\0\x7b\x2a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\ +\0\xd0\xff\xff\xff\x79\xa2\x40\xff\0\0\0\0\xb7\x04\0\0\x14\0\0\0\x05\0\x41\0\0\ +\0\0\0\x7b\x7a\x10\xff\0\0\0\0\xb7\x07\0\0\x01\0\0\0\x73\x7a\x51\xff\0\0\0\0\ +\xb7\x01\0\0\0\0\0\0\x7b\x1a\xf0\xff\0\0\0\0\x7b\x1a\xe8\xff\0\0\0\0\x7b\x1a\ +\xe0\xff\0\0\0\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\ +\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\xb7\x01\0\0\x28\0\0\0\x7b\x1a\x40\xff\0\0\0\ +\0\xbf\x61\0\0\0\0\0\0\xb7\x02\0\0\0\0\0\0\xb7\x04\0\0\x28\0\0\0\xb7\x05\0\0\ +\x01\0\0\0\x85\0\0\0\x44\0\0\0\x79\xa1\xd8\xff\0\0\0\0\x63\x1a\x5c\xff\0\0\0\0\ +\x77\x01\0\0\x20\0\0\0\x63\x1a\x60\xff\0\0\0\0\x79\xa1\xe0\xff\0\0\0\0\x63\x1a\ +\x64\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x68\xff\0\0\0\0\x79\xa1\xe8\xff\ +\0\0\0\0\x63\x1a\x6c\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x70\xff\0\0\0\0\ +\x79\xa1\xf0\xff\0\0\0\0\x63\x1a\x74\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\ +\x78\xff\0\0\0\0\x71\xa6\xd6\xff\0\0\0\0\x25\x06\xb1\0\x3c\0\0\0\x6f\x67\0\0\0\ +\0\0\0\x18\x01\0\0\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x5f\x17\0\0\0\0\0\0\x55\x07\ +\x01\0\0\0\0\0\x05\0\xab\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x6b\x1a\xfe\xff\0\0\0\0\ +\xb7\x01\0\0\x28\0\0\0\x7b\x1a\x40\xff\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\ +\x8c\xff\xff\xff\x7b\x1a\x20\xff\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\x54\ +\xff\xff\xff\x7b\x1a\x18\xff\0\0\0\0\xb7\x07\0\0\0\0\0\0\x7b\x8a\x30\xff\0\0\0\ +\0\x7b\x9a\x28\xff\0\0\0\0\x05\0\xdf\0\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\ +\x52\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\ +\0\x07\x03\0\0\xd0\xff\xff\xff\x79\xa1\x38\xff\0\0\0\0\x79\xa2\x40\xff\0\0\0\0\ +\xb7\x04\0\0\x08\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x69\xa1\xd0\ +\xff\0\0\0\0\x6b\x1a\x56\xff\0\0\0\0\x69\xa1\xd2\xff\0\0\0\0\x6b\x1a\x58\xff\0\ +\0\0\0\x71\xa1\x50\xff\0\0\0\0\x15\x01\x0f\0\0\0\0\0\x61\x71\x04\0\0\0\0\0\x71\ +\xa2\x53\xff\0\0\0\0\x15\x02\x41\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x02\ +\0\0\0\x15\x02\x3e\0\0\0\0\0\x61\xa1\x5c\xff\0\0\0\0\x63\x1a\xa0\xff\0\0\0\0\ +\x61\xa1\x60\xff\0\0\0\0\x63\x1a\xa4\xff\0\0\0\0\x69\xa1\x56\xff\0\0\0\0\x6b\ +\x1a\xa8\xff\0\0\0\0\x69\xa1\x58\xff\0\0\0\0\x6b\x1a\xaa\xff\0\0\0\0\x05\0\xf4\ +\0\0\0\0\0\x71\xa1\x51\xff\0\0\0\0\x15\x01\x5f\x01\0\0\0\0\x61\x71\x04\0\0\0\0\ +\0\x71\xa2\x53\xff\0\0\0\0\x15\x02\x3d\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\ +\0\x10\0\0\0\x15\x02\x3a\0\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\x5c\xff\xff\ +\xff\x71\xa4\x54\xff\0\0\0\0\xbf\x23\0\0\0\0\0\0\x15\x04\x02\0\0\0\0\0\xbf\xa3\ +\0\0\0\0\0\0\x07\x03\0\0\x7c\xff\xff\xff\x67\x01\0\0\x38\0\0\0\xc7\x01\0\0\x38\ +\0\0\0\x65\x01\x01\0\xff\xff\xff\xff\xbf\x32\0\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\ +\x07\x03\0\0\x6c\xff\xff\xff\x71\xa5\x55\xff\0\0\0\0\xbf\x34\0\0\0\0\0\0\x15\ +\x05\x02\0\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\x8c\xff\xff\xff\x65\x01\x01\ +\0\xff\xff\xff\xff\xbf\x43\0\0\0\0\0\0\x61\x21\x04\0\0\0\0\0\x67\x01\0\0\x20\0\ +\0\0\x61\x24\0\0\0\0\0\0\x4f\x41\0\0\0\0\0\0\x7b\x1a\xa0\xff\0\0\0\0\x61\x21\ +\x08\0\0\0\0\0\x61\x22\x0c\0\0\0\0\0\x67\x02\0\0\x20\0\0\0\x4f\x12\0\0\0\0\0\0\ +\x7b\x2a\xa8\xff\0\0\0\0\x61\x31\0\0\0\0\0\0\x61\x32\x04\0\0\0\0\0\x61\x34\x08\ +\0\0\0\0\0\x61\x33\x0c\0\0\0\0\0\x69\xa5\x58\xff\0\0\0\0\x6b\x5a\xc2\xff\0\0\0\ +\0\x69\xa5\x56\xff\0\0\0\0\x6b\x5a\xc0\xff\0\0\0\0\x67\x03\0\0\x20\0\0\0\x4f\ +\x43\0\0\0\0\0\0\x7b\x3a\xb8\xff\0\0\0\0\x67\x02\0\0\x20\0\0\0\x4f\x12\0\0\0\0\ +\0\0\x7b\x2a\xb0\xff\0\0\0\0\x05\0\xbf\0\0\0\0\0\x71\xa2\x52\xff\0\0\0\0\x15\ +\x02\x04\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x04\0\0\0\x15\x02\x01\0\0\0\ +\0\0\x05\0\xbc\xff\0\0\0\0\x57\x01\0\0\x01\0\0\0\x15\x01\x24\x01\0\0\0\0\x61\ +\xa1\x5c\xff\0\0\0\0\x63\x1a\xa0\xff\0\0\0\0\x61\xa1\x60\xff\0\0\0\0\x63\x1a\ +\xa4\xff\0\0\0\0\x05\0\xb2\0\0\0\0\0\x71\xa2\x52\xff\0\0\0\0\x15\x02\x16\0\0\0\ +\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x20\0\0\0\x15\x02\x13\0\0\0\0\0\xbf\xa2\0\ +\0\0\0\0\0\x07\x02\0\0\x5c\xff\xff\xff\x71\xa4\x54\xff\0\0\0\0\xbf\x23\0\0\0\0\ +\0\0\x15\x04\x02\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\x7c\xff\xff\xff\x57\ +\x01\0\0\0\x01\0\0\x15\x01\x01\0\0\0\0\0\xbf\x32\0\0\0\0\0\0\xbf\xa3\0\0\0\0\0\ +\0\x07\x03\0\0\x6c\xff\xff\xff\x71\xa5\x55\xff\0\0\0\0\xbf\x34\0\0\0\0\0\0\x15\ +\x05\x02\0\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\x8c\xff\xff\xff\x15\x01\xc3\ +\xff\0\0\0\0\x05\0\xc1\xff\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x08\0\0\0\ +\x15\x02\x04\x01\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\x5c\xff\xff\xff\x71\ +\xa4\x54\xff\0\0\0\0\xbf\x23\0\0\0\0\0\0\x15\x04\x02\0\0\0\0\0\xbf\xa3\0\0\0\0\ +\0\0\x07\x03\0\0\x7c\xff\xff\xff\x57\x01\0\0\x40\0\0\0\x15\x01\x01\0\0\0\0\0\ +\xbf\x32\0\0\0\0\0\0\x61\x23\x04\0\0\0\0\0\x67\x03\0\0\x20\0\0\0\x61\x24\0\0\0\ +\0\0\0\x4f\x43\0\0\0\0\0\0\x7b\x3a\xa0\xff\0\0\0\0\x61\x23\x08\0\0\0\0\0\x61\ +\x22\x0c\0\0\0\0\0\x67\x02\0\0\x20\0\0\0\x4f\x32\0\0\0\0\0\0\x7b\x2a\xa8\xff\0\ +\0\0\0\x15\x01\x78\0\0\0\0\0\x71\xa1\x55\xff\0\0\0\0\x15\x01\x76\0\0\0\0\0\x61\ +\xa1\x98\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\x94\xff\0\0\0\0\x4f\x21\0\0\ +\0\0\0\0\x7b\x1a\xb8\xff\0\0\0\0\x61\xa1\x90\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\ +\x61\xa2\x8c\xff\0\0\0\0\x05\0\x75\0\0\0\0\0\x15\x06\x54\xff\x87\0\0\0\x05\0\ +\x3e\0\0\0\0\0\x0f\x96\0\0\0\0\0\0\xbf\x62\0\0\0\0\0\0\x07\x02\0\0\x01\0\0\0\ +\x71\xa3\xff\xff\0\0\0\0\x67\x03\0\0\x03\0\0\0\x3d\x32\x22\0\0\0\0\0\x55\x01\ +\x0c\0\xc9\0\0\0\x79\xa1\x40\xff\0\0\0\0\x0f\x16\0\0\0\0\0\0\x07\x06\0\0\x02\0\ +\0\0\x79\xa1\x38\xff\0\0\0\0\xbf\x62\0\0\0\0\0\0\x79\xa3\x18\xff\0\0\0\0\xb7\ +\x04\0\0\x01\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xb7\x01\0\0\x01\0\ +\0\0\x73\x1a\x54\xff\0\0\0\0\x05\0\x15\0\0\0\0\0\x07\x08\0\0\xff\xff\xff\xff\ +\xbf\x81\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\xbf\x69\0\0\0\ +\0\0\0\x15\x01\x0f\0\0\0\0\0\xbf\x92\0\0\0\0\0\0\x79\xa1\x40\xff\0\0\0\0\x0f\ +\x12\0\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xf8\xff\xff\xff\xb7\x06\0\0\ +\x01\0\0\0\x79\xa1\x38\xff\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\x01\0\0\0\ +\x85\0\0\0\x44\0\0\0\x71\xa1\xf8\xff\0\0\0\0\x15\x01\xdb\xff\0\0\0\0\x71\xa6\ +\xf9\xff\0\0\0\0\x07\x06\0\0\x02\0\0\0\x05\0\xd8\xff\0\0\0\0\x79\xa8\x30\xff\0\ +\0\0\0\x79\xa9\x28\xff\0\0\0\0\x71\xa1\xff\xff\0\0\0\0\x67\x01\0\0\x03\0\0\0\ +\x79\xa2\x40\xff\0\0\0\0\x0f\x12\0\0\0\0\0\0\x07\x02\0\0\x08\0\0\0\x7b\x2a\x40\ +\xff\0\0\0\0\x71\xa6\xfe\xff\0\0\0\0\x25\x06\x37\0\x3c\0\0\0\xb7\x01\0\0\x01\0\ +\0\0\x6f\x61\0\0\0\0\0\0\x18\x02\0\0\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x5f\x21\0\0\ +\0\0\0\0\x55\x01\x01\0\0\0\0\0\x05\0\x30\0\0\0\0\0\x07\x07\0\0\x01\0\0\0\xbf\ +\x71\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x02\0\x0b\ +\0\0\0\x79\xa7\x10\xff\0\0\0\0\x05\0\xda\xfe\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\ +\x03\0\0\xfe\xff\xff\xff\x79\xa1\x38\xff\0\0\0\0\x79\xa2\x40\xff\0\0\0\0\xb7\ +\x04\0\0\x02\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x61\0\0\0\0\0\ +\0\x15\x01\x1b\0\x3c\0\0\0\x55\x01\xe0\xff\x2b\0\0\0\xb7\x01\0\0\0\0\0\0\x63\ +\x1a\xf8\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xf8\xff\xff\xff\x79\xa6\ +\x38\xff\0\0\0\0\xbf\x61\0\0\0\0\0\0\x79\xa2\x40\xff\0\0\0\0\xb7\x04\0\0\x04\0\ +\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x71\xa1\xfa\xff\0\0\0\0\x55\x01\ +\xd4\xff\x02\0\0\0\x71\xa1\xf9\xff\0\0\0\0\x55\x01\xd2\xff\x02\0\0\0\x71\xa1\ +\xfb\xff\0\0\0\0\x55\x01\xd0\xff\x01\0\0\0\x79\xa2\x40\xff\0\0\0\0\x07\x02\0\0\ +\x08\0\0\0\xbf\x61\0\0\0\0\0\0\x79\xa3\x20\xff\0\0\0\0\xb7\x04\0\0\x10\0\0\0\ +\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x55\ +\xff\0\0\0\0\x05\0\xc6\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x6b\x1a\xf8\xff\0\0\0\0\ +\xb7\x09\0\0\x02\0\0\0\xb7\x08\0\0\x1e\0\0\0\x05\0\xb0\xff\0\0\0\0\x15\x06\xcf\ +\xff\x87\0\0\0\x05\0\xd3\xff\0\0\0\0\x61\xa1\x78\xff\0\0\0\0\x67\x01\0\0\x20\0\ +\0\0\x61\xa2\x74\xff\0\0\0\0\x4f\x21\0\0\0\0\0\0\x7b\x1a\xb8\xff\0\0\0\0\x61\ +\xa1\x70\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\x6c\xff\0\0\0\0\x4f\x21\0\0\ +\0\0\0\0\x7b\x1a\xb0\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x07\x08\0\0\x04\0\0\0\x61\ +\x92\0\0\0\0\0\0\xb7\x05\0\0\0\0\0\0\x05\0\x4e\0\0\0\0\0\xaf\x53\0\0\0\0\0\0\ +\xbf\x85\0\0\0\0\0\0\x0f\x15\0\0\0\0\0\0\x71\x55\0\0\0\0\0\0\x67\x02\0\0\x01\0\ +\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x07\0\0\0\x4f\x02\0\0\0\0\0\0\xbf\x40\0\0\0\ +\0\0\0\x67\0\0\0\x39\0\0\0\xc7\0\0\0\x3f\0\0\0\x5f\x20\0\0\0\0\0\0\xaf\x03\0\0\ +\0\0\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x06\0\0\0\x57\0\0\0\x01\0\0\0\x67\x02\0\ +\0\x01\0\0\0\x4f\x02\0\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\0\x3a\0\0\0\xc7\0\ +\0\0\x3f\0\0\0\x5f\x20\0\0\0\0\0\0\xaf\x03\0\0\0\0\0\0\x67\x02\0\0\x01\0\0\0\ +\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x05\0\0\0\x57\0\0\0\x01\0\0\0\x4f\x02\0\0\0\0\0\ +\0\xbf\x40\0\0\0\0\0\0\x67\0\0\0\x3b\0\0\0\xc7\0\0\0\x3f\0\0\0\x5f\x20\0\0\0\0\ +\0\0\xaf\x03\0\0\0\0\0\0\x67\x02\0\0\x01\0\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\ +\x04\0\0\0\x57\0\0\0\x01\0\0\0\x4f\x02\0\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\ +\0\x3c\0\0\0\xc7\0\0\0\x3f\0\0\0\x5f\x20\0\0\0\0\0\0\xaf\x03\0\0\0\0\0\0\xbf\ +\x50\0\0\0\0\0\0\x77\0\0\0\x03\0\0\0\x57\0\0\0\x01\0\0\0\x67\x02\0\0\x01\0\0\0\ +\x4f\x02\0\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\0\x3d\0\0\0\xc7\0\0\0\x3f\0\0\ +\0\x5f\x20\0\0\0\0\0\0\xaf\x03\0\0\0\0\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x02\0\ +\0\0\x57\0\0\0\x01\0\0\0\x67\x02\0\0\x01\0\0\0\x4f\x02\0\0\0\0\0\0\xbf\x40\0\0\ +\0\0\0\0\x67\0\0\0\x3e\0\0\0\xc7\0\0\0\x3f\0\0\0\x5f\x20\0\0\0\0\0\0\xaf\x03\0\ +\0\0\0\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x01\0\0\0\x57\0\0\0\x01\0\0\0\x67\x02\ +\0\0\x01\0\0\0\x4f\x02\0\0\0\0\0\0\x57\x04\0\0\x01\0\0\0\x87\x04\0\0\0\0\0\0\ +\x5f\x24\0\0\0\0\0\0\xaf\x43\0\0\0\0\0\0\x57\x05\0\0\x01\0\0\0\x67\x02\0\0\x01\ +\0\0\0\x4f\x52\0\0\0\0\0\0\x07\x01\0\0\x01\0\0\0\xbf\x35\0\0\0\0\0\0\x15\x01\ +\x0b\0\x24\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xa0\xff\xff\xff\x0f\x13\0\0\0\ +\0\0\0\x71\x34\0\0\0\0\0\0\xbf\x40\0\0\0\0\0\0\x67\0\0\0\x38\0\0\0\xc7\0\0\0\ +\x38\0\0\0\xb7\x03\0\0\0\0\0\0\x65\0\xa9\xff\xff\xff\xff\xff\xbf\x23\0\0\0\0\0\ +\0\x05\0\xa7\xff\0\0\0\0\xbf\x31\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\ +\x20\0\0\0\x15\x01\x0b\0\0\0\0\0\x69\x72\x08\0\0\0\0\0\x3f\x21\0\0\0\0\0\0\x2f\ +\x21\0\0\0\0\0\0\x1f\x13\0\0\0\0\0\0\x63\x3a\x50\xff\0\0\0\0\xbf\xa2\0\0\0\0\0\ +\0\x07\x02\0\0\x50\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x85\0\0\0\ +\x01\0\0\0\x55\0\x02\0\0\0\0\0\x69\x70\x0a\0\0\0\0\0\x95\0\0\0\0\0\0\0\x69\0\0\ +\0\0\0\0\0\x05\0\xfd\xff\0\0\0\0\x02\0\0\0\x04\0\0\0\x0c\0\0\0\x01\0\0\0\0\0\0\ +\0\x02\0\0\0\x04\0\0\0\x28\0\0\0\x01\0\0\0\0\0\0\0\x02\0\0\0\x04\0\0\0\x02\0\0\ +\0\x80\0\0\0\0\0\0\0\x47\x50\x4c\x20\x76\x32\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x01\ +\x7a\x52\0\x08\x7c\x0b\x01\x0c\0\0\0\x18\0\0\0\x18\0\0\0\0\0\0\0\0\0\0\0\x20\ +\x11\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa0\0\ +\0\0\x04\0\xf1\xff\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd7\x01\0\0\0\0\x03\0\x38\ +\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xbf\x01\0\0\0\0\x03\0\xe0\x09\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\x68\x01\0\0\0\0\x03\0\x88\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x48\x01\ +\0\0\0\0\x03\0\xf0\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xcf\x01\0\0\0\0\x03\0\x10\ +\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb7\x01\0\0\0\0\x03\0\x88\x0a\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\x9f\x01\0\0\0\0\x03\0\xb8\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\x01\ +\0\0\0\0\x03\0\xf0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xf1\0\0\0\0\0\x03\0\x30\x0b\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x17\x02\0\0\0\0\x03\0\x40\x0b\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\xf7\x01\0\0\0\0\x03\0\x38\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xaf\x01\0\0\ +\0\0\x03\0\xb8\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x97\x01\0\0\0\0\x03\0\xe0\x0b\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\x60\x01\0\0\0\0\x03\0\xc0\x02\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\x10\x01\0\0\0\0\x03\0\xf8\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe9\0\0\0\0\0\ +\x03\0\x40\x05\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x70\x01\0\0\0\0\x03\0\xb8\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\x0f\x02\0\0\0\0\x03\0\x70\x05\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\xa7\x01\0\0\0\0\x03\0\xb0\x05\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x8f\x01\0\0\0\0\x03\ +\0\xa0\x07\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x28\x01\0\0\0\0\x03\0\xd0\x07\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\xe1\0\0\0\0\0\x03\0\xf8\x05\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x87\ +\x01\0\0\0\0\x03\0\x70\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x38\x01\0\0\0\0\x03\0\ +\x90\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\x01\0\0\0\0\x03\0\xc8\x06\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\xd9\0\0\0\0\0\x03\0\xd0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x07\ +\x02\0\0\0\0\x03\0\xd8\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xef\x01\0\0\0\0\x03\0\ +\x08\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x58\x01\0\0\0\0\x03\0\x68\x08\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\x20\x01\0\0\0\0\x03\0\x80\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd1\ +\0\0\0\0\0\x03\0\xb8\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe7\x01\0\0\0\0\x03\0\xc8\ +\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x7f\x01\0\0\0\0\x03\0\x18\x09\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\x30\x01\0\0\0\0\x03\0\x30\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc9\0\0\ +\0\0\0\x03\0\x48\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xf9\0\0\0\0\0\x03\0\xc0\x01\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\xff\x01\0\0\0\0\x03\0\x88\x0d\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\xdf\x01\0\0\0\0\x03\0\x98\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc7\x01\0\0\0\ +\0\x03\0\x30\x10\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x77\x01\0\0\0\0\x03\0\xc0\x0d\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\x50\x01\0\0\0\0\x03\0\x88\x10\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\x18\x01\0\0\0\0\x03\0\0\x11\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\x03\ +\0\x08\x11\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc1\0\0\0\0\0\x03\0\x10\x11\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\xba\0\0\0\0\0\x03\0\xf8\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\x03\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x6b\0\0\0\x11\0\x06\0\0\0\0\0\0\ +\0\0\0\x07\0\0\0\0\0\0\0\x25\0\0\0\x11\0\x05\0\0\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\ +\0\x82\0\0\0\x11\0\x05\0\x28\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\0\x01\0\0\0\x11\0\ +\x05\0\x14\0\0\0\0\0\0\0\x14\0\0\0\0\0\0\0\x40\0\0\0\x12\0\x03\0\0\0\0\0\0\0\0\ +\0\x20\x11\0\0\0\0\0\0\x28\0\0\0\0\0\0\0\x01\0\0\0\x31\0\0\0\x50\0\0\0\0\0\0\0\ +\x01\0\0\0\x33\0\0\0\xe0\x10\0\0\0\0\0\0\x01\0\0\0\x32\0\0\0\x1c\0\0\0\0\0\0\0\ +\x01\0\0\0\x2f\0\0\0\0\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x74\x6f\ +\x65\x70\x6c\x69\x74\x7a\x5f\x6b\x65\x79\0\x2e\x74\x65\x78\x74\0\x6d\x61\x70\ +\x73\0\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x63\x6f\x6e\x66\x69\x67\ +\x75\x72\x61\x74\x69\x6f\x6e\x73\0\x74\x75\x6e\x5f\x72\x73\x73\x5f\x73\x74\x65\ +\x65\x72\x69\x6e\x67\x5f\x70\x72\x6f\x67\0\x2e\x72\x65\x6c\x74\x75\x6e\x5f\x72\ +\x73\x73\x5f\x73\x74\x65\x65\x72\x69\x6e\x67\0\x5f\x6c\x69\x63\x65\x6e\x73\x65\ +\0\x2e\x72\x65\x6c\x2e\x65\x68\x5f\x66\x72\x61\x6d\x65\0\x74\x61\x70\x5f\x72\ +\x73\x73\x5f\x6d\x61\x70\x5f\x69\x6e\x64\x69\x72\x65\x63\x74\x69\x6f\x6e\x5f\ +\x74\x61\x62\x6c\x65\0\x72\x73\x73\x2e\x62\x70\x66\x2e\x63\0\x2e\x73\x74\x72\ +\x74\x61\x62\0\x2e\x73\x79\x6d\x74\x61\x62\0\x4c\x42\x42\x30\x5f\x39\0\x4c\x42\ +\x42\x30\x5f\x38\x39\0\x4c\x42\x42\x30\x5f\x37\x39\0\x4c\x42\x42\x30\x5f\x36\ +\x39\0\x4c\x42\x42\x30\x5f\x35\x39\0\x4c\x42\x42\x30\x5f\x34\x39\0\x4c\x42\x42\ +\x30\x5f\x33\x39\0\x4c\x42\x42\x30\x5f\x32\x39\0\x4c\x42\x42\x30\x5f\x38\0\x4c\ +\x42\x42\x30\x5f\x38\x38\0\x4c\x42\x42\x30\x5f\x35\x38\0\x4c\x42\x42\x30\x5f\ +\x33\x38\0\x4c\x42\x42\x30\x5f\x38\x37\0\x4c\x42\x42\x30\x5f\x36\x37\0\x4c\x42\ +\x42\x30\x5f\x34\x37\0\x4c\x42\x42\x30\x5f\x37\x36\0\x4c\x42\x42\x30\x5f\x35\ +\x36\0\x4c\x42\x42\x30\x5f\x32\x36\0\x4c\x42\x42\x30\x5f\x31\x36\0\x4c\x42\x42\ +\x30\x5f\x38\x35\0\x4c\x42\x42\x30\x5f\x36\x35\0\x4c\x42\x42\x30\x5f\x33\x35\0\ +\x4c\x42\x42\x30\x5f\x31\x35\0\x4c\x42\x42\x30\x5f\x34\0\x4c\x42\x42\x30\x5f\ +\x38\x34\0\x4c\x42\x42\x30\x5f\x37\x34\0\x4c\x42\x42\x30\x5f\x35\x34\0\x4c\x42\ +\x42\x30\x5f\x34\x34\0\x4c\x42\x42\x30\x5f\x33\x34\0\x4c\x42\x42\x30\x5f\x32\ +\x34\0\x4c\x42\x42\x30\x5f\x34\x33\0\x4c\x42\x42\x30\x5f\x33\x33\0\x4c\x42\x42\ +\x30\x5f\x32\x33\0\x4c\x42\x42\x30\x5f\x31\x33\0\x4c\x42\x42\x30\x5f\x38\x32\0\ +\x4c\x42\x42\x30\x5f\x32\x32\0\x4c\x42\x42\x30\x5f\x31\x32\0\x4c\x42\x42\x30\ +\x5f\x38\x31\0\x4c\x42\x42\x30\x5f\x37\x31\0\x4c\x42\x42\x30\x5f\x36\x31\0\x4c\ +\x42\x42\x30\x5f\x33\x31\0\x4c\x42\x42\x30\x5f\x38\x30\0\x4c\x42\x42\x30\x5f\ +\x36\x30\0\x4c\x42\x42\x30\x5f\x34\x30\0\x4c\x42\x42\x30\x5f\x33\x30\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xaa\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\x10\x17\0\0\0\0\0\0\x1f\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x1a\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\x5a\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\ +\0\0\0\x20\x11\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\x56\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd0\x16\0\0\0\0\0\0\x30\0\ +\0\0\0\0\0\0\x09\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x20\0\0\0\ +\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x60\x11\0\0\0\0\0\0\x3c\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x6c\0\0\0\x01\0\0\0\x03\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x9c\x11\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x78\0\0\0\x01\0\0\0\x02\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\xa8\x11\0\0\0\0\0\0\x30\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\x74\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\x17\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x09\0\0\0\x07\0\0\0\x08\0\0\0\0\0\0\0\x10\ +\0\0\0\0\0\0\0\xb2\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd8\x11\0\0\ +\0\0\0\0\xf8\x04\0\0\0\0\0\0\x01\0\0\0\x30\0\0\0\x08\0\0\0\0\0\0\0\x18\0\0\0\0\ +\0\0\0"; + + return 0; +err: + bpf_object__destroy_skeleton(s); + return -1; +} + +#endif /* __RSS_BPF_SKEL_H__ */ diff --git a/ebpf/trace-events b/ebpf/trace-events new file mode 100644 index 0000000000000..411b1e2be3912 --- /dev/null +++ b/ebpf/trace-events @@ -0,0 +1,4 @@ +# See docs/devel/tracing.txt for syntax documentation. + +# ebpf-rss.c +ebpf_error(const char *s1, const char *s2) "error in %s: %s" diff --git a/ebpf/trace.h b/ebpf/trace.h new file mode 100644 index 0000000000000..ad570e6691197 --- /dev/null +++ b/ebpf/trace.h @@ -0,0 +1,2 @@ +#include "trace/trace-ebpf.h" + diff --git a/hw/net/vhost_net.c b/hw/net/vhost_net.c index 24d555e764a82..23520349a2d7b 100644 --- a/hw/net/vhost_net.c +++ b/hw/net/vhost_net.c @@ -71,6 +71,8 @@ static const int user_feature_bits[] = { VIRTIO_NET_F_MTU, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_RING_PACKED, + VIRTIO_NET_F_RSS, + VIRTIO_NET_F_HASH_REPORT, /* This bit implies RARP isn't sent by QEMU out of band */ VIRTIO_NET_F_GUEST_ANNOUNCE, @@ -436,7 +438,9 @@ VHostNetState *get_vhost_net(NetClientState *nc) switch (nc->info->type) { case NET_CLIENT_DRIVER_TAP: - vhost_net = tap_get_vhost_net(nc); + if (!nc->vhost_net_disabled) { + vhost_net = tap_get_vhost_net(nc); + } break; #ifdef CONFIG_VHOST_NET_USER case NET_CLIENT_DRIVER_VHOST_USER: diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c index 09ceb02c9d932..2b5cbf1db5b97 100644 --- a/hw/net/virtio-net.c +++ b/hw/net/virtio-net.c @@ -515,11 +515,23 @@ static RxFilterInfo *virtio_net_query_rxfilter(NetClientState *nc) return info; } +static void virtio_net_allow_vhost(VirtIONet *n, bool allow) +{ + int i; + for (i = 0; i < n->max_queues; i++) { + NetClientState *nc = qemu_get_subqueue(n->nic, i)->peer; + nc->vhost_net_disabled = !allow; + } +} + static void virtio_net_reset(VirtIODevice *vdev) { VirtIONet *n = VIRTIO_NET(vdev); int i; + printf("rx received %lu\n", n->debug_rx_received); + n->debug_rx_received = 0; + /* Reset back to compatibility mode */ n->promisc = 1; n->allmulti = 0; @@ -552,6 +564,7 @@ static void virtio_net_reset(VirtIODevice *vdev) assert(!virtio_net_get_subqueue(nc)->async_tx.elem); } } + virtio_net_allow_vhost(n, true); } static void peer_test_vnet_hdr(VirtIONet *n) @@ -596,6 +609,7 @@ static void virtio_net_set_mrg_rx_bufs(VirtIONet *n, int mergeable_rx_bufs, n->guest_hdr_len = n->mergeable_rx_bufs ? sizeof(struct virtio_net_hdr_mrg_rxbuf) : sizeof(struct virtio_net_hdr); + n->rss_data.populate_hash = false; } for (i = 0; i < n->max_queues; i++) { @@ -725,9 +739,8 @@ static uint64_t virtio_net_get_features(VirtIODevice *vdev, uint64_t features, return features; } - virtio_clear_feature(&features, VIRTIO_NET_F_RSS); - virtio_clear_feature(&features, VIRTIO_NET_F_HASH_REPORT); features = vhost_net_get_features(get_vhost_net(nc->peer), features); + vdev->backend_features = features; if (n->mtu_bypass_backend && @@ -877,6 +890,8 @@ static void virtio_net_set_features(VirtIODevice *vdev, uint64_t features) VirtIONet *n = VIRTIO_NET(vdev); Error *err = NULL; int i; + NetClientState *nc = qemu_get_queue(n->nic); + bool disable_vhost = false; if (n->mtu_bypass_backend && !virtio_has_feature(vdev->backend_features, VIRTIO_NET_F_MTU)) { @@ -901,6 +916,22 @@ static void virtio_net_set_features(VirtIODevice *vdev, uint64_t features) virtio_has_feature(features, VIRTIO_NET_F_GUEST_TSO6); n->rss_data.redirect = virtio_has_feature(features, VIRTIO_NET_F_RSS); + /* + * disable vhost till next reset if there are acked features that + * we can't support with vhost + */ + disable_vhost = n->rss_data.redirect && get_vhost_net(nc->peer) && + !ebpf_rss_is_loaded(&n->ebpf_rss); + + if (n->rss_data.populate_hash && get_vhost_net(nc->peer)) { + disable_vhost = true; + } + + if (disable_vhost) { + warn_report("Not using vhost till next reset"); + virtio_net_allow_vhost(n, false); + } + if (n->has_vnet_hdr) { n->curr_guest_offloads = virtio_net_guest_offloads_by_features(features); @@ -908,7 +939,7 @@ static void virtio_net_set_features(VirtIODevice *vdev, uint64_t features) } for (i = 0; i < n->max_queues; i++) { - NetClientState *nc = qemu_get_subqueue(n->nic, i); + nc = qemu_get_subqueue(n->nic, i); if (!get_vhost_net(nc->peer)) { continue; @@ -1151,12 +1182,80 @@ static int virtio_net_handle_announce(VirtIONet *n, uint8_t cmd, } } +static void virtio_net_detach_epbf_rss(VirtIONet *n); + static void virtio_net_disable_rss(VirtIONet *n) { if (n->rss_data.enabled) { trace_virtio_net_rss_disable(); } n->rss_data.enabled = false; + + virtio_net_detach_epbf_rss(n); +} + +static bool virtio_net_attach_ebpf_to_backend(NICState *nic, int prog_fd) +{ + NetClientState *nc = qemu_get_peer(qemu_get_queue(nic), 0); + if (nc == NULL || nc->info->set_steering_ebpf == NULL) { + return false; + } + + return nc->info->set_steering_ebpf(nc, prog_fd); +} + +static void rss_data_to_rss_config(struct VirtioNetRssData *data, + struct EBPFRSSConfig *config) +{ + config->redirect = data->redirect; + config->populate_hash = data->populate_hash; + config->hash_types = data->hash_types; + config->indirections_len = data->indirections_len; + config->default_queue = data->default_queue; +} + +static bool virtio_net_attach_epbf_rss(VirtIONet *n) +{ + struct EBPFRSSConfig config = {}; + + if (!ebpf_rss_is_loaded(&n->ebpf_rss)) { + return false; + } + + rss_data_to_rss_config(&n->rss_data, &config); + + if (!ebpf_rss_set_all(&n->ebpf_rss, &config, + n->rss_data.indirections_table, n->rss_data.key)) { + return false; + } + + if (!virtio_net_attach_ebpf_to_backend(n->nic, n->ebpf_rss.program_fd)) { + return false; + } + + return true; +} + +static void virtio_net_detach_epbf_rss(VirtIONet *n) +{ + virtio_net_attach_ebpf_to_backend(n->nic, -1); +} + +static bool virtio_net_load_ebpf(VirtIONet *n) +{ + if (!n->allow_ebpf_load || !virtio_net_attach_ebpf_to_backend(n->nic, -1)) { + /* backend does't support steering ebpf */ + printf("ebpf is not loaded!\n"); + return false; + } + + return ebpf_rss_load(&n->ebpf_rss); +} + +static void virtio_net_unload_ebpf(VirtIONet *n) +{ + virtio_net_attach_ebpf_to_backend(n->nic, -1); + ebpf_rss_unload(&n->ebpf_rss); } static uint16_t virtio_net_handle_rss(VirtIONet *n, @@ -1271,6 +1370,25 @@ static uint16_t virtio_net_handle_rss(VirtIONet *n, goto error; } n->rss_data.enabled = true; + + if (!n->rss_data.populate_hash) { + if (!virtio_net_attach_epbf_rss(n)) { + /* EBPF must be loaded for vhost */ + if (get_vhost_net(qemu_get_queue(n->nic)->peer)) { + warn_report("Can't load eBPF RSS for vhost"); + goto error; + } + /* fallback to software RSS */ + warn_report("Can't load eBPF RSS - fallback to software RSS"); + n->rss_data.enabled_software_rss = true; + } + } else { + /* use software RSS for hash populating */ + /* and detach eBPF if was loaded before */ + virtio_net_detach_epbf_rss(n); + n->rss_data.enabled_software_rss = true; + } + trace_virtio_net_rss_enable(n->rss_data.hash_types, n->rss_data.indirections_len, temp.b); @@ -1656,7 +1774,7 @@ static ssize_t virtio_net_receive_rcu(NetClientState *nc, const uint8_t *buf, return -1; } - if (!no_rss && n->rss_data.enabled) { + if (!no_rss && n->rss_data.enabled && n->rss_data.enabled_software_rss) { int index = virtio_net_process_rss(nc, buf, size); if (index >= 0) { NetClientState *nc2 = qemu_get_subqueue(n->nic, index); @@ -2363,11 +2481,14 @@ static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t size) { VirtIONet *n = qemu_get_nic_opaque(nc); + ssize_t res; if ((n->rsc4_enabled || n->rsc6_enabled)) { - return virtio_net_rsc_receive(nc, buf, size); + res = virtio_net_rsc_receive(nc, buf, size); } else { - return virtio_net_do_receive(nc, buf, size); + res = virtio_net_do_receive(nc, buf, size); } + n->debug_rx_received += res; + return res; } static int32_t virtio_net_flush_tx(VirtIONetQueue *q); @@ -2759,13 +2880,6 @@ static int virtio_net_post_load_device(void *opaque, int version_id) } } - if (n->rss_data.enabled) { - trace_virtio_net_rss_enable(n->rss_data.hash_types, - n->rss_data.indirections_len, - sizeof(n->rss_data.key)); - } else { - trace_virtio_net_rss_disable(); - } return 0; } @@ -2782,6 +2896,26 @@ static int virtio_net_post_load_virtio(VirtIODevice *vdev) virtio_net_apply_guest_offloads(n); } + if (n->rss_data.enabled) { + n->rss_data.enabled_software_rss = n->rss_data.populate_hash; + if (!n->rss_data.populate_hash) { + if (!virtio_net_attach_epbf_rss(n)) { + if (get_vhost_net(qemu_get_queue(n->nic)->peer)) { + error_report("Can't post-load eBPF RSS for vhost"); + } else { + warn_report("Can't post-load eBPF RSS - fallback to software RSS"); + n->rss_data.enabled_software_rss = true; + } + } + } + + trace_virtio_net_rss_enable(n->rss_data.hash_types, + n->rss_data.indirections_len, + sizeof(n->rss_data.key)); + } else { + trace_virtio_net_rss_disable(); + } + return 0; } @@ -3336,6 +3470,9 @@ static void virtio_net_device_realize(DeviceState *dev, Error **errp) n->qdev = dev; net_rx_pkt_init(&n->rx_pkt, false); + if (virtio_has_feature(n->host_features, VIRTIO_NET_F_RSS)) { + virtio_net_load_ebpf(n); + } } static void virtio_net_device_unrealize(DeviceState *dev) @@ -3344,6 +3481,8 @@ static void virtio_net_device_unrealize(DeviceState *dev) VirtIONet *n = VIRTIO_NET(dev); int i, max_queues; + virtio_net_unload_ebpf(n); + /* This will stop vhost backend if appropriate. */ virtio_net_set_status(vdev, 0); @@ -3386,6 +3525,8 @@ static void virtio_net_instance_init(Object *obj) device_add_bootindex_property(obj, &n->nic_conf.bootindex, "bootindex", "/ethernet-phy@0", DEVICE(n)); + + ebpf_rss_init(&n->ebpf_rss); } static int virtio_net_pre_save(void *opaque) @@ -3497,6 +3638,7 @@ static Property virtio_net_properties[] = { DEFINE_PROP_INT32("speed", VirtIONet, net_conf.speed, SPEED_UNKNOWN), DEFINE_PROP_STRING("duplex", VirtIONet, net_conf.duplex_str), DEFINE_PROP_BOOL("failover", VirtIONet, failover, false), + DEFINE_PROP_BOOL("use-ebpf", VirtIONet, allow_ebpf_load, true), DEFINE_PROP_END_OF_LIST(), }; diff --git a/include/hw/virtio/virtio-net.h b/include/hw/virtio/virtio-net.h index 7e96d193aa606..963e5603cae09 100644 --- a/include/hw/virtio/virtio-net.h +++ b/include/hw/virtio/virtio-net.h @@ -21,6 +21,8 @@ #include "qemu/option_int.h" #include "qom/object.h" +#include "ebpf/ebpf_rss.h" + #define TYPE_VIRTIO_NET "virtio-net-device" OBJECT_DECLARE_SIMPLE_TYPE(VirtIONet, VIRTIO_NET) @@ -130,6 +132,7 @@ typedef struct VirtioNetRscChain { typedef struct VirtioNetRssData { bool enabled; + bool enabled_software_rss; bool redirect; bool populate_hash; uint32_t hash_types; @@ -179,6 +182,8 @@ struct VirtIONet { uint8_t nouni; uint8_t nobcast; uint8_t vhost_started; + size_t debug_rx_received; + bool allow_ebpf_load; struct { uint32_t in_use; uint32_t first_multi; @@ -209,6 +214,7 @@ struct VirtIONet { Notifier migration_state; VirtioNetRssData rss_data; struct NetRxPkt *rx_pkt; + struct EBPFRSSContext ebpf_rss; }; void virtio_net_set_netclient_name(VirtIONet *n, const char *name, diff --git a/include/net/net.h b/include/net/net.h index 919facaad2fab..4829803dc22b8 100644 --- a/include/net/net.h +++ b/include/net/net.h @@ -61,6 +61,7 @@ typedef int (SetVnetBE)(NetClientState *, bool); typedef struct SocketReadState SocketReadState; typedef void (SocketReadStateFinalize)(SocketReadState *rs); typedef void (NetAnnounce)(NetClientState *); +typedef bool (SetSteeringEBPF)(NetClientState *, int); typedef struct NetClientInfo { NetClientDriver type; @@ -82,6 +83,7 @@ typedef struct NetClientInfo { SetVnetLE *set_vnet_le; SetVnetBE *set_vnet_be; NetAnnounce *announce; + SetSteeringEBPF *set_steering_ebpf; } NetClientInfo; struct NetClientState { @@ -100,6 +102,7 @@ struct NetClientState { int vring_enable; int vnet_hdr_len; bool is_netdev; + bool vhost_net_disabled; QTAILQ_HEAD(, NetFilterState) filters; }; diff --git a/meson.build b/meson.build index 563688d682c77..27e41690e7a53 100644 --- a/meson.build +++ b/meson.build @@ -968,6 +968,13 @@ if not get_option('fuse_lseek').disabled() endif endif +# libbpf +libbpf = not_found +if 'CONFIG_EBPF' in config_host + libbpf = cc.find_library('bpf', required: true) + +endif + if get_option('cfi') cfi_flags=[] # Check for dependency on LTO @@ -1685,6 +1692,7 @@ if have_system 'backends', 'backends/tpm', 'chardev', + 'ebpf', 'hw/9pfs', 'hw/acpi', 'hw/alpha', @@ -1850,6 +1858,9 @@ subdir('accel') subdir('plugins') subdir('bsd-user') subdir('linux-user') +subdir('ebpf') + +common_ss.add(libbpf) bsd_user_ss.add(files('gdbstub.c')) specific_ss.add_all(when: 'CONFIG_BSD_USER', if_true: bsd_user_ss) @@ -2415,6 +2426,7 @@ summary_info += {'vhost-user support': config_host.has_key('CONFIG_VHOST_USER')} summary_info += {'vhost-user-blk server support': have_vhost_user_blk_server} summary_info += {'vhost-user-fs support': config_host.has_key('CONFIG_VHOST_USER_FS')} summary_info += {'vhost-vdpa support': config_host.has_key('CONFIG_VHOST_VDPA')} +summary_info += {'bpf support': config_host.has_key('CONFIG_EBPF')} summary_info += {'Trace backends': config_host['TRACE_BACKENDS']} if config_host['TRACE_BACKENDS'].split().contains('simple') summary_info += {'Trace output file': config_host['CONFIG_TRACE_FILE'] + '-'} @@ -2516,3 +2528,4 @@ if not supported_oses.contains(targetos) message('if you care about QEMU on this platform you should contact') message('us upstream at qemu-devel@nongnu.org.') endif + diff --git a/net/tap-bsd.c b/net/tap-bsd.c index 77aaf674b19dd..4f64f31e986e5 100644 --- a/net/tap-bsd.c +++ b/net/tap-bsd.c @@ -259,3 +259,8 @@ int tap_fd_get_ifname(int fd, char *ifname) { return -1; } + +int tap_fd_set_steering_ebpf(int fd, int prog_fd) +{ + return -1; +} diff --git a/net/tap-linux.c b/net/tap-linux.c index b0635e9e32cea..9584769740100 100644 --- a/net/tap-linux.c +++ b/net/tap-linux.c @@ -316,3 +316,16 @@ int tap_fd_get_ifname(int fd, char *ifname) pstrcpy(ifname, sizeof(ifr.ifr_name), ifr.ifr_name); return 0; } + +int tap_fd_set_steering_ebpf(int fd, int prog_fd) +{ + if (ioctl(fd, TUNSETSTEERINGEBPF, (void *) &prog_fd) != 0) { + error_report("Issue while setting TUNSETSTEERINGEBPF:" + " %s with fd: %d, prog_fd: %d", + strerror(errno), fd, prog_fd); + + return -1; + } + + return 0; +} diff --git a/net/tap-linux.h b/net/tap-linux.h index 2f36d100fc5d5..1d06fe0de63ba 100644 --- a/net/tap-linux.h +++ b/net/tap-linux.h @@ -31,6 +31,7 @@ #define TUNSETQUEUE _IOW('T', 217, int) #define TUNSETVNETLE _IOW('T', 220, int) #define TUNSETVNETBE _IOW('T', 222, int) +#define TUNSETSTEERINGEBPF _IOR('T', 224, int) #endif diff --git a/net/tap-solaris.c b/net/tap-solaris.c index 0475a58207b1b..d85224242b300 100644 --- a/net/tap-solaris.c +++ b/net/tap-solaris.c @@ -255,3 +255,8 @@ int tap_fd_get_ifname(int fd, char *ifname) { return -1; } + +int tap_fd_set_steering_ebpf(int fd, int prog_fd) +{ + return -1; +} diff --git a/net/tap-stub.c b/net/tap-stub.c index de525a2e69d4b..a0fa25804b6f8 100644 --- a/net/tap-stub.c +++ b/net/tap-stub.c @@ -85,3 +85,8 @@ int tap_fd_get_ifname(int fd, char *ifname) { return -1; } + +int tap_fd_set_steering_ebpf(int fd, int prog_fd) +{ + return -1; +} diff --git a/net/tap.c b/net/tap.c index b7512853f46c0..d0c1356a041ff 100644 --- a/net/tap.c +++ b/net/tap.c @@ -337,6 +337,14 @@ static void tap_poll(NetClientState *nc, bool enable) tap_write_poll(s, enable); } +static bool tap_set_steering_ebpf(NetClientState *nc, int prog_fd) +{ + TAPState *s = DO_UPCAST(TAPState, nc, nc); + assert(nc->info->type == NET_CLIENT_DRIVER_TAP); + + return tap_fd_set_steering_ebpf(s->fd, prog_fd) == 0; +} + int tap_get_fd(NetClientState *nc) { TAPState *s = DO_UPCAST(TAPState, nc, nc); @@ -362,6 +370,7 @@ static NetClientInfo net_tap_info = { .set_vnet_hdr_len = tap_set_vnet_hdr_len, .set_vnet_le = tap_set_vnet_le, .set_vnet_be = tap_set_vnet_be, + .set_steering_ebpf = tap_set_steering_ebpf, }; static TAPState *net_tap_fd_init(NetClientState *peer, diff --git a/net/tap_int.h b/net/tap_int.h index 225a49ea48438..547f8a5a28f0c 100644 --- a/net/tap_int.h +++ b/net/tap_int.h @@ -44,5 +44,6 @@ int tap_fd_set_vnet_be(int fd, int vnet_is_be); int tap_fd_enable(int fd); int tap_fd_disable(int fd); int tap_fd_get_ifname(int fd, char *ifname); +int tap_fd_set_steering_ebpf(int fd, int prog_fd); #endif /* NET_TAP_INT_H */ diff --git a/net/vhost-vdpa.c b/net/vhost-vdpa.c index fe659ec9e2422..8b142155494ad 100644 --- a/net/vhost-vdpa.c +++ b/net/vhost-vdpa.c @@ -54,6 +54,8 @@ const int vdpa_feature_bits[] = { VIRTIO_NET_F_MTU, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_RING_PACKED, + VIRTIO_NET_F_RSS, + VIRTIO_NET_F_HASH_REPORT, VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_STATUS, VHOST_INVALID_FEATURE_BIT diff --git a/tools/ebpf/Makefile.ebpf b/tools/ebpf/Makefile.ebpf new file mode 100755 index 0000000000000..d32d1680b849e --- /dev/null +++ b/tools/ebpf/Makefile.ebpf @@ -0,0 +1,33 @@ +OBJS = rss.bpf.o + +LLC ?= llc +CLANG ?= clang +INC_FLAGS = `$(CLANG) -print-file-name=include` +EXTRA_CFLAGS ?= -O2 -emit-llvm -fno-stack-protector + +ifdef linuxhdrs +LINUXINCLUDE = -I $(linuxhdrs)/arch/x86/include/uapi \ + -I $(linuxhdrs)/arch/x86/include/generated/uapi \ + -I $(linuxhdrs)/arch/x86/include/generated \ + -I $(linuxhdrs)/include/generated/uapi \ + -I $(linuxhdrs)/include/uapi \ + -I $(linuxhdrs)/include \ + -I $(linuxhdrs)/tools/lib +INC_FLAGS += -nostdinc -isystem +endif + +all: $(OBJS) + +.PHONY: clean + +clean: + rm -f $(OBJS) + +$(OBJS): %.o:%.c + $(CLANG) $(INC_FLAGS) \ + -D__KERNEL__ -D__ASM_SYSREG_H \ + -I../include $(LINUXINCLUDE) \ + $(EXTRA_CFLAGS) -c $< -o -| $(LLC) -march=bpf -filetype=obj -o $@ + bpftool gen skeleton rss.bpf.o > rss.bpf.skeleton.h + cp rss.bpf.skeleton.h ../../ebpf/ + diff --git a/tools/ebpf/rss.bpf.c b/tools/ebpf/rss.bpf.c new file mode 100644 index 0000000000000..eb377247fc3c2 --- /dev/null +++ b/tools/ebpf/rss.bpf.c @@ -0,0 +1,505 @@ +/* + * eBPF RSS program + * + * Developed by Daynix Computing LTD (http://www.daynix.com) + * + * Authors: + * Andrew Melnychenko + * Yuri Benditovich + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + * Prepare: + * Requires llvm, clang, bpftool, linux kernel tree + * + * Build rss.bpf.skeleton.h: + * make -f Makefile.ebpf clean all + */ + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include + +#define INDIRECTION_TABLE_SIZE 128 +#define HASH_CALCULATION_BUFFER_SIZE 36 + +struct rss_config_t { + __u8 redirect; + __u8 populate_hash; + __u32 hash_types; + __u16 indirections_len; + __u16 default_queue; +}; + +struct toeplitz_key_data_t { + __u32 leftmost_32_bits; + __u8 next_byte[HASH_CALCULATION_BUFFER_SIZE]; +}; + +struct packet_hash_info_t { + __u8 is_ipv4; + __u8 is_ipv6; + __u8 is_udp; + __u8 is_tcp; + __u8 is_ipv6_ext_src; + __u8 is_ipv6_ext_dst; + + __u16 src_port; + __u16 dst_port; + + union { + struct { + __be32 in_src; + __be32 in_dst; + }; + + struct { + struct in6_addr in6_src; + struct in6_addr in6_dst; + struct in6_addr in6_ext_src; + struct in6_addr in6_ext_dst; + }; + }; +}; + +struct bpf_map_def SEC("maps") +tap_rss_map_configurations = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(__u32), + .value_size = sizeof(struct rss_config_t), + .max_entries = 1, +}; + +struct bpf_map_def SEC("maps") +tap_rss_map_toeplitz_key = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(__u32), + .value_size = sizeof(struct toeplitz_key_data_t), + .max_entries = 1, +}; + +struct bpf_map_def SEC("maps") +tap_rss_map_indirection_table = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(__u32), + .value_size = sizeof(__u16), + .max_entries = INDIRECTION_TABLE_SIZE, +}; + +static inline void net_rx_rss_add_chunk(__u8 *rss_input, size_t *bytes_written, + const void *ptr, size_t size) { + __builtin_memcpy(&rss_input[*bytes_written], ptr, size); + *bytes_written += size; +} + +static inline +void net_toeplitz_add(__u32 *result, + __u8 *input, + __u32 len + , struct toeplitz_key_data_t *key) { + + __u32 accumulator = *result; + __u32 leftmost_32_bits = key->leftmost_32_bits; + __u32 byte; + + for (byte = 0; byte < HASH_CALCULATION_BUFFER_SIZE; byte++) { + __u8 input_byte = input[byte]; + __u8 key_byte = key->next_byte[byte]; + __u8 bit; + + for (bit = 0; bit < 8; bit++) { + if (input_byte & (1 << 7)) { + accumulator ^= leftmost_32_bits; + } + + leftmost_32_bits = + (leftmost_32_bits << 1) | ((key_byte & (1 << 7)) >> 7); + + input_byte <<= 1; + key_byte <<= 1; + } + } + + *result = accumulator; +} + + +static inline int ip6_extension_header_type(__u8 hdr_type) +{ + switch (hdr_type) { + case IPPROTO_HOPOPTS: + case IPPROTO_ROUTING: + case IPPROTO_FRAGMENT: + case IPPROTO_ICMPV6: + case IPPROTO_NONE: + case IPPROTO_DSTOPTS: + case IPPROTO_MH: + return 1; + default: + return 0; + } +} +/* + * According to https://www.iana.org/assignments/ipv6-parameters/ipv6-parameters.xhtml + * we suspect that there are would be no more than 11 extensions in IPv6 header, + * also there is 27 TLV options for Destination and Hop-by-hop extensions. + * Need to choose reasonable amount of maximum extensions/options we may check to find + * ext src/dst. + */ +#define IP6_EXTENSIONS_COUNT 11 +#define IP6_OPTIONS_COUNT 30 + +static inline void parse_ipv6_ext(struct __sk_buff *skb, + struct packet_hash_info_t *info, + __u8 *l4_protocol, size_t *l4_offset) +{ + if (!ip6_extension_header_type(*l4_protocol)) { + return; + } + + struct ipv6_opt_hdr ext_hdr = {}; + + for (unsigned int i = 0; i < IP6_EXTENSIONS_COUNT; ++i) { + + bpf_skb_load_bytes_relative(skb, *l4_offset, &ext_hdr, + sizeof(ext_hdr), BPF_HDR_START_NET); + + if (*l4_protocol == IPPROTO_ROUTING) { + struct ipv6_rt_hdr ext_rt = {}; + + bpf_skb_load_bytes_relative(skb, *l4_offset, &ext_rt, + sizeof(ext_rt), BPF_HDR_START_NET); + + if ((ext_rt.type == IPV6_SRCRT_TYPE_2) && + (ext_rt.hdrlen == sizeof(struct in6_addr) / 8) && + (ext_rt.segments_left == 1)) { + + bpf_skb_load_bytes_relative(skb, + *l4_offset + offsetof(struct rt2_hdr, addr), + &info->in6_ext_dst, sizeof(info->in6_ext_dst), + BPF_HDR_START_NET); + + info->is_ipv6_ext_dst = 1; + } + + } else if (*l4_protocol == IPPROTO_DSTOPTS) { + struct ipv6_opt_t { + __u8 type; + __u8 length; + } __attribute__((packed)) opt = {}; + + size_t opt_offset = sizeof(ext_hdr); + + for (unsigned int j = 0; j < IP6_OPTIONS_COUNT; ++j) { + bpf_skb_load_bytes_relative(skb, *l4_offset + opt_offset, + &opt, sizeof(opt), BPF_HDR_START_NET); + + opt_offset += (opt.type == IPV6_TLV_PAD1) ? + 1 : opt.length + sizeof(opt); + + if (opt_offset + 1 >= ext_hdr.hdrlen * 8) { + break; + } + + if (opt.type == IPV6_TLV_HAO) { + bpf_skb_load_bytes_relative(skb, + *l4_offset + opt_offset + offsetof(struct ipv6_destopt_hao, addr), + &info->is_ipv6_ext_src, sizeof(info->is_ipv6_ext_src), + BPF_HDR_START_NET); + + info->is_ipv6_ext_src = 1; + break; + } + } + } + + *l4_protocol = ext_hdr.nexthdr; + *l4_offset += (ext_hdr.hdrlen + 1) * 8; + + if (!ip6_extension_header_type(ext_hdr.nexthdr)) { + return; + } + } +} + +static __be16 parse_eth_type(struct __sk_buff *skb) +{ + unsigned int offset = 12; + __be16 ret = 0; + + bpf_skb_load_bytes_relative(skb, offset, &ret, sizeof(ret), + BPF_HDR_START_MAC); + + switch (__be16_to_cpu(ret)) { + case ETH_P_8021AD: + offset += 4; + case ETH_P_8021Q: + offset += 4; + bpf_skb_load_bytes_relative(skb, offset, &ret, sizeof(ret), + BPF_HDR_START_MAC); + default: + break; + } + + return ret; +} + +static inline void parse_packet(struct __sk_buff *skb, + struct packet_hash_info_t *info) +{ + if (!info || !skb) { + return; + } + + size_t l4_offset = 0; + __u8 l4_protocol = 0; + __u16 l3_protocol = __be16_to_cpu(parse_eth_type(skb)); + + if (l3_protocol == ETH_P_IP) { + info->is_ipv4 = 1; + + struct iphdr ip = {}; + bpf_skb_load_bytes_relative(skb, 0, &ip, sizeof(ip), + BPF_HDR_START_NET); + + info->in_src = ip.saddr; + info->in_dst = ip.daddr; + + l4_protocol = ip.protocol; + l4_offset = ip.ihl * 4; + } else if (l3_protocol == ETH_P_IPV6) { + info->is_ipv6 = 1; + + struct ipv6hdr ip6 = {}; + bpf_skb_load_bytes_relative(skb, 0, &ip6, sizeof(ip6), + BPF_HDR_START_NET); + + info->in6_src = ip6.saddr; + info->in6_dst = ip6.daddr; + + l4_protocol = ip6.nexthdr; + l4_offset = sizeof(ip6); + + parse_ipv6_ext(skb, info, &l4_protocol, &l4_offset); + } + + if (l4_protocol != 0) { + if (l4_protocol == IPPROTO_TCP) { + info->is_tcp = 1; + + struct tcphdr tcp = {}; + bpf_skb_load_bytes_relative(skb, l4_offset, &tcp, sizeof(tcp), + BPF_HDR_START_NET); + + info->src_port = tcp.source; + info->dst_port = tcp.dest; + } else if (l4_protocol == IPPROTO_UDP) { /* TODO: add udplite? */ + info->is_udp = 1; + + struct udphdr udp = {}; + bpf_skb_load_bytes_relative(skb, l4_offset, &udp, sizeof(udp), + BPF_HDR_START_NET); + + info->src_port = udp.source; + info->dst_port = udp.dest; + } + } +} + +static inline __u32 calculate_rss_hash(struct __sk_buff *skb, + struct rss_config_t *config, struct toeplitz_key_data_t *toe) +{ + __u8 rss_input[HASH_CALCULATION_BUFFER_SIZE] = {}; + size_t bytes_written = 0; + __u32 result = 0; + struct packet_hash_info_t packet_info = {}; + + parse_packet(skb, &packet_info); + + if (packet_info.is_ipv4) { + if (packet_info.is_tcp && + config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_TCPv4) { + + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in_src, + sizeof(packet_info.in_src)); + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in_dst, + sizeof(packet_info.in_dst)); + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.src_port, + sizeof(packet_info.src_port)); + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.dst_port, + sizeof(packet_info.dst_port)); + } else if (packet_info.is_udp && + config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_UDPv4) { + + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in_src, + sizeof(packet_info.in_src)); + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in_dst, + sizeof(packet_info.in_dst)); + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.src_port, + sizeof(packet_info.src_port)); + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.dst_port, + sizeof(packet_info.dst_port)); + } else if (config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_IPv4) { + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in_src, + sizeof(packet_info.in_src)); + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in_dst, + sizeof(packet_info.in_dst)); + } + } else if (packet_info.is_ipv6) { + if (packet_info.is_tcp && + config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_TCPv6) { + + if (packet_info.is_ipv6_ext_src && + config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_TCP_EX) { + + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_ext_src, + sizeof(packet_info.in6_ext_src)); + } else { + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_src, + sizeof(packet_info.in6_src)); + } + if (packet_info.is_ipv6_ext_dst && + config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_TCP_EX) { + + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_ext_dst, + sizeof(packet_info.in6_ext_dst)); + } else { + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_dst, + sizeof(packet_info.in6_dst)); + } + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.src_port, + sizeof(packet_info.src_port)); + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.dst_port, + sizeof(packet_info.dst_port)); + } else if (packet_info.is_udp && + config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_UDPv6) { + + if (packet_info.is_ipv6_ext_src && + config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_UDP_EX) { + + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_ext_src, + sizeof(packet_info.in6_ext_src)); + } else { + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_src, + sizeof(packet_info.in6_src)); + } + if (packet_info.is_ipv6_ext_dst && + config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_UDP_EX) { + + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_ext_dst, + sizeof(packet_info.in6_ext_dst)); + } else { + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_dst, + sizeof(packet_info.in6_dst)); + } + + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.src_port, + sizeof(packet_info.src_port)); + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.dst_port, + sizeof(packet_info.dst_port)); + + } else if (config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_IPv6) { + if (packet_info.is_ipv6_ext_src && + config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_IP_EX) { + + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_ext_src, + sizeof(packet_info.in6_ext_src)); + } else { + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_src, + sizeof(packet_info.in6_src)); + } + if (packet_info.is_ipv6_ext_dst && + config->hash_types & VIRTIO_NET_RSS_HASH_TYPE_IP_EX) { + + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_ext_dst, + sizeof(packet_info.in6_ext_dst)); + } else { + net_rx_rss_add_chunk(rss_input, &bytes_written, + &packet_info.in6_dst, + sizeof(packet_info.in6_dst)); + } + } + } + + if (bytes_written) { + net_toeplitz_add(&result, rss_input, bytes_written, toe); + } + + return result; +} + +SEC("tun_rss_steering") +int tun_rss_steering_prog(struct __sk_buff *skb) +{ + + struct rss_config_t *config; + struct toeplitz_key_data_t *toe; + + __u32 key = 0; + __u32 hash = 0; + + config = bpf_map_lookup_elem(&tap_rss_map_configurations, &key); + toe = bpf_map_lookup_elem(&tap_rss_map_toeplitz_key, &key); + + if (config && toe) { + if (!config->redirect) { + return config->default_queue; + } + + hash = calculate_rss_hash(skb, config, toe); + if (hash) { + __u32 table_idx = hash % config->indirections_len; + __u16 *queue = 0; + + queue = bpf_map_lookup_elem(&tap_rss_map_indirection_table, + &table_idx); + + if (queue) { + return *queue; + } + } + + return config->default_queue; + } + + return -1; +} + +char _license[] SEC("license") = "GPL v2";