diff --git a/Makefile b/Makefile index 4eafc39..5a63c0f 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,14 @@ CC = gcc CFLAGS = -Wall -D_GNU_SOURCE -Wunused + +ifeq ($(filter-out %64 %64be %64eb %64le %64el s390x, $(shell uname -m)),) + LIBDIR ?= /usr/lib64 +else + LIBDIR ?= /usr/lib +endif + +LDFLAGS = -L$(LIBDIR) + LDLIBS = -pthread -lbpf -lm diff --git a/README.md b/README.md index 87ec261..4eaf73e 100644 --- a/README.md +++ b/README.md @@ -4,17 +4,18 @@ query operations. SimpleKV supports low-latency IO via the XRP BPF interface and can also run using user-space IO (`pread()`) on mainline (and XRP compatible) kernels. -For usage instructions run `./simplekv --help` after compiling. +For usage instructions run `LD_PRELOAD="/usr/lib64/libbpf.so.0" ./simplekv --help` after compiling. # Building -SimpleKV is built using the provided Makefile. Compiling the simplekv binary +SimpleKV is built using the provided Makefile. Compiling the `simplekv` binary is as simple as running `make`. To perform IO with XRP via the `--use-xrp` flag the corresponding BPF programs in the `xrp-bpf` directory must be compiled. These BPF programs require [libbpf](https://github.com/libbpf/libbpf) and an XRP compatible kernel. Before compiling, install libbpf via your distribution's package manager or source. +Compiling these programs also requires installing `clang` and `llvm`. To compile on an XRP compatible kernel with libbpf, run: ``` @@ -26,6 +27,9 @@ Alternatively, you can compile simplekv without the BPF programs and use userspa make simplekv ``` +For the below commands, we assume your libbpf installation is located in the +default directory specified by libbpf: `/usr/lib64` (for 64-bit architectures). + # Running ## Create a Database File @@ -45,39 +49,28 @@ space. To create a 6-layer database file: ``` -./simplekv 6-layer-db 6 create +LD_PRELOAD="/usr/lib64/libbpf.so.0" ./simplekv 6-layer-db 6 create ``` ## Running the benchmark SimpleKV supports get queries and range queries, both of which can be run with various options. Usage and option docs can be reviewed by passing the `--help` flag to either command: ``` -./simplekv 6-layer-db 6 get --help -./simplekv 6-layer-db 6 range --help +LD_PRELOAD="/usr/lib64/libbpf.so.0" ./simplekv 6-layer-db 6 get --help +LD_PRELOAD="/usr/lib64/libbpf.so.0" ./simplekv 6-layer-db 6 range --help ``` ### Using XRP -Before running with XRP you must load the corresponding BPF function using the -`xrp_loader` program built via the default make target. Note that GET and RANGE operations -use separate BPF programs, so be sure to load the correct one. - -For GET operations: -``` -LD_PRELOAD="/usr/lib64/libbpf.so.0" ./xrp_loader xrp-bpf/get_op.o -``` - -For RANGE operations: -``` -LD_PRELOAD="/usr/lib64/libbpf.so.0" ./xrp_loader xrp-bpf/range_op.o -``` NOTE: You may need to change `LD_PRELOAD` depending on where `libbpf.so.0` is located on your machine. With the BPF program loaded, you can run a basic GET benchmark as follows: ``` -./simplekv 6-layer-db 6 get --requests=100000 --use-xrp +sudo LD_PRELOAD="/usr/lib64/libbpf.so.0" ./simplekv 6-layer-db 6 get --requests=100000 --use-xrp ``` +Root access is required in order to load the bpf programs. + ### CPU Configuration For consistent benchmark results you may need to disable CPU frequency scaling. diff --git a/xrp-bpf/get.c b/xrp-bpf/get.c index 7455f86..66b498d 100644 --- a/xrp-bpf/get.c +++ b/xrp-bpf/get.c @@ -54,6 +54,7 @@ static __inline void set_context_next_index(struct bpf_xrp *context, struct Scat context->next_addr[0] = query->root_pointer; context->size[0] = BLK_SIZE; } + context->fd_arr[0] = context->cur_fd; } /* Mask to prevent out of bounds memory access */ @@ -65,6 +66,8 @@ unsigned int oliver_agg_func(struct bpf_xrp *context) { Node *node = (Node *) context->data; int *curr_idx = &query->current_index; + dbg_print("fd: %d\n", context->cur_fd ); + /* Three cases: * * 1. We've found the log offset in the previous iteration and are @@ -112,6 +115,7 @@ unsigned int oliver_agg_func(struct bpf_xrp *context) { ptr__t base = query->value_ptr & ~(BLK_SIZE - 1); context->next_addr[0] = base; context->size[0] = BLK_SIZE; + context->fd_arr[0] = context->cur_fd; return 0; } @@ -119,5 +123,6 @@ unsigned int oliver_agg_func(struct bpf_xrp *context) { dbg_print("simplekv-bpf: case 3 - internal node\n"); context->next_addr[0] = decode(nxt_node(query->keys[*curr_idx & EBPF_CONTEXT_MASK], node)); context->size[0] = BLK_SIZE; + context->fd_arr[0] = context->cur_fd; return 0; } diff --git a/xrp-bpf/range.c b/xrp-bpf/range.c index 8e05190..fb99ff0 100644 --- a/xrp-bpf/range.c +++ b/xrp-bpf/range.c @@ -49,6 +49,7 @@ static __inline unsigned int process_leaf(struct bpf_xrp *context, struct RangeQ if (curr_key >= first_key) { /* Set up the next resubmit to read the value */ context->next_addr[0] = value_base(decode(node->ptr[*i & KEY_MASK])); + context->fd_arr[0] = context->cur_fd; context->size[0] = BLK_SIZE; key__t key = node->key[*i & KEY_MASK]; @@ -102,6 +103,7 @@ static __inline unsigned int process_leaf(struct bpf_xrp *context, struct RangeQ query->_state = RNG_READ_NODE; query->_node_key_ix = 0; context->next_addr[0] = node->next; + context->fd_arr[0] = context->cur_fd; context->size[0] = BLK_SIZE; return 0; } @@ -138,6 +140,7 @@ static __inline unsigned int traverse_index(struct bpf_xrp *context, struct Rang /* Grab the next node in the traversal */ context->next_addr[0] = decode(nxt_node(query->range_begin, node)); + context->fd_arr[0] = context->cur_fd; context->size[0] = BLK_SIZE; return 0; } diff --git a/xrp-bpf/simplekvspec.h b/xrp-bpf/simplekvspec.h index c5cb662..b9cb9e8 100644 --- a/xrp-bpf/simplekvspec.h +++ b/xrp-bpf/simplekvspec.h @@ -37,7 +37,6 @@ static __inline void print_query(struct Query *q) { static __inline void print_node(Node *node) { dbg_print("struct Node {\n"); - dbg_print("\tnum = %ld\n", node->num); dbg_print("\ttype = %ld\n", node->type); dbg_print("\tkey[0] = %ld\n", node->key[0]); dbg_print("\tkey[30] = %ld\n", node->key[NODE_CAPACITY - 1]); @@ -50,6 +49,4 @@ static __inline void print_node(Node *node) { #define dbg_print(...) #endif - - #endif