Implement vblock split write amp reduction algo

During a vblock split, we can save write amp by using
mpool_mblock_punch() to FALLOC_FL_PUNCH_HOLE a portion of the mblock for
both the left and right hand side destination mblocks.

Signed-off-by: Tristan Partin <[email protected]>
Co-authored-by: Nabeel Meeramohideen Mohamed <[email protected]>

Author: tristan957

lib/cn/kvset_split.c

@@ -3,22 +3,20 @@
  * Copyright (C) 2022 Micron Technology, Inc.  All rights reserved.
  */
 
+#include <stdint.h>
 #include <sys/mman.h>
 
-#include <hse/util/event_counter.h>
-#include <hse/util/assert.h>
-#include <hse/util/keycmp.h>
-#include <hse/logging/logging.h>
-#include <hse/util/perfc.h>
-
-#include <hse/limits.h>
-
 #include <hse/ikvdb/cn.h>
 #include <hse/ikvdb/ikvdb.h>
 #include <hse/ikvdb/cndb.h>
 #include <hse/ikvdb/kvset_builder.h>
-
+#include <hse/limits.h>
+#include <hse/logging/logging.h>
 #include <hse/mpool/mpool.h>
+#include <hse/util/event_counter.h>
+#include <hse/util/assert.h>
+#include <hse/util/keycmp.h>
+#include <hse/util/perfc.h>
 
 #include "kvs_mblk_desc.h"
 #include "kvset.h"
@@ -399,6 +397,13 @@ kblocks_split(
     return err;
 }
 
+struct vgroup_split_metadata {
+    bool overlaps;
+    bool accessed;
+    uint16_t vblk_idx;
+    uint offset;
+};
+
 /**
  * Return a split vblock index for the specified range of vblocks [start, end] by comparing
  * the min/max keys stored in a vblock footer against the split key.
@@ -451,6 +456,114 @@ get_vblk_split_index(
     return v;
 }
 
+static void
+find_max_key_among_overlapping_vblocks(
+    const uint32_t nvgroups,
+    const struct vgroup_split_metadata *metadatav,
+    struct kvset *const ks,
+    struct key_obj *max_key)
+{
+    for (uint32_t i = 0; i < nvgroups; i++) {
+        struct key_obj curr_key = { 0 };
+        const struct vblock_desc *vbd;
+        const struct vgroup_split_metadata *metadata = metadatav + i;
+
+        if (!metadata->overlaps)
+            continue;
+
+        vbd = kvset_get_nth_vblock_desc(ks, metadata->vblk_idx);
+
+        curr_key.ko_sfx = vbd->vbd_mblkdesc->map_base + vbd->vbd_max_koff;
+        curr_key.ko_sfx_len = vbd->vbd_max_klen;
+
+        if (key_obj_cmp(max_key, &curr_key) < 0)
+            *max_key = curr_key;
+    }
+}
+
+static merr_t
+mark_vgroup_accesses(
+    const uint32_t nvgroups,
+    struct vgroup_split_metadata *metadatav,
+    struct kvset *const ks,
+    const struct key_obj *split_key,
+    const struct key_obj *max_key)
+{
+    merr_t err;
+    bool first = true;
+    struct kv_iterator *iter;
+
+    INVARIANT(key_obj_cmp(split_key, max_key) <= 0);
+
+    err = kvset_iter_create(ks, NULL, NULL, NULL, kvset_iter_flag_mmap, &iter);
+    if (ev(err))
+        return err;
+
+    err = kvset_iter_seek(iter, split_key->ko_sfx, split_key->ko_sfx_len, NULL);
+    if (ev(err))
+        goto out;
+
+    while (true) {
+        uint vlen;
+        uint vbidx;
+        uint vboff;
+        uint complen;
+        uint64_t seqno;
+        const void *vdata;
+        enum kmd_vtype vtype;
+        struct key_obj curr_key;
+        struct kvset_iter_vctx vc;
+
+        if (iter->kvi_eof)
+            break;
+
+        err = kvset_iter_next_key(iter, &curr_key, &vc);
+        if (ev(err))
+            goto out;
+
+        /* In the event that this kvset contains the split key, skip it. */
+        if (first) {
+            first = false;
+            if (key_obj_cmp(&curr_key, split_key) == 0)
+                continue;
+        }
+
+        if (key_obj_cmp(&curr_key, max_key) > 0)
+            break;
+
+        while (kvset_iter_next_vref(iter, &vc, &seqno, &vtype, &vbidx, &vboff, &vdata, &vlen,
+                &complen)) {
+            uint64_t vgidx;
+            const struct vblock_desc *vbd;
+            struct vgroup_split_metadata *metadata;
+
+            switch (vtype) {
+            case VTYPE_UCVAL:
+            case VTYPE_CVAL:
+                vbd = kvset_get_nth_vblock_desc(ks, vbidx);
+                vgidx = atomic_read(&vbd->vbd_vgidx);
+                assert(vgidx <= nvgroups);
+                metadata = metadatav + vgidx - 1;
+                if (!metadata->accessed) {
+                    metadata->accessed = true;
+                    metadata->offset = vboff;
+                }
+                /* fallthrough */
+            case VTYPE_IVAL:
+            case VTYPE_ZVAL:
+            case VTYPE_TOMB:
+            case VTYPE_PTOMB:
+                continue;
+            }
+        }
+    }
+
+out:
+    kvset_iter_release(iter);
+
+    return err;
+}
+
 /**
  * @vbidx_left, @vbidx_right - tracks vblock index for the left and right kvsets
  * @vgidx_left, @vgidx_right - tracks vgroup index for the left and right kvsets
@@ -464,6 +577,8 @@ vblocks_split(
     struct perfc_set        *pc,
     struct kvset_split_res  *result)
 {
+    struct key_obj max_key;
+    struct vgroup_split_metadata *metadatav;
     struct vgmap *vgmap_src = ks->ks_vgmap;
     struct vgmap *vgmap_left = work[LEFT].vgmap;
     struct vgmap *vgmap_right = work[RIGHT].vgmap;
@@ -472,8 +587,8 @@ vblocks_split(
     uint16_t vbidx_left = 0, vbidx_right = 0;
     uint32_t vgidx_left = 0, vgidx_right = 0;
     uint32_t nvgroups = kvset_get_vgroups(ks), perfc_rwc = 0;
-    bool move_left = (blks_right->kblks.idc == 0);
-    bool move_right = (blks_left->kblks.idc == 0);
+    const bool move_left = (blks_right->kblks.idc == 0);
+    const bool move_right = (blks_left->kblks.idc == 0);
     uint64_t perfc_rwb = 0;
     merr_t err;
 
@@ -482,87 +597,143 @@ vblocks_split(
         return 0;
     }
 
+    metadatav = calloc(nvgroups, sizeof(*metadatav));
+    if (ev(!metadatav))
+        return merr(ENOMEM);
+
     for (uint32_t i = 0; i < nvgroups; i++) {
-        uint16_t src_start, src_end, src_split, end;
-        uint32_t vbcnt = 0;
-        bool overlap = false;
+        uint16_t start, end;
+        struct vgroup_split_metadata *metadata = metadatav + i;
 
         /* Per vgroup start and end output vblock index in the source kvset
          */
-        src_start = vgmap_vbidx_out_start(ks, i);
-        src_end = vgmap_vbidx_out_end(ks, i);
+        start = vgmap_vbidx_out_start(ks, i);
+        end = vgmap_vbidx_out_end(ks, i);
 
         if (move_left || move_right) {
-            /* If all the kblocks are on one side then all the vblocks can be safely moved
-             * to the same side
-             */
-            src_split = move_right ? src_start : src_end + 1;
-            assert(!overlap);
+            metadata->vblk_idx = move_right ? start : end + 1;
+            metadata->overlaps = false;
         } else {
-            src_split = get_vblk_split_index(ks, src_start, src_end, split_key, &overlap);
+            metadata->vblk_idx = get_vblk_split_index(ks, start, end, split_key,
+                &metadata->overlaps);
         }
-        assert(src_split >= src_start && src_split <= src_end + 1);
+        assert(metadata->vblk_idx >= start && metadata->vblk_idx <= end + 1);
+    }
+
+    max_key = *split_key;
+    find_max_key_among_overlapping_vblocks(nvgroups, metadatav, ks, &max_key);
 
-        /* Add vblocks in [src_start, end - 1] to the left kvset
+    err = mark_vgroup_accesses(nvgroups, metadatav, ks, split_key, &max_key);
+    if (ev(err))
+        goto out;
+
+    for (uint32_t i = 0; i < nvgroups; i++) {
+        uint32_t vbcnt = 0;
+        uint16_t split, start, end, last;
+        const struct vgroup_split_metadata *metadata = metadatav + i;
+
+        /* Per vgroup start and end output vblock index in the source kvset
          */
-        end = overlap ? src_split + 1 : src_split;
-        for (uint16_t j = src_start; j < end; j++) {
-            err = blk_list_append(&blks_left->vblks, kvset_get_nth_vblock_id(ks, j));
-            if (err)
-                return err;
+        start = vgmap_vbidx_out_start(ks, i);
+        end = vgmap_vbidx_out_end(ks, i);
+        split = metadata->vblk_idx;
+
+        /* Add vblocks in [start, last - 1] to the left kvset
+         */
+        last = metadata->overlaps ? split + 1 : split;
+        for (uint16_t j = start; j < last; j++) {
+            uint64_t mbid;
+
+            mbid = kvset_get_nth_vblock_id(ks, j);
+
+            err = blk_list_append(&blks_left->vblks, mbid);
+            if (ev(err))
+                goto out;
+
+            if (j == last - 1 && metadata->overlaps && metadata->accessed) {
+                off_t off;
+                uint32_t len;
+                struct mblock_props props;
+
+                /* Offset must be page aligned. Punching the rest of the vblock
+                 * from the page aligned offset up to the vblock footer.
+                 */
+                off = roundup(metadata->offset, PAGE_SIZE);
+                len = kvset_get_nth_vblock_len(ks, j) - off;
+
+                err = mpool_mblock_punch(ks->ks_mp, mbid, off, len);
+                if (ev(err))
+                    goto out;
+
+                log_debug("Punched mblock (0x%" PRIx64 ") starting at offset %u for %u bytes",
+                    mbid, metadata->offset, len);
+
+                err = mpool_mblock_props_get(ks->ks_mp, mbid, &props);
+                if (ev(err))
+                    goto out;
+
+                blks_right->bl_vused -= props.mpr_write_len - props.mpr_alloc_cap;
+                blks_left->bl_vtotal += props.mpr_alloc_cap - VBLOCK_FOOTER_LEN;
+            } else {
+                blks_left->bl_vtotal += kvset_get_nth_vblock_len(ks, j);
+            }
 
             vbcnt++;
-            blks_left->bl_vtotal += kvset_get_nth_vblock_len(ks, j);
         }
 
         if (vbcnt > 0) {
             vbidx_left += vbcnt;
 
-            err = vgmap_vbidx_set(vgmap_src, end - 1, vgmap_left, vbidx_left - 1, vgidx_left);
+            err = vgmap_vbidx_set(vgmap_src, last - 1, vgmap_left, vbidx_left - 1, vgidx_left);
             if (err)
-                return err;
+                goto out;
 
             vgidx_left++;
         }
 
         vbcnt = 0; /* reset vbcnt for the right kvset */
-        if (overlap) {
+        if (metadata->overlaps) {
             /* Append a clone of the overlapping vblock to the right kvset */
-            const uint64_t src_mbid = kvset_get_nth_vblock_id(ks, src_split);
+            off_t off;
             uint64_t clone_mbid;
+            struct mblock_props props;
+            const uint64_t src_mbid = kvset_get_nth_vblock_id(ks, split);
+
+            off = metadata->accessed ? metadata->offset : 0;
+            off = off < PAGE_SIZE ? 0 : roundup(off - PAGE_SIZE, PAGE_SIZE);
 
-            err = mpool_mblock_clone(ks->ks_mp, src_mbid, 0, 0, &clone_mbid);
+            err = mpool_mblock_clone(ks->ks_mp, src_mbid, off, 0, &clone_mbid);
             if (!err) {
                 err = blk_list_append(&blks_right->vblks, clone_mbid);
                 if (!err)
                     err = blk_list_append(result->ks[RIGHT].blks_commit, clone_mbid);
             }
 
             if (err)
-                return err;
+                goto out;
 
-            perfc_rwc++;
-            if (perfc_ison(pc, PERFC_RA_CNCOMP_RBYTES) || perfc_ison(pc, PERFC_RA_CNCOMP_WBYTES)) {
-                struct mblock_props props;
+            log_debug("Cloned mblock (0x%" PRIx64 ") starting at offset %ld", src_mbid, off);
 
-                err = mpool_mblock_props_get(ks->ks_mp, src_mbid, &props);
-                if (!ev(err))
-                    perfc_rwb += props.mpr_write_len;
-                else
-                    err = 0;
-            }
+            err = mpool_mblock_props_get(ks->ks_mp, clone_mbid, &props);
+            if (ev(err))
+                goto out;
+
+            perfc_rwc++;
+            if (perfc_ison(pc, PERFC_RA_CNCOMP_RBYTES) || perfc_ison(pc, PERFC_RA_CNCOMP_WBYTES))
+                perfc_rwb += props.mpr_write_len - off;
 
             vbcnt++;
-            blks_right->bl_vtotal += kvset_get_nth_vblock_len(ks, src_split);
-            src_split++;
+            blks_right->bl_vused -= props.mpr_write_len - props.mpr_alloc_cap;
+            blks_right->bl_vtotal += props.mpr_alloc_cap - VBLOCK_FOOTER_LEN;
+            split++;
         }
 
-        /* Add the remaining vblocks in [src_split, src_end] to the right kvset
+        /* Add the remaining vblocks in [split, end] to the right kvset
          */
-        for (uint16_t j = src_split; j <= src_end; j++) {
+        for (uint16_t j = split; j <= end; j++) {
             err = blk_list_append(&blks_right->vblks, kvset_get_nth_vblock_id(ks, j));
             if (err)
-                return err;
+                goto out;
 
             vbcnt++;
             blks_right->bl_vtotal += kvset_get_nth_vblock_len(ks, j);
@@ -571,9 +742,9 @@ vblocks_split(
         if (vbcnt > 0) {
             vbidx_right += vbcnt;
 
-            err = vgmap_vbidx_set(vgmap_src, src_end, vgmap_right, vbidx_right - 1, vgidx_right);
+            err = vgmap_vbidx_set(vgmap_src, end, vgmap_right, vbidx_right - 1, vgidx_right);
             if (err)
-                return err;
+                goto out;
 
             vgidx_right++;
         }
@@ -593,7 +764,10 @@ vblocks_split(
             perfc_add2(pc, PERFC_RA_CNCOMP_WREQS, perfc_rwc, PERFC_RA_CNCOMP_WBYTES, perfc_rwb);
     }
 
-    return 0;
+out:
+    free(metadatav);
+
+    return err;
 }
 
 /**