os/bluestore: sloppy reshard boundaries to avoid spanning blobs

Make the extent map shard target size sloppy so that we can try to avoid
sharding boundaries that create spanning blobs.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 83a69b7185e4141d8532e987c6650fc5e7e1f0ec..fa6a214203705762925041a8c6d8b2143af8cd40 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -982,6 +982,7 @@ OPTION(bluestore_compression_required_ratio, OPT_DOUBLE, .875)
 OPTION(bluestore_extent_map_shard_max_size, OPT_U32, 1200)
 OPTION(bluestore_extent_map_shard_target_size, OPT_U32, 500)
 OPTION(bluestore_extent_map_shard_min_size, OPT_U32, 150)
+OPTION(bluestore_extent_map_shard_target_size_slop, OPT_DOUBLE, .2)
 OPTION(bluestore_extent_map_inline_shard_prealloc_size, OPT_U32, 256)
 OPTION(bluestore_cache_type, OPT_STR, "2q")   // lru, 2q
 OPTION(bluestore_onode_cache_size, OPT_U32, 16*1024)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index ed95a012f53840aa3a769483b26bc546509b235a..66bc30a34176c45fe16fa94195de1e0ff7018c1b 100644 (file)
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -1488,9 +1488,10 @@ void BlueStore::ExtentMap::reshard(Onode *o)
     }
   }
   unsigned target = g_conf->bluestore_extent_map_shard_target_size;
+  unsigned slop = target * g_conf->bluestore_extent_map_shard_target_size_slop;
   unsigned extent_avg = bytes / extent_map.size();
   dout(20) << __func__ << " extent_avg " << extent_avg
-          << " target " << target << dendl;
+          << " target " << target << " slop " << slop << dendl;
 
   // reshard
   auto ep = extent_map.begin();
@@ -1500,6 +1501,7 @@ void BlueStore::ExtentMap::reshard(Onode *o)
   unsigned estimate = 0;
   unsigned offset = 0;
   vector<bluestore_onode_t::shard_info> new_shard_info;
+  unsigned max_blob_end = 0;
   while (ep != extent_map.end()) {
     dout(30) << " ep " << *ep << dendl;
     assert(!ep->blob->is_spanning());
@@ -1521,7 +1523,10 @@ void BlueStore::ExtentMap::reshard(Onode *o)
       dout(20) << __func__ << " old shard end 0x" << std::hex << shard_end
               << std::dec << dendl;
     }
-    if (estimate && estimate + extent_avg > target) {
+    // disfavor shard boundaries that span a blob
+    bool would_span = (ep->logical_offset < max_blob_end) || ep->blob_offset;
+    if (estimate &&
+       estimate + extent_avg > target + (would_span ? slop : 0)) {
       // new shard
       if (offset == 0) {
        new_shard_info.emplace_back(bluestore_onode_t::shard_info());
@@ -1537,6 +1542,10 @@ void BlueStore::ExtentMap::reshard(Onode *o)
       estimate = 0;
     }
     estimate += extent_avg;
+    uint32_t be = ep->blob_end();
+    if (be > max_blob_end) {
+      max_blob_end = be;
+    }
     ++ep;
   }
   o->onode.extent_map_shards.swap(new_shard_info);

diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 8e3b5619806000e7a3418122513d4ff3a71e6fcc..b7e436c24d5b02cfc73e5ac2e515f3bccbbb486f 100644 (file)
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -523,6 +523,11 @@ public:
       return a.logical_offset == b.logical_offset;
     }
 
+    uint32_t blob_end() {
+      return logical_offset + blob->get_blob().get_logical_length() -
+       blob_offset;
+    }
+
     bool blob_escapes_range(uint32_t o, uint32_t l) {
       uint32_t bstart = logical_offset - blob_offset;
       return (bstart < o ||