os/bluestore: avoid reencoding unchanged blobs

Keep a dirty flag on each blob, set by the accessor.

Cache the encoded blob_t and only reencode it if it was
modified.

It improves 4K random IOPS on my nvme vstart cluster
by about 9%.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 6a035c2a741d0677a6b03a8d61176f5cf8e417c8..4d9d0e8a69128bfe3cb869ca0c8bc40f552e74c8 100644 (file)
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -1182,7 +1182,7 @@ void BlueStore::BlobMap::encode(bufferlist& bl) const
   ::encode(n, bl);
   for (auto p = blob_map.begin(); n--; ++p) {
     ::encode(p->id, bl);
-    ::encode(p->blob, bl);
+    p->encode(bl);
   }
 }
 
@@ -1195,7 +1195,7 @@ void BlueStore::BlobMap::decode(bufferlist::iterator& p, Cache *c)
     int64_t id;
     ::decode(id, p);
     Blob *b = new Blob(id, c);
-    ::decode(b->blob, p);
+    b->decode(p);
     b->get();
     blob_map.insert(*b);
   }

diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 3fdb4082b6e193e6d799317b4cf73845f1566b15..93ec4da5477996c47c822285115700a4076edd38 100644 (file)
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -288,11 +288,14 @@ public:
   struct Blob : public boost::intrusive::set_base_hook<> {
     std::atomic_int nref;  ///< reference count
     int64_t id = 0;          ///< id
-  private:
-    bluestore_blob_t blob;   ///< blob metadata
-  public:
     BufferSpace bc;          ///< buffer cache
 
+  private:
+    bluestore_blob_t blob;        ///< decoded blob metadata
+    mutable bool dirty = true;    ///< true if blob != blob_bl
+    mutable bufferlist blob_bl;   ///< cached encoded blob
+
+  public:
     Blob(int64_t i, Cache *c) : nref(0), id(i), bc(c) {}
     ~Blob() {
       assert(bc.empty());
@@ -320,6 +323,10 @@ public:
       return blob;
     }
     bluestore_blob_t& dirty_blob() {
+      if (!dirty) {
+       dirty = true;
+       blob_bl.clear();
+      }
       return blob;
     }
 
@@ -333,6 +340,22 @@ public:
       if (--nref == 0)
        delete this;
     }
+
+    void encode(bufferlist& bl) const {
+      if (dirty) {
+       ::encode(blob, blob_bl);
+       dirty = false;
+      } else {
+       assert(blob_bl.length());
+      }
+      bl.append(blob_bl);
+    }
+    void decode(bufferlist::iterator& p) {
+      bufferlist::iterator start = p;
+      ::decode(blob, p);
+      start.copy(p.get_off() - start.get_off(), blob_bl);
+      dirty = false;
+    }
   };
   typedef boost::intrusive_ptr<Blob> BlobRef;