os/NewStore: delay the read of all the overlays until wal applying

The read of all the overlays can be delayed until applying the wal. If
we are doing async wal apply, this can reduce write op latency by
eliminating unnecessary reads in the write code path.

Signed-off-by: Zhiqiang Wang <zhiqiang.wang@intel.com>

diff --git a/src/os/newstore/NewStore.cc b/src/os/newstore/NewStore.cc
index 8c7289a98111cb5e566552475f625070b3ae7def..75ba0bd93e13888bcd423b2e5dad0b050e6d127f 100644 (file)
--- a/src/os/newstore/NewStore.cc
+++ b/src/os/newstore/NewStore.cc
@@ -2472,6 +2472,9 @@ int NewStore::_do_wal_transaction(wal_transaction_t& wt,
   vector<int> sync_fds;
   sync_fds.reserve(wt.ops.size());
 
+  // read all the overlay data first for apply
+  _do_read_all_overlays(wt);
+
   for (list<wal_op_t>::iterator p = wt.ops.begin(); p != wt.ops.end(); ++p) {
     switch (p->op) {
     case wal_op_t::OP_WRITE:
@@ -2605,17 +2608,7 @@ int NewStore::_wal_replay()
     }
 
     // Get the overlay data of the WAL for replay
-    for (list<wal_op_t>::iterator q = wt.ops.begin(); q != wt.ops.end(); ++q) {
-      for (vector<overlay_t>::iterator oit = q->overlays.begin();
-           oit != q->overlays.end(); ++oit) {
-        string key;
-        get_overlay_key(q->nid, oit->key, &key);
-        bufferlist bl, bl_data;
-        db->get(PREFIX_OVERLAY, key, &bl);
-        bl_data.substr_of(bl, oit->value_offset, oit->length);
-        q->data.claim_append(bl_data);
-      }
-    }
+    _do_read_all_overlays(wt);
     dout(20) << __func__ << " replay " << it->key() << dendl;
     int r = _do_wal_transaction(wt, NULL);  // don't bother with aio here
     if (r < 0)
@@ -3191,10 +3184,6 @@ int NewStore::_do_write_all_overlays(TransContext *txc,
        p != o->onode.overlay_map.end(); ) {
     dout(10) << __func__ << " overlay " << p->first
             << "~" << p->second << dendl;
-    string key;
-    get_overlay_key(o->onode.nid, p->second.key, &key);
-    bufferlist bl;
-    db->get(PREFIX_OVERLAY, key, &bl);
 
     wal_op_t *op = _get_wal_op(txc);
     op->op = wal_op_t::OP_WRITE;
@@ -3204,7 +3193,6 @@ int NewStore::_do_write_all_overlays(TransContext *txc,
     // The overlays will be removed from the db after applying the WAL
     op->nid = o->onode.nid;
     op->overlays.push_back(p->second);
-    op->data.substr_of(bl, p->second.value_offset, p->second.length);
 
     // Combine with later overlays if contiguous
     map<uint64_t,overlay_t>::iterator prev = p, next = p;
@@ -3213,14 +3201,6 @@ int NewStore::_do_write_all_overlays(TransContext *txc,
       if (prev->first + prev->second.length == next->first) {
         dout(10) << __func__ << " combining overlay " << next->first
                  << "~" << next->second << dendl;
-        string key_next;
-        get_overlay_key(o->onode.nid, next->second.key, &key_next);
-        bufferlist bl_next, bl_next_data;
-        db->get(PREFIX_OVERLAY, key_next, &bl_next);
-
-        bl_next_data.substr_of(bl_next, next->second.value_offset,
-                               next->second.length);
-        op->data.claim_append(bl_next_data);
         op->length += next->second.length;
         op->overlays.push_back(next->second);
 
@@ -3250,6 +3230,22 @@ int NewStore::_do_write_all_overlays(TransContext *txc,
   return 0;
 }
 
+void NewStore::_do_read_all_overlays(wal_transaction_t& wt)
+{
+  for (list<wal_op_t>::iterator p = wt.ops.begin(); p != wt.ops.end(); ++p) {
+    for (vector<overlay_t>::iterator q = p->overlays.begin();
+         q != p->overlays.end(); ++q) {
+      string key;
+      get_overlay_key(p->nid, q->key, &key);
+      bufferlist bl, bl_data;
+      db->get(PREFIX_OVERLAY, key, &bl);
+      bl_data.substr_of(bl, q->value_offset, q->length);
+      p->data.claim_append(bl_data);
+    }
+  }
+  return;
+}
+
 int NewStore::_do_write(TransContext *txc,
                        OnodeRef o,
                        uint64_t offset, uint64_t length,

diff --git a/src/os/newstore/NewStore.h b/src/os/newstore/NewStore.h
index c126873e970e1049a02c8c36a19cb75c38c50f8a..c4cdc2bd3f2e854e1f65d76841709c00449dcaee 100644 (file)
--- a/src/os/newstore/NewStore.h
+++ b/src/os/newstore/NewStore.h
@@ -741,6 +741,7 @@ private:
                        const bufferlist& bl);
   int _do_write_all_overlays(TransContext *txc,
                             OnodeRef o);
+  void _do_read_all_overlays(wal_transaction_t& wt);
   int _do_write(TransContext *txc,
                OnodeRef o,
                uint64_t offset, uint64_t length,