os/NewStore: avoid dup the data of the overlays in the WAL

When writing all the overlays, there is no need to dup the data in WAL.
Instead, we can reference the overlays in the WAL, and remove these
overlays after commiting them to the fs. When replaying, we can get
these data from the referenced overlays. Doing this way, we can save a
write and a deletion for each of the overlay data in the db.

Signed-off-by: Zhiqiang Wang <zhiqiang.wang@intel.com>

diff --git a/src/os/newstore/NewStore.cc b/src/os/newstore/NewStore.cc
index 39af9c45f05cef87eea7aa96a618c3e061d484d5..bb946f288f4761b5b2b72f976a4758bfb21b17c1 100644 (file)
--- a/src/os/newstore/NewStore.cc
+++ b/src/os/newstore/NewStore.cc
@@ -2383,6 +2383,23 @@ void NewStore::_kv_sync_thread()
            it != wal_cleaning.end();
            it++) {
        wal_transaction_t& wt =*(*it)->wal_txn;
+       // cleanup the data in overlays
+       for (list<wal_op_t>::iterator p = wt.ops.begin(); p != wt.ops.end(); ++p) {
+         for (vector<overlay_t>::iterator q = p->overlays.begin();
+               q != p->overlays.end(); ++q) {
+            string key;
+            get_overlay_key(p->nid, q->key, &key);
+           txc_cleanup_sync->rmkey(PREFIX_OVERLAY, key);
+         }
+       }
+       // cleanup the shared overlays. this may double delete something we
+       // did above, but that's less work than doing careful ref counting
+       // of the overlay key/value pairs.
+       for (vector<string>::iterator p = wt.shared_overlay_keys.begin();
+             p != wt.shared_overlay_keys.end(); ++p) {
+         txc_cleanup_sync->rmkey(PREFIX_OVERLAY, *p);
+       }
+       // cleanup the wal
        string key;
        get_wal_key(wt.seq, &key);
        txc_cleanup_sync->rmkey(PREFIX_WAL, key);
@@ -2586,6 +2603,19 @@ int NewStore::_wal_replay()
       derr << __func__ << " failed to decode wal txn " << it->key() << dendl;
       return -EIO;
     }
+
+    // Get the overlay data of the WAL for replay
+    for (list<wal_op_t>::iterator q = wt.ops.begin(); q != wt.ops.end(); ++q) {
+      for (vector<overlay_t>::iterator oit = q->overlays.begin();
+           oit != q->overlays.end(); ++oit) {
+        string key;
+        get_overlay_key(q->nid, oit->key, &key);
+        bufferlist bl, bl_data;
+        db->get(PREFIX_OVERLAY, key, &bl);
+        bl_data.substr_of(bl, oit->value_offset, oit->length);
+        q->data.claim_append(bl_data);
+      }
+    }
     dout(20) << __func__ << " replay " << it->key() << dendl;
     int r = _do_wal_transaction(wt, NULL);  // don't bother with aio here
     if (r < 0)
@@ -3171,10 +3201,11 @@ int NewStore::_do_write_all_overlays(TransContext *txc,
     op->offset = p->first;
     op->length = p->second.length;
     op->fid = f.fid;
+    // The overlays will be removed from the db after applying the WAL
+    op->nid = o->onode.nid;
+    op->overlays.push_back(p->second);
     op->data.substr_of(bl, p->second.value_offset, p->second.length);
 
-    txc->t->rmkey(PREFIX_OVERLAY, key);
-
     // Combine with later overlays if contiguous
     map<uint64_t,overlay_t>::iterator prev = p, next = p;
     ++next;
@@ -3191,7 +3222,7 @@ int NewStore::_do_write_all_overlays(TransContext *txc,
                                next->second.length);
         bl.claim_append(bl_next_data);
         op->length += next->second.length;
-        txc->t->rmkey(PREFIX_OVERLAY, key_next);
+       op->overlays.push_back(next->second);
 
        ++prev;
        ++next;
@@ -3202,16 +3233,15 @@ int NewStore::_do_write_all_overlays(TransContext *txc,
     p = next;
   }
 
-  // this may double delete something we did above, but that's less
-  // work than doing careful ref counting of the overlay key/value
-  // pairs.
+  // put the shared overlay keys into the WAL transaction, so that we
+  // can cleanup them later after applying the WAL
   for (set<uint64_t>::iterator p = o->onode.shared_overlays.begin();
        p != o->onode.shared_overlays.end();
        ++p) {
     dout(10) << __func__ << " shared overlay " << *p << dendl;
     string key;
     get_overlay_key(o->onode.nid, *p, &key);
-    txc->t->rmkey(PREFIX_OVERLAY, key);
+    txc->wal_txn->shared_overlay_keys.push_back(key);
   }
 
   o->onode.overlay_map.clear();

diff --git a/src/os/newstore/newstore_types.cc b/src/os/newstore/newstore_types.cc
index aa1a7105321107a821668a1d7801ac96a7b19a60..5489faf143f62dbce337282b3a9d131392772f89 100644 (file)
--- a/src/os/newstore/newstore_types.cc
+++ b/src/os/newstore/newstore_types.cc
@@ -229,7 +229,11 @@ void wal_op_t::encode(bufferlist& bl) const
   ::encode(fid, bl);
   ::encode(offset, bl);
   ::encode(length, bl);
-  ::encode(data, bl);
+  ::encode(nid, bl);
+  ::encode(overlays, bl);
+  if (!overlays.size()) {
+    ::encode(data, bl);
+  }
   ENCODE_FINISH(bl);
 }
 
@@ -240,7 +244,11 @@ void wal_op_t::decode(bufferlist::iterator& p)
   ::decode(fid, p);
   ::decode(offset, p);
   ::decode(length, p);
-  ::decode(data, p);
+  ::decode(nid, p);
+  ::decode(overlays, p);
+  if (!overlays.size()) {
+    ::decode(data, p);
+  }
   DECODE_FINISH(p);
 }
 
@@ -250,6 +258,15 @@ void wal_op_t::dump(Formatter *f) const
   f->dump_object("fid", fid);
   f->dump_unsigned("offset", offset);
   f->dump_unsigned("length", length);
+  if (overlays.size()) {
+    f->dump_unsigned("nid", nid);
+    f->open_array_section("overlays");
+    for (vector<overlay_t>::const_iterator p = overlays.begin();
+         p != overlays.end(); ++p) {
+      f->dump_object("overlay", *p);
+    }
+    f->close_section();
+  }
 }
 
 void wal_transaction_t::encode(bufferlist& bl) const
@@ -257,6 +274,7 @@ void wal_transaction_t::encode(bufferlist& bl) const
   ENCODE_START(1, 1, bl);
   ::encode(seq, bl);
   ::encode(ops, bl);
+  ::encode(shared_overlay_keys, bl);
   ENCODE_FINISH(bl);
 }
 
@@ -265,6 +283,7 @@ void wal_transaction_t::decode(bufferlist::iterator& p)
   DECODE_START(1, p);
   ::decode(seq, p);
   ::decode(ops, p);
+  ::decode(shared_overlay_keys, p);
   DECODE_FINISH(p);
 }
 
@@ -276,4 +295,10 @@ void wal_transaction_t::dump(Formatter *f) const
     f->dump_object("op", *p);
   }
   f->close_section();
+  f->open_array_section("shared_overlay_keys");
+  for (vector<string>::const_iterator p = shared_overlay_keys.begin();
+       p != shared_overlay_keys.end(); ++p) {
+    f->dump_string("shared_overlay_key", *p);
+  }
+  f->close_section();
 }

diff --git a/src/os/newstore/newstore_types.h b/src/os/newstore/newstore_types.h
index 286fc773e6791866535bd4a4b96f29e55006d8de..ca616adb94c42dec507ce3d553dfacf89ca5c0e5 100644 (file)
--- a/src/os/newstore/newstore_types.h
+++ b/src/os/newstore/newstore_types.h
@@ -151,6 +151,8 @@ struct wal_op_t {
   fid_t fid;
   uint64_t offset, length;
   bufferlist data;
+  uint64_t nid;
+  vector<overlay_t> overlays;
 
   void encode(bufferlist& bl) const;
   void decode(bufferlist::iterator& p);
@@ -164,6 +166,7 @@ WRITE_CLASS_ENCODER(wal_op_t)
 struct wal_transaction_t {
   uint64_t seq;
   list<wal_op_t> ops;
+  vector<string> shared_overlay_keys;
 
   int64_t _bytes;  ///< cached byte count