osdc/Filer: truncate large file party by party

Fixes: http://tracker.ceph.com/issues/19755
Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
(cherry picked from commit 5fab215e461e5ecc36c0f9d9ea867f6c45e80263)

Conflicts:
	src/osdc/Filer.h
	src/osdc/Filer.cc
	    - add parameter to fit _modify (need onack parameter)
	src/mds/MDCache.cc
	    - make truncate() consistency

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 60ff9772e53929ee7d55c705874e395016c9d152..f005d761058e1051481b7c2738f1edd324e3b3e1 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -430,6 +430,8 @@ OPTION(objecter_inject_no_watch_ping, OPT_BOOL, false)   // suppress watch pings
 
 // Max number of deletes at once in a single Filer::purge call
 OPTION(filer_max_purge_ops, OPT_U32, 10)
+// Max number of truncate at once in a single Filer::truncate call
+OPTION(filer_max_truncate_ops, OPT_U32, 128)
 
 OPTION(journaler_allow_split_entries, OPT_BOOL, true)
 OPTION(journaler_write_head_interval, OPT_INT, 15)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index d3e9336c36800f916405be9ed1ac0ef840ae2a39..01eb90acd519e43d84109c75f11d72e0d017dbd5 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -6249,9 +6249,8 @@ void MDCache::_truncate_inode(CInode *in, LogSegment *ls)
   filer.truncate(in->inode.ino, &in->inode.layout, *snapc,
                 pi->truncate_size, pi->truncate_from-pi->truncate_size,
                 pi->truncate_seq, ceph::real_time::min(), 0,
-                0, new C_OnFinisher(new C_IO_MDC_TruncateFinish(this, in,
-                                                                      ls),
-                                          mds->finisher));
+                new C_OnFinisher(new C_IO_MDC_TruncateFinish(this, in, ls),
+                                 mds->finisher));
 }
 
 struct C_MDC_TruncateLogged : public MDCacheContext {

diff --git a/src/osdc/Filer.cc b/src/osdc/Filer.cc
index f64a122534841c7fa276d2e79ad57b9d0d92c81b..5aef11610ac179d626fa78020cc495e5e8dfc7be 100644 (file)
--- a/src/osdc/Filer.cc
+++ b/src/osdc/Filer.cc
@@ -383,3 +383,110 @@ void Filer::_do_purge_range(PurgeRange *pr, int fin)
                     new C_OnFinisher(new C_PurgeRange(this, pr), finisher));
   }
 }
+
+// -----------------------
+struct TruncRange {
+  std::mutex lock;
+  typedef std::lock_guard<std::mutex> lock_guard;
+  typedef std::unique_lock<std::mutex> unique_lock;
+  inodeno_t ino;
+  file_layout_t layout;
+  SnapContext snapc;
+  ceph::real_time mtime;
+  int flags;
+  Context *oncommit;
+  int uncommitted;
+  uint64_t offset;
+  uint64_t length;
+  uint32_t truncate_seq;
+  TruncRange(inodeno_t i, const file_layout_t& l, const SnapContext& sc,
+            ceph::real_time t, int fl, Context *fin,
+            uint64_t off, uint64_t len, uint32_t ts)
+    : ino(i), layout(l), snapc(sc), mtime(t), flags(fl), oncommit(fin),
+      uncommitted(0), offset(off), length(len), truncate_seq(ts) {}
+};
+
+void Filer::truncate(inodeno_t ino,
+                    file_layout_t *layout,
+                    const SnapContext& snapc,
+                    uint64_t offset,
+                    uint64_t len,
+                    __u32 truncate_seq,
+                    ceph::real_time mtime,
+                    int flags,
+                    Context *oncommit)
+{
+  uint64_t period = layout->get_period();
+  uint64_t num_objs = Striper::get_num_objects(*layout, len + (offset % period));
+  if (num_objs == 1) {
+    vector<ObjectExtent> extents;
+    Striper::file_to_extents(cct, ino, layout, offset, len, 0, extents);
+    vector<OSDOp> ops(1);
+    ops[0].op.op = CEPH_OSD_OP_TRIMTRUNC;
+    ops[0].op.extent.truncate_seq = truncate_seq;
+    ops[0].op.extent.truncate_size = extents[0].offset;
+    objecter->_modify(extents[0].oid, extents[0].oloc, ops, mtime, snapc,
+                     flags, NULL, oncommit);
+    return;
+  }
+
+  if (len > 0 && (offset + len) % period)
+    len += period - ((offset + len) % period);
+
+  TruncRange *tr = new TruncRange(ino, *layout, snapc, mtime, flags, oncommit,
+                                 offset, len, truncate_seq);
+  _do_truncate_range(tr, 0);
+
+  return;
+}
+
+struct C_TruncRange : public Context {
+  Filer *filer;
+  TruncRange *tr;
+  C_TruncRange(Filer *f, TruncRange *t) : filer(f), tr(t) {}
+  void finish(int r) override {
+    filer->_do_truncate_range(tr, 1);
+  }
+};
+
+void Filer::_do_truncate_range(TruncRange *tr, int fin)
+{
+  TruncRange::unique_lock trl(tr->lock);
+  tr->uncommitted -= fin;
+  ldout(cct, 10) << "_do_truncate_range " << tr->ino << " objects " << tr->offset
+                << "~" << tr->length << " uncommitted " << tr->uncommitted
+                << dendl;
+
+  if (tr->length == 0 && tr->uncommitted == 0) {
+    tr->oncommit->complete(0);
+    trl.unlock();
+    delete tr;
+    return;
+  }
+
+  vector<ObjectExtent> extents;
+
+  int max = cct->_conf->filer_max_truncate_ops - tr->uncommitted;
+  if (max > 0 && tr->length > 0) {
+    uint64_t len = tr->layout.get_period() * max;
+    if (len > tr->length)
+      len = tr->length;
+
+    uint64_t offset = tr->offset + tr->length - len;
+    Striper::file_to_extents(cct, tr->ino, &tr->layout, offset, len, 0, extents);
+    tr->uncommitted += extents.size();
+    tr->length -= len;
+  }
+
+  trl.unlock();
+
+  // Issue objecter ops outside tr->lock to avoid lock dependency loop
+  for (const auto& p : extents) {
+    vector<OSDOp> ops(1);
+    ops[0].op.op = CEPH_OSD_OP_TRIMTRUNC;
+    ops[0].op.extent.truncate_size = p.offset;
+    ops[0].op.extent.truncate_seq = tr->truncate_seq;
+    objecter->_modify(p.oid, p.oloc, ops, tr->mtime, tr->snapc, tr->flags, NULL,
+                     new C_OnFinisher(new C_TruncRange(this, tr), finisher));
+  }
+}

diff --git a/src/osdc/Filer.h b/src/osdc/Filer.h
index 96c11b80461a065eef146d89cb5fae22e9bada0b..9cf8c92df77bb1c5330b6075183c5b32c095c31c 100644 (file)
--- a/src/osdc/Filer.h
+++ b/src/osdc/Filer.h
@@ -192,7 +192,7 @@ class Filer {
     return 0;
   }
 
-  int truncate(inodeno_t ino,
+  void truncate(inodeno_t ino,
               file_layout_t *layout,
               const SnapContext& snapc,
               uint64_t offset,
@@ -200,36 +200,8 @@ class Filer {
               __u32 truncate_seq,
               ceph::real_time mtime,
               int flags,
-              Context *onack,
-              Context *oncommit) {
-    vector<ObjectExtent> extents;
-    Striper::file_to_extents(cct, ino, layout, offset, len, 0, extents);
-    if (extents.size() == 1) {
-      vector<OSDOp> ops(1);
-      ops[0].op.op = CEPH_OSD_OP_TRIMTRUNC;
-      ops[0].op.extent.truncate_seq = truncate_seq;
-      ops[0].op.extent.truncate_size = extents[0].offset;
-      objecter->_modify(extents[0].oid, extents[0].oloc, ops, mtime, snapc,
-                       flags, onack, oncommit);
-    } else {
-      C_GatherBuilder gack(cct, onack);
-      C_GatherBuilder gcom(cct, oncommit);
-      for (vector<ObjectExtent>::iterator p = extents.begin();
-          p != extents.end();
-          ++p) {
-       vector<OSDOp> ops(1);
-       ops[0].op.op = CEPH_OSD_OP_TRIMTRUNC;
-       ops[0].op.extent.truncate_size = p->offset;
-       ops[0].op.extent.truncate_seq = truncate_seq;
-       objecter->_modify(p->oid, p->oloc, ops, mtime, snapc, flags,
-                         onack ? gack.new_sub():0,
-                         oncommit ? gcom.new_sub():0);
-      }
-      gack.activate();
-      gcom.activate();
-    }
-    return 0;
-  }
+              Context *oncommit);
+  void _do_truncate_range(struct TruncRange *pr, int fin);
 
   int zero(inodeno_t ino,
           file_layout_t *layout,