osdc/Filer: truncate large file party by party

Fixes: http://tracker.ceph.com/issues/19755
Signed-off-by: "Yan, Zheng" <zyan@redhat.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index b7787ac797937bda066db238080a186f35bf1ed2..a66819710d0d020fea6c530ca7352bfc12b19873 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -488,6 +488,8 @@ OPTION(objecter_debug_inject_relock_delay, OPT_BOOL, false)
 
 // Max number of deletes at once in a single Filer::purge call
 OPTION(filer_max_purge_ops, OPT_U32, 10)
+// Max number of truncate at once in a single Filer::truncate call
+OPTION(filer_max_truncate_ops, OPT_U32, 128)
 
 OPTION(journaler_allow_split_entries, OPT_BOOL, true)
 OPTION(journaler_write_head_interval, OPT_INT, 15)

diff --git a/src/osdc/Filer.cc b/src/osdc/Filer.cc
index 17152f78b44c56103db5373a5e7f1247fa06e827..df414adff2513ede3ee9f51798d6fd6d58b7ae56 100644 (file)
--- a/src/osdc/Filer.cc
+++ b/src/osdc/Filer.cc
@@ -377,3 +377,108 @@ void Filer::_do_purge_range(PurgeRange *pr, int fin)
                     new C_OnFinisher(new C_PurgeRange(this, pr), finisher));
   }
 }
+
+// -----------------------
+struct TruncRange {
+  std::mutex lock;
+  typedef std::lock_guard<std::mutex> lock_guard;
+  typedef std::unique_lock<std::mutex> unique_lock;
+  inodeno_t ino;
+  file_layout_t layout;
+  SnapContext snapc;
+  ceph::real_time mtime;
+  int flags;
+  Context *oncommit;
+  int uncommitted;
+  uint64_t offset;
+  uint64_t length;
+  uint32_t truncate_seq;
+  TruncRange(inodeno_t i, const file_layout_t& l, const SnapContext& sc,
+            ceph::real_time t, int fl, Context *fin,
+            uint64_t off, uint64_t len, uint32_t ts)
+    : ino(i), layout(l), snapc(sc), mtime(t), flags(fl), oncommit(fin),
+      uncommitted(0), offset(off), length(len), truncate_seq(ts) {}
+};
+
+void Filer::truncate(inodeno_t ino,
+                    file_layout_t *layout,
+                    const SnapContext& snapc,
+                    uint64_t offset,
+                    uint64_t len,
+                    __u32 truncate_seq,
+                    ceph::real_time mtime,
+                    int flags,
+                    Context *oncommit)
+{
+  uint64_t period = layout->get_period();
+  uint64_t num_objs = Striper::get_num_objects(*layout, len + (offset % period));
+  if (num_objs == 1) {
+    vector<ObjectExtent> extents;
+    Striper::file_to_extents(cct, ino, layout, offset, len, 0, extents);
+    vector<OSDOp> ops(1);
+    ops[0].op.op = CEPH_OSD_OP_TRIMTRUNC;
+    ops[0].op.extent.truncate_seq = truncate_seq;
+    ops[0].op.extent.truncate_size = extents[0].offset;
+    objecter->_modify(extents[0].oid, extents[0].oloc, ops, mtime, snapc,
+                     flags, oncommit);
+    return;
+  }
+
+  if (len > 0 && (offset + len) % period)
+    len += period - ((offset + len) % period);
+
+  TruncRange *tr = new TruncRange(ino, *layout, snapc, mtime, flags, oncommit,
+                                 offset, len, truncate_seq);
+  _do_truncate_range(tr, 0);
+}
+
+struct C_TruncRange : public Context {
+  Filer *filer;
+  TruncRange *tr;
+  C_TruncRange(Filer *f, TruncRange *t) : filer(f), tr(t) {}
+  void finish(int r) override {
+    filer->_do_truncate_range(tr, 1);
+  }
+};
+
+void Filer::_do_truncate_range(TruncRange *tr, int fin)
+{
+  TruncRange::unique_lock trl(tr->lock);
+  tr->uncommitted -= fin;
+  ldout(cct, 10) << "_do_truncate_range " << tr->ino << " objects " << tr->offset
+                << "~" << tr->length << " uncommitted " << tr->uncommitted
+                << dendl;
+
+  if (tr->length == 0 && tr->uncommitted == 0) {
+    tr->oncommit->complete(0);
+    trl.unlock();
+    delete tr;
+    return;
+  }
+
+  vector<ObjectExtent> extents;
+
+  int max = cct->_conf->filer_max_truncate_ops - tr->uncommitted;
+  if (max > 0 && tr->length > 0) {
+    uint64_t len = tr->layout.get_period() * max;
+    if (len > tr->length)
+      len = tr->length;
+
+    uint64_t offset = tr->offset + tr->length - len;
+    Striper::file_to_extents(cct, tr->ino, &tr->layout, offset, len, 0, extents);
+    tr->uncommitted += extents.size();
+    tr->length -= len;
+  }
+
+  trl.unlock();
+
+  // Issue objecter ops outside tr->lock to avoid lock dependency loop
+  for (const auto& p : extents) {
+    vector<OSDOp> ops(1);
+    ops[0].op.op = CEPH_OSD_OP_TRIMTRUNC;
+    ops[0].op.extent.truncate_size = p.offset;
+    ops[0].op.extent.truncate_seq = tr->truncate_seq;
+    objecter->_modify(p.oid, p.oloc, ops, tr->mtime, tr->snapc, tr->flags,
+                     new C_OnFinisher(new C_TruncRange(this, tr), finisher));
+  }
+}

diff --git a/src/osdc/Filer.h b/src/osdc/Filer.h
index 8f2fd3e686d1e2bdec293a930279bd6e790f479b..00b6caa8f26aa1d8cfc972c2161bf6eb7bcbfe48 100644 (file)
--- a/src/osdc/Filer.h
+++ b/src/osdc/Filer.h
@@ -193,31 +193,8 @@ class Filer {
               __u32 truncate_seq,
               ceph::real_time mtime,
               int flags,
-              Context *oncommit) {
-    vector<ObjectExtent> extents;
-    Striper::file_to_extents(cct, ino, layout, offset, len, 0, extents);
-    if (extents.size() == 1) {
-      vector<OSDOp> ops(1);
-      ops[0].op.op = CEPH_OSD_OP_TRIMTRUNC;
-      ops[0].op.extent.truncate_seq = truncate_seq;
-      ops[0].op.extent.truncate_size = extents[0].offset;
-      objecter->_modify(extents[0].oid, extents[0].oloc, ops, mtime, snapc,
-                       flags, oncommit);
-    } else {
-      C_GatherBuilder gcom(cct, oncommit);
-      for (vector<ObjectExtent>::iterator p = extents.begin();
-          p != extents.end();
-          ++p) {
-       vector<OSDOp> ops(1);
-       ops[0].op.op = CEPH_OSD_OP_TRIMTRUNC;
-       ops[0].op.extent.truncate_size = p->offset;
-       ops[0].op.extent.truncate_seq = truncate_seq;
-       objecter->_modify(p->oid, p->oloc, ops, mtime, snapc, flags,
-                         oncommit ? gcom.new_sub():0);
-      }
-      gcom.activate();
-    }
-  }
+              Context *oncommit);
+  void _do_truncate_range(struct TruncRange *pr, int fin);
 
   void zero(inodeno_t ino,
           const file_layout_t *layout,