osd: don't require RWEXCL lock for stat+write ops.

In librbd, a stat op is inserted before write op for cloned image. OSD used to use RWEXCL and such requests are processed one by one.
With this fix, OSD will use RWWRITE rather than RWEXCL for such [stat,write] request to allow multiple [stat,write] on the same object and improve performance.

Signed-off-by: Alice Zhao <brucen1030@163.com>
(cherry picked from commit 9be266b0a0304aaaaa0ca12f28fcd3e9cce1f9d7)

Conflicts:
	src/osd/PrimaryLogPG.cc [ commit dfca9f8c0d8b ("osd: move
	  PrimaryLogPG::get_rw_locks() from header to .cc.") not in
	  pacific ]

diff --git a/src/osd/OpRequest.h b/src/osd/OpRequest.h
index daa0e1993a3bd43199b36eb6cf32a6595af70d44..358ccd84f6f945f4dcd59e5d69ebd00307952a00 100644 (file)
--- a/src/osd/OpRequest.h
+++ b/src/osd/OpRequest.h
@@ -38,6 +38,7 @@ public:
   bool op_info_needs_init() const { return op_info.get_flags() == 0; }
   bool check_rmw(int flag) const { return op_info.check_rmw(flag); }
   bool may_read() const { return op_info.may_read(); }
+  bool may_read_data() const { return op_info.may_read_data(); }
   bool may_write() const { return op_info.may_write(); }
   bool may_cache() const { return op_info.may_cache(); }
   bool rwordered_forced() const { return op_info.rwordered_forced(); }

diff --git a/src/osd/PrimaryLogPG.h b/src/osd/PrimaryLogPG.h
index 2736f3de997eeb01ae938f2957ff09754d3b5f97..f95c92fa0ff21b086273c42f6c16303a6c2b3ad5 100644 (file)
--- a/src/osd/PrimaryLogPG.h
+++ b/src/osd/PrimaryLogPG.h
@@ -875,7 +875,11 @@ protected:
      * to get the second.
      */
     if (write_ordered && ctx->op->may_read()) {
-      ctx->lock_type = RWState::RWEXCL;
+      if (ctx->op->may_read_data()) {
+        ctx->lock_type = RWState::RWEXCL;
+      } else {
+        ctx->lock_type = RWState::RWWRITE;
+      }
     } else if (write_ordered) {
       ctx->lock_type = RWState::RWWRITE;
     } else {

diff --git a/src/osd/osd_op_util.cc b/src/osd/osd_op_util.cc
index 54c590ee2248ee9e842dd2b3ff770e37b14a75a4..4b75766aad4d26cabafdbbf10cb0baa25079cc49 100644 (file)
--- a/src/osd/osd_op_util.cc
+++ b/src/osd/osd_op_util.cc
@@ -16,6 +16,7 @@ bool OpInfo::check_rmw(int flag) const {
   ceph_assert(rmw_flags != 0);
   return rmw_flags & flag;
 }
+// Returns true if op performs a read (including of the object_info).
 bool OpInfo::may_read() const {
   return need_read_cap() || check_rmw(CEPH_OSD_RMW_FLAG_CLASS_READ);
 }
@@ -51,6 +52,16 @@ bool OpInfo::need_skip_promote() const {
 bool OpInfo::allows_returnvec() const {
   return check_rmw(CEPH_OSD_RMW_FLAG_RETURNVEC);
 }
+/**
+ * may_read_data()
+ * 
+ * Returns true if op reads information other than the object_info. Requires that the
+ * osd flush any prior writes prior to servicing this op. Includes any information not
+ * cached by the osd in the object_info or snapset.
+ */
+bool OpInfo::may_read_data() const {
+  return check_rmw(CEPH_OSD_RMW_FLAG_READ_DATA);
+}
 
 void OpInfo::set_rmw_flags(int flags) {
   rmw_flags |= flags;
@@ -67,6 +78,7 @@ void OpInfo::set_skip_handle_cache() { set_rmw_flags(CEPH_OSD_RMW_FLAG_SKIP_HAND
 void OpInfo::set_skip_promote() { set_rmw_flags(CEPH_OSD_RMW_FLAG_SKIP_PROMOTE); }
 void OpInfo::set_force_rwordered() { set_rmw_flags(CEPH_OSD_RMW_FLAG_RWORDERED); }
 void OpInfo::set_returnvec() { set_rmw_flags(CEPH_OSD_RMW_FLAG_RETURNVEC); }
+void OpInfo::set_read_data() { set_rmw_flags(CEPH_OSD_RMW_FLAG_READ_DATA); }
 
 
 int OpInfo::set_from_op(
@@ -101,8 +113,12 @@ int OpInfo::set_from_op(
       if (ceph_osd_op_mode_modify(iter->op.op))
        set_write();
     }
-    if (ceph_osd_op_mode_read(iter->op.op))
+    if (ceph_osd_op_mode_read(iter->op.op)) {
       set_read();
+      if (iter->op.op != CEPH_OSD_OP_STAT) {
+        set_read_data();
+      }
+    }
 
     // set READ flag if there are src_oids
     if (iter->soid.oid.name.length())
@@ -195,6 +211,7 @@ int OpInfo::set_from_op(
       // watch state (and may return early if the watch exists) or, in
       // the case of ping, is simply a read op.
       set_read();
+      set_read_data();
       // fall through
     case CEPH_OSD_OP_NOTIFY:
     case CEPH_OSD_OP_NOTIFY_ACK:

diff --git a/src/osd/osd_op_util.h b/src/osd/osd_op_util.h
index 5fb568e4073643b2d2b61e61e1e246b80f44fba3..07df4e1910f99871ccd3aac8eb725bec416b4e24 100644 (file)
--- a/src/osd/osd_op_util.h
+++ b/src/osd/osd_op_util.h
@@ -47,6 +47,7 @@ public:
 
   bool check_rmw(int flag) const ;
   bool may_read() const;
+  bool may_read_data() const;
   bool may_write() const;
   bool may_cache() const;
   bool rwordered_forced() const;
@@ -70,6 +71,7 @@ public:
   void set_skip_promote();
   void set_force_rwordered();
   void set_returnvec();
+  void set_read_data();
 
   int set_from_op(
     const MOSDOp *m,

diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index afbd0f0d0fae6cc7d9c6bd127201b4ed22ce61d5..4345f8e634ccfc85c9707053e0ad9d95b177255a 100644 (file)
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -369,6 +369,7 @@ enum {
   CEPH_OSD_RMW_FLAG_SKIP_PROMOTE      = (1 << 9),
   CEPH_OSD_RMW_FLAG_RWORDERED         = (1 << 10),
   CEPH_OSD_RMW_FLAG_RETURNVEC = (1 << 11),
+  CEPH_OSD_RMW_FLAG_READ_DATA  = (1 << 12),
 };