osd: Adjust lock type selection in PrimaryLogPG::get_rw_locks

All writes in EC are asynchronous (even if they are only updating metadata), so Cls reads in EC must use the RWEXCL lock to eliminate the possibility of race conditions.
This resolves failures with Cls queue tests on EC pools where read operations were observed overtaking in-flight writes.

Fixes: https://tracker.ceph.com/issues/74531
Signed-off-by: Matty Williams <Matty.Williams@ibm.com>

diff --git a/src/osd/OpRequest.h b/src/osd/OpRequest.h
index 255aa59d5307be6b097600a236092285152c6769..57304f9e3a9a64cef17a5602f731e6d8f71adee6 100644 (file)
--- a/src/osd/OpRequest.h
+++ b/src/osd/OpRequest.h
@@ -37,6 +37,7 @@ public:
   bool check_rmw(int flag) const { return op_info.check_rmw(flag); }
   bool may_read() const { return op_info.may_read(); }
   bool may_read_data() const { return op_info.may_read_data(); }
+  bool may_read_data_for_ec() const { return op_info.may_read_data_for_ec(); }
   bool may_write() const { return op_info.may_write(); }
   bool may_cache() const { return op_info.may_cache(); }
   bool rwordered_forced() const { return op_info.rwordered_forced(); }

diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index 2200f5e031036388f5111199500644efccc5d82b..7f69f63ea618e3b009e8553eb2b3aed8d31cd83a 100644 (file)
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -1697,7 +1697,9 @@ bool PrimaryLogPG::get_rw_locks(bool write_ordered, OpContext *ctx)
    * to get the second.
    */
   if (write_ordered && ctx->op->may_read()) {
-    if (ctx->op->may_read_data()) {
+    // In EC, reads can overtake writes unless the RWEXCL lock is held
+    if (ctx->op->may_read_data() ||
+        (pool.info.is_erasure() && ctx->op->may_read_data_for_ec())) {
       ctx->lock_type = RWState::RWEXCL;
     } else {
       ctx->lock_type = RWState::RWWRITE;

diff --git a/src/osd/osd_op_util.cc b/src/osd/osd_op_util.cc
index 2c2ad8e3ec900bfe0c6983e828ed0187180856be..1bf4945e05128e6ecdca7ec8210595885ebfe23a 100644 (file)
--- a/src/osd/osd_op_util.cc
+++ b/src/osd/osd_op_util.cc
@@ -66,13 +66,24 @@ bool OpInfo::may_read_data() const {
   return check_rmw(CEPH_OSD_RMW_FLAG_READ_DATA);
 }
 
+bool OpInfo::may_read_data_for_ec() const
+{
+  return check_rmw(CEPH_OSD_RMW_FLAG_CLASS_READ_DATA);
+}
+
 void OpInfo::set_rmw_flags(int flags) {
   rmw_flags |= flags;
 }
 
 void OpInfo::set_read() { set_rmw_flags(CEPH_OSD_RMW_FLAG_READ); }
 void OpInfo::set_write() { set_rmw_flags(CEPH_OSD_RMW_FLAG_WRITE); }
-void OpInfo::set_class_read() { set_rmw_flags(CEPH_OSD_RMW_FLAG_CLASS_READ); }
+void OpInfo::set_class_read(const bool is_erasure) {
+  int flags = CEPH_OSD_RMW_FLAG_CLASS_READ;
+  if (is_erasure) {
+    flags |= CEPH_OSD_RMW_FLAG_CLASS_READ_DATA;
+  }
+  set_rmw_flags(flags); 
+}
 void OpInfo::set_class_write() { set_rmw_flags(CEPH_OSD_RMW_FLAG_CLASS_WRITE); }
 void OpInfo::set_pg_op() { set_rmw_flags(CEPH_OSD_RMW_FLAG_PGOP); }
 void OpInfo::set_cache() { set_rmw_flags(CEPH_OSD_RMW_FLAG_CACHE); }
@@ -202,8 +213,10 @@ int OpInfo::set_from_op(
        is_write = flags & CLS_METHOD_WR;
         bool is_promote = flags & CLS_METHOD_PROMOTE;
 
-       if (is_read)
-         set_class_read();
+       if (is_read) {
+         const bool is_erasure = pool && pool->is_erasure();
+         set_class_read(is_erasure);
+       }
        if (is_write)
          set_class_write();
         if (is_promote)

diff --git a/src/osd/osd_op_util.h b/src/osd/osd_op_util.h
index ba1acae4c9e42c9e3ce438f0942602e8c9213705..5a9ea77c9666a843ec94b5cb0dc8f0a75295c28b 100644 (file)
--- a/src/osd/osd_op_util.h
+++ b/src/osd/osd_op_util.h
@@ -48,6 +48,7 @@ public:
   bool check_rmw(int flag) const ;
   bool may_read() const;
   bool may_read_data() const;
+  bool may_read_data_for_ec() const;
   bool may_write() const;
   bool may_cache() const;
   bool rwordered_forced() const;
@@ -64,7 +65,7 @@ public:
   void set_read();
   void set_write();
   void set_cache();
-  void set_class_read();
+  void set_class_read(bool is_erasure);
   void set_class_write();
   void set_pg_op();
   void set_promote();

diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index 5377ffac182e052ef4e851037b9e77d260a226f3..6b5deb6a74ea29337e8dfcb684fa0084fce06d4a 100644 (file)
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -376,6 +376,8 @@ enum {
   CEPH_OSD_RMW_FLAG_RETURNVEC = (1 << 11),
   CEPH_OSD_RMW_FLAG_READ_DATA  = (1 << 12),
   CEPH_OSD_RMW_FLAG_EC_DIRECT_READ  = (1 << 13),
+  CEPH_OSD_RMW_FLAG_EC_SYNC_READ    = (1 << 14),
+  CEPH_OSD_RMW_FLAG_CLASS_READ_DATA = (1 << 15),
 };