crimson/osd/object_context_loader: check on old watchers when loading obc

Currently, when the following events happen, an rbd image can't be opened:

1. osd stops;
2. rbd client crashes;
3. osd start again

In this case, when a client tries to open the rbd image again, it will
always see the old watchers and can't lock the rbd image.

Signed-off-by: Xuehan Xu <xxhdx1985126@gmail.com>

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index e80782eb51eddd8d7dc7d571719d7988ec44cba6..8ca71d3eb14f51b1067f5f5538941fbaa5a7cec9 100644 (file)
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -39,6 +39,7 @@
 #include "crimson/osd/osd_operations/peering_event.h"
 #include "crimson/osd/pg_recovery.h"
 #include "crimson/osd/replicated_recovery_backend.h"
+#include "crimson/osd/watch.h"
 
 using std::ostream;
 using std::set;
@@ -995,6 +996,22 @@ RWState::State PG::get_lock_type(const OpInfo &op_info)
   }
 }
 
+void PG::check_blocklisted_obc_watchers(
+  ObjectContextRef &obc)
+{
+  if (obc->watchers.empty()) {
+    for (auto &[src, winfo] : obc->obs.oi.watchers) {
+      auto watch = crimson::osd::Watch::create(
+        obc, winfo, src.second, this);
+      watch->disconnect();
+      auto [it, emplaced] = obc->watchers.emplace(src, std::move(watch));
+      assert(emplaced);
+      logger().debug("added watch for obj {}, client {}",
+        obc->get_oid(), src.second);
+    }
+  }
+}
+
 PG::load_obc_iertr::future<>
 PG::with_locked_obc(const hobject_t &hobj,
                     const OpInfo &op_info,
@@ -1004,13 +1021,17 @@ PG::with_locked_obc(const hobject_t &hobj,
     throw crimson::common::system_shutdown_exception();
   }
   const hobject_t oid = get_oid(hobj);
+  auto wrapper = [f=std::move(f), this](auto obc) {
+    check_blocklisted_obc_watchers(obc);
+    return f(obc);
+  };
   switch (get_lock_type(op_info)) {
   case RWState::RWREAD:
-      return obc_loader.with_obc<RWState::RWREAD>(oid, std::move(f));
+      return obc_loader.with_obc<RWState::RWREAD>(oid, std::move(wrapper));
   case RWState::RWWRITE:
-      return obc_loader.with_obc<RWState::RWWRITE>(oid, std::move(f));
+      return obc_loader.with_obc<RWState::RWWRITE>(oid, std::move(wrapper));
   case RWState::RWEXCL:
-      return obc_loader.with_obc<RWState::RWEXCL>(oid, std::move(f));
+      return obc_loader.with_obc<RWState::RWEXCL>(oid, std::move(wrapper));
   default:
     ceph_abort();
   };

diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
index a353634ba19f01914863aa3281a03ef05428a127..1c9bd8e3150af972e03f9d6a27754389e3cbb167 100644 (file)
--- a/src/crimson/osd/pg.h
+++ b/src/crimson/osd/pg.h
@@ -589,6 +589,7 @@ private:
   interruptible_future<> repair_object(
     const hobject_t& oid,
     eversion_t& v);
+  void check_blocklisted_obc_watchers(ObjectContextRef &obc);
 
 private:
   PG_OSDMapGate osdmap_gate;

diff --git a/src/crimson/osd/watch.cc b/src/crimson/osd/watch.cc
index 24377927735f2b08d6f2e62afb13615b27712d6f..9a899d6eccf6ec0040085a8b3f84064ba49913a5 100644 (file)
--- a/src/crimson/osd/watch.cc
+++ b/src/crimson/osd/watch.cc
@@ -90,6 +90,13 @@ seastar::future<> Watch::connect(crimson::net::ConnectionFRef conn, bool)
   return seastar::now();
 }
 
+void Watch::disconnect()
+{
+  ceph_assert(!conn);
+  timeout_timer.cancel();
+  timeout_timer.arm(std::chrono::seconds{winfo.timeout_seconds});
+}
+
 seastar::future<> Watch::send_notify_msg(NotifyRef notify)
 {
   logger().info("{} for notify(id={})", __func__, notify->ninfo.notify_id);

diff --git a/src/crimson/osd/watch.h b/src/crimson/osd/watch.h
index e2d10aa11267f011b48601c1549a2624172f7097..84c43b3fec4035e8f532e16bec382892bb1961c8 100644 (file)
--- a/src/crimson/osd/watch.h
+++ b/src/crimson/osd/watch.h
@@ -68,6 +68,7 @@ public:
   ~Watch();
 
   seastar::future<> connect(crimson::net::ConnectionFRef, bool);
+  void disconnect();
   bool is_alive() const {
     return true;
   }