From 574372409f76c2becee7264edc8a7dca150651af Mon Sep 17 00:00:00 2001
From: Xuehan Xu <xxhdx1985126@gmail.com>
Date: Sun, 7 Feb 2021 17:05:16 +0800
Subject: [PATCH] crimson/osd: interrupt all client_requests that are waiting
 for acquiring obc locks

Signed-off-by: Xuehan Xu <xxhdx1985126@gmail.com>
---
 src/crimson/osd/object_context.h | 24 ++++++++++++++++++++++++
 src/crimson/osd/pg.cc            | 12 +++++++++++-
 src/crimson/osd/pg.h             |  5 +++++
 3 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/src/crimson/osd/object_context.h b/src/crimson/osd/object_context.h
index be238851e6a92..ac80b6d5beeb1 100644
--- a/src/crimson/osd/object_context.h
+++ b/src/crimson/osd/object_context.h
@@ -107,7 +107,31 @@ private:
     });
   }
 
+  boost::intrusive::list_member_hook<> list_hook;
+  uint64_t list_link_cnt = 0;
+
 public:
+
+  template <typename ListType>
+  void append_to(ListType& list) {
+    if (list_link_cnt++ == 0) {
+      list.push_back(*this);
+    }
+  }
+
+  template <typename ListType>
+  void remove_from(ListType&& list) {
+    assert(list_link_cnt > 0);
+    if (--list_link_cnt == 0) {
+      list.erase(std::decay_t<ListType>::s_iterator_to(*this));
+    }
+  }
+
+  using obc_accessing_option_t = boost::intrusive::member_hook<
+    ObjectContext,
+    boost::intrusive::list_member_hook<>,
+    &ObjectContext::list_hook>;
+
   template<RWState::State Type, typename Func>
   auto with_lock(Func&& func) {
     switch (Type) {
diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index d40e957469db8..63c88f2b43979 100644
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -855,10 +855,13 @@ template<RWState::State State>
 PG::load_obc_ertr::future<>
 PG::with_head_obc(hobject_t oid, with_obc_func_t&& func)
 {
+  logger().debug("{} {}", __func__, oid);
+  boost::intrusive_ptr<PG> pgref{this};
   assert(oid.is_head());
   auto [obc, existed] = shard_services.obc_registry.get_cached_obc(oid);
+  obc->append_to(obc_set_accessing);
   return obc->with_lock<State>(
-    [oid=std::move(oid), existed=existed, obc=std::move(obc),
+    [oid=std::move(oid), existed=existed, obc=obc,
      func=std::move(func), this] {
     auto loaded = load_obc_ertr::make_ready_future<ObjectContextRef>(obc);
     if (existed) {
@@ -872,6 +875,9 @@ PG::with_head_obc(hobject_t oid, with_obc_func_t&& func)
     return loaded.safe_then([func=std::move(func)](auto obc) {
       return func(std::move(obc));
     });
+  }).finally([this, pgref, obc=std::move(obc)] {
+    logger().debug("with_head_obc: released {}", obc->get_oid());
+    obc->remove_from(obc_set_accessing);
   });
 }
 
@@ -1084,6 +1090,10 @@ seastar::future<> PG::stop()
 }
 
 void PG::on_change(ceph::os::Transaction &t) {
+  logger().debug("{}, {}", __func__, *this);
+  for (auto& obc : obc_set_accessing) {
+    obc.interrupt(::crimson::common::actingset_changed(is_primary()));
+  }
   recovery_backend->on_peering_interval_change(t);
   backend->on_actingset_changed({ is_primary() });
 }
diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
index 0910014756f9d..6a9ab301fe58a 100644
--- a/src/crimson/osd/pg.h
+++ b/src/crimson/osd/pg.h
@@ -490,6 +490,11 @@ public:
   using with_obc_func_t =
     std::function<load_obc_ertr::future<> (ObjectContextRef)>;
 
+  using obc_accessing_list_t = boost::intrusive::list<
+    ObjectContext,
+    ObjectContext::obc_accessing_option_t>;
+  obc_accessing_list_t obc_set_accessing;
+
   template<RWState::State State>
   load_obc_ertr::future<> with_head_obc(hobject_t oid, with_obc_func_t&& func);
 
-- 
2.39.5