ReplicatedPG: allow maybe_handle_cache to return status detailing what happened

Signed-off-by: Samuel Just <sjust@redhat.com>

diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index e499f3faa3970da0e8fa8b00467d82db4b2f1acb..0554d549e6165b4e9dbfbd8dcf7c46c77a5149cd 100644 (file)
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -1937,12 +1937,14 @@ void ReplicatedPG::do_op(OpRequestRef& op)
   execute_ctx(ctx);
 }
 
-bool ReplicatedPG::maybe_handle_cache(OpRequestRef op,
-                                     bool write_ordered,
-                                     ObjectContextRef obc,
-                                      int r, const hobject_t& missing_oid,
-                                     bool must_promote,
-                                     bool in_hit_set)
+ReplicatedPG::cache_result_t ReplicatedPG::maybe_handle_cache_detail(
+  OpRequestRef op,
+  bool write_ordered,
+  ObjectContextRef obc,
+  int r, const hobject_t& missing_oid,
+  bool must_promote,
+  bool in_hit_set,
+  ObjectContextRef *promote_obc)
 {
   if (op &&
       op->get_req() &&
@@ -1950,11 +1952,11 @@ bool ReplicatedPG::maybe_handle_cache(OpRequestRef op,
       (static_cast<MOSDOp *>(op->get_req())->get_flags() &
        CEPH_OSD_FLAG_IGNORE_CACHE)) {
     dout(20) << __func__ << ": ignoring cache due to flag" << dendl;
-    return false;
+    return cache_result_t::NOOP;
   }
   // return quickly if caching is not enabled
   if (pool.info.cache_mode == pg_pool_t::CACHEMODE_NONE)
-    return false;
+    return cache_result_t::NOOP;
 
   must_promote = must_promote || op->need_promote();
 
@@ -1976,24 +1978,24 @@ bool ReplicatedPG::maybe_handle_cache(OpRequestRef op,
   if (obc.get() && obc->is_blocked() && write_ordered) {
     // we're already doing something with this object
     dout(20) << __func__ << " blocked on " << obc->obs.oi.soid << dendl;
-    return false;
+    return cache_result_t::NOOP;
   }
 
   if (r == -ENOENT && missing_oid == hobject_t()) {
     // we know this object is logically absent (e.g., an undefined clone)
-    return false;
+    return cache_result_t::NOOP;
   }
 
   if (obc.get() && obc->obs.exists) {
     osd->logger->inc(l_osd_op_cache_hit);
-    return false;
+    return cache_result_t::NOOP;
   }
   
   MOSDOp *m = static_cast<MOSDOp*>(op->get_req());
   const object_locator_t& oloc = m->get_object_locator();
 
   if (op->need_skip_handle_cache()) {
-    return false;
+    return cache_result_t::NOOP;
   }
 
   // older versions do not proxy the feature bits.
@@ -2012,35 +2014,39 @@ bool ReplicatedPG::maybe_handle_cache(OpRequestRef op,
        if (can_proxy_read) {
          dout(20) << __func__ << " cache pool full, proxying read" << dendl;
          do_proxy_read(op);
+         return cache_result_t::HANDLED_PROXY;
        } else {
          dout(20) << __func__ << " cache pool full, redirect read" << dendl;
          do_cache_redirect(op);
+         return cache_result_t::HANDLED_REDIRECT;
        }
-       return true;
+       assert(0 == "unreachable");
       }
       dout(20) << __func__ << " cache pool full, waiting" << dendl;
-      waiting_for_cache_not_full.push_back(op);
-      return true;
+      block_write_on_full_cache(missing_oid, op);
+      return cache_result_t::BLOCKED_FULL;
     }
 
     if (!hit_set) {
-      promote_object(obc, missing_oid, oloc, op);
-      return true;
+      promote_object(obc, missing_oid, oloc, op, promote_obc);
+      return cache_result_t::BLOCKED_PROMOTE;
     } else if (op->may_write() || op->may_cache()) {
       if (can_proxy_write && !must_promote) {
         do_proxy_write(op, missing_oid);
       } else {
        // promote if can't proxy the write
-       promote_object(obc, missing_oid, oloc, op);
-       return true;
+       promote_object(obc, missing_oid, oloc, op, promote_obc);
+       return cache_result_t::BLOCKED_PROMOTE;
       }
 
       // Promote too?
       if (!op->need_skip_promote()) {
         maybe_promote(obc, missing_oid, oloc, in_hit_set,
                      pool.info.min_write_recency_for_promote,
-                     OpRequestRef());
+                     OpRequestRef(),
+                     promote_obc);
       }
+      return cache_result_t::HANDLED_PROXY;
     } else {
       bool did_proxy_read = false;
       if (can_proxy_read && !must_promote) {
@@ -2055,7 +2061,9 @@ bool ReplicatedPG::maybe_handle_cache(OpRequestRef op,
        if (!did_proxy_read) {
          wait_for_blocked_object(obc->obs.oi.soid, op);
        }
-        return true;
+       if (promote_obc)
+         *promote_obc = obc;
+        return cache_result_t::BLOCKED_PROMOTE;
       }
 
       // Promote too?
@@ -2063,32 +2071,39 @@ bool ReplicatedPG::maybe_handle_cache(OpRequestRef op,
       if (!op->need_skip_promote()) {
         promoted = maybe_promote(obc, missing_oid, oloc, in_hit_set,
                                  pool.info.min_read_recency_for_promote,
-                                 promote_op);
+                                 promote_op, promote_obc);
       }
       if (!promoted && !did_proxy_read) {
        // redirect the op if it's not proxied and not promoting
        do_cache_redirect(op);
+       return cache_result_t::HANDLED_REDIRECT;
+      } else if (did_proxy_read) {
+       return cache_result_t::HANDLED_PROXY;
+      } else {
+       assert(promoted);
+       return cache_result_t::BLOCKED_PROMOTE;
       }
     }
-    return true;
+    assert(0 == "unreachable");
+    return cache_result_t::NOOP;
 
   case pg_pool_t::CACHEMODE_FORWARD:
     do_cache_redirect(op);
-    return true;
+    return cache_result_t::HANDLED_REDIRECT;
 
   case pg_pool_t::CACHEMODE_READONLY:
     // TODO: clean this case up
     if (!obc.get() && r == -ENOENT) {
       // we don't have the object and op's a read
-      promote_object(obc, missing_oid, oloc, op);
-      return true;
+      promote_object(obc, missing_oid, oloc, op, promote_obc);
+      return cache_result_t::BLOCKED_PROMOTE;
     }
     if (!r) { // it must be a write
       do_cache_redirect(op);
-      return true;
+      return cache_result_t::HANDLED_REDIRECT;
     }
     // crap, there was a failure of some kind
-    return false;
+    return cache_result_t::NOOP;
 
   case pg_pool_t::CACHEMODE_READFORWARD:
     // Do writeback to the cache tier for writes
@@ -2096,16 +2111,16 @@ bool ReplicatedPG::maybe_handle_cache(OpRequestRef op,
       if (agent_state &&
          agent_state->evict_mode == TierAgentState::EVICT_MODE_FULL) {
        dout(20) << __func__ << " cache pool full, waiting" << dendl;
-       waiting_for_cache_not_full.push_back(op);
-       return true;
+       block_write_on_full_cache(missing_oid, op);
+       return cache_result_t::BLOCKED_FULL;
       }
-      promote_object(obc, missing_oid, oloc, op);
-      return true;
+      promote_object(obc, missing_oid, oloc, op, promote_obc);
+      return cache_result_t::BLOCKED_PROMOTE;
     }
 
     // If it is a read, we can read, we need to forward it
     do_cache_redirect(op);
-    return true;
+    return cache_result_t::HANDLED_REDIRECT;
 
   case pg_pool_t::CACHEMODE_READPROXY:
     // Do writeback to the cache tier for writes
@@ -2113,21 +2128,21 @@ bool ReplicatedPG::maybe_handle_cache(OpRequestRef op,
       if (agent_state &&
          agent_state->evict_mode == TierAgentState::EVICT_MODE_FULL) {
        dout(20) << __func__ << " cache pool full, waiting" << dendl;
-       waiting_for_cache_not_full.push_back(op);
-       return true;
+       block_write_on_full_cache(missing_oid, op);
+       return cache_result_t::BLOCKED_FULL;
       }
-      promote_object(obc, missing_oid, oloc, op);
-      return true;
+      promote_object(obc, missing_oid, oloc, op, promote_obc);
+      return cache_result_t::BLOCKED_PROMOTE;
     }
 
     // If it is a read, we can read, we need to proxy it
     do_proxy_read(op);
-    return true;
+    return cache_result_t::HANDLED_PROXY;
 
   default:
     assert(0 == "unrecognized cache_mode");
   }
-  return false;
+  return cache_result_t::NOOP;
 }
 
 bool ReplicatedPG::maybe_promote(ObjectContextRef obc,
@@ -2135,19 +2150,20 @@ bool ReplicatedPG::maybe_promote(ObjectContextRef obc,
                                 const object_locator_t& oloc,
                                 bool in_hit_set,
                                 uint32_t recency,
-                                OpRequestRef promote_op)
+                                OpRequestRef promote_op,
+                                ObjectContextRef *promote_obc)
 {
   dout(20) << __func__ << " missing_oid " << missing_oid
           << "  in_hit_set " << in_hit_set << dendl;
 
   switch (recency) {
   case 0:
-    promote_object(obc, missing_oid, oloc, promote_op);
+    promote_object(obc, missing_oid, oloc, promote_op, promote_obc);
     break;
   case 1:
     // Check if in the current hit set
     if (in_hit_set) {
-      promote_object(obc, missing_oid, oloc, promote_op);
+      promote_object(obc, missing_oid, oloc, promote_op, promote_obc);
     } else {
       // not promoting
       return false;
@@ -2570,7 +2586,8 @@ public:
 void ReplicatedPG::promote_object(ObjectContextRef obc,
                                  const hobject_t& missing_oid,
                                  const object_locator_t& oloc,
-                                 OpRequestRef op)
+                                 OpRequestRef op,
+                                 ObjectContextRef *promote_obc)
 {
   hobject_t hoid = obc ? obc->obs.oi.soid : missing_oid;
   assert(hoid != hobject_t());
@@ -2591,6 +2608,8 @@ void ReplicatedPG::promote_object(ObjectContextRef obc,
     assert(missing_oid != hobject_t());
     obc = get_object_context(missing_oid, true);
   }
+  if (promote_obc)
+    *promote_obc = obc;
 
   /*
    * Before promote complete, if there are  proxy-reads for the object,

diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h
index c122dcc8f6dcf9609ad662c26fa26e332500ef57..1fcb9ba2fe90844a7643fdd789e2071beab1ac82 100644 (file)
--- a/src/osd/ReplicatedPG.h
+++ b/src/osd/ReplicatedPG.h
@@ -1170,16 +1170,42 @@ protected:
                                   bool force_changesize=false);
   void add_interval_usage(interval_set<uint64_t>& s, object_stat_sum_t& st);
 
+
+  enum class cache_result_t {
+    NOOP,
+    BLOCKED_FULL,
+    BLOCKED_PROMOTE,
+    HANDLED_PROXY,
+    HANDLED_REDIRECT,
+  };
+  cache_result_t maybe_handle_cache_detail(OpRequestRef op,
+                                          bool write_ordered,
+                                          ObjectContextRef obc, int r,
+                                          const hobject_t& missing_oid,
+                                          bool must_promote,
+                                          bool in_hit_set,
+                                          ObjectContextRef *promote_obc);
   /**
    * This helper function is called from do_op if the ObjectContext lookup fails.
    * @returns true if the caching code is handling the Op, false otherwise.
    */
-  inline bool maybe_handle_cache(OpRequestRef op,
-                                bool write_ordered,
-                                ObjectContextRef obc, int r,
-                                const hobject_t& missing_oid,
-                                bool must_promote,
-                                bool in_hit_set = false);
+  bool maybe_handle_cache(OpRequestRef op,
+                         bool write_ordered,
+                         ObjectContextRef obc, int r,
+                         const hobject_t& missing_oid,
+                         bool must_promote,
+                         bool in_hit_set = false) {
+    return cache_result_t::NOOP != maybe_handle_cache_detail(
+      op,
+      write_ordered,
+      obc,
+      r,
+      missing_oid,
+      must_promote,
+      in_hit_set,
+      nullptr);
+  }
+
   /**
    * This helper function checks if a promotion is needed.
    */
@@ -1188,7 +1214,8 @@ protected:
                     const object_locator_t& oloc,
                     bool in_hit_set,
                     uint32_t recency,
-                    OpRequestRef promote_op);
+                    OpRequestRef promote_op,
+                    ObjectContextRef *promote_obc = nullptr);
   /**
    * This helper function tells the client to redirect their request elsewhere.
    */
@@ -1199,10 +1226,13 @@ protected:
    * this is a noop.  If a future user wants to be able to distinguish
    * these cases, a return value should be added.
    */
-  void promote_object(ObjectContextRef obc,            ///< [optional] obc
-                     const hobject_t& missing_object, ///< oid (if !obc)
-                     const object_locator_t& oloc,    ///< locator for obc|oid
-                     OpRequestRef op);                ///< [optional] client op
+  void promote_object(
+    ObjectContextRef obc,            ///< [optional] obc
+    const hobject_t& missing_object, ///< oid (if !obc)
+    const object_locator_t& oloc,    ///< locator for obc|oid
+    OpRequestRef op,                 ///< [optional] client op
+    ObjectContextRef *promote_obc = nullptr ///< [optional] new obc for object
+    );
 
   int prepare_transaction(OpContext *ctx);
   list<pair<OpRequestRef, OpContext*> > in_progress_async_reads;