osd/ReplicatedPG: redirect reads instead of promoting when full

If the cache pool is full, we are processing a read op, and we would
otherwise promote, redirect instead.  This lets us continue to process the
op without blocking or making the cache pool any more full than it is.

Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index bceaa172596a3e5e5a6f2e338fa794dd65d79ee3..b9b5c5e8e2bf1a46acd6fb4e56a49fc434e6d250 100644 (file)
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -1197,7 +1197,7 @@ void ReplicatedPG::do_op(OpRequestRef op)
   }
 
   if ((m->get_flags() & CEPH_OSD_FLAG_IGNORE_CACHE) == 0 &&
-      maybe_handle_cache(op, obc, r, missing_oid))
+      maybe_handle_cache(op, write_ordered, obc, r, missing_oid, false))
     return;
 
   if (r) {
@@ -1264,7 +1264,7 @@ void ReplicatedPG::do_op(OpRequestRef op)
          // missing the specific snap we need; requeue and wait.
          wait_for_missing_object(wait_oid, op);
        } else if (r) {
-         if (!maybe_handle_cache(op, sobc, r, wait_oid, true))
+         if (!maybe_handle_cache(op, write_ordered, sobc, r, wait_oid, true))
            osd->reply_op_error(op, r);
        } else if (sobc->obs.oi.is_whiteout()) {
          osd->reply_op_error(op, -ENOENT);
@@ -1324,7 +1324,7 @@ void ReplicatedPG::do_op(OpRequestRef op)
          // missing the specific snap we need; requeue and wait.
          wait_for_missing_object(wait_oid, op);
        } else if (r) {
-         if (!maybe_handle_cache(op, sobc, r, wait_oid, true))
+         if (!maybe_handle_cache(op, write_ordered, sobc, r, wait_oid, true))
            osd->reply_op_error(op, r);
        } else {
          dout(10) << " clone_oid " << clone_oid << " obc " << sobc << dendl;
@@ -1382,7 +1382,9 @@ void ReplicatedPG::do_op(OpRequestRef op)
   execute_ctx(ctx);
 }
 
-bool ReplicatedPG::maybe_handle_cache(OpRequestRef op, ObjectContextRef obc,
+bool ReplicatedPG::maybe_handle_cache(OpRequestRef op,
+                                     bool write_ordered,
+                                     ObjectContextRef obc,
                                       int r, const hobject_t& missing_oid,
                                      bool must_promote)
 {
@@ -1415,6 +1417,15 @@ bool ReplicatedPG::maybe_handle_cache(OpRequestRef op, ObjectContextRef obc,
     if (obc.get() && obc->obs.exists) {
       return false;
     }
+    if (agent_state &&
+       agent_state->evict_mode == TierAgentState::EVICT_MODE_FULL) {
+      if (!op->may_write() && !op->may_cache() && !write_ordered) {
+       dout(20) << __func__ << " cache pool full, redirecting read" << dendl;
+       do_cache_redirect(op, obc);
+       return true;
+      }
+      // FIXME: do something clever with writes.
+    }
     if (!must_promote && can_skip_promote(op, obc)) {
       return false;
     }
@@ -4431,7 +4442,7 @@ int ReplicatedPG::_rollback_to(OpContext *ctx, ceph_osd_op& op)
     wait_for_missing_object(missing_oid, ctx->op);
     return ret;
   }
-  if (maybe_handle_cache(ctx->op, rollback_to, ret, missing_oid, true)) {
+  if (maybe_handle_cache(ctx->op, true, rollback_to, ret, missing_oid, true)) {
     // promoting the rollback src, presumably
     return -EAGAIN;
   }

diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h
index 60faac8d9ab3f3e16bd40bbda88877c3fe7e3ac2..777f30925811146f9dff586a7379b2c7752a3ef3 100644 (file)
--- a/src/osd/ReplicatedPG.h
+++ b/src/osd/ReplicatedPG.h
@@ -910,9 +910,11 @@ protected:
    * This helper function is called from do_op if the ObjectContext lookup fails.
    * @returns true if the caching code is handling the Op, false otherwise.
    */
-  inline bool maybe_handle_cache(OpRequestRef op, ObjectContextRef obc, int r,
+  inline bool maybe_handle_cache(OpRequestRef op,
+                                bool write_ordered,
+                                ObjectContextRef obc, int r,
                                 const hobject_t& missing_oid,
-                                bool must_promote = false);
+                                bool must_promote);
   /**
    * This helper function tells the client to redirect their request elsewhere.
    */