osd: add 'proxy' cache mode

The 'forward' mode is flawed for the same reason that redirects
are flawed: they allow the reordering of requests.  Add a proxy
mode that can be used instead of forward.  (Later we can consider
dropping forward entirely?)

Fixes: #12814
Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index a003e3ac3114865f02f8a9f56d9b72121654826e..7a22245b6b2a2295b5bdd25912bd68581b0f3277 100644 (file)
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -795,7 +795,7 @@ COMMAND("osd tier rm " \
        "osd", "rw", "cli,rest")
 COMMAND("osd tier cache-mode " \
        "name=pool,type=CephPoolname " \
-       "name=mode,type=CephChoices,strings=none|writeback|forward|readonly|readforward|readproxy", \
+       "name=mode,type=CephChoices,strings=none|writeback|forward|readonly|readforward|proxy|readproxy", \
        "specify the caching mode for cache tier <pool>", "osd", "rw", "cli,rest")
 COMMAND("osd tier set-overlay " \
        "name=pool,type=CephPoolname " \

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 4051132336a2d70ed41e03fb331d1983aa4c04d3..dba25a942ff73f54f5e00985250785fbc590b3d9 100644 (file)
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -7313,15 +7313,17 @@ done:
      *  writeback:  Cache writes, promote reads from base pool
      *  readonly:   Forward writes to base pool
      *  readforward: Writes are in writeback mode, Reads are in forward mode
+     *  proxy:       Proxy all reads and writes to base pool
      *  readproxy:   Writes are in writeback mode, Reads are in proxy mode
      *
      * Hence, these are the allowed transitions:
      *
      *  none -> any
-     *  forward -> readforward || readproxy || writeback || any IF num_objects_dirty == 0
-     *  readforward -> forward || readproxy || writeback || any IF num_objects_dirty == 0
-     *  readproxy -> forward || readforward || writeback || any IF num_objects_dirty == 0
-     *  writeback -> readforward || readproxy || forward
+     *  forward -> proxy || readforward || readproxy || writeback || any IF num_objects_dirty == 0
+     *  proxy -> forward || readforward || readproxy || writeback || any IF num_objects_dirty == 0
+     *  readforward -> forward || proxy || readproxy || writeback || any IF num_objects_dirty == 0
+     *  readproxy -> forward || proxy || readforward || writeback || any IF num_objects_dirty == 0
+     *  writeback -> readforward || readproxy || forward || proxy
      *  readonly -> any
      */
 
@@ -7331,6 +7333,7 @@ done:
 
     if (p->cache_mode == pg_pool_t::CACHEMODE_WRITEBACK &&
         (mode != pg_pool_t::CACHEMODE_FORWARD &&
+         mode != pg_pool_t::CACHEMODE_PROXY &&
          mode != pg_pool_t::CACHEMODE_READFORWARD &&
          mode != pg_pool_t::CACHEMODE_READPROXY)) {
       ss << "unable to set cache-mode '" << pg_pool_t::get_cache_mode_name(mode)
@@ -7338,6 +7341,8 @@ done:
          << "' pool; only '"
          << pg_pool_t::get_cache_mode_name(pg_pool_t::CACHEMODE_FORWARD)
         << "','"
+         << pg_pool_t::get_cache_mode_name(pg_pool_t::CACHEMODE_PROXY)
+        << "','"
          << pg_pool_t::get_cache_mode_name(pg_pool_t::CACHEMODE_READFORWARD)
         << "','"
          << pg_pool_t::get_cache_mode_name(pg_pool_t::CACHEMODE_READPROXY)
@@ -7348,16 +7353,25 @@ done:
     if ((p->cache_mode == pg_pool_t::CACHEMODE_READFORWARD &&
         (mode != pg_pool_t::CACHEMODE_WRITEBACK &&
          mode != pg_pool_t::CACHEMODE_FORWARD &&
+         mode != pg_pool_t::CACHEMODE_PROXY &&
          mode != pg_pool_t::CACHEMODE_READPROXY)) ||
 
         (p->cache_mode == pg_pool_t::CACHEMODE_READPROXY &&
         (mode != pg_pool_t::CACHEMODE_WRITEBACK &&
          mode != pg_pool_t::CACHEMODE_FORWARD &&
-         mode != pg_pool_t::CACHEMODE_READFORWARD)) ||
+         mode != pg_pool_t::CACHEMODE_READFORWARD &&
+         mode != pg_pool_t::CACHEMODE_PROXY)) ||
+
+        (p->cache_mode == pg_pool_t::CACHEMODE_PROXY &&
+        (mode != pg_pool_t::CACHEMODE_WRITEBACK &&
+         mode != pg_pool_t::CACHEMODE_FORWARD &&
+         mode != pg_pool_t::CACHEMODE_READFORWARD &&
+         mode != pg_pool_t::CACHEMODE_READPROXY)) ||
 
         (p->cache_mode == pg_pool_t::CACHEMODE_FORWARD &&
         (mode != pg_pool_t::CACHEMODE_WRITEBACK &&
          mode != pg_pool_t::CACHEMODE_READFORWARD &&
+         mode != pg_pool_t::CACHEMODE_PROXY &&
          mode != pg_pool_t::CACHEMODE_READPROXY))) {
 
       const pool_stat_t& tier_stats =

diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index c4efd5c746befbd3fe36be467db2ed48e3dee07b..abefb0066c74ba9eb5371a07329744edfd16bcd2 100644 (file)
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -2255,6 +2255,7 @@ ReplicatedPG::cache_result_t ReplicatedPG::maybe_handle_cache_detail(
     return cache_result_t::NOOP;
 
   case pg_pool_t::CACHEMODE_FORWARD:
+    // FIXME: this mode allows requests to be reordered.
     do_cache_redirect(op);
     return cache_result_t::HANDLED_REDIRECT;
 
@@ -2289,6 +2290,28 @@ ReplicatedPG::cache_result_t ReplicatedPG::maybe_handle_cache_detail(
     do_cache_redirect(op);
     return cache_result_t::HANDLED_REDIRECT;
 
+  case pg_pool_t::CACHEMODE_PROXY:
+    if (!must_promote) {
+      if (op->may_write() || op->may_cache() || write_ordered) {
+       if (can_proxy_write) {
+         do_proxy_write(op, missing_oid);
+         return cache_result_t::HANDLED_PROXY;
+       }
+      } else {
+       do_proxy_read(op);
+       return cache_result_t::HANDLED_PROXY;
+      }
+    }
+    // ugh, we're forced to promote.
+    if (agent_state &&
+       agent_state->evict_mode == TierAgentState::EVICT_MODE_FULL) {
+      dout(20) << __func__ << " cache pool full, waiting" << dendl;
+      block_write_on_full_cache(missing_oid, op);
+      return cache_result_t::BLOCKED_FULL;
+    }
+    promote_object(obc, missing_oid, oloc, op, promote_obc);
+    return cache_result_t::BLOCKED_PROMOTE;
+
   case pg_pool_t::CACHEMODE_READPROXY:
     // Do writeback to the cache tier for writes
     if (op->may_write() || write_ordered || must_promote) {

diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index adfe50ff3bc8599a3a0078b17868c600596c7c56..d53aba1b8597ac37f9f6c981ca8347dde9371e89 100644 (file)
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -1125,7 +1125,8 @@ struct pg_pool_t {
     CACHEMODE_FORWARD = 2,               ///< forward if not in cache
     CACHEMODE_READONLY = 3,              ///< handle reads, forward writes [not strongly consistent]
     CACHEMODE_READFORWARD = 4,           ///< forward reads, write to cache flush later
-    CACHEMODE_READPROXY = 5              ///< proxy reads, write to cache flush later
+    CACHEMODE_READPROXY = 5,             ///< proxy reads, write to cache flush later
+    CACHEMODE_PROXY = 6,                 ///< proxy if not in cache
   } cache_mode_t;
   static const char *get_cache_mode_name(cache_mode_t m) {
     switch (m) {
@@ -1135,6 +1136,7 @@ struct pg_pool_t {
     case CACHEMODE_READONLY: return "readonly";
     case CACHEMODE_READFORWARD: return "readforward";
     case CACHEMODE_READPROXY: return "readproxy";
+    case CACHEMODE_PROXY: return "proxy";
     default: return "unknown";
     }
   }
@@ -1151,6 +1153,8 @@ struct pg_pool_t {
       return CACHEMODE_READFORWARD;
     if (s == "readproxy")
       return CACHEMODE_READPROXY;
+    if (s == "proxy")
+      return CACHEMODE_PROXY;
     return (cache_mode_t)-1;
   }
   const char *get_cache_mode_name() const {
@@ -1161,6 +1165,7 @@ struct pg_pool_t {
     case CACHEMODE_NONE:
     case CACHEMODE_FORWARD:
     case CACHEMODE_READONLY:
+    case CACHEMODE_PROXY:
       return false;
     case CACHEMODE_WRITEBACK:
     case CACHEMODE_READFORWARD: