crimson/net: make io-handler in/out dispatching aware of being switched

To prevent the previous shard-states racing on the shared data
structures after switching to a new shard-states.

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>

diff --git a/src/crimson/net/io_handler.cc b/src/crimson/net/io_handler.cc
index 2b7dee0c2d19f424dd98a339ad587f05a4e9702a..3e156ca67301eba765eaf4e191e5e212728ee525 100644 (file)
--- a/src/crimson/net/io_handler.cc
+++ b/src/crimson/net/io_handler.cc
@@ -509,6 +509,9 @@ IOHandler::do_out_dispatch(shard_states_t &ctx)
      case io_state_t::drop:
       ctx.exit_out_dispatching("dropped", conn);
       return seastar::make_ready_future<stop_t>(stop_t::yes);
+     case io_state_t::switched:
+      ctx.exit_out_dispatching("switched", conn);
+      return seastar::make_ready_future<stop_t>(stop_t::yes);
      default:
       ceph_abort("impossible");
     }
@@ -536,8 +539,13 @@ IOHandler::do_out_dispatch(shard_states_t &ctx)
       handshake_listener->notify_out_fault(
           "do_out_dispatch", eptr, states);
     } else {
-      logger().info("{} do_out_dispatch(): fault at {} -- {}",
-                    conn, io_state, e.what());
+      if (io_state != io_state_t::switched) {
+        logger().info("{} do_out_dispatch(): fault at {}, {} -- {}",
+                      conn, io_state, io_stat_printer{*this}, e.what());
+      } else {
+        logger().info("{} do_out_dispatch(): fault at {} -- {}",
+                      conn, io_state, e.what());
+      }
     }
 
     return do_out_dispatch(ctx);
@@ -781,8 +789,13 @@ void IOHandler::do_in_dispatch()
         handshake_listener->notify_out_fault(
             "do_in_dispatch", eptr, states);
       } else {
-        logger().info("{} do_in_dispatch(): fault at {} -- {}",
-                      conn, io_state, e_what);
+        if (io_state != io_state_t::switched) {
+          logger().info("{} do_in_dispatch(): fault at {}, {} -- {}",
+                        conn, io_state, io_stat_printer{*this}, e_what);
+        } else {
+          logger().info("{} do_in_dispatch(): fault at {} -- {}",
+                        conn, io_state, e_what);
+        }
       }
     }).finally([&ctx] {
       ctx.exit_in_dispatching();
@@ -854,4 +867,21 @@ IOHandler::shard_states_t::wait_io_exit_dispatching()
   ).discard_result();
 }
 
+IOHandler::shard_states_ref_t
+IOHandler::shard_states_t::create_from_previous(
+    shard_states_t &prv_states,
+    seastar::shard_id new_sid)
+{
+  auto io_state = prv_states.io_state;
+  assert(io_state != io_state_t::open);
+  auto ret = shard_states_t::create(new_sid, io_state);
+  if (io_state == io_state_t::drop) {
+    // the new gate should not never be used
+    auto fut = ret->gate.close();
+    ceph_assert_always(fut.available());
+  }
+  prv_states.set_io_state(io_state_t::switched);
+  return ret;
+}
+
 } // namespace crimson::net

diff --git a/src/crimson/net/io_handler.h b/src/crimson/net/io_handler.h
index cd85604dafdc1476afd6d04ac9262f6dfa2f460a..07f4c1cb4262a020a5e8d186ee5213b0fe36978d 100644 (file)
--- a/src/crimson/net/io_handler.h
+++ b/src/crimson/net/io_handler.h
@@ -168,10 +168,12 @@ public:
    * io behavior accordingly.
    */
   enum class io_state_t : uint8_t {
-    none,  // no IO is possible as the connection is not available to the user yet.
-    delay, // IO is delayed until open.
-    open,  // Dispatch In and Out concurrently.
-    drop   // Drop IO as the connection is closed.
+    none,    // no IO is possible as the connection is not available to the user yet.
+    delay,   // IO is delayed until open.
+    open,    // Dispatch In and Out concurrently.
+    drop,    // Drop IO as the connection is closed.
+    switched // IO is switched to a different core
+             // (is moved to maybe_prv_shard_states)
   };
   friend class fmt::formatter<io_state_t>;
 
@@ -272,6 +274,8 @@ public:
         out_dispatching = true;
         return true;
       case io_state_t::drop:
+        [[fallthrough]];
+      case io_state_t::switched:
         // do not dispatch out
         return false;
       default:
@@ -301,7 +305,8 @@ public:
     bool assert_closed_and_exit() const {
       assert(seastar::this_shard_id() == sid);
       if (gate.is_closed()) {
-        ceph_assert_always(io_state == io_state_t::drop);
+        ceph_assert_always(io_state == io_state_t::drop ||
+                           io_state == io_state_t::switched);
         ceph_assert_always(!out_dispatching);
         ceph_assert_always(!out_exit_dispatching);
         ceph_assert_always(!in_exit_dispatching);
@@ -316,6 +321,9 @@ public:
       return std::make_unique<shard_states_t>(sid, state);
     }
 
+    static shard_states_ref_t create_from_previous(
+        shard_states_t &prv_states, seastar::shard_id new_sid);
+
   private:
     const seastar::shard_id sid;
     io_state_t io_state;
@@ -476,6 +484,9 @@ struct fmt::formatter<crimson::net::IOHandler::io_state_t>
     case drop:
       name = "drop";
       break;
+    case switched:
+      name = "switched";
+      break;
     }
     return formatter<string_view>::format(name, ctx);
   }