rimson/net: cap outgoing message batch size to prevent seastar packet fragment overflow

During heavy backfill, sweep_out_pending_msgs_to_sent() could accumulate
more than 65535 buffer fragments in a single bufferlist, overflowing
seastar's uint16_t packet fragment counter and triggering an assertion.
Limit the sweep to crimson_osd_max_send_buffers fragments
per write; remaining messages are sent in subsequent dispatch iterations.

Fixes: https://tracker.ceph.com/issues/75600
Signed-off-by: Ronen Friedman <rfriedma@redhat.com>

diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in
index e5597266880c5dadf53944142f5f8992b46264ad..aa94ec0c43865838044aafed00ae5e7671fbd7dd 100644 (file)
--- a/src/common/options/crimson.yaml.in
+++ b/src/common/options/crimson.yaml.in
@@ -116,6 +116,17 @@ options:
   flags:
   - startup
 
+- name: crimson_osd_max_send_buffers
+  type: uint
+  level: advanced
+  default: 1024
+  min: 1
+  max: 65535
+  desc: Maximum buffer fragments to batch in a single outgoing network write
+  long_desc: Limits how many buffer::ptr entries are batched into one seastar
+             packet during the outgoing message sweep. Prevents overflow of the
+             seastar packet fragment counter (uint16_t).
+
 - name: crimson_allow_pg_split
   type: bool
   level: advanced

diff --git a/src/crimson/net/io_handler.cc b/src/crimson/net/io_handler.cc
index 704bbe1230bf3cfb0e751953a342367e5e8e29f2..8e4db300f10011546b8575a228ecbbac7b1d15e2 100644 (file)
--- a/src/crimson/net/io_handler.cc
+++ b/src/crimson/net/io_handler.cc
@@ -75,6 +75,8 @@ IOHandler::sweep_out_pending_msgs_to_sent(
   bool require_ack)
 {
   std::size_t num_msgs = out_pending_msgs.size();
+  const unsigned max_bufs =
+      local_conf().get_val<uint64_t>("crimson_osd_max_send_buffers");
   ceph::bufferlist bl;
 
 #ifdef UNIT_TESTS_BUILT
@@ -108,15 +110,10 @@ IOHandler::sweep_out_pending_msgs_to_sent(
 #endif
   }
 
-  std::for_each(
-      out_pending_msgs.begin(),
-      out_pending_msgs.begin()+num_msgs,
-      [this, &bl
-#ifdef UNIT_TESTS_BUILT
-        , &tags
-#endif
-      ](const MessageFRef& msg) {
-    // set priority
+  std::size_t num_swept = 0;
+  for (std::size_t i = 0; i < num_msgs; ++i) {
+    auto& msg = out_pending_msgs[i];
+
     msg->get_header().src = conn.messenger.get_myname();
 
     msg->encode(conn.features, 0);
@@ -144,15 +141,21 @@ IOHandler::sweep_out_pending_msgs_to_sent(
     auto tag = MessageFrame::tag;
     tags.push_back(tag);
 #endif
-  });
+    ++num_swept;
+    if (bl.get_num_buffers() >= max_bufs) {
+      break;
+    }
+  }
 
   if (!conn.policy.lossy) {
     out_sent_msgs.insert(
         out_sent_msgs.end(),
         std::make_move_iterator(out_pending_msgs.begin()),
-        std::make_move_iterator(out_pending_msgs.end()));
+        std::make_move_iterator(out_pending_msgs.begin() + num_swept));
   }
-  out_pending_msgs.clear();
+  out_pending_msgs.erase(
+      out_pending_msgs.begin(),
+      out_pending_msgs.begin() + num_swept);
 
 #ifdef UNIT_TESTS_BUILT
   return sweep_ret{std::move(bl), tags};

diff --git a/src/crimson/net/io_handler.h b/src/crimson/net/io_handler.h
index c86419c76b8fb7764207cffb4b0918180ae1211c..a8a6a4216d297bbf7f0a9c7e9209f27ca48e2b0b 100644 (file)
--- a/src/crimson/net/io_handler.h
+++ b/src/crimson/net/io_handler.h
@@ -450,6 +450,14 @@ public:
 #else
   ceph::bufferlist
 #endif
+  /**
+   * Encode and drain pending outgoing messages into a single bufferlist
+   * for a batched socket write. Stops early when the accumulated number
+   * of buffer fragments reaches crimson_osd_max_send_buffers to prevent
+   * overflowing seastar's uint16_t packet fragment
+   * counter. Un-swept messages remain in the queue and are picked up
+   * by the next do_out_dispatch() iteration.
+   */
   sweep_out_pending_msgs_to_sent(
       bool require_keepalive,
       std::optional<utime_t> maybe_keepalive_ack,