From e4fa5048e0b12d627f95f47e70b5cd90d1c1b388 Mon Sep 17 00:00:00 2001 From: Ronen Friedman Date: Tue, 30 Jun 2026 12:55:33 +0000 Subject: [PATCH] crimson/os/alienstore: make CnLog::_flush non-blocking CnLog::_flush() calls alien::submit_to(...).wait(), which blocks the log flusher thread until the seastar reactor finishes outputting the log entries. When the reactor writes to stderr and the pipe buffer is full (e.g. two OSDs on the same host both dumping at debug level 20), the reactor itself blocks on the write() syscall. Meanwhile BlueStore threads filling the log buffer past m_max_new block in submit_entry() waiting for the flusher - which is waiting for the reactor - which is blocked on I/O. The kv thread cannot fire commit callbacks while it is blocked on logging, so store transactions (including osdmap persistence) hang indefinitely. Fix by moving the entries into the lambda capture so _flush() can return immediately without waiting. This is a corrected version of the approach in commit 511af83e274 ("crimson/os/alienstore/alien_log: _flush concurrently", PR #55262), which was reverted in 4d52d1d22b7 because its lambda captured the EntryVector by reference - the caller would clear/reuse the vector before the lambda finished, causing a "pure virtual method called" crash (https://tracker.ceph.com/issues/64140). The fix here avoids that by swapping the entries out of the caller's vector and moving them into the lambda's capture, so the lambda owns them for their full lifetime. Fixes: https://tracker.ceph.com/issues/77826 Fixes (failure of the 1st attempt): https://tracker.ceph.com/issues/64140 Assisted-by: Claude Signed-off-by: Ronen Friedman --- src/crimson/os/alienstore/alien_log.cc | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/crimson/os/alienstore/alien_log.cc b/src/crimson/os/alienstore/alien_log.cc index 723f0f08409..74bb9ec66cd 100644 --- a/src/crimson/os/alienstore/alien_log.cc +++ b/src/crimson/os/alienstore/alien_log.cc @@ -17,18 +17,16 @@ CnLog::~CnLog() { } void CnLog::_flush(EntryVector& q, bool crash) { - // XXX: the waiting here will block the thread for an indeterministic peroid - seastar::alien::submit_to(inst, shard, [&q] { - for (auto& it : q) { + std::ignore = seastar::alien::submit_to(inst, shard, + [entries = std::move(q)] { + for (const auto& it : entries) { crimson::get_logger(it.m_subsys).log( crimson::to_log_level(it.m_prio), "{}", it.strv()); } return seastar::make_ready_future<>(); - }).wait(); - q.clear(); - return; + }); } } //namespace ceph::logging -- 2.47.3