From 5ffbdefeff4915b8f0ea4ac6d41998b7fe5a3822 Mon Sep 17 00:00:00 2001 From: Alexander Indenbaum Date: Tue, 24 Feb 2026 23:54:56 +0200 Subject: [PATCH] crimson/rbd: add external-thread API for SPDK Phase A1 integration Add C API for running Seastar reactors inside SPDK reactor threads: - rbd_crimson_configure_external_threads(): configure before spdk_reactors_start() - rbd_crimson_register_reactor(): register reactor on each SPDK reactor thread - rbd_crimson_run_one_tick(): advance reactor once per loop iteration - rbd_crimson_reactor_cleanup(): per-thread cleanup - rbd_crimson_cleanup_all(): global cleanup after spdk_reactors_fini Implement in api/external_thread.cc by wrapping Seastar app_template and reactor APIs. Lets SPDK link librbd_crimson for Phase A1 without depending on Seastar directly. Signed-off-by: Alexander Indenbaum --- ceph.spec.in | 32 +++++++++++ src/crimson/rbd/CMakeLists.txt | 19 ++++++- src/crimson/rbd/api/external_thread.cc | 73 ++++++++++++++++++++++++++ src/include/CMakeLists.txt | 6 +++ src/include/rbd/rbd_crimson.h | 21 ++++++++ src/seastar | 2 +- 6 files changed, 150 insertions(+), 3 deletions(-) create mode 100644 src/crimson/rbd/api/external_thread.cc diff --git a/ceph.spec.in b/ceph.spec.in index 40cf31e054b..a625cbd8642 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -2509,6 +2509,38 @@ fi %{_libdir}/librbd_tp.so %endif +%if 0%{with crimson} +%package -n librbd-crimson1 +Summary: Crimson RBD client library (Seastar-native) +%if 0%{?suse_version} +Group: System/Libraries +%endif +%description -n librbd-crimson1 +librbd-crimson is a Seastar-native RBD client library for the Ceph +distributed storage system. Requires Crimson build (WITH_CRIMSON=ON). + +%package -n librbd-crimson-devel +Summary: Crimson RBD client headers +%if 0%{?suse_version} +Group: Development/Libraries/C and C++ +%endif +Requires: librbd-crimson1 = %{_epoch_prefix}%{version}-%{release} +Provides: librbd-crimson1-devel = %{_epoch_prefix}%{version}-%{release} +%description -n librbd-crimson-devel +Headers and development files for librbd-crimson + +%files -n librbd-crimson1 +%{_libdir}/librbd_crimson.so.* + +%post -n librbd-crimson1 -p /sbin/ldconfig + +%postun -n librbd-crimson1 -p /sbin/ldconfig + +%files -n librbd-crimson-devel +%{_includedir}/rbd/rbd_crimson.h +%{_libdir}/librbd_crimson.so +%endif + %files -n librgw2 %{_libdir}/librgw.so.* %if %{with lttng} diff --git a/src/crimson/rbd/CMakeLists.txt b/src/crimson/rbd/CMakeLists.txt index 7a7b40fc220..aba50a61eed 100644 --- a/src/crimson/rbd/CMakeLists.txt +++ b/src/crimson/rbd/CMakeLists.txt @@ -1,5 +1,4 @@ # librbd_crimson - Seastar-native RBD client -# Phase 1: Library skeleton, RBD header read, rbd_open/close/stat set(crimson_rbd_srcs utils.cc @@ -11,9 +10,11 @@ set(crimson_rbd_srcs api/io.cc api/cluster.cc api/metadata.cc + api/external_thread.cc ) -add_library(librbd_crimson STATIC ${crimson_rbd_srcs}) +# Build shared lib when ENABLE_SHARED (for RPM packaging); else static +add_library(librbd_crimson ${CEPH_SHARED} ${crimson_rbd_srcs}) target_include_directories(librbd_crimson PUBLIC @@ -30,3 +31,17 @@ target_link_libraries(librbd_crimson PRIVATE cls_rbd_client ) + +if(ENABLE_SHARED) + set_target_properties(librbd_crimson PROPERTIES + OUTPUT_NAME rbd_crimson + VERSION 1.0.0 + SOVERSION 1 + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON) +endif() + +install(TARGETS librbd_crimson + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/src/crimson/rbd/api/external_thread.cc b/src/crimson/rbd/api/external_thread.cc new file mode 100644 index 00000000000..19ae9f35725 --- /dev/null +++ b/src/crimson/rbd/api/external_thread.cc @@ -0,0 +1,73 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 sts=2 expandtab +// +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (C) 2025 Red Hat +// +// Phase A1: External-thread integration for SPDK reactor. +// Bridges SPDK reactor loop to Seastar run_one_tick / register APIs. +// Part of librbd_crimson; SPDK links librbd_crimson, no direct Seastar dep. + +#include "include/rbd/rbd_crimson.h" + +#include + +#include +#include +#include +#include + +namespace { + +std::unique_ptr g_app; + +} // anonymous namespace + +extern "C" { + +int rbd_crimson_configure_external_threads(unsigned core_count) +{ + try { + seastar::app_template::seastar_options opts; + opts.smp_opts.thread_affinity.set_value(false); + opts.smp_opts.mbind.set_value(false); + opts.smp_opts.smp.set_value(core_count); + opts.smp_opts.lock_memory.set_value(false); + opts.log_opts.default_log_level.set_value(seastar::log_level::error); + opts.reactor_opts.no_handle_interrupt.set_value(true); + + g_app = std::make_unique(std::move(opts)); + if (g_app->configure_external_thread_mode(core_count) != 0) { + g_app.reset(); + return -1; + } + return 0; + } catch (...) { + return -1; + } +} + +void rbd_crimson_register_reactor(unsigned shard_id) +{ + g_app->register_reactor_on_this_thread(shard_id); +} + +int rbd_crimson_run_one_tick(void) +{ + return seastar::engine().run_one_tick() ? 1 : 0; +} + +void rbd_crimson_reactor_cleanup(void) +{ + seastar::smp::cleanup_cpu(); +} + +void rbd_crimson_cleanup_all(void) +{ + if (g_app) { + g_app->cleanup_external_thread_mode(); + g_app.reset(); + } +} + +} // extern "C" diff --git a/src/include/CMakeLists.txt b/src/include/CMakeLists.txt index cb9c2fea8f8..0b6d9ae084c 100644 --- a/src/include/CMakeLists.txt +++ b/src/include/CMakeLists.txt @@ -30,6 +30,12 @@ if(WITH_RBD) DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rbd) endif() +if(WITH_CRIMSON) + install(FILES + rbd/rbd_crimson.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rbd) +endif() + if(WITH_RADOSGW) install(FILES rados/librgw.h diff --git a/src/include/rbd/rbd_crimson.h b/src/include/rbd/rbd_crimson.h index 0156264165a..b16b5fba234 100644 --- a/src/include/rbd/rbd_crimson.h +++ b/src/include/rbd/rbd_crimson.h @@ -112,6 +112,27 @@ ssize_t rbd_aio_get_return_value(rbd_completion_t c); void *rbd_aio_get_arg(rbd_completion_t c); void rbd_aio_release(rbd_completion_t c); +/* Phase A1: External-thread integration for SPDK reactor + * When SPDK hosts the threads, each reactor thread runs a Seastar reactor. + * Call these from SPDK reactor loop. Part of librbd_crimson; no direct + * SPDK->Seastar dependency. Build ceph-nvmeof against librbd_crimson RPM. + */ + +/** Configure Seastar for external-thread mode. Call once before spdk_reactors_start(). */ +int rbd_crimson_configure_external_threads(unsigned core_count); + +/** Register a Seastar reactor on the current thread. Call once per SPDK reactor thread. */ +void rbd_crimson_register_reactor(unsigned shard_id); + +/** Advance the Seastar reactor one tick. Call once per reactor loop iteration. Returns 1 if more work, 0 if stopped. */ +int rbd_crimson_run_one_tick(void); + +/** Clean up the Seastar reactor on the current thread. Call when SPDK reactor thread exits. */ +void rbd_crimson_reactor_cleanup(void); + +/** Clean up all Seastar resources. Call from main thread after spdk_reactors_fini. */ +void rbd_crimson_cleanup_all(void); + #ifdef __cplusplus } #endif diff --git a/src/seastar b/src/seastar index 7347cf6f4f9..6e70097b40d 160000 --- a/src/seastar +++ b/src/seastar @@ -1 +1 @@ -Subproject commit 7347cf6f4f966929d5dc5b3fd7e34d771c9b3f85 +Subproject commit 6e70097b40d2673fc9ca86b16ebd564e300e4592 -- 2.47.3