crimson/rbd: add external-thread API for SPDK Phase A1 integration

Add C API for running Seastar reactors inside SPDK reactor threads:
- rbd_crimson_configure_external_threads(): configure before spdk_reactors_start()
- rbd_crimson_register_reactor(): register reactor on each SPDK reactor thread
- rbd_crimson_run_one_tick(): advance reactor once per loop iteration
- rbd_crimson_reactor_cleanup(): per-thread cleanup
- rbd_crimson_cleanup_all(): global cleanup after spdk_reactors_fini

Implement in api/external_thread.cc by wrapping Seastar app_template
and reactor APIs. Lets SPDK link librbd_crimson for Phase A1 without
depending on Seastar directly.

Signed-off-by: Alexander Indenbaum <aindenba@redhat.com>

diff --git a/ceph.spec.in b/ceph.spec.in
index 40cf31e054b3f8243f3b093c19280294058838f4..a625cbd8642000863e5da8f76acf7543b87050c0 100644 (file)
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -2509,6 +2509,38 @@ fi
 %{_libdir}/librbd_tp.so
 %endif
 
+%if 0%{with crimson}
+%package -n librbd-crimson1
+Summary:       Crimson RBD client library (Seastar-native)
+%if 0%{?suse_version}
+Group:         System/Libraries
+%endif
+%description -n librbd-crimson1
+librbd-crimson is a Seastar-native RBD client library for the Ceph
+distributed storage system. Requires Crimson build (WITH_CRIMSON=ON).
+
+%package -n librbd-crimson-devel
+Summary:       Crimson RBD client headers
+%if 0%{?suse_version}
+Group:         Development/Libraries/C and C++
+%endif
+Requires:      librbd-crimson1 = %{_epoch_prefix}%{version}-%{release}
+Provides:      librbd-crimson1-devel = %{_epoch_prefix}%{version}-%{release}
+%description -n librbd-crimson-devel
+Headers and development files for librbd-crimson
+
+%files -n librbd-crimson1
+%{_libdir}/librbd_crimson.so.*
+
+%post -n librbd-crimson1 -p /sbin/ldconfig
+
+%postun -n librbd-crimson1 -p /sbin/ldconfig
+
+%files -n librbd-crimson-devel
+%{_includedir}/rbd/rbd_crimson.h
+%{_libdir}/librbd_crimson.so
+%endif
+
 %files -n librgw2
 %{_libdir}/librgw.so.*
 %if %{with lttng}

diff --git a/src/crimson/rbd/CMakeLists.txt b/src/crimson/rbd/CMakeLists.txt
index 7a7b40fc2206c484bd4569be493cb8fb479733f4..aba50a61eed8bd236b3df6f9220803c749b21db0 100644 (file)
--- a/src/crimson/rbd/CMakeLists.txt
+++ b/src/crimson/rbd/CMakeLists.txt
@@ -1,5 +1,4 @@
 # librbd_crimson - Seastar-native RBD client
-# Phase 1: Library skeleton, RBD header read, rbd_open/close/stat
 
 set(crimson_rbd_srcs
   utils.cc
@@ -11,9 +10,11 @@ set(crimson_rbd_srcs
   api/io.cc
   api/cluster.cc
   api/metadata.cc
+  api/external_thread.cc
 )
 
-add_library(librbd_crimson STATIC ${crimson_rbd_srcs})
+# Build shared lib when ENABLE_SHARED (for RPM packaging); else static
+add_library(librbd_crimson ${CEPH_SHARED} ${crimson_rbd_srcs})
 
 target_include_directories(librbd_crimson
   PUBLIC
@@ -30,3 +31,17 @@ target_link_libraries(librbd_crimson
   PRIVATE
     cls_rbd_client
 )
+
+if(ENABLE_SHARED)
+  set_target_properties(librbd_crimson PROPERTIES
+    OUTPUT_NAME rbd_crimson
+    VERSION 1.0.0
+    SOVERSION 1
+    CXX_VISIBILITY_PRESET hidden
+    VISIBILITY_INLINES_HIDDEN ON)
+endif()
+
+install(TARGETS librbd_crimson
+  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})

diff --git a/src/crimson/rbd/api/external_thread.cc b/src/crimson/rbd/api/external_thread.cc
new file mode 100644 (file)
index 0000000..19ae9f3
--- /dev/null
+++ b/src/crimson/rbd/api/external_thread.cc
@@ -0,0 +1,73 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 sts=2 expandtab
+//
+// SPDX-License-Identifier: BSD-3-Clause
+// Copyright (C) 2025 Red Hat
+//
+// Phase A1: External-thread integration for SPDK reactor.
+// Bridges SPDK reactor loop to Seastar run_one_tick / register APIs.
+// Part of librbd_crimson; SPDK links librbd_crimson, no direct Seastar dep.
+
+#include "include/rbd/rbd_crimson.h"
+
+#include <memory>
+
+#include <seastar/core/app-template.hh>
+#include <seastar/core/reactor.hh>
+#include <seastar/core/smp.hh>
+#include <seastar/util/log.hh>
+
+namespace {
+
+std::unique_ptr<seastar::app_template> g_app;
+
+} // anonymous namespace
+
+extern "C" {
+
+int rbd_crimson_configure_external_threads(unsigned core_count)
+{
+  try {
+    seastar::app_template::seastar_options opts;
+    opts.smp_opts.thread_affinity.set_value(false);
+    opts.smp_opts.mbind.set_value(false);
+    opts.smp_opts.smp.set_value(core_count);
+    opts.smp_opts.lock_memory.set_value(false);
+    opts.log_opts.default_log_level.set_value(seastar::log_level::error);
+    opts.reactor_opts.no_handle_interrupt.set_value(true);
+
+    g_app = std::make_unique<seastar::app_template>(std::move(opts));
+    if (g_app->configure_external_thread_mode(core_count) != 0) {
+      g_app.reset();
+      return -1;
+    }
+    return 0;
+  } catch (...) {
+    return -1;
+  }
+}
+
+void rbd_crimson_register_reactor(unsigned shard_id)
+{
+  g_app->register_reactor_on_this_thread(shard_id);
+}
+
+int rbd_crimson_run_one_tick(void)
+{
+  return seastar::engine().run_one_tick() ? 1 : 0;
+}
+
+void rbd_crimson_reactor_cleanup(void)
+{
+  seastar::smp::cleanup_cpu();
+}
+
+void rbd_crimson_cleanup_all(void)
+{
+  if (g_app) {
+    g_app->cleanup_external_thread_mode();
+    g_app.reset();
+  }
+}
+
+} // extern "C"

diff --git a/src/include/CMakeLists.txt b/src/include/CMakeLists.txt
index cb9c2fea8f80b708b4a4500853e91382355e0f96..0b6d9ae084c6f66ad59afa7e07dcae14e6b88a49 100644 (file)
--- a/src/include/CMakeLists.txt
+++ b/src/include/CMakeLists.txt
@@ -30,6 +30,12 @@ if(WITH_RBD)
     DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rbd)
 endif()
 
+if(WITH_CRIMSON)
+  install(FILES
+    rbd/rbd_crimson.h
+    DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rbd)
+endif()
+
 if(WITH_RADOSGW)
   install(FILES
     rados/librgw.h

diff --git a/src/include/rbd/rbd_crimson.h b/src/include/rbd/rbd_crimson.h
index 0156264165a1d96b428ba7a6f20879869ec50773..b16b5fba234a3a34e33d4c37e69a711edf9028f6 100644 (file)
--- a/src/include/rbd/rbd_crimson.h
+++ b/src/include/rbd/rbd_crimson.h
@@ -112,6 +112,27 @@ ssize_t rbd_aio_get_return_value(rbd_completion_t c);
 void *rbd_aio_get_arg(rbd_completion_t c);
 void rbd_aio_release(rbd_completion_t c);
 
+/* Phase A1: External-thread integration for SPDK reactor
+ * When SPDK hosts the threads, each reactor thread runs a Seastar reactor.
+ * Call these from SPDK reactor loop. Part of librbd_crimson; no direct
+ * SPDK->Seastar dependency. Build ceph-nvmeof against librbd_crimson RPM.
+ */
+
+/** Configure Seastar for external-thread mode. Call once before spdk_reactors_start(). */
+int rbd_crimson_configure_external_threads(unsigned core_count);
+
+/** Register a Seastar reactor on the current thread. Call once per SPDK reactor thread. */
+void rbd_crimson_register_reactor(unsigned shard_id);
+
+/** Advance the Seastar reactor one tick. Call once per reactor loop iteration. Returns 1 if more work, 0 if stopped. */
+int rbd_crimson_run_one_tick(void);
+
+/** Clean up the Seastar reactor on the current thread. Call when SPDK reactor thread exits. */
+void rbd_crimson_reactor_cleanup(void);
+
+/** Clean up all Seastar resources. Call from main thread after spdk_reactors_fini. */
+void rbd_crimson_cleanup_all(void);
+
 #ifdef __cplusplus
 }
 #endif

diff --git a/src/seastar b/src/seastar
index 7347cf6f4f966929d5dc5b3fd7e34d771c9b3f85..6e70097b40d2673fc9ca86b16ebd564e300e4592 160000 (submodule)
--- a/src/seastar
+++ b/src/seastar
@@ -1 +1 @@
-Subproject commit 7347cf6f4f966929d5dc5b3fd7e34d771c9b3f85
+Subproject commit 6e70097b40d2673fc9ca86b16ebd564e300e4592