From 2d44fc89e5edc6d35b6520d9f1fb4ab668c1d9d8 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Mon, 1 Nov 2021 13:14:16 -0400 Subject: [PATCH] spawn: use explicit strand executor the default spawn::yield_context uses the polymorphic boost::asio::executor to support any executor type rgw's beast frontend always uses the same executor type for these coroutines, so we can use that type directly to avoid the overhead of type erasure and virtual function calls Signed-off-by: Casey Bodley (cherry picked from commit 9d9258e06b78bb47fd0156d9bd7bb00b52a726b0) Conflicts: src/common/async/yield_context.h src/rgw/rgw_d3n_cacherequest.h src/rgw/rgw_notify.cc src/rgw/rgw_sync_checkpoint.cc Cherry-pick notes: - src/rgw/rgw_d3n_cacherequest.h doesn't exist in Octopus - src/rgw/rgw_sync_checkpoint.cc doesn't exist in Octopus - conflicts due to rename of structs after Octopus - conflicts due to macro for conditional inclusion of beast context in Octopus --- src/common/async/yield_context.h | 15 +++++++++------ src/rgw/rgw_aio.cc | 4 ++-- src/rgw/rgw_aio_throttle.h | 4 ++-- src/rgw/rgw_asio_frontend.cc | 11 +++++------ src/test/rgw/test_rgw_dmclock_scheduler.cc | 2 +- src/test/rgw/test_rgw_reshard_wait.cc | 6 +++--- src/test/rgw/test_rgw_throttle.cc | 4 ++-- 7 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/common/async/yield_context.h b/src/common/async/yield_context.h index fdfb3f5b8fe1a..7720bc37e091d 100644 --- a/src/common/async/yield_context.h +++ b/src/common/async/yield_context.h @@ -23,15 +23,18 @@ #ifndef HAVE_BOOST_CONTEXT // hide the dependency on boost::context -namespace spawn { struct yield_context; -} #else // HAVE_BOOST_CONTEXT #include -#endif // HAVE_BOOST_CONTEXT +// use explicit executor types instead of the type-erased boost::asio::executor. +// coroutines wrap the default io_context executor with a strand executor +using yield_context = spawn::basic_yield_context< + boost::asio::executor_binder>>; +#endif // HAVE_BOOST_CONTEXT /// optional-like wrapper for a spawn::yield_context and its associated /// boost::asio::io_context. operations that take an optional_yield argument @@ -39,11 +42,11 @@ struct yield_context; /// of the blocking the thread of execution class optional_yield { boost::asio::io_context *c = nullptr; - spawn::yield_context *y = nullptr; + yield_context *y = nullptr; public: /// construct with a valid io and yield_context explicit optional_yield(boost::asio::io_context& c, - spawn::yield_context& y) noexcept + yield_context& y) noexcept : c(&c), y(&y) {} /// type tag to construct an empty object @@ -57,7 +60,7 @@ class optional_yield { boost::asio::io_context& get_io_context() const noexcept { return *c; } /// return a reference to the yield_context. only valid if non-empty - spawn::yield_context& get_yield_context() const noexcept { return *y; } + yield_context& get_yield_context() const noexcept { return *y; } }; // type tag object to construct an empty optional_yield diff --git a/src/rgw/rgw_aio.cc b/src/rgw/rgw_aio.cc index 48c83be98ac8a..14662e536122b 100644 --- a/src/rgw/rgw_aio.cc +++ b/src/rgw/rgw_aio.cc @@ -80,12 +80,12 @@ struct Handler { template Aio::OpFunc aio_abstract(Op&& op, boost::asio::io_context& context, - spawn::yield_context yield) { + yield_context yield) { return [op = std::move(op), &context, yield] (Aio* aio, AioResult& r) mutable { // arrange for the completion Handler to run on the yield_context's strand // executor so it can safely call back into Aio without locking using namespace boost::asio; - async_completion init(yield); + async_completion init(yield); auto ex = get_associated_executor(init.completion_handler); auto& ref = r.obj.get_ref(); diff --git a/src/rgw/rgw_aio_throttle.h b/src/rgw/rgw_aio_throttle.h index 764469d7ed38b..e3c485c7eb9da 100644 --- a/src/rgw/rgw_aio_throttle.h +++ b/src/rgw/rgw_aio_throttle.h @@ -83,7 +83,7 @@ class BlockingAioThrottle final : public Aio, private Throttle { // functions must be called within the coroutine strand class YieldingAioThrottle final : public Aio, private Throttle { boost::asio::io_context& context; - spawn::yield_context yield; + yield_context yield; struct Handler; // completion callback associated with the waiter @@ -97,7 +97,7 @@ class YieldingAioThrottle final : public Aio, private Throttle { public: YieldingAioThrottle(uint64_t window, boost::asio::io_context& context, - spawn::yield_context yield) + yield_context yield) : Throttle(window), context(context), yield(yield) {} diff --git a/src/rgw/rgw_asio_frontend.cc b/src/rgw/rgw_asio_frontend.cc index 321f6178299a5..c8533d8563439 100644 --- a/src/rgw/rgw_asio_frontend.cc +++ b/src/rgw/rgw_asio_frontend.cc @@ -53,13 +53,12 @@ template class StreamIO : public rgw::asio::ClientIO { CephContext* const cct; Stream& stream; - spawn::yield_context yield; + yield_context yield; parse_buffer& buffer; ceph::timespan request_timeout; public: StreamIO(CephContext *cct, Stream& stream, rgw::asio::parser_type& parser, - spawn::yield_context yield, - parse_buffer& buffer, bool is_ssl, + yield_context yield, parse_buffer& buffer, bool is_ssl, const tcp::endpoint& local_endpoint, const tcp::endpoint& remote_endpoint, ceph::timespan request_timeout) @@ -148,7 +147,7 @@ void handle_connection(boost::asio::io_context& context, SharedMutex& pause_mutex, rgw::dmclock::Scheduler *scheduler, boost::system::error_code& ec, - spawn::yield_context yield, + yield_context yield, ceph::timespan request_timeout) { // limit header to 4k, since we read it all into a single flat_buffer @@ -923,7 +922,7 @@ void AsioFrontend::accept(Listener& l, boost::system::error_code ec) #ifdef WITH_RADOSGW_BEAST_OPENSSL if (l.use_ssl) { spawn::spawn(context, - [this, s=std::move(stream)] (spawn::yield_context yield) mutable { + [this, s=std::move(stream)] (yield_context yield) mutable { Connection conn{s.socket()}; auto c = connections.add(conn); // wrap the tcp_stream in an ssl stream @@ -954,7 +953,7 @@ void AsioFrontend::accept(Listener& l, boost::system::error_code ec) { #endif // WITH_RADOSGW_BEAST_OPENSSL spawn::spawn(context, - [this, s=std::move(stream)] (spawn::yield_context yield) mutable { + [this, s=std::move(stream)] (yield_context yield) mutable { Connection conn{s.socket()}; auto c = connections.add(conn); auto buffer = std::make_unique(); diff --git a/src/test/rgw/test_rgw_dmclock_scheduler.cc b/src/test/rgw/test_rgw_dmclock_scheduler.cc index 2943df218750e..c8e16421cd27f 100644 --- a/src/test/rgw/test_rgw_dmclock_scheduler.cc +++ b/src/test/rgw/test_rgw_dmclock_scheduler.cc @@ -401,7 +401,7 @@ TEST(Queue, SpawnAsyncRequest) { boost::asio::io_context context; - spawn::spawn(context, [&] (spawn::yield_context yield) { + spawn::spawn(context, [&] (yield_context yield) { ClientCounters counters(g_ceph_context); AsyncScheduler queue(g_ceph_context, context, std::ref(counters), nullptr, [] (client_id client) -> ClientInfo* { diff --git a/src/test/rgw/test_rgw_reshard_wait.cc b/src/test/rgw/test_rgw_reshard_wait.cc index a1f55b5a71686..0c42c536f4544 100644 --- a/src/test/rgw/test_rgw_reshard_wait.cc +++ b/src/test/rgw/test_rgw_reshard_wait.cc @@ -67,7 +67,7 @@ TEST(ReshardWait, wait_yield) RGWReshardWait waiter(wait_duration); boost::asio::io_context context; - spawn::spawn(context, [&] (spawn::yield_context yield) { + spawn::spawn(context, [&] (yield_context yield) { EXPECT_EQ(0, waiter.wait(optional_yield{context, yield})); }); @@ -93,7 +93,7 @@ TEST(ReshardWait, stop_yield) boost::asio::io_context context; spawn::spawn(context, - [&] (spawn::yield_context yield) { + [&] (yield_context yield) { EXPECT_EQ(-ECANCELED, long_waiter.wait(optional_yield{context, yield})); }); @@ -136,7 +136,7 @@ TEST(ReshardWait, stop_multiple) // spawn 4 coroutines boost::asio::io_context context; { - auto async_waiter = [&] (spawn::yield_context yield) { + auto async_waiter = [&] (yield_context yield) { EXPECT_EQ(-ECANCELED, long_waiter.wait(optional_yield{context, yield})); }; spawn::spawn(context, async_waiter); diff --git a/src/test/rgw/test_rgw_throttle.cc b/src/test/rgw/test_rgw_throttle.cc index fff680012f4df..6ac4a89c9063a 100644 --- a/src/test/rgw/test_rgw_throttle.cc +++ b/src/test/rgw/test_rgw_throttle.cc @@ -173,7 +173,7 @@ TEST_F(Aio_Throttle, YieldCostOverWindow) boost::asio::io_context context; spawn::spawn(context, - [&] (spawn::yield_context yield) { + [&] (yield_context yield) { YieldingAioThrottle throttle(4, context, yield); scoped_completion op; auto c = throttle.get(obj, wait_on(op), 8, 0); @@ -195,7 +195,7 @@ TEST_F(Aio_Throttle, YieldingThrottleOverMax) boost::asio::io_context context; spawn::spawn(context, - [&] (spawn::yield_context yield) { + [&] (yield_context yield) { YieldingAioThrottle throttle(window, context, yield); for (uint64_t i = 0; i < total; i++) { using namespace std::chrono_literals; -- 2.39.5