From 08758bef9e33b65b4b2a047d8005ebdbb3f59848 Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Thu, 8 Sep 2022 19:57:22 -0700 Subject: [PATCH] crimson/osd: move send_incremental_map to OSDSingletonState Signed-off-by: Samuel Just --- src/crimson/osd/osd.cc | 28 ++-------------- src/crimson/osd/osd.h | 3 -- .../osd/osd_operations/client_request.cc | 7 ++-- src/crimson/osd/pg_shard_manager.h | 2 ++ src/crimson/osd/shard_services.cc | 32 +++++++++++++++++++ src/crimson/osd/shard_services.h | 11 +++++++ 6 files changed, 51 insertions(+), 32 deletions(-) diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc index d2bdc42f620..7ebe89f4d31 100644 --- a/src/crimson/osd/osd.cc +++ b/src/crimson/osd/osd.cc @@ -376,6 +376,7 @@ seastar::future<> OSD::start() ); }).then([this](OSDSuperblock&& sb) { superblock = std::move(sb); + pg_shard_manager.set_superblock(sb); return pg_shard_manager.get_local_map(superblock.current_epoch); }).then([this](OSDMapService::local_cached_map_t&& map) { osdmap = make_local_shared_foreign(OSDMapService::local_cached_map_t(map)); @@ -909,6 +910,7 @@ seastar::future<> OSD::handle_osd_map(crimson::net::ConnectionRef conn, superblock.clean_thru = last; } pg_shard_manager.get_meta_coll().store_superblock(t, superblock); + pg_shard_manager.set_superblock(superblock); logger().debug("OSD::handle_osd_map: do_transaction..."); return store.do_transaction( pg_shard_manager.get_meta_coll().collection(), @@ -1073,32 +1075,6 @@ seastar::future<> OSD::handle_update_log_missing_reply( return seastar::now(); } -seastar::future<> OSD::send_incremental_map(crimson::net::ConnectionRef conn, - epoch_t first) -{ - if (first >= superblock.oldest_map) { - return pg_shard_manager.load_map_bls(first, superblock.newest_map) - .then([this, conn, first](auto&& bls) { - auto m = crimson::make_message(monc->get_fsid(), - osdmap->get_encoding_features()); - m->oldest_map = first; - m->newest_map = superblock.newest_map; - m->maps = std::move(bls); - return conn->send(std::move(m)); - }); - } else { - return pg_shard_manager.load_map_bl(osdmap->get_epoch()) - .then([this, conn](auto&& bl) mutable { - auto m = crimson::make_message(monc->get_fsid(), - osdmap->get_encoding_features()); - m->oldest_map = superblock.oldest_map; - m->newest_map = superblock.newest_map; - m->maps.emplace(osdmap->get_epoch(), std::move(bl)); - return conn->send(std::move(m)); - }); - } -} - seastar::future<> OSD::handle_rep_op(crimson::net::ConnectionRef conn, Ref m) { diff --git a/src/crimson/osd/osd.h b/src/crimson/osd/osd.h index 04d3251312e..02aa2d3e25f 100644 --- a/src/crimson/osd/osd.h +++ b/src/crimson/osd/osd.h @@ -141,9 +141,6 @@ public: seastar::future<> dump_pg_state_history(Formatter*) const; void print(std::ostream&) const; - seastar::future<> send_incremental_map(crimson::net::ConnectionRef conn, - epoch_t first); - /// @return the seq id of the pg stats being sent uint64_t send_pg_stats(); diff --git a/src/crimson/osd/osd_operations/client_request.cc b/src/crimson/osd/osd_operations/client_request.cc index 88377546379..d19e5a97ee2 100644 --- a/src/crimson/osd/osd_operations/client_request.cc +++ b/src/crimson/osd/osd_operations/client_request.cc @@ -115,11 +115,11 @@ seastar::future<> ClientRequest::with_pg_int( auto instance_handle = get_instance_handle(); auto &ihref = *instance_handle; return interruptor::with_interruption( - [this, pgref, this_instance_id, &ihref]() mutable { + [this, pgref, this_instance_id, &ihref, &shard_services]() mutable { PG &pg = *pgref; if (pg.can_discard_op(*m)) { - return osd.send_incremental_map( - conn, m->get_map_epoch() + return shard_services.send_incremental_map( + std::ref(*conn), m->get_map_epoch() ).then([this, this_instance_id, pgref] { logger().debug("{}.{}: discarding", *this, this_instance_id); pgref->client_request_orderer.remove_request(*this); @@ -310,6 +310,7 @@ ClientRequest::do_process( return ihref.enter_stage(pp(*pg).send_reply, *this ).then_interruptible( [this, reply=std::move(reply)]() mutable { + logger().debug("{}: sending response", *this); return conn->send(std::move(reply)).then([] { return seastar::make_ready_future(seq_mode_t::IN_ORDER); }); diff --git a/src/crimson/osd/pg_shard_manager.h b/src/crimson/osd/pg_shard_manager.h index cc2aa6a84e1..3fdc3524d18 100644 --- a/src/crimson/osd/pg_shard_manager.h +++ b/src/crimson/osd/pg_shard_manager.h @@ -127,6 +127,8 @@ public: FORWARD_TO_OSD_SINGLETON(init_meta_coll) FORWARD_TO_OSD_SINGLETON(get_meta_coll) + FORWARD_TO_OSD_SINGLETON(set_superblock) + // Core OSDMap methods FORWARD_TO_OSD_SINGLETON(get_local_map) FORWARD_TO_OSD_SINGLETON(load_map_bl) diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc index 979b3314b49..68794184d0a 100644 --- a/src/crimson/osd/shard_services.cc +++ b/src/crimson/osd/shard_services.cc @@ -365,6 +365,7 @@ seastar::future> OSDSingletonState::load_map_bls( epoch_t first, epoch_t last) { + ceph_assert(first <= last); return seastar::map_reduce(boost::make_counting_iterator(first), boost::make_counting_iterator(last + 1), [this](epoch_t e) { @@ -665,5 +666,36 @@ seastar::future<> ShardServices::dispatch_context( }); } +seastar::future<> OSDSingletonState::send_incremental_map( + crimson::net::Connection &conn, + epoch_t first) +{ + if (first >= superblock.oldest_map) { + return load_map_bls( + first, superblock.newest_map + ).then([this, &conn, first](auto&& bls) { + auto m = crimson::make_message( + monc.get_fsid(), + osdmap->get_encoding_features()); + m->oldest_map = first; + m->newest_map = superblock.newest_map; + m->maps = std::move(bls); + return conn.send(std::move(m)); + }); + } else { + return load_map_bl(osdmap->get_epoch() + ).then([this, &conn](auto&& bl) mutable { + auto m = crimson::make_message( + monc.get_fsid(), + osdmap->get_encoding_features()); + m->oldest_map = superblock.oldest_map; + m->newest_map = superblock.newest_map; + m->maps.emplace(osdmap->get_epoch(), std::move(bl)); + return conn.send(std::move(m)); + }); + } +} + + }; diff --git a/src/crimson/osd/shard_services.h b/src/crimson/osd/shard_services.h index e1adbb9049d..2f6e026b342 100644 --- a/src/crimson/osd/shard_services.h +++ b/src/crimson/osd/shard_services.h @@ -226,6 +226,15 @@ private: return *meta_coll; } + OSDSuperblock superblock; + void set_superblock(OSDSuperblock _superblock) { + superblock = std::move(_superblock); + } + + seastar::future<> send_incremental_map( + crimson::net::Connection &conn, + epoch_t first); + auto get_pool_info(int64_t poolid) { return get_meta_coll().load_final_pool_info(poolid); } @@ -421,6 +430,8 @@ public: FORWARD_TO_OSD_SINGLETON(get_pool_info) FORWARD(with_throttle_while, with_throttle_while, local_state.throttler) + FORWARD_TO_OSD_SINGLETON(send_incremental_map) + FORWARD_TO_OSD_SINGLETON(osdmap_subscribe) FORWARD_TO_OSD_SINGLETON(queue_want_pg_temp) FORWARD_TO_OSD_SINGLETON(remove_want_pg_temp) -- 2.39.5