crimson/osd/replicated_backend: block _submit_transaction on sending the messages

Otherwise, the send operations can reorder.

Fixes: https://tracker.ceph.com/issues/57738
Signed-off-by: Samuel Just <sjust@redhat.com>

diff --git a/src/crimson/osd/replicated_backend.cc b/src/crimson/osd/replicated_backend.cc
index 894d606752975619c4619a6a29a501b1c5208097..119d7278ef6e65ed8aff41e0b3cddbf132d585a4 100644 (file)
--- a/src/crimson/osd/replicated_backend.cc
+++ b/src/crimson/osd/replicated_backend.cc
@@ -81,6 +81,7 @@ ReplicatedBackend::_submit_transaction(std::set<pg_shard_t>&& pg_shards,
     return seastar::make_ready_future<crimson::osd::acked_peers_t>(std::move(acked_peers));
   });
 
+  auto sends = std::make_unique<std::vector<seastar::future<>>>();
   for (auto pg_shard : pg_shards) {
     if (pg_shard != whoami) {
       auto m = crimson::make_message<MOSDRepOp>(
@@ -100,10 +101,13 @@ ReplicatedBackend::_submit_transaction(std::set<pg_shard_t>&& pg_shards,
       m->min_last_complete_ondisk = osd_op_p.min_last_complete_ondisk;
       m->set_rollback_to(osd_op_p.at_version);
       // TODO: set more stuff. e.g., pg_states
-      (void) shard_services.send_to_osd(pg_shard.osd, std::move(m), map_epoch);
+      sends->emplace_back(shard_services.send_to_osd(pg_shard.osd, std::move(m), map_epoch));
     }
   }
-  return {seastar::now(), std::move(all_completed)};
+  auto sends_complete = seastar::when_all_succeed(
+    sends->begin(), sends->end()
+  ).finally([sends=std::move(sends)] {});
+  return {std::move(sends_complete), std::move(all_completed)};
 }
 
 void ReplicatedBackend::on_actingset_changed(peering_info_t pi)