From a403e233edcf47eec78d5907b8685f17067bad5f Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 28 Feb 2020 14:52:02 -0600 Subject: [PATCH] msg/Policy: make stateless_server default to anon (again) Midway through the octopus cycle, we made stateless server more stateless in the sense that it would not register incoming client connections. And, in so doing, it would not enforce that client connections came from unique addresses, by closing an existing connection from the same addr when a new connection was accepted. This turned out to cause out of order OSD ops because the OSD needed that behavior. See https://tracker.ceph.com/issues/42328. We fixed that by reverting to the old behavior for all but monitor connections, where we needed it, in 507d213cc453ed86ab38619590f710f33245c652. This, in turn, breaks most OSD <-> OSD communication (and probably lots of other things) with cephadm, because we make entity_addr_t unique with a nonce that is populated by getpid()... and the containerized daemons all have pid 1. When we finally merged the follow-on fixes for the change above cephadm OSDs can't ping each other. In my view, the 'anon' connection handling is a good idea in the general case. So, let's adjust our fix for #42328 so that it is only the OSD client-side interface that registers client connections and makes them unique. Fixes: https://tracker.ceph.com/issues/44358 Signed-off-by: Sage Weil --- src/ceph_mon.cc | 8 ++++---- src/ceph_osd.cc | 2 +- src/msg/Policy.h | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/ceph_mon.cc b/src/ceph_mon.cc index f56d6914da5..3b90deecb44 100644 --- a/src/ceph_mon.cc +++ b/src/ceph_mon.cc @@ -784,17 +784,17 @@ int main(int argc, const char **argv) msgr->set_cluster_protocol(CEPH_MON_PROTOCOL); msgr->set_default_send_priority(CEPH_MSG_PRIO_HIGH); - msgr->set_default_policy(Messenger::Policy::stateless_anon_server(0)); + msgr->set_default_policy(Messenger::Policy::stateless_server(0)); msgr->set_policy(entity_name_t::TYPE_MON, Messenger::Policy::lossless_peer_reuse( CEPH_FEATURE_SERVER_LUMINOUS)); msgr->set_policy(entity_name_t::TYPE_OSD, - Messenger::Policy::stateless_anon_server( + Messenger::Policy::stateless_server( CEPH_FEATURE_SERVER_LUMINOUS)); msgr->set_policy(entity_name_t::TYPE_CLIENT, - Messenger::Policy::stateless_anon_server(0)); + Messenger::Policy::stateless_server(0)); msgr->set_policy(entity_name_t::TYPE_MDS, - Messenger::Policy::stateless_anon_server(0)); + Messenger::Policy::stateless_server(0)); // throttle client traffic Throttle *client_throttler = new Throttle(g_ceph_context, "mon_client_bytes", diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc index 1e09474058a..1bdb36b8369 100644 --- a/src/ceph_osd.cc +++ b/src/ceph_osd.cc @@ -574,7 +574,7 @@ flushjournal_out: CEPH_FEATURE_PGID64 | CEPH_FEATURE_OSDENC; - ms_public->set_default_policy(Messenger::Policy::stateless_server(0)); + ms_public->set_default_policy(Messenger::Policy::stateless_registered_server(0)); ms_public->set_policy_throttlers(entity_name_t::TYPE_CLIENT, client_byte_throttler.get(), nullptr); diff --git a/src/msg/Policy.h b/src/msg/Policy.h index 9f5ea32a0bd..10a426f2f46 100644 --- a/src/msg/Policy.h +++ b/src/msg/Policy.h @@ -65,10 +65,10 @@ public: static Policy stateful_server(uint64_t req) { return Policy(false, true, true, true, true, req); } - static Policy stateless_server(uint64_t req) { + static Policy stateless_registered_server(uint64_t req) { return Policy(true, true, false, false, true, req); } - static Policy stateless_anon_server(uint64_t req) { + static Policy stateless_server(uint64_t req) { return Policy(true, true, false, false, false, req); } static Policy lossless_peer(uint64_t req) { -- 2.47.3