From: Patrick Donnelly Date: Tue, 1 Aug 2023 18:51:21 +0000 (-0400) Subject: osd/OSDMonitor: check svc is writeable before changing pending X-Git-Tag: v18.2.5~205^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=267aaa5b898e8e6535000c179fc943055a4b888d;p=ceph.git osd/OSDMonitor: check svc is writeable before changing pending Fixes: https://tracker.ceph.com/issues/59813 Signed-off-by: Patrick Donnelly (cherry picked from commit ad295e90a6b5dc55e66e71a356bff2730a379a08) --- diff --git a/src/mon/AuthMonitor.cc b/src/mon/AuthMonitor.cc index 59adc404eb4f..ba1b77f5dc84 100644 --- a/src/mon/AuthMonitor.cc +++ b/src/mon/AuthMonitor.cc @@ -1133,11 +1133,12 @@ int AuthMonitor::validate_osd_destroy( return 0; } -int AuthMonitor::do_osd_destroy( - const EntityName& cephx_entity, - const EntityName& lockbox_entity) +void AuthMonitor::do_osd_destroy( + const EntityName& cephx_entity, + const EntityName& lockbox_entity) { ceph_assert(paxos.is_plugged()); + ceph_assert(is_writeable()); dout(10) << __func__ << " cephx " << cephx_entity << " lockbox " << lockbox_entity << dendl; @@ -1160,14 +1161,13 @@ int AuthMonitor::do_osd_destroy( if (!removed) { dout(10) << __func__ << " entities do not exist -- no-op." << dendl; - return 0; + return; } // given we have paxos plugged, this will not result in a proposal // being triggered, but it will still be needed so that we get our // pending state encoded into the paxos' pending transaction. propose_pending(); - return 0; } int _create_auth( diff --git a/src/mon/AuthMonitor.h b/src/mon/AuthMonitor.h index 993b18a02b24..d49a862d3fcc 100644 --- a/src/mon/AuthMonitor.h +++ b/src/mon/AuthMonitor.h @@ -202,9 +202,9 @@ private: EntityName& cephx_entity, EntityName& lockbox_entity, std::stringstream& ss); - int do_osd_destroy( - const EntityName& cephx_entity, - const EntityName& lockbox_entity); + void do_osd_destroy( + const EntityName& cephx_entity, + const EntityName& lockbox_entity); int do_osd_new( const auth_entity_t& cephx_entity, diff --git a/src/mon/KVMonitor.cc b/src/mon/KVMonitor.cc index 37a81a8048d4..0e9511c7c967 100644 --- a/src/mon/KVMonitor.cc +++ b/src/mon/KVMonitor.cc @@ -368,6 +368,8 @@ int KVMonitor::validate_osd_destroy( void KVMonitor::do_osd_destroy(int32_t id, uuid_d& uuid) { + ceph_assert(is_writeable()); + string dmcrypt_prefix = _get_dmcrypt_prefix(uuid, ""); string daemon_prefix = "daemon-private/osd." + stringify(id) + "/"; @@ -416,6 +418,7 @@ void KVMonitor::do_osd_new( const string& dmcrypt_key) { ceph_assert(paxos.is_plugged()); + ceph_assert(is_writeable()); string dmcrypt_key_prefix = _get_dmcrypt_prefix(uuid, "luks"); bufferlist dmcrypt_key_value; diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index 6543da85c446..1a37149c2309 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -9829,6 +9829,7 @@ static int parse_reweights(CephContext *cct, } int OSDMonitor::prepare_command_osd_destroy( + MonOpRequestRef op, int32_t id, stringstream& ss) { @@ -9860,10 +9861,11 @@ int OSDMonitor::prepare_command_osd_destroy( EntityName cephx_entity, lockbox_entity; bool idempotent_auth = false, idempotent_cks = false; - int err = mon.authmon()->validate_osd_destroy(id, uuid, - cephx_entity, - lockbox_entity, - ss); + auto&& authmon = mon.authmon(); + int err = authmon->validate_osd_destroy(id, uuid, + cephx_entity, + lockbox_entity, + ss); if (err < 0) { if (err == -ENOENT) { idempotent_auth = true; @@ -9872,21 +9874,28 @@ int OSDMonitor::prepare_command_osd_destroy( } } - auto svc = mon.kvmon(); - err = svc->validate_osd_destroy(id, uuid); + auto&& kvmon = mon.kvmon(); + err = kvmon->validate_osd_destroy(id, uuid); if (err < 0) { ceph_assert(err == -ENOENT); err = 0; idempotent_cks = true; } - if (!idempotent_auth) { - err = mon.authmon()->do_osd_destroy(cephx_entity, lockbox_entity); - ceph_assert(0 == err); + if (!idempotent_auth && !authmon->is_writeable()) { + authmon->wait_for_writeable(op, new C_RetryMessage(this, op)); + return -EAGAIN; + } + if (!idempotent_cks && !kvmon->is_writeable()) { + kvmon->wait_for_writeable(op, new C_RetryMessage(this, op)); + return -EAGAIN; } + if (!idempotent_auth) { + authmon->do_osd_destroy(cephx_entity, lockbox_entity); + } if (!idempotent_cks) { - svc->do_osd_destroy(id, uuid); + kvmon->do_osd_destroy(id, uuid); } pending_inc.new_state[id] = CEPH_OSD_DESTROYED; @@ -9901,6 +9910,7 @@ int OSDMonitor::prepare_command_osd_destroy( } int OSDMonitor::prepare_command_osd_purge( + MonOpRequestRef op, int32_t id, stringstream& ss) { @@ -9941,7 +9951,11 @@ int OSDMonitor::prepare_command_osd_purge( // no point destroying the osd again if it has already been marked destroyed if (!osdmap.is_destroyed(id)) { - err = prepare_command_osd_destroy(id, ss); + /* N.B.: up to this point, we've not changed pending at all. + * ::prepare_command_osd_destroy may return -EAGAIN if the kvmon/authmon is + * not writeable without changing pending. It will queue `op` if we should wait. + */ + err = prepare_command_osd_destroy(op, id, ss); if (err < 0) { if (err == -ENOENT) { err = 0; @@ -12722,11 +12736,17 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op, paxos.plug(); if (is_destroy) { - err = prepare_command_osd_destroy(id, ss); + err = prepare_command_osd_destroy(op, id, ss); + if (err == EAGAIN) { + return false; + } // we checked above that it should exist. ceph_assert(err != -ENOENT); } else { - err = prepare_command_osd_purge(id, ss); + err = prepare_command_osd_purge(op, id, ss); + if (err == EAGAIN) { + return false; + } if (err == -ENOENT) { err = 0; ss << "osd." << id << " does not exist."; diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h index e20da536b33a..dc3d342de1bd 100644 --- a/src/mon/OSDMonitor.h +++ b/src/mon/OSDMonitor.h @@ -718,8 +718,8 @@ public: void do_osd_create(const int32_t id, const uuid_d& uuid, const std::string& device_class, int32_t* new_id); - int prepare_command_osd_purge(int32_t id, std::stringstream& ss); - int prepare_command_osd_destroy(int32_t id, std::stringstream& ss); + int prepare_command_osd_purge(MonOpRequestRef op, int32_t id, std::stringstream& ss); + int prepare_command_osd_destroy(MonOpRequestRef op, int32_t id, std::stringstream& ss); int _prepare_command_osd_crush_remove( CrushWrapper &newcrush, int32_t id,