From: Samuel Just Date: Sat, 13 Jul 2024 02:35:11 +0000 (+0000) Subject: mon/NVMeofGwMon.cc: tabbing, line length, formatting X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=080303c4844c8eead49e6e6c15dcf84eb8b5d2d2;p=ceph-ci.git mon/NVMeofGwMon.cc: tabbing, line length, formatting - Retabs file to match emacs/vim modelines at top - Fixes bracing - Adjusts line length to 80 char Signed-off-by: Samuel Just (cherry picked from commit bff9dd47068f4a4afcdec117e841a3580cd36191) --- diff --git a/src/mon/NVMeofGwMon.cc b/src/mon/NVMeofGwMon.cc index 6111f76f425..ac4a6e199fb 100644 --- a/src/mon/NVMeofGwMon.cc +++ b/src/mon/NVMeofGwMon.cc @@ -24,92 +24,108 @@ using std::string; -void NVMeofGwMon::init() { - dout(10) << "called " << dendl; +void NVMeofGwMon::init() +{ + dout(10) << "called " << dendl; } -void NVMeofGwMon::on_restart() { - dout(10) << "called " << dendl; - last_beacon.clear(); - last_tick = ceph::coarse_mono_clock::now(); - synchronize_last_beacon(); +void NVMeofGwMon::on_restart() +{ + dout(10) << "called " << dendl; + last_beacon.clear(); + last_tick = ceph::coarse_mono_clock::now(); + synchronize_last_beacon(); } -void NVMeofGwMon::synchronize_last_beacon() { - dout(10) << "called, is leader : " << mon.is_leader() <<" active " << is_active() << dendl; - // Initialize last_beacon to identify transitions of available GWs to unavailable state - for (const auto& created_map_pair: map.created_gws) { - const auto& group_key = created_map_pair.first; - const NvmeGwMonStates& gw_created_map = created_map_pair.second; - for (const auto& gw_created_pair: gw_created_map) { - const auto& gw_id = gw_created_pair.first; - if (gw_created_pair.second.availability == gw_availability_t::GW_AVAILABLE) { - dout(10) << "synchronize last_beacon for GW :" << gw_id << dendl; - LastBeacon lb = {gw_id, group_key}; - last_beacon[lb] = last_tick; - } +void NVMeofGwMon::synchronize_last_beacon() +{ + dout(10) << "called, is leader : " << mon.is_leader() + << " active " << is_active() << dendl; + // Initialize last_beacon to identify transitions of available + // GWs to unavailable state + for (const auto& created_map_pair: map.created_gws) { + const auto& group_key = created_map_pair.first; + const NvmeGwMonStates& gw_created_map = created_map_pair.second; + for (const auto& gw_created_pair: gw_created_map) { + const auto& gw_id = gw_created_pair.first; + if (gw_created_pair.second.availability == + gw_availability_t::GW_AVAILABLE) { + dout(10) << "synchronize last_beacon for GW :" << gw_id << dendl; + LastBeacon lb = {gw_id, group_key}; + last_beacon[lb] = last_tick; } } + } } -void NVMeofGwMon::on_shutdown() { - dout(10) << "called " << dendl; +void NVMeofGwMon::on_shutdown() +{ + dout(10) << "called " << dendl; } -void NVMeofGwMon::tick() { - if (!is_active() || !mon.is_leader()) { - dout(10) << "NVMeofGwMon leader : " << mon.is_leader() << "active : " << is_active() << dendl; - return; - } - bool _propose_pending = false; +void NVMeofGwMon::tick() +{ + if (!is_active() || !mon.is_leader()) { + dout(10) << "NVMeofGwMon leader : " << mon.is_leader() + << "active : " << is_active() << dendl; + return; + } + bool _propose_pending = false; - const auto now = ceph::coarse_mono_clock::now(); - const auto nvmegw_beacon_grace = g_conf().get_val("mon_nvmeofgw_beacon_grace"); - dout(15) << "NVMeofGwMon leader got a tick, pending epoch "<< pending_map.epoch << dendl; - - const auto client_tick_period = g_conf().get_val("nvmeof_mon_client_tick_period"); - //handle exception of tick overdued in order to avoid false detection of overdued beacons, like it done in MgrMonitor::tick - if (last_tick != ceph::coarse_mono_clock::zero() - && (now - last_tick > (nvmegw_beacon_grace - client_tick_period))) { - // This case handles either local slowness (calls being delayed - // for whatever reason) or cluster election slowness (a long gap - // between calls while an election happened) - dout(10) << ": resetting beacon timeouts due to mon delay " - "(slow election?) of " << now - last_tick << " seconds" << dendl; - for (auto &i : last_beacon) { - i.second = now; - } + const auto now = ceph::coarse_mono_clock::now(); + const auto nvmegw_beacon_grace = + g_conf().get_val("mon_nvmeofgw_beacon_grace"); + dout(15) << "NVMeofGwMon leader got a tick, pending epoch " + << pending_map.epoch << dendl; + + const auto client_tick_period = + g_conf().get_val("nvmeof_mon_client_tick_period"); + // handle exception of tick overdued in order to avoid false detection of + // overdued beacons, like it done in MgrMonitor::tick + if (last_tick != ceph::coarse_mono_clock::zero() && + (now - last_tick > (nvmegw_beacon_grace - client_tick_period))) { + // This case handles either local slowness (calls being delayed + // for whatever reason) or cluster election slowness (a long gap + // between calls while an election happened) + dout(10) << ": resetting beacon timeouts due to mon delay " + "(slow election?) of " << now - last_tick << " seconds" << dendl; + for (auto &i : last_beacon) { + i.second = now; } + } - last_tick = now; - bool propose = false; - - pending_map.update_active_timers(propose); // Periodic: check active FSM timers - _propose_pending |= propose; - - const auto cutoff = now - nvmegw_beacon_grace; - for (auto &itr : last_beacon) {// Pass over all the stored beacons - auto& lb = itr.first; - auto last_beacon_time = itr.second; - if (last_beacon_time < cutoff) { - dout(10) << "beacon timeout for GW " << lb.gw_id << dendl; - pending_map.process_gw_map_gw_down(lb.gw_id, lb.group_key, propose); - _propose_pending |= propose; - last_beacon.erase(lb); - } - else { - dout(20) << "beacon live for GW key: " << lb.gw_id << dendl; - } + last_tick = now; + bool propose = false; + + // Periodic: check active FSM timers + pending_map.update_active_timers(propose); + _propose_pending |= propose; + + const auto cutoff = now - nvmegw_beacon_grace; + + // Pass over all the stored beacons + for (auto &itr : last_beacon) { + auto& lb = itr.first; + auto last_beacon_time = itr.second; + if (last_beacon_time < cutoff) { + dout(10) << "beacon timeout for GW " << lb.gw_id << dendl; + pending_map.process_gw_map_gw_down(lb.gw_id, lb.group_key, propose); + _propose_pending |= propose; + last_beacon.erase(lb); + } else { + dout(20) << "beacon live for GW key: " << lb.gw_id << dendl; } + } - pending_map.handle_abandoned_ana_groups(propose); // Periodic: take care of not handled ANA groups - _propose_pending |= propose; + // Periodic: take care of not handled ANA groups + pending_map.handle_abandoned_ana_groups(propose); + _propose_pending |= propose; - if (_propose_pending) { - dout(10) << "propose pending " <next , map-epoch " << sub->next << " " << map.epoch << dendl; - if (sub->next <= map.epoch) - { - dout(10) << "Sending map to subscriber " << sub->session->con << " " << sub->session->con->get_peer_addr() << dendl; - sub->session->con->send_message2(make_message(map)); - - if (sub->onetime) { - mon.session_map.remove_sub(sub); - } else { - sub->next = map.epoch + 1; - } + dout(10) << "sub->next , map-epoch " << sub->next + << " " << map.epoch << dendl; + if (sub->next <= map.epoch) + { + dout(10) << "Sending map to subscriber " << sub->session->con + << " " << sub->session->con->get_peer_addr() << dendl; + sub->session->con->send_message2(make_message(map)); + + if (sub->onetime) { + mon.session_map.remove_sub(sub); + } else { + sub->next = map.epoch + 1; } + } } void NVMeofGwMon::check_subs(bool t) @@ -188,345 +207,382 @@ void NVMeofGwMon::check_subs(bool t) dout(10) << "count " << mon.session_map.subs.count(type) << dendl; if (mon.session_map.subs.count(type) == 0) { - return; + return; } for (auto sub : *(mon.session_map.subs[type])) { check_sub(sub); } } -bool NVMeofGwMon::preprocess_query(MonOpRequestRef op) { - dout(20) << dendl; - - auto m = op->get_req(); - switch (m->get_type()) { - case MSG_MNVMEOF_GW_BEACON: - return preprocess_beacon(op); - - case MSG_MON_COMMAND: - try { - return preprocess_command(op); - } catch (const bad_cmd_get& e) { - bufferlist bl; - mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed()); - return true; - } - - default: - mon.no_reply(op); - derr << "Unhandled message type " << m->get_type() << dendl; - return true; - } - return false; +bool NVMeofGwMon::preprocess_query(MonOpRequestRef op) +{ + dout(20) << dendl; + + auto m = op->get_req(); + switch (m->get_type()) { + case MSG_MNVMEOF_GW_BEACON: + return preprocess_beacon(op); + + case MSG_MON_COMMAND: + try { + return preprocess_command(op); + } catch (const bad_cmd_get& e) { + bufferlist bl; + mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed()); + return true; + } + + default: + mon.no_reply(op); + derr << "Unhandled message type " << m->get_type() << dendl; + return true; + } + return false; } -bool NVMeofGwMon::prepare_update(MonOpRequestRef op) { - auto m = op->get_req(); - switch (m->get_type()) { - case MSG_MNVMEOF_GW_BEACON: - return prepare_beacon(op); - - case MSG_MON_COMMAND: - try { - return prepare_command(op); - } catch (const bad_cmd_get& e) { - bufferlist bl; - mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed()); - return false; /* nothing to propose! */ - } - - default: - mon.no_reply(op); - dout(1) << "Unhandled message type " << m->get_type() << dendl; - return false; /* nothing to propose! */ - } +bool NVMeofGwMon::prepare_update(MonOpRequestRef op) +{ + auto m = op->get_req(); + switch (m->get_type()) { + case MSG_MNVMEOF_GW_BEACON: + return prepare_beacon(op); + + case MSG_MON_COMMAND: + try { + return prepare_command(op); + } catch (const bad_cmd_get& e) { + bufferlist bl; + mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed()); + return false; /* nothing to propose! */ + } + + default: + mon.no_reply(op); + dout(1) << "Unhandled message type " << m->get_type() << dendl; + return false; /* nothing to propose! */ + } } bool NVMeofGwMon::preprocess_command(MonOpRequestRef op) { - dout(10) << dendl; - auto m = op->get_req(); - std::stringstream sstrm; - bufferlist rdata; - string rs; - int err = 0; - cmdmap_t cmdmap; - if (!cmdmap_from_json(m->cmd, &cmdmap, sstrm)) - { - string rs = sstrm.str(); - dout(4) << "Error : Invalid command " << m->cmd << "Error " << rs << dendl; - mon.reply_command(op, -EINVAL, rs, rdata, get_last_committed()); - return true; - } + dout(10) << dendl; + auto m = op->get_req(); + std::stringstream sstrm; + bufferlist rdata; + string rs; + int err = 0; + cmdmap_t cmdmap; + if (!cmdmap_from_json(m->cmd, &cmdmap, sstrm)) + { + string rs = sstrm.str(); + dout(4) << "Error : Invalid command " << m->cmd + << "Error " << rs << dendl; + mon.reply_command(op, -EINVAL, rs, rdata, get_last_committed()); + return true; + } - string prefix; - cmd_getval(cmdmap, "prefix", prefix); - dout(10) << "MonCommand : "<< prefix << dendl; - string format = cmd_getval_or(cmdmap, "format", "plain"); - boost::scoped_ptr f(Formatter::create(format)); - if (prefix == "nvme-gw show") { - std::string pool, group; - if (!f) { - f.reset(Formatter::create(format, "json-pretty", "json-pretty")); - } - cmd_getval(cmdmap, "pool", pool); - cmd_getval(cmdmap, "group", group); - auto group_key = std::make_pair(pool, group); - dout(10) <<"nvme-gw show pool "<< pool << " group "<< group << dendl; - - if (map.created_gws[group_key].size()) { - f->open_object_section("common"); - f->dump_unsigned("epoch", map.epoch); - f->dump_string("pool", pool); - f->dump_string("group", group); - f->dump_unsigned("num gws", map.created_gws[group_key].size()); - sstrm <<"[ "; - NvmeGwId gw_id; - for (auto& gw_created_pair: map.created_gws[group_key]) { - gw_id = gw_created_pair.first; - auto& st = gw_created_pair.second; - sstrm << st.ana_grp_id+1 << " "; - } - sstrm << "]"; - f->dump_string("Anagrp list", sstrm.str()); - f->close_section(); - - for (auto& gw_created_pair: map.created_gws[group_key]) { - auto& gw_id = gw_created_pair.first; - auto& state = gw_created_pair.second; - f->open_object_section("stat"); - f->dump_string("gw-id", gw_id); - f->dump_unsigned("anagrp-id",state.ana_grp_id+1); - f->dump_unsigned("performed-full-startup", state.performed_full_startup); - std::stringstream sstrm1; - sstrm1 << state.availability; - f->dump_string("Availability", sstrm1.str()); - sstrm1.str(""); - for (auto &state_itr: map.created_gws[group_key][gw_id].sm_state) { - sstrm1 << " " << state_itr.first + 1 << ": " << state.sm_state[state_itr.first] << ","; - } - f->dump_string("ana states", sstrm1.str()); - f->close_section(); - } - f->flush(rdata); - sstrm.str(""); - } - else { - sstrm << "num_gws 0"; - } - getline(sstrm, rs); - mon.reply_command(op, err, rs, rdata, get_last_committed()); - return true; + string prefix; + cmd_getval(cmdmap, "prefix", prefix); + dout(10) << "MonCommand : "<< prefix << dendl; + string format = cmd_getval_or(cmdmap, "format", "plain"); + boost::scoped_ptr f(Formatter::create(format)); + if (prefix == "nvme-gw show") { + std::string pool, group; + if (!f) { + f.reset(Formatter::create(format, "json-pretty", "json-pretty")); + } + cmd_getval(cmdmap, "pool", pool); + cmd_getval(cmdmap, "group", group); + auto group_key = std::make_pair(pool, group); + dout(10) << "nvme-gw show pool " << pool << " group " << group << dendl; + + if (map.created_gws[group_key].size()) { + f->open_object_section("common"); + f->dump_unsigned("epoch", map.epoch); + f->dump_string("pool", pool); + f->dump_string("group", group); + f->dump_unsigned("num gws", map.created_gws[group_key].size()); + sstrm << "[ "; + NvmeGwId gw_id; + for (auto& gw_created_pair: map.created_gws[group_key]) { + gw_id = gw_created_pair.first; + auto& st = gw_created_pair.second; + sstrm << st.ana_grp_id+1 << " "; + } + sstrm << "]"; + f->dump_string("Anagrp list", sstrm.str()); + f->close_section(); + + for (auto& gw_created_pair: map.created_gws[group_key]) { + auto& gw_id = gw_created_pair.first; + auto& state = gw_created_pair.second; + f->open_object_section("stat"); + f->dump_string("gw-id", gw_id); + f->dump_unsigned("anagrp-id",state.ana_grp_id+1); + f->dump_unsigned("performed-full-startup", state.performed_full_startup); + std::stringstream sstrm1; + sstrm1 << state.availability; + f->dump_string("Availability", sstrm1.str()); + sstrm1.str(""); + for (auto &state_itr: map.created_gws[group_key][gw_id].sm_state) { + sstrm1 << " " << state_itr.first + 1 << ": " + << state.sm_state[state_itr.first] << ","; + } + f->dump_string("ana states", sstrm1.str()); + f->close_section(); + } + f->flush(rdata); + sstrm.str(""); } - return false; + else { + sstrm << "num_gws 0"; + } + getline(sstrm, rs); + mon.reply_command(op, err, rs, rdata, get_last_committed()); + return true; + } + return false; } bool NVMeofGwMon::prepare_command(MonOpRequestRef op) { - dout(10) << dendl; - auto m = op->get_req(); - int rc; - std::stringstream sstrm; - bufferlist rdata; - string rs; - int err = 0; - cmdmap_t cmdmap; - bool response = false; - - if (!cmdmap_from_json(m->cmd, &cmdmap, sstrm)) - { - string rs = sstrm.str(); - mon.reply_command(op, -EINVAL, rs, rdata, get_last_committed()); - return true; - } + dout(10) << dendl; + auto m = op->get_req(); + int rc; + std::stringstream sstrm; + bufferlist rdata; + string rs; + int err = 0; + cmdmap_t cmdmap; + bool response = false; + + if (!cmdmap_from_json(m->cmd, &cmdmap, sstrm)) + { + string rs = sstrm.str(); + mon.reply_command(op, -EINVAL, rs, rdata, get_last_committed()); + return true; + } - string format = cmd_getval_or(cmdmap, "format", "plain"); - boost::scoped_ptr f(Formatter::create(format)); - - const auto prefix = cmd_getval_or(cmdmap, "prefix", string{}); - - dout(10) << "MonCommand : "<< prefix << dendl; - if (prefix == "nvme-gw create" || prefix == "nvme-gw delete") { - std::string id, pool, group; - - cmd_getval(cmdmap, "id", id); - cmd_getval(cmdmap, "pool", pool); - cmd_getval(cmdmap, "group", group); - auto group_key = std::make_pair(pool, group); - dout(10) << " id "<< id <<" pool "<< pool << " group "<< group << dendl; - if (prefix == "nvme-gw create") { - rc = pending_map.cfg_add_gw(id, group_key); - if (rc == -EINVAL) { - err = rc; - dout (4) << "Error: GW cannot be created " << id << " " << pool << " " << group << " rc " << rc << dendl; - sstrm.str(""); - } - } - else{ - rc = pending_map.cfg_delete_gw(id, group_key); - if (rc == -EINVAL) { - dout (4) << "Error: GW not found in the database " << id << " " << pool << " " << group << " rc " << rc << dendl; - err = 0; - sstrm.str(""); - } - } - if ((rc != -EEXIST) && (rc != -EINVAL)) //propose pending would be generated by the PaxosService - response = true; + string format = cmd_getval_or(cmdmap, "format", "plain"); + boost::scoped_ptr f(Formatter::create(format)); + + const auto prefix = cmd_getval_or(cmdmap, "prefix", string{}); + + dout(10) << "MonCommand : "<< prefix << dendl; + if (prefix == "nvme-gw create" || prefix == "nvme-gw delete") { + std::string id, pool, group; + + cmd_getval(cmdmap, "id", id); + cmd_getval(cmdmap, "pool", pool); + cmd_getval(cmdmap, "group", group); + auto group_key = std::make_pair(pool, group); + dout(10) << " id "<< id <<" pool "<< pool << " group "<< group << dendl; + if (prefix == "nvme-gw create") { + rc = pending_map.cfg_add_gw(id, group_key); + if (rc == -EINVAL) { + err = rc; + dout (4) << "Error: GW cannot be created " << id + << " " << pool << " " << group << " rc " << rc << dendl; + sstrm.str(""); + } + } + else{ + rc = pending_map.cfg_delete_gw(id, group_key); + if (rc == -EINVAL) { + dout (4) << "Error: GW not found in the database " << id << " " + << pool << " " << group << " rc " << rc << dendl; + err = 0; + sstrm.str(""); + } + } + // propose pending would be generated by the PaxosService + if ((rc != -EEXIST) && (rc != -EINVAL)) { + response = true; } + } - getline(sstrm, rs); - if (response == false) { - if (err < 0 && rs.length() == 0) - { - rs = cpp_strerror(err); - dout(10) << "Error command err : "<< err << " rs-len: " << rs.length() << dendl; - } - mon.reply_command(op, err, rs, rdata, get_last_committed()); + getline(sstrm, rs); + if (response == false) { + if (err < 0 && rs.length() == 0) { + rs = cpp_strerror(err); + dout(10) << "Error command err : "<< err << " rs-len: " + << rs.length() << dendl; } - else - wait_for_commit(op, new Monitor::C_Command(mon, op, 0, rs, - get_last_committed() + 1)); - return response; + mon.reply_command(op, err, rs, rdata, get_last_committed()); + } else { + wait_for_commit(op, new Monitor::C_Command(mon, op, 0, rs, + get_last_committed() + 1)); + } + return response; } -bool NVMeofGwMon::preprocess_beacon(MonOpRequestRef op) { - auto m = op->get_req(); - const BeaconSubsystems& sub = m->get_subsystems(); - dout(15) << "beacon from " << m->get_type() << " GW : " << m->get_gw_id() << " num subsystems " << sub.size() << dendl; - - return false; // allways return false to call leader's prepare beacon +bool NVMeofGwMon::preprocess_beacon(MonOpRequestRef op) +{ + auto m = op->get_req(); + const BeaconSubsystems& sub = m->get_subsystems(); + dout(15) << "beacon from " << m->get_type() + << " GW : " << m->get_gw_id() + << " num subsystems " << sub.size() << dendl; + + // allways return false to call leader's prepare beacon + return false; } -bool NVMeofGwMon::prepare_beacon(MonOpRequestRef op) { - auto m = op->get_req(); - - dout(20) << "availability " << m->get_availability() << " GW : " << m->get_gw_id() << - " osdmap_epoch " << m->get_last_osd_epoch() << " subsystems " << m->get_subsystems() << dendl; - - NvmeGwId gw_id = m->get_gw_id(); - NvmeGroupKey group_key = std::make_pair(m->get_gw_pool(), m->get_gw_group()); - gw_availability_t avail = m->get_availability(); - bool propose = false; - bool nonce_propose = false; - bool timer_propose = false; - bool gw_created = true; - NVMeofGwMap ack_map; - auto& group_gws = map.created_gws[group_key]; - auto gw = group_gws.find(gw_id); - const BeaconSubsystems& sub = m->get_subsystems(); - - if (avail == gw_availability_t::GW_CREATED) { - if (gw == group_gws.end()) { - gw_created = false; - dout(10) << "Warning: GW " << gw_id << " group_key " << group_key << " was not found in the map.Created_gws "<< map.created_gws <get_req(); + + dout(20) << "availability " << m->get_availability() + << " GW : " << m->get_gw_id() + << " osdmap_epoch " << m->get_last_osd_epoch() + << " subsystems " << m->get_subsystems() << dendl; + + NvmeGwId gw_id = m->get_gw_id(); + NvmeGroupKey group_key = std::make_pair(m->get_gw_pool(), m->get_gw_group()); + gw_availability_t avail = m->get_availability(); + bool propose = false; + bool nonce_propose = false; + bool timer_propose = false; + bool gw_created = true; + NVMeofGwMap ack_map; + auto& group_gws = map.created_gws[group_key]; + auto gw = group_gws.find(gw_id); + const BeaconSubsystems& sub = m->get_subsystems(); + + if (avail == gw_availability_t::GW_CREATED) { + if (gw == group_gws.end()) { + gw_created = false; + dout(10) << "Warning: GW " << gw_id << " group_key " << group_key + << " was not found in the map.Created_gws " + << map.created_gws << dendl; + goto set_propose; + } else { + dout(10) << "GW prepares the full startup " << gw_id + << " GW availability: " + << pending_map.created_gws[group_key][gw_id].availability + << dendl; + if (pending_map.created_gws[group_key][gw_id].availability == + gw_availability_t::GW_AVAILABLE) { + dout(4) << " Warning :GW marked as Available in the NVmeofGwMon " + << "database, performed full startup - Force gw to exit!" + << gw_id << dendl; + avail = gw_availability_t::GW_UNAVAILABLE; + // Monitor performs Force Failover for this GW in process_gw_map_gw_down + } else if ( + pending_map.created_gws[group_key][gw_id].performed_full_startup == + false) { + pending_map.created_gws[group_key][gw_id].performed_full_startup = true; + propose = true; + goto set_propose; + } } - else { // gw already created - if (gw != group_gws.end()) // if GW reports Available but in monitor's database it is Unavailable - // it means it did not perform "exit" after failover was set by NVMeofGWMon - if (pending_map.created_gws[group_key][gw_id].availability == gw_availability_t::GW_UNAVAILABLE && - pending_map.created_gws[group_key][gw_id].performed_full_startup == false && - avail == gw_availability_t::GW_AVAILABLE) { - ack_map.created_gws[group_key][gw_id] = pending_map.created_gws[group_key][gw_id]; - ack_map.epoch = map.epoch; - dout(4) << " Force gw to exit: Sending ack_map to GW: " << gw_id << dendl; - auto msg = make_message(ack_map); - mon.send_reply(op, msg.detach()); - goto false_return; - } + // gw already created + } else { + // if GW reports Available but in monitor's database it is Unavailable + if (gw != group_gws.end()) { + // it means it did not perform "exit" after failover was set by + // NVMeofGWMon + if ((pending_map.created_gws[group_key][gw_id].availability == + gw_availability_t::GW_UNAVAILABLE) && + (pending_map.created_gws[group_key][gw_id].performed_full_startup == + false) && + avail == gw_availability_t::GW_AVAILABLE) { + ack_map.created_gws[group_key][gw_id] = + pending_map.created_gws[group_key][gw_id]; + ack_map.epoch = map.epoch; + dout(4) << " Force gw to exit: Sending ack_map to GW: " + << gw_id << dendl; + auto msg = make_message(ack_map); + mon.send_reply(op, msg.detach()); + goto false_return; + } } + } - // At this stage the gw has to be in the Created_gws - if (gw == group_gws.end()) { - dout(4) << "Administratively deleted GW sends beacon " << gw_id <get_nonce_map().size()) { - if (pending_map.created_gws[group_key][gw_id].nonce_map != m->get_nonce_map()) - { - dout(10) << "nonce map of GW changed , propose pending " << gw_id << dendl; - pending_map.created_gws[group_key][gw_id].nonce_map = m->get_nonce_map(); - dout(10) << "nonce map of GW " << gw_id << " "<< pending_map.created_gws[group_key][gw_id].nonce_map << dendl; - nonce_propose = true; - } - } - else { - dout(10) << "Warning: received empty nonce map in the beacon of GW " << gw_id << " "<< dendl; + // deep copy the whole nonce map of this GW + if (m->get_nonce_map().size()) { + if (pending_map.created_gws[group_key][gw_id].nonce_map != + m->get_nonce_map()) { + dout(10) << "nonce map of GW changed , propose pending " + << gw_id << dendl; + pending_map.created_gws[group_key][gw_id].nonce_map = m->get_nonce_map(); + dout(10) << "nonce map of GW " << gw_id << " " + << pending_map.created_gws[group_key][gw_id].nonce_map << dendl; + nonce_propose = true; } + } else { + dout(10) << "Warning: received empty nonce map in the beacon of GW " + << gw_id << " " << dendl; + } - if (sub.size() == 0) { - avail = gw_availability_t::GW_UNAVAILABLE; - } - if (pending_map.created_gws[group_key][gw_id].subsystems != sub) - { - dout(10) << "subsystems of GW changed, propose pending " << gw_id << dendl; - pending_map.created_gws[group_key][gw_id].subsystems = sub; - dout(20) << "subsystems of GW " << gw_id << " "<< pending_map.created_gws[group_key][gw_id].subsystems << dendl; - nonce_propose = true; - } - pending_map.created_gws[group_key][gw_id].last_gw_map_epoch_valid = ( map.epoch == m->get_last_gwmap_epoch() ); - if (pending_map.created_gws[group_key][gw_id].last_gw_map_epoch_valid == false) { - dout(20) << "map epoch of gw is not up-to-date " << gw_id << " epoch " << map.epoch << " beacon_epoch " << m->get_last_gwmap_epoch() << dendl; - } - if (avail == gw_availability_t::GW_AVAILABLE) - { - auto now = ceph::coarse_mono_clock::now(); - // check pending_map.epoch vs m->get_version() - if different - drop the beacon - - LastBeacon lb = {gw_id, group_key}; - last_beacon[lb] = now; - epoch_t last_osd_epoch = m->get_last_osd_epoch(); - pending_map.process_gw_map_ka(gw_id, group_key, last_osd_epoch, propose); - } - else if (avail == gw_availability_t::GW_UNAVAILABLE) { // state set by GW client application - LastBeacon lb = {gw_id, group_key}; - - auto it = last_beacon.find(lb); - if (it != last_beacon.end()) { - last_beacon.erase(lb); - pending_map.process_gw_map_gw_down(gw_id, group_key, propose); - } + if (sub.size() == 0) { + avail = gw_availability_t::GW_UNAVAILABLE; + } + if (pending_map.created_gws[group_key][gw_id].subsystems != sub) { + dout(10) << "subsystems of GW changed, propose pending " << gw_id << dendl; + pending_map.created_gws[group_key][gw_id].subsystems = sub; + dout(20) << "subsystems of GW " << gw_id << " " + << pending_map.created_gws[group_key][gw_id].subsystems << dendl; + nonce_propose = true; + } + pending_map.created_gws[group_key][gw_id].last_gw_map_epoch_valid = + (map.epoch == m->get_last_gwmap_epoch()); + if (pending_map.created_gws[group_key][gw_id].last_gw_map_epoch_valid == + false) { + dout(20) << "map epoch of gw is not up-to-date " << gw_id + << " epoch " << map.epoch + << " beacon_epoch " << m->get_last_gwmap_epoch() << dendl; + } + if (avail == gw_availability_t::GW_AVAILABLE) { + auto now = ceph::coarse_mono_clock::now(); + // check pending_map.epoch vs m->get_version() - + // if different - drop the beacon + + LastBeacon lb = {gw_id, group_key}; + last_beacon[lb] = now; + epoch_t last_osd_epoch = m->get_last_osd_epoch(); + pending_map.process_gw_map_ka(gw_id, group_key, last_osd_epoch, propose); + // state set by GW client application + } else if (avail == gw_availability_t::GW_UNAVAILABLE) { + LastBeacon lb = {gw_id, group_key}; + + auto it = last_beacon.find(lb); + if (it != last_beacon.end()) { + last_beacon.erase(lb); + pending_map.process_gw_map_gw_down(gw_id, group_key, propose); } - pending_map.update_active_timers(timer_propose); // Periodic: check active FSM timers - propose |= timer_propose; - propose |= nonce_propose; + } + // Periodic: check active FSM timers + pending_map.update_active_timers(timer_propose); + propose |= timer_propose; + propose |= nonce_propose; set_propose: - if (!propose) { - if (gw_created) { - ack_map.created_gws[group_key][gw_id] = map.created_gws[group_key][gw_id];// respond with a map slice correspondent to the same GW - } - ack_map.epoch = map.epoch; - dout(20) << "ack_map " << ack_map <(ack_map); - mon.send_reply(op, msg.detach()); - } - else { - mon.no_reply(op); + if (!propose) { + if (gw_created) { + // respond with a map slice correspondent to the same GW + ack_map.created_gws[group_key][gw_id] = map.created_gws[group_key][gw_id]; } + ack_map.epoch = map.epoch; + dout(20) << "ack_map " << ack_map <(ack_map); + mon.send_reply(op, msg.detach()); + } else { + mon.no_reply(op); + } false_return: - if (propose) { - dout(10) << "decision in prepare_beacon" <