osd/OSD: osd_fast_shutdown_notify_mon not quite right

When osd_fast_shutdown and osd_fast_shutdown_notify_mon set as true, OSD marked as Down
it should be marked as Dead,

Fixed: https://tracker.ceph.com/issues/53327

Signed-off-by: Nitzan Mordechai <nmordech@redhat.com>
nd

nd

diff --git a/src/messages/MOSDMarkMeDown.h b/src/messages/MOSDMarkMeDown.h
index e9428518639c882ea042f4a774355631952a6217..88d618613d2a43da4f8eb71c33fece4e548ccc23 100644 (file)
--- a/src/messages/MOSDMarkMeDown.h
+++ b/src/messages/MOSDMarkMeDown.h
@@ -19,7 +19,7 @@
 
 class MOSDMarkMeDown final : public PaxosServiceMessage {
 private:
-  static constexpr int HEAD_VERSION = 3;
+  static constexpr int HEAD_VERSION = 4;
   static constexpr int COMPAT_VERSION = 3;
 
  public:
@@ -28,6 +28,7 @@ private:
   entity_addrvec_t target_addrs;
   epoch_t epoch = 0;
   bool request_ack = false;          // ack requested
+  bool down_and_dead = false;        // mark down and dead
 
   MOSDMarkMeDown()
     : PaxosServiceMessage{MSG_OSD_MARK_ME_DOWN, 0,
@@ -38,6 +39,12 @@ private:
                          HEAD_VERSION, COMPAT_VERSION},
       fsid(fs), target_osd(osd), target_addrs(av),
       epoch(e), request_ack(request_ack) {}
+  MOSDMarkMeDown(const uuid_d &fs, int osd, const entity_addrvec_t& av,
+                epoch_t e, bool request_ack, bool down_and_dead)
+    : PaxosServiceMessage{MSG_OSD_MARK_ME_DOWN, e,
+                         HEAD_VERSION, COMPAT_VERSION},
+      fsid(fs), target_osd(osd), target_addrs(av),
+      epoch(e), request_ack(request_ack), down_and_dead(down_and_dead) {}
  private:
   ~MOSDMarkMeDown() final {}
 
@@ -54,6 +61,8 @@ public:
     decode(target_addrs, p);
     decode(epoch, p);
     decode(request_ack, p);
+    if(header.version >= 4)
+      decode(down_and_dead, p);
   }
 
   void encode_payload(uint64_t features) override {
@@ -67,12 +76,14 @@ public:
     encode(target_addrs, payload, features);
     encode(epoch, payload);
     encode(request_ack, payload);
+    encode(down_and_dead, payload);
   }
 
   std::string_view get_type_name() const override { return "MOSDMarkMeDown"; }
   void print(std::ostream& out) const override {
     out << "MOSDMarkMeDown("
        << "request_ack=" << request_ack
+       << ", down_and_dead=" << down_and_dead
        << ", osd." << target_osd
        << ", " << target_addrs
        << ", fsid=" << fsid

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 7a62573897d6c5549bbb05e01f2a0917df385614..a0e930794a70ab0ee790917f583ab6486a9b48d4 100644 (file)
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -3058,8 +3058,14 @@ bool OSDMonitor::prepare_mark_me_down(MonOpRequestRef op)
   ceph_assert(osdmap.is_up(target_osd));
   ceph_assert(osdmap.get_addrs(target_osd) == m->target_addrs);
 
-  mon.clog->info() << "osd." << target_osd << " marked itself down";
+  mon.clog->info() << "osd." << target_osd << " marked itself " << ((m->down_and_dead) ? "down and dead" : "down");
   pending_inc.new_state[target_osd] = CEPH_OSD_UP;
+  if (m->down_and_dead) {
+    if (!pending_inc.new_xinfo.count(target_osd)) {
+      pending_inc.new_xinfo[target_osd] = osdmap.osd_xinfo[target_osd];
+    }
+    pending_inc.new_xinfo[target_osd].dead_epoch = m->get_epoch();
+  }
   if (m->request_ack)
     wait_for_finished_proposal(op, new C_AckMarkedDown(this, op));
   return true;

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index b7a68225e0602c54b3d58a4ffea59a248443617b..700fbc9b9a4576d4ff71c088802df2adc69f1a9f 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1299,7 +1299,7 @@ bool OSDService::prepare_to_stop()
 
   OSDMapRef osdmap = get_osdmap();
   if (osdmap && osdmap->is_up(whoami)) {
-    dout(0) << __func__ << " telling mon we are shutting down" << dendl;
+    dout(0) << __func__ << " telling mon we are shutting down and dead " << dendl;
     set_state(PREPARING_TO_STOP);
     monc->send_mon_message(
       new MOSDMarkMeDown(
@@ -1307,12 +1307,14 @@ bool OSDService::prepare_to_stop()
        whoami,
        osdmap->get_addrs(whoami),
        osdmap->get_epoch(),
-       true  // request ack
+       true,  // request ack
+       true   // mark as down and dead
        ));
     const auto timeout = ceph::make_timespan(cct->_conf->osd_mon_shutdown_timeout);
     is_stopping_cond.wait_for(l, timeout,
       [this] { return get_state() == STOPPING; });
   }
+
   dout(0) << __func__ << " starting shutdown" << dendl;
   set_state(STOPPING);
   return true;