mon, osd: add command to remove invalid pg-upmap-primary entries

The current rm-pg-upmap-primary command checks that the pgid exists
in the pgmap before continuing to remove it. Due to https://tracker.ceph.com/issues/66867,
some invalid pg-upmap-primary entires may exist for pools that have been removed.
Currently, these mappings are impossible to remove since the pgids no longer
exist in the pgmap.

This new command, rm-pg-upmap-primary-all, allows users the ability to remove
any and all pg-upmap-primary mappings in the osdmap at once, which includes
valid and invalid entries.

This command may also be helpful when upgrading from versions where users
are plagued by https://tracker.ceph.com/issues/61948. Users may use an upgraded
mon to remove all pg-upmap-primray entries (valid and invalid) so they continue
to upgrade to a safe version.

See manual testing for this patch here: https://tracker.ceph.com/issues/67179#note-12

Fixes: https://tracker.ceph.com/issues/67179
Fixes: https://tracker.ceph.com/issues/69760
Signed-off-by: Laura Flores <lflores@ibm.com>
(cherry picked from commit 6e9e2033bf0f4779bdfac9a3a4f29115459c8c0e)

diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index 3c161dd3b2c0e73936dbf8e7b48f5f8055c993bd..19e117cea38b877032382debd7355072a5b61f6b 100644 (file)
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -1025,6 +1025,9 @@ COMMAND("osd rm-pg-upmap-primary "
        "name=pgid,type=CephPgid ",
        "clear pg primary setting for <pgid>",
         "osd", "rw")
+COMMAND("osd rm-pg-upmap-primary-all ",
+        "clear all pg primary entries (developers only)",
+        "osd", "rw")
 COMMAND("osd primary-temp "
        "name=pgid,type=CephPgid "
        "name=id,type=CephOsdName",

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index f12ad4ea40573c34e4c357052db717b47fb633c9..c849b9d25a80ce3ff66218d4f9685676c914a77f 100644 (file)
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -12427,7 +12427,8 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
              prefix == "osd pg-upmap-items" ||
              prefix == "osd rm-pg-upmap-items" ||
             prefix == "osd pg-upmap-primary" ||
-            prefix == "osd rm-pg-upmap-primary") {
+            prefix == "osd rm-pg-upmap-primary" ||
+            prefix == "osd rm-pg-upmap-primary-all") {
     enum {
       OP_PG_UPMAP,
       OP_RM_PG_UPMAP,
@@ -12435,6 +12436,7 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
       OP_RM_PG_UPMAP_ITEMS,
       OP_PG_UPMAP_PRIMARY,
       OP_RM_PG_UPMAP_PRIMARY,
+      OP_RM_PG_UPMAP_PRIMARY_ALL,
     } upmap_option;
 
     if (prefix == "osd pg-upmap") {
@@ -12449,6 +12451,8 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
       upmap_option = OP_PG_UPMAP_PRIMARY;
     } else if (prefix == "osd rm-pg-upmap-primary") {
       upmap_option = OP_RM_PG_UPMAP_PRIMARY;
+    } else if (prefix == "osd rm-pg-upmap-primary-all") {
+      upmap_option = OP_RM_PG_UPMAP_PRIMARY_ALL;
     } else {
       ceph_abort_msg("invalid upmap option");
     }
@@ -12468,6 +12472,7 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
 
     case OP_PG_UPMAP_PRIMARY:  // fall through
     case OP_RM_PG_UPMAP_PRIMARY:
+    case OP_RM_PG_UPMAP_PRIMARY_ALL:
       min_release = ceph_release_t::reef;
       min_feature = CEPH_FEATUREMASK_SERVER_REEF;
       feature_name = "pg-upmap-primary";
@@ -12493,17 +12498,33 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
       goto wait;
     if (err < 0)
       goto reply_no_propose;
+
     pg_t pgid;
-    err = parse_pgid(cmdmap, ss, pgid);
-    if (err < 0)
-      goto reply_no_propose;
-    if (pending_inc.old_pools.count(pgid.pool())) {
-      ss << "pool of " << pgid << " is pending removal";
-      err = -ENOENT;
-      getline(ss, rs);
-      wait_for_commit(op,
-        new Monitor::C_Command(mon, op, err, rs, get_last_committed() + 1));
-      return true;
+    switch (upmap_option) {
+    case OP_RM_PG_UPMAP_PRIMARY_ALL: // no pgid to check
+      break;
+    
+    case OP_PG_UPMAP:
+    case OP_RM_PG_UPMAP:
+    case OP_PG_UPMAP_ITEMS:
+    case OP_RM_PG_UPMAP_ITEMS:
+    case OP_PG_UPMAP_PRIMARY:
+    case OP_RM_PG_UPMAP_PRIMARY:
+      err = parse_pgid(cmdmap, ss, pgid);
+      if (err < 0)
+       goto reply_no_propose;
+      if (pending_inc.old_pools.count(pgid.pool())) {
+       ss << "pool of " << pgid << " is pending removal";
+       err = -ENOENT;
+       getline(ss, rs);
+       wait_for_commit(op,
+         new Monitor::C_Command(mon, op, err, rs, get_last_committed() + 1));
+       return true;
+      }
+      break;
+    
+    default:
+      ceph_abort_msg("invalid upmap option");
     }
 
     // check pending upmap changes
@@ -12538,6 +12559,8 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
         goto wait;
       }
       break;
+    case OP_RM_PG_UPMAP_PRIMARY_ALL: // nothing to check
+      break;
 
     default:
       ceph_abort_msg("invalid upmap option");
@@ -12743,6 +12766,13 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
       }
       break;
 
+    case OP_RM_PG_UPMAP_PRIMARY_ALL:
+      {
+       osdmap.rm_all_upmap_prims(cct, &pending_inc);
+       ss << "cleared all pg_upmap_primary mappings";
+      }
+      break;
+
     default:
       ceph_abort_msg("invalid upmap option");
     }

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index b87484c1a9db9f80db6414a1ef25ea4ec86596a1..d58a8cbb8636f685d105dceb3071720b173f07c1 100644 (file)
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -5218,6 +5218,20 @@ void OSDMap::rm_all_upmap_prims(CephContext *cct, OSDMap::Incremental *pending_i
   }
 }
 
+void OSDMap::rm_all_upmap_prims(
+  CephContext *cct,
+  OSDMap::Incremental *pending_inc)
+{
+  for (const auto& [pg, _] : pg_upmap_primaries) {
+    if (pending_inc->new_pg_upmap_primary.contains(pg)) {
+        ldout(cct, 30) << __func__ << "Removing pending pg_upmap_prim for pg " << pg << dendl;
+        pending_inc->new_pg_upmap_primary.erase(pg);
+      }
+    ldout(cct, 30) << __func__ << "Removing pg_upmap_prim for pg " << pg << dendl;
+    pending_inc->old_pg_upmap_primary.insert(pg);
+  }
+}
+
 int OSDMap::calc_desired_primary_distribution(
   CephContext *cct,
   int64_t pid,

diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h
index 97d6b7e1b8bfe8b56c0dfa81df82d3127deafece..e46ebc590b74eff1bf6ade6310d624613f471388 100644 (file)
--- a/src/osd/OSDMap.h
+++ b/src/osd/OSDMap.h
@@ -1492,7 +1492,10 @@ public:
     OSDMap& tmp_osd_map,
     const std::optional<rb_policy>& rbp = std::nullopt) const;
 
-  void rm_all_upmap_prims(CephContext *cct, Incremental *pending_inc, uint64_t pid);
+  void rm_all_upmap_prims(CephContext *cct, Incremental *pending_inc, uint64_t pid); // per pool
+  void rm_all_upmap_prims(
+    CephContext *cct,
+    OSDMap::Incremental *pending_inc); // total
 
   int calc_desired_primary_distribution(
     CephContext *cct,