mon, osd: add command to remove invalid pg-upmap-primary entries

The current rm-pg-upmap-primary command checks that the pgid exists
in the pgmap before continuing to remove it. Due to https://tracker.ceph.com/issues/66867,
some invalid pg-upmap-primary entires may exist for pools that have been removed.
Currently, these mappings are impossible to remove since the pgids no longer
exist in the pgmap.

This new command, rm-pg-upmap-primary-all, allows users the ability to remove
any and all pg-upmap-primary mappings in the osdmap at once, which includes
valid and invalid entries.

This command may also be helpful when upgrading from versions where users
are plagued by https://tracker.ceph.com/issues/61948. Users may use an upgraded
mon to remove all pg-upmap-primray entries (valid and invalid) so they continue
to upgrade to a safe version.

See manual testing for this patch here: https://tracker.ceph.com/issues/67179#note-12

Fixes: https://tracker.ceph.com/issues/67179
Fixes: https://tracker.ceph.com/issues/69760
Signed-off-by: Laura Flores <lflores@ibm.com>

diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index a53d1adce9e2adc49263585d9cbb189670251961..26650a73d230b48cc56e8b4a4983d039801be787 100644 (file)
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -1035,6 +1035,9 @@ COMMAND("osd rm-pg-upmap-primary "
        "name=pgid,type=CephPgid ",
        "clear pg primary setting for <pgid>",
         "osd", "rw")
+COMMAND("osd rm-pg-upmap-primary-all ",
+        "clear all pg primary entries (developers only)",
+        "osd", "rw")
 COMMAND("osd primary-temp "
        "name=pgid,type=CephPgid "
        "name=id,type=CephOsdName",

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 4807f9dd166d314396a372a117d214f43ac6fe83..8b51901ebe2f38158fe7a34110295710ae2364a2 100644 (file)
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -12417,7 +12417,8 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
              prefix == "osd pg-upmap-items" ||
              prefix == "osd rm-pg-upmap-items" ||
             prefix == "osd pg-upmap-primary" ||
-            prefix == "osd rm-pg-upmap-primary") {
+            prefix == "osd rm-pg-upmap-primary" ||
+            prefix == "osd rm-pg-upmap-primary-all") {
     enum {
       OP_PG_UPMAP,
       OP_RM_PG_UPMAP,
@@ -12425,6 +12426,7 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
       OP_RM_PG_UPMAP_ITEMS,
       OP_PG_UPMAP_PRIMARY,
       OP_RM_PG_UPMAP_PRIMARY,
+      OP_RM_PG_UPMAP_PRIMARY_ALL,
     } upmap_option;
 
     if (prefix == "osd pg-upmap") {
@@ -12439,6 +12441,8 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
       upmap_option = OP_PG_UPMAP_PRIMARY;
     } else if (prefix == "osd rm-pg-upmap-primary") {
       upmap_option = OP_RM_PG_UPMAP_PRIMARY;
+    } else if (prefix == "osd rm-pg-upmap-primary-all") {
+      upmap_option = OP_RM_PG_UPMAP_PRIMARY_ALL;
     } else {
       ceph_abort_msg("invalid upmap option");
     }
@@ -12458,6 +12462,7 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
 
     case OP_PG_UPMAP_PRIMARY:  // fall through
     case OP_RM_PG_UPMAP_PRIMARY:
+    case OP_RM_PG_UPMAP_PRIMARY_ALL:
       min_release = ceph_release_t::reef;
       min_feature = CEPH_FEATUREMASK_SERVER_REEF;
       feature_name = "pg-upmap-primary";
@@ -12483,17 +12488,33 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
       goto wait;
     if (err < 0)
       goto reply_no_propose;
+
     pg_t pgid;
-    err = parse_pgid(cmdmap, ss, pgid);
-    if (err < 0)
-      goto reply_no_propose;
-    if (pending_inc.old_pools.count(pgid.pool())) {
-      ss << "pool of " << pgid << " is pending removal";
-      err = -ENOENT;
-      getline(ss, rs);
-      wait_for_commit(op,
-        new Monitor::C_Command(mon, op, err, rs, get_last_committed() + 1));
-      return true;
+    switch (upmap_option) {
+    case OP_RM_PG_UPMAP_PRIMARY_ALL: // no pgid to check
+      break;
+    
+    case OP_PG_UPMAP:
+    case OP_RM_PG_UPMAP:
+    case OP_PG_UPMAP_ITEMS:
+    case OP_RM_PG_UPMAP_ITEMS:
+    case OP_PG_UPMAP_PRIMARY:
+    case OP_RM_PG_UPMAP_PRIMARY:
+      err = parse_pgid(cmdmap, ss, pgid);
+      if (err < 0)
+       goto reply_no_propose;
+      if (pending_inc.old_pools.count(pgid.pool())) {
+       ss << "pool of " << pgid << " is pending removal";
+       err = -ENOENT;
+       getline(ss, rs);
+       wait_for_commit(op,
+         new Monitor::C_Command(mon, op, err, rs, get_last_committed() + 1));
+       return true;
+      }
+      break;
+    
+    default:
+      ceph_abort_msg("invalid upmap option");
     }
 
     // check pending upmap changes
@@ -12528,6 +12549,8 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
         goto wait;
       }
       break;
+    case OP_RM_PG_UPMAP_PRIMARY_ALL: // nothing to check
+      break;
 
     default:
       ceph_abort_msg("invalid upmap option");
@@ -12733,6 +12756,13 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
       }
       break;
 
+    case OP_RM_PG_UPMAP_PRIMARY_ALL:
+      {
+       osdmap.rm_all_upmap_prims(cct, &pending_inc);
+       ss << "cleared all pg_upmap_primary mappings";
+      }
+      break;
+
     default:
       ceph_abort_msg("invalid upmap option");
     }

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 86950979b9a8c6603236f852c9db1c13f753f497..ee071f1aa7134adf644b7e315d5e5535a01dbb4f 100644 (file)
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -5216,6 +5216,20 @@ void OSDMap::rm_all_upmap_prims(CephContext *cct, OSDMap::Incremental *pending_i
   }
 }
 
+void OSDMap::rm_all_upmap_prims(
+  CephContext *cct,
+  OSDMap::Incremental *pending_inc)
+{
+  for (const auto& [pg, _] : pg_upmap_primaries) {
+    if (pending_inc->new_pg_upmap_primary.contains(pg)) {
+        ldout(cct, 30) << __func__ << "Removing pending pg_upmap_prim for pg " << pg << dendl;
+        pending_inc->new_pg_upmap_primary.erase(pg);
+      }
+    ldout(cct, 30) << __func__ << "Removing pg_upmap_prim for pg " << pg << dendl;
+    pending_inc->old_pg_upmap_primary.insert(pg);
+  }
+}
+
 int OSDMap::calc_desired_primary_distribution(
   CephContext *cct,
   int64_t pid,

diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h
index 97d6b7e1b8bfe8b56c0dfa81df82d3127deafece..e46ebc590b74eff1bf6ade6310d624613f471388 100644 (file)
--- a/src/osd/OSDMap.h
+++ b/src/osd/OSDMap.h
@@ -1492,7 +1492,10 @@ public:
     OSDMap& tmp_osd_map,
     const std::optional<rb_policy>& rbp = std::nullopt) const;
 
-  void rm_all_upmap_prims(CephContext *cct, Incremental *pending_inc, uint64_t pid);
+  void rm_all_upmap_prims(CephContext *cct, Incremental *pending_inc, uint64_t pid); // per pool
+  void rm_all_upmap_prims(
+    CephContext *cct,
+    OSDMap::Incremental *pending_inc); // total
 
   int calc_desired_primary_distribution(
     CephContext *cct,