osd: asok support for "blocked scrub warning" debug

A "scrubdebug" command to artificially block/free object
ranges for scrubbing.

This asok op-code will be extended in the future to support more debug options.

Signed-off-by: Ronen Friedman <rfriedma@redhat.com>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 61666c392376ae40f2c442742b737d4dcbe39f09..b810f7d27901c10aed704abd00a42e617b984e35 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -2488,6 +2488,70 @@ std::set<int64_t> OSD::get_mapped_pools()
   return pools;
 }
 
+OSD::PGRefOrError OSD::locate_asok_target(const cmdmap_t& cmdmap,
+                                    stringstream& ss,
+                                    bool only_primary)
+{
+  string pgidstr;
+  if (!cmd_getval(cmdmap, "pgid", pgidstr)) {
+    ss << "no pgid specified";
+    return OSD::PGRefOrError{std::nullopt, -EINVAL};
+  }
+
+  pg_t pgid;
+  if (!pgid.parse(pgidstr.c_str())) {
+    ss << "couldn't parse pgid '" << pgidstr << "'";
+    return OSD::PGRefOrError{std::nullopt, -EINVAL};
+  }
+
+  spg_t pcand;
+  PGRef pg;
+  if (get_osdmap()->get_primary_shard(pgid, &pcand) && (pg = _lookup_lock_pg(pcand))) {
+    if (pg->is_primary() || !only_primary) {
+      return OSD::PGRefOrError{pg, 0};
+    }
+
+    ss << "not primary for pgid " << pgid;
+    pg->unlock();
+    return OSD::PGRefOrError{std::nullopt, -EAGAIN};
+  } else {
+    ss << "i don't have pgid " << pgid;
+    return OSD::PGRefOrError{std::nullopt, -ENOENT};
+  }
+}
+
+// note that the cmdmap is explicitly copied into asok_route_to_pg()
+int OSD::asok_route_to_pg(
+  bool only_primary,
+  std::string_view prefix,
+  cmdmap_t cmdmap,
+  Formatter* f,
+  stringstream& ss,
+  const bufferlist& inbl,
+  bufferlist& outbl,
+  std::function<void(int, const std::string&, bufferlist&)> on_finish)
+{
+  auto [target_pg, ret] = locate_asok_target(cmdmap, ss, only_primary);
+
+  if (!target_pg.has_value()) {
+    // 'ss' and 'ret' already contain the error information
+    on_finish(ret, ss.str(), outbl);
+    return ret;
+  }
+
+  //  the PG was locked by locate_asok_target()
+  try {
+    (*target_pg)->do_command(prefix, cmdmap, inbl, on_finish);
+    (*target_pg)->unlock();
+    return 0;  // the pg handler calls on_finish directly
+  } catch (const TOPNSPC::common::bad_cmd_get& e) {
+    (*target_pg)->unlock();
+    ss << e.what();
+    on_finish(ret, ss.str(), outbl);
+    return -EINVAL;
+  }
+}
+
 void OSD::asok_command(
   std::string_view prefix, const cmdmap_t& cmdmap,
   Formatter *f,
@@ -2548,6 +2612,13 @@ void OSD::asok_command(
     }
   }
 
+  // --- PG commands that will be answered even if !primary ---
+
+  else if (prefix == "scrubdebug") {
+    asok_route_to_pg(false, prefix, cmdmap, f, ss, inbl, outbl, on_finish);
+    return;
+  }
+
   // --- OSD commands follow ---
 
   else if (prefix == "status") {
@@ -4058,6 +4129,14 @@ void OSD::final_init()
     asok_hook,
     "Scrub purged_snaps vs snapmapper index");
   ceph_assert(r == 0);
+  r = admin_socket->register_command(
+    "scrubdebug "                                              \
+    "name=pgid,type=CephPgid "                                 \
+    "name=cmd,type=CephChoices,strings=block|unblock "          \
+    "name=value,type=CephString,req=false",
+    asok_hook,
+    "debug the scrubber");
+  ceph_assert(r == 0);
 
   // -- pg commands --
   // old form: ceph pg <pgid> command ...

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 90d2f800fb841a96236ca4c06d0e46493ddd70a8..383b01acbd05acb4fccea4ac7ada8d91dcfa24c0 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1155,6 +1155,16 @@ protected:
   // asok
   friend class OSDSocketHook;
   class OSDSocketHook *asok_hook;
+  using PGRefOrError = std::tuple<std::optional<PGRef>, int>;
+  PGRefOrError locate_asok_target(const cmdmap_t& cmdmap, stringstream& ss, bool only_primary);
+  int asok_route_to_pg(bool only_primary,
+    std::string_view prefix,
+    cmdmap_t cmdmap,
+    Formatter *f,
+    stringstream& ss,
+    const bufferlist& inbl,
+    bufferlist& outbl,
+    std::function<void(int, const std::string&, bufferlist&)> on_finish);
   void asok_command(
     std::string_view prefix,
     const cmdmap_t& cmdmap,

diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index 4427bff8249e2444c93c8a1aa090f29e4ea825e8..c8bf2e311cc9e0fefd077e0e12eba9ee3e019f92 100644 (file)
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -1176,6 +1176,21 @@ void PrimaryLogPG::do_command(
     outbl.append(ss.str());
   }
 
+  else if (prefix == "block" || prefix == "unblock") {
+    string value;
+    cmd_getval(cmdmap, "value", value);
+
+    if (is_primary()) {
+      ret = m_scrubber->asok_debug(prefix, value, f.get(), ss);
+      f->open_object_section("result");
+      f->dump_bool("success", true);
+      f->close_section();
+    } else {
+      ss << "Not primary";
+      ret = -EPERM;
+    }
+    outbl.append(ss.str());
+  }
   else {
     ret = -ENOSYS;
     ss << "prefix '" << prefix << "' not implemented";

diff --git a/src/osd/pg_scrubber.cc b/src/osd/pg_scrubber.cc
index 4e3544e30bac7c6c9a3dfe3bc90ba79fd5a00f07..8a85d55cd30bc42c31a1385e49b02fad1b0fb1d1 100644 (file)
--- a/src/osd/pg_scrubber.cc
+++ b/src/osd/pg_scrubber.cc
@@ -572,6 +572,12 @@ bool PgScrubber::select_range()
 
   dout(15) << __func__ << " range selected: " << m_start << " //// " << m_end << " //// "
           << m_max_end << dendl;
+
+  // debug: be 'blocked' if told so by the 'pg scrub_debug block' asok command
+  if (m_debug_blockrange > 0) {
+    m_debug_blockrange--;
+    return false;
+  }
   return true;
 }
 
@@ -1870,6 +1876,23 @@ ostream& PgScrubber::show(ostream& out) const
   return out << " [ " << m_pg_id << ": " << m_flags << " ] ";
 }
 
+int PgScrubber::asok_debug(std::string_view cmd,
+                          std::string param,
+                          Formatter* f,
+                          stringstream& ss)
+{
+  dout(10) << __func__ << " cmd: " << cmd << " param: " << param << dendl;
+
+  if (cmd == "block") {
+    // set a flag that will cause the next 'select_range' to report a blocked object
+    m_debug_blockrange = 1;
+  } else if (cmd == "unblock") {
+    // send an 'unblock' event, as if a blocked range was freed
+    m_debug_blockrange = 0;
+    m_fsm->process_event(Unblocked{});
+  }
+  return 0;
+}
 // ///////////////////// preemption_data_t //////////////////////////////////
 
 PgScrubber::preemption_data_t::preemption_data_t(PG* pg) : m_pg{pg}

diff --git a/src/osd/pg_scrubber.h b/src/osd/pg_scrubber.h
index 22f2f4a2004e7baf3f4923132afba393b0c3d277..c63d949506416a8f0c76122f22afe8a2964e6d06 100644 (file)
--- a/src/osd/pg_scrubber.h
+++ b/src/osd/pg_scrubber.h
@@ -321,11 +321,15 @@ class PgScrubber : public ScrubPgIF, public ScrubMachineListener {
     return false;
   };
 
+  int asok_debug(std::string_view cmd,
+                std::string param,
+                Formatter* f,
+                stringstream& ss) override;
+  int m_debug_blockrange{0};
+
   // -------------------------------------------------------------------------------------------
   // the I/F used by the state-machine (i.e. the implementation of ScrubMachineListener)
 
-
-
   bool select_range() final;
 
   Scrub::BlockedRangeWarning acquire_blocked_alarm() final;

diff --git a/src/osd/scrub_machine_lstnr.h b/src/osd/scrub_machine_lstnr.h
index eb9a7391e915952c197b51734998f221cfd12b82..87a5bf87d1f2b6a2716c686de3c2bb74ba3b4be4 100644 (file)
--- a/src/osd/scrub_machine_lstnr.h
+++ b/src/osd/scrub_machine_lstnr.h
@@ -84,8 +84,6 @@ struct ScrubMachineListener {
 
   virtual void replica_handling_done() = 0;
 
-  // no virtual void discard_reservation_by_primary() = 0;
-
   /// the version of 'scrub_clear_state()' that does not try to invoke FSM services
   /// (thus can be called from FSM reactions)
   virtual void clear_pgscrub_state() = 0;

diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h
index 15a6cdf4dede485e14f395feff4d6bcfd8280820..d54ad17b03db5cb5569101f72fbaa79859432ec6 100644 (file)
--- a/src/osd/scrubber_common.h
+++ b/src/osd/scrubber_common.h
@@ -263,4 +263,11 @@ struct ScrubPgIF {
   virtual void scrub_requested(scrub_level_t scrub_level,
                               scrub_type_t scrub_type,
                               requested_scrub_t& req_flags) = 0;
+
+  // --------------- debugging via the asok ------------------------------
+
+  virtual int asok_debug(std::string_view cmd,
+                        std::string param,
+                        Formatter* f,
+                        stringstream& ss) = 0;
 };