From: Ronen Friedman Date: Sun, 17 Jan 2021 15:48:43 +0000 (+0200) Subject: osd: asok support for "blocked scrub warning" debug X-Git-Tag: v17.1.0~1561^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=2ed6095535e69ac63e4e54b1dbeb2e24c3b4c2b4;p=ceph.git osd: asok support for "blocked scrub warning" debug A "scrubdebug" command to artificially block/free object ranges for scrubbing. This asok op-code will be extended in the future to support more debug options. Signed-off-by: Ronen Friedman --- diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 61666c392376..b810f7d27901 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -2488,6 +2488,70 @@ std::set OSD::get_mapped_pools() return pools; } +OSD::PGRefOrError OSD::locate_asok_target(const cmdmap_t& cmdmap, + stringstream& ss, + bool only_primary) +{ + string pgidstr; + if (!cmd_getval(cmdmap, "pgid", pgidstr)) { + ss << "no pgid specified"; + return OSD::PGRefOrError{std::nullopt, -EINVAL}; + } + + pg_t pgid; + if (!pgid.parse(pgidstr.c_str())) { + ss << "couldn't parse pgid '" << pgidstr << "'"; + return OSD::PGRefOrError{std::nullopt, -EINVAL}; + } + + spg_t pcand; + PGRef pg; + if (get_osdmap()->get_primary_shard(pgid, &pcand) && (pg = _lookup_lock_pg(pcand))) { + if (pg->is_primary() || !only_primary) { + return OSD::PGRefOrError{pg, 0}; + } + + ss << "not primary for pgid " << pgid; + pg->unlock(); + return OSD::PGRefOrError{std::nullopt, -EAGAIN}; + } else { + ss << "i don't have pgid " << pgid; + return OSD::PGRefOrError{std::nullopt, -ENOENT}; + } +} + +// note that the cmdmap is explicitly copied into asok_route_to_pg() +int OSD::asok_route_to_pg( + bool only_primary, + std::string_view prefix, + cmdmap_t cmdmap, + Formatter* f, + stringstream& ss, + const bufferlist& inbl, + bufferlist& outbl, + std::function on_finish) +{ + auto [target_pg, ret] = locate_asok_target(cmdmap, ss, only_primary); + + if (!target_pg.has_value()) { + // 'ss' and 'ret' already contain the error information + on_finish(ret, ss.str(), outbl); + return ret; + } + + // the PG was locked by locate_asok_target() + try { + (*target_pg)->do_command(prefix, cmdmap, inbl, on_finish); + (*target_pg)->unlock(); + return 0; // the pg handler calls on_finish directly + } catch (const TOPNSPC::common::bad_cmd_get& e) { + (*target_pg)->unlock(); + ss << e.what(); + on_finish(ret, ss.str(), outbl); + return -EINVAL; + } +} + void OSD::asok_command( std::string_view prefix, const cmdmap_t& cmdmap, Formatter *f, @@ -2548,6 +2612,13 @@ void OSD::asok_command( } } + // --- PG commands that will be answered even if !primary --- + + else if (prefix == "scrubdebug") { + asok_route_to_pg(false, prefix, cmdmap, f, ss, inbl, outbl, on_finish); + return; + } + // --- OSD commands follow --- else if (prefix == "status") { @@ -4058,6 +4129,14 @@ void OSD::final_init() asok_hook, "Scrub purged_snaps vs snapmapper index"); ceph_assert(r == 0); + r = admin_socket->register_command( + "scrubdebug " \ + "name=pgid,type=CephPgid " \ + "name=cmd,type=CephChoices,strings=block|unblock " \ + "name=value,type=CephString,req=false", + asok_hook, + "debug the scrubber"); + ceph_assert(r == 0); // -- pg commands -- // old form: ceph pg command ... diff --git a/src/osd/OSD.h b/src/osd/OSD.h index 90d2f800fb84..383b01acbd05 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -1155,6 +1155,16 @@ protected: // asok friend class OSDSocketHook; class OSDSocketHook *asok_hook; + using PGRefOrError = std::tuple, int>; + PGRefOrError locate_asok_target(const cmdmap_t& cmdmap, stringstream& ss, bool only_primary); + int asok_route_to_pg(bool only_primary, + std::string_view prefix, + cmdmap_t cmdmap, + Formatter *f, + stringstream& ss, + const bufferlist& inbl, + bufferlist& outbl, + std::function on_finish); void asok_command( std::string_view prefix, const cmdmap_t& cmdmap, diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc index 4427bff8249e..c8bf2e311cc9 100644 --- a/src/osd/PrimaryLogPG.cc +++ b/src/osd/PrimaryLogPG.cc @@ -1176,6 +1176,21 @@ void PrimaryLogPG::do_command( outbl.append(ss.str()); } + else if (prefix == "block" || prefix == "unblock") { + string value; + cmd_getval(cmdmap, "value", value); + + if (is_primary()) { + ret = m_scrubber->asok_debug(prefix, value, f.get(), ss); + f->open_object_section("result"); + f->dump_bool("success", true); + f->close_section(); + } else { + ss << "Not primary"; + ret = -EPERM; + } + outbl.append(ss.str()); + } else { ret = -ENOSYS; ss << "prefix '" << prefix << "' not implemented"; diff --git a/src/osd/pg_scrubber.cc b/src/osd/pg_scrubber.cc index 4e3544e30bac..8a85d55cd30b 100644 --- a/src/osd/pg_scrubber.cc +++ b/src/osd/pg_scrubber.cc @@ -572,6 +572,12 @@ bool PgScrubber::select_range() dout(15) << __func__ << " range selected: " << m_start << " //// " << m_end << " //// " << m_max_end << dendl; + + // debug: be 'blocked' if told so by the 'pg scrub_debug block' asok command + if (m_debug_blockrange > 0) { + m_debug_blockrange--; + return false; + } return true; } @@ -1870,6 +1876,23 @@ ostream& PgScrubber::show(ostream& out) const return out << " [ " << m_pg_id << ": " << m_flags << " ] "; } +int PgScrubber::asok_debug(std::string_view cmd, + std::string param, + Formatter* f, + stringstream& ss) +{ + dout(10) << __func__ << " cmd: " << cmd << " param: " << param << dendl; + + if (cmd == "block") { + // set a flag that will cause the next 'select_range' to report a blocked object + m_debug_blockrange = 1; + } else if (cmd == "unblock") { + // send an 'unblock' event, as if a blocked range was freed + m_debug_blockrange = 0; + m_fsm->process_event(Unblocked{}); + } + return 0; +} // ///////////////////// preemption_data_t ////////////////////////////////// PgScrubber::preemption_data_t::preemption_data_t(PG* pg) : m_pg{pg} diff --git a/src/osd/pg_scrubber.h b/src/osd/pg_scrubber.h index 22f2f4a2004e..c63d94950641 100644 --- a/src/osd/pg_scrubber.h +++ b/src/osd/pg_scrubber.h @@ -321,11 +321,15 @@ class PgScrubber : public ScrubPgIF, public ScrubMachineListener { return false; }; + int asok_debug(std::string_view cmd, + std::string param, + Formatter* f, + stringstream& ss) override; + int m_debug_blockrange{0}; + // ------------------------------------------------------------------------------------------- // the I/F used by the state-machine (i.e. the implementation of ScrubMachineListener) - - bool select_range() final; Scrub::BlockedRangeWarning acquire_blocked_alarm() final; diff --git a/src/osd/scrub_machine_lstnr.h b/src/osd/scrub_machine_lstnr.h index eb9a7391e915..87a5bf87d1f2 100644 --- a/src/osd/scrub_machine_lstnr.h +++ b/src/osd/scrub_machine_lstnr.h @@ -84,8 +84,6 @@ struct ScrubMachineListener { virtual void replica_handling_done() = 0; - // no virtual void discard_reservation_by_primary() = 0; - /// the version of 'scrub_clear_state()' that does not try to invoke FSM services /// (thus can be called from FSM reactions) virtual void clear_pgscrub_state() = 0; diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h index 15a6cdf4dede..d54ad17b03db 100644 --- a/src/osd/scrubber_common.h +++ b/src/osd/scrubber_common.h @@ -263,4 +263,11 @@ struct ScrubPgIF { virtual void scrub_requested(scrub_level_t scrub_level, scrub_type_t scrub_type, requested_scrub_t& req_flags) = 0; + + // --------------- debugging via the asok ------------------------------ + + virtual int asok_debug(std::string_view cmd, + std::string param, + Formatter* f, + stringstream& ss) = 0; };