From 0ab8d0c7d7c897939e34c32b55b3d079b649984d Mon Sep 17 00:00:00 2001 From: Leonid Usov Date: Wed, 27 Sep 2023 00:54:26 +0300 Subject: [PATCH] common/admin_socket: add a command to raise a signal The new command "raise [--after X]" accepts signals in the forms: '9', '-9', 'kill', '-KILL' When --after is specified, the program will fork to wait for the timeout The forked instance will bail out if it detects that the parent PID has changed which would indicate that the original parent has terminated. Forking an instance allows to schedule delivery of signals even if the original process is suspended, e.g.: ceph tell mds.a raise CONT --after 10 ceph tell mds.a raise STOP Signed-off-by: Leonid Usov Fixes: https://tracker.ceph.com/issues/63362 (cherry picked from commit 7cc54b2b7323096bf7394d8502ff232a78e0c355) --- src/common/admin_socket.cc | 308 +++++++++++++++++++++++++++++++++++++ src/common/admin_socket.h | 1 + src/test/admin_socket.cc | 215 ++++++++++++++++++++++++++ 3 files changed, 524 insertions(+) diff --git a/src/common/admin_socket.cc b/src/common/admin_socket.cc index 2211dd7340f2..df60b30067ce 100644 --- a/src/common/admin_socket.cc +++ b/src/common/admin_socket.cc @@ -13,6 +13,7 @@ */ #include #include +#include #include "common/admin_socket.h" #include "common/admin_socket_client.h" @@ -36,6 +37,7 @@ #include "include/ceph_assert.h" #include "include/compat.h" #include "include/sock_compat.h" +#include "fmt/format.h" #define dout_subsys ceph_subsys_asok #undef dout_prefix @@ -693,6 +695,303 @@ public: } }; +// Define a macro to simplify adding signals to the map +#define ADD_SIGNAL(signalName) \ + { \ + ((const char*)#signalName) + 3, signalName \ + } + +static const std::map known_signals = { + // the following 6 signals are recognized in windows according to + // https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/raise?view=msvc-170 + ADD_SIGNAL(SIGABRT), + ADD_SIGNAL(SIGFPE), + ADD_SIGNAL(SIGILL), + ADD_SIGNAL(SIGINT), + ADD_SIGNAL(SIGSEGV), + ADD_SIGNAL(SIGTERM), +#ifndef WIN32 + ADD_SIGNAL(SIGTRAP), + ADD_SIGNAL(SIGHUP), + ADD_SIGNAL(SIGBUS), + ADD_SIGNAL(SIGQUIT), + ADD_SIGNAL(SIGKILL), + ADD_SIGNAL(SIGUSR1), + ADD_SIGNAL(SIGUSR2), + ADD_SIGNAL(SIGPIPE), + ADD_SIGNAL(SIGALRM), + ADD_SIGNAL(SIGCHLD), + ADD_SIGNAL(SIGCONT), + ADD_SIGNAL(SIGSTOP), + ADD_SIGNAL(SIGTSTP), + ADD_SIGNAL(SIGTTIN), + ADD_SIGNAL(SIGTTOU), +#endif + // Add more signals as needed... +}; + +#undef ADD_SIGNAL + +static std::string strsignal_compat(int signal) { +#ifndef WIN32 + return strsignal(signal); +#else + switch (signal) { + case SIGABRT: return "SIGABRT"; + case SIGFPE: return "SIGFPE"; + case SIGILL: return "SIGILL"; + case SIGINT: return "SIGINT"; + case SIGSEGV: return "SIGSEGV"; + case SIGTERM: return "SIGTERM"; + default: return fmt::format("Signal #{}", signal); + } +#endif +} + +class RaiseHook: public AdminSocketHook { + using clock = ceph::coarse_mono_clock; + struct Killer { + CephContext* m_cct; + pid_t pid; + int signal; + clock::time_point due; + + std::string describe() + { + using std::chrono::duration_cast; + using std::chrono::seconds; + auto remaining = (due - clock::now()); + return fmt::format( + "pending signal ({}) due in {}", + strsignal_compat(signal), + duration_cast(remaining).count()); + } + + bool cancel() + { +# ifndef WIN32 + int wstatus; + int status; + if (0 == (status = waitpid(pid, &wstatus, WNOHANG))) { + status = kill(pid, SIGKILL); + if (status) { + ldout(m_cct, 5) << __func__ << "couldn't kill the killer. Error: " << strerror(errno) << dendl; + return false; + } + while (pid == waitpid(pid, &wstatus, 0)) { + if (WIFEXITED(wstatus)) { + return false; + } + if (WIFSIGNALED(wstatus)) { + return true; + } + } + } + if (status < 0) { + ldout(m_cct, 5) << __func__ << "waitpid(killer, NOHANG) returned " << status << "; " << strerror(errno) << dendl; + } else { + ldout(m_cct, 20) << __func__ << "killer process " << pid << "\"" << describe() << "\" reaped. " + << "WIFEXITED: " << WIFEXITED(wstatus) + << "WIFSIGNALED: " << WIFSIGNALED(wstatus) + << dendl; + } +# endif + return false; + } + + static std::optional fork(CephContext *m_cct, int signal_to_send, double delay) { +# ifndef WIN32 + pid_t victim = getpid(); + clock::time_point until = clock::now() + ceph::make_timespan(delay); + + int fresult = ::fork(); + if (fresult < 0) { + ldout(m_cct, 5) << __func__ << "couldn't fork the killer. Error: " << strerror(errno) << dendl; + return std::nullopt; + } + + if (fresult) { + // this is parent + return {{m_cct, fresult, signal_to_send, until}}; + } + + const ceph::signedspan poll_interval = ceph::make_timespan(0.1); + while (getppid() == victim) { + ceph::signedspan remaining = (until - clock::now()); + if (remaining.count() > 0) { + using std::chrono::duration_cast; + using std::chrono::nanoseconds; + std::this_thread::sleep_for(duration_cast(std::min(remaining, poll_interval))); + } else { + break; + } + } + + if (getppid() != victim) { + // suicide if my parent has changed + // this means that the original parent process has terminated + ldout(m_cct, 5) << __func__ << "my parent isn't what it used to be, i'm out" << strerror(errno) << dendl; + _exit(1); + } + + int status = kill(victim, signal_to_send); + if (0 != status) { + ldout(m_cct, 5) << __func__ << "couldn't kill the victim: " << strerror(errno) << dendl; + } + _exit(status); +# endif + return std::nullopt; + } + }; + + CephContext* m_cct; + std::optional killer; + + int parse_signal(std::string&& sigdesc, Formatter* f, std::ostream& errss) + { + int result = 0; + std::transform(sigdesc.begin(), sigdesc.end(), sigdesc.begin(), + [](unsigned char c) { return std::toupper(c); }); + if (0 == sigdesc.find("-")) { + sigdesc.erase(0, 1); + } + if (0 == sigdesc.find("SIG")) { + sigdesc.erase(0, 3); + } + + if (sigdesc == "L") { + f->open_object_section("known_signals"); + for (auto& [name, num] : known_signals) { + f->dump_int(name, num); + } + f->close_section(); + } else { + try { + result = std::stoi(sigdesc); + if (result < 1 || result > 64) { + errss << "signal number should be an integer in the range [1..64]" << std::endl; + return -EINVAL; + } + } catch (const std::invalid_argument&) { + auto sig_it = known_signals.find(sigdesc); + if (sig_it == known_signals.end()) { + errss << "unknown signal name; use -l to see recognized names" << std::endl; + return -EINVAL; + } + result = sig_it->second; + } + } + return result; + } + +public: + RaiseHook(CephContext* cct) : m_cct(cct) { } + static const char* get_cmddesc() + { + return "raise " + "name=signal,type=CephString,req=false " + "name=cancel,type=CephBool,req=false " + "name=after,type=CephFloat,range=0.0,req=false "; + } + + static const char* get_help() + { + return "deliver the to the daemon process, optionally delaying seconds; " + "when --after is used, the program will fork before sleeping, which allows to " + "schedule signal delivery to a stopped daemon; it's possible to --cancel a pending signal delivery. " + " can be in the forms '9', '-9', 'kill', '-KILL'. Use `raise -l` to list known signal names."; + } + + int call(std::string_view command, const cmdmap_t& cmdmap, + const bufferlist&, + Formatter* f, + std::ostream& errss, + bufferlist& out) override + { + using std::endl; + string sigdesc; + bool cancel = cmd_getval_or(cmdmap, "cancel", false); + int signal_to_send = 0; + + if (cmd_getval(cmdmap, "signal", sigdesc)) { + signal_to_send = parse_signal(std::move(sigdesc), f, errss); + if (signal_to_send < 0) { + return signal_to_send; + } + } else if (!cancel) { + errss << "signal name or number is required" << endl; + return -EINVAL; + } + + if (cancel) { + if (killer) { + if (signal_to_send == 0 || signal_to_send == killer->signal) { + if (killer->cancel()) { + errss << "cancelled " << killer->describe() << endl; + return 0; + } + killer = std::nullopt; + } + if (signal_to_send) { + errss << "signal " << signal_to_send << " is not pending" << endl; + } + } else { + errss << "no pending signal" << endl; + } + return 1; + } + + if (!signal_to_send) { + return 0; + } + + double delay = 0; + if (cmd_getval(cmdmap, "after", delay)) { + #ifdef WIN32 + errss << "'--after' functionality is unsupported on Windows" << endl; + return -ENOTSUP; + #endif + if (killer) { + if (killer->cancel()) { + errss << "cancelled " << killer->describe() << endl; + } + } + + killer = Killer::fork(m_cct, signal_to_send, delay); + + if (killer) { + errss << "scheduled " << killer->describe() << endl; + ldout(m_cct, 20) << __func__ << "scheduled " << killer->describe() << dendl; + } else { + errss << "couldn't fork the killer" << std::endl; + return -EAGAIN; + } + } else { + ldout(m_cct, 20) << __func__ << "raising " + << " (" << strsignal_compat(signal_to_send) << ")" << dendl; + // raise the signal immediately + int status = raise(signal_to_send); + + if (0 == status) { + errss << "raised signal " + << " (" << strsignal_compat(signal_to_send) << ")" << endl; + } else { + errss << "couldn't raise signal " + << " (" << strsignal_compat(signal_to_send) << ")." + << " Error: " << strerror(errno) << endl; + + ldout(m_cct, 5) << __func__ << "couldn't raise signal " + << " (" << strsignal_compat(signal_to_send) << ")." + << " Error: " << strerror(errno) << dendl; + + return 1; + } + } + + return 0; + } +}; + bool AdminSocket::init(const std::string& path) { ldout(m_cct, 5) << "init " << path << dendl; @@ -745,6 +1044,12 @@ bool AdminSocket::init(const std::string& path) register_command("get_command_descriptions", getdescs_hook.get(), "list available commands"); + raise_hook = std::make_unique(m_cct); + register_command( + RaiseHook::get_cmddesc(), + raise_hook.get(), + RaiseHook::get_help()); + th = make_named_thread("admin_socket", &AdminSocket::entry, this); add_cleanup_file(m_path.c_str()); return true; @@ -777,6 +1082,9 @@ void AdminSocket::shutdown() unregister_commands(getdescs_hook.get()); getdescs_hook.reset(); + unregister_commands(raise_hook.get()); + raise_hook.reset(); + remove_cleanup_file(m_path); m_path.clear(); } diff --git a/src/common/admin_socket.h b/src/common/admin_socket.h index 3f364a5b711c..b95a52af7beb 100644 --- a/src/common/admin_socket.h +++ b/src/common/admin_socket.h @@ -190,6 +190,7 @@ private: std::unique_ptr version_hook; std::unique_ptr help_hook; std::unique_ptr getdescs_hook; + std::unique_ptr raise_hook; std::mutex tell_lock; std::list> tell_queue; diff --git a/src/test/admin_socket.cc b/src/test/admin_socket.cc index 369e7abbf9b6..612e2364c0cb 100644 --- a/src/test/admin_socket.cc +++ b/src/test/admin_socket.cc @@ -17,12 +17,15 @@ #include "common/admin_socket.h" #include "common/admin_socket_client.h" #include "common/ceph_argparse.h" +#include "json_spirit/json_spirit.h" #include "gtest/gtest.h" +#include "fmt/format.h" #include #include #include #include +#include using namespace std; @@ -328,6 +331,218 @@ TEST(AdminSocket, bind_and_listen) { } } +class AdminSocketRaise: public ::testing::Test +{ +public: + struct TestSignal { + int sig; + const char * name; + std::atomic count; + }; + + static void SetUpTestSuite() { + signal(sig1.sig, sighandler); + signal(sig2.sig, sighandler); + } + static void TearDownTestSuite() + { + signal(sig1.sig, SIG_DFL); + signal(sig2.sig, SIG_DFL); + } + void SetUp() override + { + std::string path = get_rand_socket_path(); + asock = std::make_unique(g_ceph_context); + asock_client = std::make_unique(path); + ASSERT_TRUE(asock->init(path)); + sig1.count = 0; + sig2.count = 0; + } + void TearDown() override + { + AdminSocketTest(asock.get()).shutdown(); + } +protected: + static TestSignal sig1; + static TestSignal sig2; + + std::unique_ptr asock; + std::unique_ptr asock_client; + + static void sighandler(int signal) + { + if (signal == sig1.sig) { + sig1.count++; + } else if (signal == sig2.sig) { + sig2.count++; + } + + // Windows resets the handler upon signal delivery + // as apparently some linuxes do as well. + // The below shouldn't hurt in any case. + ::signal(signal, sighandler); + } + std::string send_raise(std::optional arg, std::optional after, bool cancel) + { + JSONFormatter f; + f.open_object_section(""); + f.dump_string("prefix", "raise"); + if (arg) { + f.dump_string("signal", *arg); + } + if (after) { + f.dump_float("after", *after); + } + if (cancel) { + f.dump_bool("cancel", true); + } + f.close_section(); + + bufferlist command; + f.flush(command); + + std::string response; + + asock_client->do_request(command.to_str(), &response); + return response; + } + + std::string send_raise_cancel(std::optional arg = std::nullopt) { + return send_raise(arg, std::nullopt, true); + } + + std::string send_raise(std::string arg, std::optional after = std::nullopt) { + return send_raise(arg, after, false); + } +}; + +AdminSocketRaise::TestSignal AdminSocketRaise::sig1 = { SIGINT, "INT", 0 }; +AdminSocketRaise::TestSignal AdminSocketRaise::sig2 = { SIGTERM, "TERM", 0 }; + +TEST_F(AdminSocketRaise, List) { + auto r = send_raise("-l"); + json_spirit::mValue v; + ASSERT_TRUE(json_spirit::read(r, v)); + ASSERT_EQ(json_spirit::Value_type::obj_type, v.type()); + EXPECT_EQ(sig1.sig, v.get_obj()[sig1.name].get_int()); + EXPECT_EQ(sig2.sig, v.get_obj()[sig2.name].get_int()); +} + +TEST_F(AdminSocketRaise, ImmediateFormats) { + std::string name1, name2; + + name1 = sig1.name; + std::transform(name1.begin(), name1.end(), name1.begin(), [](int c) { return std::tolower(c); }); + name2 = fmt::format("-{}", sig2.name); + std::transform(name2.begin(), name2.end(), name2.begin(), [](int c) { return std::tolower(c); }); + + send_raise(fmt::format("-{}", sig1.sig)); + send_raise(name1); + send_raise(name2); + send_raise(fmt::format("{}", sig2.sig)); + EXPECT_EQ(2, sig1.count.load()); + EXPECT_EQ(2, sig2.count.load()); +} + +TEST_F(AdminSocketRaise, Async) +{ + using std::chrono::milliseconds; + +#ifdef WIN32 + GTEST_SKIP() << "Windows doesn't support --after behavior"; +#endif + + ASSERT_EQ("", send_raise(fmt::format("{}", sig1.sig))); + ASSERT_EQ("", send_raise(sig2.name, 0.1)); + + EXPECT_EQ(1, sig1.count.load()); + EXPECT_EQ(0, sig2.count.load()); + + this_thread::sleep_for(milliseconds(150)); + + EXPECT_EQ(1, sig1.count.load()); + EXPECT_EQ(1, sig2.count.load()); +} + +TEST_F(AdminSocketRaise, AsyncReschedule) +{ + using std::chrono::milliseconds; + +#ifdef WIN32 + GTEST_SKIP() << "Windows doesn't support --after behavior"; +#endif + + ASSERT_EQ("", send_raise(sig1.name, 0.1)); + ASSERT_EQ("", send_raise(sig2.name, 0.2)); + + EXPECT_EQ(0, sig1.count.load()); + EXPECT_EQ(0, sig2.count.load()); + + this_thread::sleep_for(milliseconds(150)); + + // USR1 got overridden by the second async schedule + EXPECT_EQ(0, sig1.count.load()); + EXPECT_EQ(0, sig2.count.load()); + + this_thread::sleep_for(milliseconds(100)); + EXPECT_EQ(0, sig1.count.load()); + EXPECT_EQ(1, sig2.count.load()); +} + +TEST_F(AdminSocketRaise, AsyncCancel) +{ + using std::chrono::milliseconds; + +#ifdef WIN32 + GTEST_SKIP() << "Windows doesn't support --after behavior"; +#endif + + ASSERT_EQ("", send_raise(sig1.name, 0.1)); + + EXPECT_EQ(0, sig1.count.load()); + EXPECT_EQ(0, sig2.count.load()); + + ASSERT_EQ("", send_raise_cancel(sig2.name)); + + this_thread::sleep_for(milliseconds(150)); + + // cancel shouldn't have worked because the signals + // didn't match + EXPECT_EQ(1, sig1.count.load()); + + ASSERT_EQ("", send_raise(sig2.name, 0.1)); + ASSERT_EQ("", send_raise_cancel(sig2.name)); + + this_thread::sleep_for(milliseconds(150)); + + // cancel must have worked + EXPECT_EQ(0, sig2.count.load()); + + ASSERT_EQ("", send_raise(sig1.name, 0.1)); + ASSERT_EQ("", send_raise_cancel()); + + // cancel must have worked, the counter stays 1 + EXPECT_EQ(1, sig1.count.load()); +} + +TEST_F(AdminSocketRaise, StopCont) +{ + using std::chrono::duration_cast; + using std::chrono::milliseconds; + using std::chrono::system_clock; + +#ifdef WIN32 + GTEST_SKIP() << "Windows doesn't support SIGSTOP/SIGCONT and --after"; +#endif + + auto then = system_clock::now(); + ASSERT_EQ("", send_raise("CONT", 0.2)); + ASSERT_EQ("", send_raise("STOP")); + auto elapsed = system_clock::now() - then; + // give it a 1% slack + EXPECT_LE(milliseconds(198), duration_cast(elapsed)); +} + /* * Local Variables: * compile-command: "cd .. ; -- 2.47.3