From: Patrick Donnelly Date: Wed, 1 May 2024 19:09:33 +0000 (-0400) Subject: common,ceph: add output file switch to dump json to X-Git-Tag: v19.2.3~303^2~3 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=b550bf60adf14b38db2b29adb565e1469afdc2cb;p=ceph.git common,ceph: add output file switch to dump json to The `ceph tell mds.X cache dump` and `ceph tell mds.X ops` commands have a useful `--path` argument that directs the daemon to save the dump of the output to a file local the daemon. This has several advantages: * We don't need to construct the JSON output in memory, which may be many gigabytes. * Streaming writes to a local file is significantly faster than sending the data over the ceph messenger. * The command spends as little time as possible holding relevant locks. However, only some commands support this and we could make it generic to the admin socket interface. So, add a generic --daemon-output-file argument to achieve this. The main concern with this is security (telling the daemon to write to arbitrary files) but this is gated by the session caps which require "allow_all" and that's typically only valid for the client.admin. Fixes: https://tracker.ceph.com/issues/65747 Signed-off-by: Patrick Donnelly (cherry picked from commit a3414684974380c1d335f1fcafc5245dbde1c0d6) --- diff --git a/src/ceph.in b/src/ceph.in index 11a76511a8ec2..51743dd9ae8c9 100755 --- a/src/ceph.in +++ b/src/ceph.in @@ -336,6 +336,8 @@ def parse_cmdargs(args=None, target='') -> Tuple[argparse.ArgumentParser, parser.add_argument('--concise', dest='verbose', action="store_false", help="make less verbose") + parser.add_argument('--daemon-output-file', dest='daemon_output_file', + help="output file location local to the daemon for JSON produced by tell commands") parser.add_argument('-f', '--format', choices=['json', 'json-pretty', 'xml', 'xml-pretty', 'plain', 'yaml'], help="Note: yaml is only valid for orch commands", dest='output_format') @@ -580,6 +582,8 @@ def do_command(parsed_args, target, cmdargs, sigdict, inbuf, verbose): if valid_dict: if parsed_args.output_format: valid_dict['format'] = parsed_args.output_format + if parsed_args.daemon_output_file: + valid_dict['output-file'] = parsed_args.daemon_output_file if verbose: print("Submitting command: ", valid_dict, file=sys.stderr) else: diff --git a/src/common/Formatter.h b/src/common/Formatter.h index 4bfb5c41e08d5..3e94c687114c6 100644 --- a/src/common/Formatter.h +++ b/src/common/Formatter.h @@ -272,6 +272,9 @@ public: int get_len() const override { return file.tellp(); } + std::ofstream const& get_ofstream() const { + return file; + } protected: std::ostream& get_ss() override { diff --git a/src/common/admin_socket.cc b/src/common/admin_socket.cc index 58ea36c86e8ac..1e73ce0836a92 100644 --- a/src/common/admin_socket.cc +++ b/src/common/admin_socket.cc @@ -15,6 +15,8 @@ #include #include +#include + #include "common/admin_socket.h" #include "common/admin_socket_client.h" #include "common/dout.h" @@ -506,7 +508,46 @@ void AdminSocket::execute_command( empty); } - auto f = Formatter::create(format, "json-pretty", "json-pretty"); + ldout(m_cct, 20) << __func__ << ": format is " << format << " prefix is " << prefix << dendl; + + string output; + try { + cmd_getval(cmdmap, "output-file", output); + if (!output.empty()) { + ldout(m_cct, 20) << __func__ << ": output file is " << output << dendl; + } + } catch (const bad_cmd_get& e) { + output = ""; + } + + if (output == ":tmp:") { + auto path = m_cct->_conf.get_val("tmp_file_template"); + if (int fd = mkstemp(path.data()); fd >= 0) { + close(fd); + output = path; + ldout(m_cct, 20) << __func__ << ": output file created in tmp_dir is " << output << dendl; + } else { + return on_finish(-errno, "temporary output file could not be opened", empty); + } + } + + Formatter* f; + if (!output.empty()) { + if (!(format == "json" || format == "json-pretty")) { + return on_finish(-EINVAL, "unsupported format for --output-file", empty); + } + ldout(m_cct, 10) << __func__ << ": opening file for json output: " << output << dendl; + bool pretty = (format == "json-pretty"); + auto* jff = new JSONFormatterFile(output, pretty); + auto&& of = jff->get_ofstream(); + if (!of.is_open()) { + delete jff; + return on_finish(-EIO, "output file could not be opened", empty); + } + f = jff; + } else { + f = Formatter::create(format, "json-pretty", "json-pretty"); + } auto [retval, hook] = find_matched_hook(prefix, cmdmap); switch (retval) { @@ -524,10 +565,27 @@ void AdminSocket::execute_command( hook->call_async( prefix, cmdmap, f, inbl, - [f, on_finish](int r, std::string_view err, bufferlist& out) { + [f, output, on_finish, m_cct=m_cct](int r, std::string_view err, bufferlist& out) { // handle either existing output in bufferlist *or* via formatter - if (r >= 0 && out.length() == 0) { - f->flush(out); + ldout(m_cct, 10) << __func__ << ": command completed with result " << r << dendl; + if (auto* jff = dynamic_cast(f); jff != nullptr) { + ldout(m_cct, 25) << __func__ << ": flushing file" << dendl; + jff->flush(); + auto* outf = new JSONFormatter(true); + outf->open_object_section("result"); + outf->dump_string("path", output); + outf->dump_int("result", r); + outf->dump_string("output", out.to_str()); + outf->dump_int("len", jff->get_len()); + outf->close_section(); + CachedStackStringStream css; + outf->flush(*css); + delete outf; + out.clear(); + out.append(css->strv()); + } else if (r >= 0 && out.length() == 0) { + ldout(m_cct, 25) << __func__ << ": out is empty, dumping formatter" << dendl; + f->flush(out); } delete f; on_finish(r, err, out);