From 8ebfa93982a95d6ccc66faa2a828b9386af45238 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Tue, 3 Jul 2018 16:08:26 +0800 Subject: [PATCH] cephfs-journal-tool: enable purge_queue journal's event commands Resolves: http://tracker.ceph.com/issues/24604 Signed-off-by: Xuehan Xu --- src/mds/PurgeQueue.cc | 28 +++++++++++ src/mds/PurgeQueue.h | 31 +++++++++++- src/tools/cephfs/EventOutput.cc | 79 +++++++++++++++++++----------- src/tools/cephfs/JournalFilter.cc | 50 ++++++++++++++++--- src/tools/cephfs/JournalFilter.h | 13 ++++- src/tools/cephfs/JournalScanner.cc | 14 ++++-- src/tools/cephfs/JournalScanner.h | 5 +- src/tools/cephfs/JournalTool.cc | 33 +++++++++---- 8 files changed, 203 insertions(+), 50 deletions(-) diff --git a/src/mds/PurgeQueue.cc b/src/mds/PurgeQueue.cc index 0c1a722d3c429..a656257f23a6a 100644 --- a/src/mds/PurgeQueue.cc +++ b/src/mds/PurgeQueue.cc @@ -28,9 +28,22 @@ static ostream& _prefix(std::ostream *_dout, mds_rank_t rank) { return *_dout << "mds." << rank << ".purge_queue "; } +const std::map PurgeItem::actions = { + {"NONE", PurgeItem::NONE}, + {"PURGE_FILE", PurgeItem::PURGE_FILE}, + {"TRUNCATE_FILE", PurgeItem::TRUNCATE_FILE}, + {"PURGE_DIR", PurgeItem::PURGE_DIR} +}; + void PurgeItem::encode(bufferlist &bl) const { ENCODE_START(1, 1, bl); + encode(stamp, bl); + encode(pad_size, bl); + uint8_t static const pad = 0xff; + for (unsigned int i = 0; idump_int("action", action); + f->dump_int("ino", ino); + f->dump_int("size", size); + f->open_object_section("layout"); + layout.dump(f); + f->close_section(); + f->open_object_section("SnapContext"); + snapc.dump(f); + f->close_section(); + f->open_object_section("fragtree"); + fragtree.dump(f); + f->close_section(); + } + + std::string get_type_str() const; +private: + static const std::map actions; }; WRITE_CLASS_ENCODER(PurgeItem) diff --git a/src/tools/cephfs/EventOutput.cc b/src/tools/cephfs/EventOutput.cc index 56ebecf86ea68..ed8403f32ab14 100644 --- a/src/tools/cephfs/EventOutput.cc +++ b/src/tools/cephfs/EventOutput.cc @@ -37,15 +37,21 @@ int EventOutput::binary() const } for (JournalScanner::EventMap::const_iterator i = scan.events.begin(); i != scan.events.end(); ++i) { - LogEvent *le = i->second.log_event; - bufferlist le_bin; - le->encode(le_bin, CEPH_FEATURES_SUPPORTED_DEFAULT); - + bufferlist bin; std::stringstream filename; - filename << "0x" << std::hex << i->first << std::dec << "_" << le->get_type_str() << ".bin"; + if (i->second.log_event) { + LogEvent *le = i->second.log_event; + le->encode(bin, CEPH_FEATURES_SUPPORTED_DEFAULT); + filename << "0x" << std::hex << i->first << std::dec << "_" << le->get_type_str() << ".bin"; + } else if (i->second.pi) { + PurgeItem* pi = i->second.pi; + pi->encode(bin); + filename << "0x" << std::hex << i->first << std::dec << "_" << pi->get_type_str() << ".bin"; + } + std::string const file_path = path + std::string("/") + filename.str(); std::ofstream bin_file(file_path.c_str(), std::ofstream::out | std::ofstream::binary); - le_bin.write_stream(bin_file); + bin.write_stream(bin_file); bin_file.close(); if (bin_file.fail()) { return -EIO; @@ -63,12 +69,19 @@ int EventOutput::json() const jf.open_array_section("journal"); { for (JournalScanner::EventMap::const_iterator i = scan.events.begin(); i != scan.events.end(); ++i) { - LogEvent *le = i->second.log_event; - jf.open_object_section("log_event"); - { - le->dump(&jf); + if (i->second.log_event) { + LogEvent *le = i->second.log_event; + jf.open_object_section("log_event"); + { + le->dump(&jf); + } + jf.close_section(); // log_event + } else if (i->second.pi) { + PurgeItem* pi = i->second.pi; + jf.open_object_section("purge_action"); + pi->dump(&jf); + jf.close_section(); } - jf.close_section(); // log_event } } jf.close_section(); // journal @@ -86,24 +99,30 @@ int EventOutput::json() const void EventOutput::list() const { for (JournalScanner::EventMap::const_iterator i = scan.events.begin(); i != scan.events.end(); ++i) { - std::vector ev_paths; - EMetaBlob const *emb = i->second.log_event->get_metablob(); - if (emb) { - emb->get_paths(ev_paths); - } + if (i->second.log_event) { + std::vector ev_paths; + EMetaBlob const *emb = i->second.log_event->get_metablob(); + if (emb) { + emb->get_paths(ev_paths); + } - std::string detail; - if (i->second.log_event->get_type() == EVENT_UPDATE) { - EUpdate *eu = reinterpret_cast(i->second.log_event); - detail = eu->type; - } + std::string detail; + if (i->second.log_event->get_type() == EVENT_UPDATE) { + EUpdate *eu = reinterpret_cast(i->second.log_event); + detail = eu->type; + } - std::cout <second.log_event->get_stamp() << " 0x" - << std::hex << i->first << std::dec << " " - << i->second.log_event->get_type_str() << ": " - << " (" << detail << ")" << std::endl; - for (std::vector::iterator i = ev_paths.begin(); i != ev_paths.end(); ++i) { - std::cout << " " << *i << std::endl; + std::cout <second.log_event->get_stamp() << " 0x" + << std::hex << i->first << std::dec << " " + << i->second.log_event->get_type_str() << ": " + << " (" << detail << ")" << std::endl; + for (std::vector::iterator i = ev_paths.begin(); i != ev_paths.end(); ++i) { + std::cout << " " << *i << std::endl; + } + } else if (i->second.pi) { + std::cout << i->second.pi->stamp << " 0x" + << std::hex << i->first << std::dec << " " + << i->second.pi->get_type_str() << std::endl; } } } @@ -112,7 +131,11 @@ void EventOutput::summary() const { std::map type_count; for (JournalScanner::EventMap::const_iterator i = scan.events.begin(); i != scan.events.end(); ++i) { - std::string const type = i->second.log_event->get_type_str(); + std::string type; + if (i->second.log_event) + type = i->second.log_event->get_type_str(); + else if (i->second.pi) + type = i->second.pi->get_type_str(); if (type_count.count(type) == 0) { type_count[type] = 0; } diff --git a/src/tools/cephfs/JournalFilter.cc b/src/tools/cephfs/JournalFilter.cc index 6db9f4b074664..3b30ffa092f28 100644 --- a/src/tools/cephfs/JournalFilter.cc +++ b/src/tools/cephfs/JournalFilter.cc @@ -25,6 +25,24 @@ const string JournalFilter::range_separator(".."); +bool JournalFilter::apply(uint64_t pos, PurgeItem &pi) const +{ + /* Filtering by journal offset range */ + if (pos < range_start || pos >= range_end) { + return false; + } + + if (purge_action != PurgeItem::NONE) { + if (pi.action != purge_action) + return false; + } + + if (inode) { + if (inode != pi.ino) + return false; + } + return true; +} /* * Return whether a LogEvent is to be included or excluded. @@ -187,6 +205,10 @@ int JournalFilter::parse_args( } } } else if (ceph_argparse_witharg(argv, arg, &arg_str, "--path", (char*)NULL)) { + if (!type.compare("purge_queue")) { + derr << "Invalid filter arguments: purge_queue doesn't take \"--path\"." << dendl; + return -EINVAL; + } dout(4) << "Filtering by path '" << arg_str << "'" << dendl; path_expr = arg_str; } else if (ceph_argparse_witharg(argv, arg, &arg_str, "--inode", (char*)NULL)) { @@ -198,14 +220,21 @@ int JournalFilter::parse_args( return -EINVAL; } } else if (ceph_argparse_witharg(argv, arg, &arg_str, "--type", (char*)NULL)) { - std::string parse_err; - event_type = LogEvent::str_to_type(arg_str); - if (event_type == LogEvent::EventType(-1)) { - derr << "Invalid event type '" << arg_str << "': " << parse_err << dendl; - return -EINVAL; + try { + if (!type.compare("mdlog")) { + event_type = LogEvent::str_to_type(arg_str); + } else if (!type.compare("purge_queue")) { + purge_action = PurgeItem::str_to_type(arg_str); + } + } catch (std::out_of_range) { + derr << "Invalid event type '" << arg_str << "'" << dendl; + return -EINVAL; } - } else if (ceph_argparse_witharg(argv, arg, &arg_str, "--frag", (char*)NULL)) { + if (!type.compare("purge_queue")) { + derr << "Invalid filter arguments: purge_queue doesn't take \"--frag\"." << dendl; + return -EINVAL; + } std::string const frag_sep = "."; size_t sep_loc = arg_str.find(frag_sep); std::string inode_str; @@ -234,9 +263,18 @@ int JournalFilter::parse_args( frag = dirfrag_t(frag_ino, frag_t(frag_enc)); dout(4) << "dirfrag filter: '" << frag << "'" << dendl; } else if (ceph_argparse_witharg(argv, arg, &arg_str, "--dname", (char*)NULL)) { + if (!type.compare("purge_queue")) { + derr << "Invalid filter arguments: purge_queue doesn't take \"--dname\"." << dendl; + return -EINVAL; + } frag_dentry = arg_str; dout(4) << "dentry filter: '" << frag_dentry << "'" << dendl; } else if (ceph_argparse_witharg(argv, arg, &arg_str, "--client", (char*)NULL)) { + if (!type.compare("purge_queue")) { + derr << "Invalid filter arguments: purge_queue doesn't take \"--client\"." << dendl; + return -EINVAL; + } + std::string parse_err; int64_t client_num = strict_strtoll(arg_str.c_str(), 0, &parse_err); if (!parse_err.empty()) { diff --git a/src/tools/cephfs/JournalFilter.h b/src/tools/cephfs/JournalFilter.h index 4fd8ffff26880..f7a2db614a135 100644 --- a/src/tools/cephfs/JournalFilter.h +++ b/src/tools/cephfs/JournalFilter.h @@ -17,6 +17,7 @@ #include "mds/mdstypes.h" #include "mds/LogEvent.h" +#include "mds/PurgeQueue.h" /** * A set of conditions for narrowing down a search through the journal @@ -39,6 +40,11 @@ class JournalFilter /* Filtering by type */ LogEvent::EventType event_type; + std::string type; + + /* Filtering by PurgeItem::Action */ + PurgeItem::Action purge_action; + /* Filtering by dirfrag */ dirfrag_t frag; std::string frag_dentry; //< optional, filter dentry name within fragment @@ -47,14 +53,17 @@ class JournalFilter entity_name_t client_name; public: - JournalFilter() : + JournalFilter(std::string t) : range_start(0), range_end(-1), inode(0), - event_type(0) {} + event_type(0), + type(t), + purge_action(PurgeItem::NONE) {} bool get_range(uint64_t &start, uint64_t &end) const; bool apply(uint64_t pos, LogEvent &le) const; + bool apply(uint64_t pos, PurgeItem &pi) const; int parse_args( std::vector &argv, std::vector::iterator &arg); diff --git a/src/tools/cephfs/JournalScanner.cc b/src/tools/cephfs/JournalScanner.cc index a68c88441264b..adac7908fd26d 100644 --- a/src/tools/cephfs/JournalScanner.cc +++ b/src/tools/cephfs/JournalScanner.cc @@ -301,10 +301,15 @@ int JournalScanner::scan_events() valid_entry = false; } } else if (type == "purge_queue"){ - PurgeItem pi; + PurgeItem* pi = new PurgeItem(); try { auto q = le_bl.cbegin(); - ::decode(pi, q); + pi->decode(q); + if (filter.apply(read_offset, *pi)) { + events[read_offset] = EventRecord(pi, consumed); + } else { + delete pi; + } } catch (const buffer::error &err) { valid_entry = false; } @@ -347,7 +352,10 @@ JournalScanner::~JournalScanner() } dout(4) << events.size() << " events" << dendl; for (EventMap::iterator i = events.begin(); i != events.end(); ++i) { - delete i->second.log_event; + if (i->second.log_event) + delete i->second.log_event; + else if (i->second.pi) + delete i->second.pi; } events.clear(); } diff --git a/src/tools/cephfs/JournalScanner.h b/src/tools/cephfs/JournalScanner.h index dad7dc5ac65c5..89ee2fd2803f4 100644 --- a/src/tools/cephfs/JournalScanner.h +++ b/src/tools/cephfs/JournalScanner.h @@ -65,6 +65,7 @@ class JournalScanner io(io_), rank(rank_), type(type_), + filter(type_), is_mdlog(false), pointer_present(false), pointer_valid(false), @@ -89,8 +90,10 @@ class JournalScanner class EventRecord { public: EventRecord() : log_event(NULL), raw_size(0) {} - EventRecord(LogEvent *le, uint32_t rs) : log_event(le), raw_size(rs) {} + EventRecord(LogEvent *le, uint32_t rs) : log_event(le), pi(NULL), raw_size(rs) {} + EventRecord(PurgeItem* p, uint32_t rs) : log_event(NULL), pi(p), raw_size(rs) {} LogEvent *log_event; + PurgeItem *pi; uint32_t raw_size; //< Size from start offset including all encoding overhead }; diff --git a/src/tools/cephfs/JournalTool.cc b/src/tools/cephfs/JournalTool.cc index 0bbb832ed594a..aaa1d644c8aff 100644 --- a/src/tools/cephfs/JournalTool.cc +++ b/src/tools/cephfs/JournalTool.cc @@ -168,7 +168,7 @@ int JournalTool::main(std::vector &argv) r = main_journal(argv); } else if (mode == std::string("header")) { r = main_header(argv); - } else if (type == std::string("mdlog") && mode == std::string("event")) { + } else if (mode == std::string("event")) { r = main_event(argv); } else { cerr << "Bad command '" << mode << "'" << std::endl; @@ -246,7 +246,7 @@ int JournalTool::main_journal(std::vector &argv) */ int JournalTool::main_header(std::vector &argv) { - JournalFilter filter; + JournalFilter filter(type); JournalScanner js(input, rank, type, filter); int r = js.scan(false); if (r < 0) { @@ -349,6 +349,11 @@ int JournalTool::main_event(std::vector &argv) return -EINVAL; } + if (command == "recover_dentries" && type != "mdlog") { + derr << "journaler for " << type << " can't do \"recover_dentries\"." << dendl; + return -EINVAL; + } + if (arg == argv.end()) { derr << "Incomplete command line" << dendl; return -EINVAL; @@ -356,7 +361,7 @@ int JournalTool::main_event(std::vector &argv) // Parse filter options // ==================== - JournalFilter filter; + JournalFilter filter(type); r = filter.parse_args(argv, arg); if (r) { return r; @@ -539,7 +544,7 @@ int JournalTool::journal_inspect() { int r; - JournalFilter filter; + JournalFilter filter(type); JournalScanner js(input, rank, type, filter); r = js.scan(); if (r) { @@ -1043,7 +1048,13 @@ int JournalTool::erase_region(JournalScanner const &js, uint64_t const pos, uint // is needed inside the ENoOp to make up the difference. bufferlist tmp; ENoOp enoop(0); - enoop.encode_with_header(tmp, CEPH_FEATURES_SUPPORTED_DEFAULT); + PurgeItem pi; + + if (type == "mdlog") { + enoop.encode_with_header(tmp, CEPH_FEATURES_SUPPORTED_DEFAULT); + } else if (type == "purge_queue") { + pi.encode(tmp); + } dout(4) << "erase_region " << pos << " len=" << length << dendl; @@ -1056,12 +1067,16 @@ int JournalTool::erase_region(JournalScanner const &js, uint64_t const pos, uint return -EINVAL; } - // Serialize an ENoOp with the correct amount of padding - enoop = ENoOp(padding); bufferlist entry; - enoop.encode_with_header(entry, CEPH_FEATURES_SUPPORTED_DEFAULT); + if (type == "mdlog") { + // Serialize an ENoOp with the correct amount of padding + enoop = ENoOp(padding); + enoop.encode_with_header(entry, CEPH_FEATURES_SUPPORTED_DEFAULT); + } else if (type == "purge_queue") { + pi.pad_size = padding; + pi.encode(entry); + } JournalStream stream(JOURNAL_FORMAT_RESILIENT); - // Serialize region of log stream bufferlist log_data; stream.write(entry, &log_data, pos); -- 2.39.5