From: Xuehan Xu Date: Tue, 3 Jul 2018 08:08:26 +0000 (+0800) Subject: cephfs-journal-tool: enable purge_queue journal's event commands X-Git-Tag: v13.2.3~74^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=166ce6f922a941d0d37d5cf173eaef5752e780b4;p=ceph.git cephfs-journal-tool: enable purge_queue journal's event commands Resolves: http://tracker.ceph.com/issues/24604 Signed-off-by: Xuehan Xu (cherry picked from commit 5582faef0767bb2de32cb9eeeffdbca7d4323747) Conflicts: src/mds/PurgeQueue.h: Resvoled for decode src/tools/cephfs/JournalScanner.cc: Resolved in scan_events --- diff --git a/src/mds/PurgeQueue.cc b/src/mds/PurgeQueue.cc index ddf2ed5fa71..8ca2ceaf712 100644 --- a/src/mds/PurgeQueue.cc +++ b/src/mds/PurgeQueue.cc @@ -28,9 +28,16 @@ static ostream& _prefix(std::ostream *_dout, mds_rank_t rank) { return *_dout << "mds." << rank << ".purge_queue "; } +const std::map PurgeItem::actions = { + {"NONE", PurgeItem::NONE}, + {"PURGE_FILE", PurgeItem::PURGE_FILE}, + {"TRUNCATE_FILE", PurgeItem::TRUNCATE_FILE}, + {"PURGE_DIR", PurgeItem::PURGE_DIR} +}; + void PurgeItem::encode(bufferlist &bl) const { - ENCODE_START(1, 1, bl); + ENCODE_START(2, 1, bl); encode((uint8_t)action, bl); encode(ino, bl); encode(size, bl); @@ -38,12 +45,17 @@ void PurgeItem::encode(bufferlist &bl) const encode(old_pools, bl); encode(snapc, bl); encode(fragtree, bl); + encode(stamp, bl); + uint8_t static const pad = 0xff; + for (unsigned int i = 0; i= 2) { + decode(stamp, p); + } DECODE_FINISH(p); } @@ -651,3 +666,15 @@ bool PurgeQueue::drain( return false; } +std::string PurgeItem::get_type_str() const +{ + switch(action) { + case PurgeItem::NONE: return "NONE"; + case PurgeItem::PURGE_FILE: return "PURGE_FILE"; + case PurgeItem::PURGE_DIR: return "PURGE_DIR"; + case PurgeItem::TRUNCATE_FILE: return "TRUNCATE_FILE"; + default: + return "UNKNOWN"; + } +} + diff --git a/src/mds/PurgeQueue.h b/src/mds/PurgeQueue.h index 079f3763165..27c99e6e742 100644 --- a/src/mds/PurgeQueue.h +++ b/src/mds/PurgeQueue.h @@ -36,6 +36,11 @@ public: PURGE_DIR }; + utime_t stamp; + //None PurgeItem serves as NoOp for splicing out journal entries; + //so there has to be a "pad_size" to specify the size of journal + //space to be spliced. + uint32_t pad_size; Action action; inodeno_t ino; uint64_t size; @@ -45,11 +50,35 @@ public: fragtree_t fragtree; PurgeItem() - : action(NONE), ino(0), size(0) + : pad_size(0), action(NONE), ino(0), size(0) {} void encode(bufferlist &bl) const; void decode(bufferlist::iterator &p); + + static Action str_to_type(std::string_view str) { + return PurgeItem::actions.at(std::string(str)); + } + + void dump(Formatter *f) const + { + f->dump_int("action", action); + f->dump_int("ino", ino); + f->dump_int("size", size); + f->open_object_section("layout"); + layout.dump(f); + f->close_section(); + f->open_object_section("SnapContext"); + snapc.dump(f); + f->close_section(); + f->open_object_section("fragtree"); + fragtree.dump(f); + f->close_section(); + } + + std::string get_type_str() const; +private: + static const std::map actions; }; WRITE_CLASS_ENCODER(PurgeItem) diff --git a/src/mds/StrayManager.cc b/src/mds/StrayManager.cc index 08c64f61c20..ab60e06f54b 100644 --- a/src/mds/StrayManager.cc +++ b/src/mds/StrayManager.cc @@ -102,6 +102,7 @@ void StrayManager::purge(CDentry *dn) PurgeItem item; item.ino = in->inode.ino; + item.stamp = ceph_clock_now(); if (in->is_dir()) { item.action = PurgeItem::PURGE_DIR; item.fragtree = in->dirfragtree; @@ -737,6 +738,7 @@ void StrayManager::truncate(CDentry *dn) item.layout = in->inode.layout; item.snapc = *snapc; item.size = to; + item.stamp = ceph_clock_now(); purge_queue.push(item, new C_IO_PurgeStrayPurged( this, dn, true)); diff --git a/src/tools/cephfs/EventOutput.cc b/src/tools/cephfs/EventOutput.cc index 56ebecf86ea..ed8403f32ab 100644 --- a/src/tools/cephfs/EventOutput.cc +++ b/src/tools/cephfs/EventOutput.cc @@ -37,15 +37,21 @@ int EventOutput::binary() const } for (JournalScanner::EventMap::const_iterator i = scan.events.begin(); i != scan.events.end(); ++i) { - LogEvent *le = i->second.log_event; - bufferlist le_bin; - le->encode(le_bin, CEPH_FEATURES_SUPPORTED_DEFAULT); - + bufferlist bin; std::stringstream filename; - filename << "0x" << std::hex << i->first << std::dec << "_" << le->get_type_str() << ".bin"; + if (i->second.log_event) { + LogEvent *le = i->second.log_event; + le->encode(bin, CEPH_FEATURES_SUPPORTED_DEFAULT); + filename << "0x" << std::hex << i->first << std::dec << "_" << le->get_type_str() << ".bin"; + } else if (i->second.pi) { + PurgeItem* pi = i->second.pi; + pi->encode(bin); + filename << "0x" << std::hex << i->first << std::dec << "_" << pi->get_type_str() << ".bin"; + } + std::string const file_path = path + std::string("/") + filename.str(); std::ofstream bin_file(file_path.c_str(), std::ofstream::out | std::ofstream::binary); - le_bin.write_stream(bin_file); + bin.write_stream(bin_file); bin_file.close(); if (bin_file.fail()) { return -EIO; @@ -63,12 +69,19 @@ int EventOutput::json() const jf.open_array_section("journal"); { for (JournalScanner::EventMap::const_iterator i = scan.events.begin(); i != scan.events.end(); ++i) { - LogEvent *le = i->second.log_event; - jf.open_object_section("log_event"); - { - le->dump(&jf); + if (i->second.log_event) { + LogEvent *le = i->second.log_event; + jf.open_object_section("log_event"); + { + le->dump(&jf); + } + jf.close_section(); // log_event + } else if (i->second.pi) { + PurgeItem* pi = i->second.pi; + jf.open_object_section("purge_action"); + pi->dump(&jf); + jf.close_section(); } - jf.close_section(); // log_event } } jf.close_section(); // journal @@ -86,24 +99,30 @@ int EventOutput::json() const void EventOutput::list() const { for (JournalScanner::EventMap::const_iterator i = scan.events.begin(); i != scan.events.end(); ++i) { - std::vector ev_paths; - EMetaBlob const *emb = i->second.log_event->get_metablob(); - if (emb) { - emb->get_paths(ev_paths); - } + if (i->second.log_event) { + std::vector ev_paths; + EMetaBlob const *emb = i->second.log_event->get_metablob(); + if (emb) { + emb->get_paths(ev_paths); + } - std::string detail; - if (i->second.log_event->get_type() == EVENT_UPDATE) { - EUpdate *eu = reinterpret_cast(i->second.log_event); - detail = eu->type; - } + std::string detail; + if (i->second.log_event->get_type() == EVENT_UPDATE) { + EUpdate *eu = reinterpret_cast(i->second.log_event); + detail = eu->type; + } - std::cout <second.log_event->get_stamp() << " 0x" - << std::hex << i->first << std::dec << " " - << i->second.log_event->get_type_str() << ": " - << " (" << detail << ")" << std::endl; - for (std::vector::iterator i = ev_paths.begin(); i != ev_paths.end(); ++i) { - std::cout << " " << *i << std::endl; + std::cout <second.log_event->get_stamp() << " 0x" + << std::hex << i->first << std::dec << " " + << i->second.log_event->get_type_str() << ": " + << " (" << detail << ")" << std::endl; + for (std::vector::iterator i = ev_paths.begin(); i != ev_paths.end(); ++i) { + std::cout << " " << *i << std::endl; + } + } else if (i->second.pi) { + std::cout << i->second.pi->stamp << " 0x" + << std::hex << i->first << std::dec << " " + << i->second.pi->get_type_str() << std::endl; } } } @@ -112,7 +131,11 @@ void EventOutput::summary() const { std::map type_count; for (JournalScanner::EventMap::const_iterator i = scan.events.begin(); i != scan.events.end(); ++i) { - std::string const type = i->second.log_event->get_type_str(); + std::string type; + if (i->second.log_event) + type = i->second.log_event->get_type_str(); + else if (i->second.pi) + type = i->second.pi->get_type_str(); if (type_count.count(type) == 0) { type_count[type] = 0; } diff --git a/src/tools/cephfs/JournalFilter.cc b/src/tools/cephfs/JournalFilter.cc index 6db9f4b0746..3b30ffa092f 100644 --- a/src/tools/cephfs/JournalFilter.cc +++ b/src/tools/cephfs/JournalFilter.cc @@ -25,6 +25,24 @@ const string JournalFilter::range_separator(".."); +bool JournalFilter::apply(uint64_t pos, PurgeItem &pi) const +{ + /* Filtering by journal offset range */ + if (pos < range_start || pos >= range_end) { + return false; + } + + if (purge_action != PurgeItem::NONE) { + if (pi.action != purge_action) + return false; + } + + if (inode) { + if (inode != pi.ino) + return false; + } + return true; +} /* * Return whether a LogEvent is to be included or excluded. @@ -187,6 +205,10 @@ int JournalFilter::parse_args( } } } else if (ceph_argparse_witharg(argv, arg, &arg_str, "--path", (char*)NULL)) { + if (!type.compare("purge_queue")) { + derr << "Invalid filter arguments: purge_queue doesn't take \"--path\"." << dendl; + return -EINVAL; + } dout(4) << "Filtering by path '" << arg_str << "'" << dendl; path_expr = arg_str; } else if (ceph_argparse_witharg(argv, arg, &arg_str, "--inode", (char*)NULL)) { @@ -198,14 +220,21 @@ int JournalFilter::parse_args( return -EINVAL; } } else if (ceph_argparse_witharg(argv, arg, &arg_str, "--type", (char*)NULL)) { - std::string parse_err; - event_type = LogEvent::str_to_type(arg_str); - if (event_type == LogEvent::EventType(-1)) { - derr << "Invalid event type '" << arg_str << "': " << parse_err << dendl; - return -EINVAL; + try { + if (!type.compare("mdlog")) { + event_type = LogEvent::str_to_type(arg_str); + } else if (!type.compare("purge_queue")) { + purge_action = PurgeItem::str_to_type(arg_str); + } + } catch (std::out_of_range) { + derr << "Invalid event type '" << arg_str << "'" << dendl; + return -EINVAL; } - } else if (ceph_argparse_witharg(argv, arg, &arg_str, "--frag", (char*)NULL)) { + if (!type.compare("purge_queue")) { + derr << "Invalid filter arguments: purge_queue doesn't take \"--frag\"." << dendl; + return -EINVAL; + } std::string const frag_sep = "."; size_t sep_loc = arg_str.find(frag_sep); std::string inode_str; @@ -234,9 +263,18 @@ int JournalFilter::parse_args( frag = dirfrag_t(frag_ino, frag_t(frag_enc)); dout(4) << "dirfrag filter: '" << frag << "'" << dendl; } else if (ceph_argparse_witharg(argv, arg, &arg_str, "--dname", (char*)NULL)) { + if (!type.compare("purge_queue")) { + derr << "Invalid filter arguments: purge_queue doesn't take \"--dname\"." << dendl; + return -EINVAL; + } frag_dentry = arg_str; dout(4) << "dentry filter: '" << frag_dentry << "'" << dendl; } else if (ceph_argparse_witharg(argv, arg, &arg_str, "--client", (char*)NULL)) { + if (!type.compare("purge_queue")) { + derr << "Invalid filter arguments: purge_queue doesn't take \"--client\"." << dendl; + return -EINVAL; + } + std::string parse_err; int64_t client_num = strict_strtoll(arg_str.c_str(), 0, &parse_err); if (!parse_err.empty()) { diff --git a/src/tools/cephfs/JournalFilter.h b/src/tools/cephfs/JournalFilter.h index 4fd8ffff268..f7a2db614a1 100644 --- a/src/tools/cephfs/JournalFilter.h +++ b/src/tools/cephfs/JournalFilter.h @@ -17,6 +17,7 @@ #include "mds/mdstypes.h" #include "mds/LogEvent.h" +#include "mds/PurgeQueue.h" /** * A set of conditions for narrowing down a search through the journal @@ -39,6 +40,11 @@ class JournalFilter /* Filtering by type */ LogEvent::EventType event_type; + std::string type; + + /* Filtering by PurgeItem::Action */ + PurgeItem::Action purge_action; + /* Filtering by dirfrag */ dirfrag_t frag; std::string frag_dentry; //< optional, filter dentry name within fragment @@ -47,14 +53,17 @@ class JournalFilter entity_name_t client_name; public: - JournalFilter() : + JournalFilter(std::string t) : range_start(0), range_end(-1), inode(0), - event_type(0) {} + event_type(0), + type(t), + purge_action(PurgeItem::NONE) {} bool get_range(uint64_t &start, uint64_t &end) const; bool apply(uint64_t pos, LogEvent &le) const; + bool apply(uint64_t pos, PurgeItem &pi) const; int parse_args( std::vector &argv, std::vector::iterator &arg); diff --git a/src/tools/cephfs/JournalScanner.cc b/src/tools/cephfs/JournalScanner.cc index cb2ea244a09..932587bea11 100644 --- a/src/tools/cephfs/JournalScanner.cc +++ b/src/tools/cephfs/JournalScanner.cc @@ -301,10 +301,15 @@ int JournalScanner::scan_events() valid_entry = false; } } else if (type == "purge_queue"){ - PurgeItem pi; + PurgeItem* pi = new PurgeItem(); try { - bufferlist::iterator q = le_bl.begin(); - ::decode(pi, q); + auto q = le_bl.begin(); + pi->decode(q); + if (filter.apply(read_offset, *pi)) { + events[read_offset] = EventRecord(pi, consumed); + } else { + delete pi; + } } catch (const buffer::error &err) { valid_entry = false; } @@ -347,7 +352,10 @@ JournalScanner::~JournalScanner() } dout(4) << events.size() << " events" << dendl; for (EventMap::iterator i = events.begin(); i != events.end(); ++i) { - delete i->second.log_event; + if (i->second.log_event) + delete i->second.log_event; + else if (i->second.pi) + delete i->second.pi; } events.clear(); } diff --git a/src/tools/cephfs/JournalScanner.h b/src/tools/cephfs/JournalScanner.h index dad7dc5ac65..89ee2fd2803 100644 --- a/src/tools/cephfs/JournalScanner.h +++ b/src/tools/cephfs/JournalScanner.h @@ -65,6 +65,7 @@ class JournalScanner io(io_), rank(rank_), type(type_), + filter(type_), is_mdlog(false), pointer_present(false), pointer_valid(false), @@ -89,8 +90,10 @@ class JournalScanner class EventRecord { public: EventRecord() : log_event(NULL), raw_size(0) {} - EventRecord(LogEvent *le, uint32_t rs) : log_event(le), raw_size(rs) {} + EventRecord(LogEvent *le, uint32_t rs) : log_event(le), pi(NULL), raw_size(rs) {} + EventRecord(PurgeItem* p, uint32_t rs) : log_event(NULL), pi(p), raw_size(rs) {} LogEvent *log_event; + PurgeItem *pi; uint32_t raw_size; //< Size from start offset including all encoding overhead }; diff --git a/src/tools/cephfs/JournalTool.cc b/src/tools/cephfs/JournalTool.cc index 95a06111cdb..7dc6552fa4a 100644 --- a/src/tools/cephfs/JournalTool.cc +++ b/src/tools/cephfs/JournalTool.cc @@ -168,7 +168,7 @@ int JournalTool::main(std::vector &argv) r = main_journal(argv); } else if (mode == std::string("header")) { r = main_header(argv); - } else if (type == std::string("mdlog") && mode == std::string("event")) { + } else if (mode == std::string("event")) { r = main_event(argv); } else { cerr << "Bad command '" << mode << "'" << std::endl; @@ -237,7 +237,7 @@ int JournalTool::main_journal(std::vector &argv) */ int JournalTool::main_header(std::vector &argv) { - JournalFilter filter; + JournalFilter filter(type); JournalScanner js(input, rank, type, filter); int r = js.scan(false); if (r < 0) { @@ -340,6 +340,11 @@ int JournalTool::main_event(std::vector &argv) return -EINVAL; } + if (command == "recover_dentries" && type != "mdlog") { + derr << "journaler for " << type << " can't do \"recover_dentries\"." << dendl; + return -EINVAL; + } + if (arg == argv.end()) { derr << "Incomplete command line" << dendl; return -EINVAL; @@ -347,7 +352,7 @@ int JournalTool::main_event(std::vector &argv) // Parse filter options // ==================== - JournalFilter filter; + JournalFilter filter(type); r = filter.parse_args(argv, arg); if (r) { return r; @@ -530,7 +535,7 @@ int JournalTool::journal_inspect() { int r; - JournalFilter filter; + JournalFilter filter(type); JournalScanner js(input, rank, type, filter); r = js.scan(); if (r) { @@ -1034,7 +1039,13 @@ int JournalTool::erase_region(JournalScanner const &js, uint64_t const pos, uint // is needed inside the ENoOp to make up the difference. bufferlist tmp; ENoOp enoop(0); - enoop.encode_with_header(tmp, CEPH_FEATURES_SUPPORTED_DEFAULT); + PurgeItem pi; + + if (type == "mdlog") { + enoop.encode_with_header(tmp, CEPH_FEATURES_SUPPORTED_DEFAULT); + } else if (type == "purge_queue") { + pi.encode(tmp); + } dout(4) << "erase_region " << pos << " len=" << length << dendl; @@ -1047,12 +1058,16 @@ int JournalTool::erase_region(JournalScanner const &js, uint64_t const pos, uint return -EINVAL; } - // Serialize an ENoOp with the correct amount of padding - enoop = ENoOp(padding); bufferlist entry; - enoop.encode_with_header(entry, CEPH_FEATURES_SUPPORTED_DEFAULT); + if (type == "mdlog") { + // Serialize an ENoOp with the correct amount of padding + enoop = ENoOp(padding); + enoop.encode_with_header(entry, CEPH_FEATURES_SUPPORTED_DEFAULT); + } else if (type == "purge_queue") { + pi.pad_size = padding; + pi.encode(entry); + } JournalStream stream(JOURNAL_FORMAT_RESILIENT); - // Serialize region of log stream bufferlist log_data; stream.write(entry, &log_data, pos);