From: Yan, Zheng Date: Fri, 6 Jul 2018 01:29:25 +0000 (+0800) Subject: mds: reduce memory fragments of EMetaBlob X-Git-Tag: v14.0.1~331^2~1 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=d12f3e311854d372d69dbf998e552a875ea9f621;p=ceph.git mds: reduce memory fragments of EMetaBlob avoid using shared_ptr list Signed-off-by: "Yan, Zheng" --- diff --git a/src/include/encoding.h b/src/include/encoding.h index 81e86a0103a1b..3b21569c46dd6 100644 --- a/src/include/encoding.h +++ b/src/include/encoding.h @@ -651,9 +651,8 @@ inline std::enable_if_t decode(n, p); ls.clear(); while (n--) { - T v; - decode(v, p); - ls.push_back(v); + ls.emplace_back(); + decode(ls.back(), p); } } @@ -1165,9 +1164,8 @@ inline void decode(std::deque& ls, bufferlist::const_iterator& p) decode(n, p); ls.clear(); while (n--) { - T v; - decode(v, p); - ls.push_back(v); + ls.emplace_back(); + decode(ls.back(), p); } } diff --git a/src/mds/LogEvent.h b/src/mds/LogEvent.h index 7ded7d7a14cb4..11aa4a82eb763 100644 --- a/src/mds/LogEvent.h +++ b/src/mds/LogEvent.h @@ -70,7 +70,9 @@ public: explicit LogEvent(int t) : _type(t), _start_off(0), _segment(0) { } + LogEvent(const LogEvent&) = delete; virtual ~LogEvent() { } + LogEvent& operator=(const LogEvent&) = delete; string get_type_str() const; static EventType str_to_type(std::string_view str); diff --git a/src/mds/events/EMetaBlob.h b/src/mds/events/EMetaBlob.h index cf6701f0a6b0d..a1077f37636b2 100644 --- a/src/mds/events/EMetaBlob.h +++ b/src/mds/events/EMetaBlob.h @@ -77,9 +77,6 @@ public: __u8 state{0}; CInode::mempool_old_inode_map old_inodes; // XXX should not be part of mempool; wait for std::pmr to simplify - fullbit(const fullbit& o); - const fullbit& operator=(const fullbit& o); - fullbit(std::string_view d, snapid_t df, snapid_t dl, version_t v, const CInode::mempool_inode& i, const fragtree_t &dft, const CInode::mempool_xattr_map &xa, std::string_view sym, @@ -100,7 +97,9 @@ public: decode(p); } fullbit() {} + fullbit(const fullbit&) = delete; ~fullbit() {} + fullbit& operator=(const fullbit&) = delete; void encode(bufferlist& bl, uint64_t features) const; void decode(bufferlist::const_iterator &bl); @@ -181,7 +180,7 @@ public: void decode(bufferlist::const_iterator &bl); void dump(Formatter *f) const; static void generate_test_instances(list& ls); - void print(ostream& out) { + void print(ostream& out) const { out << " nullbit dn " << dn << " [" << dnfirst << "," << dnlast << "] dnv " << dnv << " dirty=" << dirty << std::endl; } @@ -207,12 +206,14 @@ public: private: mutable bufferlist dnbl; mutable bool dn_decoded; - mutable list > dfull; - mutable list dremote; - mutable list dnull; + mutable list dfull; + mutable vector dremote; + mutable vector dnull; public: dirlump() : state(0), nfull(0), nremote(0), nnull(0), dn_decoded(true) { } + dirlump(const dirlump&) = delete; + dirlump& operator=(const dirlump&) = delete; bool is_complete() const { return state & STATE_COMPLETE; } void mark_complete() { state |= STATE_COMPLETE; } @@ -225,26 +226,36 @@ public: bool is_dirty_dft() { return state & STATE_DIRTYDFT; } void mark_dirty_dft() { state |= STATE_DIRTYDFT; } - const list > &get_dfull() const { return dfull; } - const list &get_dremote() const { return dremote; } - const list &get_dnull() const { return dnull; } + const list &get_dfull() const { return dfull; } + list &_get_dfull() { return dfull; } + const vector &get_dremote() const { return dremote; } + const vector &get_dnull() const { return dnull; } - void add_dnull(nullbit const &n) { dnull.push_back(n); }; - void add_dfull(std::shared_ptr const &p) { dfull.push_back(p); }; - void add_dremote(remotebit const &r) { dremote.push_back(r); }; + template< class... Args> + void add_dfull(Args&&... args) { + dfull.emplace_back(std::forward(args)...); + } + template< class... Args> + void add_dremote(Args&&... args) { + dremote.emplace_back(std::forward(args)...); + } + template< class... Args> + void add_dnull(Args&&... args) { + dnull.emplace_back(std::forward(args)...); + } - void print(dirfrag_t dirfrag, ostream& out) { + void print(dirfrag_t dirfrag, ostream& out) const { out << "dirlump " << dirfrag << " v " << fnode.version << " state " << state << " num " << nfull << "/" << nremote << "/" << nnull << std::endl; _decode_bits(); - for (list >::iterator p = dfull.begin(); p != dfull.end(); ++p) - (*p)->print(out); - for (list::iterator p = dremote.begin(); p != dremote.end(); ++p) - p->print(out); - for (list::iterator p = dnull.begin(); p != dnull.end(); ++p) - p->print(out); + for (const auto& p : dfull) + p.print(out); + for (const auto& p : dremote) + p.print(out); + for (const auto& p : dnull) + p.print(out); } string state_string() const { @@ -290,16 +301,16 @@ public: WRITE_CLASS_ENCODER_FEATURES(dirlump) // my lumps. preserve the order we added them in a list. - list lump_order; + vector lump_order; map lump_map; - list > roots; + list roots; public: - list > table_tids; // tableclient transactions + vector > table_tids; // tableclient transactions inodeno_t opened_ino; public: inodeno_t renamed_dirino; - list renamed_dir_frags; + vector renamed_dir_frags; private: // ino (pre)allocation. may involve both inotable AND session state. @@ -310,7 +321,7 @@ private: entity_name_t client_name; // session // inodes i've truncated - list truncate_start; // start truncate + vector truncate_start; // start truncate map truncate_finish; // finished truncate (started in segment blah) public: @@ -318,8 +329,8 @@ public: private: // idempotent op(s) - list > client_reqs; - list > client_flushes; + vector > client_reqs; + vector > client_flushes; public: void encode(bufferlist& bl, uint64_t features) const; @@ -342,14 +353,13 @@ private: inotablev(0), sessionmapv(0), allocated_ino(0), last_subtree_map(0), event_seq(0) {} + EMetaBlob(const EMetaBlob&) = delete; ~EMetaBlob() { } + EMetaBlob& operator=(const EMetaBlob&) = delete; void print(ostream& out) { - for (list::iterator p = lump_order.begin(); - p != lump_order.end(); - ++p) { - lump_map[*p].print(*p, out); - } + for (const auto &p : lump_order) + lump_map[p].print(p, out); } void add_client_req(metareqid_t r, uint64_t tid=0) { @@ -400,10 +410,8 @@ private: void add_null_dentry(dirlump& lump, CDentry *dn, bool dirty) { // add the dir lump.nnull++; - lump.add_dnull(nullbit(dn->get_name(), - dn->first, dn->last, - dn->get_projected_version(), - dirty)); + lump.add_dnull(dn->get_name(), dn->first, dn->last, + dn->get_projected_version(), dirty); } void add_remote_dentry(CDentry *dn, bool dirty) { @@ -419,11 +427,8 @@ private: rdt = dn->get_projected_linkage()->get_remote_d_type(); } lump.nremote++; - lump.add_dremote(remotebit(dn->get_name(), - dn->first, dn->last, - dn->get_projected_version(), - rino, rdt, - dirty)); + lump.add_dremote(dn->get_name(), dn->first, dn->last, + dn->get_projected_version(), rino, rdt, dirty); } // return remote pointer to to-be-journaled inode @@ -455,14 +460,9 @@ private: sr->encode(snapbl); lump.nfull++; - lump.add_dfull(std::shared_ptr(new fullbit(dn->get_name(), - dn->first, dn->last, - dn->get_projected_version(), - *pi, in->dirfragtree, - *in->get_projected_xattrs(), - in->symlink, - in->oldest_snap, snapbl, - state, &in->old_inodes))); + lump.add_dfull(dn->get_name(), dn->first, dn->last, dn->get_projected_version(), + *pi, in->dirfragtree, *in->get_projected_xattrs(), in->symlink, + in->oldest_snap, snapbl, state, &in->old_inodes); } // convenience: primary or remote? figure it out. @@ -506,19 +506,17 @@ private: if (sr) sr->encode(snapbl); - for (list >::iterator p = roots.begin(); p != roots.end(); ++p) { - if ((*p)->inode.ino == in->ino()) { + for (auto p = roots.begin(); p != roots.end(); ++p) { + if (p->inode.ino == in->ino()) { roots.erase(p); break; } } string empty; - roots.push_back(std::shared_ptr(new fullbit(empty, in->first, in->last, 0, pi, - pdft, px, in->symlink, - in->oldest_snap, snapbl, - dirty ? fullbit::STATE_DIRTY : 0, - &in->old_inodes))); + roots.emplace_back(empty, in->first, in->last, 0, pi, pdft, px, in->symlink, + in->oldest_snap, snapbl, (dirty ? fullbit::STATE_DIRTY : 0), + &in->old_inodes); } dirlump& add_dir(CDir *dir, bool dirty, bool complete=false) { diff --git a/src/mds/journal.cc b/src/mds/journal.cc index 86054158ae281..2759bb7c11c24 100644 --- a/src/mds/journal.cc +++ b/src/mds/journal.cc @@ -731,26 +731,23 @@ void EMetaBlob::dirlump::dump(Formatter *f) const f->dump_int("nnull", nnull); f->open_array_section("full bits"); - for (list >::const_iterator - iter = dfull.begin(); iter != dfull.end(); ++iter) { + for (const auto& iter : dfull) { f->open_object_section("fullbit"); - (*iter)->dump(f); + iter.dump(f); f->close_section(); // fullbit } f->close_section(); // full bits f->open_array_section("remote bits"); - for (list::const_iterator - iter = dremote.begin(); iter != dremote.end(); ++iter) { + for (const auto& iter : dremote) { f->open_object_section("remotebit"); - (*iter).dump(f); + iter.dump(f); f->close_section(); // remotebit } f->close_section(); // remote bits f->open_array_section("null bits"); - for (list::const_iterator - iter = dnull.begin(); iter != dnull.end(); ++iter) { + for (const auto& iter : dnull) { f->open_object_section("null bit"); - (*iter).dump(f); + iter.dump(f); f->close_section(); // null bit } f->close_section(); // null bits @@ -806,7 +803,7 @@ void EMetaBlob::decode(bufferlist::const_iterator &bl) decode(rootbl, bl); if (rootbl.length()) { auto p = rootbl.cbegin(); - roots.push_back(std::shared_ptr(new fullbit(p))); + roots.emplace_back(p); } } decode(table_tids, bl); @@ -866,17 +863,13 @@ void EMetaBlob::get_inodes( dl._decode_bits(); // Record inodes of fullbits - list > const &fb_list = dl.get_dfull(); - for (list >::const_iterator - iter = fb_list.begin(); iter != fb_list.end(); ++iter) { - inodes.insert((*iter)->inode.ino); + for (const auto& iter : dl.get_dfull()) { + inodes.insert(iter.inode.ino); } // Record inodes of remotebits - list const &rb_list = dl.get_dremote(); - for (list::const_iterator - iter = rb_list.begin(); iter != rb_list.end(); ++iter) { - inodes.insert(iter->ino); + for (const auto& iter : dl.get_dremote()) { + inodes.insert(iter.ino); } } } @@ -894,23 +887,16 @@ void EMetaBlob::get_dentries(std::map > &dentri // Get all bits dl._decode_bits(); - list > const &fb_list = dl.get_dfull(); - list const &nb_list = dl.get_dnull(); - list const &rb_list = dl.get_dremote(); // For all bits, store dentry - for (list >::const_iterator - iter = fb_list.begin(); iter != fb_list.end(); ++iter) { - dentries[df].insert((*iter)->dn); - + for (const auto& iter : dl.get_dfull()) { + dentries[df].insert(iter.dn); } - for (list::const_iterator - iter = nb_list.begin(); iter != nb_list.end(); ++iter) { - dentries[df].insert(iter->dn); + for (const auto& iter : dl.get_dremote()) { + dentries[df].insert(iter.dn); } - for (list::const_iterator - iter = rb_list.begin(); iter != rb_list.end(); ++iter) { - dentries[df].insert(iter->dn); + for (const auto& iter : dl.get_dnull()) { + dentries[df].insert(iter.dn); } } } @@ -950,26 +936,19 @@ void EMetaBlob::get_paths( dirlump const &dl = i->second; dl._decode_bits(); - list > const &fb_list = dl.get_dfull(); - list const &nb_list = dl.get_dnull(); - list const &rb_list = dl.get_dremote(); - - for (list >::const_iterator - iter = fb_list.begin(); iter != fb_list.end(); ++iter) { - std::string_view dentry = (*iter)->dn; + for (const auto& iter : dl.get_dfull()) { + std::string_view dentry = iter.dn; children[dir_ino].emplace_back(dentry); - ino_locations[(*iter)->inode.ino] = Location(dir_ino, dentry); + ino_locations[iter.inode.ino] = Location(dir_ino, dentry); } - for (list::const_iterator - iter = nb_list.begin(); iter != nb_list.end(); ++iter) { - std::string_view dentry = iter->dn; + for (const auto& iter : dl.get_dremote()) { + std::string_view dentry = iter.dn; children[dir_ino].emplace_back(dentry); } - for (list::const_iterator - iter = rb_list.begin(); iter != rb_list.end(); ++iter) { - std::string_view dentry = iter->dn; + for (const auto& iter : dl.get_dnull()) { + std::string_view dentry = iter.dn; children[dir_ino].emplace_back(dentry); } } @@ -984,26 +963,20 @@ void EMetaBlob::get_paths( dirlump const &dl = i->second; dl._decode_bits(); - list > const &fb_list = dl.get_dfull(); - for (list >::const_iterator - iter = fb_list.begin(); iter != fb_list.end(); ++iter) { - std::string_view dentry = (*iter)->dn; - if (children.find((*iter)->inode.ino) == children.end()) { + for (const auto& iter : dl.get_dfull()) { + std::string_view dentry = iter.dn; + if (children.find(iter.inode.ino) == children.end()) { leaf_locations.push_back(Location(dir_ino, dentry)); } } - list const &nb_list = dl.get_dnull(); - for (list::const_iterator - iter = nb_list.begin(); iter != nb_list.end(); ++iter) { - std::string_view dentry = iter->dn; + for (const auto& iter : dl.get_dremote()) { + std::string_view dentry = iter.dn; leaf_locations.push_back(Location(dir_ino, dentry)); } - list const &rb_list = dl.get_dremote(); - for (list::const_iterator - iter = rb_list.begin(); iter != rb_list.end(); ++iter) { - std::string_view dentry = iter->dn; + for (const auto& iter : dl.get_dnull()) { + std::string_view dentry = iter.dn; leaf_locations.push_back(Location(dir_ino, dentry)); } } @@ -1032,34 +1005,31 @@ void EMetaBlob::get_paths( void EMetaBlob::dump(Formatter *f) const { f->open_array_section("lumps"); - for (list::const_iterator i = lump_order.begin(); - i != lump_order.end(); ++i) { + for (const auto& d : lump_order) { f->open_object_section("lump"); f->open_object_section("dirfrag"); - f->dump_stream("dirfrag") << *i; + f->dump_stream("dirfrag") << d; f->close_section(); // dirfrag f->open_object_section("dirlump"); - lump_map.at(*i).dump(f); + lump_map.at(d).dump(f); f->close_section(); // dirlump f->close_section(); // lump } f->close_section(); // lumps f->open_array_section("roots"); - for (list >::const_iterator i = roots.begin(); - i != roots.end(); ++i) { + for (const auto& iter : roots) { f->open_object_section("root"); - (*i)->dump(f); + iter.dump(f); f->close_section(); // root } f->close_section(); // roots f->open_array_section("tableclient tranactions"); - for (list >::const_iterator i = table_tids.begin(); - i != table_tids.end(); ++i) { + for (const auto& p : table_tids) { f->open_object_section("transaction"); - f->dump_int("tid", i->first); - f->dump_int("version", i->second); + f->dump_int("tid", p.first); + f->dump_int("version", p.second); f->close_section(); // transaction } f->close_section(); // tableclient transactions @@ -1067,9 +1037,8 @@ void EMetaBlob::dump(Formatter *f) const f->dump_int("renamed directory inodeno", renamed_dirino); f->open_array_section("renamed directory fragments"); - for (list::const_iterator i = renamed_dir_frags.begin(); - i != renamed_dir_frags.end(); ++i) { - f->dump_int("frag", *i); + for (const auto& p : renamed_dir_frags) { + f->dump_int("frag", p); } f->close_section(); // renamed directory fragments @@ -1085,17 +1054,15 @@ void EMetaBlob::dump(Formatter *f) const f->close_section(); // client name f->open_array_section("inodes starting a truncate"); - for(list::const_iterator i = truncate_start.begin(); - i != truncate_start.end(); ++i) { - f->dump_int("inodeno", *i); + for(const auto& ino : truncate_start) { + f->dump_int("inodeno", ino); } f->close_section(); // truncate inodes f->open_array_section("inodes finishing a truncated"); - for(map::const_iterator i = truncate_finish.begin(); - i != truncate_finish.end(); ++i) { + for(const auto& p : truncate_finish) { f->open_object_section("inode+segment"); - f->dump_int("inodeno", i->first); - f->dump_int("truncate starting segment", i->second); + f->dump_int("inodeno", p.first); + f->dump_int("truncate starting segment", p.second); f->close_section(); // truncated inode } f->close_section(); // truncate finish inodes @@ -1108,11 +1075,10 @@ void EMetaBlob::dump(Formatter *f) const f->close_section(); // destroyed inodes f->open_array_section("client requests"); - for(list >::const_iterator i = client_reqs.begin(); - i != client_reqs.end(); ++i) { + for(const auto& p : client_reqs) { f->open_object_section("Client request"); - f->dump_stream("request ID") << i->first; - f->dump_int("oldest request on client", i->second); + f->dump_stream("request ID") << p.first; + f->dump_int("oldest request on client", p.second); f->close_section(); // request } f->close_section(); // client requests @@ -1131,16 +1097,16 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDSlaveUpdate *slaveup) ceph_assert(g_conf()->mds_kill_journal_replay_at != 1); - for (list >::iterator p = roots.begin(); p != roots.end(); ++p) { - CInode *in = mds->mdcache->get_inode((*p)->inode.ino); + for (auto& p : roots) { + CInode *in = mds->mdcache->get_inode(p.inode.ino); bool isnew = in ? false:true; if (!in) - in = new CInode(mds->mdcache, false); - (*p)->update_inode(mds, in); + in = new CInode(mds->mdcache, false, 2, CEPH_NOSNAP); + p.update_inode(mds, in); if (isnew) mds->mdcache->add_inode(in); - if ((*p)->is_dirty()) in->_mark_dirty(logseg); + if (p.is_dirty()) in->_mark_dirty(logseg); dout(10) << "EMetaBlob.replay " << (isnew ? " added root ":" updated root ") << *in << dendl; } @@ -1154,10 +1120,10 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDSlaveUpdate *slaveup) dout(10) << "EMetaBlob.replay don't have renamed ino " << renamed_dirino << dendl; int nnull = 0; - for (list::iterator lp = lump_order.begin(); lp != lump_order.end(); ++lp) { - dirlump &lump = lump_map[*lp]; + for (const auto& lp : lump_order) { + dirlump &lump = lump_map[lp]; if (lump.nnull) { - dout(10) << "EMetaBlob.replay found null dentry in dir " << *lp << dendl; + dout(10) << "EMetaBlob.replay found null dentry in dir " << lp << dendl; nnull += lump.nnull; } } @@ -1169,25 +1135,23 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDSlaveUpdate *slaveup) set linked; // walk through my dirs (in order!) - for (list::iterator lp = lump_order.begin(); - lp != lump_order.end(); - ++lp) { - dout(10) << "EMetaBlob.replay dir " << *lp << dendl; - dirlump &lump = lump_map[*lp]; + for (const auto& lp : lump_order) { + dout(10) << "EMetaBlob.replay dir " << lp << dendl; + dirlump &lump = lump_map[lp]; // the dir - CDir *dir = mds->mdcache->get_force_dirfrag(*lp, true); + CDir *dir = mds->mdcache->get_force_dirfrag(lp, true); if (!dir) { // hmm. do i have the inode? - CInode *diri = mds->mdcache->get_inode((*lp).ino); + CInode *diri = mds->mdcache->get_inode((lp).ino); if (!diri) { - if (MDS_INO_IS_MDSDIR(lp->ino)) { - ceph_assert(MDS_INO_MDSDIR(mds->get_nodeid()) != lp->ino); - diri = mds->mdcache->create_system_inode(lp->ino, S_IFDIR|0755); + if (MDS_INO_IS_MDSDIR(lp.ino)) { + ceph_assert(MDS_INO_MDSDIR(mds->get_nodeid()) != lp.ino); + diri = mds->mdcache->create_system_inode(lp.ino, S_IFDIR|0755); diri->state_clear(CInode::STATE_AUTH); dout(10) << "EMetaBlob.replay created base " << *diri << dendl; } else { - dout(0) << "EMetaBlob.replay missing dir ino " << (*lp).ino << dendl; + dout(0) << "EMetaBlob.replay missing dir ino " << lp.ino << dendl; mds->clog->error() << "failure replaying journal (EMetaBlob)"; mds->damaged(); ceph_abort(); // Should be unreachable because damaged() calls respawn() @@ -1195,9 +1159,9 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDSlaveUpdate *slaveup) } // create the dirfrag - dir = diri->get_or_open_dirfrag(mds->mdcache, (*lp).frag); + dir = diri->get_or_open_dirfrag(mds->mdcache, lp.frag); - if (MDS_INO_IS_BASE(lp->ino)) + if (MDS_INO_IS_BASE(lp.ino)) mds->mdcache->adjust_subtree_auth(dir, CDIR_AUTH_UNDEF); dout(10) << "EMetaBlob.replay added dir " << *dir << dendl; @@ -1244,37 +1208,34 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDSlaveUpdate *slaveup) lump._decode_bits(); // full dentry+inode pairs - for (list >::const_iterator pp = lump.get_dfull().begin(); - pp != lump.get_dfull().end(); - ++pp) { - std::shared_ptr p = *pp; - CDentry *dn = dir->lookup_exact_snap(p->dn, p->dnlast); + for (auto& fb : lump._get_dfull()) { + CDentry *dn = dir->lookup_exact_snap(fb.dn, fb.dnlast); if (!dn) { - dn = dir->add_null_dentry(p->dn, p->dnfirst, p->dnlast); - dn->set_version(p->dnv); - if (p->is_dirty()) dn->_mark_dirty(logseg); + dn = dir->add_null_dentry(fb.dn, fb.dnfirst, fb.dnlast); + dn->set_version(fb.dnv); + if (fb.is_dirty()) dn->_mark_dirty(logseg); dout(10) << "EMetaBlob.replay added (full) " << *dn << dendl; } else { - dn->set_version(p->dnv); - if (p->is_dirty()) dn->_mark_dirty(logseg); - dout(10) << "EMetaBlob.replay for [" << p->dnfirst << "," << p->dnlast << "] had " << *dn << dendl; - dn->first = p->dnfirst; - ceph_assert(dn->last == p->dnlast); + dn->set_version(fb.dnv); + if (fb.is_dirty()) dn->_mark_dirty(logseg); + dout(10) << "EMetaBlob.replay for [" << fb.dnfirst << "," << fb.dnlast << "] had " << *dn << dendl; + dn->first = fb.dnfirst; + ceph_assert(dn->last == fb.dnlast); } if (lump.is_importing()) dn->state_set(CDentry::STATE_AUTH); - CInode *in = mds->mdcache->get_inode(p->inode.ino, p->dnlast); + CInode *in = mds->mdcache->get_inode(fb.inode.ino, fb.dnlast); if (!in) { - in = new CInode(mds->mdcache, dn->is_auth(), p->dnfirst, p->dnlast); - p->update_inode(mds, in); + in = new CInode(mds->mdcache, dn->is_auth(), fb.dnfirst, fb.dnlast); + fb.update_inode(mds, in); mds->mdcache->add_inode(in); if (!dn->get_linkage()->is_null()) { if (dn->get_linkage()->is_primary()) { unlinked[dn->get_linkage()->get_inode()] = dir; stringstream ss; ss << "EMetaBlob.replay FIXME had dentry linked to wrong inode " << *dn - << " " << *dn->get_linkage()->get_inode() << " should be " << p->inode.ino; + << " " << *dn->get_linkage()->get_inode() << " should be " << fb.inode.ino; dout(0) << ss.str() << dendl; mds->clog->warn(ss); } @@ -1285,8 +1246,8 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDSlaveUpdate *slaveup) dir->link_primary_inode(dn, in); dout(10) << "EMetaBlob.replay added " << *in << dendl; } else { - in->first = p->dnfirst; - p->update_inode(mds, in); + in->first = fb.dnfirst; + fb.update_inode(mds, in); if (dn->get_linkage()->get_inode() != in && in->get_parent_dn()) { dout(10) << "EMetaBlob.replay unlinking " << *in << dendl; unlinked[in] = in->get_parent_dir(); @@ -1298,7 +1259,7 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDSlaveUpdate *slaveup) unlinked[dn->get_linkage()->get_inode()] = dir; stringstream ss; ss << "EMetaBlob.replay FIXME had dentry linked to wrong inode " << *dn - << " " << *dn->get_linkage()->get_inode() << " should be " << p->inode.ino; + << " " << *dn->get_linkage()->get_inode() << " should be " << fb.inode.ino; dout(0) << ss.str() << dendl; mds->clog->warn(ss); } @@ -1309,16 +1270,16 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDSlaveUpdate *slaveup) dir->link_primary_inode(dn, in); dout(10) << "EMetaBlob.replay linked " << *in << dendl; } else { - dout(10) << "EMetaBlob.replay for [" << p->dnfirst << "," << p->dnlast << "] had " << *in << dendl; + dout(10) << "EMetaBlob.replay for [" << fb.dnfirst << "," << fb.dnlast << "] had " << *in << dendl; } - ceph_assert(in->first == p->dnfirst || - (in->is_multiversion() && in->first > p->dnfirst)); + ceph_assert(in->first == fb.dnfirst || + (in->is_multiversion() && in->first > fb.dnfirst)); } - if (p->is_dirty()) + if (fb.is_dirty()) in->_mark_dirty(logseg); - if (p->is_dirty_parent()) - in->mark_dirty_parent(logseg, p->is_dirty_pool()); - if (p->need_snapflush()) + if (fb.is_dirty_parent()) + in->mark_dirty_parent(logseg, fb.is_dirty_pool()); + if (fb.need_snapflush()) logseg->open_files.push_back(&in->item_open_file); if (dn->is_auth()) in->state_set(CInode::STATE_AUTH); @@ -1328,14 +1289,12 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDSlaveUpdate *slaveup) } // remote dentries - for (list::const_iterator p = lump.get_dremote().begin(); - p != lump.get_dremote().end(); - ++p) { - CDentry *dn = dir->lookup_exact_snap(p->dn, p->dnlast); + for (const auto& rb : lump.get_dremote()) { + CDentry *dn = dir->lookup_exact_snap(rb.dn, rb.dnlast); if (!dn) { - dn = dir->add_remote_dentry(p->dn, p->ino, p->d_type, p->dnfirst, p->dnlast); - dn->set_version(p->dnv); - if (p->dirty) dn->_mark_dirty(logseg); + dn = dir->add_remote_dentry(rb.dn, rb.ino, rb.d_type, rb.dnfirst, rb.dnlast); + dn->set_version(rb.dnv); + if (rb.dirty) dn->_mark_dirty(logseg); dout(10) << "EMetaBlob.replay added " << *dn << dendl; } else { if (!dn->get_linkage()->is_null()) { @@ -1344,34 +1303,32 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDSlaveUpdate *slaveup) unlinked[dn->get_linkage()->get_inode()] = dir; stringstream ss; ss << "EMetaBlob.replay FIXME had dentry linked to wrong inode " << *dn - << " " << *dn->get_linkage()->get_inode() << " should be remote " << p->ino; + << " " << *dn->get_linkage()->get_inode() << " should be remote " << rb.ino; dout(0) << ss.str() << dendl; } dir->unlink_inode(dn, false); } - dir->link_remote_inode(dn, p->ino, p->d_type); - dn->set_version(p->dnv); - if (p->dirty) dn->_mark_dirty(logseg); - dout(10) << "EMetaBlob.replay for [" << p->dnfirst << "," << p->dnlast << "] had " << *dn << dendl; - dn->first = p->dnfirst; - ceph_assert(dn->last == p->dnlast); + dir->link_remote_inode(dn, rb.ino, rb.d_type); + dn->set_version(rb.dnv); + if (rb.dirty) dn->_mark_dirty(logseg); + dout(10) << "EMetaBlob.replay for [" << rb.dnfirst << "," << rb.dnlast << "] had " << *dn << dendl; + dn->first = rb.dnfirst; + ceph_assert(dn->last == rb.dnlast); } if (lump.is_importing()) dn->state_set(CDentry::STATE_AUTH); } // null dentries - for (list::const_iterator p = lump.get_dnull().begin(); - p != lump.get_dnull().end(); - ++p) { - CDentry *dn = dir->lookup_exact_snap(p->dn, p->dnlast); + for (const auto& nb : lump.get_dnull()) { + CDentry *dn = dir->lookup_exact_snap(nb.dn, nb.dnlast); if (!dn) { - dn = dir->add_null_dentry(p->dn, p->dnfirst, p->dnlast); - dn->set_version(p->dnv); - if (p->dirty) dn->_mark_dirty(logseg); + dn = dir->add_null_dentry(nb.dn, nb.dnfirst, nb.dnlast); + dn->set_version(nb.dnv); + if (nb.dirty) dn->_mark_dirty(logseg); dout(10) << "EMetaBlob.replay added (nullbit) " << *dn << dendl; } else { - dn->first = p->dnfirst; + dn->first = nb.dnfirst; if (!dn->get_linkage()->is_null()) { dout(10) << "EMetaBlob.replay unlinking " << *dn << dendl; CInode *in = dn->get_linkage()->get_inode(); @@ -1384,10 +1341,10 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDSlaveUpdate *slaveup) dir->unlink_inode(dn); } } - dn->set_version(p->dnv); - if (p->dirty) dn->_mark_dirty(logseg); + dn->set_version(nb.dnv); + if (nb.dirty) dn->_mark_dirty(logseg); dout(10) << "EMetaBlob.replay had " << *dn << dendl; - ceph_assert(dn->last == p->dnlast); + ceph_assert(dn->last == nb.dnlast); } olddir = dir; if (lump.is_importing()) @@ -1442,15 +1399,15 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDSlaveUpdate *slaveup) // if we are the srci importer, we'll also have some dirfrags we have to open up... if (renamed_diri->authority() != CDIR_AUTH_UNDEF) { - for (list::iterator p = renamed_dir_frags.begin(); p != renamed_dir_frags.end(); ++p) { - CDir *dir = renamed_diri->get_dirfrag(*p); + for (const auto& p : renamed_dir_frags) { + CDir *dir = renamed_diri->get_dirfrag(p); if (dir) { // we already had the inode before, and we already adjusted this subtree accordingly. dout(10) << " already had+adjusted rename import bound " << *dir << dendl; ceph_assert(olddir); continue; } - dir = renamed_diri->get_or_open_dirfrag(mds->mdcache, *p); + dir = renamed_diri->get_or_open_dirfrag(mds->mdcache, p); dout(10) << " creating new rename import bound " << *dir << dendl; dir->state_clear(CDir::STATE_AUTH); mds->mdcache->adjust_subtree_auth(dir, CDIR_AUTH_UNDEF); @@ -1483,14 +1440,12 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDSlaveUpdate *slaveup) } // table client transactions - for (list >::iterator p = table_tids.begin(); - p != table_tids.end(); - ++p) { - dout(10) << "EMetaBlob.replay noting " << get_mdstable_name(p->first) - << " transaction " << p->second << dendl; - MDSTableClient *client = mds->get_table_client(p->first); + for (const auto& p : table_tids) { + dout(10) << "EMetaBlob.replay noting " << get_mdstable_name(p.first) + << " transaction " << p.second << dendl; + MDSTableClient *client = mds->get_table_client(p.first); if (client) - client->got_journaled_agree(p->second, logseg); + client->got_journaled_agree(p.second, logseg); } // opened ino? @@ -1576,19 +1531,15 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDSlaveUpdate *slaveup) } // truncating inodes - for (list::iterator p = truncate_start.begin(); - p != truncate_start.end(); - ++p) { - CInode *in = mds->mdcache->get_inode(*p); + for (const auto& ino : truncate_start) { + CInode *in = mds->mdcache->get_inode(ino); ceph_assert(in); mds->mdcache->add_recovered_truncate(in, logseg); } - for (map::iterator p = truncate_finish.begin(); - p != truncate_finish.end(); - ++p) { - LogSegment *ls = mds->mdlog->get_segment(p->second); + for (const auto& p : truncate_finish) { + LogSegment *ls = mds->mdlog->get_segment(p.second); if (ls) { - CInode *in = mds->mdcache->get_inode(p->first); + CInode *in = mds->mdcache->get_inode(p.first); ceph_assert(in); mds->mdcache->remove_recovered_truncate(in, ls); } @@ -1616,35 +1567,31 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDSlaveUpdate *slaveup) } // client requests - for (list >::iterator p = client_reqs.begin(); - p != client_reqs.end(); - ++p) { - if (p->first.name.is_client()) { - dout(10) << "EMetaBlob.replay request " << p->first << " trim_to " << p->second << dendl; + for (const auto& p : client_reqs) { + if (p.first.name.is_client()) { + dout(10) << "EMetaBlob.replay request " << p.first << " trim_to " << p.second << dendl; inodeno_t created = allocated_ino ? allocated_ino : used_preallocated_ino; // if we allocated an inode, there should be exactly one client request id. ceph_assert(created == inodeno_t() || client_reqs.size() == 1); - Session *session = mds->sessionmap.get_session(p->first.name); + Session *session = mds->sessionmap.get_session(p.first.name); if (session) { - session->add_completed_request(p->first.tid, created); - if (p->second) - session->trim_completed_requests(p->second); + session->add_completed_request(p.first.tid, created); + if (p.second) + session->trim_completed_requests(p.second); } } } // client flushes - for (list >::iterator p = client_flushes.begin(); - p != client_flushes.end(); - ++p) { - if (p->first.name.is_client()) { - dout(10) << "EMetaBlob.replay flush " << p->first << " trim_to " << p->second << dendl; - Session *session = mds->sessionmap.get_session(p->first.name); + for (const auto& p : client_flushes) { + if (p.first.name.is_client()) { + dout(10) << "EMetaBlob.replay flush " << p.first << " trim_to " << p.second << dendl; + Session *session = mds->sessionmap.get_session(p.first.name); if (session) { - session->add_completed_flush(p->first.tid); - if (p->second) - session->trim_completed_flushes(p->second); + session->add_completed_flush(p.first.tid); + if (p.second) + session->trim_completed_flushes(p.second); } } } @@ -3172,20 +3119,20 @@ bool EMetaBlob::rewrite_truncate_finish(MDSRank const *mds, { bool modified = false; map new_trunc_finish; - for (std::map::iterator i = truncate_finish.begin(); - i != truncate_finish.end(); ++i) { - if (old_to_new.count(i->second)) { + for (const auto& p : truncate_finish) { + auto q = old_to_new.find(p.second); + if (q != old_to_new.end()) { dout(20) << __func__ << " applying segment seq mapping " - << i->second << " -> " << old_to_new.find(i->second)->second << dendl; - new_trunc_finish[i->first] = old_to_new.find(i->second)->second; + << p.second << " -> " << q->second << dendl; + new_trunc_finish.emplace(p.first, q->second); modified = true; } else { dout(20) << __func__ << " no segment seq mapping found for " - << i->second << dendl; - new_trunc_finish[i->first] = i->second; + << p.second << dendl; + new_trunc_finish.insert(p); } } - truncate_finish = new_trunc_finish; + truncate_finish.swap(new_trunc_finish); return modified; } diff --git a/src/tools/ceph-dencoder/types.h b/src/tools/ceph-dencoder/types.h index 255c23f00d6d9..34f9da6d17687 100644 --- a/src/tools/ceph-dencoder/types.h +++ b/src/tools/ceph-dencoder/types.h @@ -261,57 +261,57 @@ TYPE(InoTable) TYPE_STRAYDATA(SnapServer) #include "mds/events/ECommitted.h" -TYPE_FEATUREFUL(ECommitted) +TYPE_FEATUREFUL_NOCOPY(ECommitted) #include "mds/events/EExport.h" -TYPE_FEATUREFUL(EExport) +TYPE_FEATUREFUL_NOCOPY(EExport) #include "mds/events/EFragment.h" -TYPE_FEATUREFUL(EFragment) +TYPE_FEATUREFUL_NOCOPY(EFragment) #include "mds/events/EImportFinish.h" -TYPE_FEATUREFUL(EImportFinish) +TYPE_FEATUREFUL_NOCOPY(EImportFinish) #include "mds/events/EImportStart.h" -TYPE_FEATUREFUL(EImportStart) +TYPE_FEATUREFUL_NOCOPY(EImportStart) #include "mds/events/EMetaBlob.h" TYPE_FEATUREFUL_NOCOPY(EMetaBlob::fullbit) TYPE(EMetaBlob::remotebit) TYPE(EMetaBlob::nullbit) -TYPE_FEATUREFUL(EMetaBlob::dirlump) -TYPE_FEATUREFUL(EMetaBlob) +TYPE_FEATUREFUL_NOCOPY(EMetaBlob::dirlump) +TYPE_FEATUREFUL_NOCOPY(EMetaBlob) #include "mds/events/EOpen.h" -TYPE_FEATUREFUL(EOpen) +TYPE_FEATUREFUL_NOCOPY(EOpen) #include "mds/events/EResetJournal.h" -TYPE_FEATUREFUL(EResetJournal) +TYPE_FEATUREFUL_NOCOPY(EResetJournal) #include "mds/events/ESession.h" -TYPE_FEATUREFUL(ESession) +TYPE_FEATUREFUL_NOCOPY(ESession) #include "mds/events/ESessions.h" -TYPE_FEATUREFUL(ESessions) +TYPE_FEATUREFUL_NOCOPY(ESessions) #include "mds/events/ESlaveUpdate.h" TYPE(link_rollback) TYPE(rmdir_rollback) TYPE(rename_rollback::drec) TYPE(rename_rollback) -TYPE_FEATUREFUL(ESlaveUpdate) +TYPE_FEATUREFUL_NOCOPY(ESlaveUpdate) #include "mds/events/ESubtreeMap.h" -TYPE_FEATUREFUL(ESubtreeMap) +TYPE_FEATUREFUL_NOCOPY(ESubtreeMap) #include "mds/events/ETableClient.h" -TYPE_FEATUREFUL(ETableClient) +TYPE_FEATUREFUL_NOCOPY(ETableClient) #include "mds/events/ETableServer.h" -TYPE_FEATUREFUL(ETableServer) +TYPE_FEATUREFUL_NOCOPY(ETableServer) #include "mds/events/EUpdate.h" -TYPE_FEATUREFUL(EUpdate) +TYPE_FEATUREFUL_NOCOPY(EUpdate) #ifdef WITH_RBD #include "librbd/journal/Types.h" diff --git a/src/tools/cephfs/JournalTool.cc b/src/tools/cephfs/JournalTool.cc index 2d62846b27dee..227b334f8eb4d 100644 --- a/src/tools/cephfs/JournalTool.cc +++ b/src/tools/cephfs/JournalTool.cc @@ -658,10 +658,7 @@ int JournalTool::recover_dentries( int r = 0; // Replay fullbits (dentry+inode) - for (list::const_iterator lp = metablob.lump_order.begin(); - lp != metablob.lump_order.end(); ++lp) - { - dirfrag_t const &frag = *lp; + for (const auto& frag : metablob.lump_order) { EMetaBlob::dirlump const &lump = metablob.lump_map.find(frag)->second; lump._decode_bits(); object_t frag_oid = InodeStore::get_object_name(frag.ino, frag.frag, ""); @@ -724,12 +721,7 @@ int JournalTool::recover_dentries( std::set read_keys; // Compose list of potentially-existing dentries we would like to fetch - list > const &fb_list = - lump.get_dfull(); - for (list >::const_iterator fbi = - fb_list.begin(); fbi != fb_list.end(); ++fbi) { - EMetaBlob::fullbit const &fb = *(*fbi); - + for (const auto& fb : lump.get_dfull()) { // Get a key like "foobar_head" std::string key; dentry_key_t dn_key(fb.dnlast, fb.dn.c_str()); @@ -737,12 +729,7 @@ int JournalTool::recover_dentries( read_keys.insert(key); } - list const &rb_list = - lump.get_dremote(); - for (list::const_iterator rbi = - rb_list.begin(); rbi != rb_list.end(); ++rbi) { - EMetaBlob::remotebit const &rb = *rbi; - + for(const auto& rb : lump.get_dremote()) { // Get a key like "foobar_head" std::string key; dentry_key_t dn_key(rb.dnlast, rb.dn.c_str()); @@ -750,8 +737,7 @@ int JournalTool::recover_dentries( read_keys.insert(key); } - list const &nb_list = lump.get_dnull(); - for (auto& nb : nb_list) { + for (const auto& nb : lump.get_dnull()) { // Get a key like "foobar_head" std::string key; dentry_key_t dn_key(nb.dnlast, nb.dn.c_str()); @@ -773,10 +759,7 @@ int JournalTool::recover_dentries( // Compose list of dentries we will write back std::map write_vals; - for (list >::const_iterator fbi = - fb_list.begin(); fbi != fb_list.end(); ++fbi) { - EMetaBlob::fullbit const &fb = *(*fbi); - + for (const auto& fb : lump.get_dfull()) { // Get a key like "foobar_head" std::string key; dentry_key_t dn_key(fb.dnlast, fb.dn.c_str()); @@ -842,10 +825,7 @@ int JournalTool::recover_dentries( } } - for (list::const_iterator rbi = - rb_list.begin(); rbi != rb_list.end(); ++rbi) { - EMetaBlob::remotebit const &rb = *rbi; - + for(const auto& rb : lump.get_dremote()) { // Get a key like "foobar_head" std::string key; dentry_key_t dn_key(rb.dnlast, rb.dn.c_str()); @@ -906,7 +886,7 @@ int JournalTool::recover_dentries( } std::set null_vals; - for (auto& nb : nb_list) { + for (const auto& nb : lump.get_dnull()) { std::string key; dentry_key_t dn_key(nb.dnlast, nb.dn.c_str()); dn_key.encode(key); @@ -974,9 +954,7 @@ int JournalTool::recover_dentries( * important because clients use them to infer completeness * of directories */ - for (list >::const_iterator p = - metablob.roots.begin(); p != metablob.roots.end(); ++p) { - EMetaBlob::fullbit const &fb = *(*p); + for (const auto& fb : metablob.roots) { inodeno_t ino = fb.inode.ino; dout(4) << "updating root 0x" << std::hex << ino << std::dec << dendl; @@ -1047,12 +1025,11 @@ int JournalTool::erase_region(JournalScanner const &js, uint64_t const pos, uint // of an ENoOp, and our trailing start ptr. Calculate how much padding // is needed inside the ENoOp to make up the difference. bufferlist tmp; - ENoOp enoop(0); - PurgeItem pi; - if (type == "mdlog") { + ENoOp enoop(0); enoop.encode_with_header(tmp, CEPH_FEATURES_SUPPORTED_DEFAULT); } else if (type == "purge_queue") { + PurgeItem pi; pi.encode(tmp); } @@ -1070,9 +1047,10 @@ int JournalTool::erase_region(JournalScanner const &js, uint64_t const pos, uint bufferlist entry; if (type == "mdlog") { // Serialize an ENoOp with the correct amount of padding - enoop = ENoOp(padding); + ENoOp enoop(padding); enoop.encode_with_header(entry, CEPH_FEATURES_SUPPORTED_DEFAULT); } else if (type == "purge_queue") { + PurgeItem pi; pi.pad_size = padding; pi.encode(entry); }