From: Yan, Zheng Date: Fri, 26 Jan 2018 01:18:30 +0000 (+0800) Subject: mds: add dirfrags whose child inodes have caps to open file table X-Git-Tag: v13.1.0~436^2~5 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=242ed5bba8c160677adb2edf80d8df84bf37c1c3;p=ceph.git mds: add dirfrags whose child inodes have caps to open file table When opening a file in RO mode, if client has the corresponding inode in its cache and issued caps of the inode already include caps that RO file needs, client does not immediately tell mds what caps it wants. From mds' point of view, the inode just has client caps. Open file table should handle above scenario. But adding all inodes with caps to the open file table can be expensive. So we only add dirfrags whose child inodes have caps to the open file table. MDS will fetch these dirfrags during recovery. Hopefully, most RO opened inodes will be populated into the cache. Signed-off-by: Yan, Zheng --- diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc index 5fffead64ab7..ce8eb31cca71 100644 --- a/src/mds/CDir.cc +++ b/src/mds/CDir.cc @@ -289,6 +289,18 @@ bool CDir::check_rstats(bool scrub) return good; } +void CDir::adjust_num_inodes_with_caps(int d) +{ + // FIXME: smarter way to decide if adding 'this' to open file table + if (num_inodes_with_caps == 0 && d > 0) + cache->open_file_table.add_dirfrag(this); + else if (num_inodes_with_caps > 0 && num_inodes_with_caps == -d) + cache->open_file_table.remove_dirfrag(this); + + num_inodes_with_caps += d; + assert(num_inodes_with_caps >= 0); +} + CDentry *CDir::lookup(std::string_view name, snapid_t snap) { dout(20) << "lookup (" << snap << ", '" << name << "')" << dendl; @@ -569,6 +581,8 @@ void CDir::link_inode_work( CDentry *dn, CInode *in) if (in->state_test(CInode::STATE_TRACKEDBYOFT)) inode->mdcache->open_file_table.notify_link(in); + if (in->is_any_caps()) + adjust_num_inodes_with_caps(1); // adjust auth pin count if (in->auth_pins + in->nested_auth_pins) @@ -648,6 +662,8 @@ void CDir::unlink_inode_work( CDentry *dn ) if (in->state_test(CInode::STATE_TRACKEDBYOFT)) inode->mdcache->open_file_table.notify_unlink(in); + if (in->is_any_caps()) + adjust_num_inodes_with_caps(-1); // unlink auth_pin count if (in->auth_pins + in->nested_auth_pins) @@ -840,6 +856,9 @@ void CDir::steal_dentry(CDentry *dn) if (pi->accounted_rstat.rctime > fnode.rstat.rctime) fnode.rstat.rctime = pi->accounted_rstat.rctime; + if (in->is_any_caps()) + adjust_num_inodes_with_caps(1); + // move dirty inode rstat to new dirfrag if (in->is_dirty_rstat()) dirty_rstat_inodes.push_back(&in->dirty_rstat_item); @@ -921,6 +940,7 @@ void CDir::finish_old_fragment(list& waiters, bool repl num_head_items = num_head_null = 0; num_snap_items = num_snap_null = 0; + adjust_num_inodes_with_caps(-num_inodes_with_caps); // this mirrors init_fragment_pins() if (is_auth()) diff --git a/src/mds/CDir.h b/src/mds/CDir.h index a81053f6951d..e8877a813511 100644 --- a/src/mds/CDir.h +++ b/src/mds/CDir.h @@ -91,7 +91,8 @@ public: static const unsigned STATE_ASSIMRSTAT = (1<<15); // assimilating inode->frag rstats static const unsigned STATE_DIRTYDFT = (1<<16); // dirty dirfragtree static const unsigned STATE_BADFRAG = (1<<17); // bad dirfrag - static const unsigned STATE_AUXSUBTREE = (1<<18); // no subtree merge + static const unsigned STATE_TRACKEDBYOFT = (1<<18); // tracked by open file table + static const unsigned STATE_AUXSUBTREE = (1<<19); // no subtree merge // common states static const unsigned STATE_CLEAN = 0; @@ -106,14 +107,16 @@ public: STATE_IMPORTBOUND | STATE_EXPORTBOUND | STATE_FROZENTREE | - STATE_STICKY); + STATE_STICKY | + STATE_TRACKEDBYOFT); static const unsigned MASK_STATE_EXPORT_KEPT = (STATE_EXPORTING | STATE_IMPORTBOUND | STATE_EXPORTBOUND | STATE_FROZENTREE | STATE_FROZENDIR | - STATE_STICKY); + STATE_STICKY | + STATE_TRACKEDBYOFT); static const unsigned MASK_STATE_FRAGMENT_KEPT = (STATE_DIRTY | STATE_EXPORTBOUND | @@ -337,6 +340,8 @@ protected: int num_dirty; + int num_inodes_with_caps = 0; + // state version_t committing_version; version_t committed_version; @@ -429,6 +434,8 @@ protected: return num_dirty; } + void adjust_num_inodes_with_caps(int d); + int64_t get_frag_size() const { return get_projected_fnode()->fragstat.size(); } diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc index 017e2863e73a..27a460930847 100644 --- a/src/mds/CInode.cc +++ b/src/mds/CInode.cc @@ -2892,10 +2892,11 @@ Capability *CInode::add_client_cap(client_t client, Session *session, SnapRealm containing_realm = find_snaprealm(); containing_realm->inodes_with_caps.push_back(&item_caps); dout(10) << __func__ << " first cap, joining realm " << *containing_realm << dendl; - } - if (client_caps.empty()) mdcache->num_inodes_with_caps++; + if (parent) + parent->dir->adjust_num_inodes_with_caps(1); + } Capability *cap = new Capability(this, ++mdcache->last_cap_id, client); assert(client_caps.count(client) == 0); @@ -2936,6 +2937,8 @@ void CInode::remove_client_cap(client_t client) item_caps.remove_myself(); containing_realm = NULL; mdcache->num_inodes_with_caps--; + if (parent) + parent->dir->adjust_num_inodes_with_caps(-1); } //clean up advisory locks diff --git a/src/mds/OpenFileTable.cc b/src/mds/OpenFileTable.cc index 7ae14c41c928..10c6b2c566cc 100644 --- a/src/mds/OpenFileTable.cc +++ b/src/mds/OpenFileTable.cc @@ -13,6 +13,7 @@ */ #include "mds/CInode.h" +#include "mds/CDir.h" #include "mds/MDSRank.h" #include "mds/MDCache.h" #include "osdc/Objecter.h" @@ -110,6 +111,29 @@ void OpenFileTable::remove_inode(CInode *in) put_ref(in); } +void OpenFileTable::add_dirfrag(CDir *dir) +{ + dout(10) << __func__ << " " << *dir << dendl; + assert(!dir->state_test(CDir::STATE_TRACKEDBYOFT)); + dir->state_set(CDir::STATE_TRACKEDBYOFT); + auto ret = dirfrags.insert(dir->dirfrag()); + assert(ret.second); + get_ref(dir->get_inode()); + dirty_items.emplace(dir->ino(), 0); +} + +void OpenFileTable::remove_dirfrag(CDir *dir) +{ + dout(10) << __func__ << " " << *dir << dendl; + assert(dir->state_test(CDir::STATE_TRACKEDBYOFT)); + dir->state_clear(CDir::STATE_TRACKEDBYOFT); + auto p = dirfrags.find(dir->dirfrag()); + assert(p != dirfrags.end()); + dirfrags.erase(p); + dirty_items.emplace(dir->ino(), 0); + put_ref(dir->get_inode()); +} + void OpenFileTable::notify_link(CInode *in) { dout(10) << __func__ << " " << *in << dendl; @@ -246,11 +270,31 @@ void OpenFileTable::commit(MDSInternalContextBase *c, uint64_t log_seq, int op_p using ceph::encode; for (auto& it : dirty_items) { + list fgls; auto p = anchor_map.find(it.first); + if (p != anchor_map.end()) { + for (auto q = dirfrags.lower_bound(dirfrag_t(it.first, 0)); + q != dirfrags.end() && q->ino == it.first; + ++q) + fgls.push_back(q->frag); + } + if (first_commit) { auto q = loaded_anchor_map.find(it.first); if (q != loaded_anchor_map.end()) { bool same = (p != anchor_map.end() && p->second == q->second); + if (same) { + auto r = loaded_dirfrags.lower_bound(dirfrag_t(it.first, 0)); + for (auto fg : fgls) { + if (r == loaded_dirfrags.end() || !(*r == dirfrag_t(it.first, fg))) { + same = false; + break; + } + ++r; + } + if (same && r != loaded_dirfrags.end() && r->ino == it.first) + same = false; + } loaded_anchor_map.erase(q); if (same) continue; @@ -264,9 +308,9 @@ void OpenFileTable::commit(MDSInternalContextBase *c, uint64_t log_seq, int op_p if (p != anchor_map.end()) { bufferlist bl; encode(p->second, bl); + encode(fgls, bl); write_size += bl.length() + sizeof(__u32); - to_update[key].swap(bl); } else { to_remove.emplace(key); @@ -290,6 +334,7 @@ void OpenFileTable::commit(MDSInternalContextBase *c, uint64_t log_seq, int op_p } } loaded_anchor_map.clear(); + loaded_dirfrags.clear(); } commit_func(true); @@ -325,8 +370,10 @@ void OpenFileTable::_load_finish(int op_r, int header_r, int values_r, if (op_r < 0) { derr << __func__ << " got " << cpp_strerror(op_r) << dendl; clear_on_commit = true; - if (!first) + if (!first) { loaded_anchor_map.clear(); + loaded_dirfrags.clear(); + } goto out; } @@ -357,11 +404,17 @@ void OpenFileTable::_load_finish(int op_r, int header_r, int values_r, decode(anchor, p); assert(ino == anchor.ino); anchor.auth = MDS_RANK_NONE; + + list fgls; + decode(fgls, p); + for (auto fg : fgls) + loaded_dirfrags.insert(loaded_dirfrags.end(), dirfrag_t(anchor.ino, fg)); } } catch (buffer::error &e) { derr << __func__ << ": corrupted header/values: " << e.what() << dendl; clear_on_commit = true; loaded_anchor_map.clear(); + loaded_dirfrags.clear(); goto out; } @@ -464,8 +517,8 @@ void OpenFileTable::_open_ino_finish(inodeno_t ino, int r) num_opening_inodes--; if (num_opening_inodes == 0) { if (prefetch_state == DIR_INODES) { - prefetch_state = FILE_INODES; - _prefetch_inodes(); + prefetch_state = DIRFRAGS; + _prefetch_dirfrags(); } else if (prefetch_state == FILE_INODES) { prefetch_state = DONE; logseg_destroyed_inos.clear(); @@ -478,6 +531,68 @@ void OpenFileTable::_open_ino_finish(inodeno_t ino, int r) } } +void OpenFileTable::_prefetch_dirfrags() +{ + dout(10) << __func__ << dendl; + assert(prefetch_state == DIRFRAGS); + + MDCache *mdcache = mds->mdcache; + list fetch_queue; + + CInode *last_in = nullptr; + for (auto df : loaded_dirfrags) { + CInode *diri; + if (last_in && last_in->ino() == df.ino) { + diri = last_in; + } else { + diri = mdcache->get_inode(df.ino); + if (!diri) + continue; + last_in = diri; + } + if (diri->state_test(CInode::STATE_REJOINUNDEF)) + continue; + + CDir *dir = diri->get_dirfrag(df.frag); + if (dir) { + if (dir->is_auth() && !dir->is_complete()) + fetch_queue.push_back(dir); + } else { + list fgls; + diri->dirfragtree.get_leaves_under(df.frag, fgls); + for (auto fg : fgls) { + if (diri->is_auth()) { + dir = diri->get_or_open_dirfrag(mdcache, fg); + } else { + dir = diri->get_dirfrag(fg); + } + if (dir && dir->is_auth() && !dir->is_complete()) + fetch_queue.push_back(dir); + } + } + } + + MDSGatherBuilder gather(g_ceph_context); + for (auto dir : fetch_queue) { + if (dir->state_test(CDir::STATE_REJOINUNDEF)) + assert(dir->get_inode()->dirfragtree.is_leaf(dir->get_frag())); + dir->fetch(gather.new_sub()); + } + + auto finish_func = [this](int r) { + prefetch_state = FILE_INODES; + _prefetch_inodes(); + }; + if (gather.has_subs()) { + gather.set_finisher( + new MDSInternalContextWrapper(mds, + new FunctionContext(finish_func))); + gather.activate(); + } else { + finish_func(0); + } +} + void OpenFileTable::_prefetch_inodes() { dout(10) << __func__ << " state " << prefetch_state << dendl; diff --git a/src/mds/OpenFileTable.h b/src/mds/OpenFileTable.h index 0a51f0fff2a4..243caefa2eac 100644 --- a/src/mds/OpenFileTable.h +++ b/src/mds/OpenFileTable.h @@ -30,6 +30,8 @@ public: void add_inode(CInode *in); void remove_inode(CInode *in); + void add_dirfrag(CDir *dir); + void remove_dirfrag(CDir *dir); void notify_link(CInode *in); void notify_unlink(CInode *in); bool is_any_dirty() const { return !dirty_items.empty(); } @@ -65,6 +67,7 @@ protected: MDSRank *mds; map anchor_map; + set dirfrags; std::map dirty_items; // ino -> dirty state static const unsigned DIRTY_NEW = 1; @@ -82,6 +85,7 @@ protected: void _commit_finish(int r, uint64_t log_seq, MDSInternalContextBase *fin); map loaded_anchor_map; + set loaded_dirfrags; list waiting_for_load; bool load_done = false; @@ -92,14 +96,16 @@ protected: enum { DIR_INODES = 1, - FILE_INODES = 2, - DONE = 3, + DIRFRAGS = 2, + FILE_INODES = 3, + DONE = 4, }; unsigned prefetch_state = 0; unsigned num_opening_inodes = 0; list waiting_for_prefetch; void _open_ino_finish(inodeno_t ino, int r); void _prefetch_inodes(); + void _prefetch_dirfrags(); std::map > logseg_destroyed_inos; std::set destroyed_inos_set;