mds: add dirfrags whose child inodes have caps to open file table

When opening a file in RO mode, if client has the corresponding inode in
its cache and issued caps of the inode already include caps that RO file
needs, client does not immediately tell mds what caps it wants. From
mds' point of view, the inode just has client caps.

Open file table should handle above scenario. But adding all inodes with
caps to the open file table can be expensive. So we only add dirfrags
whose child inodes have caps to the open file table. MDS will fetch
these dirfrags during recovery. Hopefully, most RO opened inodes will
be populated into the cache.

Signed-off-by: Yan, Zheng <zyan@redhat.com>

diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc
index 5fffead64ab7f60bbf5d9838a8f6739822313045..ce8eb31cca712b7b74373984c0b8ef4dd59333a0 100644 (file)
--- a/src/mds/CDir.cc
+++ b/src/mds/CDir.cc
@@ -289,6 +289,18 @@ bool CDir::check_rstats(bool scrub)
   return good;
 }
 
+void CDir::adjust_num_inodes_with_caps(int d)
+{
+  // FIXME: smarter way to decide if adding 'this' to open file table
+  if (num_inodes_with_caps == 0 && d > 0)
+    cache->open_file_table.add_dirfrag(this);
+  else if (num_inodes_with_caps > 0 && num_inodes_with_caps == -d)
+    cache->open_file_table.remove_dirfrag(this);
+
+  num_inodes_with_caps += d;
+  assert(num_inodes_with_caps >= 0);
+}
+
 CDentry *CDir::lookup(std::string_view name, snapid_t snap)
 { 
   dout(20) << "lookup (" << snap << ", '" << name << "')" << dendl;
@@ -569,6 +581,8 @@ void CDir::link_inode_work( CDentry *dn, CInode *in)
 
   if (in->state_test(CInode::STATE_TRACKEDBYOFT))
     inode->mdcache->open_file_table.notify_link(in);
+  if (in->is_any_caps())
+    adjust_num_inodes_with_caps(1);
   
   // adjust auth pin count
   if (in->auth_pins + in->nested_auth_pins)
@@ -648,6 +662,8 @@ void CDir::unlink_inode_work( CDentry *dn )
 
     if (in->state_test(CInode::STATE_TRACKEDBYOFT))
       inode->mdcache->open_file_table.notify_unlink(in);
+    if (in->is_any_caps())
+      adjust_num_inodes_with_caps(-1);
     
     // unlink auth_pin count
     if (in->auth_pins + in->nested_auth_pins)
@@ -840,6 +856,9 @@ void CDir::steal_dentry(CDentry *dn)
       if (pi->accounted_rstat.rctime > fnode.rstat.rctime)
        fnode.rstat.rctime = pi->accounted_rstat.rctime;
 
+      if (in->is_any_caps())
+       adjust_num_inodes_with_caps(1);
+
       // move dirty inode rstat to new dirfrag
       if (in->is_dirty_rstat())
        dirty_rstat_inodes.push_back(&in->dirty_rstat_item);
@@ -921,6 +940,7 @@ void CDir::finish_old_fragment(list<MDSInternalContextBase*>& waiters, bool repl
 
   num_head_items = num_head_null = 0;
   num_snap_items = num_snap_null = 0;
+  adjust_num_inodes_with_caps(-num_inodes_with_caps);
 
   // this mirrors init_fragment_pins()
   if (is_auth()) 

diff --git a/src/mds/CDir.h b/src/mds/CDir.h
index a81053f6951d9b03ee15646cb201b86645416f65..e8877a8135114d5e5813ec1a7f803fdda366ea30 100644 (file)
--- a/src/mds/CDir.h
+++ b/src/mds/CDir.h
@@ -91,7 +91,8 @@ public:
   static const unsigned STATE_ASSIMRSTAT =    (1<<15);  // assimilating inode->frag rstats
   static const unsigned STATE_DIRTYDFT =      (1<<16);  // dirty dirfragtree
   static const unsigned STATE_BADFRAG =       (1<<17);  // bad dirfrag
-  static const unsigned STATE_AUXSUBTREE =    (1<<18);  // no subtree merge
+  static const unsigned STATE_TRACKEDBYOFT =  (1<<18);  // tracked by open file table
+  static const unsigned STATE_AUXSUBTREE =    (1<<19);  // no subtree merge
 
   // common states
   static const unsigned STATE_CLEAN =  0;
@@ -106,14 +107,16 @@ public:
    STATE_IMPORTBOUND |
    STATE_EXPORTBOUND |
    STATE_FROZENTREE |
-   STATE_STICKY);
+   STATE_STICKY |
+   STATE_TRACKEDBYOFT);
   static const unsigned MASK_STATE_EXPORT_KEPT = 
   (STATE_EXPORTING |
    STATE_IMPORTBOUND |
    STATE_EXPORTBOUND |
    STATE_FROZENTREE |
    STATE_FROZENDIR |
-   STATE_STICKY);
+   STATE_STICKY |
+   STATE_TRACKEDBYOFT);
   static const unsigned MASK_STATE_FRAGMENT_KEPT = 
   (STATE_DIRTY |
    STATE_EXPORTBOUND |
@@ -337,6 +340,8 @@ protected:
 
   int num_dirty;
 
+  int num_inodes_with_caps = 0;
+
   // state
   version_t committing_version;
   version_t committed_version;
@@ -429,6 +434,8 @@ protected:
     return num_dirty;
   }
 
+  void adjust_num_inodes_with_caps(int d);
+
   int64_t get_frag_size() const {
     return get_projected_fnode()->fragstat.size();
   }

diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index 017e2863e73ab0f85de0e616123f0a1382b61664..27a460930847fc9a07c468cf758eee7099c5c61d 100644 (file)
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -2892,10 +2892,11 @@ Capability *CInode::add_client_cap(client_t client, Session *session, SnapRealm
       containing_realm = find_snaprealm();
     containing_realm->inodes_with_caps.push_back(&item_caps);
     dout(10) << __func__ << " first cap, joining realm " << *containing_realm << dendl;
-  }
 
-  if (client_caps.empty())
     mdcache->num_inodes_with_caps++;
+    if (parent)
+      parent->dir->adjust_num_inodes_with_caps(1);
+  }
   
   Capability *cap = new Capability(this, ++mdcache->last_cap_id, client);
   assert(client_caps.count(client) == 0);
@@ -2936,6 +2937,8 @@ void CInode::remove_client_cap(client_t client)
     item_caps.remove_myself();
     containing_realm = NULL;
     mdcache->num_inodes_with_caps--;
+    if (parent)
+      parent->dir->adjust_num_inodes_with_caps(-1);
   }
 
   //clean up advisory locks

diff --git a/src/mds/OpenFileTable.cc b/src/mds/OpenFileTable.cc
index 7ae14c41c92875aa591ca52a9f3e5378d124ac3c..10c6b2c566cca4d52e6772939a74289608a0eefb 100644 (file)
--- a/src/mds/OpenFileTable.cc
+++ b/src/mds/OpenFileTable.cc
@@ -13,6 +13,7 @@
  */
 
 #include "mds/CInode.h"
+#include "mds/CDir.h"
 #include "mds/MDSRank.h"
 #include "mds/MDCache.h"
 #include "osdc/Objecter.h"
@@ -110,6 +111,29 @@ void OpenFileTable::remove_inode(CInode *in)
   put_ref(in);
 }
 
+void OpenFileTable::add_dirfrag(CDir *dir)
+{
+  dout(10) << __func__ << " " << *dir << dendl;
+  assert(!dir->state_test(CDir::STATE_TRACKEDBYOFT));
+  dir->state_set(CDir::STATE_TRACKEDBYOFT);
+  auto ret = dirfrags.insert(dir->dirfrag());
+  assert(ret.second);
+  get_ref(dir->get_inode());
+  dirty_items.emplace(dir->ino(), 0);
+}
+
+void OpenFileTable::remove_dirfrag(CDir *dir)
+{
+  dout(10) << __func__ << " " << *dir << dendl;
+  assert(dir->state_test(CDir::STATE_TRACKEDBYOFT));
+  dir->state_clear(CDir::STATE_TRACKEDBYOFT);
+  auto p = dirfrags.find(dir->dirfrag());
+  assert(p != dirfrags.end());
+  dirfrags.erase(p);
+  dirty_items.emplace(dir->ino(), 0);
+  put_ref(dir->get_inode());
+}
+
 void OpenFileTable::notify_link(CInode *in)
 {
   dout(10) << __func__ << " " << *in << dendl;
@@ -246,11 +270,31 @@ void OpenFileTable::commit(MDSInternalContextBase *c, uint64_t log_seq, int op_p
 
   using ceph::encode;
   for (auto& it : dirty_items) {
+    list<frag_t> fgls;
     auto p = anchor_map.find(it.first);
+    if (p != anchor_map.end()) {
+      for (auto q = dirfrags.lower_bound(dirfrag_t(it.first, 0));
+          q != dirfrags.end() && q->ino == it.first;
+          ++q)
+       fgls.push_back(q->frag);
+    }
+
     if (first_commit) {
       auto q = loaded_anchor_map.find(it.first);
       if (q != loaded_anchor_map.end()) {
        bool same = (p != anchor_map.end() && p->second == q->second);
+       if (same) {
+         auto r = loaded_dirfrags.lower_bound(dirfrag_t(it.first, 0));
+         for (auto fg : fgls) {
+           if (r == loaded_dirfrags.end() || !(*r == dirfrag_t(it.first, fg))) {
+             same = false;
+             break;
+           }
+           ++r;
+         }
+         if (same && r != loaded_dirfrags.end() && r->ino == it.first)
+           same = false;
+       }
        loaded_anchor_map.erase(q);
        if (same)
          continue;
@@ -264,9 +308,9 @@ void OpenFileTable::commit(MDSInternalContextBase *c, uint64_t log_seq, int op_p
     if (p != anchor_map.end()) {
       bufferlist bl;
       encode(p->second, bl);
+      encode(fgls, bl);
 
       write_size += bl.length() + sizeof(__u32);
-
       to_update[key].swap(bl);
     } else {
       to_remove.emplace(key);
@@ -290,6 +334,7 @@ void OpenFileTable::commit(MDSInternalContextBase *c, uint64_t log_seq, int op_p
       }
     }
     loaded_anchor_map.clear();
+    loaded_dirfrags.clear();
   }
 
   commit_func(true);
@@ -325,8 +370,10 @@ void OpenFileTable::_load_finish(int op_r, int header_r, int values_r,
   if (op_r < 0) {
     derr << __func__ << " got " << cpp_strerror(op_r) << dendl;
     clear_on_commit = true;
-    if (!first)
+    if (!first) {
       loaded_anchor_map.clear();
+      loaded_dirfrags.clear();
+    }
     goto out;
   }
 
@@ -357,11 +404,17 @@ void OpenFileTable::_load_finish(int op_r, int header_r, int values_r,
       decode(anchor, p);
       assert(ino == anchor.ino);
       anchor.auth = MDS_RANK_NONE;
+
+      list<frag_t> fgls;
+      decode(fgls, p);
+      for (auto fg : fgls)
+       loaded_dirfrags.insert(loaded_dirfrags.end(), dirfrag_t(anchor.ino, fg));
     }
   } catch (buffer::error &e) {
     derr << __func__ << ": corrupted header/values: " << e.what() << dendl;
     clear_on_commit = true;
     loaded_anchor_map.clear();
+    loaded_dirfrags.clear();
     goto out;
   }
 
@@ -464,8 +517,8 @@ void OpenFileTable::_open_ino_finish(inodeno_t ino, int r)
   num_opening_inodes--;
   if (num_opening_inodes == 0) {
     if (prefetch_state == DIR_INODES)  {
-      prefetch_state = FILE_INODES;
-      _prefetch_inodes();
+      prefetch_state = DIRFRAGS;
+      _prefetch_dirfrags();
     } else if (prefetch_state == FILE_INODES) {
       prefetch_state = DONE;
       logseg_destroyed_inos.clear();
@@ -478,6 +531,68 @@ void OpenFileTable::_open_ino_finish(inodeno_t ino, int r)
   }
 }
 
+void OpenFileTable::_prefetch_dirfrags()
+{
+  dout(10) << __func__ << dendl;
+  assert(prefetch_state == DIRFRAGS);
+
+  MDCache *mdcache = mds->mdcache;
+  list<CDir*> fetch_queue;
+
+  CInode *last_in = nullptr;
+  for (auto df : loaded_dirfrags) {
+    CInode *diri;
+    if (last_in && last_in->ino() == df.ino) {
+      diri = last_in;
+    } else {
+      diri = mdcache->get_inode(df.ino);
+      if (!diri)
+       continue;
+      last_in = diri;
+    }
+    if (diri->state_test(CInode::STATE_REJOINUNDEF))
+      continue;
+
+    CDir *dir = diri->get_dirfrag(df.frag);
+    if (dir) {
+      if (dir->is_auth() && !dir->is_complete())
+       fetch_queue.push_back(dir);
+    } else {
+      list<frag_t> fgls;
+      diri->dirfragtree.get_leaves_under(df.frag, fgls);
+      for (auto fg : fgls) {
+       if (diri->is_auth()) {
+         dir = diri->get_or_open_dirfrag(mdcache, fg);
+       } else {
+         dir = diri->get_dirfrag(fg);
+       }
+       if (dir && dir->is_auth() && !dir->is_complete())
+         fetch_queue.push_back(dir);
+      }
+    }
+  }
+
+  MDSGatherBuilder gather(g_ceph_context);
+  for (auto dir : fetch_queue) {
+    if (dir->state_test(CDir::STATE_REJOINUNDEF))
+      assert(dir->get_inode()->dirfragtree.is_leaf(dir->get_frag()));
+    dir->fetch(gather.new_sub());
+  }
+
+  auto finish_func = [this](int r) {
+    prefetch_state = FILE_INODES;
+    _prefetch_inodes();
+  };
+  if (gather.has_subs()) {
+    gather.set_finisher(
+       new MDSInternalContextWrapper(mds,
+         new FunctionContext(finish_func)));
+    gather.activate();
+  } else {
+    finish_func(0);
+  }
+}
+
 void OpenFileTable::_prefetch_inodes()
 {
   dout(10) << __func__ << " state " << prefetch_state << dendl;

diff --git a/src/mds/OpenFileTable.h b/src/mds/OpenFileTable.h
index 0a51f0fff2a420bb1e3279c47389f08886b2f8ef..243caefa2eac52462f8da6504dc84fddd2e75f2f 100644 (file)
--- a/src/mds/OpenFileTable.h
+++ b/src/mds/OpenFileTable.h
@@ -30,6 +30,8 @@ public:
 
   void add_inode(CInode *in);
   void remove_inode(CInode *in);
+  void add_dirfrag(CDir *dir);
+  void remove_dirfrag(CDir *dir);
   void notify_link(CInode *in);
   void notify_unlink(CInode *in);
   bool is_any_dirty() const { return !dirty_items.empty(); }
@@ -65,6 +67,7 @@ protected:
   MDSRank *mds;
 
   map<inodeno_t, Anchor> anchor_map;
+  set<dirfrag_t> dirfrags;
 
   std::map<inodeno_t, unsigned> dirty_items; // ino -> dirty state
   static const unsigned DIRTY_NEW = 1;
@@ -82,6 +85,7 @@ protected:
   void _commit_finish(int r, uint64_t log_seq, MDSInternalContextBase *fin);
 
   map<inodeno_t, Anchor> loaded_anchor_map;
+  set<dirfrag_t> loaded_dirfrags;
   list<MDSInternalContextBase*> waiting_for_load;
   bool load_done = false;
 
@@ -92,14 +96,16 @@ protected:
 
   enum {
     DIR_INODES = 1,
-    FILE_INODES = 2,
-    DONE = 3,
+    DIRFRAGS = 2,
+    FILE_INODES = 3,
+    DONE = 4,
   };
   unsigned prefetch_state = 0;
   unsigned num_opening_inodes = 0;
   list<MDSInternalContextBase*> waiting_for_prefetch;
   void _open_ino_finish(inodeno_t ino, int r);
   void _prefetch_inodes();
+  void _prefetch_dirfrags();
 
   std::map<uint64_t, vector<inodeno_t> > logseg_destroyed_inos;
   std::set<inodeno_t> destroyed_inos_set;