mds: replicate random pin state

This is slightly evil in its current form. The MDS should use locks to
transmit state changes but right now it's just set when the CInode is
replicated. This replication of this state marker is necessary for
failover situations where we want the randomly pinned subtree to remain
pinned across failovers.

Note: this problem does not exist for the ephemeral distributed pins
because simple knowledge of the immediate parent's setting (which is
replicated normally) is sufficient to determine if the CInode is
ephemerally distributed. Ditto for regular export pins.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>

diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc
index 8cf2cb2cdf81e3644df1d69502433421f3cd2b16..ec50eeb41d0433433f0eb4970b1f21f29cbd9e3e 100644 (file)
--- a/src/mds/CDir.cc
+++ b/src/mds/CDir.cc
@@ -1857,7 +1857,7 @@ CDentry *CDir::_load_dentry(
         if (in->inode.is_dirty_rstat())
           in->mark_dirty_rstat();
 
-        in->maybe_ephemeral_rand();
+        in->maybe_ephemeral_rand(true);
         //in->hack_accessed = false;
         //in->hack_load_stamp = ceph_clock_now();
         //num_new_inodes_loaded++;

diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index 57d5649078619122d7c695eb504e315100141468..39239f8fcd9aec76dbe81bea8b66f51a741776b8 100644 (file)
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -835,8 +835,7 @@ CDir *CInode::add_dirfrag(CDir *dir)
     dir->get(CDir::PIN_STICKY);
   }
 
-  maybe_export_pin();
-  maybe_ephemeral_dist();
+  maybe_pin();
 
   return dir;
 }
@@ -4276,9 +4275,20 @@ void CInode::decode_import(bufferlist::const_iterator& p,
 
   _decode_base(p);
 
-  unsigned s;
-  decode(s, p);
-  state_set(STATE_AUTH | (s & MASK_STATE_EXPORTED));
+  {
+    unsigned s;
+    decode(s, p);
+    s &= MASK_STATE_EXPORTED;
+
+    if (s & STATE_RANDEPHEMERALPIN) {
+      set_ephemeral_rand(true);
+    }
+    if (s & STATE_DISTEPHEMERALPIN) {
+      set_ephemeral_dist(true);
+    }
+
+    state_set(STATE_AUTH | s);
+  }
 
   if (is_dirty()) {
     get(PIN_DIRTY);
@@ -5378,7 +5388,7 @@ void CInode::set_ephemeral_rand(bool yes)
   }
 }
 
-void CInode::maybe_ephemeral_rand()
+void CInode::maybe_ephemeral_rand(bool fresh)
 {
   if (!mdcache->get_export_ephemeral_random_config()) {
     dout(15) << __func__ << " config false: cannot ephemeral random pin " << *this << dendl;
@@ -5396,6 +5406,8 @@ void CInode::maybe_ephemeral_rand()
     dout(10) << __func__ << " already ephemeral random pinned: requeueing " << *this << dendl;
     queue_export_pin(mdcache->hash_into_rank_bucket(ino()));
     return;
+  } else if (!fresh) {
+    return;
   }
 
   double threshold = get_ephemeral_rand();
@@ -5432,6 +5444,34 @@ void CInode::set_export_pin(mds_rank_t rank)
   get_projected_inode()->export_pin = rank;
 }
 
+void CInode::check_pin_policy()
+{
+  const CInode *in = this;
+  mds_rank_t etarget = MDS_RANK_NONE;
+  while (true) {
+    if (in->is_system())
+      break;
+    const CDentry *pdn = in->get_parent_dn();
+    if (!pdn)
+      break;
+    if (in->get_inode().nlink == 0) {
+      // ignore export pin for unlinked directory
+      return;
+    } else if (etarget != MDS_RANK_NONE && in->has_ephemeral_policy()) {
+      return;
+    } else if (in->get_inode().export_pin >= 0) {
+      /* clear any epin policy */
+      set_ephemeral_dist(false);
+      set_ephemeral_rand(false);
+      return;
+    } else if (etarget == MDS_RANK_NONE && in->is_ephemerally_pinned()) {
+      /* If a parent overrides a grandparent ephemeral pin policy with an export pin, we use that export pin instead. */
+      etarget = mdcache->hash_into_rank_bucket(in->ino());
+    }
+    in = pdn->get_dir()->inode;
+  }
+}
+
 mds_rank_t CInode::get_export_pin(bool inherit, bool ephemeral) const
 {
   /* An inode that is export pinned may not necessarily be a subtree root, we
@@ -5450,7 +5490,7 @@ mds_rank_t CInode::get_export_pin(bool inherit, bool ephemeral) const
     if (in->get_inode().nlink == 0) {
       // ignore export pin for unlinked directory
       return MDS_RANK_NONE;
-    } else if (etarget != MDS_RANK_NONE && (in->get_inode().export_ephemeral_random_pin > 0.0 || in->get_inode().export_ephemeral_distributed_pin)) {
+    } else if (etarget != MDS_RANK_NONE && in->has_ephemeral_policy()) {
       return etarget;
     } else if (in->get_inode().export_pin >= 0) {
       return in->get_inode().export_pin;

diff --git a/src/mds/CInode.h b/src/mds/CInode.h
index 0c0d97ff0874ce0e850a9ad4a31eb786fb20aae8..83fb13e4d82967126b87556dd3978ea9bfd8f94e 100644 (file)
--- a/src/mds/CInode.h
+++ b/src/mds/CInode.h
@@ -331,12 +331,25 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
   static const int STATE_ORPHAN =      STATE_NOTIFYREF;
 
   static const int MASK_STATE_EXPORTED =
-    (STATE_DIRTY|STATE_NEEDSRECOVER|STATE_DIRTYPARENT|STATE_DIRTYPOOL);
+    (STATE_DIRTY|STATE_NEEDSRECOVER|STATE_DIRTYPARENT|STATE_DIRTYPOOL|
+    STATE_DISTEPHEMERALPIN|STATE_RANDEPHEMERALPIN);
   static const int MASK_STATE_EXPORT_KEPT =
     (STATE_FROZEN|STATE_AMBIGUOUSAUTH|STATE_EXPORTINGCAPS|
      STATE_QUEUEDEXPORTPIN|STATE_TRACKEDBYOFT|STATE_DELAYEDEXPORTPIN|
      STATE_DISTEPHEMERALPIN|STATE_RANDEPHEMERALPIN);
 
+  /* These are for "permanent" state markers that are passed around between
+   * MDS. Nothing protects/updates it like a typical MDS lock.
+   *
+   * Currently, we just use this for REPLICATED inodes. The reason we need to
+   * replicate the random epin state is because the directory inode is still
+   * under the authority of the parent subtree. So it's not exported normally
+   * and we can't pass around the state that way. The importer of the dirfrags
+   * still needs to know that the inode is random pinned though otherwise it
+   * doesn't know that the dirfrags are pinned.
+   */
+  static const int MASK_STATE_REPLICATED = STATE_RANDEPHEMERALPIN;
+
   // -- waiters --
   static const uint64_t WAIT_DIR         = (1<<0);
   static const uint64_t WAIT_FROZEN      = (1<<1);
@@ -917,6 +930,8 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
   void queue_export_pin(mds_rank_t target);
   void maybe_export_pin(bool update=false);
 
+  void check_pin_policy();
+
   void set_ephemeral_dist(bool yes);
   void maybe_ephemeral_dist(bool update=false);
   void maybe_ephemeral_dist_children(bool update=false);
@@ -927,18 +942,28 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
 
   double get_ephemeral_rand(bool inherit=true) const;
   void set_ephemeral_rand(bool yes);
-  void maybe_ephemeral_rand();
+  void maybe_ephemeral_rand(bool fresh=false);
   void setxattr_ephemeral_rand(double prob=0.0);
   bool is_ephemeral_rand() const {
     return state_test(STATE_RANDEPHEMERALPIN);
   }
 
+  bool has_ephemeral_policy() const {
+    return get_inode().export_ephemeral_random_pin > 0.0 ||
+           get_inode().export_ephemeral_distributed_pin;
+  }
   bool is_ephemerally_pinned() const {
     return state_test(STATE_DISTEPHEMERALPIN) ||
            state_test(STATE_RANDEPHEMERALPIN);
   }
   bool is_exportable(mds_rank_t dest) const;
 
+  void maybe_pin() {
+    maybe_export_pin();
+    maybe_ephemeral_dist();
+    maybe_ephemeral_rand();
+  }
+
   void print(std::ostream& out) override;
   void dump(ceph::Formatter *f, int flags = DUMP_DEFAULT) const;
 

diff --git a/src/mds/MDBalancer.cc b/src/mds/MDBalancer.cc
index e4de95ec0f46551a50c644bdbeeb8ebaf6ac82fa..0c8ec4965bcf6535256a1f98bb4085189355d8b5 100644 (file)
--- a/src/mds/MDBalancer.cc
+++ b/src/mds/MDBalancer.cc
@@ -99,6 +99,8 @@ void MDBalancer::handle_export_pins(void)
     auto cur = it++;
     CInode *in = *cur;
     ceph_assert(in->is_dir());
+
+    in->check_pin_policy();
     mds_rank_t export_pin = in->get_export_pin(false);
     if (export_pin >= mds->mdsmap->get_max_mds()) {
       dout(20) << " delay export_pin=" << export_pin << " on " << *in << dendl;
@@ -179,9 +181,12 @@ void MDBalancer::handle_export_pins(void)
                  dendl;
     }
 
-    if (export_pin >= 0 && export_pin < mds->mdsmap->get_max_mds() 
-       && export_pin != mds->get_nodeid()) {
-      mds->mdcache->migrator->export_dir(cd, export_pin);
+    if (export_pin >= 0 && export_pin < mds->mdsmap->get_max_mds()) {
+      if (export_pin == mds->get_nodeid()) {
+        cd->get_inode()->check_pin_policy();
+      } else {
+        mds->mdcache->migrator->export_dir(cd, export_pin);
+      }
     }
   }
 }

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index b06d4cc2f92b96c7faadb90f36053d6054d1c0cf..58ce50128fbe56b6a38da135d0a1e84bfa1bd4b2 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -983,6 +983,11 @@ void MDCache::adjust_subtree_auth(CDir *dir, mds_authority_t auth, bool adjust_p
     }
   }
 
+  if (dir->is_auth()) {
+    /* do this now that we are auth for the CDir */
+    dir->inode->maybe_pin();
+  }
+
   show_subtrees();
 }
 
@@ -10795,7 +10800,7 @@ void MDCache::encode_replica_dentry(CDentry *dn, mds_rank_t to, bufferlist& bl)
 void MDCache::encode_replica_inode(CInode *in, mds_rank_t to, bufferlist& bl,
                              uint64_t features)
 {
-  ENCODE_START(1, 1, bl);
+  ENCODE_START(2, 1, bl);
   ceph_assert(in->is_auth());
   encode(in->inode.ino, bl);  // bleh, minor assymetry here
   encode(in->last, bl);
@@ -10805,6 +10810,10 @@ void MDCache::encode_replica_inode(CInode *in, mds_rank_t to, bufferlist& bl,
 
   in->_encode_base(bl, features);
   in->_encode_locks_state_for_replica(bl, mds->get_state() < MDSMap::STATE_ACTIVE);
+
+  __u32 state = in->state;
+  encode(state, bl);
+
   ENCODE_FINISH(bl);
 }
 
@@ -10901,7 +10910,7 @@ void MDCache::decode_replica_dentry(CDentry *&dn, bufferlist::const_iterator& p,
 
 void MDCache::decode_replica_inode(CInode *&in, bufferlist::const_iterator& p, CDentry *dn, MDSContext::vec& finished)
 {
-  DECODE_START(1, p);
+  DECODE_START(2, p);
   inodeno_t ino;
   snapid_t last;
   __u32 nonce;
@@ -10935,6 +10944,17 @@ void MDCache::decode_replica_inode(CInode *&in, bufferlist::const_iterator& p, C
     if (!dn->get_linkage()->is_primary() || dn->get_linkage()->get_inode() != in)
       dout(10) << __func__ << " different linkage in dentry " << *dn << dendl;
   }
+
+  if (struct_v >= 2) {
+    __u32 s;
+    decode(s, p);
+    s &= CInode::MASK_STATE_REPLICATED;
+    if (s & CInode::STATE_RANDEPHEMERALPIN) {
+      dout(10) << "replica inode is random ephemeral pinned" << dendl;
+      in->set_ephemeral_rand(true);
+    }
+  }
+
   DECODE_FINISH(p); 
 }
 

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 58ff0fd0b052a5afd13b693139d9d3392a1ee480..a261fc4c31105fb3d1467ece7de3d56d4084ec3b 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -6011,7 +6011,7 @@ public:
     } else if (newi->inode.is_dir()) {
       // We do this now so that the linkages on the new directory are stable.
       newi->maybe_ephemeral_dist();
-      newi->maybe_ephemeral_rand();
+      newi->maybe_ephemeral_rand(true);
     }
 
     // hit pop

diff --git a/src/mds/journal.cc b/src/mds/journal.cc
index 68b58495ae582f4005e4dfb0c13932772410e7d4..70face21f1c53bdc15033e2932d96a287f6f1aa2 100644 (file)
--- a/src/mds/journal.cc
+++ b/src/mds/journal.cc
@@ -530,7 +530,7 @@ void EMetaBlob::fullbit::update_inode(MDSRank *mds, CInode *in)
     if (is_export_ephemeral_random()) {
       dout(15) << "random ephemeral pin on " << *in << dendl;
       in->set_ephemeral_rand(true);
-      in->maybe_ephemeral_rand();
+      in->maybe_ephemeral_rand(true);
     }
     in->maybe_ephemeral_dist();
     in->maybe_export_pin();