mds: fix replica dentry linkage updates

Previously we have a broken hack that would drop a replicated
null dentry if it got new linkage.  We already get an explicit
message if it was unlinked, that unlinks it cleanly.  Do the
same for links, and replicate the newly linked inode as
needed.  This is much cleaner and more correct.

Specifically, this fixes a problem where a create (link) and
unlink are pipelined by the same client under the same xlock,
so that the previous hack (in the handle_lock handler) never
triggers because the lock state doesn't toggle between the link
and unlink.

send_dentry_link sends current, not projected, linkage

diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h
index 0ae4745b24b609d3dee1c1ab31220183914413df..6ff5819842cd0e8e2ecd69426d686610429a61e0 100644 (file)
--- a/src/include/ceph_fs.h
+++ b/src/include/ceph_fs.h
@@ -36,7 +36,7 @@
  * client-facing protocol.
  */
 #define CEPH_OSD_PROTOCOL     8 /* cluster internal */
-#define CEPH_MDS_PROTOCOL     9 /* cluster internal */
+#define CEPH_MDS_PROTOCOL    10 /* cluster internal */
 #define CEPH_MON_PROTOCOL     5 /* cluster internal */
 #define CEPH_OSDC_PROTOCOL   24 /* server/client */
 #define CEPH_MDSC_PROTOCOL   32 /* server/client */

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 7954949b020532be678dcbd8d3e873eff5fefdc1..c212ef9eed177034d15a376a415c495c2844593c 100644 (file)
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -2397,19 +2397,6 @@ void Locker::handle_simple_lock(SimpleLock *lock, MLock *m)
     lock->decode_locked_state(m->get_data());
     lock->set_state(LOCK_SYNC);
     lock->finish_waiters(SimpleLock::WAIT_RD|SimpleLock::WAIT_STABLE);
-
-    // special case: trim replica no-longer-null dentry?
-    if (lock->get_type() == CEPH_LOCK_DN) {
-      CDentry *dn = (CDentry*)lock->get_parent();
-      if (dn->get_linkage()->is_null() && m->get_data().length() > 0) {
-       dout(10) << "handle_simple_lock replica dentry null -> non-null, must trim " 
-                << *dn << dendl;
-       assert(dn->get_num_ref() == 0);
-       map<int, MCacheExpire*> expiremap;
-       mdcache->trim_dentry(dn, expiremap);
-       mdcache->send_expire_messages(expiremap);
-      }
-    }
     break;
     
   case LOCK_AC_LOCK:

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 2d935f76cbfd9f50d35de8d5d8be6de2f9792a49..4f0bdfc5672dc3908e78178286c4fa54dcd92c6f 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -64,6 +64,7 @@
 #include "messages/MInodeFileCaps.h"
 
 #include "messages/MLock.h"
+#include "messages/MDentryLink.h"
 #include "messages/MDentryUnlink.h"
 
 #include "messages/MClientRequest.h"
@@ -5642,6 +5643,9 @@ void MDCache::dispatch(Message *m)
 
 
 
+  case MSG_MDS_DENTRYLINK:
+    handle_dentry_link((MDentryLink*)m);
+    break;
   case MSG_MDS_DENTRYUNLINK:
     handle_dentry_unlink((MDentryUnlink*)m);
     break;
@@ -7999,17 +8003,99 @@ void MDCache::handle_dir_update(MDirUpdate *m)
 
 
 
+// LINK
+
+void MDCache::send_dentry_link(CDentry *dn)
+{
+  dout(7) << "send_dentry_link " << *dn << dendl;
+
+  for (map<int,int>::iterator p = dn->replicas_begin(); 
+       p != dn->replicas_end(); 
+       p++) {
+    if (mds->mdsmap->get_state(p->first) < MDSMap::STATE_REJOIN) 
+      continue;
+    CDentry::linkage_t *dnl = dn->get_linkage();
+    MDentryLink *m = new MDentryLink(dn->get_dir()->dirfrag(), dn->name,
+                                    dnl->is_primary());
+    if (dnl->is_primary()) {
+      dout(10) << "  primary " << *dnl->get_inode() << dendl;
+      replicate_inode(dnl->get_inode(), p->first, m->bl);
+    } else if (dnl->is_remote()) {
+      inodeno_t ino = dnl->get_remote_ino();
+      __u8 d_type = dnl->get_remote_d_type();
+      dout(10) << "  remote " << ino << " " << d_type << dendl;
+      ::encode(ino, m->bl);
+      ::encode(d_type, m->bl);
+    } else
+      assert(0);   // aie, bad caller!
+    mds->send_message_mds(m, p->first);
+  }
+}
+
+void MDCache::handle_dentry_link(MDentryLink *m)
+{
+  CDir *dir = get_dirfrag(m->get_dirfrag());
+  assert(dir);
+  CDentry *dn = dir->lookup(m->get_dn());
+  assert(dn);
+
+  dout(7) << "handle_dentry_link on " << *dn << dendl;
+  CDentry::linkage_t *dnl = dn->get_linkage();
+
+  assert(!dn->is_auth());
+  assert(dnl->is_null());
+
+  bufferlist::iterator p = m->bl.begin();
+  list<Context*> finished;
+  
+  if (m->get_is_primary()) {
+    // primary link.
+    add_replica_inode(p, dn, finished);
+  } else {
+    // remote link, easy enough.
+    inodeno_t ino;
+    __u8 d_type;
+    ::decode(ino, p);
+    ::decode(d_type, p);
+    dir->link_remote_inode(dn, ino, d_type);
+  }
+  
+  if (!finished.empty())
+    mds->queue_waiters(finished);
+
+  delete m;
+  return;
+}
+
 
 // UNLINK
 
+void MDCache::send_dentry_unlink(CDentry *dn, CDentry *straydn)
+{
+  dout(10) << "send_dentry_unlink " << *dn << dendl;
+  // share unlink news with replicas
+  for (map<int,int>::iterator it = dn->replicas_begin();
+       it != dn->replicas_end();
+       it++) {
+    MDentryUnlink *unlink = new MDentryUnlink(dn->get_dir()->dirfrag(), dn->name);
+    if (straydn) {
+      replicate_inode(get_myin(), it->first, unlink->straybl);
+      replicate_dir(straydn->get_dir()->inode->get_parent_dn()->get_dir(), it->first, unlink->straybl);
+      replicate_dentry(straydn->get_dir()->inode->get_parent_dn(), it->first, unlink->straybl);
+      replicate_inode(straydn->get_dir()->inode, it->first, unlink->straybl);
+      replicate_dir(straydn->get_dir(), it->first, unlink->straybl);
+      replicate_dentry(straydn, it->first, unlink->straybl);
+    }
+    mds->send_message_mds(unlink, it->first);
+  }
+}
+
 void MDCache::handle_dentry_unlink(MDentryUnlink *m)
 {
   CDir *dir = get_dirfrag(m->get_dirfrag());
-
   if (!dir) {
     dout(7) << "handle_dentry_unlink don't have dirfrag " << m->get_dirfrag() << dendl;
-  }
-  else {
+  } else {
     CDentry *dn = dir->lookup(m->get_dn());
     if (!dn) {
       dout(7) << "handle_dentry_unlink don't have dentry " << *dir << " dn " << m->get_dn() << dendl;

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 15f87b27dfc86f556f7c58733b58716d8c98496b..536800fc51e76586178d0775584cf608311d19fc 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -49,6 +49,7 @@ class MDiscover;
 class MDiscoverReply;
 class MCacheExpire;
 class MDirUpdate;
+class MDentryLink;
 class MDentryUnlink;
 class MLock;
 
@@ -1023,8 +1024,12 @@ public:
   CInode *add_replica_inode(bufferlist::iterator& p, CDentry *dn, list<Context*>& finished);
   CDentry *add_replica_stray(bufferlist &bl, int from);
 
-protected:
   // -- namespace --
+public:
+  void send_dentry_link(CDentry *dn);
+  void send_dentry_unlink(CDentry *dn, CDentry *straydn);
+protected:
+  void handle_dentry_link(MDentryLink *m);
   void handle_dentry_unlink(MDentryUnlink *m);
 
 

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 6ba9788b5c6ccfa048b728f4617febf0bad326c6..0021c0a7c4581beaf21c7254a0b91ad933a59cab 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -2158,6 +2158,8 @@ public:
 
     mdr->apply();
 
+    mds->mdcache->send_dentry_link(dn);
+
     mds->server->reply_request(mdr, 0);
   }
 };
@@ -2865,6 +2867,9 @@ public:
 
     mdr->apply();
 
+    mds->mdcache->send_dentry_link(dn);
+
+
     // hit pop
     mds->balancer->hit_inode(mdr->now, newi, META_POP_IWR);
     //mds->balancer->hit_dir(mdr->now, dn->get_dir(), META_POP_DWR);
@@ -3165,6 +3170,8 @@ void Server::_link_local_finish(MDRequest *mdr, CDentry *dn, CInode *targeti,
 
   mdr->apply();
   
+  mds->mdcache->send_dentry_link(dn);
+
   // bump target popularity
   mds->balancer->hit_inode(mdr->now, targeti, META_POP_IWR);
   //mds->balancer->hit_dir(mdr->now, dn->get_dir(), META_POP_DWR);
@@ -3283,6 +3290,8 @@ void Server::_link_remote_finish(MDRequest *mdr, bool inc,
   }
 
   mdr->apply();
+
+  mds->mdcache->send_dentry_link(dn);
   
   // commit anchor update?
   if (mdr->more()->dst_reanchor_atid) 
@@ -3810,22 +3819,7 @@ void Server::_unlink_local_finish(MDRequest *mdr,
   dn->mark_dirty(dnpv, mdr->ls);  
   mdr->apply();
   
-  // share unlink news with replicas
-  for (map<int,int>::iterator it = dn->replicas_begin();
-       it != dn->replicas_end();
-       it++) {
-    dout(7) << "_unlink_local_finish sending MDentryUnlink to mds" << it->first << dendl;
-    MDentryUnlink *unlink = new MDentryUnlink(dn->get_dir()->dirfrag(), dn->name);
-    if (straydn) {
-      mdcache->replicate_inode(mds->mdcache->get_myin(), it->first, unlink->straybl);
-      mdcache->replicate_dir(straydn->get_dir()->inode->get_parent_dn()->get_dir(), it->first, unlink->straybl);
-      mdcache->replicate_dentry(straydn->get_dir()->inode->get_parent_dn(), it->first, unlink->straybl);
-      mdcache->replicate_inode(straydn->get_dir()->inode, it->first, unlink->straybl);
-      mdcache->replicate_dir(straydn->get_dir(), it->first, unlink->straybl);
-      mdcache->replicate_dentry(straydn, it->first, unlink->straybl);
-    }
-    mds->send_message_mds(unlink, it->first);
-  }
+  mds->mdcache->send_dentry_unlink(dn, straydn);
   
   // commit anchor update?
   if (mdr->more()->dst_reanchor_atid) 

diff --git a/src/messages/MDentryLink.h b/src/messages/MDentryLink.h
new file mode 100644 (file)
index 0000000..7b90f6f
--- /dev/null
+++ b/src/messages/MDentryLink.h
@@ -0,0 +1,59 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+
+#ifndef __MDENTRYLINK_H
+#define __MDENTRYLINK_H
+
+class MDentryLink : public Message {
+  dirfrag_t dirfrag;
+  nstring dn;
+  bool is_primary;
+
+ public:
+  dirfrag_t get_dirfrag() { return dirfrag; }
+  nstring& get_dn() { return dn; }
+  bool get_is_primary() { return is_primary; }
+
+  bufferlist bl;
+
+  MDentryLink() :
+    Message(MSG_MDS_DENTRYLINK) { }
+  MDentryLink(dirfrag_t df, nstring& n, bool p) :
+    Message(MSG_MDS_DENTRYLINK),
+    dirfrag(df),
+    dn(n),
+    is_primary(p) {}
+
+  const char *get_type_name() { return "dentry_link";}
+  void print(ostream& o) {
+    o << "dentry_link(" << dirfrag << " " << dn << ")";
+  }
+  
+  void decode_payload() {
+    bufferlist::iterator p = payload.begin();
+    ::decode(dirfrag, p);
+    ::decode(dn, p);
+    ::decode(is_primary, p);
+    ::decode(bl, p);
+  }
+  void encode_payload() {
+    ::encode(dirfrag, payload);
+    ::encode(dn, payload);
+    ::encode(is_primary, payload);
+    ::encode(bl, payload);
+  }
+};
+
+#endif

diff --git a/src/msg/Message.cc b/src/msg/Message.cc
index b3495ed0ba355abe0bd132a844800be780e516a9..97df383ddffb45a805179e7e3737e5545bb80e8a 100644 (file)
--- a/src/msg/Message.cc
+++ b/src/msg/Message.cc
@@ -112,6 +112,7 @@ using namespace std;
 
 
 #include "messages/MDentryUnlink.h"
+#include "messages/MDentryLink.h"
 
 #include "messages/MHeartbeat.h"
 
@@ -461,7 +462,10 @@ Message *decode_message(ceph_msg_header& header, ceph_msg_footer& footer,
 
 
   case MSG_MDS_DENTRYUNLINK:
-    m = new MDentryUnlink();
+    m = new MDentryUnlink;
+    break;
+  case MSG_MDS_DENTRYLINK:
+    m = new MDentryLink;
     break;
 
   case MSG_MDS_HEARTBEAT:

diff --git a/src/msg/Message.h b/src/msg/Message.h
index 77946a2cb0c9a3ed665d0c23b662cd909312e261..cc16d5784b10d53b82cf86f253bed14391c3c048 100644 (file)
--- a/src/msg/Message.h
+++ b/src/msg/Message.h
@@ -95,6 +95,7 @@
 #define MSG_MDS_FRAGMENTNOTIFY     0x209
 #define MSG_MDS_OFFLOAD_TARGETS    0x20a
 #define MSG_MDS_OFFLOAD_COMPLETE   0x20b
+#define MSG_MDS_DENTRYLINK         0x20c
 
 #define MSG_MDS_LOCK               0x300
 #define MSG_MDS_INODEFILECAPS      0x301