mds: fix file purge race

Handle the case where a new inode ref appears while we are
purging an inode.  If so, we just truncate it to 0, so that next
time we go through purge_stray() we don't have to do the work
over again.

This can happen if a client goes snooping in the stray dir (or
who knows what else!).

diff --git a/src/TODO b/src/TODO
index 6cb74a4c3421f4aeb254606ae4fae5a3d27f76eb..e1d3bd7926adf915bd35f6758528f7e27acfe881 100644 (file)
--- a/src/TODO
+++ b/src/TODO
@@ -83,26 +83,6 @@ cp: writing `/c/ceph2.2/bin/gs-gpl': Bad file descriptor
 
 ?- kclient: socket creation
 
-- mds file purge should truncate in place, or remove from namespace before purge.  otherwise new ref can appear before inode is destroyed.
-mds/MDCache.cc: In function 'void MDCache::remove_inode(CInode*)':
-mds/MDCache.cc:217: FAILED assert(o->get_num_ref() == 0)
- 1: /tmp/cmds.20091211.084324(_Z18__ceph_assert_failPKcS0_iS0_+0x34) [0x9656ea]
- 2: /tmp/cmds.20091211.084324(_ZN7MDCache12remove_inodeEP6CInode+0x1ad) [0x7af283]
- 3: /tmp/cmds.20091211.084324(_ZN7MDCache19_purge_stray_loggedEP7CDentrymP10LogSegment+0x115) [0x7af3b5]
- 4: /tmp/cmds.20091211.084324(_ZN22C_MDC_PurgeStrayLogged6finishEi+0x34) [0x83286e]
- 5: /tmp/cmds.20091211.084324(_Z15finish_contextsRSt4listIP7ContextSaIS1_EEi+0x130) [0x736d96]
- 6: /tmp/cmds.20091211.084324(_ZN9Journaler13_finish_flushEil7utime_tb+0x873) [0x915f4d]
- 7: /tmp/cmds.20091211.084324(_ZN9Journaler7C_Flush6finishEi+0x43) [0x91d5eb]
- 8: /tmp/cmds.20091211.084324(_ZN8Objecter19handle_osd_op_replyEP11MOSDOpReply+0xcf5) [0x8e7415]
- 9: /tmp/cmds.20091211.084324(_ZN3MDS9_dispatchEP7Message+0x1f04) [0x715dda]
- 10: /tmp/cmds.20091211.084324(_ZN3MDS11ms_dispatchEP7Message+0x2f) [0x716dc1]
- 11: /tmp/cmds.20091211.084324(_ZN9Messenger19ms_deliver_dispatchEP7Message+0x54) [0x70a658]
- 12: /tmp/cmds.20091211.084324(_ZN15SimpleMessenger8Endpoint14dispatch_entryEv+0x4df) [0x6f78af]
- 13: /tmp/cmds.20091211.084324(_ZN15SimpleMessenger8Endpoint14DispatchThread5entryEv+0x19) [0x70ce77]
- 14: /tmp/cmds.20091211.084324(_ZN6Thread11_entry_funcEPv+0x20) [0x704b0c]
- 15: /lib/libpthread.so.0 [0x7f3ea5bf2fc7]
- 16: /lib/libc.so.6(clone+0x6d) [0x7f3ea4e355ad]
-
 - snaprealm thing
 ceph3:~# find /c
 /c

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index af2f7341f054cb3cf9dd3c955e26eee7e6f4219a..3249aa87bc5b517ab95ceff77fe9867af377fe94 100644 (file)
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -1044,6 +1044,17 @@ void Locker::file_update_finish(CInode *in, Mutation *mut, bool share, client_t
   
   if (share && in->is_auth() && in->filelock.is_stable())
     share_inode_max_size(in);
+
+  // unlinked stray?  may need to purge (e.g., after all caps are released)
+  if (in->inode.nlink == 0 &&
+      in->get_parent_dn() &&
+      in->get_parent_dn()->get_dir()->get_inode()->is_stray() &&
+      !in->is_any_caps()) {
+    if (!in->is_auth())
+      mdcache->touch_dentry_bottom(in->get_parent_dn());  // move to bottom of lru so that we trim quickly!
+    else if (!in->is_replicated())
+      mdcache->eval_stray(in->get_parent_dn());
+  }
 }
 
 Capability* Locker::issue_new_caps(CInode *in,

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 64c710440cb7c96ad766e140e3397abb0de8ced1..a8434428d72ccb1b2b10f9769edac66db1996d08 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -5317,7 +5317,8 @@ void MDCache::remove_client_cap(CInode *in, client_t client)
 
   if (in->is_auth()) {
     // make sure we clear out the client byte range
-    if (in->get_projected_inode()->client_ranges.count(client))
+    if (in->get_projected_inode()->client_ranges.count(client) &&
+       !(in->inode.nlink == 0 && !in->is_any_caps()))    // unless it's unlink + stray
       mds->locker->check_inode_max_size(in);
   } else {
     mds->locker->request_inode_file_caps(in);
@@ -6940,39 +6941,67 @@ public:
     cache->_purge_stray_logged(dn, pdv, ls);
   }
 };
+class C_MDC_PurgeStrayLoggedTruncate : public Context {
+  MDCache *cache;
+  CDentry *dn;
+  LogSegment *ls;
+public:
+  C_MDC_PurgeStrayLoggedTruncate(MDCache *c, CDentry *d, LogSegment *s) : 
+    cache(c), dn(d), ls(s) { }
+  void finish(int r) {
+    cache->_purge_stray_logged_truncate(dn, ls);
+  }
+};
 
 void MDCache::_purge_stray_purged(CDentry *dn)
 {
-  dout(10) << "_purge_stray_purged " << *dn << dendl;
-
   CInode *in = dn->get_projected_linkage()->get_inode();
+  dout(10) << "_purge_stray_purged " << *dn << " " << *in << dendl;
 
-  // kill dentry.
-  version_t pdv = dn->pre_dirty();
+  if (in->get_num_ref() == (int)in->is_dirty()) {
+    // kill dentry.
+    version_t pdv = dn->pre_dirty();
+    
+    EUpdate *le = new EUpdate(mds->mdlog, "purge_stray");
+    mds->mdlog->start_entry(le);
+    
+    le->metablob.add_dir_context(dn->dir);
+    le->metablob.add_null_dentry(dn, true);
+    le->metablob.add_destroyed_inode(in->ino());
 
-  EUpdate *le = new EUpdate(mds->mdlog, "purge_stray");
-  mds->mdlog->start_entry(le);
+    mds->mdlog->submit_entry(le, new C_MDC_PurgeStrayLogged(this, dn, pdv, mds->mdlog->get_current_segment()));
+  } else {
+    // new refs.. just truncate to 0
+    EUpdate *le = new EUpdate(mds->mdlog, "purge_stray truncate");
+    mds->mdlog->start_entry(le);
+    
+    inode_t *pi = in->project_inode();
+    pi->size = 0;
+    pi->client_ranges.clear();
+    pi->truncate_size = 0;
+    pi->truncate_from = 0;
+    pi->version = in->pre_dirty();
 
-  le->metablob.add_dir_context(dn->dir);
-  le->metablob.add_null_dentry(dn, true);
-  le->metablob.add_destroyed_inode(in->ino());
+    le->metablob.add_dir_context(dn->dir);
+    le->metablob.add_primary_dentry(dn, true, in);
 
-  mds->mdlog->submit_entry(le, new C_MDC_PurgeStrayLogged(this, dn, pdv, mds->mdlog->get_current_segment()));
+    mds->mdlog->submit_entry(le, new C_MDC_PurgeStrayLoggedTruncate(this, dn, mds->mdlog->get_current_segment()));
+  }
 }
 
 void MDCache::_purge_stray_logged(CDentry *dn, version_t pdv, LogSegment *ls)
 {
-  dout(10) << "_purge_stray_logged " << *dn << " " << *dn->get_projected_linkage()->get_inode() << dendl;
+  CInode *in = dn->get_projected_linkage()->get_inode();
+  dout(10) << "_purge_stray_logged " << *dn << " " << *in << dendl;
 
   dn->state_clear(CDentry::STATE_PURGING);
   dn->put(CDentry::PIN_PURGING);
 
+  assert(!in->state_test(CInode::STATE_RECOVERING));
+
   // unlink and remove dentry
-  CInode *in = dn->get_projected_linkage()->get_inode();
   if (in->is_dirty())
     in->mark_clean();
-  if (in->state_test(CInode::STATE_RECOVERING))
-    unqueue_file_recover(in);
   if (dn->is_dirty())
     dn->mark_clean();
   remove_inode(in);
@@ -6982,6 +7011,19 @@ void MDCache::_purge_stray_logged(CDentry *dn, version_t pdv, LogSegment *ls)
   touch_dentry_bottom(dn);  // drop dn as quickly as possible.
 }
 
+void MDCache::_purge_stray_logged_truncate(CDentry *dn, LogSegment *ls)
+{
+  CInode *in = dn->get_projected_linkage()->get_inode();
+  dout(10) << "_purge_stray_logged_truncate " << *dn << " " << *in << dendl;
+
+  dn->state_clear(CDentry::STATE_PURGING);
+  dn->put(CDentry::PIN_PURGING);
+
+  in->pop_and_dirty_projected_inode(ls);
+
+  eval_stray(dn);
+}
+
 
 
 void MDCache::reintegrate_stray(CDentry *straydn, CDentry *rdn)

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 7df1214b6b41372c56f9c319acddc15d573f7d7c..50087a099268a71a0d60a40001b775bd350f684f 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -978,7 +978,9 @@ protected:
   void purge_stray(CDentry *dn);
   void _purge_stray_purged(CDentry *dn);
   void _purge_stray_logged(CDentry *dn, version_t pdv, LogSegment *ls);
+  void _purge_stray_logged_truncate(CDentry *dn, LogSegment *ls);
   friend class C_MDC_PurgeStrayLogged;
+  friend class C_MDC_PurgeStrayLoggedTruncate;
   friend class C_MDC_PurgeStrayPurged;
   void reintegrate_stray(CDentry *dn, CDentry *rlink);
   void migrate_stray(CDentry *dn, int src, int dest);