From: Sage Weil Date: Tue, 23 Feb 2010 23:51:39 +0000 (-0800) Subject: mds: fix file purge race X-Git-Tag: v0.20~415 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=c51148cd6d66f0720538838017d65c51f82025ca;p=ceph.git mds: fix file purge race Handle the case where a new inode ref appears while we are purging an inode. If so, we just truncate it to 0, so that next time we go through purge_stray() we don't have to do the work over again. This can happen if a client goes snooping in the stray dir (or who knows what else!). --- diff --git a/src/TODO b/src/TODO index 6cb74a4c3421..e1d3bd7926ad 100644 --- a/src/TODO +++ b/src/TODO @@ -83,26 +83,6 @@ cp: writing `/c/ceph2.2/bin/gs-gpl': Bad file descriptor ?- kclient: socket creation -- mds file purge should truncate in place, or remove from namespace before purge. otherwise new ref can appear before inode is destroyed. -mds/MDCache.cc: In function 'void MDCache::remove_inode(CInode*)': -mds/MDCache.cc:217: FAILED assert(o->get_num_ref() == 0) - 1: /tmp/cmds.20091211.084324(_Z18__ceph_assert_failPKcS0_iS0_+0x34) [0x9656ea] - 2: /tmp/cmds.20091211.084324(_ZN7MDCache12remove_inodeEP6CInode+0x1ad) [0x7af283] - 3: /tmp/cmds.20091211.084324(_ZN7MDCache19_purge_stray_loggedEP7CDentrymP10LogSegment+0x115) [0x7af3b5] - 4: /tmp/cmds.20091211.084324(_ZN22C_MDC_PurgeStrayLogged6finishEi+0x34) [0x83286e] - 5: /tmp/cmds.20091211.084324(_Z15finish_contextsRSt4listIP7ContextSaIS1_EEi+0x130) [0x736d96] - 6: /tmp/cmds.20091211.084324(_ZN9Journaler13_finish_flushEil7utime_tb+0x873) [0x915f4d] - 7: /tmp/cmds.20091211.084324(_ZN9Journaler7C_Flush6finishEi+0x43) [0x91d5eb] - 8: /tmp/cmds.20091211.084324(_ZN8Objecter19handle_osd_op_replyEP11MOSDOpReply+0xcf5) [0x8e7415] - 9: /tmp/cmds.20091211.084324(_ZN3MDS9_dispatchEP7Message+0x1f04) [0x715dda] - 10: /tmp/cmds.20091211.084324(_ZN3MDS11ms_dispatchEP7Message+0x2f) [0x716dc1] - 11: /tmp/cmds.20091211.084324(_ZN9Messenger19ms_deliver_dispatchEP7Message+0x54) [0x70a658] - 12: /tmp/cmds.20091211.084324(_ZN15SimpleMessenger8Endpoint14dispatch_entryEv+0x4df) [0x6f78af] - 13: /tmp/cmds.20091211.084324(_ZN15SimpleMessenger8Endpoint14DispatchThread5entryEv+0x19) [0x70ce77] - 14: /tmp/cmds.20091211.084324(_ZN6Thread11_entry_funcEPv+0x20) [0x704b0c] - 15: /lib/libpthread.so.0 [0x7f3ea5bf2fc7] - 16: /lib/libc.so.6(clone+0x6d) [0x7f3ea4e355ad] - - snaprealm thing ceph3:~# find /c /c diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc index af2f7341f054..3249aa87bc5b 100644 --- a/src/mds/Locker.cc +++ b/src/mds/Locker.cc @@ -1044,6 +1044,17 @@ void Locker::file_update_finish(CInode *in, Mutation *mut, bool share, client_t if (share && in->is_auth() && in->filelock.is_stable()) share_inode_max_size(in); + + // unlinked stray? may need to purge (e.g., after all caps are released) + if (in->inode.nlink == 0 && + in->get_parent_dn() && + in->get_parent_dn()->get_dir()->get_inode()->is_stray() && + !in->is_any_caps()) { + if (!in->is_auth()) + mdcache->touch_dentry_bottom(in->get_parent_dn()); // move to bottom of lru so that we trim quickly! + else if (!in->is_replicated()) + mdcache->eval_stray(in->get_parent_dn()); + } } Capability* Locker::issue_new_caps(CInode *in, diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc index 64c710440cb7..a8434428d72c 100644 --- a/src/mds/MDCache.cc +++ b/src/mds/MDCache.cc @@ -5317,7 +5317,8 @@ void MDCache::remove_client_cap(CInode *in, client_t client) if (in->is_auth()) { // make sure we clear out the client byte range - if (in->get_projected_inode()->client_ranges.count(client)) + if (in->get_projected_inode()->client_ranges.count(client) && + !(in->inode.nlink == 0 && !in->is_any_caps())) // unless it's unlink + stray mds->locker->check_inode_max_size(in); } else { mds->locker->request_inode_file_caps(in); @@ -6940,39 +6941,67 @@ public: cache->_purge_stray_logged(dn, pdv, ls); } }; +class C_MDC_PurgeStrayLoggedTruncate : public Context { + MDCache *cache; + CDentry *dn; + LogSegment *ls; +public: + C_MDC_PurgeStrayLoggedTruncate(MDCache *c, CDentry *d, LogSegment *s) : + cache(c), dn(d), ls(s) { } + void finish(int r) { + cache->_purge_stray_logged_truncate(dn, ls); + } +}; void MDCache::_purge_stray_purged(CDentry *dn) { - dout(10) << "_purge_stray_purged " << *dn << dendl; - CInode *in = dn->get_projected_linkage()->get_inode(); + dout(10) << "_purge_stray_purged " << *dn << " " << *in << dendl; - // kill dentry. - version_t pdv = dn->pre_dirty(); + if (in->get_num_ref() == (int)in->is_dirty()) { + // kill dentry. + version_t pdv = dn->pre_dirty(); + + EUpdate *le = new EUpdate(mds->mdlog, "purge_stray"); + mds->mdlog->start_entry(le); + + le->metablob.add_dir_context(dn->dir); + le->metablob.add_null_dentry(dn, true); + le->metablob.add_destroyed_inode(in->ino()); - EUpdate *le = new EUpdate(mds->mdlog, "purge_stray"); - mds->mdlog->start_entry(le); + mds->mdlog->submit_entry(le, new C_MDC_PurgeStrayLogged(this, dn, pdv, mds->mdlog->get_current_segment())); + } else { + // new refs.. just truncate to 0 + EUpdate *le = new EUpdate(mds->mdlog, "purge_stray truncate"); + mds->mdlog->start_entry(le); + + inode_t *pi = in->project_inode(); + pi->size = 0; + pi->client_ranges.clear(); + pi->truncate_size = 0; + pi->truncate_from = 0; + pi->version = in->pre_dirty(); - le->metablob.add_dir_context(dn->dir); - le->metablob.add_null_dentry(dn, true); - le->metablob.add_destroyed_inode(in->ino()); + le->metablob.add_dir_context(dn->dir); + le->metablob.add_primary_dentry(dn, true, in); - mds->mdlog->submit_entry(le, new C_MDC_PurgeStrayLogged(this, dn, pdv, mds->mdlog->get_current_segment())); + mds->mdlog->submit_entry(le, new C_MDC_PurgeStrayLoggedTruncate(this, dn, mds->mdlog->get_current_segment())); + } } void MDCache::_purge_stray_logged(CDentry *dn, version_t pdv, LogSegment *ls) { - dout(10) << "_purge_stray_logged " << *dn << " " << *dn->get_projected_linkage()->get_inode() << dendl; + CInode *in = dn->get_projected_linkage()->get_inode(); + dout(10) << "_purge_stray_logged " << *dn << " " << *in << dendl; dn->state_clear(CDentry::STATE_PURGING); dn->put(CDentry::PIN_PURGING); + assert(!in->state_test(CInode::STATE_RECOVERING)); + // unlink and remove dentry - CInode *in = dn->get_projected_linkage()->get_inode(); if (in->is_dirty()) in->mark_clean(); - if (in->state_test(CInode::STATE_RECOVERING)) - unqueue_file_recover(in); if (dn->is_dirty()) dn->mark_clean(); remove_inode(in); @@ -6982,6 +7011,19 @@ void MDCache::_purge_stray_logged(CDentry *dn, version_t pdv, LogSegment *ls) touch_dentry_bottom(dn); // drop dn as quickly as possible. } +void MDCache::_purge_stray_logged_truncate(CDentry *dn, LogSegment *ls) +{ + CInode *in = dn->get_projected_linkage()->get_inode(); + dout(10) << "_purge_stray_logged_truncate " << *dn << " " << *in << dendl; + + dn->state_clear(CDentry::STATE_PURGING); + dn->put(CDentry::PIN_PURGING); + + in->pop_and_dirty_projected_inode(ls); + + eval_stray(dn); +} + void MDCache::reintegrate_stray(CDentry *straydn, CDentry *rdn) diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h index 7df1214b6b41..50087a099268 100644 --- a/src/mds/MDCache.h +++ b/src/mds/MDCache.h @@ -978,7 +978,9 @@ protected: void purge_stray(CDentry *dn); void _purge_stray_purged(CDentry *dn); void _purge_stray_logged(CDentry *dn, version_t pdv, LogSegment *ls); + void _purge_stray_logged_truncate(CDentry *dn, LogSegment *ls); friend class C_MDC_PurgeStrayLogged; + friend class C_MDC_PurgeStrayLoggedTruncate; friend class C_MDC_PurgeStrayPurged; void reintegrate_stray(CDentry *dn, CDentry *rlink); void migrate_stray(CDentry *dn, int src, int dest);