From: Yan, Zheng Date: Sat, 8 Dec 2012 14:43:32 +0000 (+0800) Subject: mds: fix race between send_dentry_link() and cache expire X-Git-Tag: v0.57~281^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=0002546205a46306433b3c85b079bbb64fb45f09;p=ceph.git mds: fix race between send_dentry_link() and cache expire MDentryLink message can race with cache expire, When it arrives at the target MDS, it's possible there is no corresponding dentry in the cache. If this race happens, we should expire the replica inode encoded in the MDentryLink message. But to expire an inode, the MDS need to know which subtree does the inode belong to, so modify the MDentryLink message to include this information. Signed-off-by: Yan, Zheng --- diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc index 25f726924e39..21861158e8b5 100644 --- a/src/mds/MDCache.cc +++ b/src/mds/MDCache.cc @@ -9279,14 +9279,15 @@ void MDCache::send_dentry_link(CDentry *dn) { dout(7) << "send_dentry_link " << *dn << dendl; + CDir *subtree = get_subtree_root(dn->get_dir()); for (map::iterator p = dn->replicas_begin(); p != dn->replicas_end(); p++) { if (mds->mdsmap->get_state(p->first) < MDSMap::STATE_REJOIN) continue; CDentry::linkage_t *dnl = dn->get_linkage(); - MDentryLink *m = new MDentryLink(dn->get_dir()->dirfrag(), dn->name, - dnl->is_primary()); + MDentryLink *m = new MDentryLink(subtree->dirfrag(), dn->get_dir()->dirfrag(), + dn->name, dnl->is_primary()); if (dnl->is_primary()) { dout(10) << " primary " << *dnl->get_inode() << dendl; replicate_inode(dnl->get_inode(), p->first, m->bl); @@ -9305,32 +9306,48 @@ void MDCache::send_dentry_link(CDentry *dn) /* This function DOES put the passed message before returning */ void MDCache::handle_dentry_link(MDentryLink *m) { - CDir *dir = get_dirfrag(m->get_dirfrag()); - assert(dir); - CDentry *dn = dir->lookup(m->get_dn()); - assert(dn); - dout(7) << "handle_dentry_link on " << *dn << dendl; - CDentry::linkage_t *dnl = dn->get_linkage(); + CDentry *dn = NULL; + CDir *dir = get_dirfrag(m->get_dirfrag()); + if (!dir) { + dout(7) << "handle_dentry_link don't have dirfrag " << m->get_dirfrag() << dendl; + } else { + dn = dir->lookup(m->get_dn()); + if (!dn) { + dout(7) << "handle_dentry_link don't have dentry " << *dir << " dn " << m->get_dn() << dendl; + } else { + dout(7) << "handle_dentry_link on " << *dn << dendl; + CDentry::linkage_t *dnl = dn->get_linkage(); - assert(!dn->is_auth()); - assert(dnl->is_null()); + assert(!dn->is_auth()); + assert(dnl->is_null()); + } + } bufferlist::iterator p = m->bl.begin(); list finished; - - if (m->get_is_primary()) { - // primary link. - add_replica_inode(p, dn, finished); - } else { - // remote link, easy enough. - inodeno_t ino; - __u8 d_type; - ::decode(ino, p); - ::decode(d_type, p); - dir->link_remote_inode(dn, ino, d_type); + if (dn) { + if (m->get_is_primary()) { + // primary link. + add_replica_inode(p, dn, finished); + } else { + // remote link, easy enough. + inodeno_t ino; + __u8 d_type; + ::decode(ino, p); + ::decode(d_type, p); + dir->link_remote_inode(dn, ino, d_type); + } + } else if (m->get_is_primary()) { + CInode *in = add_replica_inode(p, NULL, finished); + assert(in->get_num_ref() == 0); + assert(in->get_parent_dn() == NULL); + MCacheExpire* expire = new MCacheExpire(mds->get_nodeid()); + expire->add_inode(m->get_subtree(), in->vino(), in->get_replica_nonce()); + mds->send_message_mds(expire, m->get_source().num()); + remove_inode(in); } - + if (!finished.empty()) mds->queue_waiters(finished); @@ -9362,6 +9379,11 @@ void MDCache::send_dentry_unlink(CDentry *dn, CDentry *straydn, MDRequest *mdr) /* This function DOES put the passed message before returning */ void MDCache::handle_dentry_unlink(MDentryUnlink *m) { + // straydn + CDentry *straydn = NULL; + if (m->straybl.length()) + straydn = add_replica_stray(m->straybl, m->get_source().num()); + CDir *dir = get_dirfrag(m->get_dirfrag()); if (!dir) { dout(7) << "handle_dentry_unlink don't have dirfrag " << m->get_dirfrag() << dendl; @@ -9373,13 +9395,6 @@ void MDCache::handle_dentry_unlink(MDentryUnlink *m) dout(7) << "handle_dentry_unlink on " << *dn << dendl; CDentry::linkage_t *dnl = dn->get_linkage(); - // straydn - CDentry *straydn = NULL; - if (m->straybl.length()) { - int from = m->get_source().num(); - straydn = add_replica_stray(m->straybl, from); - } - // open inode? if (dnl->is_primary()) { CInode *in = dnl->get_inode(); @@ -9402,8 +9417,9 @@ void MDCache::handle_dentry_unlink(MDentryUnlink *m) migrator->export_caps(in); lru.lru_bottouch(straydn); // move stray to end of lru - + straydn = NULL; } else { + assert(!straydn); assert(dnl->is_remote()); dn->dir->unlink_inode(dn); } @@ -9414,6 +9430,15 @@ void MDCache::handle_dentry_unlink(MDentryUnlink *m) } } + // race with trim_dentry() + if (straydn) { + assert(straydn->get_num_ref() == 0); + assert(straydn->get_linkage()->is_null()); + map expiremap; + trim_dentry(straydn, expiremap); + send_expire_messages(expiremap); + } + m->put(); return; } diff --git a/src/messages/MDentryLink.h b/src/messages/MDentryLink.h index ed02bc20e6bf..b351532799b6 100644 --- a/src/messages/MDentryLink.h +++ b/src/messages/MDentryLink.h @@ -17,11 +17,13 @@ #define CEPH_MDENTRYLINK_H class MDentryLink : public Message { + dirfrag_t subtree; dirfrag_t dirfrag; string dn; bool is_primary; public: + dirfrag_t get_subtree() { return subtree; } dirfrag_t get_dirfrag() { return dirfrag; } string& get_dn() { return dn; } bool get_is_primary() { return is_primary; } @@ -30,8 +32,9 @@ class MDentryLink : public Message { MDentryLink() : Message(MSG_MDS_DENTRYLINK) { } - MDentryLink(dirfrag_t df, string& n, bool p) : + MDentryLink(dirfrag_t r, dirfrag_t df, string& n, bool p) : Message(MSG_MDS_DENTRYLINK), + subtree(r), dirfrag(df), dn(n), is_primary(p) {} @@ -46,12 +49,14 @@ public: void decode_payload() { bufferlist::iterator p = payload.begin(); + ::decode(subtree, p); ::decode(dirfrag, p); ::decode(dn, p); ::decode(is_primary, p); ::decode(bl, p); } void encode_payload(uint64_t features) { + ::encode(subtree, payload); ::encode(dirfrag, payload); ::encode(dn, payload); ::encode(is_primary, payload);