From: Yan, Zheng Date: Wed, 22 Jun 2016 12:34:41 +0000 (+0800) Subject: mds: using cap_reconnect_t to track cap recconect X-Git-Tag: v10.2.3~114^2~12 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=0897fc4f15804a25a667bf7e495dc4ff1209127b;p=ceph.git mds: using cap_reconnect_t to track cap recconect Previous commit extended cap_reconnect_t to include snapflush related information. This information are needed in various places Signed-off-by: Yan, Zheng (cherry picked from commit 25b6011dcf07c1fc663cbe29ffd119b66545a0ac) --- diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc index df0b2bbeb49a6..0168b56581142 100644 --- a/src/mds/CInode.cc +++ b/src/mds/CInode.cc @@ -2817,18 +2817,18 @@ void CInode::move_to_realm(SnapRealm *realm) containing_realm = realm; } -Capability *CInode::reconnect_cap(client_t client, ceph_mds_cap_reconnect& icr, Session *session) +Capability *CInode::reconnect_cap(client_t client, const cap_reconnect_t& icr, Session *session) { Capability *cap = get_client_cap(client); if (cap) { // FIXME? - cap->merge(icr.wanted, icr.issued); + cap->merge(icr.capinfo.wanted, icr.capinfo.issued); } else { cap = add_client_cap(client, session); - cap->set_wanted(icr.wanted); - cap->issue_norevoke(icr.issued); + cap->set_cap_id(icr.capinfo.cap_id); + cap->set_wanted(icr.capinfo.wanted); + cap->issue_norevoke(icr.capinfo.issued); cap->reset_seq(); - cap->set_cap_id(icr.cap_id); } cap->set_last_issue_stamp(ceph_clock_now(g_ceph_context)); return cap; diff --git a/src/mds/CInode.h b/src/mds/CInode.h index 217257cca90fa..4181191341e20 100644 --- a/src/mds/CInode.h +++ b/src/mds/CInode.h @@ -987,7 +987,7 @@ public: void remove_client_cap(client_t client); void move_to_realm(SnapRealm *realm); - Capability *reconnect_cap(client_t client, ceph_mds_cap_reconnect& icr, Session *session); + Capability *reconnect_cap(client_t client, const cap_reconnect_t& icr, Session *session); void clear_client_caps_after_export(); void export_client_caps(std::map& cl); diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc index 101b3820db6e4..a8523f0b4c067 100644 --- a/src/mds/MDCache.cc +++ b/src/mds/MDCache.cc @@ -3937,17 +3937,13 @@ void MDCache::rejoin_send_rejoins() if (mds->is_rejoin()) { map > client_exports; - for (map >::iterator p = cap_exports.begin(); - p != cap_exports.end(); - ++p) { + for (auto p = cap_exports.begin(); p != cap_exports.end(); ++p) { assert(cap_export_targets.count(p->first)); mds_rank_t target = cap_export_targets[p->first]; if (rejoins.count(target) == 0) continue; rejoins[target]->cap_exports[p->first] = p->second; - for (map::iterator q = p->second.begin(); - q != p->second.end(); - ++q) + for (auto q = p->second.begin(); q != p->second.end(); ++q) client_exports[q->first].insert(target); } for (map >::iterator p = client_exports.begin(); @@ -4274,14 +4270,10 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak) map > imported_caps; // check cap exports - for (map >::iterator p = weak->cap_exports.begin(); - p != weak->cap_exports.end(); - ++p) { + for (auto p = weak->cap_exports.begin(); p != weak->cap_exports.end(); ++p) { CInode *in = get_inode(p->first); assert(!in || in->is_auth()); - for (map::iterator q = p->second.begin(); - q != p->second.end(); - ++q) { + for (auto q = p->second.begin(); q != p->second.end(); ++q) { dout(10) << " claiming cap import " << p->first << " client." << q->first << " on " << *in << dendl; Capability *cap = rejoin_import_cap(in, q->first, q->second, from); Capability::Import& im = imported_caps[p->first][q->first]; @@ -4303,15 +4295,11 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak) // check cap exports. rejoin_client_map.insert(weak->client_map.begin(), weak->client_map.end()); - for (map >::iterator p = weak->cap_exports.begin(); - p != weak->cap_exports.end(); - ++p) { + for (auto p = weak->cap_exports.begin(); p != weak->cap_exports.end(); ++p) { CInode *in = get_inode(p->first); assert(in && in->is_auth()); // note - for (map::iterator q = p->second.begin(); - q != p->second.end(); - ++q) { + for (auto q = p->second.begin(); q != p->second.end(); ++q) { dout(10) << " claiming cap import " << p->first << " client." << q->first << dendl; cap_imports[p->first][q->first][from] = q->second; } @@ -5066,7 +5054,7 @@ void MDCache::handle_cache_rejoin_ack(MMDSCacheRejoin *ack) // mark client caps stale. MClientCaps *m = new MClientCaps(CEPH_CAP_OP_EXPORT, p->first, 0, - cap_exports[p->first][q->first].cap_id, 0, + cap_exports[p->first][q->first].capinfo.cap_id, 0, mds->get_osd_epoch_barrier()); m->set_cap_peer(q->second.cap_id, q->second.issue_seq, q->second.mseq, from, 0); mds->send_message_client_counted(m, session); @@ -5206,12 +5194,9 @@ void MDCache::rejoin_open_ino_finish(inodeno_t ino, int ret) } else if (ret == mds->get_nodeid()) { assert(get_inode(ino)); } else { - map > >::iterator p; - p = cap_imports.find(ino); + auto p = cap_imports.find(ino); assert(p != cap_imports.end()); - for (map >::iterator q = p->second.begin(); - q != p->second.end(); - ++q) { + for (auto q = p->second.begin(); q != p->second.end(); ++q) { assert(q->second.count(MDS_RANK_NONE)); assert(q->second.size() == 1); rejoin_export_caps(p->first, q->first, q->second[MDS_RANK_NONE], ret); @@ -5256,9 +5241,7 @@ bool MDCache::process_imported_caps() { dout(10) << "process_imported_caps" << dendl; - for (map > >::iterator p = cap_imports.begin(); - p != cap_imports.end(); - ++p) { + for (auto p = cap_imports.begin(); p != cap_imports.end(); ++p) { CInode *in = get_inode(p->first); if (in) { assert(in->is_auth()); @@ -5325,8 +5308,7 @@ bool MDCache::process_imported_caps() // process cap imports // ino -> client -> frommds -> capex - for (map > >::iterator p = cap_imports.begin(); - p != cap_imports.end(); ) { + for (auto p = cap_imports.begin(); p != cap_imports.end(); ) { CInode *in = get_inode(p->first); if (!in) { dout(10) << " still missing ino " << p->first @@ -5335,20 +5317,16 @@ bool MDCache::process_imported_caps() continue; } assert(in->is_auth()); - for (map >::iterator q = p->second.begin(); - q != p->second.end(); - ++q) { + for (auto q = p->second.begin(); q != p->second.end(); ++q) { Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(q->first.v)); assert(session); - for (map::iterator r = q->second.begin(); - r != q->second.end(); - ++r) { - add_reconnected_cap(in, q->first, inodeno_t(r->second.snaprealm)); + for (auto r = q->second.begin(); r != q->second.end(); ++r) { + add_reconnected_cap(in, q->first, inodeno_t(r->second.capinfo.snaprealm)); Capability *cap = in->reconnect_cap(q->first, r->second, session); if (r->first >= 0) { if (cap->get_last_seq() == 0) // don't increase mseq if cap already exists cap->inc_mseq(); - do_cap_import(session, in, cap, r->second.cap_id, 0, 0, r->first, 0); + do_cap_import(session, in, cap, r->second.capinfo.cap_id, 0, 0, r->first, 0); Capability::Import& im = rejoin_imported_caps[r->first][p->first][q->first]; im.cap_id = cap->get_cap_id(); @@ -5522,7 +5500,7 @@ void MDCache::clean_open_file_lists() -Capability* MDCache::rejoin_import_cap(CInode *in, client_t client, ceph_mds_cap_reconnect& icr, mds_rank_t frommds) +Capability* MDCache::rejoin_import_cap(CInode *in, client_t client, const cap_reconnect_t& icr, mds_rank_t frommds) { dout(10) << "rejoin_import_cap for client." << client << " from mds." << frommds << " on " << *in << dendl; @@ -5534,7 +5512,7 @@ Capability* MDCache::rejoin_import_cap(CInode *in, client_t client, ceph_mds_cap if (frommds >= 0) { if (cap->get_last_seq() == 0) // don't increase mseq if cap already exists cap->inc_mseq(); - do_cap_import(session, in, cap, icr.cap_id, 0, 0, frommds, 0); + do_cap_import(session, in, cap, icr.capinfo.cap_id, 0, 0, frommds, 0); } return cap; @@ -5546,13 +5524,9 @@ void MDCache::export_remaining_imported_caps() stringstream warn_str; - for (map > >::iterator p = cap_imports.begin(); - p != cap_imports.end(); - ++p) { + for (auto p = cap_imports.begin(); p != cap_imports.end(); ++p) { warn_str << " ino " << p->first << "\n"; - for (map >::iterator q = p->second.begin(); - q != p->second.end(); - ++q) { + for (auto q = p->second.begin(); q != p->second.end(); ++q) { Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(q->first.v)); if (session) { // mark client caps stale. @@ -5581,12 +5555,12 @@ void MDCache::export_remaining_imported_caps() void MDCache::try_reconnect_cap(CInode *in, Session *session) { client_t client = session->info.get_client(); - ceph_mds_cap_reconnect *rc = get_replay_cap_reconnect(in->ino(), client); + const cap_reconnect_t *rc = get_replay_cap_reconnect(in->ino(), client); if (rc) { in->reconnect_cap(client, *rc, session); dout(10) << "try_reconnect_cap client." << client - << " reconnect wanted " << ccap_string(rc->wanted) - << " issue " << ccap_string(rc->issued) + << " reconnect wanted " << ccap_string(rc->capinfo.wanted) + << " issue " << ccap_string(rc->capinfo.issued) << " on " << *in << dendl; remove_replay_cap_reconnect(in->ino(), client); diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h index 5aadc13981ee3..d60e8f35c5192 100644 --- a/src/mds/MDCache.h +++ b/src/mds/MDCache.h @@ -490,10 +490,10 @@ protected: map > > rejoin_slave_exports; map rejoin_client_map; - map > cap_exports; // ino -> client -> capex + map > cap_exports; // ino -> client -> capex map cap_export_targets; // ino -> auth mds - map > > cap_imports; // ino -> client -> frommds -> capex + map > > cap_imports; // ino -> client -> frommds -> capex map cap_imports_dirty; set cap_imports_missing; map > cap_reconnect_waiters; @@ -530,16 +530,16 @@ public: void rejoin_start(MDSInternalContext *rejoin_done_); void rejoin_gather_finish(); void rejoin_send_rejoins(); - void rejoin_export_caps(inodeno_t ino, client_t client, ceph_mds_cap_reconnect& capinfo, + void rejoin_export_caps(inodeno_t ino, client_t client, const cap_reconnect_t& icr, int target=-1) { - cap_exports[ino][client] = capinfo; + cap_exports[ino][client] = icr; cap_export_targets[ino] = target; } - void rejoin_recovered_caps(inodeno_t ino, client_t client, cap_reconnect_t& icr, + void rejoin_recovered_caps(inodeno_t ino, client_t client, const cap_reconnect_t& icr, mds_rank_t frommds=MDS_RANK_NONE) { - cap_imports[ino][client][frommds] = icr.capinfo; + cap_imports[ino][client][frommds] = icr; } - ceph_mds_cap_reconnect *get_replay_cap_reconnect(inodeno_t ino, client_t client) { + const cap_reconnect_t *get_replay_cap_reconnect(inodeno_t ino, client_t client) { if (cap_imports.count(ino) && cap_imports[ino].count(client) && cap_imports[ino][client].count(MDS_RANK_NONE)) { @@ -581,7 +581,7 @@ public: map& splits); void do_realm_invalidate_and_update_notify(CInode *in, int snapop, bool nosend=false); void send_snaps(map& splits); - Capability* rejoin_import_cap(CInode *in, client_t client, ceph_mds_cap_reconnect& icr, mds_rank_t frommds); + Capability* rejoin_import_cap(CInode *in, client_t client, const cap_reconnect_t& icr, mds_rank_t frommds); void finish_snaprealm_reconnect(client_t client, SnapRealm *realm, snapid_t seq); void try_reconnect_cap(CInode *in, Session *session); void export_remaining_imported_caps(); diff --git a/src/mds/MDSDaemon.h b/src/mds/MDSDaemon.h index 4d6296c5df2ab..b23e24a042d2f 100644 --- a/src/mds/MDSDaemon.h +++ b/src/mds/MDSDaemon.h @@ -39,7 +39,7 @@ #include "Beacon.h" -#define CEPH_MDS_PROTOCOL 27 /* cluster internal */ +#define CEPH_MDS_PROTOCOL 28 /* cluster internal */ class filepath; diff --git a/src/mds/Server.cc b/src/mds/Server.cc index 313744718b0b1..0246833d518aa 100644 --- a/src/mds/Server.cc +++ b/src/mds/Server.cc @@ -809,7 +809,7 @@ void Server::handle_client_reconnect(MClientReconnect *m) // we recovered it, and it's ours. take note. dout(15) << "open cap realm " << inodeno_t(p->second.capinfo.snaprealm) << " on " << *in << dendl; - in->reconnect_cap(from, p->second.capinfo, session); + in->reconnect_cap(from, p->second, session); mdcache->add_reconnected_cap(in, from, inodeno_t(p->second.capinfo.snaprealm)); recover_filelocks(in, p->second.flockbl, m->get_orig_source().num()); continue; @@ -819,11 +819,13 @@ void Server::handle_client_reconnect(MClientReconnect *m) // not mine. dout(10) << "non-auth " << *in << ", will pass off to authority" << dendl; // add to cap export list. - mdcache->rejoin_export_caps(p->first, from, p->second.capinfo, + p->second.path.clear(); // we don't need path + mdcache->rejoin_export_caps(p->first, from, p->second, in->authority().first); } else { // don't know if the inode is mine dout(10) << "missing ino " << p->first << ", will load later" << dendl; + p->second.path.clear(); // we don't need path mdcache->rejoin_recovered_caps(p->first, from, p->second, MDS_RANK_NONE); } } diff --git a/src/messages/MMDSCacheRejoin.h b/src/messages/MMDSCacheRejoin.h index 111a24460176f..4bb47eb284c54 100644 --- a/src/messages/MMDSCacheRejoin.h +++ b/src/messages/MMDSCacheRejoin.h @@ -21,6 +21,7 @@ #include "mds/CInode.h" #include "mds/CDir.h" +#include "mds/mdstypes.h" // sent from replica to auth @@ -163,7 +164,7 @@ class MMDSCacheRejoin : public Message { map strong_inodes; // open - map > cap_exports; + map > cap_exports; map client_map; bufferlist imported_caps;