From: Yan, Zheng Date: Wed, 16 May 2018 09:54:01 +0000 (+0800) Subject: mds: send cap export message when exporting non-auth caps to auth mds X-Git-Tag: v13.2.1~76^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=a9bce4106317b9f41ee7f3ec6b175c8db4fa2321;p=ceph.git mds: send cap export message when exporting non-auth caps to auth mds The cap export message makes client open session for the importer mds. Otherwise client may never open session for the importer mds. The importer mds can not get cap renew message from the client. It evicts the client session (opened while importing caps) after a certain time. Signed-off-by: "Yan, Zheng" Fixes: http://tracker.ceph.com/issues/24052 (cherry picked from commit 7385b37272af6ce07c6e5649cd8ea00644df7173) Conflicts: src/mds/Migrator.cc - replace cbegin() with begin() as per https://tracker.ceph.com/issues/24296#note-3 --- diff --git a/src/mds/MDSDaemon.h b/src/mds/MDSDaemon.h index b9d331971c9b..8a2cad101edc 100644 --- a/src/mds/MDSDaemon.h +++ b/src/mds/MDSDaemon.h @@ -29,7 +29,7 @@ #include "MDSMap.h" #include "MDSRank.h" -#define CEPH_MDS_PROTOCOL 32 /* cluster internal */ +#define CEPH_MDS_PROTOCOL 33 /* cluster internal */ class AuthAuthorizeHandlerRegistry; class Message; diff --git a/src/mds/Migrator.cc b/src/mds/Migrator.cc index b7e1b03ea885..35f1e60008cb 100644 --- a/src/mds/Migrator.cc +++ b/src/mds/Migrator.cc @@ -156,6 +156,9 @@ void Migrator::dispatch(Message *m) case MSG_MDS_EXPORTCAPS: handle_export_caps(static_cast(m)); break; + case MSG_MDS_EXPORTCAPSACK: + handle_export_caps_ack(static_cast(m)); + break; case MSG_MDS_GATHERCAPS: handle_gather_caps(static_cast(m)); break; @@ -3092,10 +3095,16 @@ void Migrator::finish_import_inode_caps(CInode *in, mds_rank_t peer, bool auth_c cap->mark_importing(); } - Capability::Import& im = import_map[it.first]; - im.cap_id = cap->get_cap_id(); - im.mseq = auth_cap ? it.second.mseq : cap->get_mseq(); - im.issue_seq = cap->get_last_seq() + 1; + // Always ask exporter mds to send cap export messages for auth caps. + // For non-auth caps, ask exporter mds to send cap export messages to + // clients who haven't opened sessions. The cap export messages will + // make clients open sessions. + if (auth_cap || session->connection == nullptr) { + Capability::Import& im = import_map[it.first]; + im.cap_id = cap->get_cap_id(); + im.mseq = auth_cap ? it.second.mseq : cap->get_mseq(); + im.issue_seq = cap->get_last_seq() + 1; + } if (peer >= 0) { cap->merge(it.second, auth_cap); @@ -3301,16 +3310,55 @@ void Migrator::export_caps(CInode *in) mds->send_message_mds(ex, dest); } +/* This function DOES put the passed message before returning*/ +void Migrator::handle_export_caps_ack(MExportCapsAck *ack) +{ + mds_rank_t from = ack->get_source().num(); + CInode *in = cache->get_inode(ack->ino); + if (in) { + assert(!in->is_auth()); + + dout(10) << "handle_export_caps_ack " << *ack << " from " + << ack->get_source() << " on " << *in << dendl; + + map imported_caps; + map caps_ids; + auto blp = ack->cap_bl.begin(); + decode(imported_caps, blp); + decode(caps_ids, blp); + + for (auto& it : imported_caps) { + Capability *cap = in->get_client_cap(it.first); + if (!cap || cap->get_cap_id() != caps_ids.at(it.first)) + continue; + + dout(7) << __func__ << " telling client." << it.first + << " exported caps on " << *in << dendl; + MClientCaps *m = new MClientCaps(CEPH_CAP_OP_EXPORT, in->ino(), 0, + cap->get_cap_id(), cap->get_mseq(), + mds->get_osd_epoch_barrier()); + m->set_cap_peer(it.second.cap_id, it.second.issue_seq, it.second.mseq, from, 0); + mds->send_message_client_counted(m, it.first); + + in->remove_client_cap(it.first); + } + + mds->locker->request_inode_file_caps(in); + mds->locker->try_eval(in, CEPH_CAP_LOCKS); + } + + ack->put(); +} + void Migrator::handle_gather_caps(MGatherCaps *m) { CInode *in = cache->get_inode(m->ino); - if (!in) goto out; dout(10) << "handle_gather_caps " << *m << " from " << m->get_source() - << " on " << *in - << dendl; + << " on " << *in << dendl; + if (in->is_any_caps() && !in->is_auth() && !in->is_ambiguous_auth() && @@ -3386,14 +3434,25 @@ void Migrator::logged_import_caps(CInode *in, // force open client sessions and finish cap import mds->server->finish_force_open_sessions(imported_session_map); - map imported_caps; - auto it = peer_exports.find(in); assert(it != peer_exports.end()); // clients will release caps from the exporter when they receive the cap import message. + map imported_caps; finish_import_inode_caps(in, from, false, imported_session_map, it->second, imported_caps); mds->locker->eval(in, CEPH_CAP_LOCKS, true); + + if (!imported_caps.empty()) { + MExportCapsAck *ack = new MExportCapsAck(in->ino()); + map peer_caps_ids; + for (auto &p : imported_caps ) + peer_caps_ids[p.first] = it->second.at(p.first).cap_id; + + encode(imported_caps, ack->cap_bl); + encode(peer_caps_ids, ack->cap_bl); + mds->send_message_mds(ack, from); + } + in->auth_unpin(this); } diff --git a/src/mds/Migrator.h b/src/mds/Migrator.h index 148b2fb4fd2c..92f3913a8ad6 100644 --- a/src/mds/Migrator.h +++ b/src/mds/Migrator.h @@ -194,6 +194,7 @@ protected: void handle_export_finish(MExportDirFinish *m); void handle_export_caps(MExportCaps *m); + void handle_export_caps_ack(MExportCapsAck *m); void logged_import_caps(CInode *in, mds_rank_t from, map >& imported_session_map, diff --git a/src/messages/MExportCapsAck.h b/src/messages/MExportCapsAck.h index c11178f58fbd..d2039b43a05c 100644 --- a/src/messages/MExportCapsAck.h +++ b/src/messages/MExportCapsAck.h @@ -22,6 +22,7 @@ class MExportCapsAck : public Message { public: inodeno_t ino; + bufferlist cap_bl; MExportCapsAck() : Message(MSG_MDS_EXPORTCAPSACK) {} @@ -39,12 +40,13 @@ public: void encode_payload(uint64_t features) override { using ceph::encode; encode(ino, payload); + encode(cap_bl, payload); } void decode_payload() override { bufferlist::iterator p = payload.begin(); decode(ino, p); + decode(cap_bl, p); } - }; #endif