mds: send cap export message when exporting non-auth caps to auth mds

The cap export message makes client open session for the importer mds.
Otherwise client may never open session for the importer mds. The
importer mds can not get cap renew message from the client. It evicts
the client session (opened while importing caps) after a certain time.

Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
Fixes: http://tracker.ceph.com/issues/24052
(cherry picked from commit 7385b37272af6ce07c6e5649cd8ea00644df7173)
Conflicts:
	src/mds/MDSDaemon.h
	src/mds/Migrator.cc
	src/messages/MExportCapsAck.h

diff --git a/src/mds/MDSDaemon.h b/src/mds/MDSDaemon.h
index 8dac42c1b6263b80bb96504b751092f8e3561c11..68e8a0c5593551ebf235e0eb793e210a5257664d 100644 (file)
--- a/src/mds/MDSDaemon.h
+++ b/src/mds/MDSDaemon.h
@@ -29,7 +29,7 @@
 #include "MDSMap.h"
 #include "MDSRank.h"
 
-#define CEPH_MDS_PROTOCOL    30 /* cluster internal */
+#define CEPH_MDS_PROTOCOL    31 /* cluster internal */
 
 class AuthAuthorizeHandlerRegistry;
 class Message;

diff --git a/src/mds/Migrator.cc b/src/mds/Migrator.cc
index 40a89626bc798710342ca357424d873370d4ecf8..7fbd1375ef3caf213ab1489548ccffac53b69a3f 100644 (file)
--- a/src/mds/Migrator.cc
+++ b/src/mds/Migrator.cc
@@ -156,6 +156,9 @@ void Migrator::dispatch(Message *m)
   case MSG_MDS_EXPORTCAPS:
     handle_export_caps(static_cast<MExportCaps*>(m));
     break;
+  case MSG_MDS_EXPORTCAPSACK:
+    handle_export_caps_ack(static_cast<MExportCapsAck*>(m));
+    break;
   case MSG_MDS_GATHERCAPS:
     handle_gather_caps(static_cast<MGatherCaps*>(m));
     break;
@@ -3090,10 +3093,16 @@ void Migrator::finish_import_inode_caps(CInode *in, mds_rank_t peer, bool auth_c
        cap->mark_importing();
     }
 
-    Capability::Import& im = import_map[it.first];
-    im.cap_id = cap->get_cap_id();
-    im.mseq = auth_cap ? it.second.mseq : cap->get_mseq();
-    im.issue_seq = cap->get_last_seq() + 1;
+    // Always ask exporter mds to send cap export messages for auth caps.
+    // For non-auth caps, ask exporter mds to send cap export messages to
+    // clients who haven't opened sessions. The cap export messages will
+    // make clients open sessions.
+    if (auth_cap || session->connection == nullptr) {
+      Capability::Import& im = import_map[it.first];
+      im.cap_id = cap->get_cap_id();
+      im.mseq = auth_cap ? it.second.mseq : cap->get_mseq();
+      im.issue_seq = cap->get_last_seq() + 1;
+    }
 
     if (peer >= 0) {
       cap->merge(it.second, auth_cap);
@@ -3299,16 +3308,55 @@ void Migrator::export_caps(CInode *in)
   mds->send_message_mds(ex, dest);
 }
 
+/* This function DOES put the passed message before returning*/
+void Migrator::handle_export_caps_ack(MExportCapsAck *ack)
+{
+  mds_rank_t from = ack->get_source().num();
+  CInode *in = cache->get_inode(ack->ino);
+  if (in) {
+    assert(!in->is_auth());
+
+    dout(10) << "handle_export_caps_ack " << *ack << " from "
+            << ack->get_source() << " on " << *in << dendl;
+
+    map<client_t,Capability::Import> imported_caps;
+    map<client_t,uint64_t> caps_ids;
+    auto blp = ack->cap_bl.begin();
+    ::decode(imported_caps, blp);
+    ::decode(caps_ids, blp);
+
+    for (auto& it : imported_caps) {
+      Capability *cap = in->get_client_cap(it.first);
+      if (!cap || cap->get_cap_id() != caps_ids.at(it.first))
+       continue;
+
+      dout(7) << __func__ << " telling client." << it.first
+             << " exported caps on " << *in << dendl;
+      MClientCaps *m = new MClientCaps(CEPH_CAP_OP_EXPORT, in->ino(), 0,
+                                      cap->get_cap_id(), cap->get_mseq(),
+                                      mds->get_osd_epoch_barrier());
+      m->set_cap_peer(it.second.cap_id, it.second.issue_seq, it.second.mseq, from, 0);
+      mds->send_message_client_counted(m, it.first);
+
+      in->remove_client_cap(it.first);
+    }
+
+    mds->locker->request_inode_file_caps(in);
+    mds->locker->try_eval(in, CEPH_CAP_LOCKS);
+  }
+
+  ack->put();
+}
+
 void Migrator::handle_gather_caps(MGatherCaps *m)
 {
   CInode *in = cache->get_inode(m->ino);
-
   if (!in)
     goto out;
 
   dout(10) << "handle_gather_caps " << *m << " from " << m->get_source()
-           << " on " << *in
-          << dendl;
+           << " on " << *in << dendl;
+
   if (in->is_any_caps() &&
       !in->is_auth() &&
       !in->is_ambiguous_auth() &&
@@ -3384,14 +3432,25 @@ void Migrator::logged_import_caps(CInode *in,
   // force open client sessions and finish cap import
   mds->server->finish_force_open_sessions(imported_session_map);
 
-  map<client_t,Capability::Import> imported_caps;
-
   auto it = peer_exports.find(in);
   assert(it != peer_exports.end());
 
   // clients will release caps from the exporter when they receive the cap import message.
+  map<client_t,Capability::Import> imported_caps;
   finish_import_inode_caps(in, from, false, imported_session_map, it->second, imported_caps);
   mds->locker->eval(in, CEPH_CAP_LOCKS, true);
+
+  if (!imported_caps.empty()) {
+    MExportCapsAck *ack = new MExportCapsAck(in->ino());
+    map<client_t,uint64_t> peer_caps_ids;
+    for (auto &p : imported_caps )
+      peer_caps_ids[p.first] = it->second.at(p.first).cap_id;
+
+    ::encode(imported_caps, ack->cap_bl);
+    ::encode(peer_caps_ids, ack->cap_bl);
+    mds->send_message_mds(ack, from);
+  }
+
   in->auth_unpin(this);
 }
 

diff --git a/src/mds/Migrator.h b/src/mds/Migrator.h
index 148b2fb4fd2c01422109e9f36455929c5269a291..92f3913a8ad6d5066fddd9417acafaaff7352fcf 100644 (file)
--- a/src/mds/Migrator.h
+++ b/src/mds/Migrator.h
@@ -194,6 +194,7 @@ protected:
   void handle_export_finish(MExportDirFinish *m);
 
   void handle_export_caps(MExportCaps *m);
+  void handle_export_caps_ack(MExportCapsAck *m);
   void logged_import_caps(CInode *in,
                          mds_rank_t from,
                          map<client_t,pair<Session*,uint64_t> >& imported_session_map,

diff --git a/src/messages/MExportCapsAck.h b/src/messages/MExportCapsAck.h
index 5a21d9465de31fc866b4bd726301e691bfb6491f..74286e6ac84e922b0d7f6294b0ec69ca7e433d54 100644 (file)
--- a/src/messages/MExportCapsAck.h
+++ b/src/messages/MExportCapsAck.h
@@ -22,6 +22,7 @@
 class MExportCapsAck : public Message {
  public:  
   inodeno_t ino;
+  bufferlist cap_bl;
 
   MExportCapsAck() :
     Message(MSG_MDS_EXPORTCAPSACK) {}
@@ -38,12 +39,13 @@ public:
 
   void encode_payload(uint64_t features) override {
     ::encode(ino, payload);
+    ::encode(cap_bl, payload);
   }
   void decode_payload() override {
-    bufferlist::iterator p = payload.begin();
+    auto p = payload.begin();
     ::decode(ino, p);
+    ::decode(cap_bl, p);
   }
-
 };
 
 #endif