mds: send cap export message when exporting non-auth caps to auth mds

The cap export message makes client open session for the importer mds.
Otherwise client may never open session for the importer mds. The
importer mds can not get cap renew message from the client. It evicts
the client session (opened while importing caps) after a certain time.

Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
Fixes: http://tracker.ceph.com/issues/24052

diff --git a/src/mds/MDSDaemon.h b/src/mds/MDSDaemon.h
index b9d331971c9bf3b4a8482acf904e265c2b47e6c7..8a2cad101edcb4bef54765281ba0b7dea63867e0 100644 (file)
--- a/src/mds/MDSDaemon.h
+++ b/src/mds/MDSDaemon.h
@@ -29,7 +29,7 @@
 #include "MDSMap.h"
 #include "MDSRank.h"
 
-#define CEPH_MDS_PROTOCOL    32 /* cluster internal */
+#define CEPH_MDS_PROTOCOL    33 /* cluster internal */
 
 class AuthAuthorizeHandlerRegistry;
 class Message;

diff --git a/src/mds/Migrator.cc b/src/mds/Migrator.cc
index 6de085257903680f5661686b65c25dbcc339100b..d8ac9154a345c5d2d449bb6cedb381f0874b94c5 100644 (file)
--- a/src/mds/Migrator.cc
+++ b/src/mds/Migrator.cc
@@ -156,6 +156,9 @@ void Migrator::dispatch(Message *m)
   case MSG_MDS_EXPORTCAPS:
     handle_export_caps(static_cast<MExportCaps*>(m));
     break;
+  case MSG_MDS_EXPORTCAPSACK:
+    handle_export_caps_ack(static_cast<MExportCapsAck*>(m));
+    break;
   case MSG_MDS_GATHERCAPS:
     handle_gather_caps(static_cast<MGatherCaps*>(m));
     break;
@@ -3092,10 +3095,16 @@ void Migrator::finish_import_inode_caps(CInode *in, mds_rank_t peer, bool auth_c
        cap->mark_importing();
     }
 
-    Capability::Import& im = import_map[it.first];
-    im.cap_id = cap->get_cap_id();
-    im.mseq = auth_cap ? it.second.mseq : cap->get_mseq();
-    im.issue_seq = cap->get_last_seq() + 1;
+    // Always ask exporter mds to send cap export messages for auth caps.
+    // For non-auth caps, ask exporter mds to send cap export messages to
+    // clients who haven't opened sessions. The cap export messages will
+    // make clients open sessions.
+    if (auth_cap || session->connection == nullptr) {
+      Capability::Import& im = import_map[it.first];
+      im.cap_id = cap->get_cap_id();
+      im.mseq = auth_cap ? it.second.mseq : cap->get_mseq();
+      im.issue_seq = cap->get_last_seq() + 1;
+    }
 
     if (peer >= 0) {
       cap->merge(it.second, auth_cap);
@@ -3301,16 +3310,55 @@ void Migrator::export_caps(CInode *in)
   mds->send_message_mds(ex, dest);
 }
 
+/* This function DOES put the passed message before returning*/
+void Migrator::handle_export_caps_ack(MExportCapsAck *ack)
+{
+  mds_rank_t from = ack->get_source().num();
+  CInode *in = cache->get_inode(ack->ino);
+  if (in) {
+    assert(!in->is_auth());
+
+    dout(10) << "handle_export_caps_ack " << *ack << " from "
+            << ack->get_source() << " on " << *in << dendl;
+
+    map<client_t,Capability::Import> imported_caps;
+    map<client_t,uint64_t> caps_ids;
+    auto blp = ack->cap_bl.cbegin();
+    decode(imported_caps, blp);
+    decode(caps_ids, blp);
+
+    for (auto& it : imported_caps) {
+      Capability *cap = in->get_client_cap(it.first);
+      if (!cap || cap->get_cap_id() != caps_ids.at(it.first))
+       continue;
+
+      dout(7) << __func__ << " telling client." << it.first
+             << " exported caps on " << *in << dendl;
+      MClientCaps *m = new MClientCaps(CEPH_CAP_OP_EXPORT, in->ino(), 0,
+                                      cap->get_cap_id(), cap->get_mseq(),
+                                      mds->get_osd_epoch_barrier());
+      m->set_cap_peer(it.second.cap_id, it.second.issue_seq, it.second.mseq, from, 0);
+      mds->send_message_client_counted(m, it.first);
+
+      in->remove_client_cap(it.first);
+    }
+
+    mds->locker->request_inode_file_caps(in);
+    mds->locker->try_eval(in, CEPH_CAP_LOCKS);
+  }
+
+  ack->put();
+}
+
 void Migrator::handle_gather_caps(MGatherCaps *m)
 {
   CInode *in = cache->get_inode(m->ino);
-
   if (!in)
     goto out;
 
   dout(10) << "handle_gather_caps " << *m << " from " << m->get_source()
-           << " on " << *in
-          << dendl;
+           << " on " << *in << dendl;
+
   if (in->is_any_caps() &&
       !in->is_auth() &&
       !in->is_ambiguous_auth() &&
@@ -3383,14 +3431,25 @@ void Migrator::logged_import_caps(CInode *in,
   // force open client sessions and finish cap import
   mds->server->finish_force_open_sessions(imported_session_map);
 
-  map<client_t,Capability::Import> imported_caps;
-
   auto it = peer_exports.find(in);
   assert(it != peer_exports.end());
 
   // clients will release caps from the exporter when they receive the cap import message.
+  map<client_t,Capability::Import> imported_caps;
   finish_import_inode_caps(in, from, false, imported_session_map, it->second, imported_caps);
   mds->locker->eval(in, CEPH_CAP_LOCKS, true);
+
+  if (!imported_caps.empty()) {
+    MExportCapsAck *ack = new MExportCapsAck(in->ino());
+    map<client_t,uint64_t> peer_caps_ids;
+    for (auto &p : imported_caps )
+      peer_caps_ids[p.first] = it->second.at(p.first).cap_id;
+
+    encode(imported_caps, ack->cap_bl);
+    encode(peer_caps_ids, ack->cap_bl);
+    mds->send_message_mds(ack, from);
+  }
+
   in->auth_unpin(this);
 }
 

diff --git a/src/mds/Migrator.h b/src/mds/Migrator.h
index b9aaa3a19f63feaee8fbbe560a5ae418a4ce614d..232252637683acdf089c9143c0cad8276e1f8401 100644 (file)
--- a/src/mds/Migrator.h
+++ b/src/mds/Migrator.h
@@ -194,6 +194,7 @@ protected:
   void handle_export_finish(MExportDirFinish *m);
 
   void handle_export_caps(MExportCaps *m);
+  void handle_export_caps_ack(MExportCapsAck *m);
   void logged_import_caps(CInode *in,
                          mds_rank_t from,
                          map<client_t,pair<Session*,uint64_t> >& imported_session_map,

diff --git a/src/messages/MExportCapsAck.h b/src/messages/MExportCapsAck.h
index 65891a9a17112bfe18fe601c311d513c8dd2867c..03b89bcd39d49dfd53a3a7ba9a95be6f72580796 100644 (file)
--- a/src/messages/MExportCapsAck.h
+++ b/src/messages/MExportCapsAck.h
@@ -22,6 +22,7 @@
 class MExportCapsAck : public Message {
  public:  
   inodeno_t ino;
+  bufferlist cap_bl;
 
   MExportCapsAck() :
     Message(MSG_MDS_EXPORTCAPSACK) {}
@@ -39,12 +40,13 @@ public:
   void encode_payload(uint64_t features) override {
     using ceph::encode;
     encode(ino, payload);
+    encode(cap_bl, payload);
   }
   void decode_payload() override {
     auto p = payload.cbegin();
     decode(ino, p);
+    decode(cap_bl, p);
   }
-
 };
 
 #endif