client: use finisher to abort MDS request

When a request is interrupted, libfuse first locks an internal mutex,
then calls the interrupt callback. libfuse need to lock the same mutex
when unregistering interrupt callback. We unregister interrupt callback
while client_lock is locked, so we can't acquiring the client_lock in
the interrupt callback.

Signed-off-by: Yan, Zheng <zyan@redhat.com>

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 2299913fe7fa5a01d5989905ff856eba620e1440..40ea02fe4248ac06f89d95b97dbd58744c506af1 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -165,6 +165,7 @@ Client::Client(Messenger *m, MonClient *mc)
     getgroups_cb_handle(NULL),
     async_ino_invalidator(m->cct),
     async_dentry_invalidator(m->cct),
+    interrupt_finisher(m->cct),
     objecter_finisher(m->cct),
     tick_event(NULL),
     monclient(mc), messenger(m), whoami(m->get_myname().num()),
@@ -456,6 +457,12 @@ void Client::shutdown()
     async_dentry_invalidator.stop();
   }
 
+  if (switch_interrupt_cb) {
+    ldout(cct, 10) << "shutdown stopping interrupt finisher" << dendl;
+    interrupt_finisher.wait_for_empty();
+    interrupt_finisher.stop();
+  }
+
   objectcacher->stop();  // outside of client_lock! this does a join.
 
   client_lock.Lock();
@@ -7612,6 +7619,7 @@ void Client::ll_register_switch_interrupt_cb(client_switch_interrupt_callback_t
   if (cb == NULL)
     return;
   switch_interrupt_cb = cb;
+  interrupt_finisher.start();
 }
 
 void Client::ll_register_getgroups_cb(client_getgroups_callback_t cb, void *handle)
@@ -9666,16 +9674,28 @@ int Client::ll_flock(Fh *fh, int cmd, uint64_t owner, void *fuse_req)
   return _flock(fh, cmd, owner, fuse_req);
 }
 
+class C_Client_RequestInterrupt : public Context  {
+private:
+  Client *client;
+  MetaRequest *req;
+public:
+  C_Client_RequestInterrupt(Client *c, MetaRequest *r) : client(c), req(r) {
+    req->get();
+  }
+  void finish(int r) {
+    Mutex::Locker l(client->client_lock);
+    assert(req->head.op == CEPH_MDS_OP_SETFILELOCK);
+    client->_interrupt_filelock(req);
+    client->put_request(req);
+  }
+};
+
 void Client::ll_interrupt(void *d)
 {
-  Mutex::Locker lock(client_lock);
-
   MetaRequest *req = static_cast<MetaRequest*>(d);
   ldout(cct, 3) << "ll_interrupt tid " << req->get_tid() << dendl;
   tout(cct) << "ll_interrupt tid " << req->get_tid() << std::endl;
-
-  assert(req->head.op == CEPH_MDS_OP_SETFILELOCK);
-  _interrupt_filelock(req);
+  interrupt_finisher.queue(new C_Client_RequestInterrupt(this, req));
 }
 
 // =========================================

diff --git a/src/client/Client.h b/src/client/Client.h
index f4582a5581d3d9d155f002c8945e7e036e7be1c2..7d69a72857601d2a9e03b9f19804733f24bcdab7 100644 (file)
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -238,6 +238,7 @@ class Client : public Dispatcher {
 
   Finisher async_ino_invalidator;
   Finisher async_dentry_invalidator;
+  Finisher interrupt_finisher;
   Finisher objecter_finisher;
 
   Context *tick_event;
@@ -403,6 +404,7 @@ protected:
   friend class C_Block_Sync; // Calls block map and protected helpers
   friend class C_C_Tick; // Asserts on client_lock
   friend class C_Client_SyncCommit; // Asserts on client_lock
+  friend class C_Client_RequestInterrupt;
 
   //int get_cache_size() { return lru.lru_get_size(); }
   //void set_cache_size(int m) { lru.lru_set_max(m); }