client: make fsync wait for unsafe directory operations

Signed-off-by: Yan, Zheng <zyan@redhat.com>

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 3dc71646f55baa051a970c86f6c8591becc9a4c8..a1305566a48cbca3f25563f898de65948a8c5af1 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -2006,6 +2006,17 @@ void Client::handle_client_request_forward(MClientRequestForward *fwd)
   fwd->put();
 }
 
+bool Client::is_dir_operation(MetaRequest *req)
+{
+  int op = req->get_op();
+  if (op == CEPH_MDS_OP_MKNOD || op == CEPH_MDS_OP_LINK ||
+      op == CEPH_MDS_OP_UNLINK || op == CEPH_MDS_OP_RENAME ||
+      op == CEPH_MDS_OP_MKDIR || op == CEPH_MDS_OP_RMDIR ||
+      op == CEPH_MDS_OP_SYMLINK || op == CEPH_MDS_OP_CREATE)
+    return true;
+  return false;
+}
+
 void Client::handle_client_reply(MClientReply *reply)
 {
   mds_rank_t mds_num = mds_rank_t(reply->get_source().num());
@@ -2071,6 +2082,11 @@ void Client::handle_client_reply(MClientReply *reply)
   if (!is_safe) {
     request->got_unsafe = true;
     session->unsafe_requests.push_back(&request->unsafe_item);
+    if (is_dir_operation(request)) {
+      Inode *dir = request->inode();
+      assert(dir);
+      dir->unsafe_dir_ops.push_back(&request->unsafe_dir_item);
+    }
   }
 
   // Only signal the caller once (on the first reply):
@@ -2095,6 +2111,8 @@ void Client::handle_client_reply(MClientReply *reply)
     // we're done, clean up
     if (request->got_unsafe) {
       request->unsafe_item.remove_myself();
+      request->unsafe_dir_item.remove_myself();
+      signal_cond_list(request->waitfor_safe);
     }
     request->item.remove_myself();
     mds_requests.erase(tid);
@@ -2468,6 +2486,8 @@ void Client::kick_requests_closed(MetaSession *session)
       if (req->got_unsafe) {
        lderr(cct) << "kick_requests_closed removing unsafe request " << req->get_tid() << dendl;
        req->unsafe_item.remove_myself();
+       req->unsafe_dir_item.remove_myself();
+       signal_cond_list(req->waitfor_safe);
        mds_requests.erase(req->get_tid());
        put_request(req);
       }
@@ -7738,6 +7758,21 @@ int Client::_fsync(Fh *f, bool syncdataonly)
     }
   }
 
+  if (!in->unsafe_dir_ops.empty()) {
+    MetaRequest *req = in->unsafe_dir_ops.back();
+    uint64_t last_tid = req->get_tid();
+    ldout(cct, 15) << "waiting on unsafe requests, last tid " << last_tid <<  dendl;
+
+    do {
+      req->get();
+      wait_on_list(req->waitfor_safe);
+      put_request(req);
+      if (in->unsafe_dir_ops.empty())
+       break;
+      req = in->unsafe_dir_ops.front();
+    } while (req->tid < last_tid);
+  }
+
   if (!r) {
     if (flushed_metadata)
       wait_sync_caps(in, wait_on_flush);
@@ -9545,6 +9580,7 @@ int Client::_rmdir(Inode *dir, const char *name, int uid, int gid)
   if (res < 0)
     goto fail;
   if (req->get_op() == CEPH_MDS_OP_RMDIR) {
+    req->set_inode(dir);
     req->set_dentry(de);
     req->set_other_inode(in);
   } else {

diff --git a/src/client/Client.h b/src/client/Client.h
index 4485ff82266baf9fbba76664b7cea87ecf17541d..b1ac162b693adb94d21df9bf4f43fb64277afb38 100644 (file)
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -319,6 +319,7 @@ public:
   void kick_requests_closed(MetaSession *session);
   void handle_client_request_forward(MClientRequestForward *reply);
   void handle_client_reply(MClientReply *reply);
+  bool is_dir_operation(MetaRequest *request);
 
   bool   initialized;
   bool   authenticated;

diff --git a/src/client/Inode.h b/src/client/Inode.h
index 4b2e32872dcac59bf76ea18d207086e6c74c96a2..04a8215a1c94c79b40a5b3d4ad22d34fab251f55 100644 (file)
--- a/src/client/Inode.h
+++ b/src/client/Inode.h
@@ -19,6 +19,7 @@ class Dir;
 struct SnapRealm;
 struct Inode;
 class ceph_lock_state_t;
+class MetaRequest;
 
 struct Cap {
   MetaSession *session;
@@ -300,6 +301,8 @@ struct Inode {
   ceph_lock_state_t *fcntl_locks;
   ceph_lock_state_t *flock_locks;
 
+  xlist<MetaRequest*> unsafe_dir_ops;
+
   Inode(CephContext *cct_, vinodeno_t vino, ceph_file_layout *newlayout)
     : cct(cct_), ino(vino.ino), snapid(vino.snapid),
       rdev(0), mode(0), uid(0), gid(0), nlink(0),

diff --git a/src/client/MetaRequest.h b/src/client/MetaRequest.h
index f580b99ab3986659bcb7f1e38a2ade8d7f6aab75..378a61e60d7f25ff6bdc103374decb48ce706ed0 100644 (file)
--- a/src/client/MetaRequest.h
+++ b/src/client/MetaRequest.h
@@ -72,10 +72,12 @@ public:
 
   xlist<MetaRequest*>::item item;
   xlist<MetaRequest*>::item unsafe_item;
+  xlist<MetaRequest*>::item unsafe_dir_item;
   Mutex lock; //for get/set sync
 
   Cond  *caller_cond;          // who to take up
   Cond  *dispatch_cond;        // who to kick back
+  list<Cond*> waitfor_safe;
 
   Inode *target;
 
@@ -94,7 +96,7 @@ public:
     ref(1), reply(0), 
     kick(false), aborted(false), success(false),
     readdir_offset(0), readdir_end(false), readdir_num(0),
-    got_unsafe(false), item(this), unsafe_item(this),
+    got_unsafe(false), item(this), unsafe_item(this), unsafe_dir_item(this),
     lock("MetaRequest lock"),
     caller_cond(0), dispatch_cond(0),
     target(0) {

diff --git a/src/client/fuse_ll.cc b/src/client/fuse_ll.cc
index f9c380c9cb4fde8556fad576baa754bc7654fec3..72ac98d23f6acda0a6dd0532300123058864e261 100644 (file)
--- a/src/client/fuse_ll.cc
+++ b/src/client/fuse_ll.cc
@@ -817,7 +817,7 @@ const static struct fuse_lowlevel_ops fuse_ll_oper = {
  opendir: fuse_ll_opendir,
  readdir: fuse_ll_readdir,
  releasedir: fuse_ll_releasedir,
- fsyncdir: 0,
+ fsyncdir: fuse_ll_fsync,
  statfs: fuse_ll_statfs,
  setxattr: fuse_ll_setxattr,
  getxattr: fuse_ll_getxattr,