client: implement fsync() for cache-disabled mode

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 4cee7310b05dd9c87b073be7597a972a9137372b..3eb5b4a23d73bfdb29861e76506c26caac9adc6e 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -3077,29 +3077,43 @@ int Client::_read(Fh *f, __s64 offset, __u64 size, bufferlist *bl)
 
 
 /*
- * hack -- 
- *  until we properly implement synchronous writes wrt buffer cache,
- *  make sure we delay shutdown until they're all safe on disk!
+ * we keep count of uncommitted sync writes on the inode, so that
+ * fsync can DDRT.
  */
-class C_Client_HackUnsafe : public Context {
+class C_Client_SyncCommit : public Context {
   Client *cl;
+  Inode *in;
 public:
-  C_Client_HackUnsafe(Client *c) : cl(c) {}
+  C_Client_SyncCommit(Client *c, Inode *i) : cl(c), in(i) {}
   void finish(int) {
-    cl->hack_sync_write_safe();
+    cl->sync_write_commit(in);
   }
 };
 
-void Client::hack_sync_write_safe()
+void Client::sync_write_commit(Inode *in)
 {
   client_lock.Lock();
   assert(unsafe_sync_write > 0);
   unsafe_sync_write--;
-  dout(15) << "hack_sync_write_safe unsafe_sync_write = " << unsafe_sync_write << dendl;
+  in->uncommitted_writes--;
+
+  if (in->uncommitted_writes == 0) {
+    dout(15) << "sync_write_commit all sync writes committed on ino " << in->inode.ino << dendl;
+    for (list<Cond*>::iterator p = in->waitfor_commit.begin();
+        p != in->waitfor_commit.end();
+        p++)
+      (*p)->Signal();
+    in->waitfor_commit.clear();
+  }
+
+  dout(15) << "sync_write_commit unsafe_sync_write = " << unsafe_sync_write << dendl;
   if (unsafe_sync_write == 0 && unmounting) {
-    dout(10) << "hack_sync_write_safe -- no more unsafe writes, unmount can proceed" << dendl;
+    dout(10) << "sync_write_comit -- no more unsafe writes, unmount can proceed" << dendl;
     mount_cond.Signal();
   }
+
+  put_inode(in);
+
   client_lock.Unlock();
 }
 
@@ -3175,11 +3189,17 @@ int Client::_write(Fh *f, __s64 offset, __u64 size, const char *buf)
       cond.Wait(client_lock);
     }
 
+    // avoid livelock with fsync
+    if (in->uncommitted_writes > 0 &&
+       !in->waitfor_commit.empty())
+      _fsync(f, true);
+
     // prepare write
     Cond cond;
     bool done = false;
-    C_Cond *onfinish = new C_Cond(&cond, &done);
-    C_Client_HackUnsafe *onsafe = new C_Client_HackUnsafe(this);
+    Context *onfinish = new C_Cond(&cond, &done);
+    in->get();
+    Context *onsafe = new C_Client_SyncCommit(this, in);
     unsafe_sync_write++;
     in->sync_writes++;
     
@@ -3310,13 +3330,23 @@ int Client::_fsync(Fh *f, bool syncdataonly)
     dout(0) << "fsync - not syncing metadata yet.. implement me" << dendl;
   }
 
-  // data?
-  Cond cond;
-  bool done = false;
-  if (!objectcacher->commit_set(in->ino(),
-                                new C_Cond(&cond, &done))) {
-    // wait for callback
-    while (!done) cond.Wait(client_lock);
+  if (g_conf.client_oc) {
+    // data?
+    Cond cond;
+    bool done = false;
+    if (!objectcacher->commit_set(in->ino(),
+                                 new C_Cond(&cond, &done))) {
+      // wait for callback
+      while (!done) cond.Wait(client_lock);
+    }
+  } else {
+    while (in->uncommitted_writes > 0) {
+      Cond cond;
+      dout(10) << "ino " << in->inode.ino << " has " << in->uncommitted_writes << " uncommitted, waiting" << dendl;
+      in->waitfor_commit.push_back(&cond);
+      cond.Wait(client_lock);
+    }    
+    dout(10) << "ino " << in->inode.ino << " has no uncommitted writes" << dendl;
   }
   return r;
 }

diff --git a/src/client/Client.h b/src/client/Client.h
index 30d2ec7a16776558c551e09f576737711e732426..401ff4400545fbe776f70c490fa99303e2ca6662 100644 (file)
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -164,10 +164,12 @@ class Inode {
   // for sync i/o mode
   int       sync_reads;   // sync reads in progress
   int       sync_writes;  // sync writes in progress
+  int       uncommitted_writes;  // sync writes missing commits
 
   list<Cond*>       waitfor_write;
   list<Cond*>       waitfor_read;
   list<Cond*>       waitfor_lazy;
+  list<Cond*>       waitfor_commit;
   list<Context*>    waitfor_no_read, waitfor_no_write;
 
   // <hack>
@@ -208,7 +210,7 @@ class Inode {
     ref(0), ll_ref(0), 
     dir(0), dn(0), symlink(0),
     fc(_oc, ino, layout),
-    sync_reads(0), sync_writes(0),
+    sync_reads(0), sync_writes(0), uncommitted_writes(0),
     hack_balance_reads(false)
   {
     inode.ino = ino;
@@ -531,7 +533,7 @@ public:
   int    unsafe_sync_write;
 public:
   entity_name_t get_myname() { return messenger->get_myname(); } 
-  void hack_sync_write_safe();
+  void sync_write_commit(Inode *in);
 
 protected:
   Filer                 *filer;     

diff --git a/src/client/SyntheticClient.cc b/src/client/SyntheticClient.cc
index 4b29600ce61b3284702140c2c2ef14404df6d74f..bd683308c1c9c813978545b666ea971cec7a41a4 100644 (file)
--- a/src/client/SyntheticClient.cc
+++ b/src/client/SyntheticClient.cc
@@ -1778,7 +1778,7 @@ int SyntheticClient::write_file(string& fn, int size, int wrsize)   // size is i
   sleep(5);
   // END temporary hack piece 2 --Esteban
 #endif
-
+  client->fsync(fd, true);
   client->close(fd);
   delete[] buf;