FileStore: Added O_DSYNC write scheme

lfn_open can now open file handle in O_DSYNC mode based on the
filestore_odsync_write config option. This should help in getting
more stable and higher performance out in case of SSDs configured as
OSD data store.

Signed-off-by: Somnath Roy <somnath.roy@sandisk.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index dfc12af7c26ec096886efd1853eae3b5f85a300b..23ba3e18a2cb29ba7f8e4c4270143b25f4ba8a0c 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -981,6 +981,9 @@ OPTION(filestore_wbthrottle_xfs_inodes_start_flusher, OPT_U64, 500)
 OPTION(filestore_wbthrottle_btrfs_inodes_hard_limit, OPT_U64, 5000)
 OPTION(filestore_wbthrottle_xfs_inodes_hard_limit, OPT_U64, 5000)
 
+//Introduce a O_DSYNC write in the filestore
+OPTION(filestore_odsync_write, OPT_BOOL, false)
+
 // Tests index failure paths
 OPTION(filestore_index_retry_probability, OPT_DOUBLE, 0)
 

diff --git a/src/os/filestore/FileStore.cc b/src/os/filestore/FileStore.cc
index 9d0eeac2e3e2d850dcab70df17be3e65e0db5216..abf2be1eee56b7844ac470541eb827560175d32b 100644 (file)
--- a/src/os/filestore/FileStore.cc
+++ b/src/os/filestore/FileStore.cc
@@ -240,6 +240,9 @@ int FileStore::lfn_open(const coll_t& cid,
 
   if (create)
     flags |= O_CREAT;
+  if (g_conf->filestore_odsync_write) {
+    flags |= O_DSYNC;
+  }
 
   Index index2;
   if (!index) {
@@ -464,7 +467,9 @@ int FileStore::lfn_unlink(const coll_t& cid, const ghobject_t& o,
 
     if (!force_clear_omap) {
       if (hardlink == 0) {
-         wbthrottle.clear_object(o); // should be only non-cache ref
+          if (!m_disable_wbthrottle) {
+           wbthrottle.clear_object(o); // should be only non-cache ref
+          }
          fdcache.clear(o);
          return 0;
       } else if (hardlink == 1) {
@@ -483,7 +488,9 @@ int FileStore::lfn_unlink(const coll_t& cid, const ghobject_t& o,
       if (g_conf->filestore_debug_inject_read_err) {
        debug_obj_on_delete(o);
       }
-      wbthrottle.clear_object(o); // should be only non-cache ref
+      if (!m_disable_wbthrottle) {
+        wbthrottle.clear_object(o); // should be only non-cache ref
+      }
       fdcache.clear(o);
     } else {
       /* Ensure that replay of this op doesn't result in the object_map
@@ -521,6 +528,8 @@ FileStore::FileStore(const std::string &base, const std::string &jdev, osflagbit
   next_osr_id(0),
   throttle_ops(g_conf->filestore_caller_concurrency),
   throttle_bytes(g_conf->filestore_caller_concurrency),
+  m_disable_wbthrottle(g_conf->filestore_odsync_write || 
+                      !g_conf->filestore_wbthrottle_enable),
   m_ondisk_finisher_num(g_conf->filestore_ondisk_finisher_threads),
   m_apply_finisher_num(g_conf->filestore_apply_finisher_threads),
   op_tp(g_ceph_context, "FileStore::op_tp", "tp_fstore_op", g_conf->filestore_op_threads, "filestore_op_threads"),
@@ -752,7 +761,9 @@ void FileStore::create_backend(long f_type)
   switch (f_type) {
 #if defined(__linux__)
   case BTRFS_SUPER_MAGIC:
-    wbthrottle.set_fs(WBThrottle::BTRFS);
+    if (!m_disable_wbthrottle){
+      wbthrottle.set_fs(WBThrottle::BTRFS);
+    }
     break;
 
   case XFS_SUPER_MAGIC:
@@ -1594,8 +1605,14 @@ int FileStore::mount()
       index->cleanup();
     }
   }
-
-  wbthrottle.start();
+  if (!m_disable_wbthrottle) {
+    wbthrottle.start();
+  } else {
+    dout(0) << "mount INFO: WbThrottle is disabled" << dendl;
+    if (g_conf->filestore_odsync_write) {
+      dout(0) << "mount INFO: O_DSYNC write is enabled" << dendl;
+    }
+  }
   sync_thread.create("filestore_sync");
 
   if (!(generic_flags & SKIP_JOURNAL_REPLAY)) {
@@ -1654,8 +1671,9 @@ stop_sync:
   sync_cond.Signal();
   lock.Unlock();
   sync_thread.join();
-
-  wbthrottle.stop();
+  if (!m_disable_wbthrottle) {
+    wbthrottle.stop();
+  }
 close_current_fd:
   VOID_TEMP_FAILURE_RETRY(::close(current_fd));
   current_fd = -1;
@@ -1722,7 +1740,9 @@ int FileStore::umount()
   sync_cond.Signal();
   lock.Unlock();
   sync_thread.join();
-  wbthrottle.stop();
+  if (!m_disable_wbthrottle){
+    wbthrottle.stop();
+  }
   op_tp.stop();
 
   journal_stop();
@@ -1838,7 +1858,9 @@ void FileStore::op_queue_release_throttle(Op *o)
 
 void FileStore::_do_op(OpSequencer *osr, ThreadPool::TPHandle &handle)
 {
-  wbthrottle.throttle();
+  if (!m_disable_wbthrottle) {
+    wbthrottle.throttle();
+  }
   // inject a stall?
   if (g_conf->filestore_inject_stall) {
     int orig = g_conf->filestore_inject_stall;
@@ -3219,12 +3241,16 @@ int FileStore::_write(const coll_t& cid, const ghobject_t& oid,
     int rc = backend->_crc_update_write(**fd, offset, len, bl);
     assert(rc >= 0);
   }
-
-  // flush?
-  if (!replaying &&
-      g_conf->filestore_wbthrottle_enable)
+ 
+  if (replaying || m_disable_wbthrottle) {
+    if (fadvise_flags & CEPH_OSD_OP_FLAG_FADVISE_DONTNEED) {
+        posix_fadvise(**fd, 0, 0, POSIX_FADV_DONTNEED);
+    }
+  } else {
     wbthrottle.queue_wb(fd, oid, offset, len,
-                         fadvise_flags & CEPH_OSD_OP_FLAG_FADVISE_DONTNEED);
+        fadvise_flags & CEPH_OSD_OP_FLAG_FADVISE_DONTNEED);
+  }
+ 
   lfn_close(fd);
 
  out:
@@ -3732,7 +3758,9 @@ void FileStore::sync_entry()
       logger->tinc(l_os_commit_len, dur);
 
       apply_manager.commit_finish();
-      wbthrottle.clear();
+      if (!m_disable_wbthrottle) {
+        wbthrottle.clear();
+      }
 
       logger->set(l_os_committing, 0);
 

diff --git a/src/os/filestore/FileStore.h b/src/os/filestore/FileStore.h
index 9450d73d8047b7d9f68dfe4753be52782cdd3ed5..d81f8b0691b02917216157902eb63f04eb7baab8 100644 (file)
--- a/src/os/filestore/FileStore.h
+++ b/src/os/filestore/FileStore.h
@@ -340,6 +340,7 @@ private:
   WBThrottle wbthrottle;
 
   atomic_t next_osr_id;
+  bool m_disable_wbthrottle;
   deque<OpSequencer*> op_queue;
   BackoffThrottle throttle_ops, throttle_bytes;
   const int m_ondisk_finisher_num;