From 22bdecd9c76bc30a223406053f2df40e29dbe372 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@newdream.net>
Date: Mon, 1 Feb 2010 11:24:02 -0800
Subject: [PATCH] journal: align (by default) even when !directio; directio by
 default; log latency

---
 src/config.cc         |  5 ++++-
 src/config.h          |  1 +
 src/os/FileJournal.cc | 18 +++++++++++++-----
 src/os/FileJournal.h  |  5 +----
 4 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/src/config.cc b/src/config.cc
index 5ac40a6e870ed..098a355d2da43 100644
--- a/src/config.cc
+++ b/src/config.cc
@@ -539,7 +539,8 @@ static struct config_option config_optionsp[] = {
 	OPTION(ebofs_max_prefetch, 0, OPT_INT, 1000), // 4k blocks
 	OPTION(ebofs_realloc, 0, OPT_BOOL, false),    // hrm, this can cause bad fragmentation, don't use!
 	OPTION(ebofs_verify_csum_on_read, 0, OPT_BOOL, true),
-	OPTION(journal_dio, 0, OPT_BOOL, false),
+	OPTION(journal_dio, 0, OPT_BOOL, true),
+	OPTION(journal_block_align, 0, OPT_BOOL, true),
 	OPTION(journal_max_write_bytes, 0, OPT_INT, 0),
 	OPTION(journal_max_write_entries, 0, OPT_INT, 100),
 	OPTION(bdev_lock, 0, OPT_BOOL, true),
@@ -692,6 +693,8 @@ static bool init_g_conf()
   int i;
   config_option *opt;
 
+  memset(&g_conf, 0, sizeof(g_conf));
+
   for (i = 0; i<len; i++) {
     opt = &config_optionsp[i];
     if (opt->val_ptr) {
diff --git a/src/config.h b/src/config.h
index 3ef85867c3df9..8714d822ec9e2 100644
--- a/src/config.h
+++ b/src/config.h
@@ -355,6 +355,7 @@ struct md_config_t {
   
   // journal
   bool journal_dio;
+  bool journal_block_align;
   int journal_max_write_bytes;
   int journal_max_write_entries;
 
diff --git a/src/os/FileJournal.cc b/src/os/FileJournal.cc
index 72321f2ec9c8b..796db99da45f1 100644
--- a/src/os/FileJournal.cc
+++ b/src/os/FileJournal.cc
@@ -34,7 +34,7 @@ int FileJournal::_open(bool forwrite, bool create)
 
   if (forwrite) {
     flags = O_RDWR;
-    if (directio) flags |= O_DIRECT;
+    if (directio) flags |= O_DIRECT | O_SYNC;
   } else {
     flags = O_RDONLY;
   }
@@ -88,7 +88,9 @@ int FileJournal::_open(bool forwrite, bool create)
   memset(zero_buf, 0, header.alignment);
 
   dout(2) << "_open " << fn << " fd " << fd 
-	  << ": " << max_size << " bytes, block size " << block_size << dendl;
+	  << ": " << max_size 
+	  << " bytes, block size " << block_size
+	  << " bytes, directio = " << directio << dendl;
 
   return 0;
 }
@@ -107,7 +109,7 @@ int FileJournal::create()
   header.fsid = fsid;
   header.max_size = max_size;
   header.block_size = block_size;
-  if (directio)
+  if (g_conf.journal_block_align || directio)
     header.alignment = block_size;
   else
     header.alignment = 16;  // at least stay word aligned on 64bit machines...
@@ -450,6 +452,8 @@ void FileJournal::do_write(bufferlist& bl)
 	   << (hbp.length() ? " + header":"")
 	   << dendl;
   
+  utime_t from = g_clock.now();
+
   // header
   if (hbp.length())
     ::pwrite(fd, hbp.c_str(), hbp.length(), 0);
@@ -484,15 +488,19 @@ void FileJournal::do_write(bufferlist& bl)
     pos += (*it).length();
   }
   if (!directio) {
-    dout(20) << "do_write fsync start" << dendl;
+    dout(20) << "do_write fsync" << dendl;
 #ifdef DARWIN
     ::fsync(fd);
 #else
     ::fdatasync(fd);
+    //::sync_file_range(fd, write_pos, bl.length(),
+    //SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE|SYNC_FILE_RANGE_WAIT_AFTER);
 #endif
-    dout(20) << "do_write fsync finish" << dendl;
   }
 
+  utime_t lat = g_clock.now() - from;    
+  dout(20) << "do_write latency " << lat << dendl;
+
   write_lock.Lock();    
 
   writing = false;
diff --git a/src/os/FileJournal.h b/src/os/FileJournal.h
index c0a95054d585e..60ba57bfe6b1b 100644
--- a/src/os/FileJournal.h
+++ b/src/os/FileJournal.h
@@ -132,10 +132,7 @@ private:
   } write_thread;
 
   off64_t get_top() {
-    if (directio)
-      return block_size;
-    else
-      return sizeof(header);
+    return ROUND_UP_TO(sizeof(header), block_size);
   }
 
  public:
-- 
2.39.5