From b14ced7bce3373a154695e9971b27b4d7e9488e8 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <ifedotov@mirantis.com>
Date: Fri, 26 May 2017 15:33:05 +0300
Subject: [PATCH] os/bluestore: keep statfs replica in memory to avoid
 expensive KV access

Signed-off-by: Igor Fedotov <ifedotov@mirantis.com>
---
 src/os/bluestore/BlueStore.cc | 52 ++++++++++++++++++++++-------------
 src/os/bluestore/BlueStore.h  | 11 ++++++++
 2 files changed, 44 insertions(+), 19 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 34e424c1f835d..26bd3dac20e14 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -4678,6 +4678,24 @@ int BlueStore::_open_collections(int *errors)
   return 0;
 }
 
+void BlueStore::open_statfs()
+{
+  bufferlist bl;
+  int r = db->get(PREFIX_STAT, "bluestore_statfs", &bl);
+  if (r >= 0) {
+    if (size_t(bl.length()) >= sizeof(vstatfs.values)) {
+      auto it = bl.begin();
+      vstatfs.decode(it);
+    }
+    else {
+      dout(10) << __func__ << " store_statfs is corrupt, using empty" << dendl;
+    }
+  }
+  else {
+    dout(10) << __func__ << " store_statfs missed, using empty" << dendl;
+  }
+}
+
 int BlueStore::_setup_block_symlink_or_file(
   string name,
   string epath,
@@ -5792,27 +5810,16 @@ int BlueStore::statfs(struct store_statfs_t *buf)
     }
   }
 
-  bufferlist bl;
-  int r = db->get(PREFIX_STAT, "bluestore_statfs", &bl);
-  if (r >= 0) {
-     volatile_statfs vstatfs;
-     if (size_t(bl.length()) >= sizeof(vstatfs.values)) {
-       auto it = bl.begin();
-       vstatfs.decode(it);
-
-       buf->allocated = vstatfs.allocated();
-       buf->stored = vstatfs.stored();
-       buf->compressed = vstatfs.compressed();
-       buf->compressed_original = vstatfs.compressed_original();
-       buf->compressed_allocated = vstatfs.compressed_allocated();
-     } else {
-       dout(10) << __func__ << " store_statfs is corrupt, using empty" << dendl;
-     }
-  } else {
-    dout(10) << __func__ << " store_statfs missed, using empty" << dendl;
+  {
+    std::lock_guard<std::mutex> l(vstatfs_lock);
+    
+    buf->allocated = vstatfs.allocated();
+    buf->stored = vstatfs.stored();
+    buf->compressed = vstatfs.compressed();
+    buf->compressed_original = vstatfs.compressed_original();
+    buf->compressed_allocated = vstatfs.compressed_allocated();
   }
 
-
   dout(20) << __func__ << *buf << dendl;
   return 0;
 }
@@ -7370,6 +7377,7 @@ int BlueStore::_open_super_meta()
     dout(10) << __func__ << " min_alloc_size 0x" << std::hex << min_alloc_size
 	     << std::dec << dendl;
   }
+  open_statfs();
   _set_alloc_sizes();
   _set_throttle_params();
 
@@ -7483,6 +7491,11 @@ void BlueStore::_txc_update_store_statfs(TransContext *txc)
   logger->inc(l_bluestore_compressed_allocated, txc->statfs_delta.compressed_allocated());
   logger->inc(l_bluestore_compressed_original, txc->statfs_delta.compressed_original());
 
+  {
+    std::lock_guard<std::mutex> l(vstatfs_lock);
+    vstatfs += txc->statfs_delta;
+  }
+
   bufferlist bl;
   txc->statfs_delta.encode(bl);
 
@@ -8318,6 +8331,7 @@ void BlueStore::_kv_finalize_thread()
 	_txc_state_proc(txc);
 	kv_committed.pop_front();
       }
+
       for (auto b : deferred_stable) {
 	auto p = b->txcs.begin();
 	while (p != b->txcs.end()) {
diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 66c8a26faf7ac..83fe45d275217 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -1408,6 +1408,12 @@ public:
     void reset() {
       *this = volatile_statfs();
     }
+    volatile_statfs& operator+=(const volatile_statfs& other) {
+      for (size_t i = 0; i < STATFS_LAST; ++i) {
+	values[i] += other.values[i];
+      }
+      return *this;
+    }
     int64_t& allocated() {
       return values[STATFS_ALLOCATED];
     }
@@ -1896,6 +1902,9 @@ private:
 
   // cache trim control
 
+  std::mutex vstatfs_lock;
+  volatile_statfs vstatfs;
+
   struct MempoolThread : public Thread {
     BlueStore *store;
     Cond cond;
@@ -1960,6 +1969,8 @@ private:
 
   int _open_super_meta();
 
+  void open_statfs();
+
   int _reconcile_bluefs_freespace();
   int _balance_bluefs_freespace(PExtentVector *extents);
   void _commit_bluefs_freespace(const PExtentVector& extents);
-- 
2.39.5