From 1cf4ff41c6449dd78df4ef5f5e234c1a1c6da3b6 Mon Sep 17 00:00:00 2001
From: Yuan Zhou <yuan.zhou@intel.com>
Date: Thu, 20 Dec 2018 16:20:37 +0800
Subject: [PATCH] tools: clean up immutable object cache

- drop "ceph_" prefix
- prefix all options with immutable_object_cache_
- adding configurable immutable_object_cache_max_inflight_ops

Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
---
 src/common/options.cc                         | 24 ++++-----
 .../CMakeLists.txt                            |  0
 .../test_DomainSocket.cc                      |  4 +-
 .../test_SimplePolicy.cc                      |  2 +-
 .../test_main.cc                              |  0
 .../test_sync_file.cc                         | 30 ++++++------
 src/tools/CMakeLists.txt                      |  2 +-
 .../CMakeLists.txt                            |  0
 .../CacheClient.cc                            | 49 ++++++++++++-------
 .../CacheClient.h                             |  1 -
 .../CacheController.cc                        |  2 +-
 .../CacheController.h                         |  3 --
 .../CacheServer.cc                            | 11 +++--
 .../CacheServer.h                             |  0
 .../CacheSession.cc                           | 19 ++++---
 .../CacheSession.h                            | 10 ++--
 .../ObjectCacheFile.cc                        | 22 ++++-----
 .../ObjectCacheFile.h                         | 14 +++---
 .../ObjectCacheStore.cc                       | 17 +++----
 .../ObjectCacheStore.h                        | 10 ++--
 .../Policy.h                                  |  4 +-
 .../SimplePolicy.cc                           |  9 ++--
 .../SimplePolicy.h                            |  9 ++--
 .../SocketCommon.h                            |  0
 .../main.cc                                   |  9 ++--
 25 files changed, 137 insertions(+), 114 deletions(-)
 rename src/test/{ceph_immutable_object_cache => immutable_object_cache}/CMakeLists.txt (100%)
 rename src/test/{ceph_immutable_object_cache => immutable_object_cache}/test_DomainSocket.cc (97%)
 rename src/test/{ceph_immutable_object_cache => immutable_object_cache}/test_SimplePolicy.cc (99%)
 rename src/test/{ceph_immutable_object_cache => immutable_object_cache}/test_main.cc (100%)
 rename src/test/{ceph_immutable_object_cache => immutable_object_cache}/test_sync_file.cc (76%)
 rename src/tools/{ceph_immutable_object_cache => immutable_object_cache}/CMakeLists.txt (100%)
 rename src/tools/{ceph_immutable_object_cache => immutable_object_cache}/CacheClient.cc (77%)
 rename src/tools/{ceph_immutable_object_cache => immutable_object_cache}/CacheClient.h (98%)
 rename src/tools/{ceph_immutable_object_cache => immutable_object_cache}/CacheController.cc (98%)
 rename src/tools/{ceph_immutable_object_cache => immutable_object_cache}/CacheController.h (92%)
 rename src/tools/{ceph_immutable_object_cache => immutable_object_cache}/CacheServer.cc (87%)
 rename src/tools/{ceph_immutable_object_cache => immutable_object_cache}/CacheServer.h (100%)
 rename src/tools/{ceph_immutable_object_cache => immutable_object_cache}/CacheSession.cc (82%)
 rename src/tools/{ceph_immutable_object_cache => immutable_object_cache}/CacheSession.h (74%)
 rename src/tools/{ceph_immutable_object_cache => immutable_object_cache}/ObjectCacheFile.cc (74%)
 rename src/tools/{ceph_immutable_object_cache => immutable_object_cache}/ObjectCacheFile.h (78%)
 rename src/tools/{ceph_immutable_object_cache => immutable_object_cache}/ObjectCacheStore.cc (95%)
 rename src/tools/{ceph_immutable_object_cache => immutable_object_cache}/ObjectCacheStore.h (92%)
 rename src/tools/{ceph_immutable_object_cache => immutable_object_cache}/Policy.h (92%)
 rename src/tools/{ceph_immutable_object_cache => immutable_object_cache}/SimplePolicy.cc (94%)
 rename src/tools/{ceph_immutable_object_cache => immutable_object_cache}/SimplePolicy.h (88%)
 rename src/tools/{ceph_immutable_object_cache => immutable_object_cache}/SocketCommon.h (100%)
 rename src/tools/{ceph_immutable_object_cache => immutable_object_cache}/main.cc (90%)

diff --git a/src/common/options.cc b/src/common/options.cc
index be3d63fa113..3f3c795e018 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -7035,21 +7035,21 @@ static std::vector<Option> get_rbd_options() {
     .set_default(false)
     .set_description("whether to block writes to the cache before the aio_write call completes"),
 
-    Option("rbd_shared_cache_enabled", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
-    .set_default(false)
-    .set_description("whether to enable shared ssd caching"),
-
-    Option("rbd_shared_cache_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
+    Option("immutable_object_cache_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
     .set_default("/tmp")
-    .set_description("shared ssd caching data dir"),
+    .set_description("immutable object cache data dir"),
 
-    Option("rbd_shared_cache_sock", Option::TYPE_STR, Option::LEVEL_ADVANCED)
-    .set_default("/tmp/rbd_shared_ro_cache_sock")
-    .set_description("shared ssd caching domain socket"),
+    Option("immutable_object_cache_sock", Option::TYPE_STR, Option::LEVEL_ADVANCED)
+    .set_default("/var/run/ceph/immutable_object_cache_sock")
+    .set_description("immutable object cache domain socket"),
 
-    Option("rbd_shared_cache_entries", Option::TYPE_INT, Option::LEVEL_ADVANCED)
-    .set_default(4096)
-    .set_description("shared ssd caching data entries"),
+    Option("immutable_object_cache_max_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
+    .set_default(1_G)
+    .set_description("max immutable object cache data size"),
+
+    Option("immutable_object_cache_max_inflight_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
+    .set_default(128)
+    .set_description("max immutable object caching inflight ops"),
 
     Option("rbd_concurrent_management_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
     .set_default(10)
diff --git a/src/test/ceph_immutable_object_cache/CMakeLists.txt b/src/test/immutable_object_cache/CMakeLists.txt
similarity index 100%
rename from src/test/ceph_immutable_object_cache/CMakeLists.txt
rename to src/test/immutable_object_cache/CMakeLists.txt
diff --git a/src/test/ceph_immutable_object_cache/test_DomainSocket.cc b/src/test/immutable_object_cache/test_DomainSocket.cc
similarity index 97%
rename from src/test/ceph_immutable_object_cache/test_DomainSocket.cc
rename to src/test/immutable_object_cache/test_DomainSocket.cc
index e28109011f1..a7ae75c3a9d 100644
--- a/src/test/ceph_immutable_object_cache/test_DomainSocket.cc
+++ b/src/test/immutable_object_cache/test_DomainSocket.cc
@@ -11,8 +11,8 @@
 #include "global/global_init.h"
 #include "global/global_context.h"
 
-#include "tools/ceph_immutable_object_cache/CacheClient.h"
-#include "tools/ceph_immutable_object_cache/CacheServer.h"
+#include "tools/immutable_object_cache/CacheClient.h"
+#include "tools/immutable_object_cache/CacheServer.h"
 
 using namespace ceph::immutable_obj_cache;
 
diff --git a/src/test/ceph_immutable_object_cache/test_SimplePolicy.cc b/src/test/immutable_object_cache/test_SimplePolicy.cc
similarity index 99%
rename from src/test/ceph_immutable_object_cache/test_SimplePolicy.cc
rename to src/test/immutable_object_cache/test_SimplePolicy.cc
index f83a7e10f9f..808b09b7ad7 100644
--- a/src/test/ceph_immutable_object_cache/test_SimplePolicy.cc
+++ b/src/test/immutable_object_cache/test_SimplePolicy.cc
@@ -5,7 +5,7 @@
 #include <list>
 #include <gtest/gtest.h>
 
-#include "tools/ceph_immutable_object_cache/SimplePolicy.h"
+#include "tools/immutable_object_cache/SimplePolicy.h"
 
 using namespace ceph::immutable_obj_cache;
 
diff --git a/src/test/ceph_immutable_object_cache/test_main.cc b/src/test/immutable_object_cache/test_main.cc
similarity index 100%
rename from src/test/ceph_immutable_object_cache/test_main.cc
rename to src/test/immutable_object_cache/test_main.cc
diff --git a/src/test/ceph_immutable_object_cache/test_sync_file.cc b/src/test/immutable_object_cache/test_sync_file.cc
similarity index 76%
rename from src/test/ceph_immutable_object_cache/test_sync_file.cc
rename to src/test/immutable_object_cache/test_sync_file.cc
index 19352d409e3..f4b196b42ed 100644
--- a/src/test/ceph_immutable_object_cache/test_sync_file.cc
+++ b/src/test/immutable_object_cache/test_sync_file.cc
@@ -11,22 +11,22 @@
 #include "global/global_context.h"
 #include <experimental/filesystem>
 
-#include "tools/ceph_immutable_object_cache/ObjectCacheFile.h"
+#include "tools/immutable_object_cache/ObjectCacheFile.h"
 
 using namespace ceph::immutable_obj_cache;
 namespace efs = std::experimental::filesystem;
 
-class TestSyncFile :public ::testing::Test {
+class TestObjectCacheFile :public ::testing::Test {
 public:
   std::string m_cache_root_dir;
 
-  TestSyncFile(){}
-  ~TestSyncFile(){}
+  TestObjectCacheFile(){}
+  ~TestObjectCacheFile(){}
   static void SetUpTestCase() {}
   static void TearDownTestCase() {}
 
   void SetUp() override {
-    m_cache_root_dir = g_ceph_context->_conf.get_val<std::string>("rbd_shared_cache_path")
+    m_cache_root_dir = g_ceph_context->_conf.get_val<std::string>("immutable_object_cache_path")
       + "/ceph_immutable_obj_cache/";
 
     if (efs::exists(m_cache_root_dir)) {
@@ -41,25 +41,25 @@ public:
 
 };
 
-TEST_F(TestSyncFile, test_create_file) {
-  SyncFile* m_sync_file = new SyncFile(g_ceph_context, "test_sync_file");
+TEST_F(TestObjectCacheFile, test_create_file) {
+  ObjectCacheFile* m_sync_file = new ObjectCacheFile(g_ceph_context, "test_sync_file");
   ASSERT_TRUE(m_sync_file->create() >  0);
   ASSERT_TRUE(m_sync_file->get_file_size() == 0);
   delete m_sync_file;
 }
 
 
-TEST_F(TestSyncFile, test_open_file) {
-  SyncFile* m_sync_file = new SyncFile(g_ceph_context, "test_sync_file");
+TEST_F(TestObjectCacheFile, test_open_file) {
+  ObjectCacheFile* m_sync_file = new ObjectCacheFile(g_ceph_context, "test_sync_file");
   ASSERT_EQ(m_sync_file->open_file(), -1);
   ASSERT_GT(m_sync_file->create(), 0);
   ASSERT_GT(m_sync_file->open_file(), 0);
   delete m_sync_file;
 }
 
-TEST_F(TestSyncFile, test_write_object_to_file) {
-  SyncFile* m_sync_file_1 = new SyncFile(g_ceph_context, "test_sync_file_1");
-  SyncFile* m_sync_file_2 = new SyncFile(g_ceph_context, "test_sync_file_2");
+TEST_F(TestObjectCacheFile, test_write_object_to_file) {
+  ObjectCacheFile* m_sync_file_1 = new ObjectCacheFile(g_ceph_context, "test_sync_file_1");
+  ObjectCacheFile* m_sync_file_2 = new ObjectCacheFile(g_ceph_context, "test_sync_file_2");
   ASSERT_GT(m_sync_file_1->create(), 0);
   ASSERT_GT(m_sync_file_2->create(), 0);
   ASSERT_TRUE(m_sync_file_1->get_file_size() == 0);
@@ -78,9 +78,9 @@ TEST_F(TestSyncFile, test_write_object_to_file) {
   delete buf_2;
 }
 
-TEST_F(TestSyncFile, test_read_object_from_file) {
-  SyncFile* m_sync_file_1 = new SyncFile(g_ceph_context, "test_sync_file_1");
-  SyncFile* m_sync_file_2 = new SyncFile(g_ceph_context, "test_sync_file_2");
+TEST_F(TestObjectCacheFile, test_read_object_from_file) {
+  ObjectCacheFile* m_sync_file_1 = new ObjectCacheFile(g_ceph_context, "test_sync_file_1");
+  ObjectCacheFile* m_sync_file_2 = new ObjectCacheFile(g_ceph_context, "test_sync_file_2");
   bufferlist* buf_1 = new ceph::bufferlist();
   bufferlist* buf_2 = new ceph::bufferlist();
 
diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt
index f8c53a3334d..6c99795b968 100644
--- a/src/tools/CMakeLists.txt
+++ b/src/tools/CMakeLists.txt
@@ -122,5 +122,5 @@ if(WITH_RBD)
   endif()
 endif(WITH_RBD)
 
-add_subdirectory(ceph_immutable_object_cache)
+add_subdirectory(immutable_object_cache)
 add_subdirectory(ceph-dencoder)
diff --git a/src/tools/ceph_immutable_object_cache/CMakeLists.txt b/src/tools/immutable_object_cache/CMakeLists.txt
similarity index 100%
rename from src/tools/ceph_immutable_object_cache/CMakeLists.txt
rename to src/tools/immutable_object_cache/CMakeLists.txt
diff --git a/src/tools/ceph_immutable_object_cache/CacheClient.cc b/src/tools/immutable_object_cache/CacheClient.cc
similarity index 77%
rename from src/tools/ceph_immutable_object_cache/CacheClient.cc
rename to src/tools/immutable_object_cache/CacheClient.cc
index 7a54c70fcb6..c3070a8f533 100644
--- a/src/tools/ceph_immutable_object_cache/CacheClient.cc
+++ b/src/tools/immutable_object_cache/CacheClient.cc
@@ -6,7 +6,7 @@
 #define dout_context g_ceph_context
 #define dout_subsys ceph_subsys_immutable_obj_cache
 #undef dout_prefix
-#define dout_prefix *_dout << "ceph::cache::CacheControllerSocketClient: " << this << " " \
+#define dout_prefix *_dout << "ceph::cache::CacheClient: " << this << " " \
                            << __func__ << ": "
 
 
@@ -54,7 +54,8 @@ namespace immutable_obj_cache {
     if(close_ec) {
        ldout(cct, 20) << "close: " << close_ec.message() << dendl;
     }
-    ldout(cct, 20) << "session don't work, later all request will be dispatched to rados layer" << dendl;
+    ldout(cct, 20) << "session don't work, later all request will be" <<
+                      " dispatched to rados layer" << dendl;
   }
 
   int CacheClient::connect() {
@@ -62,8 +63,9 @@ namespace immutable_obj_cache {
     m_dm_socket.connect(m_ep, ec);
     if(ec) {
       if(ec == boost::asio::error::connection_refused) {
-        ldout(cct, 20) << ec.message() << " : maybe rbd-cache Controller don't startup. "
-                  << "Now data will be read from ceph cluster " << dendl;
+        ldout(cct, 20) << ec.message()
+                       << " : immutable-object-cache daemon is down?"
+                       << "Now data will be read from ceph cluster " << dendl;
       } else {
         ldout(cct, 20) << "connect: " << ec.message() << dendl;
       }
@@ -86,7 +88,8 @@ namespace immutable_obj_cache {
     return 0;
   }
 
-  int CacheClient::register_volume(std::string pool_name, std::string vol_name, uint64_t vol_size, Context* on_finish) {
+  int CacheClient::register_volume(std::string pool_name, std::string vol_name,
+                                   uint64_t vol_size, Context* on_finish) {
     // cache controller will init layout
     rbdsc_req_type_t *message = new rbdsc_req_type_t();
     message->type = RBDSC_REGISTER;
@@ -99,7 +102,9 @@ namespace immutable_obj_cache {
     uint64_t ret;
     boost::system::error_code ec;
 
-    ret = boost::asio::write(m_dm_socket, boost::asio::buffer((char*)message, message->size()), ec);
+    ret = boost::asio::write(m_dm_socket,
+      boost::asio::buffer((char*)message, message->size()), ec);
+
     if(ec) {
       ldout(cct, 20) << "write fails : " << ec.message() << dendl;
       return -1;
@@ -111,7 +116,9 @@ namespace immutable_obj_cache {
     }
 
     // hard code TODO
-    ret = boost::asio::read(m_dm_socket, boost::asio::buffer(m_recv_buffer, RBDSC_MSG_LEN), ec);
+    ret = boost::asio::read(m_dm_socket,
+      boost::asio::buffer(m_recv_buffer, RBDSC_MSG_LEN), ec);
+
     if(ec == boost::asio::error::eof) {
       ldout(cct, 20) << "recv eof" << dendl;
       return -1;
@@ -147,7 +154,8 @@ namespace immutable_obj_cache {
   }
 
   // if occur any error, we just return false. Then read from rados.
-  int CacheClient::lookup_object(std::string pool_name, std::string vol_name, std::string object_id, Context* on_finish) {
+  int CacheClient::lookup_object(std::string pool_name, std::string vol_name,
+                                 std::string object_id, Context* on_finish) {
     rbdsc_req_type_t *message = new rbdsc_req_type_t();
     message->type = RBDSC_READ;
     memcpy(message->pool_name, pool_name.c_str(), pool_name.size());
@@ -158,18 +166,19 @@ namespace immutable_obj_cache {
     message->length = 0;
 
     boost::asio::async_write(m_dm_socket,
-                             boost::asio::buffer((char*)message, message->size()),
-                             boost::asio::transfer_exactly(RBDSC_MSG_LEN),
-        [this, on_finish, message](const boost::system::error_code& err, size_t cb) {
+      boost::asio::buffer((char*)message, message->size()),
+      boost::asio::transfer_exactly(RBDSC_MSG_LEN),
+      [this, on_finish, message](const boost::system::error_code& err, size_t cb) {
           delete message;
           if(err) {
-            ldout(cct, 20) << "lookup_object: async_write fails." << err.message() << dendl;
+            ldout(cct, 20) << "async_write failed"
+                           << err.message() << dendl;
             close();
             on_finish->complete(false);
             return;
           }
           if(cb != RBDSC_MSG_LEN) {
-            ldout(cct, 20) << "lookup_object: async_write fails. in-complete request" << dendl;
+            ldout(cct, 20) << "async_write failed in-complete request" << dendl;
             close();
             on_finish->complete(false);
             return;
@@ -182,14 +191,17 @@ namespace immutable_obj_cache {
 
   void CacheClient::get_result(Context* on_finish) {
     char* lookup_result = new char[RBDSC_MSG_LEN + 1];
-    boost::asio::async_read(m_dm_socket, boost::asio::buffer(lookup_result, RBDSC_MSG_LEN),
-                            boost::asio::transfer_exactly(RBDSC_MSG_LEN),
-        [this, lookup_result, on_finish](const boost::system::error_code& err, size_t cb) {
+    boost::asio::async_read(m_dm_socket,
+      boost::asio::buffer(lookup_result, RBDSC_MSG_LEN),
+      boost::asio::transfer_exactly(RBDSC_MSG_LEN),
+      [this, lookup_result, on_finish](const boost::system::error_code& err,
+                                       size_t cb) {
           if(err == boost::asio::error::eof ||
             err == boost::asio::error::connection_reset ||
             err == boost::asio::error::operation_aborted ||
             err == boost::asio::error::bad_descriptor) {
-            ldout(cct, 20) << "fail to read lookup result" << err.message() << dendl;
+            ldout(cct, 20) << "fail to read lookup result"
+                           << err.message() << dendl;
             close();
             on_finish->complete(false);
             delete lookup_result;
@@ -197,7 +209,8 @@ namespace immutable_obj_cache {
           }
 
           if(err) {
-            ldout(cct, 1) << "fail to read lookup result" << err.message() << dendl;
+            ldout(cct, 1) << "fail to read lookup result"
+                          << err.message() << dendl;
             close();
             on_finish->complete(false);
             delete lookup_result;
diff --git a/src/tools/ceph_immutable_object_cache/CacheClient.h b/src/tools/immutable_object_cache/CacheClient.h
similarity index 98%
rename from src/tools/ceph_immutable_object_cache/CacheClient.h
rename to src/tools/immutable_object_cache/CacheClient.h
index ad1f9f96b8d..dca081cbf5a 100644
--- a/src/tools/ceph_immutable_object_cache/CacheClient.h
+++ b/src/tools/immutable_object_cache/CacheClient.h
@@ -9,7 +9,6 @@
 #include <boost/bind.hpp>
 #include <boost/asio/error.hpp>
 #include <boost/algorithm/string.hpp>
-#include "librbd/ImageCtx.h"
 #include "include/ceph_assert.h"
 #include "include/Context.h"
 #include "SocketCommon.h"
diff --git a/src/tools/ceph_immutable_object_cache/CacheController.cc b/src/tools/immutable_object_cache/CacheController.cc
similarity index 98%
rename from src/tools/ceph_immutable_object_cache/CacheController.cc
rename to src/tools/immutable_object_cache/CacheController.cc
index 5b4652c4e27..e6a1e92220f 100644
--- a/src/tools/ceph_immutable_object_cache/CacheController.cc
+++ b/src/tools/immutable_object_cache/CacheController.cc
@@ -45,7 +45,7 @@ void CacheController::handle_signal(int signum){}
 
 void CacheController::run() {
   try {
-    std::string controller_path = m_cct->_conf.get_val<std::string>("rbd_shared_cache_sock");
+    std::string controller_path = m_cct->_conf.get_val<std::string>("immutable_object_cache_sock");
     std::remove(controller_path.c_str());
 
     m_cache_server = new CacheServer(m_cct, controller_path,
diff --git a/src/tools/ceph_immutable_object_cache/CacheController.h b/src/tools/immutable_object_cache/CacheController.h
similarity index 92%
rename from src/tools/ceph_immutable_object_cache/CacheController.h
rename to src/tools/immutable_object_cache/CacheController.h
index 39a85eae652..08fed8b37d1 100644
--- a/src/tools/ceph_immutable_object_cache/CacheController.h
+++ b/src/tools/immutable_object_cache/CacheController.h
@@ -12,10 +12,7 @@
 #include "common/Mutex.h"
 #include "common/WorkQueue.h"
 #include "include/rados/librados.hpp"
-#include "include/rbd/librbd.h"
 #include "include/ceph_assert.h"
-#include "librbd/ImageCtx.h"
-#include "librbd/ImageState.h"
 #include "CacheServer.h"
 #include "ObjectCacheStore.h"
 
diff --git a/src/tools/ceph_immutable_object_cache/CacheServer.cc b/src/tools/immutable_object_cache/CacheServer.cc
similarity index 87%
rename from src/tools/ceph_immutable_object_cache/CacheServer.cc
rename to src/tools/immutable_object_cache/CacheServer.cc
index 1b2fe2ec4a6..5d94b069c05 100644
--- a/src/tools/ceph_immutable_object_cache/CacheServer.cc
+++ b/src/tools/immutable_object_cache/CacheServer.cc
@@ -8,7 +8,7 @@
 #define dout_context g_ceph_context
 #define dout_subsys ceph_subsys_immutable_obj_cache
 #undef dout_prefix
-#define dout_prefix *_dout << "ceph::cache::CacheControllerSocket: " << this << " " \
+#define dout_prefix *_dout << "ceph::cache::CacheServer: " << this << " " \
                            << __func__ << ": "
 
 
@@ -17,7 +17,8 @@ using boost::asio::local::stream_protocol;
 namespace ceph {
 namespace immutable_obj_cache {
 
-CacheServer::CacheServer(CephContext* cct, const std::string& file, ProcessMsg processmsg)
+CacheServer::CacheServer(CephContext* cct, const std::string& file,
+                         ProcessMsg processmsg)
   : cct(cct), m_server_process_msg(processmsg),
     m_local_path(file), m_acceptor(m_io_service) {}
 
@@ -87,13 +88,15 @@ int CacheServer::start_accept() {
 
 void CacheServer::accept() {
 
-  CacheSessionPtr new_session(new CacheSession(m_session_id, m_io_service, m_server_process_msg, cct));
+  CacheSessionPtr new_session(new CacheSession(m_session_id, m_io_service,
+                                               m_server_process_msg, cct));
   m_acceptor.async_accept(new_session->socket(),
       boost::bind(&CacheServer::handle_accept, this, new_session,
         boost::asio::placeholders::error));
 }
 
-void CacheServer::handle_accept(CacheSessionPtr new_session, const boost::system::error_code& error) {
+void CacheServer::handle_accept(CacheSessionPtr new_session,
+                                const boost::system::error_code& error) {
   ldout(cct, 20) << dendl;
   if (error) {
     // operation_absort
diff --git a/src/tools/ceph_immutable_object_cache/CacheServer.h b/src/tools/immutable_object_cache/CacheServer.h
similarity index 100%
rename from src/tools/ceph_immutable_object_cache/CacheServer.h
rename to src/tools/immutable_object_cache/CacheServer.h
diff --git a/src/tools/ceph_immutable_object_cache/CacheSession.cc b/src/tools/immutable_object_cache/CacheSession.cc
similarity index 82%
rename from src/tools/ceph_immutable_object_cache/CacheSession.cc
rename to src/tools/immutable_object_cache/CacheSession.cc
index 1c06bc7785c..417d198a6fb 100644
--- a/src/tools/ceph_immutable_object_cache/CacheSession.cc
+++ b/src/tools/immutable_object_cache/CacheSession.cc
@@ -15,8 +15,10 @@
 namespace ceph {
 namespace immutable_obj_cache {
 
-CacheSession::CacheSession(uint64_t session_id, boost::asio::io_service& io_service, ProcessMsg processmsg, CephContext* cct)
-    : m_session_id(session_id), m_dm_socket(io_service), process_msg(processmsg), cct(cct)
+CacheSession::CacheSession(uint64_t session_id, io_service& io_service,
+                           ProcessMsg processmsg, CephContext* cct)
+    : m_session_id(session_id), m_dm_socket(io_service),
+      process_msg(processmsg), cct(cct)
     {}
 
 CacheSession::~CacheSession() {
@@ -42,7 +44,8 @@ void CacheSession::start() {
 }
 
 void CacheSession::handing_request() {
-  boost::asio::async_read(m_dm_socket, boost::asio::buffer(m_buffer, RBDSC_MSG_LEN),
+  boost::asio::async_read(m_dm_socket,
+                          boost::asio::buffer(m_buffer, RBDSC_MSG_LEN),
                           boost::asio::transfer_exactly(RBDSC_MSG_LEN),
                           boost::bind(&CacheSession::handle_read,
                                       shared_from_this(),
@@ -50,7 +53,8 @@ void CacheSession::handing_request() {
                                       boost::asio::placeholders::bytes_transferred));
 }
 
-void CacheSession::handle_read(const boost::system::error_code& err, size_t bytes_transferred) {
+void CacheSession::handle_read(const boost::system::error_code& err,
+                               size_t bytes_transferred) {
   if (err == boost::asio::error::eof ||
      err == boost::asio::error::connection_reset ||
      err == boost::asio::error::operation_aborted ||
@@ -73,14 +77,15 @@ void CacheSession::handle_read(const boost::system::error_code& err, size_t byte
   process_msg(m_session_id, std::string(m_buffer, bytes_transferred));
 }
 
-void CacheSession::handle_write(const boost::system::error_code& error, size_t bytes_transferred) {
+void CacheSession::handle_write(const boost::system::error_code& error,
+                                size_t bytes_transferred) {
   if (error) {
-    ldout(cct, 20) << "session: async_write fails: " << error.message() << dendl;
+    ldout(cct, 20) << "async_write failed: " << error.message() << dendl;
     assert(0);
   }
 
   if(bytes_transferred != RBDSC_MSG_LEN) {
-    ldout(cct, 20) << "session : reply in-complete. "<<dendl;
+    ldout(cct, 20) << "reply in-complete. "<<dendl;
     assert(0);
   }
 
diff --git a/src/tools/ceph_immutable_object_cache/CacheSession.h b/src/tools/immutable_object_cache/CacheSession.h
similarity index 74%
rename from src/tools/ceph_immutable_object_cache/CacheSession.h
rename to src/tools/immutable_object_cache/CacheSession.h
index b45e878a41b..76a8543b340 100644
--- a/src/tools/ceph_immutable_object_cache/CacheSession.h
+++ b/src/tools/immutable_object_cache/CacheSession.h
@@ -16,13 +16,15 @@
 
 
 using boost::asio::local::stream_protocol;
+using boost::asio::io_service;
 
 namespace ceph {
 namespace immutable_obj_cache {
 
 class CacheSession : public std::enable_shared_from_this<CacheSession> {
 public:
-  CacheSession(uint64_t session_id, boost::asio::io_service& io_service, ProcessMsg processmsg, CephContext* cct);
+  CacheSession(uint64_t session_id, io_service& io_service,
+               ProcessMsg processmsg, CephContext* cct);
   ~CacheSession();
 
   stream_protocol::socket& socket();
@@ -32,9 +34,11 @@ public:
 
 private:
 
-  void handle_read(const boost::system::error_code& error, size_t bytes_transferred);
+  void handle_read(const boost::system::error_code& error,
+                   size_t bytes_transferred);
 
-  void handle_write(const boost::system::error_code& error, size_t bytes_transferred);
+  void handle_write(const boost::system::error_code& error,
+                    size_t bytes_transferred);
 
 public:
   void send(std::string msg);
diff --git a/src/tools/ceph_immutable_object_cache/ObjectCacheFile.cc b/src/tools/immutable_object_cache/ObjectCacheFile.cc
similarity index 74%
rename from src/tools/ceph_immutable_object_cache/ObjectCacheFile.cc
rename to src/tools/immutable_object_cache/ObjectCacheFile.cc
index 9786be1214a..538a59edc53 100644
--- a/src/tools/ceph_immutable_object_cache/ObjectCacheFile.cc
+++ b/src/tools/immutable_object_cache/ObjectCacheFile.cc
@@ -21,20 +21,20 @@
 namespace ceph {
 namespace immutable_obj_cache {
 
-SyncFile::SyncFile(CephContext *cct, const std::string &name)
+ObjectCacheFile::ObjectCacheFile(CephContext *cct, const std::string &name)
   : cct(cct), m_fd(-1) {
-  m_name = cct->_conf.get_val<std::string>("rbd_shared_cache_path") + "/ceph_immutable_obj_cache/" + name;
+  m_name = cct->_conf.get_val<std::string>("immutable_object_cache_path") + "/ceph_immutable_obj_cache/" + name;
   ldout(cct, 20) << "file path=" << m_name << dendl;
 }
 
-SyncFile::~SyncFile() {
+ObjectCacheFile::~ObjectCacheFile() {
   // TODO force proper cleanup
   if (m_fd != -1) {
     ::close(m_fd);
   }
 }
 
-int SyncFile::open_file() {
+int ObjectCacheFile::open_file() {
   m_fd = ::open(m_name.c_str(), O_RDONLY);
   if(m_fd == -1) {
     lderr(cct) << "open fails : " << std::strerror(errno) << dendl;
@@ -42,7 +42,7 @@ int SyncFile::open_file() {
   return m_fd;
 }
 
-int SyncFile::create() {
+int ObjectCacheFile::create() {
   m_fd = ::open(m_name.c_str(), O_CREAT | O_NOATIME | O_RDWR | O_SYNC,
                   S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
   if(m_fd == -1) {
@@ -51,15 +51,15 @@ int SyncFile::create() {
   return m_fd;
 }
 
-void SyncFile::read(uint64_t offset, uint64_t length, ceph::bufferlist *bl, Context *on_finish) {
+void ObjectCacheFile::read(uint64_t offset, uint64_t length, ceph::bufferlist *bl, Context *on_finish) {
   on_finish->complete(read_object_from_file(bl, offset, length));
 }
 
-void SyncFile::write(uint64_t offset, ceph::bufferlist &&bl, bool fdatasync, Context *on_finish) {
+void ObjectCacheFile::write(uint64_t offset, ceph::bufferlist &&bl, bool fdatasync, Context *on_finish) {
   on_finish->complete(write_object_to_file(bl, bl.length()));
 }
 
-int SyncFile::write_object_to_file(ceph::bufferlist read_buf, uint64_t object_len) {
+int ObjectCacheFile::write_object_to_file(ceph::bufferlist read_buf, uint64_t object_len) {
 
   ldout(cct, 20) << "cache file name:" << m_name
                  << ", length:" << object_len <<  dendl;
@@ -74,7 +74,7 @@ int SyncFile::write_object_to_file(ceph::bufferlist read_buf, uint64_t object_le
   return ret;
 }
 
-int SyncFile::read_object_from_file(ceph::bufferlist* read_buf, uint64_t object_off, uint64_t object_len) {
+int ObjectCacheFile::read_object_from_file(ceph::bufferlist* read_buf, uint64_t object_off, uint64_t object_len) {
 
   ldout(cct, 20) << "offset:" << object_off
                  << ", length:" << object_len <<  dendl;
@@ -92,7 +92,7 @@ int SyncFile::read_object_from_file(ceph::bufferlist* read_buf, uint64_t object_
   return ret;
 }
 
-uint64_t SyncFile::get_file_size() {
+uint64_t ObjectCacheFile::get_file_size() {
   struct stat buf;
   if(m_fd == -1) {
     lderr(cct)<<"get_file_size fail: file is closed status." << dendl;
@@ -107,5 +107,5 @@ uint64_t SyncFile::get_file_size() {
 }
 
 
+} // namespace immutable_obj_cache
 } // namespace cache
-} // namespace librbd
diff --git a/src/tools/ceph_immutable_object_cache/ObjectCacheFile.h b/src/tools/immutable_object_cache/ObjectCacheFile.h
similarity index 78%
rename from src/tools/ceph_immutable_object_cache/ObjectCacheFile.h
rename to src/tools/immutable_object_cache/ObjectCacheFile.h
index d29b44177de..a8fe4350f40 100644
--- a/src/tools/ceph_immutable_object_cache/ObjectCacheFile.h
+++ b/src/tools/immutable_object_cache/ObjectCacheFile.h
@@ -1,8 +1,8 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
-#ifndef CEPH_CACHE_OBJECT_CACHE_SYNC_FILE_H
-#define CEPH_CACHE_OBJECT_CACHE_SYNC_FILE_H
+#ifndef CEPH_CACHE_OBJECT_CACHE_FILE_H
+#define CEPH_CACHE_OBJECT_CACHE_FILE_H
 
 #include "include/Context.h"
 #include "include/buffer_fwd.h"
@@ -14,10 +14,10 @@
 namespace ceph {
 namespace immutable_obj_cache {
 
-class SyncFile {
+class ObjectCacheFile {
 public:
-  SyncFile(CephContext *cct, const std::string &name);
-  ~SyncFile();
+  ObjectCacheFile(CephContext *cct, const std::string &name);
+  ~ObjectCacheFile();
 
   // TODO use IO queue instead of individual commands so operations can be
   // submitted in batch
@@ -41,7 +41,7 @@ private:
   int m_fd;
 };
 
+} // namespace immutable_obj_cache
 } // namespace cache
-} // namespace librbd
 
-#endif // CEPH_LIBRBD_CACHE_STORE_SYNC_FILE
+#endif // CEPH_CACHE_OBJECT_CACHE_FILE_H
diff --git a/src/tools/ceph_immutable_object_cache/ObjectCacheStore.cc b/src/tools/immutable_object_cache/ObjectCacheStore.cc
similarity index 95%
rename from src/tools/ceph_immutable_object_cache/ObjectCacheStore.cc
rename to src/tools/immutable_object_cache/ObjectCacheStore.cc
index 7409e8f43fd..83ad6ad53b0 100644
--- a/src/tools/ceph_immutable_object_cache/ObjectCacheStore.cc
+++ b/src/tools/immutable_object_cache/ObjectCacheStore.cc
@@ -2,7 +2,6 @@
 // vim: ts=8 sw=2 smarttab
 
 #include "ObjectCacheStore.h"
-#include "include/Context.h"
 #include "librbd/Utils.h"
 
 #include <experimental/filesystem>
@@ -23,9 +22,9 @@ ObjectCacheStore::ObjectCacheStore(CephContext *cct, ContextWQ* work_queue)
         m_ioctxs_lock("ceph::cache::ObjectCacheStore::m_ioctxs_lock") {
 
   object_cache_entries =
-    m_cct->_conf.get_val<int64_t>("rbd_shared_cache_entries");
+    m_cct->_conf.get_val<Option::size_t>("immutable_object_cache_max_size");
 
-  std::string cache_path = m_cct->_conf.get_val<std::string>("rbd_shared_cache_path");
+  std::string cache_path = m_cct->_conf.get_val<std::string>("immutable_object_cache_path");
   m_cache_root_dir = cache_path + "/ceph_immutable_obj_cache/";
 
   //TODO(): allow to set cache level
@@ -86,7 +85,7 @@ int ObjectCacheStore::init_cache(std::string pool_name, std::string vol_name, ui
 
   std::string vol_cache_dir = m_cache_root_dir + pool_name + "_" + vol_name;
 
-  int dir = dir_num - 1;
+  int dir = m_dir_num - 1;
   while (dir >= 0) {
     efs::create_directories(vol_cache_dir + "/" + std::to_string(dir));
     dir --;
@@ -154,12 +153,12 @@ int ObjectCacheStore::handle_promote_callback(int ret, bufferlist* read_buf,
     read_buf->append(std::string(object_size - ret, '0'));
   }
 
-  if (dir_num > 0) {
+  if (m_dir_num > 0) {
     auto const pos = cache_file_name.find_last_of('.');
-    cache_dir = cache_dir + "/" + std::to_string(stoul(cache_file_name.substr(pos+1)) % dir_num);
+    cache_dir = cache_dir + "/" + std::to_string(stoul(cache_file_name.substr(pos+1)) % m_dir_num);
   }
   // write to cache
-  SyncFile cache_file(m_cct, cache_dir + "/" + cache_file_name);
+  ObjectCacheFile cache_file(m_cct, cache_dir + "/" + cache_file_name);
   cache_file.create();
 
   ret = cache_file.write_object_to_file(*read_buf, object_size);
@@ -253,9 +252,9 @@ int ObjectCacheStore::do_evict(std::string cache_file) {
 
   std::string cache_dir = m_cache_root_dir + pool_name + "_" + vol_name;
 
-   if (dir_num > 0) {
+   if (m_dir_num > 0) {
     auto const pos = cache_file.find_last_of('.');
-    cache_dir = cache_dir + "/" + std::to_string(stoul(cache_file.substr(pos+1)) % dir_num);
+    cache_dir = cache_dir + "/" + std::to_string(stoul(cache_file.substr(pos+1)) % m_dir_num);
   }
   std::string cache_file_path = cache_dir + "/" + cache_file;
 
diff --git a/src/tools/ceph_immutable_object_cache/ObjectCacheStore.h b/src/tools/immutable_object_cache/ObjectCacheStore.h
similarity index 92%
rename from src/tools/ceph_immutable_object_cache/ObjectCacheStore.h
rename to src/tools/immutable_object_cache/ObjectCacheStore.h
index 7db93b27420..e8c44f0fee8 100644
--- a/src/tools/ceph_immutable_object_cache/ObjectCacheStore.h
+++ b/src/tools/immutable_object_cache/ObjectCacheStore.h
@@ -8,16 +8,16 @@
 #include "common/errno.h"
 #include "common/ceph_context.h"
 #include "common/Mutex.h"
+#include "include/Context.h"
 #include "include/rados/librados.hpp"
-#include "include/rbd/librbd.h"
-#include "librbd/ImageCtx.h"
-#include "librbd/ImageState.h"
+
 #include "ObjectCacheFile.h"
 #include "SimplePolicy.h"
 
 
 using librados::Rados;
 using librados::IoCtx;
+class ContextWQ;
 
 namespace ceph {
 namespace immutable_obj_cache {
@@ -60,14 +60,14 @@ class ObjectCacheStore
     std::map<std::string, librados::IoCtx*> m_ioctxs;
     Mutex m_ioctxs_lock;
 
-    SyncFile *m_cache_file;
+    ObjectCacheFile *m_cache_file;
 
     Policy* m_policy;
     std::thread* evict_thd;
     bool m_evict_go = false;
 
     //TODO(): make this configurable
-    int dir_num = 10;
+    int m_dir_num = 10;
     uint64_t object_cache_entries;
     std::string m_cache_root_dir;
 };
diff --git a/src/tools/ceph_immutable_object_cache/Policy.h b/src/tools/immutable_object_cache/Policy.h
similarity index 92%
rename from src/tools/ceph_immutable_object_cache/Policy.h
rename to src/tools/immutable_object_cache/Policy.h
index b68e5fa6cc4..f1eef1b1a92 100644
--- a/src/tools/ceph_immutable_object_cache/Policy.h
+++ b/src/tools/immutable_object_cache/Policy.h
@@ -1,8 +1,8 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
-#ifndef CEPH_CACHE_POLICY_HPP
-#define CEPH_CACHE_POLICY_HPP
+#ifndef CEPH_CACHE_POLICY_H
+#define CEPH_CACHE_POLICY_H
 
 #include <list>
 #include <string>
diff --git a/src/tools/ceph_immutable_object_cache/SimplePolicy.cc b/src/tools/immutable_object_cache/SimplePolicy.cc
similarity index 94%
rename from src/tools/ceph_immutable_object_cache/SimplePolicy.cc
rename to src/tools/immutable_object_cache/SimplePolicy.cc
index 81775297403..119b8c76b06 100644
--- a/src/tools/ceph_immutable_object_cache/SimplePolicy.cc
+++ b/src/tools/immutable_object_cache/SimplePolicy.cc
@@ -16,11 +16,13 @@
 namespace ceph {
 namespace immutable_obj_cache {
 
-SimplePolicy::SimplePolicy(CephContext *cct, uint64_t block_num, float watermark)
-  : cct(cct), m_watermark(watermark), m_entry_count(block_num), inflight_ops(0),
+SimplePolicy::SimplePolicy(CephContext *cct, uint64_t block_num,
+                           float watermark)
+  : cct(cct), m_watermark(watermark), m_entry_count(block_num),
     m_cache_map_lock("rbd::cache::SimplePolicy::m_cache_map_lock"),
     m_free_list_lock("rbd::cache::SimplePolicy::m_free_list_lock") {
   ldout(cct, 20) << dendl;
+  m_max_inflight_ops = cct->_conf.get_val<uint64_t>("immutable_object_cache_max_inflight_ops");
   for(uint64_t i = 0; i < m_entry_count; i++) {
     m_free_list.push_back(new Entry());
   }
@@ -87,7 +89,8 @@ cache_status_t SimplePolicy::lookup_object(std::string file_name) {
   return entry->status;
 }
 
-void SimplePolicy::update_status(std::string file_name, cache_status_t new_status) {
+void SimplePolicy::update_status(std::string file_name,
+                                 cache_status_t new_status) {
   ldout(cct, 20) << "update status for: " << file_name
                  << " new status = " << new_status << dendl;
 
diff --git a/src/tools/ceph_immutable_object_cache/SimplePolicy.h b/src/tools/immutable_object_cache/SimplePolicy.h
similarity index 88%
rename from src/tools/ceph_immutable_object_cache/SimplePolicy.h
rename to src/tools/immutable_object_cache/SimplePolicy.h
index ac0fbf86849..a75bf3ca3d7 100644
--- a/src/tools/ceph_immutable_object_cache/SimplePolicy.h
+++ b/src/tools/immutable_object_cache/SimplePolicy.h
@@ -1,8 +1,8 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
-#ifndef CEPH_CACHE_SIMPLE_POLICY_HPP
-#define CEPH_CACHE_SIMPLE_POLICY_HPP
+#ifndef CEPH_CACHE_SIMPLE_POLICY_H
+#define CEPH_CACHE_SIMPLE_POLICY_H
 
 #include "common/ceph_context.h"
 #include "common/RWLock.h"
@@ -50,7 +50,8 @@ private:
   CephContext* cct;
   float m_watermark;
   uint64_t m_entry_count;
-  std::atomic<uint8_t> inflight_ops;
+  uint64_t m_max_inflight_ops;
+  std::atomic<uint64_t> inflight_ops = 0;
 
   std::unordered_map<std::string, Entry*> m_cache_map;
   RWLock m_cache_map_lock;
@@ -63,4 +64,4 @@ private:
 
 } // namespace immutable_obj_cache
 } // namespace ceph
-#endif
+#endif // CEPH_CACHE_SIMPLE_POLICY_H
diff --git a/src/tools/ceph_immutable_object_cache/SocketCommon.h b/src/tools/immutable_object_cache/SocketCommon.h
similarity index 100%
rename from src/tools/ceph_immutable_object_cache/SocketCommon.h
rename to src/tools/immutable_object_cache/SocketCommon.h
diff --git a/src/tools/ceph_immutable_object_cache/main.cc b/src/tools/immutable_object_cache/main.cc
similarity index 90%
rename from src/tools/ceph_immutable_object_cache/main.cc
rename to src/tools/immutable_object_cache/main.cc
index 28b0a274893..a030c5cdbbe 100644
--- a/src/tools/ceph_immutable_object_cache/main.cc
+++ b/src/tools/immutable_object_cache/main.cc
@@ -19,7 +19,7 @@ void usage() {
   std::cout << "  -m monaddress[:port]      connect to specified monitor\n";
   std::cout << "  --keyring=<path>          path to keyring for local cluster\n";
   std::cout << "  --log-file=<logfile>       file to log debug output\n";
-  std::cout << "  --debug-rbd-cachecontroller=<log-level>/<memory-level>  set rbd-mirror debug level\n";
+  std::cout << "  --debug-immutable-obj-cache=<log-level>/<memory-level>  set debug level\n";
   generic_server_usage();
 }
 
@@ -51,6 +51,7 @@ int main(int argc, const char **argv)
   }
 
   common_init_finish(g_ceph_context);
+  global_init_chdir(g_ceph_context);
 
   init_async_signal_handler();
   register_async_signal_handler(SIGHUP, sighup_handler);
@@ -60,10 +61,8 @@ int main(int argc, const char **argv)
   std::vector<const char*> cmd_args;
   argv_to_vec(argc, argv, cmd_args);
 
-  // disable unnecessary librbd cache
-  g_ceph_context->_conf.set_val_or_die("rbd_cache", "false");
-
-  cachectl = new ceph::immutable_obj_cache::CacheController(g_ceph_context, cmd_args);
+  cachectl = new ceph::immutable_obj_cache::CacheController(g_ceph_context,
+                                                            cmd_args);
   int r = cachectl->init();
   if (r < 0) {
     std::cerr << "failed to initialize: " << cpp_strerror(r) << std::endl;
-- 
2.39.5