tools: maintain cache size counter for immutable obj cache daemon

Do not assume 4M object size since we allow to cache any-sized objects.

Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>

diff --git a/src/test/immutable_object_cache/test_SimplePolicy.cc b/src/test/immutable_object_cache/test_SimplePolicy.cc
index 70f333ebec0de052c549339b79cd5c4ef751f7e7..348d988ea5123ad118c4c603277642f138613f82 100644 (file)
--- a/src/test/immutable_object_cache/test_SimplePolicy.cc
+++ b/src/test/immutable_object_cache/test_SimplePolicy.cc
@@ -20,18 +20,19 @@ std::string generate_file_name(uint64_t index) {
 class TestSimplePolicy :public ::testing::Test {
 public:
   SimplePolicy* m_simple_policy;
-  const uint64_t m_entry_num;
+  const uint64_t m_cache_size;
   uint64_t m_entry_index;
   std::vector<std::string> m_promoted_lru;
   std::vector<std::string> m_promoting_lru;
 
-  TestSimplePolicy() : m_entry_num(100), m_entry_index(0) {}
+  TestSimplePolicy() : m_cache_size(100), m_entry_index(0) {}
   ~TestSimplePolicy() {}
   static void SetUpTestCase() {}
   static void TearDownTestCase() {}
   void SetUp() override {
-    m_simple_policy = new SimplePolicy(g_ceph_context, m_entry_num, 0.1);
-    for (uint64_t i = 0; i < m_entry_num / 2; i++, m_entry_index++) {
+    m_simple_policy = new SimplePolicy(g_ceph_context, m_cache_size, 0.1);
+    // populate 50 entries
+    for (uint64_t i = 0; i < m_cache_size / 2; i++, m_entry_index++) {
       insert_entry_into_promoted_lru(generate_file_name(m_entry_index));
     }
   }
@@ -45,28 +46,28 @@ public:
   }
 
   void insert_entry_into_promoted_lru(std::string cache_file_name) {
-    ASSERT_EQ(m_entry_num - m_promoted_lru.size() - m_promoting_lru.size(), m_simple_policy->get_free_entry_num());
+    ASSERT_EQ(m_cache_size - m_promoted_lru.size(), m_simple_policy->get_free_size());
     ASSERT_EQ(m_promoting_lru.size(), m_simple_policy->get_promoting_entry_num());
     ASSERT_EQ(m_promoted_lru.size(), m_simple_policy->get_promoted_entry_num());
     ASSERT_EQ(OBJ_CACHE_NONE, m_simple_policy->get_status(cache_file_name));
 
     m_simple_policy->lookup_object(cache_file_name);
     ASSERT_EQ(OBJ_CACHE_SKIP, m_simple_policy->get_status(cache_file_name));
-    ASSERT_EQ(m_entry_num - m_promoted_lru.size() - m_promoting_lru.size() - 1, m_simple_policy->get_free_entry_num());
+    ASSERT_EQ(m_cache_size - m_promoted_lru.size(), m_simple_policy->get_free_size());
     ASSERT_EQ(m_promoting_lru.size() + 1, m_simple_policy->get_promoting_entry_num());
     ASSERT_EQ(m_promoted_lru.size(), m_simple_policy->get_promoted_entry_num());
 
-    m_simple_policy->update_status(cache_file_name, OBJ_CACHE_PROMOTED);
+    m_simple_policy->update_status(cache_file_name, OBJ_CACHE_PROMOTED, 1);
     m_promoted_lru.push_back(cache_file_name);
     ASSERT_EQ(OBJ_CACHE_PROMOTED, m_simple_policy->get_status(cache_file_name));
 
-    ASSERT_EQ(m_entry_num - m_promoted_lru.size() - m_promoting_lru.size(), m_simple_policy->get_free_entry_num());
+    ASSERT_EQ(m_cache_size - m_promoted_lru.size(), m_simple_policy->get_free_size());
     ASSERT_EQ(m_promoting_lru.size(), m_simple_policy->get_promoting_entry_num());
     ASSERT_EQ(m_promoted_lru.size(), m_simple_policy->get_promoted_entry_num());
   }
 
   void insert_entry_into_promoting_lru(std::string cache_file_name) {
-    ASSERT_EQ(m_entry_num - m_promoted_lru.size() - m_promoting_lru.size(), m_simple_policy->get_free_entry_num());
+    ASSERT_EQ(m_cache_size - m_promoted_lru.size(), m_simple_policy->get_free_size());
     ASSERT_EQ(m_promoting_lru.size(), m_simple_policy->get_promoting_entry_num());
     ASSERT_EQ(m_promoted_lru.size(), m_simple_policy->get_promoted_entry_num());
     ASSERT_EQ(OBJ_CACHE_NONE, m_simple_policy->get_status(cache_file_name));
@@ -74,7 +75,7 @@ public:
     m_simple_policy->lookup_object(cache_file_name);
     m_promoting_lru.push_back(cache_file_name);
     ASSERT_EQ(OBJ_CACHE_SKIP, m_simple_policy->get_status(cache_file_name));
-    ASSERT_EQ(m_entry_num - m_promoted_lru.size() - m_promoting_lru.size(), m_simple_policy->get_free_entry_num());
+    ASSERT_EQ(m_cache_size - m_promoted_lru.size(), m_simple_policy->get_free_size());
     ASSERT_EQ(m_promoting_lru.size(), m_simple_policy->get_promoting_entry_num());
     ASSERT_EQ(m_promoted_lru.size(), m_simple_policy->get_promoted_entry_num());
   }
@@ -82,22 +83,22 @@ public:
 
 TEST_F(TestSimplePolicy, test_lookup_miss_and_no_free) {
   // exhaust cache space
-  uint64_t left_entry_num = m_entry_num - m_promoted_lru.size() - m_promoting_lru.size();
-  for(uint64_t i = 0; i < left_entry_num; i++, ++m_entry_index) {
+  uint64_t left_entry_num = m_cache_size - m_promoted_lru.size();
+  for (uint64_t i = 0; i < left_entry_num; i++, ++m_entry_index) {
     insert_entry_into_promoted_lru(generate_file_name(m_entry_index));
   }
-  ASSERT_TRUE(0 == m_simple_policy->get_free_entry_num());
+  ASSERT_TRUE(0 == m_simple_policy->get_free_size());
   ASSERT_TRUE(m_simple_policy->lookup_object("no_this_cache_file_name") == OBJ_CACHE_SKIP);
 }
 
-TEST_F(TestSimplePolicy, test_lookup_miss_and_free_space) {
-  ASSERT_TRUE(m_entry_num - m_promoting_lru.size() - m_promoted_lru.size() == m_simple_policy->get_free_entry_num());
+TEST_F(TestSimplePolicy, test_lookup_miss_and_have_free) {
+  ASSERT_TRUE(m_cache_size - m_promoted_lru.size() == m_simple_policy->get_free_size());
   ASSERT_TRUE(m_simple_policy->lookup_object("miss_but_have_free_space_file_name") == OBJ_CACHE_NONE);
   ASSERT_TRUE(m_simple_policy->get_status("miss_but_have_free_space_file_name") == OBJ_CACHE_SKIP);
 }
 
 TEST_F(TestSimplePolicy, test_lookup_hit_and_promoting) {
-  ASSERT_TRUE(m_entry_num - m_promoting_lru.size() - m_promoted_lru.size() == m_simple_policy->get_free_entry_num());
+  ASSERT_TRUE(m_cache_size - m_promoted_lru.size() == m_simple_policy->get_free_size());
   insert_entry_into_promoting_lru("promoting_file_1");
   insert_entry_into_promoting_lru("promoting_file_2");
   insert_entry_into_promoted_lru(generate_file_name(++m_entry_index));
@@ -118,13 +119,13 @@ TEST_F(TestSimplePolicy, test_lookup_hit_and_promoting) {
 
 TEST_F(TestSimplePolicy, test_lookup_hit_and_promoted) {
   ASSERT_TRUE(m_promoted_lru.size() == m_simple_policy->get_promoted_entry_num());
-  for(uint64_t index = 0; index < m_entry_index; index++) {
+  for (uint64_t index = 0; index < m_entry_index; index++) {
     ASSERT_TRUE(m_simple_policy->get_status(generate_file_name(index)) == OBJ_CACHE_PROMOTED);
   }
 }
 
 TEST_F(TestSimplePolicy, test_update_state_from_promoting_to_none) {
-  ASSERT_TRUE(m_entry_num - m_promoting_lru.size() - m_promoted_lru.size() == m_simple_policy->get_free_entry_num());
+  ASSERT_TRUE(m_cache_size - m_promoted_lru.size() == m_simple_policy->get_free_size());
   insert_entry_into_promoting_lru("promoting_to_none_file_1");
   insert_entry_into_promoting_lru("promoting_to_none_file_2");
   insert_entry_into_promoted_lru(generate_file_name(++m_entry_index));
@@ -168,7 +169,7 @@ TEST_F(TestSimplePolicy, test_update_state_from_promoting_to_none) {
 
 TEST_F(TestSimplePolicy, test_update_state_from_promoted_to_none) {
   ASSERT_TRUE(m_promoted_lru.size() == m_simple_policy->get_promoted_entry_num());
-  for(uint64_t index = 0; index < m_entry_index; index++) {
+  for (uint64_t index = 0; index < m_entry_index; index++) {
     ASSERT_TRUE(m_simple_policy->get_status(generate_file_name(index)) == OBJ_CACHE_PROMOTED);
     m_simple_policy->update_status(generate_file_name(index), OBJ_CACHE_NONE);
     ASSERT_TRUE(m_simple_policy->get_status(generate_file_name(index)) == OBJ_CACHE_NONE);
@@ -178,7 +179,7 @@ TEST_F(TestSimplePolicy, test_update_state_from_promoted_to_none) {
 }
 
 TEST_F(TestSimplePolicy, test_update_state_from_promoting_to_promoted) {
-  ASSERT_TRUE(m_entry_num - m_promoting_lru.size() - m_promoted_lru.size() == m_simple_policy->get_free_entry_num());
+  ASSERT_TRUE(m_cache_size - m_promoted_lru.size() == m_simple_policy->get_free_size());
   insert_entry_into_promoting_lru("promoting_to_promoted_file_1");
   insert_entry_into_promoting_lru("promoting_to_promoted_file_2");
   insert_entry_into_promoting_lru("promoting_to_promoted_file_3");
@@ -210,23 +211,24 @@ TEST_F(TestSimplePolicy, test_update_state_from_promoting_to_promoted) {
 TEST_F(TestSimplePolicy, test_evict_list_0) {
   std::list<std::string> evict_entry_list;
   // 0.1 is watermark
-  ASSERT_TRUE((float)m_simple_policy->get_free_entry_num() > m_entry_num*0.1);
+  ASSERT_TRUE((float)m_simple_policy->get_free_size() > m_cache_size*0.1);
   m_simple_policy->get_evict_list(&evict_entry_list);
   ASSERT_TRUE(evict_entry_list.size() == 0);
 }
 
 TEST_F(TestSimplePolicy, test_evict_list_10) {
-  uint64_t left_entry_num = m_entry_num - m_promoted_lru.size() - m_promoting_lru.size();
-  for(uint64_t i = 0; i < left_entry_num; i++, ++m_entry_index) {
+  uint64_t left_entry_num = m_cache_size - m_promoted_lru.size();
+  for (uint64_t i = 0; i < left_entry_num; i++, ++m_entry_index) {
     insert_entry_into_promoted_lru(generate_file_name(m_entry_index));
   }
-  ASSERT_TRUE(0 == m_simple_policy->get_free_entry_num());
+  ASSERT_TRUE(0 == m_simple_policy->get_free_size());
   std::list<std::string> evict_entry_list;
   m_simple_policy->get_evict_list(&evict_entry_list);
-  ASSERT_TRUE(10 == evict_entry_list.size());
-  ASSERT_TRUE(m_entry_num - 10  == m_simple_policy->get_promoted_entry_num());
+  // evict 10% of old entries
+  ASSERT_TRUE(m_cache_size*0.1 == evict_entry_list.size());
+  ASSERT_TRUE(m_cache_size - m_cache_size*0.1  == m_simple_policy->get_promoted_entry_num());
 
-  for(auto it = evict_entry_list.begin(); it != evict_entry_list.end(); it++) {
+  for (auto it = evict_entry_list.begin(); it != evict_entry_list.end(); it++) {
     ASSERT_TRUE(*it == m_promoted_lru.front());
     m_promoted_lru.erase(m_promoted_lru.begin());
   }

diff --git a/src/tools/immutable_object_cache/ObjectCacheStore.cc b/src/tools/immutable_object_cache/ObjectCacheStore.cc
index b79ec408b2512bc06ec1458dd8d7075868665806..d268890af0615d1bcb4d81d23b330e761b3db134 100644 (file)
--- a/src/tools/immutable_object_cache/ObjectCacheStore.cc
+++ b/src/tools/immutable_object_cache/ObjectCacheStore.cc
@@ -28,7 +28,7 @@ ObjectCacheStore::ObjectCacheStore(CephContext *cct)
   m_cache_root_dir = cache_path + "/ceph_immutable_obj_cache/";
 
   //TODO(): allow to set cache level
-  m_policy = new SimplePolicy(m_cct, object_cache_max_size/(4096*1024), 0.1);
+  m_policy = new SimplePolicy(m_cct, object_cache_max_size, 0.1);
 }
 
 ObjectCacheStore::~ObjectCacheStore() {
@@ -162,7 +162,7 @@ int ObjectCacheStore::handle_promote_callback(int ret, bufferlist* read_buf,
 
   // update metadata
   ceph_assert(OBJ_CACHE_SKIP == m_policy->get_status(cache_file_name));
-  m_policy->update_status(cache_file_name, OBJ_CACHE_PROMOTED);
+  m_policy->update_status(cache_file_name, OBJ_CACHE_PROMOTED, ret);
   ceph_assert(OBJ_CACHE_PROMOTED == m_policy->get_status(cache_file_name));
 
   delete read_buf;

diff --git a/src/tools/immutable_object_cache/Policy.h b/src/tools/immutable_object_cache/Policy.h
index f1eef1b1a920dd39c2e107460cf51591b918d0d6..cda8e69c917d67517c491b9c446bd60e0a82bea4 100644 (file)
--- a/src/tools/immutable_object_cache/Policy.h
+++ b/src/tools/immutable_object_cache/Policy.h
@@ -23,7 +23,7 @@ public:
   virtual ~Policy(){};
   virtual cache_status_t lookup_object(std::string) = 0;
   virtual int evict_entry(std::string) = 0;
-  virtual void update_status(std::string, cache_status_t) = 0;
+  virtual void update_status(std::string, cache_status_t, uint64_t size=0) = 0;
   virtual cache_status_t get_status(std::string) = 0;
   virtual void get_evict_list(std::list<std::string>* obj_list) = 0;
 };

diff --git a/src/tools/immutable_object_cache/SimplePolicy.cc b/src/tools/immutable_object_cache/SimplePolicy.cc
index ab02def4fe0cc6660147b81ebcdab9080ffe2e9b..8fe70b4e00d5ff1ab8bf2a98f078821936357e1d 100644 (file)
--- a/src/tools/immutable_object_cache/SimplePolicy.cc
+++ b/src/tools/immutable_object_cache/SimplePolicy.cc
@@ -13,30 +13,24 @@
 namespace ceph {
 namespace immutable_obj_cache {
 
-SimplePolicy::SimplePolicy(CephContext *cct, uint64_t block_num,
+SimplePolicy::SimplePolicy(CephContext *cct, uint64_t cache_size,
                            float watermark)
-  : cct(cct), m_watermark(watermark), m_entry_count(block_num),
-    m_cache_map_lock("rbd::cache::SimplePolicy::m_cache_map_lock"),
-    m_free_list_lock("rbd::cache::SimplePolicy::m_free_list_lock") {
+  : cct(cct), m_watermark(watermark), m_max_cache_size(cache_size),
+    m_cache_map_lock("rbd::cache::SimplePolicy::m_cache_map_lock") {
   ldout(cct, 20) << dendl;
   m_max_inflight_ops = cct->_conf.get_val<uint64_t>("immutable_object_cache_max_inflight_ops");
-  for(uint64_t i = 0; i < m_entry_count; i++) {
-    m_free_list.push_back(new Entry());
-  }
+  m_cache_size = 0;
 }
 
 SimplePolicy::~SimplePolicy() {
   ldout(cct, 20) << dendl;
 
   for (auto it: m_cache_map) {
-    Entry* entry = reinterpret_cast<Entry*>(it.second);
+    Entry* entry = (it.second);
     delete entry;
   }
 
-  for(auto it : m_free_list) {
-    Entry* entry = it;
-    delete entry;
-  }
+
 }
 
 cache_status_t SimplePolicy::alloc_entry(std::string file_name) {
@@ -49,20 +43,16 @@ cache_status_t SimplePolicy::alloc_entry(std::string file_name) {
     ldout(cct, 20) << "object is under promoting: " << file_name << dendl;
     return OBJ_CACHE_SKIP;
   }
-  m_free_list_lock.lock();
 
-  if (m_free_list.size() && (inflight_ops < m_max_inflight_ops)) {
-    Entry* entry = m_free_list.front();
+  if ((m_cache_size < m_max_cache_size) && (inflight_ops < m_max_inflight_ops)) {
+    Entry* entry = new Entry();
     ceph_assert(entry != nullptr);
-    m_free_list.pop_front();
-    m_free_list_lock.unlock();
     m_cache_map[file_name] = entry;
     wlocker.unlock();
     update_status(file_name, OBJ_CACHE_SKIP);
     return OBJ_CACHE_NONE; // start promotion request
   }
 
-  m_free_list_lock.unlock();
   // if there's no free entry, return skip to read from rados
   return OBJ_CACHE_SKIP;
 }
@@ -90,7 +80,7 @@ cache_status_t SimplePolicy::lookup_object(std::string file_name) {
 }
 
 void SimplePolicy::update_status(std::string file_name,
-                                 cache_status_t new_status) {
+                                 cache_status_t new_status, uint64_t size) {
   ldout(cct, 20) << "update status for: " << file_name
                  << " new status = " << new_status << dendl;
 
@@ -116,6 +106,8 @@ void SimplePolicy::update_status(std::string file_name,
   if (entry->status == OBJ_CACHE_SKIP && new_status== OBJ_CACHE_PROMOTED) {
     m_promoted_lru.lru_insert_top(entry);
     entry->status = new_status;
+    entry->size = size;
+    m_cache_size += entry->size;
     inflight_ops--;
     return;
   }
@@ -125,10 +117,7 @@ void SimplePolicy::update_status(std::string file_name,
     // mark this entry as free
     entry->file_name = "";
     entry->status = new_status;
-    {
-      Mutex::Locker free_list_locker(m_free_list_lock);
-      m_free_list.push_back(entry);
-    }
+
     m_cache_map.erase(entry_it);
     inflight_ops--;
     return;
@@ -137,14 +126,14 @@ void SimplePolicy::update_status(std::string file_name,
   // to evict
   if (entry->status == OBJ_CACHE_PROMOTED && new_status== OBJ_CACHE_NONE) {
     // mark this entry as free
+    uint64_t size = entry->size;
     entry->file_name = "";
+    entry->size = 0;
     entry->status = new_status;
-    {
-      Mutex::Locker free_list_locker(m_free_list_lock);
-      m_free_list.push_back(entry);
-    }
+
     m_promoted_lru.lru_remove(entry);
     m_cache_map.erase(entry_it);
+    m_cache_size -= size;
     return;
   }
 
@@ -175,8 +164,8 @@ void SimplePolicy::get_evict_list(std::list<std::string>* obj_list) {
 
   RWLock::WLocker locker(m_cache_map_lock);
   // check free ratio, pop entries from LRU
-  if ((float)m_free_list.size() / m_entry_count < m_watermark) {
-    int evict_num = m_entry_count * 0.1; //TODO(): make this configurable
+  if ((double)m_cache_size / m_max_cache_size > (1 - m_watermark)) {
+    int evict_num = m_cache_map.size() * 0.1; //TODO(): make this configurable
     for (int i = 0; i < evict_num; i++) {
       Entry* entry = reinterpret_cast<Entry*>(m_promoted_lru.lru_expire());
       if (entry == nullptr) {
@@ -190,9 +179,8 @@ void SimplePolicy::get_evict_list(std::list<std::string>* obj_list) {
 }
 
 // for unit test
-uint64_t SimplePolicy::get_free_entry_num() {
-  Mutex::Locker free_list_locker(m_free_list_lock);
-  return m_free_list.size();
+uint64_t SimplePolicy::get_free_size() {
+  return m_max_cache_size - m_cache_size;
 }
 
 uint64_t SimplePolicy::get_promoting_entry_num() {

diff --git a/src/tools/immutable_object_cache/SimplePolicy.h b/src/tools/immutable_object_cache/SimplePolicy.h
index 7dce7840fd322ad519807e5f0e7515d09ed71a53..7d97bdc9a15998436d8e092f6c795d5ab4efacf4 100644 (file)
--- a/src/tools/immutable_object_cache/SimplePolicy.h
+++ b/src/tools/immutable_object_cache/SimplePolicy.h
@@ -24,13 +24,14 @@ public:
   cache_status_t lookup_object(std::string file_name);
   cache_status_t get_status(std::string file_name);
 
-  void update_status(std::string file_name, cache_status_t new_status);
+  void update_status(std::string file_name,
+                     cache_status_t new_status, uint64_t size=0);
 
   int evict_entry(std::string file_name);
 
   void get_evict_list(std::list<std::string>* obj_list);
 
-  uint64_t get_free_entry_num();
+  uint64_t get_free_size();
   uint64_t get_promoting_entry_num();
   uint64_t get_promoted_entry_num();
   std::string get_evict_entry();
@@ -43,19 +44,23 @@ private:
       cache_status_t status;
       Entry() : status(OBJ_CACHE_NONE){}
       std::string file_name;
+      uint64_t size;
   };
 
   CephContext* cct;
   float m_watermark;
   uint64_t m_entry_count;
   uint64_t m_max_inflight_ops;
+  uint64_t m_max_cache_size;
   std::atomic<uint64_t> inflight_ops = 0;
 
   std::unordered_map<std::string, Entry*> m_cache_map;
   RWLock m_cache_map_lock;
 
+
   std::deque<Entry*> m_free_list;
-  Mutex m_free_list_lock;
+
+  std::atomic<uint64_t> m_cache_size;
 
   LRU m_promoted_lru;
 };