From 83752060b6f4caff53df06e322a4455cce2d3b97 Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Fri, 22 Feb 2019 10:44:04 +0800 Subject: [PATCH] tools: maintain cache size counter for immutable obj cache daemon Do not assume 4M object size since we allow to cache any-sized objects. Signed-off-by: Yuan Zhou --- .../test_SimplePolicy.cc | 56 ++++++++++--------- .../ObjectCacheStore.cc | 4 +- src/tools/immutable_object_cache/Policy.h | 2 +- .../immutable_object_cache/SimplePolicy.cc | 52 +++++++---------- .../immutable_object_cache/SimplePolicy.h | 11 +++- 5 files changed, 60 insertions(+), 65 deletions(-) diff --git a/src/test/immutable_object_cache/test_SimplePolicy.cc b/src/test/immutable_object_cache/test_SimplePolicy.cc index 70f333ebec0de..348d988ea5123 100644 --- a/src/test/immutable_object_cache/test_SimplePolicy.cc +++ b/src/test/immutable_object_cache/test_SimplePolicy.cc @@ -20,18 +20,19 @@ std::string generate_file_name(uint64_t index) { class TestSimplePolicy :public ::testing::Test { public: SimplePolicy* m_simple_policy; - const uint64_t m_entry_num; + const uint64_t m_cache_size; uint64_t m_entry_index; std::vector m_promoted_lru; std::vector m_promoting_lru; - TestSimplePolicy() : m_entry_num(100), m_entry_index(0) {} + TestSimplePolicy() : m_cache_size(100), m_entry_index(0) {} ~TestSimplePolicy() {} static void SetUpTestCase() {} static void TearDownTestCase() {} void SetUp() override { - m_simple_policy = new SimplePolicy(g_ceph_context, m_entry_num, 0.1); - for (uint64_t i = 0; i < m_entry_num / 2; i++, m_entry_index++) { + m_simple_policy = new SimplePolicy(g_ceph_context, m_cache_size, 0.1); + // populate 50 entries + for (uint64_t i = 0; i < m_cache_size / 2; i++, m_entry_index++) { insert_entry_into_promoted_lru(generate_file_name(m_entry_index)); } } @@ -45,28 +46,28 @@ public: } void insert_entry_into_promoted_lru(std::string cache_file_name) { - ASSERT_EQ(m_entry_num - m_promoted_lru.size() - m_promoting_lru.size(), m_simple_policy->get_free_entry_num()); + ASSERT_EQ(m_cache_size - m_promoted_lru.size(), m_simple_policy->get_free_size()); ASSERT_EQ(m_promoting_lru.size(), m_simple_policy->get_promoting_entry_num()); ASSERT_EQ(m_promoted_lru.size(), m_simple_policy->get_promoted_entry_num()); ASSERT_EQ(OBJ_CACHE_NONE, m_simple_policy->get_status(cache_file_name)); m_simple_policy->lookup_object(cache_file_name); ASSERT_EQ(OBJ_CACHE_SKIP, m_simple_policy->get_status(cache_file_name)); - ASSERT_EQ(m_entry_num - m_promoted_lru.size() - m_promoting_lru.size() - 1, m_simple_policy->get_free_entry_num()); + ASSERT_EQ(m_cache_size - m_promoted_lru.size(), m_simple_policy->get_free_size()); ASSERT_EQ(m_promoting_lru.size() + 1, m_simple_policy->get_promoting_entry_num()); ASSERT_EQ(m_promoted_lru.size(), m_simple_policy->get_promoted_entry_num()); - m_simple_policy->update_status(cache_file_name, OBJ_CACHE_PROMOTED); + m_simple_policy->update_status(cache_file_name, OBJ_CACHE_PROMOTED, 1); m_promoted_lru.push_back(cache_file_name); ASSERT_EQ(OBJ_CACHE_PROMOTED, m_simple_policy->get_status(cache_file_name)); - ASSERT_EQ(m_entry_num - m_promoted_lru.size() - m_promoting_lru.size(), m_simple_policy->get_free_entry_num()); + ASSERT_EQ(m_cache_size - m_promoted_lru.size(), m_simple_policy->get_free_size()); ASSERT_EQ(m_promoting_lru.size(), m_simple_policy->get_promoting_entry_num()); ASSERT_EQ(m_promoted_lru.size(), m_simple_policy->get_promoted_entry_num()); } void insert_entry_into_promoting_lru(std::string cache_file_name) { - ASSERT_EQ(m_entry_num - m_promoted_lru.size() - m_promoting_lru.size(), m_simple_policy->get_free_entry_num()); + ASSERT_EQ(m_cache_size - m_promoted_lru.size(), m_simple_policy->get_free_size()); ASSERT_EQ(m_promoting_lru.size(), m_simple_policy->get_promoting_entry_num()); ASSERT_EQ(m_promoted_lru.size(), m_simple_policy->get_promoted_entry_num()); ASSERT_EQ(OBJ_CACHE_NONE, m_simple_policy->get_status(cache_file_name)); @@ -74,7 +75,7 @@ public: m_simple_policy->lookup_object(cache_file_name); m_promoting_lru.push_back(cache_file_name); ASSERT_EQ(OBJ_CACHE_SKIP, m_simple_policy->get_status(cache_file_name)); - ASSERT_EQ(m_entry_num - m_promoted_lru.size() - m_promoting_lru.size(), m_simple_policy->get_free_entry_num()); + ASSERT_EQ(m_cache_size - m_promoted_lru.size(), m_simple_policy->get_free_size()); ASSERT_EQ(m_promoting_lru.size(), m_simple_policy->get_promoting_entry_num()); ASSERT_EQ(m_promoted_lru.size(), m_simple_policy->get_promoted_entry_num()); } @@ -82,22 +83,22 @@ public: TEST_F(TestSimplePolicy, test_lookup_miss_and_no_free) { // exhaust cache space - uint64_t left_entry_num = m_entry_num - m_promoted_lru.size() - m_promoting_lru.size(); - for(uint64_t i = 0; i < left_entry_num; i++, ++m_entry_index) { + uint64_t left_entry_num = m_cache_size - m_promoted_lru.size(); + for (uint64_t i = 0; i < left_entry_num; i++, ++m_entry_index) { insert_entry_into_promoted_lru(generate_file_name(m_entry_index)); } - ASSERT_TRUE(0 == m_simple_policy->get_free_entry_num()); + ASSERT_TRUE(0 == m_simple_policy->get_free_size()); ASSERT_TRUE(m_simple_policy->lookup_object("no_this_cache_file_name") == OBJ_CACHE_SKIP); } -TEST_F(TestSimplePolicy, test_lookup_miss_and_free_space) { - ASSERT_TRUE(m_entry_num - m_promoting_lru.size() - m_promoted_lru.size() == m_simple_policy->get_free_entry_num()); +TEST_F(TestSimplePolicy, test_lookup_miss_and_have_free) { + ASSERT_TRUE(m_cache_size - m_promoted_lru.size() == m_simple_policy->get_free_size()); ASSERT_TRUE(m_simple_policy->lookup_object("miss_but_have_free_space_file_name") == OBJ_CACHE_NONE); ASSERT_TRUE(m_simple_policy->get_status("miss_but_have_free_space_file_name") == OBJ_CACHE_SKIP); } TEST_F(TestSimplePolicy, test_lookup_hit_and_promoting) { - ASSERT_TRUE(m_entry_num - m_promoting_lru.size() - m_promoted_lru.size() == m_simple_policy->get_free_entry_num()); + ASSERT_TRUE(m_cache_size - m_promoted_lru.size() == m_simple_policy->get_free_size()); insert_entry_into_promoting_lru("promoting_file_1"); insert_entry_into_promoting_lru("promoting_file_2"); insert_entry_into_promoted_lru(generate_file_name(++m_entry_index)); @@ -118,13 +119,13 @@ TEST_F(TestSimplePolicy, test_lookup_hit_and_promoting) { TEST_F(TestSimplePolicy, test_lookup_hit_and_promoted) { ASSERT_TRUE(m_promoted_lru.size() == m_simple_policy->get_promoted_entry_num()); - for(uint64_t index = 0; index < m_entry_index; index++) { + for (uint64_t index = 0; index < m_entry_index; index++) { ASSERT_TRUE(m_simple_policy->get_status(generate_file_name(index)) == OBJ_CACHE_PROMOTED); } } TEST_F(TestSimplePolicy, test_update_state_from_promoting_to_none) { - ASSERT_TRUE(m_entry_num - m_promoting_lru.size() - m_promoted_lru.size() == m_simple_policy->get_free_entry_num()); + ASSERT_TRUE(m_cache_size - m_promoted_lru.size() == m_simple_policy->get_free_size()); insert_entry_into_promoting_lru("promoting_to_none_file_1"); insert_entry_into_promoting_lru("promoting_to_none_file_2"); insert_entry_into_promoted_lru(generate_file_name(++m_entry_index)); @@ -168,7 +169,7 @@ TEST_F(TestSimplePolicy, test_update_state_from_promoting_to_none) { TEST_F(TestSimplePolicy, test_update_state_from_promoted_to_none) { ASSERT_TRUE(m_promoted_lru.size() == m_simple_policy->get_promoted_entry_num()); - for(uint64_t index = 0; index < m_entry_index; index++) { + for (uint64_t index = 0; index < m_entry_index; index++) { ASSERT_TRUE(m_simple_policy->get_status(generate_file_name(index)) == OBJ_CACHE_PROMOTED); m_simple_policy->update_status(generate_file_name(index), OBJ_CACHE_NONE); ASSERT_TRUE(m_simple_policy->get_status(generate_file_name(index)) == OBJ_CACHE_NONE); @@ -178,7 +179,7 @@ TEST_F(TestSimplePolicy, test_update_state_from_promoted_to_none) { } TEST_F(TestSimplePolicy, test_update_state_from_promoting_to_promoted) { - ASSERT_TRUE(m_entry_num - m_promoting_lru.size() - m_promoted_lru.size() == m_simple_policy->get_free_entry_num()); + ASSERT_TRUE(m_cache_size - m_promoted_lru.size() == m_simple_policy->get_free_size()); insert_entry_into_promoting_lru("promoting_to_promoted_file_1"); insert_entry_into_promoting_lru("promoting_to_promoted_file_2"); insert_entry_into_promoting_lru("promoting_to_promoted_file_3"); @@ -210,23 +211,24 @@ TEST_F(TestSimplePolicy, test_update_state_from_promoting_to_promoted) { TEST_F(TestSimplePolicy, test_evict_list_0) { std::list evict_entry_list; // 0.1 is watermark - ASSERT_TRUE((float)m_simple_policy->get_free_entry_num() > m_entry_num*0.1); + ASSERT_TRUE((float)m_simple_policy->get_free_size() > m_cache_size*0.1); m_simple_policy->get_evict_list(&evict_entry_list); ASSERT_TRUE(evict_entry_list.size() == 0); } TEST_F(TestSimplePolicy, test_evict_list_10) { - uint64_t left_entry_num = m_entry_num - m_promoted_lru.size() - m_promoting_lru.size(); - for(uint64_t i = 0; i < left_entry_num; i++, ++m_entry_index) { + uint64_t left_entry_num = m_cache_size - m_promoted_lru.size(); + for (uint64_t i = 0; i < left_entry_num; i++, ++m_entry_index) { insert_entry_into_promoted_lru(generate_file_name(m_entry_index)); } - ASSERT_TRUE(0 == m_simple_policy->get_free_entry_num()); + ASSERT_TRUE(0 == m_simple_policy->get_free_size()); std::list evict_entry_list; m_simple_policy->get_evict_list(&evict_entry_list); - ASSERT_TRUE(10 == evict_entry_list.size()); - ASSERT_TRUE(m_entry_num - 10 == m_simple_policy->get_promoted_entry_num()); + // evict 10% of old entries + ASSERT_TRUE(m_cache_size*0.1 == evict_entry_list.size()); + ASSERT_TRUE(m_cache_size - m_cache_size*0.1 == m_simple_policy->get_promoted_entry_num()); - for(auto it = evict_entry_list.begin(); it != evict_entry_list.end(); it++) { + for (auto it = evict_entry_list.begin(); it != evict_entry_list.end(); it++) { ASSERT_TRUE(*it == m_promoted_lru.front()); m_promoted_lru.erase(m_promoted_lru.begin()); } diff --git a/src/tools/immutable_object_cache/ObjectCacheStore.cc b/src/tools/immutable_object_cache/ObjectCacheStore.cc index b79ec408b2512..d268890af0615 100644 --- a/src/tools/immutable_object_cache/ObjectCacheStore.cc +++ b/src/tools/immutable_object_cache/ObjectCacheStore.cc @@ -28,7 +28,7 @@ ObjectCacheStore::ObjectCacheStore(CephContext *cct) m_cache_root_dir = cache_path + "/ceph_immutable_obj_cache/"; //TODO(): allow to set cache level - m_policy = new SimplePolicy(m_cct, object_cache_max_size/(4096*1024), 0.1); + m_policy = new SimplePolicy(m_cct, object_cache_max_size, 0.1); } ObjectCacheStore::~ObjectCacheStore() { @@ -162,7 +162,7 @@ int ObjectCacheStore::handle_promote_callback(int ret, bufferlist* read_buf, // update metadata ceph_assert(OBJ_CACHE_SKIP == m_policy->get_status(cache_file_name)); - m_policy->update_status(cache_file_name, OBJ_CACHE_PROMOTED); + m_policy->update_status(cache_file_name, OBJ_CACHE_PROMOTED, ret); ceph_assert(OBJ_CACHE_PROMOTED == m_policy->get_status(cache_file_name)); delete read_buf; diff --git a/src/tools/immutable_object_cache/Policy.h b/src/tools/immutable_object_cache/Policy.h index f1eef1b1a920d..cda8e69c917d6 100644 --- a/src/tools/immutable_object_cache/Policy.h +++ b/src/tools/immutable_object_cache/Policy.h @@ -23,7 +23,7 @@ public: virtual ~Policy(){}; virtual cache_status_t lookup_object(std::string) = 0; virtual int evict_entry(std::string) = 0; - virtual void update_status(std::string, cache_status_t) = 0; + virtual void update_status(std::string, cache_status_t, uint64_t size=0) = 0; virtual cache_status_t get_status(std::string) = 0; virtual void get_evict_list(std::list* obj_list) = 0; }; diff --git a/src/tools/immutable_object_cache/SimplePolicy.cc b/src/tools/immutable_object_cache/SimplePolicy.cc index ab02def4fe0cc..8fe70b4e00d5f 100644 --- a/src/tools/immutable_object_cache/SimplePolicy.cc +++ b/src/tools/immutable_object_cache/SimplePolicy.cc @@ -13,30 +13,24 @@ namespace ceph { namespace immutable_obj_cache { -SimplePolicy::SimplePolicy(CephContext *cct, uint64_t block_num, +SimplePolicy::SimplePolicy(CephContext *cct, uint64_t cache_size, float watermark) - : cct(cct), m_watermark(watermark), m_entry_count(block_num), - m_cache_map_lock("rbd::cache::SimplePolicy::m_cache_map_lock"), - m_free_list_lock("rbd::cache::SimplePolicy::m_free_list_lock") { + : cct(cct), m_watermark(watermark), m_max_cache_size(cache_size), + m_cache_map_lock("rbd::cache::SimplePolicy::m_cache_map_lock") { ldout(cct, 20) << dendl; m_max_inflight_ops = cct->_conf.get_val("immutable_object_cache_max_inflight_ops"); - for(uint64_t i = 0; i < m_entry_count; i++) { - m_free_list.push_back(new Entry()); - } + m_cache_size = 0; } SimplePolicy::~SimplePolicy() { ldout(cct, 20) << dendl; for (auto it: m_cache_map) { - Entry* entry = reinterpret_cast(it.second); + Entry* entry = (it.second); delete entry; } - for(auto it : m_free_list) { - Entry* entry = it; - delete entry; - } + } cache_status_t SimplePolicy::alloc_entry(std::string file_name) { @@ -49,20 +43,16 @@ cache_status_t SimplePolicy::alloc_entry(std::string file_name) { ldout(cct, 20) << "object is under promoting: " << file_name << dendl; return OBJ_CACHE_SKIP; } - m_free_list_lock.lock(); - if (m_free_list.size() && (inflight_ops < m_max_inflight_ops)) { - Entry* entry = m_free_list.front(); + if ((m_cache_size < m_max_cache_size) && (inflight_ops < m_max_inflight_ops)) { + Entry* entry = new Entry(); ceph_assert(entry != nullptr); - m_free_list.pop_front(); - m_free_list_lock.unlock(); m_cache_map[file_name] = entry; wlocker.unlock(); update_status(file_name, OBJ_CACHE_SKIP); return OBJ_CACHE_NONE; // start promotion request } - m_free_list_lock.unlock(); // if there's no free entry, return skip to read from rados return OBJ_CACHE_SKIP; } @@ -90,7 +80,7 @@ cache_status_t SimplePolicy::lookup_object(std::string file_name) { } void SimplePolicy::update_status(std::string file_name, - cache_status_t new_status) { + cache_status_t new_status, uint64_t size) { ldout(cct, 20) << "update status for: " << file_name << " new status = " << new_status << dendl; @@ -116,6 +106,8 @@ void SimplePolicy::update_status(std::string file_name, if (entry->status == OBJ_CACHE_SKIP && new_status== OBJ_CACHE_PROMOTED) { m_promoted_lru.lru_insert_top(entry); entry->status = new_status; + entry->size = size; + m_cache_size += entry->size; inflight_ops--; return; } @@ -125,10 +117,7 @@ void SimplePolicy::update_status(std::string file_name, // mark this entry as free entry->file_name = ""; entry->status = new_status; - { - Mutex::Locker free_list_locker(m_free_list_lock); - m_free_list.push_back(entry); - } + m_cache_map.erase(entry_it); inflight_ops--; return; @@ -137,14 +126,14 @@ void SimplePolicy::update_status(std::string file_name, // to evict if (entry->status == OBJ_CACHE_PROMOTED && new_status== OBJ_CACHE_NONE) { // mark this entry as free + uint64_t size = entry->size; entry->file_name = ""; + entry->size = 0; entry->status = new_status; - { - Mutex::Locker free_list_locker(m_free_list_lock); - m_free_list.push_back(entry); - } + m_promoted_lru.lru_remove(entry); m_cache_map.erase(entry_it); + m_cache_size -= size; return; } @@ -175,8 +164,8 @@ void SimplePolicy::get_evict_list(std::list* obj_list) { RWLock::WLocker locker(m_cache_map_lock); // check free ratio, pop entries from LRU - if ((float)m_free_list.size() / m_entry_count < m_watermark) { - int evict_num = m_entry_count * 0.1; //TODO(): make this configurable + if ((double)m_cache_size / m_max_cache_size > (1 - m_watermark)) { + int evict_num = m_cache_map.size() * 0.1; //TODO(): make this configurable for (int i = 0; i < evict_num; i++) { Entry* entry = reinterpret_cast(m_promoted_lru.lru_expire()); if (entry == nullptr) { @@ -190,9 +179,8 @@ void SimplePolicy::get_evict_list(std::list* obj_list) { } // for unit test -uint64_t SimplePolicy::get_free_entry_num() { - Mutex::Locker free_list_locker(m_free_list_lock); - return m_free_list.size(); +uint64_t SimplePolicy::get_free_size() { + return m_max_cache_size - m_cache_size; } uint64_t SimplePolicy::get_promoting_entry_num() { diff --git a/src/tools/immutable_object_cache/SimplePolicy.h b/src/tools/immutable_object_cache/SimplePolicy.h index 7dce7840fd322..7d97bdc9a1599 100644 --- a/src/tools/immutable_object_cache/SimplePolicy.h +++ b/src/tools/immutable_object_cache/SimplePolicy.h @@ -24,13 +24,14 @@ public: cache_status_t lookup_object(std::string file_name); cache_status_t get_status(std::string file_name); - void update_status(std::string file_name, cache_status_t new_status); + void update_status(std::string file_name, + cache_status_t new_status, uint64_t size=0); int evict_entry(std::string file_name); void get_evict_list(std::list* obj_list); - uint64_t get_free_entry_num(); + uint64_t get_free_size(); uint64_t get_promoting_entry_num(); uint64_t get_promoted_entry_num(); std::string get_evict_entry(); @@ -43,19 +44,23 @@ private: cache_status_t status; Entry() : status(OBJ_CACHE_NONE){} std::string file_name; + uint64_t size; }; CephContext* cct; float m_watermark; uint64_t m_entry_count; uint64_t m_max_inflight_ops; + uint64_t m_max_cache_size; std::atomic inflight_ops = 0; std::unordered_map m_cache_map; RWLock m_cache_map_lock; + std::deque m_free_list; - Mutex m_free_list_lock; + + std::atomic m_cache_size; LRU m_promoted_lru; }; -- 2.39.5