From 1c6bc3893bf4cadae344ef19290f5c7e55dd5a62 Mon Sep 17 00:00:00 2001 From: Deepika Upadhyay Date: Wed, 15 Apr 2020 17:31:46 +0530 Subject: [PATCH] split mempool allocation for bluestore_cache_other While doing root cause analysis it bluestore_cache_other gives a bit of a crude estimate, something more helpful would be to have it split into the following fields: - bluestore_cache_onode - bluestore_cache_meta - bluestore_onode_attrs Signed-off-by: Deepika Upadhyay --- src/include/mempool.h | 5 ++++- src/os/bluestore/BlueStore.cc | 18 +++++++++--------- src/os/bluestore/BlueStore.h | 21 +++++++++++---------- src/os/bluestore/bluestore_types.h | 4 +++- src/test/objectstore/store_test.cc | 7 +++++-- 5 files changed, 32 insertions(+), 23 deletions(-) diff --git a/src/include/mempool.h b/src/include/mempool.h index 79cf862c8cd7b..1f597b8c42dab 100644 --- a/src/include/mempool.h +++ b/src/include/mempool.h @@ -151,12 +151,15 @@ namespace mempool { f(bluestore_alloc) \ f(bluestore_cache_data) \ f(bluestore_cache_onode) \ + f(bluestore_cache_meta) \ f(bluestore_cache_other) \ f(bluestore_fsck) \ f(bluestore_txc) \ - f(bluestore_writing_deferred) \ + f(bluestore_writing_deferred) \ f(bluestore_writing) \ f(bluefs) \ + f(bluefs_file_reader) \ + f(bluefs_file_writer) \ f(buffer_anon) \ f(buffer_meta) \ f(osd) \ diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 212f90b8a3bc6..42f2b56c9c130 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -1646,7 +1646,7 @@ void BlueStore::OnodeSpace::rename( OnodeRef& oldo, const ghobject_t& old_oid, const ghobject_t& new_oid, - const mempool::bluestore_cache_other::string& new_okey) + const mempool::bluestore_cache_meta::string& new_okey) { std::lock_guard l(cache->lock); ldout(cache->cct, 30) << __func__ << " " << old_oid << " -> " << new_oid @@ -3292,7 +3292,7 @@ BlueStore::Onode* BlueStore::Onode::decode( auto p = v.front().begin_deep(); on->onode.decode(p); for (auto& i : on->onode.attrs) { - i.second.reassign_to_mempool(mempool::mempool_bluestore_cache_other); + i.second.reassign_to_mempool(mempool::mempool_bluestore_cache_onode); } // initialize extent_map @@ -3301,7 +3301,7 @@ BlueStore::Onode* BlueStore::Onode::decode( denc(on->extent_map.inline_bl, p); on->extent_map.decode_some(on->extent_map.inline_bl); on->extent_map.inline_bl.reassign_to_mempool( - mempool::mempool_bluestore_cache_other); + mempool::mempool_bluestore_cache_onode); } else { on->extent_map.init_shards(false, false); @@ -10416,7 +10416,7 @@ int BlueStore::getattr( int r; { std::shared_lock l(c->lock); - mempool::bluestore_cache_other::string k(name); + mempool::bluestore_cache_meta::string k(name); OnodeRef o = c->get_onode(oid, false); if (!o || !o->exists) { @@ -14687,10 +14687,10 @@ int BlueStore::_setattr(TransContext *txc, if (val.is_partial()) { auto& b = o->onode.attrs[name.c_str()] = bufferptr(val.c_str(), val.length()); - b.reassign_to_mempool(mempool::mempool_bluestore_cache_other); + b.reassign_to_mempool(mempool::mempool_bluestore_cache_onode); } else { auto& b = o->onode.attrs[name.c_str()] = val; - b.reassign_to_mempool(mempool::mempool_bluestore_cache_other); + b.reassign_to_mempool(mempool::mempool_bluestore_cache_onode); } txc->write_onode(o); dout(10) << __func__ << " " << c->cid << " " << o->oid @@ -14713,10 +14713,10 @@ int BlueStore::_setattrs(TransContext *txc, if (p->second.is_partial()) { auto& b = o->onode.attrs[p->first.c_str()] = bufferptr(p->second.c_str(), p->second.length()); - b.reassign_to_mempool(mempool::mempool_bluestore_cache_other); + b.reassign_to_mempool(mempool::mempool_bluestore_cache_onode); } else { auto& b = o->onode.attrs[p->first.c_str()] = p->second; - b.reassign_to_mempool(mempool::mempool_bluestore_cache_other); + b.reassign_to_mempool(mempool::mempool_bluestore_cache_onode); } } txc->write_onode(o); @@ -15128,7 +15128,7 @@ int BlueStore::_rename(TransContext *txc, << new_oid << dendl; int r; ghobject_t old_oid = oldo->oid; - mempool::bluestore_cache_other::string new_okey; + mempool::bluestore_cache_meta::string new_okey; if (newo) { if (newo->exists) { diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index 74e9dd5176bae..37d0979092b7c 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -285,7 +285,7 @@ public: boost::intrusive::list_member_hook<>, &Buffer::state_item> > state_list_t; - mempool::bluestore_cache_other::map> + mempool::bluestore_cache_meta::map> buffer_map; // we use a bare intrusive list here instead of std::map because @@ -474,7 +474,7 @@ public: // we use a bare pointer because we don't want to affect the ref // count - mempool::bluestore_cache_other::unordered_map sb_map; + mempool::bluestore_cache_meta::unordered_map sb_map; SharedBlobRef lookup(uint64_t sbid) { std::lock_guard l(lock); @@ -689,7 +689,7 @@ public: #endif }; typedef boost::intrusive_ptr BlobRef; - typedef mempool::bluestore_cache_other::map blob_map_t; + typedef mempool::bluestore_cache_meta::map blob_map_t; /// a logical extent, pointing to (some portion of) a blob typedef boost::intrusive::set_base_hook > ExtentBase; //making an alias to avoid build warnings @@ -797,7 +797,7 @@ public: bool loaded = false; ///< true if shard is loaded bool dirty = false; ///< true if shard is dirty and needs reencoding }; - mempool::bluestore_cache_other::vector shards; ///< shards + mempool::bluestore_cache_meta::vector shards; ///< shards ceph::buffer::list inline_bl; ///< cached encoded map, if unsharded; empty=>dirty @@ -1064,7 +1064,7 @@ public: ghobject_t oid; /// key under PREFIX_OBJ where we are stored - mempool::bluestore_cache_other::string key; + mempool::bluestore_cache_meta::string key; boost::intrusive::list_member_hook<> lru_item; @@ -1086,7 +1086,7 @@ public: ceph::condition_variable flush_cond; ///< wait here for uncommitted txns Onode(Collection *c, const ghobject_t& o, - const mempool::bluestore_cache_other::string& k) + const mempool::bluestore_cache_meta::string& k) : nref(0), c(c), oid(o), @@ -1288,7 +1288,7 @@ public: private: /// forward lookups - mempool::bluestore_cache_other::unordered_map onode_map; + mempool::bluestore_cache_onode::unordered_map onode_map; friend struct Collection; // for split_cache() @@ -1304,7 +1304,7 @@ public: OnodeRef lookup(const ghobject_t& o); void rename(OnodeRef& o, const ghobject_t& old_oid, const ghobject_t& new_oid, - const mempool::bluestore_cache_other::string& new_okey); + const mempool::bluestore_cache_meta::string& new_okey); void clear(); bool empty(); @@ -2211,8 +2211,9 @@ private: MetaCache(BlueStore *s) : MempoolCache(s) {}; virtual uint64_t _get_used_bytes() const { - return mempool::bluestore_cache_other::allocated_bytes() + - mempool::bluestore_cache_onode::allocated_bytes(); + return mempool::bluestore_cache_meta::allocated_bytes() + + mempool::bluestore_cache_other::allocated_bytes() + + mempool::bluestore_cache_onode::allocated_bytes(); } virtual std::string get_cache_name() const { diff --git a/src/os/bluestore/bluestore_types.h b/src/os/bluestore/bluestore_types.h index a9c3bf3ae3a89..7821a052111d8 100644 --- a/src/os/bluestore/bluestore_types.h +++ b/src/os/bluestore/bluestore_types.h @@ -936,7 +936,9 @@ std::ostream& operator<<(std::ostream& out, const bluestore_shared_blob_t& o); struct bluestore_onode_t { uint64_t nid = 0; ///< numeric id (locally unique) uint64_t size = 0; ///< object size - std::map attrs; ///< attrs + // FIXME: bufferptr does not have a mempool + std::map attrs; +// mempool::bluestore_cache_onode::map attrs; ///< attrs struct shard_info { uint32_t offset = 0; ///< logical offset for start of shard diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc index b5ec4c3079347..4884232b925a5 100644 --- a/src/test/objectstore/store_test.cc +++ b/src/test/objectstore/store_test.cc @@ -6438,15 +6438,18 @@ TEST_P(StoreTestSpecificAUSize, TooManyBlobsTest) { #if defined(WITH_BLUESTORE) void get_mempool_stats(uint64_t* total_bytes, uint64_t* total_items) { + uint64_t meta_allocated = mempool::bluestore_cache_meta::allocated_bytes(); uint64_t onode_allocated = mempool::bluestore_cache_onode::allocated_bytes(); uint64_t other_allocated = mempool::bluestore_cache_other::allocated_bytes(); + uint64_t meta_items = mempool::bluestore_cache_meta::allocated_items(); uint64_t onode_items = mempool::bluestore_cache_onode::allocated_items(); uint64_t other_items = mempool::bluestore_cache_other::allocated_items(); - cout << "onode(" << onode_allocated << "/" << onode_items + cout << "meta(" << meta_allocated << "/" << meta_items + << ") onode(" << onode_allocated << "/" << onode_items << ") other(" << other_allocated << "/" << other_items << ")" << std::endl; - *total_bytes = onode_allocated + other_allocated; + *total_bytes = meta_allocated + onode_allocated + other_allocated; *total_items = onode_items; } -- 2.39.5