From: Igor Fedotov Date: Wed, 8 Jul 2020 13:54:33 +0000 (+0300) Subject: os/bluestore: modify cache shutdown X-Git-Tag: v15.2.5~5^2~1 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=a1318f4a4a97142ade49272a7e96c020e8c46de5;p=ceph.git os/bluestore: modify cache shutdown Signed-off-by: Igor Fedotov (cherry picked from commit fafaa0809d4c213923659aacc6f1b6b493157e73) Conflicts: trivial - src/os/bluestore/BlueStore.h --- diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 8827a3f0548e7..442a93c66bb27 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -856,6 +856,7 @@ struct LruOnodeCacheShard : public BlueStore::OnodeCacheShard { ++num_pinned; } ++num; // we count both pinned and unpinned entries + dout(20) << __func__ << " " << this << " " << o->oid << " added, num=" << num << dendl; } void _rm(BlueStore::Onode* o) override { @@ -867,19 +868,20 @@ struct LruOnodeCacheShard : public BlueStore::OnodeCacheShard { } ceph_assert(num); --num; + dout(20) << __func__ << " " << this << " " << " " << o->oid << " removed, num=" << num << dendl; } void _pin(BlueStore::Onode* o) override { lru.erase(lru.iterator_to(*o)); ++num_pinned; - dout(30) << __func__ << " " << o->oid << " pinned" << dendl; + dout(20) << __func__ << this << " " << " " << " " << o->oid << " pinned" << dendl; } void _unpin(BlueStore::Onode* o) override { lru.push_front(*o); ceph_assert(num_pinned); --num_pinned; - dout(30) << __func__ << " " << o->oid << " unpinned" << dendl; + dout(20) << __func__ << this << " " << " " << " " << o->oid << " unpinned" << dendl; } void _trim_to(uint64_t new_size) override @@ -895,7 +897,7 @@ struct LruOnodeCacheShard : public BlueStore::OnodeCacheShard { num -= n; while (n-- > 0) { BlueStore::Onode *o = &*p; - dout(30) << __func__ << " rm " << o->oid << " " + dout(20) << __func__ << " rm " << o->oid << " " << o->nref << " " << o->cached << " " << o->pinned << dendl; if (p != lru.begin()) { lru.erase(p--); @@ -905,7 +907,7 @@ struct LruOnodeCacheShard : public BlueStore::OnodeCacheShard { } auto pinned = !o->pop_cache(); ceph_assert(!pinned); - o->c->onode_map.remove(o->oid); + o->c->onode_map._remove(o->oid); } } void move_pinned(OnodeCacheShard *to, BlueStore::Onode *o) override @@ -1630,13 +1632,19 @@ BlueStore::OnodeRef BlueStore::OnodeSpace::add(const ghobject_t& oid, << dendl; return p->second; } - ldout(cache->cct, 30) << __func__ << " " << oid << " " << o << dendl; + ldout(cache->cct, 20) << __func__ << " " << oid << " " << o << dendl; onode_map[oid] = o; cache->_add(o.get(), 1); cache->_trim(); return o; } +void BlueStore::OnodeSpace::_remove(const ghobject_t& oid) +{ + ldout(cache->cct, 20) << __func__ << " " << oid << " " << dendl; + onode_map.erase(oid); +} + BlueStore::OnodeRef BlueStore::OnodeSpace::lookup(const ghobject_t& oid) { ldout(cache->cct, 30) << __func__ << dendl; @@ -1674,7 +1682,7 @@ BlueStore::OnodeRef BlueStore::OnodeSpace::lookup(const ghobject_t& oid) void BlueStore::OnodeSpace::clear() { std::lock_guard l(cache->lock); - ldout(cache->cct, 10) << __func__ << dendl; + ldout(cache->cct, 10) << __func__ << " " << onode_map.size()<< dendl; for (auto &p : onode_map) { cache->_rm(p.second.get()); } @@ -7078,7 +7086,7 @@ int BlueStore::_mount(bool kv_only, bool open_db) out_stop: _kv_stop(); out_coll: - _flush_cache(); + _shutdown_cache(); out_db: _close_db_and_around(false); out_bdev: @@ -7102,7 +7110,7 @@ int BlueStore::umount() mempool_thread.shutdown(); dout(20) << __func__ << " stopping kv thread" << dendl; _kv_stop(); - _flush_cache(); + _shutdown_cache(); dout(20) << __func__ << " closing" << dendl; } @@ -8222,7 +8230,7 @@ int BlueStore::_fsck(BlueStore::FSCKDepth depth, bool repair) out_scan: mempool_thread.shutdown(); - _flush_cache(); + _shutdown_cache(); out_db: _close_db_and_around(false); out_bdev: @@ -15400,22 +15408,15 @@ void BlueStore::generate_db_histogram(Formatter *f) } -void BlueStore::_flush_cache() +void BlueStore::_shutdown_cache() { dout(10) << __func__ << dendl; - for (auto i : onode_cache_shards) { - i->flush(); - ceph_assert(i->empty()); - } for (auto i : buffer_cache_shards) { i->flush(); ceph_assert(i->empty()); } for (auto& p : coll_map) { - if (!p.second->onode_map.empty()) { - derr << __func__ << " stray onodes on " << p.first << dendl; - p.second->onode_map.dump<0>(cct); - } + p.second->onode_map.clear(); if (!p.second->shared_blob_set.empty()) { derr << __func__ << " stray shared blobs on " << p.first << dendl; p.second->shared_blob_set.dump<0>(cct); @@ -15424,6 +15425,9 @@ void BlueStore::_flush_cache() ceph_assert(p.second->shared_blob_set.empty()); } coll_map.clear(); + for (auto i : onode_cache_shards) { + ceph_assert(i->empty()); + } } // For external caller. diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index 29bba03a7b492..cac64a08d5df4 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -1287,6 +1287,8 @@ public: friend class Collection; // for split_cache() + friend struct LruOnodeCacheShard; + void _remove(const ghobject_t& oid); public: OnodeSpace(OnodeCacheShard *c) : cache(c) {} ~OnodeSpace() { @@ -1295,9 +1297,6 @@ public: OnodeRef add(const ghobject_t& oid, OnodeRef& o); OnodeRef lookup(const ghobject_t& o); - void remove(const ghobject_t& oid) { - onode_map.erase(oid); - } void rename(OnodeRef& o, const ghobject_t& old_oid, const ghobject_t& new_oid, const mempool::bluestore_cache_other::string& new_okey); @@ -2569,7 +2568,7 @@ public: void get_db_statistics(Formatter *f) override; void generate_db_histogram(Formatter *f) override; - void _flush_cache(); + void _shutdown_cache(); int flush_cache(ostream *os = NULL) override; void dump_perf_counters(Formatter *f) override { f->open_object_section("perf_counters");