os/bluestore: modify cache shutdown

Signed-off-by: Igor Fedotov <ifedotov@suse.com>
(cherry picked from commit fafaa0809d4c213923659aacc6f1b6b493157e73)

 Conflicts:
 trivial - src/os/bluestore/BlueStore.h

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 8827a3f0548e72ec6009363647b861043b246a51..442a93c66bb276ff0121155d1856ffd1ff506087 100644 (file)
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -856,6 +856,7 @@ struct LruOnodeCacheShard : public BlueStore::OnodeCacheShard {
       ++num_pinned;
     }
     ++num; // we count both pinned and unpinned entries
+    dout(20) << __func__ << " " << this << " " << o->oid << " added, num=" << num << dendl;
   }
   void _rm(BlueStore::Onode* o) override
   {
@@ -867,19 +868,20 @@ struct LruOnodeCacheShard : public BlueStore::OnodeCacheShard {
     }
     ceph_assert(num);
     --num;
+    dout(20) << __func__ << " " << this << " " << " " << o->oid << " removed, num=" << num << dendl;
   }
   void _pin(BlueStore::Onode* o) override
   {
     lru.erase(lru.iterator_to(*o));
     ++num_pinned;
-    dout(30) << __func__ << " " << o->oid << " pinned" << dendl;
+    dout(20) << __func__ << this << " " << " " << " " << o->oid << " pinned" << dendl;
   }
   void _unpin(BlueStore::Onode* o) override
   {
     lru.push_front(*o);
     ceph_assert(num_pinned);
     --num_pinned;
-    dout(30) << __func__ << " " << o->oid << " unpinned" << dendl;
+    dout(20) << __func__ << this << " " << " " << " " << o->oid << " unpinned" << dendl;
   }
 
   void _trim_to(uint64_t new_size) override
@@ -895,7 +897,7 @@ struct LruOnodeCacheShard : public BlueStore::OnodeCacheShard {
     num -= n;
     while (n-- > 0) {
       BlueStore::Onode *o = &*p;
-      dout(30) << __func__ << "  rm " << o->oid << " "
+      dout(20) << __func__ << "  rm " << o->oid << " "
                << o->nref << " " << o->cached << " " << o->pinned << dendl;
       if (p != lru.begin()) {
         lru.erase(p--);
@@ -905,7 +907,7 @@ struct LruOnodeCacheShard : public BlueStore::OnodeCacheShard {
       }
       auto pinned = !o->pop_cache();
       ceph_assert(!pinned);
-      o->c->onode_map.remove(o->oid);
+      o->c->onode_map._remove(o->oid);
     }
   }
   void move_pinned(OnodeCacheShard *to, BlueStore::Onode *o) override
@@ -1630,13 +1632,19 @@ BlueStore::OnodeRef BlueStore::OnodeSpace::add(const ghobject_t& oid,
                          << dendl;
     return p->second;
   }
-  ldout(cache->cct, 30) << __func__ << " " << oid << " " << o << dendl;
+  ldout(cache->cct, 20) << __func__ << " " << oid << " " << o << dendl;
   onode_map[oid] = o;
   cache->_add(o.get(), 1);
   cache->_trim();
   return o;
 }
 
+void BlueStore::OnodeSpace::_remove(const ghobject_t& oid)
+{
+  ldout(cache->cct, 20) << __func__ << " " << oid << " " << dendl;
+  onode_map.erase(oid);
+}
+
 BlueStore::OnodeRef BlueStore::OnodeSpace::lookup(const ghobject_t& oid)
 {
   ldout(cache->cct, 30) << __func__ << dendl;
@@ -1674,7 +1682,7 @@ BlueStore::OnodeRef BlueStore::OnodeSpace::lookup(const ghobject_t& oid)
 void BlueStore::OnodeSpace::clear()
 {
   std::lock_guard l(cache->lock);
-  ldout(cache->cct, 10) << __func__ << dendl;
+  ldout(cache->cct, 10) << __func__ << " " << onode_map.size()<< dendl;
   for (auto &p : onode_map) {
     cache->_rm(p.second.get());
   }
@@ -7078,7 +7086,7 @@ int BlueStore::_mount(bool kv_only, bool open_db)
  out_stop:
   _kv_stop();
  out_coll:
-  _flush_cache();
+  _shutdown_cache();
  out_db:
   _close_db_and_around(false);
  out_bdev:
@@ -7102,7 +7110,7 @@ int BlueStore::umount()
     mempool_thread.shutdown();
     dout(20) << __func__ << " stopping kv thread" << dendl;
     _kv_stop();
-    _flush_cache();
+    _shutdown_cache();
     dout(20) << __func__ << " closing" << dendl;
 
   }
@@ -8222,7 +8230,7 @@ int BlueStore::_fsck(BlueStore::FSCKDepth depth, bool repair)
 
 out_scan:
   mempool_thread.shutdown();
-  _flush_cache();
+  _shutdown_cache();
 out_db:
   _close_db_and_around(false);
 out_bdev:
@@ -15400,22 +15408,15 @@ void BlueStore::generate_db_histogram(Formatter *f)
 
 }
 
-void BlueStore::_flush_cache()
+void BlueStore::_shutdown_cache()
 {
   dout(10) << __func__ << dendl;
-  for (auto i : onode_cache_shards) {
-    i->flush();
-    ceph_assert(i->empty());
-  }
   for (auto i : buffer_cache_shards) {
     i->flush();
     ceph_assert(i->empty());
   }
   for (auto& p : coll_map) {
-    if (!p.second->onode_map.empty()) {
-      derr << __func__ << " stray onodes on " << p.first << dendl;
-      p.second->onode_map.dump<0>(cct);
-    }
+    p.second->onode_map.clear();
     if (!p.second->shared_blob_set.empty()) {
       derr << __func__ << " stray shared blobs on " << p.first << dendl;
       p.second->shared_blob_set.dump<0>(cct);
@@ -15424,6 +15425,9 @@ void BlueStore::_flush_cache()
     ceph_assert(p.second->shared_blob_set.empty());
   }
   coll_map.clear();
+  for (auto i : onode_cache_shards) {
+    ceph_assert(i->empty());
+  }
 }
 
 // For external caller.

diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 29bba03a7b492b260e87ce048286fd270775a9aa..cac64a08d5df482c6c00ee07b5847d22ca839a1c 100644 (file)
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -1287,6 +1287,8 @@ public:
 
     friend class Collection; // for split_cache()
 
+    friend struct LruOnodeCacheShard;
+    void _remove(const ghobject_t& oid);
   public:
     OnodeSpace(OnodeCacheShard *c) : cache(c) {}
     ~OnodeSpace() {
@@ -1295,9 +1297,6 @@ public:
 
     OnodeRef add(const ghobject_t& oid, OnodeRef& o);
     OnodeRef lookup(const ghobject_t& o);
-    void remove(const ghobject_t& oid) {
-      onode_map.erase(oid);
-    }
     void rename(OnodeRef& o, const ghobject_t& old_oid,
                const ghobject_t& new_oid,
                const mempool::bluestore_cache_other::string& new_okey);
@@ -2569,7 +2568,7 @@ public:
 
   void get_db_statistics(Formatter *f) override;
   void generate_db_histogram(Formatter *f) override;
-  void _flush_cache();
+  void _shutdown_cache();
   int flush_cache(ostream *os = NULL) override;
   void dump_perf_counters(Formatter *f) override {
     f->open_object_section("perf_counters");