os/bluestore: correct zoned freelist when device write pointers are ahead

If we find that the device's write pointers are ahead of ours, then we
need to advance the allocator *and* update the ondisk freelist so that we
don't see the adjustment on the next bluestore restart too.

(Actually, we could perhaps not manage the freelist at all and rely
entirely on the device's write pointers and a metadata scan on startup
w/ the null freelist mode, but it is cheap to track, and I like having
some redundancy so that we see if/when the device does something
unexpected.)

Signed-off-by: Sage Weil <sage@newdream.net>

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 459f7070b1bca0697b704880db86625db4b9b72a..f1269c757b5f17f5c5c77fcebec4a81347c0aa5a 100644 (file)
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -5578,7 +5578,7 @@ int BlueStore::_create_alloc()
   return 0;
 }
 
-int BlueStore::_init_alloc()
+int BlueStore::_init_alloc(std::map<uint64_t, uint64_t> *zone_adjustments)
 {
   int r = _create_alloc();
   if (r < 0) {
@@ -5616,6 +5616,7 @@ int BlueStore::_init_alloc()
                 << " device write pointer 0x" << p
                 << " > bluestore pointer 0x" << zones[i].write_pointer
                 << ", advancing 0x" << delta << std::dec << dendl;
+       (*zone_adjustments)[zones[i].write_pointer] = delta;
        zones[i].num_dead_bytes += delta;
        zones[i].write_pointer = p;
       }
@@ -5633,6 +5634,7 @@ int BlueStore::_init_alloc()
            << ", free 0x" << shared_alloc.a->get_free()
            << ", fragmentation " << shared_alloc.a->get_fragmentation()
            << std::dec << dendl;
+
     return 0;
   }
 #endif
@@ -5692,6 +5694,25 @@ int BlueStore::_init_alloc()
   return 0;
 }
 
+void BlueStore::_post_init_alloc(const std::map<uint64_t, uint64_t>& zone_adjustments)
+{
+#ifdef HAVE_LIBZBD
+  assert(bdev->is_smr());
+  dout(1) << __func__ << " adjusting freelist based on device write pointers" << dendl;
+  auto f = dynamic_cast<ZonedFreelistManager*>(fm);
+  ceph_assert(f);
+  KeyValueDB::Transaction t = db->get_transaction();
+  for (auto& i : zone_adjustments) {
+    // allocate AND release since this gap is now dead space
+    // note that the offset is imprecise, but only need to select the zone
+    f->allocate(i.first, i.second, t);
+    f->release(i.first, i.second, t);
+  }
+  int r = db->submit_transaction_sync(t);
+  ceph_assert(r == 0);
+#endif
+}
+
 void BlueStore::_close_alloc()
 {
   ceph_assert(bdev);
@@ -6077,6 +6098,10 @@ int BlueStore::_open_db_and_around(bool read_only, bool to_repair)
     }
   }
 
+  // SMR devices may require a freelist adjustment, but that can only happen after
+  // the db is read-write. we'll stash pending changes here.
+  std::map<uint64_t, uint64_t> zone_adjustments;
+
   int r = _open_path();
   if (r < 0)
     return r;
@@ -6114,7 +6139,7 @@ int BlueStore::_open_db_and_around(bool read_only, bool to_repair)
   if (r < 0)
     goto out_db;
 
-  r = _init_alloc();
+  r = _init_alloc(&zone_adjustments);
   if (r < 0)
     goto out_fm;
 
@@ -6130,6 +6155,11 @@ int BlueStore::_open_db_and_around(bool read_only, bool to_repair)
     goto out_alloc;
   }
 
+  if (!read_only && !zone_adjustments.empty()) {
+    // for SMR devices that have freelist mismatch with device write pointers
+    _post_init_alloc(zone_adjustments);
+  }
+
   // when function is called in repair mode (to_repair=true) we skip db->open()/create()
   // we can't change bluestore allocation so no need to invlidate allocation-file
   if (fm->is_null_manager() && !read_only && !to_repair) {

diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 587503fd039d7232f2a7709782a62aa0bf022344..110389fd545ec4a215a7bcdff15753f7cc6b037e 100644 (file)
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -2394,7 +2394,8 @@ private:
   void _close_fm();
   int _write_out_fm_meta(uint64_t target_size);
   int _create_alloc();
-  int _init_alloc();
+  int _init_alloc(std::map<uint64_t, uint64_t> *zone_adjustments);
+  void _post_init_alloc(const std::map<uint64_t, uint64_t>& zone_adjustments);
   void _close_alloc();
   int _open_collections();
   void _fsck_collections(int64_t* errors);