os/bluestore: Added rescue procedure for bluefs log replay

This is a procedure tries to find on disk unreachable extents and pretend they were
already a part of bluefs log. If this gives proper crc, accept it.

Fixes: https://tracker.ceph.com/issues/46552
Signed-off-by: Adam Kupczyk <akupczyk@redhat.com>
(cherry-picked from commit 40e94a106a8adff6304905d59e1634801d2c9890)

Conflicts:
	src/os/bluestore/BlueFS.cc

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index b0562247dda99cac9344264548cfffa0e3550c54..cf1ba8c0a5e31df2b3ca3c6b80143e08bede7a04 100644 (file)
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -914,6 +914,7 @@ OPTION(bluefs_sync_write, OPT_BOOL)
 OPTION(bluefs_allocator, OPT_STR)     // stupid | bitmap
 OPTION(bluefs_preextend_wal_files, OPT_BOOL)  // this *requires* that rocksdb has recycling enabled
 OPTION(bluefs_log_replay_check_allocations, OPT_BOOL)
+OPTION(bluefs_replay_recovery, OPT_BOOL)
 
 OPTION(bluestore_bluefs, OPT_BOOL)
 OPTION(bluestore_bluefs_env_mirror, OPT_BOOL) // mirror to normal Env for debug

diff --git a/src/common/options.cc b/src/common/options.cc
index f3c848579dd60a04a57fe10c014fd9b9131a3e8f..87a708c9ba45588361339b16944441d7de9c62f1 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -4022,6 +4022,13 @@ std::vector<Option> get_global_options() {
       .set_default(true)
       .set_description("Enables checks for allocations consistency during log replay"),
 
+    Option("bluefs_replay_recovery", Option::TYPE_BOOL, Option::LEVEL_DEV)
+    .set_default(false)
+    .set_description("Attempt to read bluefs log so large that it became unreadable.")
+    .set_long_description("If BlueFS log grows to extreme sizes (200GB+) it is likely that it becames unreadable. "
+                         "This options enables heuristics that scans devices for missing data. "
+                         "DO NOT ENABLE BY DEFAULT"),
+
     Option("bluestore_bluefs", Option::TYPE_BOOL, Option::LEVEL_DEV)
     .set_default(true)
     .set_flag(Option::FLAG_CREATE)

diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc
index e90b5694b1df88ec95c6c8d20aeac9659fd876d2..1c21d443aa341cdb4b5d1592eb0b6f1528be1ae3 100644 (file)
--- a/src/os/bluestore/BlueFS.cc
+++ b/src/os/bluestore/BlueFS.cc
@@ -1003,7 +1003,10 @@ int BlueFS::_replay(bool noop, bool to_stdout)
     {
       int r = _read(log_reader, &log_reader->buf, read_pos, super.block_size,
                    &bl, NULL);
-      ceph_assert(r == (int)super.block_size);
+      if (r != (int)super.block_size && cct->_conf->bluefs_replay_recovery) {
+       r += do_replay_recovery_read(log_reader, pos, read_pos + r, super.block_size - r, &bl);
+      }
+      assert(r == (int)super.block_size);
       read_pos += r;
     }
     uint64_t more = 0;
@@ -1056,10 +1059,17 @@ int BlueFS::_replay(bool noop, bool to_stdout)
       bufferlist t;
       int r = _read(log_reader, &log_reader->buf, read_pos, more, &t, NULL);
       if (r < (int)more) {
-       derr  << __func__ << " 0x" << std::hex << pos
-              << ": stop: len is 0x" << bl.length() + more << std::dec
-              << ", which is past eof" << dendl;
-       break;
+       dout(10) << __func__ << " 0x" << std::hex << pos
+                 << ": stop: len is 0x" << bl.length() + more << std::dec
+                 << ", which is past eof" << dendl;
+       if (cct->_conf->bluefs_replay_recovery) {
+         //try to search for more data
+         r += do_replay_recovery_read(log_reader, pos, read_pos + r, more - r, &t);
+         if (r < (int)more) {
+           //in normal mode we must read r==more, for recovery it is too strict
+           break;
+         }
+       }
       }
       ceph_assert(r == (int)more);
       bl.claim_append(t);
@@ -2004,6 +2014,12 @@ int BlueFS::_read(
         buf->bl_off = off & super.block_mask();
         uint64_t x_off = 0;
         auto p = h->file->fnode.seek(buf->bl_off, &x_off);
+       if (p == h->file->fnode.extents.end()) {
+         dout(5) << __func__ << " reading less then required "
+                 << ret << "<" << ret + len << dendl;
+         break;
+       }
+
         uint64_t want = round_up_to(len + (off & ~super.block_mask()),
                                    super.block_size);
         want = std::max(want, buf->max_prefetch);
@@ -3582,6 +3598,191 @@ bool BlueFS::wal_is_rotational()
   return bdev[BDEV_SLOW]->is_rotational();
 }
 
+/*
+  Algorithm.
+  do_replay_recovery_read is used when bluefs log abruptly ends, but it seems that more data should be there.
+  Idea is to search disk for definiton of extents that will be accompanied with bluefs log in future,
+  and try if using it will produce healthy bluefs transaction.
+  We encode already known bluefs log extents and search disk for these bytes.
+  When we find it, we decode following bytes as extent.
+  We read that whole extent and then check if merged with existing log part gives a proper bluefs transaction.
+ */
+int BlueFS::do_replay_recovery_read(FileReader *log_reader,
+                                   size_t replay_pos,
+                                   size_t read_offset,
+                                   size_t read_len,
+                                   bufferlist* bl) {
+  dout(1) << __func__ << " replay_pos=0x" << std::hex << replay_pos <<
+    " needs 0x" << read_offset << "~" << read_len << std::dec << dendl;
+
+  bluefs_fnode_t& log_fnode = log_reader->file->fnode;
+  bufferlist bin_extents;
+  ceph::encode(log_fnode.extents, bin_extents);
+  dout(2) << __func__ << " log file encoded extents length = " << bin_extents.length() << dendl;
+
+  // cannot process if too small to effectively search
+  ceph_assert(bin_extents.length() >= 32);
+  bufferlist last_32;
+  last_32.substr_of(bin_extents, bin_extents.length() - 32, 32);
+
+  //read fixed part from replay_pos to end of bluefs_log extents
+  bufferlist fixed;
+  uint64_t e_off = 0;
+  auto e = log_fnode.seek(replay_pos, &e_off);
+  ceph_assert(e != log_fnode.extents.end());
+  int r = bdev[e->bdev]->read(e->offset + e_off, e->length - e_off, &fixed, ioc[e->bdev],
+                                 cct->_conf->bluefs_buffered_io);
+  ceph_assert(r == 0);
+  //capture dev of last good extent
+  uint8_t last_e_dev = e->bdev;
+  uint64_t last_e_off = e->offset;
+  ++e;
+  while (e != log_fnode.extents.end()) {
+    r = bdev[e->bdev]->read(e->offset, e->length, &fixed, ioc[e->bdev],
+                                 cct->_conf->bluefs_buffered_io);
+    ceph_assert(r == 0);
+    last_e_dev = e->bdev;
+    ++e;
+  }
+  ceph_assert(replay_pos + fixed.length() == read_offset);
+
+  dout(2) << __func__ << " valid data in log = " << fixed.length() << dendl;
+
+  struct compare {
+    bool operator()(const bluefs_extent_t& a, const bluefs_extent_t& b) const {
+      if (a.bdev < b.bdev) return true;
+      if (a.offset < b.offset) return true;
+      return a.length < b.length;
+    }
+  };
+  std::set<bluefs_extent_t, compare> extents_rejected;
+  for (int dcnt = 0; dcnt < 3; dcnt++) {
+    uint8_t dev = (last_e_dev + dcnt) % MAX_BDEV;
+    if (bdev[dev] == nullptr) continue;
+    dout(2) << __func__ << " processing " << get_device_name(dev) << dendl;
+    interval_set<uint64_t> disk_regions;
+    disk_regions.insert(0, bdev[dev]->get_size());
+    for (auto f : file_map) {
+      auto& e = f.second->fnode.extents;
+      for (auto& p : e) {
+       if (p.bdev == dev) {
+         disk_regions.erase(p.offset, p.length);
+       }
+      }
+    }
+    size_t disk_regions_count = disk_regions.num_intervals();
+    dout(5) << __func__ << " " << disk_regions_count << " regions to scan on " << get_device_name(dev) << dendl;
+
+    auto reg = disk_regions.lower_bound(last_e_off);
+    //for all except first, start from beginning
+    last_e_off = 0;
+    if (reg == disk_regions.end()) {
+      reg = disk_regions.begin();
+    }
+    const uint64_t chunk_size = 4 * 1024 * 1024;
+    const uint64_t page_size = 4096;
+    const uint64_t max_extent_size = 16;
+    uint64_t overlay_size = last_32.length() + max_extent_size;
+    for (size_t i = 0; i < disk_regions_count; reg++, i++) {
+      if (reg == disk_regions.end()) {
+       reg = disk_regions.begin();
+      }
+      uint64_t pos = reg.get_start();
+      uint64_t len = reg.get_len();
+
+      std::unique_ptr<char[]> raw_data_p{new char[page_size + chunk_size]};
+      char* raw_data = raw_data_p.get();
+      memset(raw_data, 0, page_size);
+
+      while (len > last_32.length()) {
+       uint64_t chunk_len = len > chunk_size ? chunk_size : len;
+       dout(5) << __func__ << " read "
+               << get_device_name(dev) << ":0x" << std::hex << pos << "+" << chunk_len << std::dec << dendl;
+       r = bdev[dev]->read_random(pos, chunk_len, raw_data + page_size, cct->_conf->bluefs_buffered_io);
+       ceph_assert(r == 0);
+
+       //search for fixed_last_32
+       char* chunk_b = raw_data + page_size;
+       char* chunk_e = chunk_b + chunk_len;
+
+       char* search_b = chunk_b - overlay_size;
+       char* search_e = chunk_e;
+
+       for (char* sp = search_b; ; sp += last_32.length()) {
+         sp = (char*)memmem(sp, search_e - sp, last_32.c_str(), last_32.length());
+         if (sp == nullptr) {
+           break;
+         }
+
+         char* n = sp + last_32.length();
+         dout(5) << __func__ << " checking location 0x" << std::hex << pos + (n - chunk_b) << std::dec << dendl;
+         bufferlist test;
+         test.append(n, std::min<size_t>(max_extent_size, chunk_e - n));
+         bluefs_extent_t ne;
+         try {
+           bufferlist::const_iterator p = test.begin();
+           ceph::decode(ne, p);
+         } catch (buffer::error& e) {
+           continue;
+         }
+         if (extents_rejected.count(ne) != 0) {
+           dout(5) << __func__ << " extent " << ne << " already refected" <<dendl;
+           continue;
+         }
+         //insert as rejected already. if we succeed, it wouldn't make difference.
+         extents_rejected.insert(ne);
+
+         if (ne.bdev >= MAX_BDEV ||
+             bdev[ne.bdev] == nullptr ||
+             ne.length > 16 * 1024 * 1024 ||
+             (ne.length & 4095) != 0 ||
+             ne.offset + ne.length > bdev[ne.bdev]->get_size() ||
+             (ne.offset & 4095) != 0) {
+           dout(5) << __func__ << " refusing extent " << ne << dendl;
+           continue;
+         }
+         dout(5) << __func__ << " checking extent " << ne << dendl;
+
+         //read candidate extent - whole
+         bufferlist candidate;
+         candidate.append(fixed);
+         r = bdev[ne.bdev]->read(ne.offset, ne.length, &candidate, ioc[ne.bdev],
+                               cct->_conf->bluefs_buffered_io);
+         ceph_assert(r == 0);
+
+         //check if transaction & crc is ok
+         bluefs_transaction_t t;
+         try {
+           bufferlist::const_iterator p = candidate.cbegin();
+           decode(t, p);
+         }
+         catch (buffer::error& e) {
+           dout(5) << __func__ << " failed match" << dendl;
+           continue;
+         }
+
+         //success, it seems a probable candidate
+         uint64_t l = std::min<uint64_t>(ne.length, read_len);
+         //trim to required size
+         bufferlist requested_read;
+         requested_read.substr_of(candidate, fixed.length(), l);
+         bl->append(requested_read);
+         dout(5) << __func__ << " successful extension of log " << l << "/" << read_len << dendl;
+         log_fnode.append_extent(ne);
+         log_fnode.recalc_allocated();
+         log_reader->buf.pos += l;
+         return l;
+       }
+       //save overlay for next search
+       memcpy(search_b, chunk_e - overlay_size, overlay_size);
+       pos += chunk_len;
+       len -= chunk_len;
+      }
+    }
+  }
+  return 0;
+}
+
 void BlueFS::debug_inject_duplicate_gift(unsigned id,
   uint64_t offset,
   uint64_t len)

diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h
index 4ad026960109b4f81ea091c0d992d20924446cd1..e8ef684602d6e920c200615ae216ab5492a4f998 100644 (file)
--- a/src/os/bluestore/BlueFS.h
+++ b/src/os/bluestore/BlueFS.h
@@ -590,6 +590,11 @@ public:
     std::lock_guard l(lock);
     return _truncate(h, offset);
   }
+  int do_replay_recovery_read(FileReader *log,
+                             size_t log_pos,
+                             size_t read_offset,
+                             size_t read_len,
+                             bufferlist* bl);
 
   /// test purpose methods
   void debug_inject_duplicate_gift(unsigned bdev, uint64_t offset, uint64_t len);