os/bluestore: introduce a cooldown period for failed BlueFS allocations.

When using bluefs_shared_alloc_size one might get a long-lasting state when
that large chunks are not available any more and fallback to shared
device min alloc size occurs. The introduced cooldown is intended to
prevent repetitive allocation attempts with bluefs_shared_alloc_size for
a while. The rationale is to eliminate performance penalty these failing
attempts might cause.

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
(cherry picked from commit e52bcc852ce51ab99138420f9069e2f59e1cb706)

 Conflicts:
	src/common/options/global.yaml.in
 (legacy options declarations, no yamls in pacific)

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index ea103d7de9ebb047527a73422ccefe5c44b1ff66..27bd36f3c2aa2d8e42ad35e5fe57805161972a66 100644 (file)
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -906,6 +906,7 @@ OPTION(objectstore_blackhole, OPT_BOOL)
 
 OPTION(bluefs_alloc_size, OPT_U64)
 OPTION(bluefs_shared_alloc_size, OPT_U64)
+OPTION(bluefs_failed_shared_alloc_cooldown, OPT_DOUBLE)
 OPTION(bluefs_max_prefetch, OPT_U64)
 OPTION(bluefs_min_log_runway, OPT_U64)  // alloc when we get this low
 OPTION(bluefs_max_log_runway, OPT_U64)  // alloc this much at a time

diff --git a/src/common/options.cc b/src/common/options.cc
index 4dc69cb0db3abff1b06124e50a57fad4becf8e64..b6f18e30b2188373b498559c1e7d4929210aa3f5 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -4278,6 +4278,18 @@ std::vector<Option> get_global_options() {
     .set_default(64_K)
     .set_description("Allocation unit size for primary/shared device"),
 
+    Option("bluefs_failed_shared_alloc_cooldown", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
+    .set_default(600)
+    .set_description(
+        "Duration(in seconds) until the next attempt to use "
+        "'bluefs_shared_alloc_size' after facing ENOSPC failure.")
+    .set_long_description(
+        "Cooldown period(in seconds) when BlueFS uses shared/slow device "
+        "allocation size instead of 'bluefs_shared_alloc_size' one after facing "
+        "recoverable (via fallback to smaller chunk size) ENOSPC failure. Intended "
+        "primarily to avoid repetitive unsuccessful allocations which might be "
+        " expensive."),
+
     Option("bluefs_max_prefetch", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
     .set_default(1_M)
     .set_description(""),

diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc
index ed915a3c461f1e2572b45953b2c4472dcd6c158e..9596b7f5b112d0e0349af9729c67c1e935baf324 100644 (file)
--- a/src/os/bluestore/BlueFS.cc
+++ b/src/os/bluestore/BlueFS.cc
@@ -1,6 +1,6 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
-
+#include <chrono>
 #include "boost/algorithm/string.hpp" 
 #include "bluestore_common.h"
 #include "BlueFS.h"
@@ -28,6 +28,8 @@ using std::set;
 using std::string;
 using std::to_string;
 using std::vector;
+using std::chrono::duration;
+using std::chrono::seconds;
 
 using ceph::bufferlist;
 using ceph::decode;
@@ -3613,17 +3615,37 @@ int BlueFS::_allocate(uint8_t id, uint64_t len,
                       bool permit_dev_fallback)
 {
   dout(10) << __func__ << " len 0x" << std::hex << len
-           << " alloc unit hint 0x" << alloc_unit
-           << std::dec << " from " << (int)id << dendl;
+           << " au 0x" << alloc_unit
+           << std::dec << " from " << (int)id
+           << " cooldown " << cooldown_deadline
+           << dendl;
   ceph_assert(id < alloc.size());
   int64_t alloc_len = 0;
   PExtentVector extents;
   uint64_t hint = 0;
   int64_t need = len;
+  bool shared = is_shared_alloc(id);
+  auto shared_unit = shared_alloc ? shared_alloc->alloc_unit : 0;
+  bool was_cooldown = false;
   if (alloc[id]) {
     if (!alloc_unit) {
       alloc_unit = alloc_size[id];
     }
+    // do not attempt shared_allocator with bluefs alloc unit
+    // when cooling down, fallback to slow dev alloc unit.
+    if (shared && alloc_unit != shared_unit) {
+       if (std::chrono::duration_cast<seconds>(real_clock::now().time_since_epoch()).count() <
+           cooldown_deadline) {
+         logger->inc(l_bluefs_alloc_shared_size_fallbacks);
+         alloc_unit = shared_unit;
+         was_cooldown = true;
+       } else if (cooldown_deadline.fetch_and(0)) {
+         // we might get false cooldown_deadline reset at this point
+         // but that's mostly harmless.
+         dout(1) << __func__ << " shared allocation cooldown period elapsed"
+                 << dendl;
+       }
+    }
     need = round_up_to(len, alloc_unit);
     if (!node->extents.empty() && node->extents.back().bdev == id) {
       hint = node->extents.back().end();
@@ -3637,6 +3659,14 @@ int BlueFS::_allocate(uint8_t id, uint64_t len,
       if (alloc_len > 0) {
         alloc[id]->release(extents);
       }
+      if (!was_cooldown && shared) {
+        auto delay_s = cct->_conf->bluefs_failed_shared_alloc_cooldown;
+        cooldown_deadline = delay_s +
+          std::chrono::duration_cast<seconds>(real_clock::now().time_since_epoch()).count();
+        dout(1) << __func__ << " shared allocation cooldown set for "
+                << delay_s << "s"
+                << dendl;
+      }
       dout(1) << __func__ << " unable to allocate 0x" << std::hex << need
              << " on bdev " << (int)id
               << ", allocator name " << alloc[id]->get_name()
@@ -3653,8 +3683,8 @@ int BlueFS::_allocate(uint8_t id, uint64_t len,
                << " unable to allocate 0x" << std::hex << need
               << " on bdev " << (int)id << std::dec << dendl;
     }
-    if (alloc[id] && is_shared_alloc(id) && alloc_unit != shared_alloc->alloc_unit) {
-      alloc_unit = shared_alloc->alloc_unit;
+    if (alloc[id] && shared && alloc_unit != shared_unit) {
+      alloc_unit = shared_unit;
       dout(20) << __func__ << " fallback to bdev "
               << (int)id
                << " with alloc unit 0x" << std::hex << alloc_unit
@@ -3690,7 +3720,7 @@ int BlueFS::_allocate(uint8_t id, uint64_t len,
       logger->set(max_bytes_pcounters[id], used);
       max_bytes[id] = used;
     }
-    if (is_shared_alloc(id)) {
+    if (shared) {
       shared_alloc->bluefs_used += alloc_len;
     }
   }

diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h
index 2f736c828631dbb18c886eaa2d170244a2e6fa9a..ec1b4c8ba1382208c788d8ae38c67e42b97fc7af 100644 (file)
--- a/src/os/bluestore/BlueFS.h
+++ b/src/os/bluestore/BlueFS.h
@@ -379,6 +379,7 @@ private:
   inline bool is_shared_alloc(unsigned id) const {
     return id == shared_alloc_id;
   }
+  std::atomic<int64_t> cooldown_deadline = 0;
 
   class SocketHook;
   SocketHook* asok_hook = nullptr;