os/bluestore: warn on no per-pool omap

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/doc/rados/operations/health-checks.rst b/doc/rados/operations/health-checks.rst
index e83cd479b6669bf82ab4f1f56ee31e302a7a2d9a..2c22b7157ff19ac672946c68c84386618ba36ba6 100644 (file)
--- a/doc/rados/operations/health-checks.rst
+++ b/doc/rados/operations/health-checks.rst
@@ -356,6 +356,27 @@ This warning can be disabled with::
 
   ceph config set global bluestore_warn_on_legacy_statfs false
 
+BLUESTORE_NO_PER_POOL_OMAP
+__________________________
+
+Starting with the Octopus release, BlueStore tracks omap space utilization
+by pool, and one or more OSDs have volumes that were created prior to
+Octopus.  If all OSDs are not running BlueStore with the new tracking
+enabled, the cluster will report and approximate value for per-pool omap usage
+based on the most recent deep-scrub.
+
+The old OSDs can be updated to track by pool by stopping each OSD,
+running a repair operation, and the restarting it.  For example, if
+``osd.123`` needed to be updated,::
+
+  systemctl stop ceph-osd@123
+  ceph-bluestore-tool repair --path /var/lib/ceph/osd/ceph-123
+  systemctl start ceph-osd@123
+
+This warning can be disabled with::
+
+  ceph config set global bluestore_warn_on_no_per_pool_omap false
+
 
 BLUESTORE_DISK_SIZE_MISMATCH
 ____________________________

diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/tasks/2-upgrade.yaml b/qa/suites/fs/upgrade/featureful_client/old_client/tasks/2-upgrade.yaml
index 701fbea6424284c576c51073ad85cce99e6866ae..5fcf8c2a565d46cddc72275a7895c0e1ab5f1896 100644 (file)
--- a/qa/suites/fs/upgrade/featureful_client/old_client/tasks/2-upgrade.yaml
+++ b/qa/suites/fs/upgrade/featureful_client/old_client/tasks/2-upgrade.yaml
@@ -13,6 +13,7 @@ overrides:
     conf:
       global:
         bluestore warn on legacy statfs: false
+        bluestore warn on no per pool omap: false
       mon:
         mon warn on osd down out interval zero: false
 

diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/2-upgrade.yaml b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/2-upgrade.yaml
index 701fbea6424284c576c51073ad85cce99e6866ae..5fcf8c2a565d46cddc72275a7895c0e1ab5f1896 100644 (file)
--- a/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/2-upgrade.yaml
+++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/2-upgrade.yaml
@@ -13,6 +13,7 @@ overrides:
     conf:
       global:
         bluestore warn on legacy statfs: false
+        bluestore warn on no per pool omap: false
       mon:
         mon warn on osd down out interval zero: false
 

diff --git a/qa/suites/upgrade/mimic-x-singleton/0-cluster/start.yaml b/qa/suites/upgrade/mimic-x-singleton/0-cluster/start.yaml
index eb338ea161b7ccc11f93fb4042f042ab6bbb9372..dc4fd9fd1ec0cd46d0b4c7792fedcfabb6fbdf1d 100644 (file)
--- a/qa/suites/upgrade/mimic-x-singleton/0-cluster/start.yaml
+++ b/qa/suites/upgrade/mimic-x-singleton/0-cluster/start.yaml
@@ -13,6 +13,7 @@ overrides:
         ms dump corrupt message level: 0
         ms bind msgr2: false
         bluestore warn on legacy statfs: false
+        bluestore warn on no per pool omap: false
       mds:
         debug ms: 1
         debug mds: 20

diff --git a/qa/suites/upgrade/mimic-x/parallel/1-ceph-install/mimic.yaml b/qa/suites/upgrade/mimic-x/parallel/1-ceph-install/mimic.yaml
index 6c9bd0d2160ce86e9368edc13ae873b8206a4a33..bd9f1767b8d532eb8896b2d8e4ce6b878f694c33 100644 (file)
--- a/qa/suites/upgrade/mimic-x/parallel/1-ceph-install/mimic.yaml
+++ b/qa/suites/upgrade/mimic-x/parallel/1-ceph-install/mimic.yaml
@@ -34,6 +34,7 @@ tasks:
       global:
         mon warn on pool no app: false
         bluestore_warn_on_legacy_statfs: false
+        bluestore warn on no per pool omap: false
 - exec:
     osd.0:
       - ceph osd require-osd-release mimic

diff --git a/qa/suites/upgrade/mimic-x/stress-split/1-ceph-install/mimic.yaml b/qa/suites/upgrade/mimic-x/stress-split/1-ceph-install/mimic.yaml
index ad5b0e236101b9d7673987e275f2fe0acf877a4f..f450b832d1e53abf4c551c8d264e24dd4cda480a 100644 (file)
--- a/qa/suites/upgrade/mimic-x/stress-split/1-ceph-install/mimic.yaml
+++ b/qa/suites/upgrade/mimic-x/stress-split/1-ceph-install/mimic.yaml
@@ -16,6 +16,7 @@ tasks:
     conf:
       global:
         bluestore_warn_on_legacy_statfs: false
+        bluestore warn on no per pool omap: false
 - exec:
     osd.0:
       - ceph osd require-osd-release mimic

diff --git a/qa/suites/upgrade/nautilus-x-singleton/0-cluster/start.yaml b/qa/suites/upgrade/nautilus-x-singleton/0-cluster/start.yaml
index c3c9566ac536dbd82c1b3848ad013fbd0264f111..499bca95c44f084b7bce3941741e848286277ad9 100644 (file)
--- a/qa/suites/upgrade/nautilus-x-singleton/0-cluster/start.yaml
+++ b/qa/suites/upgrade/nautilus-x-singleton/0-cluster/start.yaml
@@ -12,6 +12,7 @@ overrides:
       global:
         ms dump corrupt message level: 0
         ms bind msgr2: false
+        bluestore warn on no per pool omap: false
       mds:
         debug ms: 1
         debug mds: 20

diff --git a/qa/suites/upgrade/nautilus-x/parallel/1-ceph-install/nautilus.yaml b/qa/suites/upgrade/nautilus-x/parallel/1-ceph-install/nautilus.yaml
index 0ed40672cb0a66c69853823e33402b5a27d35b37..9720192def080a85ddf989a01a8352ed58524f65 100644 (file)
--- a/qa/suites/upgrade/nautilus-x/parallel/1-ceph-install/nautilus.yaml
+++ b/qa/suites/upgrade/nautilus-x/parallel/1-ceph-install/nautilus.yaml
@@ -26,6 +26,7 @@ tasks:
       global:
         mon warn on pool no app: false
         bluestore_warn_on_legacy_statfs: false
+        bluestore warn on no per pool omap: false
 - exec:
     osd.0:
       - ceph osd set-require-min-compat-client nautilus

diff --git a/qa/suites/upgrade/nautilus-x/stress-split/1-ceph-install/nautilus.yaml b/qa/suites/upgrade/nautilus-x/stress-split/1-ceph-install/nautilus.yaml
index b1d523f539649be6e5fdded6024649ffee4111c4..a5bf71d07b1ff090e7aa54b2face6a73fab6f4cb 100644 (file)
--- a/qa/suites/upgrade/nautilus-x/stress-split/1-ceph-install/nautilus.yaml
+++ b/qa/suites/upgrade/nautilus-x/stress-split/1-ceph-install/nautilus.yaml
@@ -8,6 +8,7 @@ tasks:
     conf:
       global:
         bluestore_warn_on_legacy_statfs: false
+        bluestore warn on no per pool omap: false
 - exec:
     osd.0:
       - ceph osd require-osd-release nautilus

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index be221c6593673f986b2b997eae3cdcd4709c4547..66f6aa6c4b9d5bda98c382771c764ec62e041268 100644 (file)
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -1072,6 +1072,7 @@ OPTION(bluestore_no_per_pool_stats_tolerance, OPT_STR)
 OPTION(bluestore_warn_on_bluefs_spillover, OPT_BOOL)
 OPTION(bluestore_warn_on_legacy_statfs, OPT_BOOL)
 OPTION(bluestore_fsck_error_on_no_per_pool_omap, OPT_BOOL)
+OPTION(bluestore_warn_on_no_per_pool_omap, OPT_BOOL)
 OPTION(bluestore_log_op_age, OPT_DOUBLE)
 OPTION(bluestore_log_omap_iterator_age, OPT_DOUBLE)
 OPTION(bluestore_log_collection_list_age, OPT_DOUBLE)

diff --git a/src/common/options.cc b/src/common/options.cc
index 7f107a4beb55ff77e7c7a9b64085f30a251f4177..7e07607ceb780f738dfa6d40d623ad265e692707 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -4868,6 +4868,10 @@ std::vector<Option> get_global_options() {
     .set_default(false)
     .set_description("Make fsck error (instead of warn) when objects without per-pool omap are found"),
 
+    Option("bluestore_warn_on_no_per_pool_omap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(true)
+    .set_description("Enable health indication on lack of per-pool omap"),
+
     Option("bluestore_log_op_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
     .set_default(5)
     .set_description("log operation if it's slower than this age (seconds)"),

diff --git a/src/mon/PGMap.cc b/src/mon/PGMap.cc
index 606afe88ec2a4f2eaa196a845147f1f3ce401d82..df5ee60a6d7a8bfd2e69645b3c788d3d80119b49 100644 (file)
--- a/src/mon/PGMap.cc
+++ b/src/mon/PGMap.cc
@@ -3068,6 +3068,8 @@ void PGMap::get_health_checks(
        summary = "Legacy BlueStore stats reporting detected";
       } else if (asum.first == "BLUESTORE_DISK_SIZE_MISMATCH") {
        summary = "BlueStore has dangerous mismatch between block device and free list sizes";
+      } else if (asum.first == "BLUESTORE_NO_PER_POOL_OMAP") {
+       summary = "Legacy BlueStore does not track omap usage by pool";
       }
       summary += " on ";
       summary += stringify(asum.second.first);

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 9dad291300d6f84ca4519aaf6712c6201a90a17e..ceea70e660abef4468eb520b1eaba82aebc81087 100644 (file)
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -4135,6 +4135,7 @@ const char **BlueStore::get_tracked_conf_keys() const
     "bluestore_cache_autotune_interval",
     "bluestore_no_per_pool_stats_tolerance",
     "bluestore_warn_on_legacy_statfs",
+    "bluestore_warn_on_no_per_pool_omap",
     NULL
   };
   return KEYS;
@@ -4147,6 +4148,9 @@ void BlueStore::handle_conf_change(const ConfigProxy& conf,
       changed.count("bluestore_warn_on_legacy_statfs")) {
     _check_legacy_statfs_alert();
   }
+  if (changed.count("bluestore_warn_on_no_per_pool_omap")) {
+    _check_no_per_pool_omap_alert();
+  }
 
   if (changed.count("bluestore_csum_type")) {
     _set_csum();
@@ -8453,6 +8457,18 @@ void BlueStore::_check_legacy_statfs_alert()
   legacy_statfs_alert = s;
 }
 
+void BlueStore::_check_no_per_pool_omap_alert()
+{
+  string s;
+  if (!per_pool_omap &&
+      cct->_conf->bluestore_warn_on_no_per_pool_omap) {
+    s = "legacy (not per-pool) omap detected, "
+      "suggest to run store repair to measure per-pool omap usage";
+  }
+  std::lock_guard l(qlock);
+  no_per_pool_omap_alert = s;
+}
+
 // ---------------
 // cache
 
@@ -9979,6 +9995,7 @@ int BlueStore::_open_super_meta()
     } else {
       dout(10) << __func__ << " per_pool_omap not present" << dendl;
     }
+    _check_no_per_pool_omap_alert();
   }
 
   _open_statfs();
@@ -14255,6 +14272,11 @@ void BlueStore::_log_alerts(osd_alert_list_t& alerts)
       "BLUEFS_SPILLOVER",
       spillover_alert);
   }
+  if (!no_per_pool_omap_alert.empty()) {
+    alerts.emplace(
+      "BLUESTORE_NO_PER_POOL_OMAP",
+      no_per_pool_omap_alert);
+  }
   string s0(failed_cmode);
 
   if (!failed_compressors.empty()) {

diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 17e0649e5d76d93a3f4afeec6cafa7508fbbde14..60d9838744fea0f66b3910d7b93d2892771a5fe7 100644 (file)
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -2604,6 +2604,7 @@ private:
   set<string> failed_compressors;
   string spillover_alert;
   string legacy_statfs_alert;
+  string no_per_pool_omap_alert;
   string disk_size_mismatch_alert;
 
   void _log_alerts(osd_alert_list_t& alerts);
@@ -2632,6 +2633,7 @@ private:
   }
 
   void _check_legacy_statfs_alert();
+  void _check_no_per_pool_omap_alert();
   void _set_disk_size_mismatch_alert(const string& s) {
     std::lock_guard l(qlock);
     disk_size_mismatch_alert = s;