From b8501164ef0315b4bee2c62b7a5ead97cf05b4b8 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@redhat.com>
Date: Thu, 8 Aug 2019 16:57:22 -0500
Subject: [PATCH] os/bluestore: warn on no per-pool omap

Signed-off-by: Sage Weil <sage@redhat.com>
---
 doc/rados/operations/health-checks.rst        | 21 ++++++++++++++++++
 .../old_client/tasks/2-upgrade.yaml           |  1 +
 .../upgraded_client/tasks/2-upgrade.yaml      |  1 +
 .../mimic-x-singleton/0-cluster/start.yaml    |  1 +
 .../parallel/1-ceph-install/mimic.yaml        |  1 +
 .../stress-split/1-ceph-install/mimic.yaml    |  1 +
 .../nautilus-x-singleton/0-cluster/start.yaml |  1 +
 .../parallel/1-ceph-install/nautilus.yaml     |  1 +
 .../stress-split/1-ceph-install/nautilus.yaml |  1 +
 src/common/legacy_config_opts.h               |  1 +
 src/common/options.cc                         |  4 ++++
 src/mon/PGMap.cc                              |  2 ++
 src/os/bluestore/BlueStore.cc                 | 22 +++++++++++++++++++
 src/os/bluestore/BlueStore.h                  |  2 ++
 14 files changed, 60 insertions(+)

diff --git a/doc/rados/operations/health-checks.rst b/doc/rados/operations/health-checks.rst
index e83cd479b666..2c22b7157ff1 100644
--- a/doc/rados/operations/health-checks.rst
+++ b/doc/rados/operations/health-checks.rst
@@ -356,6 +356,27 @@ This warning can be disabled with::
 
   ceph config set global bluestore_warn_on_legacy_statfs false
 
+BLUESTORE_NO_PER_POOL_OMAP
+__________________________
+
+Starting with the Octopus release, BlueStore tracks omap space utilization
+by pool, and one or more OSDs have volumes that were created prior to
+Octopus.  If all OSDs are not running BlueStore with the new tracking
+enabled, the cluster will report and approximate value for per-pool omap usage
+based on the most recent deep-scrub.
+
+The old OSDs can be updated to track by pool by stopping each OSD,
+running a repair operation, and the restarting it.  For example, if
+``osd.123`` needed to be updated,::
+
+  systemctl stop ceph-osd@123
+  ceph-bluestore-tool repair --path /var/lib/ceph/osd/ceph-123
+  systemctl start ceph-osd@123
+
+This warning can be disabled with::
+
+  ceph config set global bluestore_warn_on_no_per_pool_omap false
+
 
 BLUESTORE_DISK_SIZE_MISMATCH
 ____________________________
diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/tasks/2-upgrade.yaml b/qa/suites/fs/upgrade/featureful_client/old_client/tasks/2-upgrade.yaml
index 701fbea64242..5fcf8c2a565d 100644
--- a/qa/suites/fs/upgrade/featureful_client/old_client/tasks/2-upgrade.yaml
+++ b/qa/suites/fs/upgrade/featureful_client/old_client/tasks/2-upgrade.yaml
@@ -13,6 +13,7 @@ overrides:
     conf:
       global:
         bluestore warn on legacy statfs: false
+        bluestore warn on no per pool omap: false
       mon:
         mon warn on osd down out interval zero: false
 
diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/2-upgrade.yaml b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/2-upgrade.yaml
index 701fbea64242..5fcf8c2a565d 100644
--- a/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/2-upgrade.yaml
+++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/2-upgrade.yaml
@@ -13,6 +13,7 @@ overrides:
     conf:
       global:
         bluestore warn on legacy statfs: false
+        bluestore warn on no per pool omap: false
       mon:
         mon warn on osd down out interval zero: false
 
diff --git a/qa/suites/upgrade/mimic-x-singleton/0-cluster/start.yaml b/qa/suites/upgrade/mimic-x-singleton/0-cluster/start.yaml
index eb338ea161b7..dc4fd9fd1ec0 100644
--- a/qa/suites/upgrade/mimic-x-singleton/0-cluster/start.yaml
+++ b/qa/suites/upgrade/mimic-x-singleton/0-cluster/start.yaml
@@ -13,6 +13,7 @@ overrides:
         ms dump corrupt message level: 0
         ms bind msgr2: false
         bluestore warn on legacy statfs: false
+        bluestore warn on no per pool omap: false
       mds:
         debug ms: 1
         debug mds: 20
diff --git a/qa/suites/upgrade/mimic-x/parallel/1-ceph-install/mimic.yaml b/qa/suites/upgrade/mimic-x/parallel/1-ceph-install/mimic.yaml
index 6c9bd0d2160c..bd9f1767b8d5 100644
--- a/qa/suites/upgrade/mimic-x/parallel/1-ceph-install/mimic.yaml
+++ b/qa/suites/upgrade/mimic-x/parallel/1-ceph-install/mimic.yaml
@@ -34,6 +34,7 @@ tasks:
       global:
         mon warn on pool no app: false
         bluestore_warn_on_legacy_statfs: false
+        bluestore warn on no per pool omap: false
 - exec:
     osd.0:
       - ceph osd require-osd-release mimic
diff --git a/qa/suites/upgrade/mimic-x/stress-split/1-ceph-install/mimic.yaml b/qa/suites/upgrade/mimic-x/stress-split/1-ceph-install/mimic.yaml
index ad5b0e236101..f450b832d1e5 100644
--- a/qa/suites/upgrade/mimic-x/stress-split/1-ceph-install/mimic.yaml
+++ b/qa/suites/upgrade/mimic-x/stress-split/1-ceph-install/mimic.yaml
@@ -16,6 +16,7 @@ tasks:
     conf:
       global:
         bluestore_warn_on_legacy_statfs: false
+        bluestore warn on no per pool omap: false
 - exec:
     osd.0:
       - ceph osd require-osd-release mimic
diff --git a/qa/suites/upgrade/nautilus-x-singleton/0-cluster/start.yaml b/qa/suites/upgrade/nautilus-x-singleton/0-cluster/start.yaml
index c3c9566ac536..499bca95c44f 100644
--- a/qa/suites/upgrade/nautilus-x-singleton/0-cluster/start.yaml
+++ b/qa/suites/upgrade/nautilus-x-singleton/0-cluster/start.yaml
@@ -12,6 +12,7 @@ overrides:
       global:
         ms dump corrupt message level: 0
         ms bind msgr2: false
+        bluestore warn on no per pool omap: false
       mds:
         debug ms: 1
         debug mds: 20
diff --git a/qa/suites/upgrade/nautilus-x/parallel/1-ceph-install/nautilus.yaml b/qa/suites/upgrade/nautilus-x/parallel/1-ceph-install/nautilus.yaml
index 0ed40672cb0a..9720192def08 100644
--- a/qa/suites/upgrade/nautilus-x/parallel/1-ceph-install/nautilus.yaml
+++ b/qa/suites/upgrade/nautilus-x/parallel/1-ceph-install/nautilus.yaml
@@ -26,6 +26,7 @@ tasks:
       global:
         mon warn on pool no app: false
         bluestore_warn_on_legacy_statfs: false
+        bluestore warn on no per pool omap: false
 - exec:
     osd.0:
       - ceph osd set-require-min-compat-client nautilus
diff --git a/qa/suites/upgrade/nautilus-x/stress-split/1-ceph-install/nautilus.yaml b/qa/suites/upgrade/nautilus-x/stress-split/1-ceph-install/nautilus.yaml
index b1d523f53964..a5bf71d07b1f 100644
--- a/qa/suites/upgrade/nautilus-x/stress-split/1-ceph-install/nautilus.yaml
+++ b/qa/suites/upgrade/nautilus-x/stress-split/1-ceph-install/nautilus.yaml
@@ -8,6 +8,7 @@ tasks:
     conf:
       global:
         bluestore_warn_on_legacy_statfs: false
+        bluestore warn on no per pool omap: false
 - exec:
     osd.0:
       - ceph osd require-osd-release nautilus
diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index be221c659367..66f6aa6c4b9d 100644
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -1072,6 +1072,7 @@ OPTION(bluestore_no_per_pool_stats_tolerance, OPT_STR)
 OPTION(bluestore_warn_on_bluefs_spillover, OPT_BOOL)
 OPTION(bluestore_warn_on_legacy_statfs, OPT_BOOL)
 OPTION(bluestore_fsck_error_on_no_per_pool_omap, OPT_BOOL)
+OPTION(bluestore_warn_on_no_per_pool_omap, OPT_BOOL)
 OPTION(bluestore_log_op_age, OPT_DOUBLE)
 OPTION(bluestore_log_omap_iterator_age, OPT_DOUBLE)
 OPTION(bluestore_log_collection_list_age, OPT_DOUBLE)
diff --git a/src/common/options.cc b/src/common/options.cc
index 7f107a4beb55..7e07607ceb78 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -4868,6 +4868,10 @@ std::vector<Option> get_global_options() {
     .set_default(false)
     .set_description("Make fsck error (instead of warn) when objects without per-pool omap are found"),
 
+    Option("bluestore_warn_on_no_per_pool_omap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(true)
+    .set_description("Enable health indication on lack of per-pool omap"),
+
     Option("bluestore_log_op_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
     .set_default(5)
     .set_description("log operation if it's slower than this age (seconds)"),
diff --git a/src/mon/PGMap.cc b/src/mon/PGMap.cc
index 606afe88ec2a..df5ee60a6d7a 100644
--- a/src/mon/PGMap.cc
+++ b/src/mon/PGMap.cc
@@ -3068,6 +3068,8 @@ void PGMap::get_health_checks(
 	summary = "Legacy BlueStore stats reporting detected";
       } else if (asum.first == "BLUESTORE_DISK_SIZE_MISMATCH") {
 	summary = "BlueStore has dangerous mismatch between block device and free list sizes";
+      } else if (asum.first == "BLUESTORE_NO_PER_POOL_OMAP") {
+	summary = "Legacy BlueStore does not track omap usage by pool";
       }
       summary += " on ";
       summary += stringify(asum.second.first);
diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 9dad291300d6..ceea70e660ab 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -4135,6 +4135,7 @@ const char **BlueStore::get_tracked_conf_keys() const
     "bluestore_cache_autotune_interval",
     "bluestore_no_per_pool_stats_tolerance",
     "bluestore_warn_on_legacy_statfs",
+    "bluestore_warn_on_no_per_pool_omap",
     NULL
   };
   return KEYS;
@@ -4147,6 +4148,9 @@ void BlueStore::handle_conf_change(const ConfigProxy& conf,
       changed.count("bluestore_warn_on_legacy_statfs")) {
     _check_legacy_statfs_alert();
   }
+  if (changed.count("bluestore_warn_on_no_per_pool_omap")) {
+    _check_no_per_pool_omap_alert();
+  }
 
   if (changed.count("bluestore_csum_type")) {
     _set_csum();
@@ -8453,6 +8457,18 @@ void BlueStore::_check_legacy_statfs_alert()
   legacy_statfs_alert = s;
 }
 
+void BlueStore::_check_no_per_pool_omap_alert()
+{
+  string s;
+  if (!per_pool_omap &&
+      cct->_conf->bluestore_warn_on_no_per_pool_omap) {
+    s = "legacy (not per-pool) omap detected, "
+      "suggest to run store repair to measure per-pool omap usage";
+  }
+  std::lock_guard l(qlock);
+  no_per_pool_omap_alert = s;
+}
+
 // ---------------
 // cache
 
@@ -9979,6 +9995,7 @@ int BlueStore::_open_super_meta()
     } else {
       dout(10) << __func__ << " per_pool_omap not present" << dendl;
     }
+    _check_no_per_pool_omap_alert();
   }
 
   _open_statfs();
@@ -14255,6 +14272,11 @@ void BlueStore::_log_alerts(osd_alert_list_t& alerts)
       "BLUEFS_SPILLOVER",
       spillover_alert);
   }
+  if (!no_per_pool_omap_alert.empty()) {
+    alerts.emplace(
+      "BLUESTORE_NO_PER_POOL_OMAP",
+      no_per_pool_omap_alert);
+  }
   string s0(failed_cmode);
 
   if (!failed_compressors.empty()) {
diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 17e0649e5d76..60d9838744fe 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -2604,6 +2604,7 @@ private:
   set<string> failed_compressors;
   string spillover_alert;
   string legacy_statfs_alert;
+  string no_per_pool_omap_alert;
   string disk_size_mismatch_alert;
 
   void _log_alerts(osd_alert_list_t& alerts);
@@ -2632,6 +2633,7 @@ private:
   }
 
   void _check_legacy_statfs_alert();
+  void _check_no_per_pool_omap_alert();
   void _set_disk_size_mismatch_alert(const string& s) {
     std::lock_guard l(qlock);
     disk_size_mismatch_alert = s;
-- 
2.47.3