osd, mon: New pool recovery priority range -10 to 10

Use OSD_POOL_PRIORITY_MAX and OSD_POOL_PRIORITY_MIN constants
Scale legacy priorities if exceeds maximum

Signed-off-by: David Zafman <dzafman@redhat.com>
(cherry picked from commit 444aa9f9fe0bbbdb34bf2108044598785db6ef65)

diff --git a/doc/rados/operations/pools.rst b/doc/rados/operations/pools.rst
index 662b34ff31a17cd00dcc56f58e468da542c2f2b1..98bb27babd94188812e68f836d1d99b8ac90707f 100644 (file)
--- a/doc/rados/operations/pools.rst
+++ b/doc/rados/operations/pools.rst
@@ -601,8 +601,10 @@ You may set values for the following keys:
 
 ``recovery_priority``
 
-:Description: When a value is set it will boost the computed reservation priority
-              by this amount. This value should be less than 30.
+:Description: When a value is set it will increase or decrease the computed
+              reservation priority. This value must be in the range -10 to
+              10.  Use a negative priority for less important pools so they
+              have lower priority than any new pools.
 
 :Type: Integer
 :Default: ``0``

diff --git a/qa/standalone/osd/osd-backfill-prio.sh b/qa/standalone/osd/osd-backfill-prio.sh
index 6491cd19efd6086b82d65bd7a2cff95bd7a11c79..2a69ba12d03f9677d7ad48dfb5ece2871e729e01 100755 (executable)
--- a/qa/standalone/osd/osd-backfill-prio.sh
+++ b/qa/standalone/osd/osd-backfill-prio.sh
@@ -30,8 +30,8 @@ function run() {
     export objects=50
     export poolprefix=test
     export FORCE_PRIO="254"     # See OSD_BACKFILL_PRIORITY_FORCED
-    export DEGRADED_PRIO="140"  # See OSD_BACKFILL_DEGRADED_PRIORITY_BASE
-    export NORMAL_PRIO="100"    # See OSD_BACKFILL_PRIORITY_BASE
+    export DEGRADED_PRIO="150"  # See OSD_BACKFILL_DEGRADED_PRIORITY_BASE + 10
+    export NORMAL_PRIO="110"    # See OSD_BACKFILL_PRIORITY_BASE + 10
 
     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
     for func in $funcs ; do

diff --git a/qa/standalone/osd/osd-recovery-prio.sh b/qa/standalone/osd/osd-recovery-prio.sh
index 9b017800f674e55881f67391e6cca79fb49af8df..d246dda66db1aac3ed8506c95333737e75e7d35b 100755 (executable)
--- a/qa/standalone/osd/osd-recovery-prio.sh
+++ b/qa/standalone/osd/osd-recovery-prio.sh
@@ -29,7 +29,7 @@ function run() {
     export objects=200
     export poolprefix=test
     export FORCE_PRIO="255"    # See OSD_RECOVERY_PRIORITY_FORCED
-    export NORMAL_PRIO="180"   # See OSD_RECOVERY_PRIORITY_BASE
+    export NORMAL_PRIO="190"   # See OSD_RECOVERY_PRIORITY_BASE + 10
 
     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
     for func in $funcs ; do

diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh
index 827d081090b7f6936114017a02240b7c42ffab14..49b7a4d27c15c3b6eac945f5b8c84959807262ed 100755 (executable)
--- a/qa/workunits/cephtool/test.sh
+++ b/qa/workunits/cephtool/test.sh
@@ -2022,10 +2022,12 @@ function test_mon_osd_pool_set()
   ceph osd pool get $TEST_POOL_GETSET recovery_priority | expect_false grep '.'
   ceph osd pool set $TEST_POOL_GETSET recovery_priority 5 
   ceph osd pool get $TEST_POOL_GETSET recovery_priority | grep 'recovery_priority: 5'
+  ceph osd pool set $TEST_POOL_GETSET recovery_priority -5
+  ceph osd pool get $TEST_POOL_GETSET recovery_priority | grep 'recovery_priority: -5'
   ceph osd pool set $TEST_POOL_GETSET recovery_priority 0
   ceph osd pool get $TEST_POOL_GETSET recovery_priority | expect_false grep '.'
-  expect_false ceph osd pool set $TEST_POOL_GETSET recovery_priority -1
-  expect_false ceph osd pool set $TEST_POOL_GETSET recovery_priority 30
+  expect_false ceph osd pool set $TEST_POOL_GETSET recovery_priority -11
+  expect_false ceph osd pool set $TEST_POOL_GETSET recovery_priority 11
 
   ceph osd pool get $TEST_POOL_GETSET recovery_op_priority | expect_false grep '.'
   ceph osd pool set $TEST_POOL_GETSET recovery_op_priority 5 

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 940c8d376077ab4777fe30956dbfae1c73975708..a991f1f652ae0ee6785f12e0b08f22e09b1c321d 100644 (file)
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -646,6 +646,11 @@ void OSDMonitor::on_active()
 
   if (mon->is_leader()) {
     mon->clog->debug() << "osdmap " << osdmap;
+    if (!priority_convert) {
+      // Only do this once at start-up
+      convert_pool_priorities();
+      priority_convert = true;
+    }
   } else {
     list<MonOpRequestRef> ls;
     take_all_failures(ls);
@@ -7617,12 +7622,9 @@ int OSDMonitor::prepare_command_pool_set(const cmdmap_t& cmdmap,
         ss << "error parsing int value '" << val << "': " << interr;
         return -EINVAL;
       }
-      if (n < 0) {
-        ss << "pool recovery_priority can not be negative";
-        return -EINVAL;
-      } else if (n >= 30) {
-        ss << "pool recovery_priority should be less than 30 due to "
-           << "Ceph internal implementation restrictions";
+      if (n > OSD_POOL_PRIORITY_MAX || n < OSD_POOL_PRIORITY_MIN) {
+        ss << "pool recovery_priority must be between " << OSD_POOL_PRIORITY_MIN
+          << " and " << OSD_POOL_PRIORITY_MAX;
         return -EINVAL;
       }
     } else if (var == "pg_autoscale_bias") {
@@ -13162,3 +13164,55 @@ void OSDMonitor::_pool_op_reply(MonOpRequestRef op,
                                         ret, epoch, get_last_committed(), blp);
   mon->send_reply(op, reply);
 }
+
+void OSDMonitor::convert_pool_priorities(void)
+{
+  pool_opts_t::key_t key = pool_opts_t::get_opt_desc("recovery_priority").key;
+  int64_t max_prio = 0;
+  int64_t min_prio = 0;
+  for (const auto &i : osdmap.get_pools()) {
+    const auto &pool = i.second;
+
+    if (pool.opts.is_set(key)) {
+      int64_t prio;
+      pool.opts.get(key, &prio);
+      if (prio > max_prio)
+       max_prio = prio;
+      if (prio < min_prio)
+       min_prio = prio;
+    }
+  }
+  if (max_prio <= OSD_POOL_PRIORITY_MAX && min_prio >= OSD_POOL_PRIORITY_MIN) {
+    dout(20) << __func__ << " nothing to fix" << dendl;
+    return;
+  }
+  // Current pool priorities exceeds new maximum
+  for (const auto &i : osdmap.get_pools()) {
+    const auto pool_id = i.first;
+    pg_pool_t pool = i.second;
+
+    int64_t prio = 0;
+    pool.opts.get(key, &prio);
+    int64_t n;
+
+    if (prio > 0 && max_prio > OSD_POOL_PRIORITY_MAX) { // Likely scenario
+      // Scaled priority range 0 to OSD_POOL_PRIORITY_MAX
+      n = (float)prio / max_prio * OSD_POOL_PRIORITY_MAX;
+    } else if (prio < 0 && min_prio < OSD_POOL_PRIORITY_MIN) {
+      // Scaled  priority range OSD_POOL_PRIORITY_MIN to 0
+      n = (float)prio / min_prio * OSD_POOL_PRIORITY_MIN;
+    } else {
+      continue;
+    }
+    if (n == 0) {
+      pool.opts.unset(key);
+    } else {
+      pool.opts.set(key, static_cast<int64_t>(n));
+    }
+    dout(10) << __func__ << " pool " << pool_id
+            << " recovery_priority adjusted "
+            << prio << " to " << n << dendl;
+    pool.last_change = pending_inc.epoch;
+    pending_inc.new_pools[pool_id] = pool;
+  }
+}

diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h
index 9055ec452602bd9197f3692e02052115aeb84765..4968a2d6270544b1f220c1fe92206eeebbe90eaa 100644 (file)
--- a/src/mon/OSDMonitor.h
+++ b/src/mon/OSDMonitor.h
@@ -220,6 +220,7 @@ public:
   set<int>             pending_metadata_rm;
   map<int, failure_info_t> failure_info;
   map<int,utime_t>    down_pending_out;  // osd down -> out
+  bool priority_convert = false;
 
   map<int,double> osd_weight;
 
@@ -701,6 +702,7 @@ public:
       pending_inc.new_flags &= ~flag;
     }
   }
+  void convert_pool_priorities(void);
 };
 
 #endif

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 8e8fceafdc50ee0afbbc889330eb6e9487add807..2ffdcc449cc1da787a9e51ec8067c86c4a7a48b3 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -2455,11 +2455,22 @@ bool PG::set_force_backfill(bool b)
   return did;
 }
 
-inline int PG::clamp_recovery_priority(int priority)
+int PG::clamp_recovery_priority(int priority, int pool_recovery_priority)
 {
   static_assert(OSD_RECOVERY_PRIORITY_MIN < OSD_RECOVERY_PRIORITY_MAX, "Invalid priority range");
   static_assert(OSD_RECOVERY_PRIORITY_MIN >= 0, "Priority range must match unsigned type");
 
+  // User can't set this too high anymore, but might be a legacy value
+  if (pool_recovery_priority > OSD_POOL_PRIORITY_MAX)
+    pool_recovery_priority = OSD_POOL_PRIORITY_MAX;
+  if (pool_recovery_priority < OSD_POOL_PRIORITY_MIN)
+    pool_recovery_priority = OSD_POOL_PRIORITY_MIN;
+  // Shift range from min to max to 0 to max - min
+  pool_recovery_priority += (0 - OSD_POOL_PRIORITY_MIN);
+  ceph_assert(pool_recovery_priority >= 0 && pool_recovery_priority <= (OSD_POOL_PRIORITY_MAX - OSD_POOL_PRIORITY_MIN));
+
+  priority += pool_recovery_priority;
+
   // Clamp to valid range
   if (priority > OSD_RECOVERY_PRIORITY_MAX) {
     return OSD_RECOVERY_PRIORITY_MAX;
@@ -2487,7 +2498,7 @@ unsigned PG::get_recovery_priority()
     int64_t pool_recovery_priority = 0;
     pool.info.opts.get(pool_opts_t::RECOVERY_PRIORITY, &pool_recovery_priority);
 
-    ret = clamp_recovery_priority(pool_recovery_priority + ret);
+    ret = clamp_recovery_priority(ret, pool_recovery_priority);
   }
   dout(20) << __func__ << " recovery priority is " << ret << dendl;
   return static_cast<unsigned>(ret);
@@ -2518,7 +2529,7 @@ unsigned PG::get_backfill_priority()
     int64_t pool_recovery_priority = 0;
     pool.info.opts.get(pool_opts_t::RECOVERY_PRIORITY, &pool_recovery_priority);
 
-    ret = clamp_recovery_priority(pool_recovery_priority + ret);
+    ret = clamp_recovery_priority(ret, pool_recovery_priority);
   }
 
   dout(20) << __func__ << " backfill priority is " << ret << dendl;

diff --git a/src/osd/PG.h b/src/osd/PG.h
index 790c004bed753e514adf6b807070a86f0c21a20a..8f844a7c5999168a7f5bec15c03a9662057eddd8 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -1432,7 +1432,7 @@ protected:
   bool needs_backfill() const;
 
   /// clip calculated priority to reasonable range
-  inline int clamp_recovery_priority(int priority);
+  int clamp_recovery_priority(int prio, int pool_recovery_prio);
   /// get log recovery reservation priority
   unsigned get_recovery_priority();
   /// get backfill reservation priority

diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index 90b4edea6181a50167a5e9021870765685532c0b..a2a22ab7754f9eb708f1a753e95a1ca4965d258d 100644 (file)
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -67,6 +67,10 @@
 #define CEPH_OSD_FEATURE_INCOMPAT_RECOVERY_DELETES CompatSet::Feature(16, "deletes in missing set")
 
 
+/// pool priority range set by user
+#define OSD_POOL_PRIORITY_MAX 10
+#define OSD_POOL_PRIORITY_MIN -OSD_POOL_PRIORITY_MAX
+
 /// min recovery priority for MBackfillReserve
 #define OSD_RECOVERY_PRIORITY_MIN 0