mon/OSDMonitor: add prior_version arg to 'osd crush set', 'osd setcrushmap'

If the prior_version doesn't match, reject the update.

Note that we also allow the crush_version-1 iff the proposed map is
identical to the current map in order to make the command idempotent.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh
index 02051906e804fcdc7a2244ad73f8e6309f3a12ba..791eb961af3113e4d7ba5271fa095c069cc3c504 100755 (executable)
--- a/qa/workunits/cephtool/test.sh
+++ b/qa/workunits/cephtool/test.sh
@@ -1449,11 +1449,6 @@ function test_mon_osd()
   ceph osd in 0
   ceph osd dump | grep ^osd.0 | grep 'weight 0.5'
 
-  f=$TEMP_DIR/map.$$
-  ceph osd getcrushmap -o $f
-  [ -s $f ]
-  ceph osd setcrushmap -i $f
-  rm $f
   ceph osd getmap -o $f
   [ -s $f ]
   rm $f
@@ -1582,6 +1577,31 @@ function test_mon_osd()
   ceph osd stat | grep up,
 }
 
+function test_mon_crush()
+{
+  f=$TEMP_DIR/map.$$
+  ceph osd getcrushmap -o $f 2> $f.epoch
+  [ -s $f ]
+  epoch=`cat $f.epoch`
+  [ "$epoch" -gt 1 ]
+  nextepoch=$(( $epoch + 1 ))
+  echo epoch $epoch nextepoch $nextepoch
+  rm -f $f.epoch
+  expect_false ceph osd setcrushmap $nextepoch -i $f
+  ceph osd setcrushmap $epoch -i $f 2> $f.epoch
+  gotepoch=`cat $f.epoch`
+  echo gotepoch $gotepoch
+  rm -f $f.epoch
+  [ "$gotepoch" -eq "$nextepoch" ]
+  # should be idempotent
+  ceph osd setcrushmap $epoch -i $f 2> $f.epoch
+  gotepoch=`cat $f.epoch`
+  echo epoch $gotepoch
+  rm -f $f.epoch
+  [ "$gotepoch" -eq "$nextepoch" ]
+  rm $f
+}
+
 function test_mon_osd_pool()
 {
   #
@@ -2349,6 +2369,7 @@ MON_TESTS+=" auth_profiles"
 MON_TESTS+=" mon_misc"
 MON_TESTS+=" mon_mon"
 MON_TESTS+=" mon_osd"
+MON_TESTS+=" mon_crush"
 MON_TESTS+=" mon_osd_create_destroy"
 MON_TESTS+=" mon_osd_pool"
 MON_TESTS+=" mon_osd_pool_quota"

diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index 255e267b5559da3b9d191663a72aab957939937a..1379b2a421e20bea656594981beae8b7cb3b13fa 100644 (file)
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -493,9 +493,11 @@ COMMAND("osd crush rule dump " \
 COMMAND("osd crush dump", \
        "dump crush map", \
        "osd", "r", "cli,rest")
-COMMAND("osd setcrushmap", "set crush map from input file", \
+COMMAND("osd setcrushmap name=prior_version,type=CephInt,req=false", \
+       "set crush map from input file",        \
        "osd", "rw", "cli,rest")
-COMMAND("osd crush set", "set crush map from input file", \
+COMMAND("osd crush set name=prior_version,type=CephInt,req=false", \
+       "set crush map from input file",        \
        "osd", "rw", "cli,rest")
 COMMAND("osd crush add-bucket " \
        "name=name,type=CephString,goodchars=[A-Za-z0-9-_.] " \

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 941a2f89de832b41011290eb94b1b71aeb00672d..e9e0cbcbfd1ffcb55cff6b44d895f575fe3f34e3 100644 (file)
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -7222,6 +7222,11 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
  
   if (prefix == "osd setcrushmap" ||
       (prefix == "osd crush set" && !osdid_present)) {
+    if (pending_inc.crush.length()) {
+      dout(10) << __func__ << " waiting for pending crush update " << dendl;
+      wait_for_finished_proposal(op, new C_RetryMessage(this, op));
+      return true;
+    }
     dout(10) << "prepare_command setting new crush map" << dendl;
     bufferlist data(m->get_data());
     CrushWrapper crush;
@@ -7235,6 +7240,32 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
       goto reply;
     }
 
+    int64_t prior_version = 0;
+    if (cmd_getval(g_ceph_context, cmdmap, "prior_version", prior_version)) {
+      if (prior_version == osdmap.get_crush_version() - 1) {
+       // see if we are a resend of the last update.  this is imperfect
+       // (multiple racing updaters may not both get reliable success)
+       // but we expect crush updaters (via this interface) to be rare-ish.
+       bufferlist current, proposed;
+       osdmap.crush->encode(current, mon->get_quorum_con_features());
+       crush.encode(proposed, mon->get_quorum_con_features());
+       if (current.contents_equal(proposed)) {
+         dout(10) << __func__
+                  << " proposed matches current and version equals previous"
+                  << dendl;
+         err = 0;
+         ss << osdmap.get_crush_version();
+         goto reply;
+       }
+      }
+      if (prior_version != osdmap.get_crush_version()) {
+       err = -EPERM;
+       ss << "prior_version " << prior_version << " != crush version "
+          << osdmap.get_crush_version();
+       goto reply;
+      }
+    }
+
     if (!validate_crush_against_features(&crush, ss)) {
       err = -EINVAL;
       goto reply;