From: Sage Weil <sage@newdream.net>
Date: Fri, 26 Feb 2021 16:42:52 +0000 (-0500)
Subject: mon/ConfigMonitor: make config changes via KVMonitor's pending set
X-Git-Tag: v17.1.0~2780^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F39723%2Fhead;p=ceph.git

mon/ConfigMonitor: make config changes via KVMonitor's pending set

We need to ensure that changes we make to the kv store (config/...)
are proposed via KVMonitor so that they are properly versioned there
and shared with subscribers (notably, the mgr).

Fixes: bb7ebc41532aeb23cff2241ab07b3f01c2f57ddd
Signed-off-by: Sage Weil <sage@newdream.net>
---

diff --git a/src/mon/ConfigMonitor.cc b/src/mon/ConfigMonitor.cc
index d305aa4a2429..7f0ee21e6715 100644
--- a/src/mon/ConfigMonitor.cc
+++ b/src/mon/ConfigMonitor.cc
@@ -96,7 +96,14 @@ void ConfigMonitor::encode_pending(MonitorDBStore::TransactionRef t)
 {
   dout(10) << " " << (version+1) << dendl;
   put_last_committed(t, version+1);
+  // NOTE: caller should have done encode_pending_to_kvmon() and
+  // kvmon->propose_pending() to commit the actual config changes.
+}
 
+void ConfigMonitor::encode_pending_to_kvmon()
+{
+  // we need to pass our data through KVMonitor so that it is properly
+  // versioned and shared with subscribers.
   for (auto& [key, value] : pending_cleanup) {
     if (pending.count(key) == 0) {
       derr << __func__ << " repair: adjusting config key '" << key << "'"
@@ -113,7 +120,7 @@ void ConfigMonitor::encode_pending(MonitorDBStore::TransactionRef t)
     bufferlist metabl;
     ::encode(ceph_clock_now(), metabl);
     ::encode(pending_description, metabl);
-    t->put(KV_PREFIX, history, metabl);
+    mon.kvmon()->enqueue_set(history, metabl);
   }
   for (auto& p : pending) {
     string key = KEY_PREFIX + p.first;
@@ -122,17 +129,17 @@ void ConfigMonitor::encode_pending(MonitorDBStore::TransactionRef t)
       if (p.second && *p.second == q->second) {
 	continue;
       }
-      t->put(KV_PREFIX, history + "-" + p.first, q->second);
+      mon.kvmon()->enqueue_set(history + "-" + p.first, q->second);
     } else if (!p.second) {
       continue;
     }
     if (p.second) {
       dout(20) << __func__ << " set " << key << dendl;
-      t->put(KV_PREFIX, key, *p.second);
-      t->put(KV_PREFIX, history + "+" + p.first, *p.second);
-    } else {
+      mon.kvmon()->enqueue_set(key, *p.second);
+      mon.kvmon()->enqueue_set(history + "+" + p.first, *p.second);
+   } else {
       dout(20) << __func__ << " rm " << key << dendl;
-      t->erase(KV_PREFIX, key);
+      mon.kvmon()->enqueue_rm(key);
     }
   }
 }
@@ -513,6 +520,13 @@ bool ConfigMonitor::prepare_command(MonOpRequestRef op)
   std::stringstream ss;
   int err = -EINVAL;
 
+  // make sure kv is writeable.
+  if (!mon.kvmon()->is_writeable()) {
+    dout(10) << __func__ << " waiting for kv mon to be writeable" << dendl;
+    mon.kvmon()->wait_for_writeable(op, new C_RetryMessage(this, op));
+    return false;
+  }
+
   cmdmap_t cmdmap;
   if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
     string rs = ss.str();
@@ -724,7 +738,12 @@ update:
     err = 0;
     goto reply;
   }
-  force_immediate_propose();  // faster response
+  // immediately propose *with* KV mon
+  encode_pending_to_kvmon();
+  paxos.plug();
+  mon.kvmon()->propose_pending();
+  paxos.unplug();
+  force_immediate_propose();
   wait_for_finished_proposal(
     op,
     new Monitor::C_Command(
@@ -743,7 +762,11 @@ void ConfigMonitor::tick()
   if (!pending_cleanup.empty()) {
     changed = true;
   }
-  if (changed) {
+  if (changed && mon.kvmon()->is_writeable()) {
+    paxos.plug();
+    encode_pending_to_kvmon();
+    mon.kvmon()->propose_pending();
+    paxos.unplug();
     propose_pending();
   }
 }
diff --git a/src/mon/ConfigMonitor.h b/src/mon/ConfigMonitor.h
index 8d05dc3b46b6..e6c12a3d7f01 100644
--- a/src/mon/ConfigMonitor.h
+++ b/src/mon/ConfigMonitor.h
@@ -20,6 +20,8 @@ class ConfigMonitor : public PaxosService
 
   std::map<std::string,ceph::buffer::list> current;
 
+  void encode_pending_to_kvmon();
+
 public:
   ConfigMonitor(Monitor &m, Paxos &p, const std::string& service_name);
 
diff --git a/src/mon/KVMonitor.h b/src/mon/KVMonitor.h
index 8c0204390f53..c14c16380ee3 100644
--- a/src/mon/KVMonitor.h
+++ b/src/mon/KVMonitor.h
@@ -55,4 +55,15 @@ public:
   void check_all_subs();
 
   bool maybe_send_update(Subscription *sub);
+
+
+  // used by other services to adjust kv content; note that callers MUST ensure that
+  // propose_pending() is called and a commit is forced to provide atomicity and
+  // proper subscriber notifications.
+  void enqueue_set(const std::string& key, bufferlist &v) {
+    pending[key] = v;
+  }
+  void enqueue_rm(const std::string& key) {
+    pending[key] = boost::none;
+  }
 };