From: Sage Weil Date: Mon, 15 Feb 2021 22:56:58 +0000 (-0500) Subject: mon: convert ConfigKeyService -> KVMonitor X-Git-Tag: v17.1.0~2900^2~11 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=bb7ebc41532aeb23cff2241ab07b3f01c2f57ddd;p=ceph.git mon: convert ConfigKeyService -> KVMonitor Convert this into a normal PaxosService. This gives us (1) code uniformity and also (2) a changelog in the form of the paxos commits for recently changed keys that we can use to allow clients to subscribe to changes. For upgrades this is pretty painless: - the actual kv data is in the same place - an old mon will still see updates made by a new monitor - a new mon will still see updates made by an old monitor, but won't get a paxosservice version bump. Signed-off-by: Sage Weil --- diff --git a/src/mon/AuthMonitor.cc b/src/mon/AuthMonitor.cc index 2b27dc6f530b..660a641a4855 100644 --- a/src/mon/AuthMonitor.cc +++ b/src/mon/AuthMonitor.cc @@ -17,7 +17,6 @@ #include "mon/AuthMonitor.h" #include "mon/Monitor.h" #include "mon/MonitorDBStore.h" -#include "mon/ConfigKeyService.h" #include "mon/OSDMonitor.h" #include "mon/MDSMonitor.h" #include "mon/ConfigMonitor.h" diff --git a/src/mon/CMakeLists.txt b/src/mon/CMakeLists.txt index 93f02c8bd5d1..088fa6a0cdd6 100644 --- a/src/mon/CMakeLists.txt +++ b/src/mon/CMakeLists.txt @@ -20,7 +20,7 @@ set(lib_mon_srcs ElectionLogic.cc ConnectionTracker.cc HealthMonitor.cc - ConfigKeyService.cc + KVMonitor.cc ../mds/MDSAuthCaps.cc ../mgr/mgr_commands.cc ../osd/OSDCap.cc diff --git a/src/mon/ConfigKeyService.cc b/src/mon/ConfigKeyService.cc deleted file mode 100644 index a75f0dcbeb82..000000000000 --- a/src/mon/ConfigKeyService.cc +++ /dev/null @@ -1,414 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2013 Inktank, Inc - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - * - */ - -#include -#include -#include - -#include "mon/Monitor.h" -#include "mon/ConfigKeyService.h" -#include "mon/MonitorDBStore.h" -#include "mon/OSDMonitor.h" -#include "common/errno.h" -#include "include/stringify.h" - -#include "include/ceph_assert.h" // re-clobber ceph_assert() -#define dout_subsys ceph_subsys_mon -#undef dout_prefix -#define dout_prefix _prefix(_dout, mon, this) -using namespace TOPNSPC::common; - -using namespace std::literals; -using std::cerr; -using std::cout; -using std::dec; -using std::hex; -using std::list; -using std::map; -using std::make_pair; -using std::ostream; -using std::ostringstream; -using std::pair; -using std::set; -using std::setfill; -using std::string; -using std::stringstream; -using std::to_string; -using std::vector; -using std::unique_ptr; - -using ceph::bufferlist; -using ceph::decode; -using ceph::encode; -using ceph::Formatter; -using ceph::JSONFormatter; -using ceph::mono_clock; -using ceph::mono_time; -using ceph::parse_timespan; -using ceph::timespan_str; - -static ostream& _prefix(std::ostream *_dout, const Monitor &mon, - const ConfigKeyService *service) -{ - return *_dout << "mon." << mon.name << "@" << mon.rank - << "(" << mon.get_state_name() << ").config_key"; -} - -const string CONFIG_PREFIX = "mon_config_key"; - -ConfigKeyService::ConfigKeyService(Monitor &m, Paxos &p) - : mon(m), - paxos(p) -{} - -bool ConfigKeyService::in_quorum() const -{ - return (mon.is_leader() || mon.is_peon()); -} - -int ConfigKeyService::store_get(const string &key, bufferlist &bl) -{ - return mon.store->get(CONFIG_PREFIX, key, bl); -} - -void ConfigKeyService::get_store_prefixes(set& s) const -{ - s.insert(CONFIG_PREFIX); -} - -void ConfigKeyService::store_put(const string &key, bufferlist &bl, Context *cb) -{ - MonitorDBStore::TransactionRef t = paxos.get_pending_transaction(); - t->put(CONFIG_PREFIX, key, bl); - if (cb) - paxos.queue_pending_finisher(cb); - paxos.trigger_propose(); -} - -void ConfigKeyService::store_delete(const string &key, Context *cb) -{ - MonitorDBStore::TransactionRef t = paxos.get_pending_transaction(); - store_delete(t, key); - if (cb) - paxos.queue_pending_finisher(cb); - paxos.trigger_propose(); -} - -void ConfigKeyService::store_delete( - MonitorDBStore::TransactionRef t, - const string &key) -{ - t->erase(CONFIG_PREFIX, key); -} - -bool ConfigKeyService::store_exists(const string &key) -{ - return mon.store->exists(CONFIG_PREFIX, key); -} - -void ConfigKeyService::store_list(stringstream &ss) -{ - KeyValueDB::Iterator iter = - mon.store->get_iterator(CONFIG_PREFIX); - - JSONFormatter f(true); - f.open_array_section("keys"); - - while (iter->valid()) { - string key(iter->key()); - f.dump_string("key", key); - iter->next(); - } - f.close_section(); - f.flush(ss); -} - -bool ConfigKeyService::store_has_prefix(const string &prefix) -{ - KeyValueDB::Iterator iter = - mon.store->get_iterator(CONFIG_PREFIX); - - while (iter->valid()) { - string key(iter->key()); - size_t p = key.find(prefix); - if (p != string::npos && p == 0) { - return true; - } - iter->next(); - } - return false; -} - -static bool is_binary_string(const string& s) -{ - for (auto c : s) { - // \n and \t are escaped in JSON; other control characters are not. - if ((c < 0x20 && c != '\n' && c != '\t') || c >= 0x7f) { - return true; - } - } - return false; -} - -void ConfigKeyService::store_dump(stringstream &ss, const string& prefix) -{ - KeyValueDB::Iterator iter = - mon.store->get_iterator(CONFIG_PREFIX); - - dout(10) << __func__ << " prefix '" << prefix << "'" << dendl; - if (prefix.size()) { - iter->lower_bound(prefix); - } - - JSONFormatter f(true); - f.open_object_section("config-key store"); - - while (iter->valid()) { - if (prefix.size() && - iter->key().find(prefix) != 0) { - break; - } - string s = iter->value().to_str(); - if (is_binary_string(s)) { - ostringstream ss; - ss << "<<< binary blob of length " << s.size() << " >>>"; - f.dump_string(iter->key().c_str(), ss.str()); - } else { - f.dump_string(iter->key().c_str(), s); - } - iter->next(); - } - f.close_section(); - f.flush(ss); -} - -void ConfigKeyService::store_delete_prefix( - MonitorDBStore::TransactionRef t, - const string &prefix) -{ - KeyValueDB::Iterator iter = - mon.store->get_iterator(CONFIG_PREFIX); - - while (iter->valid()) { - string key(iter->key()); - - size_t p = key.find(prefix); - if (p != string::npos && p == 0) { - store_delete(t, key); - } - iter->next(); - } -} - -bool ConfigKeyService::dispatch(MonOpRequestRef op) -{ - Message *m = op->get_req(); - ceph_assert(m != NULL); - dout(10) << __func__ << " " << *m << dendl; - - if (!in_quorum()) { - dout(1) << __func__ << " not in quorum -- waiting" << dendl; - paxos.wait_for_readable(op, new Monitor::C_RetryMessage(&mon, op)); - return false; - } - - ceph_assert(m->get_type() == MSG_MON_COMMAND); - - MMonCommand *cmd = static_cast(m); - - ceph_assert(!cmd->cmd.empty()); - - int ret = 0; - stringstream ss; - bufferlist rdata; - - string prefix; - cmdmap_t cmdmap; - - if (!TOPNSPC::common::cmdmap_from_json(cmd->cmd, &cmdmap, ss)) { - return false; - } - - cmd_getval(cmdmap, "prefix", prefix); - string key; - cmd_getval(cmdmap, "key", key); - - if (prefix == "config-key get") { - ret = store_get(key, rdata); - if (ret < 0) { - ceph_assert(!rdata.length()); - ss << "error obtaining '" << key << "': " << cpp_strerror(ret); - goto out; - } - ss << "obtained '" << key << "'"; - - } else if (prefix == "config-key put" || - prefix == "config-key set") { - if (!mon.is_leader()) { - mon.forward_request_leader(op); - // we forward the message; so return now. - return true; - } - - bufferlist data; - string val; - if (cmd_getval(cmdmap, "val", val)) { - // they specified a value in the command instead of a file - data.append(val); - } else if (cmd->get_data_len() > 0) { - // they specified '-i ' - data = cmd->get_data(); - } - if (data.length() > (size_t) g_conf()->mon_config_key_max_entry_size) { - ret = -EFBIG; // File too large - ss << "error: entry size limited to " - << g_conf()->mon_config_key_max_entry_size << " bytes. " - << "Use 'mon config key max entry size' to manually adjust"; - goto out; - } - - ss << "set " << key; - - // we'll reply to the message once the proposal has been handled - store_put(key, data, - new Monitor::C_Command(mon, op, 0, ss.str(), 0)); - // return for now; we'll put the message once it's done. - return true; - - } else if (prefix == "config-key del" || - prefix == "config-key rm") { - if (!mon.is_leader()) { - mon.forward_request_leader(op); - return true; - } - - if (!store_exists(key)) { - ret = 0; - ss << "no such key '" << key << "'"; - goto out; - } - store_delete(key, new Monitor::C_Command(mon, op, 0, "key deleted", 0)); - // return for now; we'll put the message once it's done - return true; - - } else if (prefix == "config-key exists") { - bool exists = store_exists(key); - ss << "key '" << key << "'"; - if (exists) { - ss << " exists"; - ret = 0; - } else { - ss << " doesn't exist"; - ret = -ENOENT; - } - - } else if (prefix == "config-key list" || - prefix == "config-key ls") { - stringstream tmp_ss; - store_list(tmp_ss); - rdata.append(tmp_ss); - ret = 0; - - } else if (prefix == "config-key dump") { - string prefix; - cmd_getval(cmdmap, "key", prefix); - stringstream tmp_ss; - store_dump(tmp_ss, prefix); - rdata.append(tmp_ss); - ret = 0; - - } - -out: - if (!cmd->get_source().is_mon()) { - string rs = ss.str(); - mon.reply_command(op, ret, rs, rdata, 0); - } - - return (ret == 0); -} - -string _get_dmcrypt_prefix(const uuid_d& uuid, const string k) -{ - return "dm-crypt/osd/" + stringify(uuid) + "/" + k; -} - -int ConfigKeyService::validate_osd_destroy( - const int32_t id, - const uuid_d& uuid) -{ - string dmcrypt_prefix = _get_dmcrypt_prefix(uuid, ""); - string daemon_prefix = - "daemon-private/osd." + stringify(id) + "/"; - - if (!store_has_prefix(dmcrypt_prefix) && - !store_has_prefix(daemon_prefix)) { - return -ENOENT; - } - return 0; -} - -void ConfigKeyService::do_osd_destroy(int32_t id, uuid_d& uuid) -{ - string dmcrypt_prefix = _get_dmcrypt_prefix(uuid, ""); - string daemon_prefix = - "daemon-private/osd." + stringify(id) + "/"; - - MonitorDBStore::TransactionRef t = paxos.get_pending_transaction(); - for (auto p : { dmcrypt_prefix, daemon_prefix }) { - store_delete_prefix(t, p); - } - - paxos.trigger_propose(); -} - -int ConfigKeyService::validate_osd_new( - const uuid_d& uuid, - const string& dmcrypt_key, - stringstream& ss) -{ - string dmcrypt_prefix = _get_dmcrypt_prefix(uuid, "luks"); - bufferlist value; - value.append(dmcrypt_key); - - if (store_exists(dmcrypt_prefix)) { - bufferlist existing_value; - int err = store_get(dmcrypt_prefix, existing_value); - if (err < 0) { - dout(10) << __func__ << " unable to get dm-crypt key from store (r = " - << err << ")" << dendl; - return err; - } - if (existing_value.contents_equal(value)) { - // both values match; this will be an idempotent op. - return EEXIST; - } - ss << "dm-crypt key already exists and does not match"; - return -EEXIST; - } - return 0; -} - -void ConfigKeyService::do_osd_new( - const uuid_d& uuid, - const string& dmcrypt_key) -{ - ceph_assert(paxos.is_plugged()); - - string dmcrypt_key_prefix = _get_dmcrypt_prefix(uuid, "luks"); - bufferlist dmcrypt_key_value; - dmcrypt_key_value.append(dmcrypt_key); - // store_put() will call trigger_propose - store_put(dmcrypt_key_prefix, dmcrypt_key_value, nullptr); -} diff --git a/src/mon/ConfigKeyService.h b/src/mon/ConfigKeyService.h deleted file mode 100644 index 71ee59bd858e..000000000000 --- a/src/mon/ConfigKeyService.h +++ /dev/null @@ -1,63 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2013 Inktank, Inc - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - * - */ -#ifndef CEPH_MON_CONFIG_KEY_SERVICE_H -#define CEPH_MON_CONFIG_KEY_SERVICE_H - -#include "include/Context.h" -#include "mon/MonOpRequest.h" -#include "mon/MonitorDBStore.h" - -class Paxos; -class Monitor; - -class ConfigKeyService -{ -public: - ConfigKeyService(Monitor &m, Paxos &p); - ~ConfigKeyService() {} - - bool dispatch(MonOpRequestRef op); - - int validate_osd_destroy(const int32_t id, const uuid_d& uuid); - void do_osd_destroy(int32_t id, uuid_d& uuid); - int validate_osd_new( - const uuid_d& uuid, - const std::string& dmcrypt_key, - std::stringstream& ss); - void do_osd_new(const uuid_d& uuid, const std::string& dmcrypt_key); - - void get_store_prefixes(std::set& s) const; - -private: - Monitor &mon; - Paxos &paxos; - - bool in_quorum() const; - - int store_get(const std::string &key, ceph::buffer::list &bl); - void store_put(const std::string &key, ceph::buffer::list &bl, Context *cb = NULL); - void store_delete(MonitorDBStore::TransactionRef t, const std::string &key); - void store_delete(const std::string &key, Context *cb = NULL); - void store_delete_prefix( - MonitorDBStore::TransactionRef t, - const std::string &prefix); - void store_list(std::stringstream &ss); - void store_dump(std::stringstream &ss, const std::string& prefix); - bool store_exists(const std::string &key); - bool store_has_prefix(const std::string &prefix); - - static const std::string STORE_PREFIX; -}; - -#endif // CEPH_MON_CONFIG_KEY_SERVICE_H diff --git a/src/mon/ConfigMonitor.cc b/src/mon/ConfigMonitor.cc index ddeb8d8b995d..2d62c0ca521a 100644 --- a/src/mon/ConfigMonitor.cc +++ b/src/mon/ConfigMonitor.cc @@ -5,6 +5,7 @@ #include "mon/Monitor.h" #include "mon/ConfigMonitor.h" +#include "mon/KVMonitor.h" #include "mon/MgrMonitor.h" #include "mon/OSDMonitor.h" #include "messages/MConfig.h" @@ -112,7 +113,7 @@ void ConfigMonitor::encode_pending(MonitorDBStore::TransactionRef t) bufferlist metabl; ::encode(ceph_clock_now(), metabl); ::encode(pending_description, metabl); - t->put(CONFIG_PREFIX, history, metabl); + t->put(KV_PREFIX, history, metabl); } for (auto& p : pending) { string key = KEY_PREFIX + p.first; @@ -121,17 +122,17 @@ void ConfigMonitor::encode_pending(MonitorDBStore::TransactionRef t) if (p.second && *p.second == q->second) { continue; } - t->put(CONFIG_PREFIX, history + "-" + p.first, q->second); + t->put(KV_PREFIX, history + "-" + p.first, q->second); } else if (!p.second) { continue; } if (p.second) { dout(20) << __func__ << " set " << key << dendl; - t->put(CONFIG_PREFIX, key, *p.second); - t->put(CONFIG_PREFIX, history + "+" + p.first, *p.second); + t->put(KV_PREFIX, key, *p.second); + t->put(KV_PREFIX, history + "+" + p.first, *p.second); } else { dout(20) << __func__ << " rm " << key << dendl; - t->erase(CONFIG_PREFIX, key); + t->erase(KV_PREFIX, key); } } } @@ -764,7 +765,7 @@ void ConfigMonitor::load_config() }; unsigned num = 0; - KeyValueDB::Iterator it = mon.store->get_iterator(CONFIG_PREFIX); + KeyValueDB::Iterator it = mon.store->get_iterator(KV_PREFIX); it->lower_bound(KEY_PREFIX); config_map.clear(); current.clear(); @@ -873,7 +874,7 @@ void ConfigMonitor::load_changeset(version_t v, ConfigChangeSet *ch) { ch->version = v; string prefix = HISTORY_PREFIX + stringify(v) + "/"; - KeyValueDB::Iterator it = mon.store->get_iterator(CONFIG_PREFIX); + KeyValueDB::Iterator it = mon.store->get_iterator(KV_PREFIX); it->lower_bound(prefix); while (it->valid() && it->key().find(prefix) == 0) { if (it->key() == prefix) { diff --git a/src/mon/KVMonitor.cc b/src/mon/KVMonitor.cc new file mode 100644 index 000000000000..0fd279d11f7e --- /dev/null +++ b/src/mon/KVMonitor.cc @@ -0,0 +1,419 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "mon/Monitor.h" +#include "mon/KVMonitor.h" +#include "include/stringify.h" + +#define dout_subsys ceph_subsys_mon +#undef dout_prefix +#define dout_prefix _prefix(_dout, mon, this) + +static ostream& _prefix(std::ostream *_dout, const Monitor &mon, + const KVMonitor *hmon) { + return *_dout << "mon." << mon.name << "@" << mon.rank + << "(" << mon.get_state_name() << ").kv "; +} + +const string KV_PREFIX = "mon_config_key"; + +const int MAX_HISTORY = 50; + + +static bool is_binary_string(const string& s) +{ + for (auto c : s) { + // \n and \t are escaped in JSON; other control characters are not. + if ((c < 0x20 && c != '\n' && c != '\t') || c >= 0x7f) { + return true; + } + } + return false; +} + + +KVMonitor::KVMonitor(Monitor &m, Paxos &p, const string& service_name) + : PaxosService(m, p, service_name) { +} + +void KVMonitor::init() +{ + dout(10) << __func__ << dendl; +} + +void KVMonitor::create_initial() +{ + dout(10) << __func__ << dendl; + version = 0; + pending.clear(); +} + +void KVMonitor::update_from_paxos(bool *need_bootstrap) +{ + if (version == get_last_committed()) { + return; + } + version = get_last_committed(); + dout(10) << __func__ << " " << version << dendl; + //check_all_subs(); +} + +void KVMonitor::create_pending() +{ + dout(10) << " " << version << dendl; + pending.clear(); +} + +void KVMonitor::encode_pending(MonitorDBStore::TransactionRef t) +{ + dout(10) << " " << (version+1) << dendl; + put_last_committed(t, version+1); + + // record the delta for this commit point + bufferlist bl; + encode(pending, bl); + put_version(t, version+1, bl); + + // make actual changes + for (auto& p : pending) { + string key = p.first; + if (p.second) { + dout(20) << __func__ << " set " << key << dendl; + t->put(KV_PREFIX, key, *p.second); + } else { + dout(20) << __func__ << " rm " << key << dendl; + t->erase(KV_PREFIX, key); + } + } +} + +version_t KVMonitor::get_trim_to() const +{ + // we don't need that many old states, but keep a few + if (version > MAX_HISTORY) { + return version - MAX_HISTORY; + } + return 0; +} + +void KVMonitor::get_store_prefixes(set& s) const +{ + s.insert(service_name); + s.insert(KV_PREFIX); +} + +void KVMonitor::tick() +{ + if (!is_active() || !mon.is_leader()) { + return; + } + dout(10) << __func__ << dendl; +} + +void KVMonitor::on_active() +{ +} + + +bool KVMonitor::preprocess_query(MonOpRequestRef op) +{ + switch (op->get_req()->get_type()) { + case MSG_MON_COMMAND: + try { + return preprocess_command(op); + } catch (const bad_cmd_get& e) { + bufferlist bl; + mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed()); + return true; + } + } + return false; +} + +bool KVMonitor::preprocess_command(MonOpRequestRef op) +{ + auto m = op->get_req(); + std::stringstream ss; + int err = 0; + + cmdmap_t cmdmap; + if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) { + string rs = ss.str(); + mon.reply_command(op, -EINVAL, rs, get_last_committed()); + return true; + } + string format; + cmd_getval(cmdmap, "format", format, string("plain")); + boost::scoped_ptr f(Formatter::create(format)); + + string prefix; + cmd_getval(cmdmap, "prefix", prefix); + string key; + cmd_getval(cmdmap, "key", key); + + bufferlist odata; + + if (prefix == "config-key get") { + err = mon.store->get(KV_PREFIX, key, odata); + } + else if (prefix == "config-key exists") { + bool exists = mon.store->exists(KV_PREFIX, key); + ss << "key '" << key << "'"; + if (exists) { + ss << " exists"; + err = 0; + } else { + ss << " doesn't exist"; + err = -ENOENT; + } + } + else if (prefix == "config-key list" || + prefix == "config-key ls") { + if (!f) { + f.reset(Formatter::create("json-pretty")); + } + KeyValueDB::Iterator iter = mon.store->get_iterator(KV_PREFIX); + f->open_array_section("keys"); + while (iter->valid()) { + string key(iter->key()); + f->dump_string("key", key); + iter->next(); + } + f->close_section(); + + stringstream tmp_ss; + f->flush(tmp_ss); + odata.append(tmp_ss); + err = 0; + } + else if (prefix == "config-key dump") { + if (!f) { + f.reset(Formatter::create("json-pretty")); + } + + KeyValueDB::Iterator iter = mon.store->get_iterator(KV_PREFIX); + if (key.size()) { + iter->lower_bound(key); + } + f->open_object_section("config-key store"); + while (iter->valid()) { + if (key.size() && + iter->key().find(key) != 0) { + break; + } + string s = iter->value().to_str(); + if (is_binary_string(s)) { + ostringstream ss; + ss << "<<< binary blob of length " << s.size() << " >>>"; + f->dump_string(iter->key().c_str(), ss.str()); + } else { + f->dump_string(iter->key().c_str(), s); + } + iter->next(); + } + f->close_section(); + + stringstream tmp_ss; + f->flush(tmp_ss); + odata.append(tmp_ss); + err = 0; + } + else { + return false; + } + + mon.reply_command(op, err, ss.str(), odata, get_last_committed()); + return true; +} + +bool KVMonitor::prepare_update(MonOpRequestRef op) +{ + Message *m = op->get_req(); + dout(7) << "prepare_update " << *m + << " from " << m->get_orig_source_inst() << dendl; + switch (m->get_type()) { + case MSG_MON_COMMAND: + try { + return prepare_command(op); + } catch (const bad_cmd_get& e) { + bufferlist bl; + mon.reply_command(op, -EINVAL, e.what(), bl, get_last_committed()); + return true; + } + } + return false; +} + + +bool KVMonitor::prepare_command(MonOpRequestRef op) +{ + auto m = op->get_req(); + std::stringstream ss; + int err = 0; + bufferlist odata; + + cmdmap_t cmdmap; + if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) { + string rs = ss.str(); + mon.reply_command(op, -EINVAL, rs, get_last_committed()); + return true; + } + + string prefix; + cmd_getval(cmdmap, "prefix", prefix); + string key; + if (!cmd_getval(cmdmap, "key", key)) { + err = -EINVAL; + ss << "must specify a key"; + goto reply; + } + + + if (prefix == "config-key set" || + prefix == "config-key put") { + bufferlist data; + string val; + if (cmd_getval(cmdmap, "val", val)) { + // they specified a value in the command instead of a file + data.append(val); + } else if (m->get_data_len() > 0) { + // they specified '-i ' + data = m->get_data(); + } + if (data.length() > (size_t) g_conf()->mon_config_key_max_entry_size) { + err = -EFBIG; // File too large + ss << "error: entry size limited to " + << g_conf()->mon_config_key_max_entry_size << " bytes. " + << "Use 'mon config key max entry size' to manually adjust"; + goto reply; + } + + ss << "set " << key; + pending[key] = data; + goto update; + } + else if (prefix == "config-key del" || + prefix == "config-key rm") { + ss << "key deleted"; + pending[key] = boost::none; + goto update; + } + else { + ss << "unknown command " << prefix; + err = -EINVAL; + } + +reply: + mon.reply_command(op, err, ss.str(), odata, get_last_committed()); + return false; + +update: + // see if there is an actual change + if (pending.empty()) { + err = 0; + goto reply; + } + force_immediate_propose(); // faster response + wait_for_finished_proposal( + op, + new Monitor::C_Command( + mon, op, 0, ss.str(), odata, + get_last_committed() + 1)); + return true; +} + + + + +static string _get_dmcrypt_prefix(const uuid_d& uuid, const string k) +{ + return "dm-crypt/osd/" + stringify(uuid) + "/" + k; +} + +bool KVMonitor::_have_prefix(const string &prefix) +{ + KeyValueDB::Iterator iter = mon.store->get_iterator(KV_PREFIX); + + while (iter->valid()) { + string key(iter->key()); + size_t p = key.find(prefix); + if (p != string::npos && p == 0) { + return true; + } + iter->next(); + } + return false; +} + +int KVMonitor::validate_osd_destroy( + const int32_t id, + const uuid_d& uuid) +{ + string dmcrypt_prefix = _get_dmcrypt_prefix(uuid, ""); + string daemon_prefix = + "daemon-private/osd." + stringify(id) + "/"; + + if (!_have_prefix(dmcrypt_prefix) && + !_have_prefix(daemon_prefix)) { + return -ENOENT; + } + return 0; +} + +void KVMonitor::do_osd_destroy(int32_t id, uuid_d& uuid) +{ + string dmcrypt_prefix = _get_dmcrypt_prefix(uuid, ""); + string daemon_prefix = + "daemon-private/osd." + stringify(id) + "/"; + + for (auto& prefix : { dmcrypt_prefix, daemon_prefix }) { + KeyValueDB::Iterator iter = mon.store->get_iterator(KV_PREFIX); + iter->lower_bound(prefix); + if (iter->key().find(prefix) != 0) { + break; + } + pending[iter->key()] = boost::none; + } + + paxos.trigger_propose(); +} + +int KVMonitor::validate_osd_new( + const uuid_d& uuid, + const string& dmcrypt_key, + stringstream& ss) +{ + string dmcrypt_prefix = _get_dmcrypt_prefix(uuid, "luks"); + bufferlist value; + value.append(dmcrypt_key); + + if (mon.store->exists(KV_PREFIX, dmcrypt_prefix)) { + bufferlist existing_value; + int err = mon.store->get(KV_PREFIX, dmcrypt_prefix, existing_value); + if (err < 0) { + dout(10) << __func__ << " unable to get dm-crypt key from store (r = " + << err << ")" << dendl; + return err; + } + if (existing_value.contents_equal(value)) { + // both values match; this will be an idempotent op. + return EEXIST; + } + ss << "dm-crypt key already exists and does not match"; + return -EEXIST; + } + return 0; +} + +void KVMonitor::do_osd_new( + const uuid_d& uuid, + const string& dmcrypt_key) +{ + ceph_assert(paxos.is_plugged()); + + string dmcrypt_key_prefix = _get_dmcrypt_prefix(uuid, "luks"); + bufferlist dmcrypt_key_value; + dmcrypt_key_value.append(dmcrypt_key); + + pending[dmcrypt_key_prefix] = dmcrypt_key_value; +} diff --git a/src/mon/KVMonitor.h b/src/mon/KVMonitor.h new file mode 100644 index 000000000000..28c71f003c2a --- /dev/null +++ b/src/mon/KVMonitor.h @@ -0,0 +1,52 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#pragma once + +#include + +#include "mon/PaxosService.h" + +class MonSession; + +extern const std::string KV_PREFIX; + +class KVMonitor : public PaxosService +{ + version_t version = 0; + std::map> pending; + + bool _have_prefix(const string &prefix); + +public: + KVMonitor(Monitor &m, Paxos &p, const std::string& service_name); + + void init() override; + + void get_store_prefixes(set& s) const override; + + bool preprocess_command(MonOpRequestRef op); + bool prepare_command(MonOpRequestRef op); + + bool preprocess_query(MonOpRequestRef op) override; + bool prepare_update(MonOpRequestRef op) override; + + void create_initial() override; + void update_from_paxos(bool *need_bootstrap) override; + void create_pending() override; + void encode_pending(MonitorDBStore::TransactionRef t) override; + version_t get_trim_to() const override; + + void encode_full(MonitorDBStore::TransactionRef t) override { } + + void on_active() override; + void tick() override; + + int validate_osd_destroy(const int32_t id, const uuid_d& uuid); + void do_osd_destroy(int32_t id, uuid_d& uuid); + int validate_osd_new( + const uuid_d& uuid, + const std::string& dmcrypt_key, + std::stringstream& ss); + void do_osd_new(const uuid_d& uuid, const std::string& dmcrypt_key); +}; diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index cc040c88e994..323ad96bbad8 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -1200,7 +1200,7 @@ COMMAND("osd tier add-cache " "osd", "rw") /* - * mon/ConfigKeyService.cc + * mon/KVMonitor.cc */ COMMAND("config-key get " diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index c58e6e528a43..30eb07410b20 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -85,7 +85,7 @@ #include "MgrMonitor.h" #include "MgrStatMonitor.h" #include "ConfigMonitor.h" -#include "mon/ConfigKeyService.h" +#include "KVMonitor.h" #include "mon/HealthMonitor.h" #include "common/config.h" #include "common/cmdparse.h" @@ -248,8 +248,7 @@ Monitor::Monitor(CephContext* cct_, string nm, MonitorDBStore *s, paxos_service[PAXOS_MGRSTAT].reset(new MgrStatMonitor(*this, *paxos, "mgrstat")); paxos_service[PAXOS_HEALTH].reset(new HealthMonitor(*this, *paxos, "health")); paxos_service[PAXOS_CONFIG].reset(new ConfigMonitor(*this, *paxos, "config")); - - config_key_service = std::make_unique(*this, *paxos); + paxos_service[PAXOS_KV].reset(new KVMonitor(*this, *paxos, "kv")); bool r = mon_caps.parse("allow *", NULL); ceph_assert(r); @@ -1369,7 +1368,6 @@ set Monitor::get_sync_targets_names() for (auto& svc : paxos_service) { svc->get_store_prefixes(targets); } - config_key_service->get_store_prefixes(targets); return targets; } @@ -3533,7 +3531,7 @@ void Monitor::handle_command(MonOpRequestRef op) } if (module == "config-key") { - config_key_service->dispatch(op); + kvmon()->dispatch(op); return; } diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h index 1b8bd8c7c6df..b6111137b82b 100644 --- a/src/mon/Monitor.h +++ b/src/mon/Monitor.h @@ -99,7 +99,6 @@ enum { l_mon_last, }; -class ConfigKeyService; class PaxosService; class AdminSocketHook; @@ -678,14 +677,16 @@ public: return (class ConfigMonitor*) paxos_service[PAXOS_CONFIG].get(); } + class KVMonitor *kvmon() { + return (class KVMonitor*) paxos_service[PAXOS_KV].get(); + } + friend class Paxos; friend class OSDMonitor; friend class MDSMonitor; friend class MonmapMonitor; friend class LogMonitor; - friend class ConfigKeyService; - - std::unique_ptr config_key_service; + friend class KVMonitor; // -- sessions -- MonSessionMap session_map; diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index fecc25be10b5..3d2528ec0022 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -27,7 +27,7 @@ #include "mon/MDSMonitor.h" #include "mon/MgrStatMonitor.h" #include "mon/AuthMonitor.h" -#include "mon/ConfigKeyService.h" +#include "mon/KVMonitor.h" #include "mon/MonitorDBStore.h" #include "mon/Session.h" @@ -9329,7 +9329,7 @@ int OSDMonitor::prepare_command_osd_new( || params.count("cephx_lockbox_secret") || params.count("dmcrypt_key"); - ConfigKeyService *svc = nullptr; + KVMonitor *svc = nullptr; AuthMonitor::auth_entity_t cephx_entity, lockbox_entity; if (has_secrets) { @@ -9373,7 +9373,7 @@ int OSDMonitor::prepare_command_osd_new( } if (has_lockbox) { - svc = mon.config_key_service.get(); + svc = mon.kvmon(); err = svc->validate_osd_new(uuid, dmcrypt_key, ss); if (err < 0) { return err; @@ -9558,7 +9558,7 @@ int OSDMonitor::prepare_command_osd_destroy( } } - auto svc = mon.config_key_service.get(); + auto svc = mon.kvmon(); err = svc->validate_osd_destroy(id, uuid); if (err < 0) { ceph_assert(err == -ENOENT); diff --git a/src/mon/mon_types.h b/src/mon/mon_types.h index d14358d1f65c..ec4b74897708 100644 --- a/src/mon/mon_types.h +++ b/src/mon/mon_types.h @@ -35,13 +35,12 @@ enum { PAXOS_MGRSTAT, PAXOS_HEALTH, PAXOS_CONFIG, + PAXOS_KV, PAXOS_NUM }; #define CEPH_MON_ONDISK_MAGIC "ceph mon volume v012" -extern const std::string CONFIG_PREFIX; - // map of entity_type -> features -> count struct FeatureMap { std::map> m;