From: Joao Eduardo Luis Date: Tue, 2 May 2017 22:58:51 +0000 (+0100) Subject: mon: add `osd new` X-Git-Tag: v12.1.0~266^2~5 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=c9e6cac1cfae73cc4bce4a995ed2bba5dc85ae4b;p=ceph.git mon: add `osd new` Signed-off-by: Joao Eduardo Luis --- diff --git a/src/mon/AuthMonitor.cc b/src/mon/AuthMonitor.cc index 3c41a2b8aeb..000c7b39299 100644 --- a/src/mon/AuthMonitor.cc +++ b/src/mon/AuthMonitor.cc @@ -18,6 +18,7 @@ #include "mon/Monitor.h" #include "mon/MonitorDBStore.h" #include "mon/ConfigKeyService.h" +#include "mon/OSDMonitor.h" #include "messages/MMonCommand.h" #include "messages/MAuth.h" @@ -696,9 +697,18 @@ bool AuthMonitor::entity_is_pending(EntityName& entity) } int AuthMonitor::exists_and_matches_entity( - EntityName& name, - EntityAuth& auth, - map& caps, + const auth_entity_t& entity, + bool has_secret, + stringstream& ss) +{ + return exists_and_matches_entity(entity.name, entity.auth, + entity.auth.caps, has_secret, ss); +} + +int AuthMonitor::exists_and_matches_entity( + const EntityName& name, + const EntityAuth& auth, + const map& caps, bool has_secret, stringstream& ss) { @@ -819,6 +829,162 @@ int AuthMonitor::do_osd_destroy( return 0; } +bufferlist _encode_cap(const string& cap) +{ + bufferlist bl; + ::encode(cap, bl); + return bl; +} + +int _create_auth( + EntityAuth& auth, + const string& key, + const map& caps) +{ + if (key.empty()) + return -EINVAL; + try { + auth.key.decode_base64(key); + } catch (buffer::error& e) { + return -EINVAL; + } + auth.caps = caps; + return 0; +} + +int AuthMonitor::validate_osd_new( + int32_t id, + const uuid_d& uuid, + const string& cephx_secret, + const string& lockbox_secret, + auth_entity_t& cephx_entity, + auth_entity_t& lockbox_entity, + stringstream& ss) +{ + + dout(10) << __func__ << " osd." << id << " uuid " << uuid << dendl; + + map cephx_caps = { + { "osd", _encode_cap("allow *") }, + { "mon", _encode_cap("allow profile osd") }, + { "mgr", _encode_cap("allow profile osd") } + }; + map lockbox_caps = { + { "mon", _encode_cap("allow command \"config-key get\" " + "with key=\"dm-crypt/osd/" + + stringify(uuid) + + "/luks\"") } + }; + + bool has_lockbox = !lockbox_secret.empty(); + + string cephx_name = "osd." + stringify(id); + string lockbox_name = "client.osd-lockbox." + stringify(uuid); + + if (!cephx_entity.name.from_str(cephx_name)) { + dout(10) << __func__ << " invalid cephx entity '" + << cephx_name << "'" << dendl; + ss << "invalid cephx key entity '" << cephx_name << "'"; + return -EINVAL; + } + + if (has_lockbox) { + if (!lockbox_entity.name.from_str(lockbox_name)) { + dout(10) << __func__ << " invalid cephx lockbox entity '" + << lockbox_name << "'" << dendl; + ss << "invalid cephx lockbox entity '" << lockbox_name << "'"; + return -EINVAL; + } + } + + if (entity_is_pending(cephx_entity.name) || + (has_lockbox && entity_is_pending(lockbox_entity.name))) { + // If we have pending entities for either the cephx secret or the + // lockbox secret, then our safest bet is to retry the command at + // a later time. These entities may be pending because an `osd new` + // command has been run (which is unlikely, due to the nature of + // the operation, which will force a paxos proposal), or (more likely) + // because a competing client created those entities before we handled + // the `osd new` command. Regardless, let's wait and see. + return -EAGAIN; + } + + if (!is_valid_cephx_key(cephx_secret)) { + ss << "invalid cephx secret."; + return -EINVAL; + } + + if (has_lockbox && !is_valid_cephx_key(lockbox_secret)) { + ss << "invalid cephx lockbox secret."; + return -EINVAL; + } + + int err = _create_auth(cephx_entity.auth, cephx_secret, cephx_caps); + assert(0 == err); + + err = exists_and_matches_entity(cephx_entity, true, ss); + + if (err != -ENOENT) { + if (err < 0) { + return err; + } + assert(0 == err); + } + + if (has_lockbox) { + err = _create_auth(lockbox_entity.auth, lockbox_secret, lockbox_caps); + assert(err == 0); + err = exists_and_matches_entity(lockbox_entity, true, ss); + if (err != -ENOENT) { + if (err < 0) { + return err; + } + assert(0 == err); + } + } + + return 0; +} + +int AuthMonitor::do_osd_new( + const auth_entity_t& cephx_entity, + const auth_entity_t& lockbox_entity, + bool has_lockbox) +{ + assert(paxos->is_plugged()); + + dout(10) << __func__ << " cephx " << cephx_entity.name + << " lockbox "; + if (has_lockbox) { + *_dout << lockbox_entity.name; + } else { + *_dout << "n/a"; + } + *_dout << dendl; + + // we must have validated before reaching this point. + // if keys exist, then this means they also match; otherwise we would + // have failed before calling this function. + bool cephx_exists = mon->key_server.contains(cephx_entity.name); + + if (!cephx_exists) { + int err = add_entity(cephx_entity.name, cephx_entity.auth); + assert(0 == err); + } + + if (has_lockbox && + !mon->key_server.contains(lockbox_entity.name)) { + int err = add_entity(lockbox_entity.name, lockbox_entity.auth); + assert(0 == err); + } + + // given we have paxos plugged, this will not result in a proposal + // being triggered, but it will still be needed so that we get our + // pending state encoded into the paxos' pending transaction. + propose_pending(); + return 0; +} + bool AuthMonitor::prepare_command(MonOpRequestRef op) { MMonCommand *m = static_cast(op->get_req()); @@ -1135,7 +1301,6 @@ bool AuthMonitor::prepare_command(MonOpRequestRef op) get_last_committed() + 1)); return true; } - done: rdata.append(ds); getline(ss, rs, '\0'); diff --git a/src/mon/AuthMonitor.h b/src/mon/AuthMonitor.h index 33badd9a9db..18b847d6456 100644 --- a/src/mon/AuthMonitor.h +++ b/src/mon/AuthMonitor.h @@ -103,6 +103,12 @@ public: } }; + struct auth_entity_t { + EntityName name; + EntityAuth auth; + }; + + private: vector pending_auth; version_t last_rotating_ver; @@ -161,9 +167,13 @@ private: bool entity_is_pending(EntityName& entity); int exists_and_matches_entity( - EntityName& name, - EntityAuth& auth, - map& caps, + const auth_entity_t& entity, + bool has_secret, + stringstream& ss); + int exists_and_matches_entity( + const EntityName& name, + const EntityAuth& auth, + const map& caps, bool has_secret, stringstream& ss); int remove_entity(const EntityName &entity); @@ -193,7 +203,32 @@ private: const EntityName& cephx_entity, const EntityName& lockbox_entity); + int do_osd_new( + const auth_entity_t& cephx_entity, + const auth_entity_t& lockbox_entity, + bool has_lockbox); + int validate_osd_new( + int32_t id, + const uuid_d& uuid, + const string& cephx_secret, + const string& lockbox_secret, + auth_entity_t& cephx_entity, + auth_entity_t& lockbox_entity, + stringstream& ss); + void dump_info(Formatter *f); + + bool is_valid_cephx_key(const string& k) { + if (k.empty()) + return false; + + EntityAuth ea; + try { + ea.key.decode_base64(k); + return true; + } catch (buffer::error& e) { /* fallthrough */ } + return false; + } }; diff --git a/src/mon/ConfigKeyService.cc b/src/mon/ConfigKeyService.cc index 7b79943a656..1c128f12f49 100644 --- a/src/mon/ConfigKeyService.cc +++ b/src/mon/ConfigKeyService.cc @@ -19,6 +19,7 @@ #include "mon/Monitor.h" #include "mon/ConfigKeyService.h" #include "mon/MonitorDBStore.h" +#include "mon/OSDMonitor.h" #include "common/errno.h" #include "include/stringify.h" @@ -246,10 +247,14 @@ out: return (ret == 0); } +string _get_dmcrypt_prefix(const uuid_d& uuid, const string k) +{ + return "dm-crypt/osd/" + stringify(uuid) + "/" + k; +} + void ConfigKeyService::do_osd_destroy(int32_t id, uuid_d& uuid) { - string dmcrypt_prefix = - "dm-crypt/osd/" + stringify(uuid) + "/"; + string dmcrypt_prefix = _get_dmcrypt_prefix(uuid, ""); string daemon_prefix = "daemon-private/osd." + stringify(id) + "/"; @@ -260,3 +265,43 @@ void ConfigKeyService::do_osd_destroy(int32_t id, uuid_d& uuid) paxos->trigger_propose(); } + +int ConfigKeyService::validate_osd_new( + const uuid_d& uuid, + const string& dmcrypt_key, + stringstream& ss) +{ + string dmcrypt_prefix = _get_dmcrypt_prefix(uuid, "luks"); + bufferlist value; + value.append(dmcrypt_key); + + if (store_exists(dmcrypt_prefix)) { + bufferlist existing_value; + int err = store_get(dmcrypt_prefix, existing_value); + if (err < 0) { + dout(10) << __func__ << " unable to get dm-crypt key from store (r = " + << err << ")" << dendl; + return err; + } + if (existing_value.contents_equal(value)) { + // both values match; this will be an idempotent op. + return 0; + } + ss << "dm-crypt key already exists and does not match"; + return -EEXIST; + } + return 0; +} + +void ConfigKeyService::do_osd_new( + const uuid_d& uuid, + const string& dmcrypt_key) +{ + assert(paxos->is_plugged()); + + string dmcrypt_key_prefix = _get_dmcrypt_prefix(uuid, "luks"); + bufferlist dmcrypt_key_value; + dmcrypt_key_value.append(dmcrypt_key); + // store_put() will call trigger_propose + store_put(dmcrypt_key_prefix, dmcrypt_key_value, nullptr); +} diff --git a/src/mon/ConfigKeyService.h b/src/mon/ConfigKeyService.h index 631264e8a67..6866e604e39 100644 --- a/src/mon/ConfigKeyService.h +++ b/src/mon/ConfigKeyService.h @@ -67,6 +67,11 @@ public: void service_tick() override { } void do_osd_destroy(int32_t id, uuid_d& uuid); + int validate_osd_new( + const uuid_d& uuid, + const string& dmcrypt_key, + stringstream& ss); + void do_osd_new(const uuid_d& uuid, const string& dmcrypt_key); int get_type() override { return QuorumService::SERVICE_CONFIG_KEY; diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index 4494d44d6b1..0b4fa4e4e58 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -714,6 +714,13 @@ COMMAND("osd create " \ "name=uuid,type=CephUUID,req=false " \ "name=id,type=CephOsdName,req=false", \ "create new osd (with optional UUID and ID)", "osd", "rw", "cli,rest") +COMMAND("osd new " \ + "name=uuid,type=CephUUID,req=true " \ + "name=id,type=CephOsdName,req=false", \ + "Create a new OSD. If supplied, the `id` to be replaced needs to " \ + "exist and have been previously destroyed. " \ + "Reads secrets from JSON file via `-i ` (see man page).", \ + "osd", "rw", "cli,rest") COMMAND("osd blacklist " \ "name=blacklistop,type=CephChoices,strings=add|rm " \ "name=addr,type=CephEntityAddr " \ diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index baca99b4653..e38ca254b86 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -6531,7 +6531,104 @@ int OSDMonitor::prepare_command_osd_create( // osd is about to exist return -EAGAIN; } + return 0; +} + +int OSDMonitor::prepare_command_osd_new( + MonOpRequestRef op, + const map& cmdmap, + const map& secrets, + stringstream &ss) +{ + uuid_d uuid; + string uuidstr; + int64_t id = -1; + + assert(paxos->is_plugged()); + + dout(10) << __func__ << " " << op << dendl; + + /* validate command. abort now if something's wrong. */ + if (!cmd_getval(g_ceph_context, cmdmap, "uuid", uuidstr)) { + ss << "requires the OSD's UUID to be specified."; + return -EINVAL; + } + + if (!uuid.parse(uuidstr.c_str())) { + ss << "invalid UUID value '" << uuidstr << "'."; + return -EINVAL; + } else if (osdmap.identify_osd(uuid) >= 0) { + ss << "UUID already exists; please provide a unique UUID."; + return -EEXIST; + } + + if (!cmd_getval(g_ceph_context, cmdmap, "replace_id", id)) { + ss << "requires an existing OSD id."; + return -EINVAL; + } else if (id < 0) { + ss << "requires an existing OSD id, greater or equal than 0."; + return -ENOENT; + } else if (!osdmap.exists(id)) { + ss << "osd." << id << " does not exist."; + return -ENOENT; + } else if (!osdmap.is_destroyed(id)) { + ss << "osd." << id << " has not been destroyed."; + return -EBUSY; + } + + string cephx_secret, lockbox_secret, dmcrypt_key; + bool has_lockbox = false; + + if (secrets.count("cephx_secret") == 0) { + ss << "requires a cephx secret."; + return -EINVAL; + } + cephx_secret = secrets.at("cephx_secret"); + + if (secrets.count("cephx_lockbox_secret") > 0) { + if (secrets.count("dmcrypt_key") == 0) { + ss << "requires both a cephx lockbox secret and a dm-crypt key."; + return -EINVAL; + } + has_lockbox = true; + lockbox_secret = secrets.at("cephx_lockbox_secret"); + dmcrypt_key = secrets.at("dmcrypt_key"); + } + + AuthMonitor::auth_entity_t cephx_entity, lockbox_entity; + int err = mon->authmon()->validate_osd_new(id, uuid, + cephx_secret, + lockbox_secret, + cephx_entity, + lockbox_entity, + ss); + if (err < 0) { + return err; + } + + ConfigKeyService* svc = (ConfigKeyService*)mon->config_key_service; + if (has_lockbox) { + err = svc->validate_osd_new(uuid, dmcrypt_key, ss); + if (err < 0) { + return err; + } + } + // perform updates. + err = mon->authmon()->do_osd_new(cephx_entity, + lockbox_entity, + has_lockbox); + assert(0 == err); + + if (has_lockbox) { + svc->do_osd_new(uuid, dmcrypt_key); + } + + pending_inc.new_weight[id] = CEPH_OSD_OUT; + pending_inc.new_state[id] |= CEPH_OSD_DESTROYED | CEPH_OSD_NEW; + pending_inc.new_uuid[id] = uuid; + + ss << "new osd." << id << " uuid " << uuid; return 0; } @@ -8580,6 +8677,48 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op, force_immediate_propose(); return true; + } else if (prefix == "osd new") { + + // make sure authmon is writeable. + if (!mon->authmon()->is_writeable()) { + dout(10) << __func__ << " waiting for auth mon to be writeable for " + << "osd destroy" << dendl; + mon->authmon()->wait_for_writeable(op, new C_RetryMessage(this, op)); + return false; + } + + map secrets_map; + + bufferlist bl = m->get_data(); + string secrets_json = bl.to_str(); + dout(20) << __func__ << " osd new json = " << secrets_json << dendl; + + err = get_json_str_map(secrets_json, ss, &secrets_map); + if (err < 0) + goto reply; + + if (secrets_map.empty()) { + ss << "'osd new' requires a populated json file as input."; + err = -EINVAL; + goto reply; + } + + dout(20) << __func__ << " osd new secrets " << secrets_map << dendl; + + paxos->plug(); + err = prepare_command_osd_new(op, cmdmap, secrets_map, ss); + paxos->unplug(); + + if (err < 0) { + goto reply; + } + + getline(ss, rs); + wait_for_finished_proposal(op, + new Monitor::C_Command(mon, op, 0, rs, get_last_committed() + 1)); + force_immediate_propose(); + return true; + } else if (prefix == "osd create") { // optional id provided? diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h index dffe54d70e3..5da26c31001 100644 --- a/src/mon/OSDMonitor.h +++ b/src/mon/OSDMonitor.h @@ -498,6 +498,12 @@ public: bool has_ancestor, bool unlink_only); int prepare_command_osd_remove(int32_t id); + int prepare_command_osd_new( + MonOpRequestRef op, + const map& cmdmap, + const map& secrets, + stringstream &ss); + int prepare_command_pool_set(map &cmdmap, stringstream& ss);