From: Jason Dillaman Date: Wed, 2 Oct 2019 19:14:23 +0000 (-0400) Subject: rbd: include new mirror peer direction and image site status state X-Git-Tag: v15.1.0~1245^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=856fe3d4c0c6d5cc9255acc206ed94a97dfa4eea;p=ceph.git rbd: include new mirror peer direction and image site status state Signed-off-by: Jason Dillaman --- diff --git a/src/test/cli/rbd/help.t b/src/test/cli/rbd/help.t index 803a61d782a3..a26869a1f612 100644 --- a/src/test/cli/rbd/help.t +++ b/src/test/cli/rbd/help.t @@ -1674,6 +1674,7 @@ [--remote-cluster ] [--remote-mon-host ] [--remote-key-file ] + [--direction ] Add a mirroring peer to a pool. @@ -1689,6 +1690,8 @@ --remote-cluster arg remote cluster name --remote-mon-host arg remote mon host(s) --remote-key-file arg path to file containing remote key + --direction arg mirroring direction (rx-only, rx-tx) + [default: rx-tx] rbd help mirror pool peer bootstrap create usage: rbd mirror pool peer bootstrap create @@ -1748,8 +1751,10 @@ Positional arguments pool name peer uuid - peer parameter [client, cluster, mon-host, key-file] + peer parameter + (direction, site-name, client, mon-host, key-file) new value for specified key + (rx-only, tx-only, or rx-tx for direction) Optional arguments -p [ --pool ] arg pool name diff --git a/src/tools/rbd/Utils.cc b/src/tools/rbd/Utils.cc index e8cd5f293c3c..d78534151575 100644 --- a/src/tools/rbd/Utils.cc +++ b/src/tools/rbd/Utils.cc @@ -906,6 +906,10 @@ int get_local_mirror_image_status( } std::string timestr(time_t t) { + if (t == 0) { + return ""; + } + struct tm tm; localtime_r(&t, &tm); @@ -932,5 +936,73 @@ bool is_not_user_snap_namespace(librbd::Image* image, return namespace_type != RBD_SNAP_NAMESPACE_TYPE_USER; } +void get_mirror_peer_sites( + librados::IoCtx& io_ctx, + std::vector* mirror_peers) { + librados::IoCtx default_io_ctx; + default_io_ctx.dup(io_ctx); + default_io_ctx.set_namespace(""); + + mirror_peers->clear(); + + librbd::RBD rbd; + int r = rbd.mirror_peer_site_list(default_io_ctx, mirror_peers); + if (r < 0 && r != -ENOENT) { + std::cerr << "rbd: failed to list mirror peers" << std::endl; + } +} + +void get_mirror_peer_fsid_to_names( + const std::vector& mirror_peers, + std::map* fsid_to_name) { + fsid_to_name->clear(); + for (auto& peer : mirror_peers) { + if (!peer.fsid.empty() && !peer.site_name.empty()) { + (*fsid_to_name)[peer.fsid] = peer.site_name; + } + } +} + +void populate_unknown_mirror_image_site_statuses( + const std::vector& mirror_peers, + librbd::mirror_image_global_status_t* global_status) { + std::set missing_fsids; + librbd::mirror_peer_direction_t mirror_peer_direction = + RBD_MIRROR_PEER_DIRECTION_RX_TX; + for (auto& peer : mirror_peers) { + if (peer.uuid == mirror_peers.begin()->uuid) { + mirror_peer_direction = peer.direction; + } else if (mirror_peer_direction != RBD_MIRROR_PEER_DIRECTION_RX_TX && + mirror_peer_direction != peer.direction) { + mirror_peer_direction = RBD_MIRROR_PEER_DIRECTION_RX_TX; + } + + if (!peer.fsid.empty() && peer.direction != RBD_MIRROR_PEER_DIRECTION_TX) { + missing_fsids.insert(peer.fsid); + } + } + + if (mirror_peer_direction != RBD_MIRROR_PEER_DIRECTION_TX) { + missing_fsids.insert(RBD_MIRROR_IMAGE_STATUS_LOCAL_FSID); + } + + std::vector site_statuses; + site_statuses.reserve(missing_fsids.size()); + + for (auto& site_status : global_status->site_statuses) { + if (missing_fsids.count(site_status.fsid) > 0) { + missing_fsids.erase(site_status.fsid); + site_statuses.push_back(site_status); + } + } + + for (auto& fsid : missing_fsids) { + site_statuses.push_back({fsid, MIRROR_IMAGE_STATUS_STATE_UNKNOWN, + "status not found", 0, false}); + } + + std::swap(global_status->site_statuses, site_statuses); +} + } // namespace utils } // namespace rbd diff --git a/src/tools/rbd/Utils.h b/src/tools/rbd/Utils.h index c0bf0be0b70f..c9812ce61d69 100644 --- a/src/tools/rbd/Utils.h +++ b/src/tools/rbd/Utils.h @@ -207,6 +207,16 @@ std::string timestr(time_t t); // duplicate here to not include librbd_internal lib uint64_t get_rbd_default_features(CephContext* cct); +void get_mirror_peer_sites( + librados::IoCtx& io_ctx, + std::vector* mirror_peers); +void get_mirror_peer_fsid_to_names( + const std::vector& mirror_peers, + std::map* fsid_to_name); +void populate_unknown_mirror_image_site_statuses( + const std::vector& mirror_peers, + librbd::mirror_image_global_status_t* global_status); + } // namespace utils } // namespace rbd diff --git a/src/tools/rbd/action/MirrorImage.cc b/src/tools/rbd/action/MirrorImage.cc index c9c4cf73b143..ff079a2adbab 100644 --- a/src/tools/rbd/action/MirrorImage.cc +++ b/src/tools/rbd/action/MirrorImage.cc @@ -275,6 +275,12 @@ int execute_status(const po::variables_map &vm, return r; } + std::vector mirror_peers; + utils::get_mirror_peer_sites(io_ctx, &mirror_peers); + + std::map peer_fsid_to_name; + utils::get_mirror_peer_fsid_to_names(mirror_peers, &peer_fsid_to_name); + librbd::mirror_image_global_status_t status; r = image.mirror_image_get_global_status(&status, sizeof(status)); if (r < 0) { @@ -283,13 +289,23 @@ int execute_status(const po::variables_map &vm, return r; } + utils::populate_unknown_mirror_image_site_statuses(mirror_peers, &status); + std::string instance_id; MirrorDaemonServiceInfo daemon_service_info(io_ctx); librbd::mirror_image_site_status_t local_status; - utils::get_local_mirror_image_status(status, &local_status); - - if (local_status.up) { + int local_site_r = utils::get_local_mirror_image_status( + status, &local_status); + status.site_statuses.erase( + std::remove_if(status.site_statuses.begin(), + status.site_statuses.end(), + [](auto& status) { + return (status.fsid == RBD_MIRROR_IMAGE_STATUS_LOCAL_FSID); + }), + status.site_statuses.end()); + + if (local_site_r >= 0 && local_status.up) { r = image.mirror_image_get_instance_id(&instance_id); if (r == -EOPNOTSUPP) { std::cerr << "rbd: newer release of Ceph OSDs required to map image " @@ -304,31 +320,75 @@ int execute_status(const po::variables_map &vm, } } - std::string state = utils::mirror_image_site_status_state(local_status); - std::string last_update = ( - local_status.last_update == 0 ? - "" : utils::timestr(local_status.last_update)); - if (formatter != nullptr) { formatter->open_object_section("image"); formatter->dump_string("name", image_name); formatter->dump_string("global_id", status.info.global_id); - formatter->dump_string("state", state); - formatter->dump_string("description", local_status.description); - daemon_service_info.dump(instance_id, formatter); - formatter->dump_string("last_update", last_update); + if (local_site_r >= 0) { + formatter->dump_string("state", utils::mirror_image_site_status_state( + local_status)); + formatter->dump_string("description", local_status.description); + daemon_service_info.dump(instance_id, formatter); + formatter->dump_string("last_update", utils::timestr( + local_status.last_update)); + } + if (!status.site_statuses.empty()) { + formatter->open_array_section("peer_sites"); + for (auto& status : status.site_statuses) { + formatter->open_object_section("peer_site"); + + auto name_it = peer_fsid_to_name.find(status.fsid); + formatter->dump_string("site_name", + (name_it != peer_fsid_to_name.end() ? name_it->second : "")); + formatter->dump_string("fsid", status.fsid); + + formatter->dump_string("state", utils::mirror_image_site_status_state( + status)); + formatter->dump_string("description", status.description); + formatter->dump_string("last_update", utils::timestr( + status.last_update)); + formatter->close_section(); // peer_site + } + formatter->close_section(); // peer_sites + } formatter->close_section(); // image formatter->flush(std::cout); } else { std::cout << image_name << ":\n" - << " global_id: " << status.info.global_id << "\n" - << " state: " << state << "\n" - << " description: " << local_status.description << "\n"; - if (!instance_id.empty()) { - std::cout << " service: " << - daemon_service_info.get_description(instance_id) << "\n"; + << " global_id: " << status.info.global_id << "\n"; + if (local_site_r >= 0) { + std::cout << " state: " << utils::mirror_image_site_status_state( + local_status) << "\n" + << " description: " << local_status.description << "\n"; + if (!instance_id.empty()) { + std::cout << " service: " << + daemon_service_info.get_description(instance_id) << "\n"; + } + std::cout << " last_update: " << utils::timestr( + local_status.last_update) << std::endl; + } + if (!status.site_statuses.empty()) { + std::cout << " peer_sites:" << std::endl; + + bool first_site = true; + for (auto& site : status.site_statuses) { + if (!first_site) { + std::cout << std::endl; + } + first_site = false; + + auto name_it = peer_fsid_to_name.find(site.fsid); + std::cout << " name: " + << (name_it != peer_fsid_to_name.end() ? name_it->second : + site.fsid) + << std::endl + << " state: " << utils::mirror_image_site_status_state( + site) << std::endl + << " description: " << site.description << std::endl + << " last_update: " << utils::timestr( + site.last_update) << std::endl; + } } - std::cout << " last_update: " << last_update << std::endl; } return 0; diff --git a/src/tools/rbd/action/MirrorPool.cc b/src/tools/rbd/action/MirrorPool.cc index 1c85f8d960d5..457f528122a3 100644 --- a/src/tools/rbd/action/MirrorPool.cc +++ b/src/tools/rbd/action/MirrorPool.cc @@ -66,7 +66,7 @@ int set_site_name(librados::Rados& rados, const std::string& site_name) { struct MirrorPeerDirection {}; void validate(boost::any& v, const std::vector& values, - MirrorPeerDirection *target_type, int) { + MirrorPeerDirection *target_type, int permit_tx) { po::validators::check_first_occurrence(v); const std::string &s = po::validators::get_single_string(values); @@ -74,11 +74,20 @@ void validate(boost::any& v, const std::vector& values, v = boost::any(RBD_MIRROR_PEER_DIRECTION_RX); } else if (s == "rx-tx") { v = boost::any(RBD_MIRROR_PEER_DIRECTION_RX_TX); + } else if (permit_tx != 0 && s == "tx-only") { + v = boost::any(RBD_MIRROR_PEER_DIRECTION_TX); } else { throw po::validation_error(po::validation_error::invalid_option_value); } } +void add_direction_optional(po::options_description *options) { + options->add_options() + ("direction", po::value(), + "mirroring direction (rx-only, rx-tx)\n" + "[default: rx-tx]"); +} + int validate_mirroring_enabled(librados::IoCtx& io_ctx) { librbd::RBD rbd; rbd_mirror_mode_t mirror_mode; @@ -245,23 +254,14 @@ int format_mirror_peers(librados::IoCtx& io_ctx, at::Format::Formatter formatter, const std::vector &peers, bool config_key) { - TextTable tbl; if (formatter != nullptr) { formatter->open_array_section("peers"); } else { - std::cout << "Peers: "; + std::cout << "Peer Sites: "; if (peers.empty()) { - std::cout << "none" << std::endl; - } else { - tbl.define_column("", TextTable::LEFT, TextTable::LEFT); - tbl.define_column("UUID", TextTable::LEFT, TextTable::LEFT); - tbl.define_column("NAME", TextTable::LEFT, TextTable::LEFT); - tbl.define_column("CLIENT", TextTable::LEFT, TextTable::LEFT); - if (config_key) { - tbl.define_column("MON_HOST", TextTable::LEFT, TextTable::LEFT); - tbl.define_column("KEY", TextTable::LEFT, TextTable::LEFT); - } + std::cout << "none"; } + std::cout << std::endl; } for (auto &peer : peers) { @@ -273,32 +273,58 @@ int format_mirror_peers(librados::IoCtx& io_ctx, } } + std::string direction; + switch (peer.direction) { + case RBD_MIRROR_PEER_DIRECTION_RX: + direction = "rx-only"; + break; + case RBD_MIRROR_PEER_DIRECTION_TX: + direction = "tx-only"; + break; + case RBD_MIRROR_PEER_DIRECTION_RX_TX: + direction = "rx-tx"; + break; + default: + direction = "unknown"; + break; + } + if (formatter != nullptr) { formatter->open_object_section("peer"); formatter->dump_string("uuid", peer.uuid); - formatter->dump_string("cluster_name", peer.site_name); + formatter->dump_string("direction", direction); + formatter->dump_string("site_name", peer.site_name); + formatter->dump_string("fsid", peer.fsid); formatter->dump_string("client_name", peer.client_name); for (auto& pair : attributes) { formatter->dump_string(pair.first.c_str(), pair.second); } formatter->close_section(); } else { - tbl << " " - << peer.uuid - << peer.site_name - << peer.client_name; + std::cout << std::endl + << "UUID: " << peer.uuid << std::endl + << "Name: " << peer.site_name << std::endl; + if (peer.direction != RBD_MIRROR_PEER_DIRECTION_RX || + !peer.fsid.empty()) { + std::cout << "FSID: " << peer.fsid << std::endl; + } + std::cout << "Direction: " << direction << std::endl; + if (peer.direction != RBD_MIRROR_PEER_DIRECTION_TX || + !peer.client_name.empty()) { + std::cout << "Client: " << peer.client_name << std::endl; + } if (config_key) { - tbl << attributes["mon_host"] - << attributes["key"]; + std::cout << "Mon Host: " << attributes["mon_host"] << std::endl + << "Key: " << attributes["key"] << std::endl; + } + if (peer.site_name != peers.rbegin()->site_name) { + std::cout << std::endl; } - tbl << TextTable::endrow; } } if (formatter != nullptr) { formatter->close_section(); - } else { - std::cout << std::endl << tbl; } return 0; } @@ -565,11 +591,14 @@ public: librados::IoCtx &io_ctx, OrderedThrottle &throttle, const std::string &image_name, const std::map &instance_ids, + const std::vector& mirror_peers, + const std::map &peer_fsid_to_name, const MirrorDaemonServiceInfo &daemon_service_info, at::Format::Formatter formatter) : ImageRequestBase(io_ctx, throttle, image_name), - m_instance_ids(instance_ids), m_daemon_service_info(daemon_service_info), - m_formatter(formatter) { + m_instance_ids(instance_ids), m_mirror_peers(mirror_peers), + m_peer_fsid_to_name(peer_fsid_to_name), + m_daemon_service_info(daemon_service_info), m_formatter(formatter) { } protected: @@ -590,39 +619,94 @@ protected: return; } + utils::populate_unknown_mirror_image_site_statuses( + m_mirror_peers, &m_mirror_image_global_status); + librbd::mirror_image_site_status_t local_status; - utils::get_local_mirror_image_status( + int local_site_r = utils::get_local_mirror_image_status( m_mirror_image_global_status, &local_status); - - std::string state = utils::mirror_image_site_status_state(local_status); - std::string instance_id = (local_status.up && + m_mirror_image_global_status.site_statuses.erase( + std::remove_if(m_mirror_image_global_status.site_statuses.begin(), + m_mirror_image_global_status.site_statuses.end(), + [](auto& status) { + return (status.fsid == RBD_MIRROR_IMAGE_STATUS_LOCAL_FSID); + }), + m_mirror_image_global_status.site_statuses.end()); + + std::string instance_id = (local_site_r >= 0 && local_status.up && m_instance_ids.count(m_image_id)) ? m_instance_ids.find(m_image_id)->second : ""; - std::string last_update = ( - local_status.last_update == 0 ? - "" : utils::timestr(local_status.last_update)); if (m_formatter != nullptr) { m_formatter->open_object_section("image"); m_formatter->dump_string("name", m_mirror_image_global_status.name); m_formatter->dump_string( "global_id", m_mirror_image_global_status.info.global_id); - m_formatter->dump_string("state", state); - m_formatter->dump_string("description", local_status.description); - m_daemon_service_info.dump(instance_id, m_formatter); - m_formatter->dump_string("last_update", last_update); + if (local_site_r >= 0) { + m_formatter->dump_string("state", utils::mirror_image_site_status_state( + local_status)); + m_formatter->dump_string("description", local_status.description); + m_daemon_service_info.dump(instance_id, m_formatter); + m_formatter->dump_string("last_update", utils::timestr( + local_status.last_update)); + } + if (!m_mirror_image_global_status.site_statuses.empty()) { + m_formatter->open_array_section("peer_sites"); + for (auto& status : m_mirror_image_global_status.site_statuses) { + m_formatter->open_object_section("peer_site"); + + auto name_it = m_peer_fsid_to_name.find(status.fsid); + m_formatter->dump_string("site_name", + (name_it != m_peer_fsid_to_name.end() ? name_it->second : "")); + m_formatter->dump_string("fsid", status.fsid); + + m_formatter->dump_string( + "state", utils::mirror_image_site_status_state(status)); + m_formatter->dump_string("description", status.description); + m_formatter->dump_string("last_update", utils::timestr( + status.last_update)); + m_formatter->close_section(); // peer_site + } + m_formatter->close_section(); // peer_sites + } m_formatter->close_section(); // image } else { - std::cout << "\n" << m_mirror_image_global_status.name << ":\n" - << " global_id: " - << m_mirror_image_global_status.info.global_id << "\n" - << " state: " << state << "\n" - << " description: " << local_status.description << "\n"; - if (!instance_id.empty()) { - std::cout << " service: " - << m_daemon_service_info.get_description(instance_id) << "\n"; + std::cout << std::endl + << m_mirror_image_global_status.name << ":" << std::endl + << " global_id: " + << m_mirror_image_global_status.info.global_id << std::endl; + if (local_site_r >= 0) { + std::cout << " state: " << utils::mirror_image_site_status_state( + local_status) << std::endl + << " description: " << local_status.description << std::endl; + if (!instance_id.empty()) { + std::cout << " service: " << + m_daemon_service_info.get_description(instance_id) << std::endl; + } + std::cout << " last_update: " << utils::timestr( + local_status.last_update) << std::endl; + } + if (!m_mirror_image_global_status.site_statuses.empty()) { + std::cout << " peer_sites:" << std::endl; + bool first_site = true; + for (auto& site : m_mirror_image_global_status.site_statuses) { + if (!first_site) { + std::cout << std::endl; + } + first_site = false; + + auto name_it = m_peer_fsid_to_name.find(site.fsid); + std::cout << " name: " + << (name_it != m_peer_fsid_to_name.end() ? name_it->second : + site.fsid) + << std::endl + << " state: " << utils::mirror_image_site_status_state( + site) << std::endl + << " description: " << site.description << std::endl + << " last_update: " << utils::timestr( + site.last_update) << std::endl; + } } - std::cout << " last_update: " << last_update << std::endl; } } @@ -632,6 +716,8 @@ protected: private: const std::map &m_instance_ids; + const std::vector &m_mirror_peers; + const std::map &m_peer_fsid_to_name; const MirrorDaemonServiceInfo &m_daemon_service_info; at::Format::Formatter m_formatter; std::string m_image_id; @@ -756,10 +842,8 @@ void get_peer_bootstrap_import_arguments(po::options_description *positional, "bootstrap token file (or '-' for stdin)"); options->add_options() ("token-path", po::value(), - "bootstrap token file (or '-' for stdin)") - ("direction", po::value(), - "mirroring direction (rx-only, rx-tx)\n" - "[default: rx-tx]"); + "bootstrap token file (or '-' for stdin)"); + add_direction_optional(options); } int execute_peer_bootstrap_import( @@ -853,6 +937,7 @@ void get_peer_add_arguments(po::options_description *positional, ("remote-mon-host", po::value(), "remote mon host(s)") ("remote-key-file", po::value(), "path to file containing remote key"); + add_direction_optional(options); } int execute_peer_add(const po::variables_map &vm, @@ -901,10 +986,15 @@ int execute_peer_add(const po::variables_map &vm, return -EINVAL; } + rbd_mirror_peer_direction_t mirror_peer_direction = + RBD_MIRROR_PEER_DIRECTION_RX_TX; + if (vm.count("direction")) { + mirror_peer_direction = vm["direction"].as(); + } + std::string uuid; r = rbd.mirror_peer_site_add( - io_ctx, &uuid, RBD_MIRROR_PEER_DIRECTION_RX_TX, remote_cluster, - remote_client_name); + io_ctx, &uuid, mirror_peer_direction, remote_cluster, remote_client_name); if (r < 0) { std::cerr << "rbd: error adding mirror peer" << std::endl; return r; @@ -969,8 +1059,10 @@ void get_peer_set_arguments(po::options_description *positional, at::add_pool_options(positional, options, false); add_uuid_option(positional); positional->add_options() - ("key", "peer parameter [client, cluster, mon-host, key-file]") - ("value", "new value for specified key"); + ("key", "peer parameter\n" + "(direction, site-name, client, mon-host, key-file)") + ("value", "new value for specified key\n" + "(rx-only, tx-only, or rx-tx for direction)"); } int execute_peer_set(const po::variables_map &vm, @@ -989,8 +1081,8 @@ int execute_peer_set(const po::variables_map &vm, return r; } - std::set valid_keys{{"client", "cluster", "mon-host", - "key-file"}}; + std::set valid_keys{{"direction", "site-name", "cluster", + "client", "mon-host", "key-file"}}; std::string key = utils::get_positional_argument(vm, arg_index++); if (valid_keys.find(key) == valid_keys.end()) { std::cerr << "rbd: must specify "; @@ -1033,14 +1125,27 @@ int execute_peer_set(const po::variables_map &vm, if (key == "client") { r = rbd.mirror_peer_site_set_client_name(io_ctx, uuid.c_str(), value.c_str()); - } else if (key == "cluster") { + } else if (key == "site-name" || key == "cluster") { r = rbd.mirror_peer_site_set_name(io_ctx, uuid.c_str(), value.c_str()); + } else if (key == "direction") { + MirrorPeerDirection tag; + boost::any direction; + try { + validate(direction, {value}, &tag, 1); + } catch (...) { + std::cerr << "rbd: invalid direction" << std::endl; + return -EINVAL; + } + + r = rbd.mirror_peer_site_set_direction( + io_ctx, uuid, boost::any_cast(direction)); } else { r = update_peer_config_key(io_ctx, uuid, key, value); - if (r == -ENOENT) { - std::cerr << "rbd: mirror peer " << uuid << " does not exist" - << std::endl; - } + } + + if (r == -ENOENT) { + std::cerr << "rbd: mirror peer " << uuid << " does not exist" + << std::endl; } if (r < 0) { @@ -1252,7 +1357,8 @@ int execute_info(const po::variables_map &vm, if (formatter != nullptr) { formatter->dump_string("site_name", site_name); } else { - std::cout << "Site Name: " << site_name << std::endl; + std::cout << "Site Name: " << site_name << std::endl + << std::endl; } r = format_mirror_peers(io_ctx, formatter, mirror_peers, @@ -1360,6 +1466,12 @@ int execute_status(const po::variables_map &vm, int ret = 0; if (verbose) { + std::vector mirror_peers; + utils::get_mirror_peer_sites(io_ctx, &mirror_peers); + + std::map peer_fsid_to_name; + utils::get_mirror_peer_fsid_to_names(mirror_peers, &peer_fsid_to_name); + if (formatter != nullptr) { formatter->open_array_section("images"); } @@ -1394,7 +1506,8 @@ int execute_status(const po::variables_map &vm, } ImageRequestGenerator generator( - io_ctx, instance_ids, daemon_service_info, formatter); + io_ctx, instance_ids, mirror_peers, peer_fsid_to_name, + daemon_service_info, formatter); ret = generator.execute(); if (formatter != nullptr) {