From: Sage Weil Date: Mon, 4 Jun 2018 19:23:35 +0000 (-0500) Subject: osd/OSDMap: make cluster addrs addrvecs too X-Git-Tag: v14.0.1~951^2~29 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=9fb1e521c7c75c124b0dbf193e8b65ff1b5f461e;p=ceph.git osd/OSDMap: make cluster addrs addrvecs too This will allow a single-step upgrade from msgr1 to msgr2: as each nautilus OSD starts up, it will bind to both msgr1 and msgr2 addrs (assuming it starts up after the mons have been upgraded). Peer OSDs will use the msgr2 address if they can, or the legacy one if they can't. Once require_osd_release >= nautilus OSDs will stop binding to msgr1 ports at all. Signed-off-by: Sage Weil --- diff --git a/src/messages/MOSDBoot.h b/src/messages/MOSDBoot.h index 2f06412859e16..beb6a587f2034 100644 --- a/src/messages/MOSDBoot.h +++ b/src/messages/MOSDBoot.h @@ -27,8 +27,8 @@ class MOSDBoot : public PaxosServiceMessage { public: OSDSuperblock sb; - entity_addr_t hb_back_addr, hb_front_addr; - entity_addr_t cluster_addr; + entity_addrvec_t hb_back_addrs, hb_front_addrs; + entity_addrvec_t cluster_addrs; epoch_t boot_epoch; // last epoch this daemon was added to the map (if any) map metadata; ///< misc metadata about this osd uint64_t osd_features; @@ -38,15 +38,15 @@ class MOSDBoot : public PaxosServiceMessage { boot_epoch(0), osd_features(0) { } MOSDBoot(OSDSuperblock& s, epoch_t e, epoch_t be, - const entity_addr_t& hb_back_addr_ref, - const entity_addr_t& hb_front_addr_ref, - const entity_addr_t& cluster_addr_ref, + const entity_addrvec_t& hb_back_addr_ref, + const entity_addrvec_t& hb_front_addr_ref, + const entity_addrvec_t& cluster_addr_ref, uint64_t feat) : PaxosServiceMessage(MSG_OSD_BOOT, e, HEAD_VERSION, COMPAT_VERSION), sb(s), - hb_back_addr(hb_back_addr_ref), - hb_front_addr(hb_front_addr_ref), - cluster_addr(cluster_addr_ref), + hb_back_addrs(hb_back_addr_ref), + hb_front_addrs(hb_front_addr_ref), + cluster_addrs(cluster_addr_ref), boot_epoch(be), osd_features(feat) { } @@ -66,10 +66,10 @@ public: using ceph::encode; paxos_encode(); encode(sb, payload); - encode(hb_back_addr, payload, features); - encode(cluster_addr, payload, features); + encode(hb_back_addrs, payload, features); + encode(cluster_addrs, payload, features); encode(boot_epoch, payload); - encode(hb_front_addr, payload, features); + encode(hb_front_addrs, payload, features); encode(metadata, payload); encode(osd_features, payload); } @@ -77,10 +77,10 @@ public: auto p = payload.cbegin(); paxos_decode(p); decode(sb, p); - decode(hb_back_addr, p); - decode(cluster_addr, p); + decode(hb_back_addrs, p); + decode(cluster_addrs, p); decode(boot_epoch, p); - decode(hb_front_addr, p); + decode(hb_front_addrs, p); decode(metadata, p); decode(osd_features, p); } diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index 683c4d59fcea6..231283189a51f 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -2671,7 +2671,7 @@ bool OSDMonitor::preprocess_boot(MonOpRequestRef op) // already booted? if (osdmap.is_up(from) && osdmap.get_addrs(from) == m->get_orig_source_addrs() && - osdmap.get_cluster_addr(from) == m->cluster_addr) { + osdmap.get_cluster_addrs(from) == m->cluster_addrs) { // yup. dout(7) << "preprocess_boot dup from " << m->get_orig_source() << " " << m->get_orig_source_addrs() @@ -2716,10 +2716,12 @@ bool OSDMonitor::prepare_boot(MonOpRequestRef op) { op->mark_osdmon_event(__func__); MOSDBoot *m = static_cast(op->get_req()); - dout(7) << __func__ << " from " << m->get_orig_source_inst() << " sb " << m->sb - << " cluster_addr " << m->cluster_addr - << " hb_back_addr " << m->hb_back_addr - << " hb_front_addr " << m->hb_front_addr + dout(7) << __func__ << " from " << m->get_source() + << " sb " << m->sb + << " client_addrs" << m->get_connection()->get_peer_addrs() + << " cluster_addrs " << m->cluster_addrs + << " hb_back_addrs " << m->hb_back_addrs + << " hb_front_addrs " << m->hb_front_addrs << dendl; assert(m->get_orig_source().is_osd()); @@ -2742,7 +2744,7 @@ bool OSDMonitor::prepare_boot(MonOpRequestRef op) << osdmap.get_addrs(from) << dendl; // preprocess should have caught these; if not, assert. assert(osdmap.get_addrs(from) != m->get_orig_source_addrs() || - osdmap.get_cluster_addr(from) != m->cluster_addr); + osdmap.get_cluster_addrs(from) != m->cluster_addrs); assert(osdmap.get_uuid(from) == m->sb.osd_fsid); if (pending_inc.new_state.count(from) == 0 || @@ -2759,9 +2761,9 @@ bool OSDMonitor::prepare_boot(MonOpRequestRef op) } else { // mark new guy up. pending_inc.new_up_client[from] = m->get_orig_source_addrs(); - pending_inc.new_up_cluster[from] = m->cluster_addr; - pending_inc.new_hb_back_up[from] = m->hb_back_addr; - pending_inc.new_hb_front_up[from] = m->hb_front_addr; + pending_inc.new_up_cluster[from] = m->cluster_addrs; + pending_inc.new_hb_back_up[from] = m->hb_back_addrs; + pending_inc.new_hb_front_up[from] = m->hb_front_addrs; down_pending_out.erase(from); // if any diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index efa86a78c702e..cd816223bf758 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -1055,8 +1055,8 @@ bool OSDService::should_share_map(entity_name_t name, Connection *con, if (con->get_messenger() == osd->cluster_messenger && con != osd->cluster_messenger->get_loopback_connection() && osdmap->is_up(name.num()) && - (osdmap->get_cluster_addr(name.num()) == con->get_peer_addr() || - osdmap->get_hb_back_addr(name.num()) == con->get_peer_addr())) { + (osdmap->get_cluster_addrs(name.num()) == con->get_peer_addrs() || + osdmap->get_hb_back_addrs(name.num()) == con->get_peer_addrs())) { // remember epoch_t has = std::max(get_peer_epoch(name.num()), epoch); @@ -1102,9 +1102,9 @@ void OSDService::share_map( send_incremental_map(epoch, con, osdmap); } else if (con->get_messenger() == osd->cluster_messenger && osdmap->is_up(name.num()) && - (osdmap->get_cluster_addr(name.num()) == con->get_peer_addr() || - osdmap->get_hb_back_addr(name.num()) == con->get_peer_addr())) { - dout(10) << name << " " << con->get_peer_addr() + (osdmap->get_cluster_addrs(name.num()) == con->get_peer_addrs() || + osdmap->get_hb_back_addrs(name.num()) == con->get_peer_addrs())) { + dout(10) << name << " " << con->get_peer_addrs() << " has old map " << epoch << " < " << osdmap->get_epoch() << dendl; note_peer_epoch(name.num(), osdmap->get_epoch()); @@ -5338,55 +5338,89 @@ bool OSD::_is_healthy() void OSD::_send_boot() { dout(10) << "_send_boot" << dendl; - entity_addr_t cluster_addr = cluster_messenger->get_myaddr(); - Connection *local_connection = cluster_messenger->get_loopback_connection().get(); - if (cluster_addr.is_blank_ip()) { - int port = cluster_addr.get_port(); - cluster_addr = client_messenger->get_myaddr(); - cluster_addr.set_port(port); - cluster_messenger->set_addr_unknowns(cluster_addr); - dout(10) << " assuming cluster_addr ip matches client_addr" << dendl; - } else { - if (auto session = local_connection->get_priv(); !session) { - cluster_messenger->ms_deliver_handle_fast_connect(local_connection); + Connection *local_connection = + cluster_messenger->get_loopback_connection().get(); + entity_addrvec_t client_addrs = client_messenger->get_myaddrs(); + entity_addrvec_t cluster_addrs = cluster_messenger->get_myaddrs(); + entity_addrvec_t hb_back_addrs = hb_back_server_messenger->get_myaddrs(); + entity_addrvec_t hb_front_addrs = hb_front_server_messenger->get_myaddrs(); + + dout(20) << " initial client_addrs " << client_addrs + << ", cluster_addrs " << cluster_addrs + << ", hb_back_addrs " << hb_back_addrs + << ", hb_front_addrs " << hb_front_addrs + << dendl; + + for (auto& a : cluster_addrs.v) { + if (a.is_blank_ip()) { + int port = a.get_port(); + for (auto& b : client_addrs.v) { + if (a.get_type() == b.get_type() && + a.get_family() == b.get_family()) { + dout(10) << " assuming cluster_addr " << a + << " matches client_addr " << b << dendl; + a = b; + a.set_port(port); + cluster_messenger->set_addr_unknowns(a); + break; + } + } } } + if (auto session = local_connection->get_priv(); !session) { + cluster_messenger->ms_deliver_handle_fast_connect(local_connection); + } - entity_addr_t hb_back_addr = hb_back_server_messenger->get_myaddr(); local_connection = hb_back_server_messenger->get_loopback_connection().get(); - if (hb_back_addr.is_blank_ip()) { - int port = hb_back_addr.get_port(); - hb_back_addr = cluster_addr; - hb_back_addr.set_port(port); - hb_back_server_messenger->set_addr_unknowns(hb_back_addr); - dout(10) << " assuming hb_back_addr ip matches cluster_addr" << dendl; - } else { - if (auto session = local_connection->get_priv(); !session) { - hb_back_server_messenger->ms_deliver_handle_fast_connect(local_connection); + for (auto& a : hb_back_addrs.v) { + if (a.is_blank_ip()) { + int port = a.get_port(); + for (auto& b : cluster_addrs.v) { + if (a.get_type() == b.get_type() && + a.get_family() == b.get_family()) { + dout(10) << " assuming hb_back_addr " << a + << " matches cluster_addr " << b << dendl; + a = b; + a.set_port(port); + hb_back_server_messenger->set_addr_unknowns(a); + break; + } + } } } + if (auto session = local_connection->get_priv(); !session) { + hb_back_server_messenger->ms_deliver_handle_fast_connect(local_connection); + } - entity_addr_t hb_front_addr = hb_front_server_messenger->get_myaddr(); local_connection = hb_front_server_messenger->get_loopback_connection().get(); - if (hb_front_addr.is_blank_ip()) { - int port = hb_front_addr.get_port(); - hb_front_addr = client_messenger->get_myaddr(); - hb_front_addr.set_port(port); - hb_front_server_messenger->set_addr_unknowns(hb_front_addr); - dout(10) << " assuming hb_front_addr ip matches client_addr" << dendl; - } else { - if (auto session = local_connection->get_priv(); !session) { - hb_front_server_messenger->ms_deliver_handle_fast_connect(local_connection); + for (auto& a : hb_front_addrs.v) { + if (a.is_blank_ip()) { + int port = a.get_port(); + for (auto& b : client_addrs.v) { + if (a.get_type() == b.get_type() && + a.get_family() == b.get_family()) { + dout(10) << " assuming hb_front_addr " << a + << " matches client_addr " << b << dendl; + a = b; + a.set_port(port); + hb_front_server_messenger->set_addr_unknowns(a); + break; + } + } } } + if (auto session = local_connection->get_priv(); !session) { + hb_front_server_messenger->ms_deliver_handle_fast_connect(local_connection); + } - MOSDBoot *mboot = new MOSDBoot(superblock, get_osdmap_epoch(), service.get_boot_epoch(), - hb_back_addr, hb_front_addr, cluster_addr, - CEPH_FEATURES_ALL); - dout(10) << " client_addr " << client_messenger->get_myaddr() - << ", cluster_addr " << cluster_addr - << ", hb_back_addr " << hb_back_addr - << ", hb_front_addr " << hb_front_addr + MOSDBoot *mboot = new MOSDBoot( + superblock, get_osdmap_epoch(), service.get_boot_epoch(), + hb_back_addrs, hb_front_addrs, cluster_addrs, + CEPH_FEATURES_ALL); + dout(10) << " final client_addrs " << client_addrs + << ", cluster_addrs " << cluster_addrs + << ", hb_back_addrs " << hb_back_addrs + << ", hb_front_addrs " << hb_front_addrs << dendl; _collect_metadata(&mboot->metadata); monc->send_mon_message(mboot); @@ -7029,7 +7063,7 @@ void OSD::wait_for_new_map(OpRequestRef op) void OSD::note_down_osd(int peer) { assert(osd_lock.is_locked()); - cluster_messenger->mark_down(osdmap->get_cluster_addr(peer)); + cluster_messenger->mark_down_addrs(osdmap->get_cluster_addrs(peer)); heartbeat_lock.Lock(); failure_queue.erase(peer); @@ -7981,11 +8015,12 @@ bool OSD::require_same_peer_instance(const Message *m, OSDMapRef& map, int from = m->get_source().num(); if (map->is_down(from) || - (map->get_cluster_addr(from) != m->get_source_inst().addr)) { + (map->get_cluster_addrs(from) != m->get_source_addrs())) { dout(5) << "from dead osd." << from << ", marking down, " << " msg was " << m->get_source_inst().addr - << " expected " << (map->is_up(from) ? - map->get_cluster_addr(from) : entity_addr_t()) + << " expected " + << (map->is_up(from) ? + map->get_cluster_addrs(from) : entity_addrvec_t()) << dendl; ConnectionRef con = m->get_connection(); con->mark_down(); diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc index 67bb651ec63d9..01b623318687d 100644 --- a/src/osd/OSDMap.cc +++ b/src/osd/OSDMap.cc @@ -464,6 +464,31 @@ void OSDMap::Incremental::encode_classic(bufferlist& bl, uint64_t features) cons encode(new_hb_front_up, bl, features); } +template +static void encode_addrvec_map_as_addr(const T& m, bufferlist& bl, uint64_t f) +{ + uint32_t n = m.size(); + encode(n, bl); + for (auto& i : m) { + encode(i.first, bl); + encode(i.second.legacy_addr(), bl, f); + } +} + +template +static void encode_addrvec_pvec_as_addr(const T& m, bufferlist& bl, uint64_t f) +{ + uint32_t n = m.size(); + encode(n, bl); + for (auto& i : m) { + if (i) { + encode(i->legacy_addr(), bl, f); + } else { + encode(entity_addr_t(), bl, f); + } + } +} + void OSDMap::Incremental::encode(bufferlist& bl, uint64_t features) const { using ceph::encode; @@ -511,12 +536,7 @@ void OSDMap::Incremental::encode(bufferlist& bl, uint64_t features) const if (v >= 7) { encode(new_up_client, bl, features); } else { - uint32_t n = new_up_client.size(); - encode(n, bl); - for (auto& i : new_up_client) { - encode(i.first, bl); - encode(i.second.legacy_addr(), bl, features); - } + encode_addrvec_map_as_addr(new_up_client, bl, features); } if (v >= 5) { encode(new_state, bl); @@ -548,22 +568,36 @@ void OSDMap::Incremental::encode(bufferlist& bl, uint64_t features) const } { - uint8_t target_v = 6; + uint8_t target_v = 7; if (!HAVE_FEATURE(features, SERVER_LUMINOUS)) { target_v = 2; + } else if (!HAVE_FEATURE(features, SERVER_NAUTILUS)) { + target_v = 6; } ENCODE_START(target_v, 1, bl); // extended, osd-only data - encode(new_hb_back_up, bl, features); + if (target_v < 7) { + encode_addrvec_map_as_addr(new_hb_back_up, bl, features); + } else { + encode(new_hb_back_up, bl, features); + } encode(new_up_thru, bl); encode(new_last_clean_interval, bl); encode(new_lost, bl); encode(new_blacklist, bl, features); encode(old_blacklist, bl, features); - encode(new_up_cluster, bl, features); + if (target_v < 7) { + encode_addrvec_map_as_addr(new_up_cluster, bl, features); + } else { + encode(new_up_cluster, bl, features); + } encode(cluster_snapshot, bl); encode(new_uuid, bl); encode(new_xinfo, bl); - encode(new_hb_front_up, bl, features); + if (target_v < 7) { + encode_addrvec_map_as_addr(new_hb_front_up, bl, features); + } else { + encode(new_hb_front_up, bl, features); + } encode(features, bl); // NOTE: features arg, not the member if (target_v >= 3) { encode(new_nearfull_ratio, bl); @@ -778,7 +812,7 @@ void OSDMap::Incremental::decode(bufferlist::const_iterator& bl) } { - DECODE_START(6, bl); // extended, osd-only data + DECODE_START(7, bl); // extended, osd-only data decode(new_hb_back_up, bl); decode(new_up_thru, bl); decode(new_last_clean_interval, bl); @@ -928,11 +962,19 @@ void OSDMap::Incremental::dump(Formatter *f) const f->dump_int("osd", upclient.first); f->dump_stream("public_addr") << upclient.second.legacy_addr(); f->dump_object("public_addrs", upclient.second); - f->dump_stream("cluster_addr") << new_up_cluster.find(upclient.first)->second; - f->dump_stream("heartbeat_back_addr") << new_hb_back_up.find(upclient.first)->second; - map::const_iterator q; - if ((q = new_hb_front_up.find(upclient.first)) != new_hb_front_up.end()) - f->dump_stream("heartbeat_front_addr") << q->second; + if (auto p = new_up_cluster.find(upclient.first); + p != new_up_cluster.end()) { + f->dump_stream("cluster_addr") << p->second.legacy_addr(); + f->dump_object("cluster_addrs", p->second); + } + if (auto p = new_hb_back_up.find(upclient.first); + p != new_hb_back_up.end()) { + f->dump_object("heartbeat_back_addrs", p->second); + } + if (auto p = new_hb_front_up.find(upclient.first); + p != new_hb_front_up.end()) { + f->dump_object("heartbeat_front_addrs", p->second); + } f->close_section(); } f->close_section(); @@ -1187,9 +1229,9 @@ void OSDMap::set_max_osd(int m) osd_info.resize(m); osd_xinfo.resize(m); osd_addrs->client_addrs.resize(m); - osd_addrs->cluster_addr.resize(m); - osd_addrs->hb_back_addr.resize(m); - osd_addrs->hb_front_addr.resize(m); + osd_addrs->cluster_addrs.resize(m); + osd_addrs->hb_back_addrs.resize(m); + osd_addrs->hb_front_addrs.resize(m); osd_uuid->resize(m); if (osd_primary_affinity) osd_primary_affinity->resize(m, CEPH_OSD_DEFAULT_PRIMARY_AFFINITY); @@ -1331,7 +1373,7 @@ int OSDMap::identify_osd(const entity_addr_t& addr) const { for (int i=0; iosd_addrs->client_addrs[i] = o->osd_addrs->client_addrs[i]; else diff++; - if ( n->osd_addrs->cluster_addr[i] && o->osd_addrs->cluster_addr[i] && - *n->osd_addrs->cluster_addr[i] == *o->osd_addrs->cluster_addr[i]) - n->osd_addrs->cluster_addr[i] = o->osd_addrs->cluster_addr[i]; + if ( n->osd_addrs->cluster_addrs[i] && o->osd_addrs->cluster_addrs[i] && + *n->osd_addrs->cluster_addrs[i] == *o->osd_addrs->cluster_addrs[i]) + n->osd_addrs->cluster_addrs[i] = o->osd_addrs->cluster_addrs[i]; else diff++; - if ( n->osd_addrs->hb_back_addr[i] && o->osd_addrs->hb_back_addr[i] && - *n->osd_addrs->hb_back_addr[i] == *o->osd_addrs->hb_back_addr[i]) - n->osd_addrs->hb_back_addr[i] = o->osd_addrs->hb_back_addr[i]; + if ( n->osd_addrs->hb_back_addrs[i] && o->osd_addrs->hb_back_addrs[i] && + *n->osd_addrs->hb_back_addrs[i] == *o->osd_addrs->hb_back_addrs[i]) + n->osd_addrs->hb_back_addrs[i] = o->osd_addrs->hb_back_addrs[i]; else diff++; - if ( n->osd_addrs->hb_front_addr[i] && o->osd_addrs->hb_front_addr[i] && - *n->osd_addrs->hb_front_addr[i] == *o->osd_addrs->hb_front_addr[i]) - n->osd_addrs->hb_front_addr[i] = o->osd_addrs->hb_front_addr[i]; + if ( n->osd_addrs->hb_front_addrs[i] && o->osd_addrs->hb_front_addrs[i] && + *n->osd_addrs->hb_front_addrs[i] == *o->osd_addrs->hb_front_addrs[i]) + n->osd_addrs->hb_front_addrs[i] = o->osd_addrs->hb_front_addrs[i]; else diff++; } @@ -1886,9 +1930,9 @@ int OSDMap::apply_incremental(const Incremental &inc) osd_xinfo[osd] = osd_xinfo_t(); set_primary_affinity(osd, CEPH_OSD_DEFAULT_PRIMARY_AFFINITY); osd_addrs->client_addrs[osd].reset(new entity_addrvec_t()); - osd_addrs->cluster_addr[osd].reset(new entity_addr_t()); - osd_addrs->hb_front_addr[osd].reset(new entity_addr_t()); - osd_addrs->hb_back_addr[osd].reset(new entity_addr_t()); + osd_addrs->cluster_addrs[osd].reset(new entity_addrvec_t()); + osd_addrs->hb_front_addrs[osd].reset(new entity_addrvec_t()); + osd_addrs->hb_back_addrs[osd].reset(new entity_addrvec_t()); osd_state[osd] = 0; } else { osd_state[osd] ^= s; @@ -1899,19 +1943,17 @@ int OSDMap::apply_incremental(const Incremental &inc) osd_state[client.first] |= CEPH_OSD_EXISTS | CEPH_OSD_UP; osd_addrs->client_addrs[client.first].reset( new entity_addrvec_t(client.second)); - osd_addrs->hb_back_addr[client.first].reset( - new entity_addr_t(inc.new_hb_back_up.find(client.first)->second)); - const auto j = inc.new_hb_front_up.find(client.first); - if (j != inc.new_hb_front_up.end()) - osd_addrs->hb_front_addr[client.first].reset(new entity_addr_t(j->second)); - else - osd_addrs->hb_front_addr[client.first].reset(); + osd_addrs->hb_back_addrs[client.first].reset( + new entity_addrvec_t(inc.new_hb_back_up.find(client.first)->second)); + osd_addrs->hb_front_addrs[client.first].reset( + new entity_addrvec_t(inc.new_hb_front_up.find(client.first)->second)); osd_info[client.first].up_from = epoch; } for (const auto &cluster : inc.new_up_cluster) - osd_addrs->cluster_addr[cluster.first].reset(new entity_addr_t(cluster.second)); + osd_addrs->cluster_addrs[cluster.first].reset( + new entity_addrvec_t(cluster.second)); // info for (const auto &thru : inc.new_up_thru) @@ -2524,15 +2566,15 @@ void OSDMap::encode_classic(bufferlist& bl, uint64_t features) const // extended __u16 ev = 10; encode(ev, bl); - encode(osd_addrs->hb_back_addr, bl, features); + encode(osd_addrs->hb_back_addrs, bl, features); encode(osd_info, bl); encode(blacklist, bl, features); - encode(osd_addrs->cluster_addr, bl, features); + encode(osd_addrs->cluster_addrs, bl, features); encode(cluster_snapshot_epoch, bl); encode(cluster_snapshot, bl); encode(*osd_uuid, bl); encode(osd_xinfo, bl); - encode(osd_addrs->hb_front_addr, bl, features); + encode(osd_addrs->hb_front_addrs, bl, features); } void OSDMap::encode(bufferlist& bl, uint64_t features) const @@ -2606,15 +2648,7 @@ void OSDMap::encode(bufferlist& bl, uint64_t features) const if (v >= 8) { encode(osd_addrs->client_addrs, bl, features); } else { - uint32_t n = osd_addrs->client_addrs.size(); - encode(n, bl); - for (auto& i : osd_addrs->client_addrs) { - if (i) { - encode(i->legacy_addr(), bl, features); - } else { - encode(entity_addr_t(), bl, features); - } - } + encode_addrvec_pvec_as_addr(osd_addrs->client_addrs, bl, features); } encode(*pg_temp, bl); @@ -2652,14 +2686,20 @@ void OSDMap::encode(bufferlist& bl, uint64_t features) const { // NOTE: any new encoding dependencies must be reflected by // SIGNIFICANT_FEATURES - uint8_t target_v = 6; + uint8_t target_v = 7; if (!HAVE_FEATURE(features, SERVER_LUMINOUS)) { target_v = 1; } else if (!HAVE_FEATURE(features, SERVER_MIMIC)) { target_v = 5; + } else if (!HAVE_FEATURE(features, SERVER_NAUTILUS)) { + target_v = 6; } ENCODE_START(target_v, 1, bl); // extended, osd-only data - encode(osd_addrs->hb_back_addr, bl, features); + if (target_v < 7) { + encode_addrvec_pvec_as_addr(osd_addrs->hb_back_addrs, bl, features); + } else { + encode(osd_addrs->hb_back_addrs, bl, features); + } encode(osd_info, bl); { // put this in a sorted, ordered map<> so that we encode in a @@ -2669,12 +2709,20 @@ void OSDMap::encode(bufferlist& bl, uint64_t features) const blacklist_map.insert(make_pair(addr.first, addr.second)); encode(blacklist_map, bl, features); } - encode(osd_addrs->cluster_addr, bl, features); + if (target_v < 7) { + encode_addrvec_pvec_as_addr(osd_addrs->cluster_addrs, bl, features); + } else { + encode(osd_addrs->cluster_addrs, bl, features); + } encode(cluster_snapshot_epoch, bl); encode(cluster_snapshot, bl); encode(*osd_uuid, bl); encode(osd_xinfo, bl); - encode(osd_addrs->hb_front_addr, bl, features); + if (target_v < 7) { + encode_addrvec_pvec_as_addr(osd_addrs->hb_front_addrs, bl, features); + } else { + encode(osd_addrs->hb_front_addrs, bl, features); + } if (target_v >= 2) { encode(nearfull_ratio, bl); encode(full_ratio, bl); @@ -2804,16 +2852,16 @@ void OSDMap::decode_classic(bufferlist::const_iterator& p) __u16 ev = 0; if (v >= 5) decode(ev, p); - decode(osd_addrs->hb_back_addr, p); + decode(osd_addrs->hb_back_addrs, p); decode(osd_info, p); if (v < 5) decode(pool_name, p); decode(blacklist, p); if (ev >= 6) - decode(osd_addrs->cluster_addr, p); + decode(osd_addrs->cluster_addrs, p); else - osd_addrs->cluster_addr.resize(osd_addrs->client_addrs.size()); + osd_addrs->cluster_addrs.resize(osd_addrs->client_addrs.size()); if (ev >= 7) { decode(cluster_snapshot_epoch, p); @@ -2831,9 +2879,9 @@ void OSDMap::decode_classic(bufferlist::const_iterator& p) osd_xinfo.resize(max_osd); if (ev >= 10) - decode(osd_addrs->hb_front_addr, p); + decode(osd_addrs->hb_front_addrs, p); else - osd_addrs->hb_front_addr.resize(osd_addrs->hb_back_addr.size()); + osd_addrs->hb_front_addrs.resize(osd_addrs->hb_back_addrs.size()); osd_primary_affinity.reset(); @@ -2931,16 +2979,16 @@ void OSDMap::decode(bufferlist::const_iterator& bl) } { - DECODE_START(6, bl); // extended, osd-only data - decode(osd_addrs->hb_back_addr, bl); + DECODE_START(7, bl); // extended, osd-only data + decode(osd_addrs->hb_back_addrs, bl); decode(osd_info, bl); decode(blacklist, bl); - decode(osd_addrs->cluster_addr, bl); + decode(osd_addrs->cluster_addrs, bl); decode(cluster_snapshot_epoch, bl); decode(cluster_snapshot, bl); decode(*osd_uuid, bl); decode(osd_xinfo, bl); - decode(osd_addrs->hb_front_addr, bl); + decode(osd_addrs->hb_front_addrs, bl); if (struct_v >= 2) { decode(nearfull_ratio, bl); decode(full_ratio, bl); @@ -3100,9 +3148,10 @@ void OSDMap::dump(Formatter *f) const get_info(i).dump(f); f->dump_stream("public_addr") << get_addrs(i).legacy_addr(); f->dump_object("public_addrs", get_addrs(i)); - f->dump_stream("cluster_addr") << get_cluster_addr(i); - f->dump_stream("heartbeat_back_addr") << get_hb_back_addr(i); - f->dump_stream("heartbeat_front_addr") << get_hb_front_addr(i); + f->dump_stream("cluster_addr") << get_cluster_addrs(i).legacy_addr(); + f->dump_object("cluster_addrs", get_cluster_addrs(i)); + f->dump_object("heartbeat_back_addrs", get_hb_back_addrs(i)); + f->dump_object("heartbeat_front_addr", get_hb_front_addrs(i)); set st; get_state(i, st); @@ -3354,9 +3403,9 @@ void OSDMap::print(ostream& out) const out << " primary_affinity " << get_primary_affinityf(i); const osd_info_t& info(get_info(i)); out << " " << info; - out << " " << get_addrs(i) << " " << get_cluster_addr(i) - << " " << get_hb_back_addr(i) - << " " << get_hb_front_addr(i); + out << " " << get_addrs(i) << " " << get_cluster_addrs(i) + << " " << get_hb_back_addrs(i) + << " " << get_hb_front_addrs(i); set st; get_state(i, st); out << " " << st; diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h index 6c7b1362a953f..753094d769fe8 100644 --- a/src/osd/OSDMap.h +++ b/src/osd/OSDMap.h @@ -376,7 +376,7 @@ public: mempool::osdmap::map > new_erasure_code_profiles; mempool::osdmap::vector old_erasure_code_profiles; mempool::osdmap::map new_up_client; - mempool::osdmap::map new_up_cluster; + mempool::osdmap::map new_up_cluster; mempool::osdmap::map new_state; // XORed onto previous state. mempool::osdmap::map new_weight; mempool::osdmap::map > new_pg_temp; // [] to remove @@ -390,8 +390,8 @@ public: mempool::osdmap::map new_blacklist; mempool::osdmap::vector old_blacklist; - mempool::osdmap::map new_hb_back_up; - mempool::osdmap::map new_hb_front_up; + mempool::osdmap::map new_hb_back_up; + mempool::osdmap::map new_hb_front_up; mempool::osdmap::map> new_pg_upmap; mempool::osdmap::map>> new_pg_upmap_items; @@ -525,13 +525,12 @@ private: struct addrs_s { mempool::osdmap::vector > client_addrs; - mempool::osdmap::vector > cluster_addr; - mempool::osdmap::vector > hb_back_addr; - mempool::osdmap::vector > hb_front_addr; + mempool::osdmap::vector > cluster_addrs; + mempool::osdmap::vector > hb_back_addrs; + mempool::osdmap::vector > hb_front_addrs; }; std::shared_ptr osd_addrs; - entity_addr_t _blank_addr; entity_addrvec_t _blank_addrvec; mempool::osdmap::vector<__u32> osd_weight; // 16.16 fixed point, 0x10000 = "in", 0 = "out" @@ -632,7 +631,7 @@ public: if (o.osd_primary_affinity) osd_primary_affinity.reset(new mempool::osdmap::vector<__u32>(*o.osd_primary_affinity)); - // NOTE: this still references shared entity_addr_t's. + // NOTE: this still references shared entity_addrvec_t's. osd_addrs.reset(new addrs_s(*o.osd_addrs)); // NOTE: we do not copy crush. note that apply_incremental will @@ -899,40 +898,23 @@ public: return osd_addrs->client_addrs[osd] ? *osd_addrs->client_addrs[osd] : _blank_addrvec; } - entity_addrvec_t get_cluster_addrs(int osd) const { - assert(exists(osd)); - if (!osd_addrs->cluster_addr[osd]) - return entity_addrvec_t(); - return entity_addrvec_t(*osd_addrs->cluster_addr[osd]); - } - entity_addrvec_t get_hb_back_addrs(int osd) const { - assert(exists(osd)); - return entity_addrvec_t(osd_addrs->hb_back_addr[osd] ? - *osd_addrs->hb_back_addr[osd] : _blank_addr); - } - entity_addrvec_t get_hb_front_addrs(int osd) const { - assert(exists(osd)); - return entity_addrvec_t(osd_addrs->hb_front_addr[osd] ? - *osd_addrs->hb_front_addr[osd] : _blank_addr); - } const entity_addrvec_t& get_most_recent_addrs(int osd) const { return get_addrs(osd); } - - const entity_addr_t &get_cluster_addr(int osd) const { + const entity_addrvec_t &get_cluster_addrs(int osd) const { assert(exists(osd)); - return osd_addrs->cluster_addr[osd] ? - *osd_addrs->cluster_addr[osd] : _blank_addr; + return osd_addrs->cluster_addrs[osd] ? + *osd_addrs->cluster_addrs[osd] : _blank_addrvec; } - const entity_addr_t &get_hb_back_addr(int osd) const { + const entity_addrvec_t &get_hb_back_addrs(int osd) const { assert(exists(osd)); - return osd_addrs->hb_back_addr[osd] ? - *osd_addrs->hb_back_addr[osd] : _blank_addr; + return osd_addrs->hb_back_addrs[osd] ? + *osd_addrs->hb_back_addrs[osd] : _blank_addrvec; } - const entity_addr_t &get_hb_front_addr(int osd) const { + const entity_addrvec_t &get_hb_front_addrs(int osd) const { assert(exists(osd)); - return osd_addrs->hb_front_addr[osd] ? - *osd_addrs->hb_front_addr[osd] : _blank_addr; + return osd_addrs->hb_front_addrs[osd] ? + *osd_addrs->hb_front_addrs[osd] : _blank_addrvec; } const uuid_d& get_uuid(int osd) const { diff --git a/src/test/osd/TestOSDMap.cc b/src/test/osd/TestOSDMap.cc index 8a690f9963b3c..45fed3458ac16 100644 --- a/src/test/osd/TestOSDMap.cc +++ b/src/test/osd/TestOSDMap.cc @@ -42,16 +42,17 @@ public: osdmap.build_simple(g_ceph_context, 0, fsid, num_osds); OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1); pending_inc.fsid = osdmap.get_fsid(); - entity_addr_t sample_addr; + entity_addrvec_t sample_addrs; + sample_addrs.v.push_back(entity_addr_t()); uuid_d sample_uuid; for (int i = 0; i < num_osds; ++i) { sample_uuid.generate_random(); - sample_addr.nonce = i; + sample_addrs.v[0].nonce = i; pending_inc.new_state[i] = CEPH_OSD_EXISTS | CEPH_OSD_NEW; - pending_inc.new_up_client[i] = entity_addrvec_t(sample_addr); - pending_inc.new_up_cluster[i] = sample_addr; - pending_inc.new_hb_back_up[i] = sample_addr; - pending_inc.new_hb_front_up[i] = sample_addr; + pending_inc.new_up_client[i] = sample_addrs; + pending_inc.new_up_cluster[i] = sample_addrs; + pending_inc.new_hb_back_up[i] = sample_addrs; + pending_inc.new_hb_front_up[i] = sample_addrs; pending_inc.new_weight[i] = CEPH_OSD_IN; pending_inc.new_uuid[i] = sample_uuid; }