]> git.apps.os.sepia.ceph.com Git - ceph.git/commitdiff
osd/OSDMap: make cluster addrs addrvecs too
authorSage Weil <sage@redhat.com>
Mon, 4 Jun 2018 19:23:35 +0000 (14:23 -0500)
committerSage Weil <sage@redhat.com>
Tue, 3 Jul 2018 18:01:23 +0000 (13:01 -0500)
This will allow a single-step upgrade from msgr1 to msgr2: as each
nautilus OSD starts up, it will bind to both msgr1 and msgr2 addrs
(assuming it starts up after the mons have been upgraded).  Peer OSDs will
use the msgr2 address if they can, or the legacy one if they can't.
Once require_osd_release >= nautilus OSDs will stop binding to msgr1
ports at all.

Signed-off-by: Sage Weil <sage@redhat.com>
src/messages/MOSDBoot.h
src/mon/OSDMonitor.cc
src/osd/OSD.cc
src/osd/OSDMap.cc
src/osd/OSDMap.h
src/test/osd/TestOSDMap.cc

index 2f06412859e16f8325505cf81a270ef9b97063c1..beb6a587f20348d76a95a61ff25c9b325fdeeda1 100644 (file)
@@ -27,8 +27,8 @@ class MOSDBoot : public PaxosServiceMessage {
 
  public:
   OSDSuperblock sb;
-  entity_addr_t hb_back_addr, hb_front_addr;
-  entity_addr_t cluster_addr;
+  entity_addrvec_t hb_back_addrs, hb_front_addrs;
+  entity_addrvec_t cluster_addrs;
   epoch_t boot_epoch;  // last epoch this daemon was added to the map (if any)
   map<string,string> metadata; ///< misc metadata about this osd
   uint64_t osd_features;
@@ -38,15 +38,15 @@ class MOSDBoot : public PaxosServiceMessage {
       boot_epoch(0), osd_features(0)
   { }
   MOSDBoot(OSDSuperblock& s, epoch_t e, epoch_t be,
-          const entity_addr_t& hb_back_addr_ref,
-          const entity_addr_t& hb_front_addr_ref,
-           const entity_addr_t& cluster_addr_ref,
+          const entity_addrvec_t& hb_back_addr_ref,
+          const entity_addrvec_t& hb_front_addr_ref,
+           const entity_addrvec_t& cluster_addr_ref,
           uint64_t feat)
     : PaxosServiceMessage(MSG_OSD_BOOT, e, HEAD_VERSION, COMPAT_VERSION),
       sb(s),
-      hb_back_addr(hb_back_addr_ref),
-      hb_front_addr(hb_front_addr_ref),
-      cluster_addr(cluster_addr_ref),
+      hb_back_addrs(hb_back_addr_ref),
+      hb_front_addrs(hb_front_addr_ref),
+      cluster_addrs(cluster_addr_ref),
       boot_epoch(be),
       osd_features(feat)
   { }
@@ -66,10 +66,10 @@ public:
     using ceph::encode;
     paxos_encode();
     encode(sb, payload);
-    encode(hb_back_addr, payload, features);
-    encode(cluster_addr, payload, features);
+    encode(hb_back_addrs, payload, features);
+    encode(cluster_addrs, payload, features);
     encode(boot_epoch, payload);
-    encode(hb_front_addr, payload, features);
+    encode(hb_front_addrs, payload, features);
     encode(metadata, payload);
     encode(osd_features, payload);
   }
@@ -77,10 +77,10 @@ public:
     auto p = payload.cbegin();
     paxos_decode(p);
     decode(sb, p);
-    decode(hb_back_addr, p);
-    decode(cluster_addr, p);
+    decode(hb_back_addrs, p);
+    decode(cluster_addrs, p);
     decode(boot_epoch, p);
-    decode(hb_front_addr, p);
+    decode(hb_front_addrs, p);
     decode(metadata, p);
     decode(osd_features, p);
   }
index 683c4d59fcea67f32f453881375a426d4def6e0c..231283189a51f12ce5524fee2bb4ce90998bb469 100644 (file)
@@ -2671,7 +2671,7 @@ bool OSDMonitor::preprocess_boot(MonOpRequestRef op)
   // already booted?
   if (osdmap.is_up(from) &&
       osdmap.get_addrs(from) == m->get_orig_source_addrs() &&
-      osdmap.get_cluster_addr(from) == m->cluster_addr) {
+      osdmap.get_cluster_addrs(from) == m->cluster_addrs) {
     // yup.
     dout(7) << "preprocess_boot dup from " << m->get_orig_source()
            << " " << m->get_orig_source_addrs()
@@ -2716,10 +2716,12 @@ bool OSDMonitor::prepare_boot(MonOpRequestRef op)
 {
   op->mark_osdmon_event(__func__);
   MOSDBoot *m = static_cast<MOSDBoot*>(op->get_req());
-  dout(7) << __func__ << " from " << m->get_orig_source_inst() << " sb " << m->sb
-         << " cluster_addr " << m->cluster_addr
-         << " hb_back_addr " << m->hb_back_addr
-         << " hb_front_addr " << m->hb_front_addr
+  dout(7) << __func__ << " from " << m->get_source()
+         << " sb " << m->sb
+         << " client_addrs" << m->get_connection()->get_peer_addrs()
+         << " cluster_addrs " << m->cluster_addrs
+         << " hb_back_addrs " << m->hb_back_addrs
+         << " hb_front_addrs " << m->hb_front_addrs
          << dendl;
 
   assert(m->get_orig_source().is_osd());
@@ -2742,7 +2744,7 @@ bool OSDMonitor::prepare_boot(MonOpRequestRef op)
            << osdmap.get_addrs(from) << dendl;
     // preprocess should have caught these;  if not, assert.
     assert(osdmap.get_addrs(from) != m->get_orig_source_addrs() ||
-           osdmap.get_cluster_addr(from) != m->cluster_addr);
+           osdmap.get_cluster_addrs(from) != m->cluster_addrs);
     assert(osdmap.get_uuid(from) == m->sb.osd_fsid);
 
     if (pending_inc.new_state.count(from) == 0 ||
@@ -2759,9 +2761,9 @@ bool OSDMonitor::prepare_boot(MonOpRequestRef op)
   } else {
     // mark new guy up.
     pending_inc.new_up_client[from] = m->get_orig_source_addrs();
-    pending_inc.new_up_cluster[from] = m->cluster_addr;
-    pending_inc.new_hb_back_up[from] = m->hb_back_addr;
-    pending_inc.new_hb_front_up[from] = m->hb_front_addr;
+    pending_inc.new_up_cluster[from] = m->cluster_addrs;
+    pending_inc.new_hb_back_up[from] = m->hb_back_addrs;
+    pending_inc.new_hb_front_up[from] = m->hb_front_addrs;
 
     down_pending_out.erase(from);  // if any
 
index efa86a78c702e926444bc9b42c30e5626b1fc1b1..cd816223bf758861c713684896f732e12df643db 100644 (file)
@@ -1055,8 +1055,8 @@ bool OSDService::should_share_map(entity_name_t name, Connection *con,
   if (con->get_messenger() == osd->cluster_messenger &&
       con != osd->cluster_messenger->get_loopback_connection() &&
       osdmap->is_up(name.num()) &&
-      (osdmap->get_cluster_addr(name.num()) == con->get_peer_addr() ||
-       osdmap->get_hb_back_addr(name.num()) == con->get_peer_addr())) {
+      (osdmap->get_cluster_addrs(name.num()) == con->get_peer_addrs() ||
+       osdmap->get_hb_back_addrs(name.num()) == con->get_peer_addrs())) {
     // remember
     epoch_t has = std::max(get_peer_epoch(name.num()), epoch);
 
@@ -1102,9 +1102,9 @@ void OSDService::share_map(
       send_incremental_map(epoch, con, osdmap);
     } else if (con->get_messenger() == osd->cluster_messenger &&
         osdmap->is_up(name.num()) &&
-        (osdmap->get_cluster_addr(name.num()) == con->get_peer_addr() ||
-            osdmap->get_hb_back_addr(name.num()) == con->get_peer_addr())) {
-      dout(10) << name << " " << con->get_peer_addr()
+        (osdmap->get_cluster_addrs(name.num()) == con->get_peer_addrs() ||
+            osdmap->get_hb_back_addrs(name.num()) == con->get_peer_addrs())) {
+      dout(10) << name << " " << con->get_peer_addrs()
                       << " has old map " << epoch << " < "
                       << osdmap->get_epoch() << dendl;
       note_peer_epoch(name.num(), osdmap->get_epoch());
@@ -5338,55 +5338,89 @@ bool OSD::_is_healthy()
 void OSD::_send_boot()
 {
   dout(10) << "_send_boot" << dendl;
-  entity_addr_t cluster_addr = cluster_messenger->get_myaddr();
-  Connection *local_connection = cluster_messenger->get_loopback_connection().get();
-  if (cluster_addr.is_blank_ip()) {
-    int port = cluster_addr.get_port();
-    cluster_addr = client_messenger->get_myaddr();
-    cluster_addr.set_port(port);
-    cluster_messenger->set_addr_unknowns(cluster_addr);
-    dout(10) << " assuming cluster_addr ip matches client_addr" << dendl;
-  } else {
-    if (auto session = local_connection->get_priv(); !session) {
-      cluster_messenger->ms_deliver_handle_fast_connect(local_connection);
+  Connection *local_connection =
+    cluster_messenger->get_loopback_connection().get();
+  entity_addrvec_t client_addrs = client_messenger->get_myaddrs();
+  entity_addrvec_t cluster_addrs = cluster_messenger->get_myaddrs();
+  entity_addrvec_t hb_back_addrs = hb_back_server_messenger->get_myaddrs();
+  entity_addrvec_t hb_front_addrs = hb_front_server_messenger->get_myaddrs();
+
+  dout(20) << " initial client_addrs " << client_addrs
+          << ", cluster_addrs " << cluster_addrs
+          << ", hb_back_addrs " << hb_back_addrs
+          << ", hb_front_addrs " << hb_front_addrs
+          << dendl;
+
+  for (auto& a : cluster_addrs.v) {
+    if (a.is_blank_ip()) {
+      int port = a.get_port();
+      for (auto& b : client_addrs.v) {
+       if (a.get_type() == b.get_type() &&
+           a.get_family() == b.get_family()) {
+         dout(10) << " assuming cluster_addr " << a
+                  << " matches client_addr " << b << dendl;
+         a = b;
+         a.set_port(port);
+         cluster_messenger->set_addr_unknowns(a);
+         break;
+       }
+      }
     }
   }
+  if (auto session = local_connection->get_priv(); !session) {
+    cluster_messenger->ms_deliver_handle_fast_connect(local_connection);
+  }
 
-  entity_addr_t hb_back_addr = hb_back_server_messenger->get_myaddr();
   local_connection = hb_back_server_messenger->get_loopback_connection().get();
-  if (hb_back_addr.is_blank_ip()) {
-    int port = hb_back_addr.get_port();
-    hb_back_addr = cluster_addr;
-    hb_back_addr.set_port(port);
-    hb_back_server_messenger->set_addr_unknowns(hb_back_addr);
-    dout(10) << " assuming hb_back_addr ip matches cluster_addr" << dendl;
-  } else {
-    if (auto session = local_connection->get_priv(); !session) {
-      hb_back_server_messenger->ms_deliver_handle_fast_connect(local_connection);
+  for (auto& a : hb_back_addrs.v) {
+    if (a.is_blank_ip()) {
+      int port = a.get_port();
+      for (auto& b : cluster_addrs.v) {
+       if (a.get_type() == b.get_type() &&
+           a.get_family() == b.get_family()) {
+         dout(10) << " assuming hb_back_addr " << a
+                  << " matches cluster_addr " << b << dendl;
+         a = b;
+         a.set_port(port);
+         hb_back_server_messenger->set_addr_unknowns(a);
+         break;
+       }
+      }
     }
   }
+  if (auto session = local_connection->get_priv(); !session) {
+    hb_back_server_messenger->ms_deliver_handle_fast_connect(local_connection);
+  }
 
-  entity_addr_t hb_front_addr = hb_front_server_messenger->get_myaddr();
   local_connection = hb_front_server_messenger->get_loopback_connection().get();
-  if (hb_front_addr.is_blank_ip()) {
-    int port = hb_front_addr.get_port();
-    hb_front_addr = client_messenger->get_myaddr();
-    hb_front_addr.set_port(port);
-    hb_front_server_messenger->set_addr_unknowns(hb_front_addr);
-    dout(10) << " assuming hb_front_addr ip matches client_addr" << dendl;
-  } else {
-    if (auto session = local_connection->get_priv(); !session) {
-      hb_front_server_messenger->ms_deliver_handle_fast_connect(local_connection);
+  for (auto& a : hb_front_addrs.v) {
+    if (a.is_blank_ip()) {
+      int port = a.get_port();
+      for (auto& b : client_addrs.v) {
+       if (a.get_type() == b.get_type() &&
+           a.get_family() == b.get_family()) {
+         dout(10) << " assuming hb_front_addr " << a
+                  << " matches client_addr " << b << dendl;
+         a = b;
+         a.set_port(port);
+         hb_front_server_messenger->set_addr_unknowns(a);
+         break;
+       }
+      }
     }
   }
+  if (auto session = local_connection->get_priv(); !session) {
+    hb_front_server_messenger->ms_deliver_handle_fast_connect(local_connection);
+  }
 
-  MOSDBoot *mboot = new MOSDBoot(superblock, get_osdmap_epoch(), service.get_boot_epoch(),
-                                 hb_back_addr, hb_front_addr, cluster_addr,
-                                CEPH_FEATURES_ALL);
-  dout(10) << " client_addr " << client_messenger->get_myaddr()
-          << ", cluster_addr " << cluster_addr
-          << ", hb_back_addr " << hb_back_addr
-          << ", hb_front_addr " << hb_front_addr
+  MOSDBoot *mboot = new MOSDBoot(
+    superblock, get_osdmap_epoch(), service.get_boot_epoch(),
+    hb_back_addrs, hb_front_addrs, cluster_addrs,
+    CEPH_FEATURES_ALL);
+  dout(10) << " final client_addrs " << client_addrs
+          << ", cluster_addrs " << cluster_addrs
+          << ", hb_back_addrs " << hb_back_addrs
+          << ", hb_front_addrs " << hb_front_addrs
           << dendl;
   _collect_metadata(&mboot->metadata);
   monc->send_mon_message(mboot);
@@ -7029,7 +7063,7 @@ void OSD::wait_for_new_map(OpRequestRef op)
 void OSD::note_down_osd(int peer)
 {
   assert(osd_lock.is_locked());
-  cluster_messenger->mark_down(osdmap->get_cluster_addr(peer));
+  cluster_messenger->mark_down_addrs(osdmap->get_cluster_addrs(peer));
 
   heartbeat_lock.Lock();
   failure_queue.erase(peer);
@@ -7981,11 +8015,12 @@ bool OSD::require_same_peer_instance(const Message *m, OSDMapRef& map,
   int from = m->get_source().num();
 
   if (map->is_down(from) ||
-      (map->get_cluster_addr(from) != m->get_source_inst().addr)) {
+      (map->get_cluster_addrs(from) != m->get_source_addrs())) {
     dout(5) << "from dead osd." << from << ", marking down, "
            << " msg was " << m->get_source_inst().addr
-           << " expected " << (map->is_up(from) ?
-                               map->get_cluster_addr(from) : entity_addr_t())
+           << " expected "
+           << (map->is_up(from) ?
+               map->get_cluster_addrs(from) : entity_addrvec_t())
            << dendl;
     ConnectionRef con = m->get_connection();
     con->mark_down();
index 67bb651ec63d9ee1bf226305b7badc95438c26a6..01b623318687d2c22896320aac7b975d78d8d6fd 100644 (file)
@@ -464,6 +464,31 @@ void OSDMap::Incremental::encode_classic(bufferlist& bl, uint64_t features) cons
   encode(new_hb_front_up, bl, features);
 }
 
+template<class T>
+static void encode_addrvec_map_as_addr(const T& m, bufferlist& bl, uint64_t f)
+{
+  uint32_t n = m.size();
+  encode(n, bl);
+  for (auto& i : m) {
+    encode(i.first, bl);
+    encode(i.second.legacy_addr(), bl, f);
+  }
+}
+
+template<class T>
+static void encode_addrvec_pvec_as_addr(const T& m, bufferlist& bl, uint64_t f)
+{
+  uint32_t n = m.size();
+  encode(n, bl);
+  for (auto& i : m) {
+    if (i) {
+      encode(i->legacy_addr(), bl, f);
+    } else {
+      encode(entity_addr_t(), bl, f);
+    }
+  }
+}
+
 void OSDMap::Incremental::encode(bufferlist& bl, uint64_t features) const
 {
   using ceph::encode;
@@ -511,12 +536,7 @@ void OSDMap::Incremental::encode(bufferlist& bl, uint64_t features) const
     if (v >= 7) {
       encode(new_up_client, bl, features);
     } else {
-      uint32_t n = new_up_client.size();
-      encode(n, bl);
-      for (auto& i : new_up_client) {
-       encode(i.first, bl);
-       encode(i.second.legacy_addr(), bl, features);
-      }
+      encode_addrvec_map_as_addr(new_up_client, bl, features);
     }
     if (v >= 5) {
       encode(new_state, bl);
@@ -548,22 +568,36 @@ void OSDMap::Incremental::encode(bufferlist& bl, uint64_t features) const
   }
 
   {
-    uint8_t target_v = 6;
+    uint8_t target_v = 7;
     if (!HAVE_FEATURE(features, SERVER_LUMINOUS)) {
       target_v = 2;
+    } else if (!HAVE_FEATURE(features, SERVER_NAUTILUS)) {
+      target_v = 6;
     }
     ENCODE_START(target_v, 1, bl); // extended, osd-only data
-    encode(new_hb_back_up, bl, features);
+    if (target_v < 7) {
+      encode_addrvec_map_as_addr(new_hb_back_up, bl, features);
+    } else {
+      encode(new_hb_back_up, bl, features);
+    }
     encode(new_up_thru, bl);
     encode(new_last_clean_interval, bl);
     encode(new_lost, bl);
     encode(new_blacklist, bl, features);
     encode(old_blacklist, bl, features);
-    encode(new_up_cluster, bl, features);
+    if (target_v < 7) {
+      encode_addrvec_map_as_addr(new_up_cluster, bl, features);
+    } else {
+      encode(new_up_cluster, bl, features);
+    }
     encode(cluster_snapshot, bl);
     encode(new_uuid, bl);
     encode(new_xinfo, bl);
-    encode(new_hb_front_up, bl, features);
+    if (target_v < 7) {
+      encode_addrvec_map_as_addr(new_hb_front_up, bl, features);
+    } else {
+      encode(new_hb_front_up, bl, features);
+    }
     encode(features, bl);         // NOTE: features arg, not the member
     if (target_v >= 3) {
       encode(new_nearfull_ratio, bl);
@@ -778,7 +812,7 @@ void OSDMap::Incremental::decode(bufferlist::const_iterator& bl)
   }
 
   {
-    DECODE_START(6, bl); // extended, osd-only data
+    DECODE_START(7, bl); // extended, osd-only data
     decode(new_hb_back_up, bl);
     decode(new_up_thru, bl);
     decode(new_last_clean_interval, bl);
@@ -928,11 +962,19 @@ void OSDMap::Incremental::dump(Formatter *f) const
     f->dump_int("osd", upclient.first);
     f->dump_stream("public_addr") << upclient.second.legacy_addr();
     f->dump_object("public_addrs", upclient.second);
-    f->dump_stream("cluster_addr") << new_up_cluster.find(upclient.first)->second;
-    f->dump_stream("heartbeat_back_addr") << new_hb_back_up.find(upclient.first)->second;
-    map<int32_t, entity_addr_t>::const_iterator q;
-    if ((q = new_hb_front_up.find(upclient.first)) != new_hb_front_up.end())
-      f->dump_stream("heartbeat_front_addr") << q->second;
+    if (auto p = new_up_cluster.find(upclient.first);
+       p != new_up_cluster.end()) {
+      f->dump_stream("cluster_addr") << p->second.legacy_addr();
+      f->dump_object("cluster_addrs", p->second);
+    }
+    if (auto p = new_hb_back_up.find(upclient.first);
+       p != new_hb_back_up.end()) {
+      f->dump_object("heartbeat_back_addrs", p->second);
+    }
+    if (auto p = new_hb_front_up.find(upclient.first);
+       p != new_hb_front_up.end()) {
+      f->dump_object("heartbeat_front_addrs", p->second);
+    }
     f->close_section();
   }
   f->close_section();
@@ -1187,9 +1229,9 @@ void OSDMap::set_max_osd(int m)
   osd_info.resize(m);
   osd_xinfo.resize(m);
   osd_addrs->client_addrs.resize(m);
-  osd_addrs->cluster_addr.resize(m);
-  osd_addrs->hb_back_addr.resize(m);
-  osd_addrs->hb_front_addr.resize(m);
+  osd_addrs->cluster_addrs.resize(m);
+  osd_addrs->hb_back_addrs.resize(m);
+  osd_addrs->hb_front_addrs.resize(m);
   osd_uuid->resize(m);
   if (osd_primary_affinity)
     osd_primary_affinity->resize(m, CEPH_OSD_DEFAULT_PRIMARY_AFFINITY);
@@ -1331,7 +1373,7 @@ int OSDMap::identify_osd(const entity_addr_t& addr) const
 {
   for (int i=0; i<max_osd; i++)
     if (exists(i) && (get_addrs(i).contains(addr) ||
-                     get_cluster_addr(i) == addr))
+                     get_cluster_addrs(i).contains(addr)))
       return i;
   return -1;
 }
@@ -1347,8 +1389,10 @@ int OSDMap::identify_osd(const uuid_d& u) const
 int OSDMap::identify_osd_on_all_channels(const entity_addr_t& addr) const
 {
   for (int i=0; i<max_osd; i++)
-    if (exists(i) && (get_addrs(i).contains(addr) || get_cluster_addr(i) == addr ||
-       get_hb_back_addr(i) == addr || get_hb_front_addr(i) == addr))
+    if (exists(i) && (get_addrs(i).contains(addr) ||
+                     get_cluster_addrs(i).contains(addr) ||
+                     get_hb_back_addrs(i).contains(addr) ||
+                     get_hb_front_addrs(i).contains(addr)))
       return i;
   return -1;
 }
@@ -1357,7 +1401,7 @@ int OSDMap::find_osd_on_ip(const entity_addr_t& ip) const
 {
   for (int i=0; i<max_osd; i++)
     if (exists(i) && (get_addrs(i).is_same_host(ip) ||
-                     get_cluster_addr(i).is_same_host(ip)))
+                     get_cluster_addrs(i).is_same_host(ip)))
       return i;
   return -1;
 }
@@ -1514,19 +1558,19 @@ void OSDMap::dedup(const OSDMap *o, OSDMap *n)
       n->osd_addrs->client_addrs[i] = o->osd_addrs->client_addrs[i];
     else
       diff++;
-    if ( n->osd_addrs->cluster_addr[i] &&  o->osd_addrs->cluster_addr[i] &&
-       *n->osd_addrs->cluster_addr[i] == *o->osd_addrs->cluster_addr[i])
-      n->osd_addrs->cluster_addr[i] = o->osd_addrs->cluster_addr[i];
+    if ( n->osd_addrs->cluster_addrs[i] &&  o->osd_addrs->cluster_addrs[i] &&
+       *n->osd_addrs->cluster_addrs[i] == *o->osd_addrs->cluster_addrs[i])
+      n->osd_addrs->cluster_addrs[i] = o->osd_addrs->cluster_addrs[i];
     else
       diff++;
-    if ( n->osd_addrs->hb_back_addr[i] &&  o->osd_addrs->hb_back_addr[i] &&
-       *n->osd_addrs->hb_back_addr[i] == *o->osd_addrs->hb_back_addr[i])
-      n->osd_addrs->hb_back_addr[i] = o->osd_addrs->hb_back_addr[i];
+    if ( n->osd_addrs->hb_back_addrs[i] &&  o->osd_addrs->hb_back_addrs[i] &&
+       *n->osd_addrs->hb_back_addrs[i] == *o->osd_addrs->hb_back_addrs[i])
+      n->osd_addrs->hb_back_addrs[i] = o->osd_addrs->hb_back_addrs[i];
     else
       diff++;
-    if ( n->osd_addrs->hb_front_addr[i] &&  o->osd_addrs->hb_front_addr[i] &&
-       *n->osd_addrs->hb_front_addr[i] == *o->osd_addrs->hb_front_addr[i])
-      n->osd_addrs->hb_front_addr[i] = o->osd_addrs->hb_front_addr[i];
+    if ( n->osd_addrs->hb_front_addrs[i] &&  o->osd_addrs->hb_front_addrs[i] &&
+       *n->osd_addrs->hb_front_addrs[i] == *o->osd_addrs->hb_front_addrs[i])
+      n->osd_addrs->hb_front_addrs[i] = o->osd_addrs->hb_front_addrs[i];
     else
       diff++;
   }
@@ -1886,9 +1930,9 @@ int OSDMap::apply_incremental(const Incremental &inc)
       osd_xinfo[osd] = osd_xinfo_t();
       set_primary_affinity(osd, CEPH_OSD_DEFAULT_PRIMARY_AFFINITY);
       osd_addrs->client_addrs[osd].reset(new entity_addrvec_t());
-      osd_addrs->cluster_addr[osd].reset(new entity_addr_t());
-      osd_addrs->hb_front_addr[osd].reset(new entity_addr_t());
-      osd_addrs->hb_back_addr[osd].reset(new entity_addr_t());
+      osd_addrs->cluster_addrs[osd].reset(new entity_addrvec_t());
+      osd_addrs->hb_front_addrs[osd].reset(new entity_addrvec_t());
+      osd_addrs->hb_back_addrs[osd].reset(new entity_addrvec_t());
       osd_state[osd] = 0;
     } else {
       osd_state[osd] ^= s;
@@ -1899,19 +1943,17 @@ int OSDMap::apply_incremental(const Incremental &inc)
     osd_state[client.first] |= CEPH_OSD_EXISTS | CEPH_OSD_UP;
     osd_addrs->client_addrs[client.first].reset(
       new entity_addrvec_t(client.second));
-    osd_addrs->hb_back_addr[client.first].reset(
-      new entity_addr_t(inc.new_hb_back_up.find(client.first)->second));
-    const auto j = inc.new_hb_front_up.find(client.first);
-    if (j != inc.new_hb_front_up.end())
-      osd_addrs->hb_front_addr[client.first].reset(new entity_addr_t(j->second));
-    else
-      osd_addrs->hb_front_addr[client.first].reset();
+    osd_addrs->hb_back_addrs[client.first].reset(
+      new entity_addrvec_t(inc.new_hb_back_up.find(client.first)->second));
+    osd_addrs->hb_front_addrs[client.first].reset(
+      new entity_addrvec_t(inc.new_hb_front_up.find(client.first)->second));
 
     osd_info[client.first].up_from = epoch;
   }
 
   for (const auto &cluster : inc.new_up_cluster)
-    osd_addrs->cluster_addr[cluster.first].reset(new entity_addr_t(cluster.second));
+    osd_addrs->cluster_addrs[cluster.first].reset(
+      new entity_addrvec_t(cluster.second));
 
   // info
   for (const auto &thru : inc.new_up_thru)
@@ -2524,15 +2566,15 @@ void OSDMap::encode_classic(bufferlist& bl, uint64_t features) const
   // extended
   __u16 ev = 10;
   encode(ev, bl);
-  encode(osd_addrs->hb_back_addr, bl, features);
+  encode(osd_addrs->hb_back_addrs, bl, features);
   encode(osd_info, bl);
   encode(blacklist, bl, features);
-  encode(osd_addrs->cluster_addr, bl, features);
+  encode(osd_addrs->cluster_addrs, bl, features);
   encode(cluster_snapshot_epoch, bl);
   encode(cluster_snapshot, bl);
   encode(*osd_uuid, bl);
   encode(osd_xinfo, bl);
-  encode(osd_addrs->hb_front_addr, bl, features);
+  encode(osd_addrs->hb_front_addrs, bl, features);
 }
 
 void OSDMap::encode(bufferlist& bl, uint64_t features) const
@@ -2606,15 +2648,7 @@ void OSDMap::encode(bufferlist& bl, uint64_t features) const
     if (v >= 8) {
       encode(osd_addrs->client_addrs, bl, features);
     } else {
-      uint32_t n = osd_addrs->client_addrs.size();
-      encode(n, bl);
-      for (auto& i : osd_addrs->client_addrs) {
-       if (i) {
-         encode(i->legacy_addr(), bl, features);
-       } else {
-         encode(entity_addr_t(), bl, features);
-       }
-      }
+      encode_addrvec_pvec_as_addr(osd_addrs->client_addrs, bl, features);
     }
 
     encode(*pg_temp, bl);
@@ -2652,14 +2686,20 @@ void OSDMap::encode(bufferlist& bl, uint64_t features) const
   {
     // NOTE: any new encoding dependencies must be reflected by
     // SIGNIFICANT_FEATURES
-    uint8_t target_v = 6;
+    uint8_t target_v = 7;
     if (!HAVE_FEATURE(features, SERVER_LUMINOUS)) {
       target_v = 1;
     } else if (!HAVE_FEATURE(features, SERVER_MIMIC)) {
       target_v = 5;
+    } else if (!HAVE_FEATURE(features, SERVER_NAUTILUS)) {
+      target_v = 6;
     }
     ENCODE_START(target_v, 1, bl); // extended, osd-only data
-    encode(osd_addrs->hb_back_addr, bl, features);
+    if (target_v < 7) {
+      encode_addrvec_pvec_as_addr(osd_addrs->hb_back_addrs, bl, features);
+    } else {
+      encode(osd_addrs->hb_back_addrs, bl, features);
+    }
     encode(osd_info, bl);
     {
       // put this in a sorted, ordered map<> so that we encode in a
@@ -2669,12 +2709,20 @@ void OSDMap::encode(bufferlist& bl, uint64_t features) const
        blacklist_map.insert(make_pair(addr.first, addr.second));
       encode(blacklist_map, bl, features);
     }
-    encode(osd_addrs->cluster_addr, bl, features);
+    if (target_v < 7) {
+      encode_addrvec_pvec_as_addr(osd_addrs->cluster_addrs, bl, features);
+    } else {
+      encode(osd_addrs->cluster_addrs, bl, features);
+    }
     encode(cluster_snapshot_epoch, bl);
     encode(cluster_snapshot, bl);
     encode(*osd_uuid, bl);
     encode(osd_xinfo, bl);
-    encode(osd_addrs->hb_front_addr, bl, features);
+    if (target_v < 7) {
+      encode_addrvec_pvec_as_addr(osd_addrs->hb_front_addrs, bl, features);
+    } else {
+      encode(osd_addrs->hb_front_addrs, bl, features);
+    }
     if (target_v >= 2) {
       encode(nearfull_ratio, bl);
       encode(full_ratio, bl);
@@ -2804,16 +2852,16 @@ void OSDMap::decode_classic(bufferlist::const_iterator& p)
   __u16 ev = 0;
   if (v >= 5)
     decode(ev, p);
-  decode(osd_addrs->hb_back_addr, p);
+  decode(osd_addrs->hb_back_addrs, p);
   decode(osd_info, p);
   if (v < 5)
     decode(pool_name, p);
 
   decode(blacklist, p);
   if (ev >= 6)
-    decode(osd_addrs->cluster_addr, p);
+    decode(osd_addrs->cluster_addrs, p);
   else
-    osd_addrs->cluster_addr.resize(osd_addrs->client_addrs.size());
+    osd_addrs->cluster_addrs.resize(osd_addrs->client_addrs.size());
 
   if (ev >= 7) {
     decode(cluster_snapshot_epoch, p);
@@ -2831,9 +2879,9 @@ void OSDMap::decode_classic(bufferlist::const_iterator& p)
     osd_xinfo.resize(max_osd);
 
   if (ev >= 10)
-    decode(osd_addrs->hb_front_addr, p);
+    decode(osd_addrs->hb_front_addrs, p);
   else
-    osd_addrs->hb_front_addr.resize(osd_addrs->hb_back_addr.size());
+    osd_addrs->hb_front_addrs.resize(osd_addrs->hb_back_addrs.size());
 
   osd_primary_affinity.reset();
 
@@ -2931,16 +2979,16 @@ void OSDMap::decode(bufferlist::const_iterator& bl)
   }
 
   {
-    DECODE_START(6, bl); // extended, osd-only data
-    decode(osd_addrs->hb_back_addr, bl);
+    DECODE_START(7, bl); // extended, osd-only data
+    decode(osd_addrs->hb_back_addrs, bl);
     decode(osd_info, bl);
     decode(blacklist, bl);
-    decode(osd_addrs->cluster_addr, bl);
+    decode(osd_addrs->cluster_addrs, bl);
     decode(cluster_snapshot_epoch, bl);
     decode(cluster_snapshot, bl);
     decode(*osd_uuid, bl);
     decode(osd_xinfo, bl);
-    decode(osd_addrs->hb_front_addr, bl);
+    decode(osd_addrs->hb_front_addrs, bl);
     if (struct_v >= 2) {
       decode(nearfull_ratio, bl);
       decode(full_ratio, bl);
@@ -3100,9 +3148,10 @@ void OSDMap::dump(Formatter *f) const
       get_info(i).dump(f);
       f->dump_stream("public_addr") << get_addrs(i).legacy_addr();
       f->dump_object("public_addrs", get_addrs(i));
-      f->dump_stream("cluster_addr") << get_cluster_addr(i);
-      f->dump_stream("heartbeat_back_addr") << get_hb_back_addr(i);
-      f->dump_stream("heartbeat_front_addr") << get_hb_front_addr(i);
+      f->dump_stream("cluster_addr") << get_cluster_addrs(i).legacy_addr();
+      f->dump_object("cluster_addrs", get_cluster_addrs(i));
+      f->dump_object("heartbeat_back_addrs", get_hb_back_addrs(i));
+      f->dump_object("heartbeat_front_addr", get_hb_front_addrs(i));
 
       set<string> st;
       get_state(i, st);
@@ -3354,9 +3403,9 @@ void OSDMap::print(ostream& out) const
        out << " primary_affinity " << get_primary_affinityf(i);
       const osd_info_t& info(get_info(i));
       out << " " << info;
-      out << " " << get_addrs(i) << " " << get_cluster_addr(i)
-         << " " << get_hb_back_addr(i)
-         << " " << get_hb_front_addr(i);
+      out << " " << get_addrs(i) << " " << get_cluster_addrs(i)
+         << " " << get_hb_back_addrs(i)
+         << " " << get_hb_front_addrs(i);
       set<string> st;
       get_state(i, st);
       out << " " << st;
index 6c7b1362a953f7173e6f21f674766c747e9e93ee..753094d769fe84e91bb5046a32c49efd2f37eb83 100644 (file)
@@ -376,7 +376,7 @@ public:
     mempool::osdmap::map<string,map<string,string> > new_erasure_code_profiles;
     mempool::osdmap::vector<string> old_erasure_code_profiles;
     mempool::osdmap::map<int32_t,entity_addrvec_t> new_up_client;
-    mempool::osdmap::map<int32_t,entity_addr_t> new_up_cluster;
+    mempool::osdmap::map<int32_t,entity_addrvec_t> new_up_cluster;
     mempool::osdmap::map<int32_t,uint32_t> new_state;             // XORed onto previous state.
     mempool::osdmap::map<int32_t,uint32_t> new_weight;
     mempool::osdmap::map<pg_t,mempool::osdmap::vector<int32_t> > new_pg_temp;     // [] to remove
@@ -390,8 +390,8 @@ public:
 
     mempool::osdmap::map<entity_addr_t,utime_t> new_blacklist;
     mempool::osdmap::vector<entity_addr_t> old_blacklist;
-    mempool::osdmap::map<int32_t, entity_addr_t> new_hb_back_up;
-    mempool::osdmap::map<int32_t, entity_addr_t> new_hb_front_up;
+    mempool::osdmap::map<int32_t, entity_addrvec_t> new_hb_back_up;
+    mempool::osdmap::map<int32_t, entity_addrvec_t> new_hb_front_up;
 
     mempool::osdmap::map<pg_t,mempool::osdmap::vector<int32_t>> new_pg_upmap;
     mempool::osdmap::map<pg_t,mempool::osdmap::vector<pair<int32_t,int32_t>>> new_pg_upmap_items;
@@ -525,13 +525,12 @@ private:
 
   struct addrs_s {
     mempool::osdmap::vector<std::shared_ptr<entity_addrvec_t> > client_addrs;
-    mempool::osdmap::vector<std::shared_ptr<entity_addr_t> > cluster_addr;
-    mempool::osdmap::vector<std::shared_ptr<entity_addr_t> > hb_back_addr;
-    mempool::osdmap::vector<std::shared_ptr<entity_addr_t> > hb_front_addr;
+    mempool::osdmap::vector<std::shared_ptr<entity_addrvec_t> > cluster_addrs;
+    mempool::osdmap::vector<std::shared_ptr<entity_addrvec_t> > hb_back_addrs;
+    mempool::osdmap::vector<std::shared_ptr<entity_addrvec_t> > hb_front_addrs;
   };
   std::shared_ptr<addrs_s> osd_addrs;
 
-  entity_addr_t _blank_addr;
   entity_addrvec_t _blank_addrvec;
 
   mempool::osdmap::vector<__u32>   osd_weight;   // 16.16 fixed point, 0x10000 = "in", 0 = "out"
@@ -632,7 +631,7 @@ public:
     if (o.osd_primary_affinity)
       osd_primary_affinity.reset(new mempool::osdmap::vector<__u32>(*o.osd_primary_affinity));
 
-    // NOTE: this still references shared entity_addr_t's.
+    // NOTE: this still references shared entity_addrvec_t's.
     osd_addrs.reset(new addrs_s(*o.osd_addrs));
 
     // NOTE: we do not copy crush.  note that apply_incremental will
@@ -899,40 +898,23 @@ public:
     return osd_addrs->client_addrs[osd] ?
       *osd_addrs->client_addrs[osd] : _blank_addrvec;
   }
-  entity_addrvec_t get_cluster_addrs(int osd) const {
-    assert(exists(osd));
-    if (!osd_addrs->cluster_addr[osd])
-      return entity_addrvec_t();
-    return entity_addrvec_t(*osd_addrs->cluster_addr[osd]);
-  }
-  entity_addrvec_t get_hb_back_addrs(int osd) const {
-    assert(exists(osd));
-    return entity_addrvec_t(osd_addrs->hb_back_addr[osd] ?
-                           *osd_addrs->hb_back_addr[osd] : _blank_addr);
-  }
-  entity_addrvec_t get_hb_front_addrs(int osd) const {
-    assert(exists(osd));
-    return entity_addrvec_t(osd_addrs->hb_front_addr[osd] ?
-                           *osd_addrs->hb_front_addr[osd] : _blank_addr);
-  }
   const entity_addrvec_t& get_most_recent_addrs(int osd) const {
     return get_addrs(osd);
   }
-
-  const entity_addr_t &get_cluster_addr(int osd) const {
+  const entity_addrvec_t &get_cluster_addrs(int osd) const {
     assert(exists(osd));
-    return osd_addrs->cluster_addr[osd] ?
-      *osd_addrs->cluster_addr[osd] : _blank_addr;
+    return osd_addrs->cluster_addrs[osd] ?
+      *osd_addrs->cluster_addrs[osd] : _blank_addrvec;
   }
-  const entity_addr_t &get_hb_back_addr(int osd) const {
+  const entity_addrvec_t &get_hb_back_addrs(int osd) const {
     assert(exists(osd));
-    return osd_addrs->hb_back_addr[osd] ?
-      *osd_addrs->hb_back_addr[osd] : _blank_addr;
+    return osd_addrs->hb_back_addrs[osd] ?
+      *osd_addrs->hb_back_addrs[osd] : _blank_addrvec;
   }
-  const entity_addr_t &get_hb_front_addr(int osd) const {
+  const entity_addrvec_t &get_hb_front_addrs(int osd) const {
     assert(exists(osd));
-    return osd_addrs->hb_front_addr[osd] ?
-      *osd_addrs->hb_front_addr[osd] : _blank_addr;
+    return osd_addrs->hb_front_addrs[osd] ?
+      *osd_addrs->hb_front_addrs[osd] : _blank_addrvec;
   }
 
   const uuid_d& get_uuid(int osd) const {
index 8a690f9963b3cd9b4a99f83c82d3623aced1c957..45fed3458ac16fd0e5caea3cf59d1528db323a03 100644 (file)
@@ -42,16 +42,17 @@ public:
     osdmap.build_simple(g_ceph_context, 0, fsid, num_osds);
     OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1);
     pending_inc.fsid = osdmap.get_fsid();
-    entity_addr_t sample_addr;
+    entity_addrvec_t sample_addrs;
+    sample_addrs.v.push_back(entity_addr_t());
     uuid_d sample_uuid;
     for (int i = 0; i < num_osds; ++i) {
       sample_uuid.generate_random();
-      sample_addr.nonce = i;
+      sample_addrs.v[0].nonce = i;
       pending_inc.new_state[i] = CEPH_OSD_EXISTS | CEPH_OSD_NEW;
-      pending_inc.new_up_client[i] = entity_addrvec_t(sample_addr);
-      pending_inc.new_up_cluster[i] = sample_addr;
-      pending_inc.new_hb_back_up[i] = sample_addr;
-      pending_inc.new_hb_front_up[i] = sample_addr;
+      pending_inc.new_up_client[i] = sample_addrs;
+      pending_inc.new_up_cluster[i] = sample_addrs;
+      pending_inc.new_hb_back_up[i] = sample_addrs;
+      pending_inc.new_hb_front_up[i] = sample_addrs;
       pending_inc.new_weight[i] = CEPH_OSD_IN;
       pending_inc.new_uuid[i] = sample_uuid;
     }