mon/OSDMonitor: calculate new mapping in parallel

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 8d0538992d9a0174c7de932ab99c4ed741c122d1..aeb58f7b19f11846a8a533d8d4701ff8c7544ff5 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -261,6 +261,7 @@ OPTION(mon_compact_on_trim, OPT_BOOL, true)       // compact (a prefix) when we
 OPTION(mon_osd_cache_size, OPT_INT, 10)  // the size of osdmaps cache, not to rely on underlying store's cache
 
 OPTION(mon_cpu_threads, OPT_INT, 4)
+OPTION(mon_osd_mapping_pgs_per_chunk, OPT_INT, 4096)
 OPTION(mon_tick_interval, OPT_INT, 5)
 OPTION(mon_session_timeout, OPT_INT, 300)    // must send keepalive or subscribe
 OPTION(mon_subscribe_interval, OPT_DOUBLE, 24*3600)  // for legacy clients only

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 2f748e7ab1c07c552b0f53e6ac02758116fcefa3..0628574b7a39a81a466a5f9eff6e05677562938d 100644 (file)
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -75,9 +75,14 @@ static ostream& _prefix(std::ostream *_dout, Monitor *mon, const OSDMap& osdmap)
                << ").osd e" << osdmap.get_epoch() << " ";
 }
 
-OSDMonitor::OSDMonitor(CephContext *cct, Monitor *mn, Paxos *p, const string& service_name)
+OSDMonitor::OSDMonitor(
+  CephContext *cct,
+  Monitor *mn,
+  Paxos *p,
+  const string& service_name)
  : PaxosService(mn, p, service_name),
    cct(cct),
+   mapper(mn->cct, &mn->cpu_tp),
    inc_osd_cache(g_conf->mon_osd_cache_size),
    full_osd_cache(g_conf->mon_osd_cache_size),
    last_attempted_minwait_time(utime_t()),
@@ -337,6 +342,20 @@ void OSDMonitor::update_msgr_features()
   }
 }
 
+void OSDMonitor::_calc_mapping(const OSDMap& osdmap, OSDMapMapping *mapping)
+{
+  utime_t start = ceph_clock_now();
+  if (g_conf->mon_cpu_threads) {
+    auto job = mapper.queue(osdmap, mapping,
+                           g_conf->mon_osd_mapping_pgs_per_chunk);
+    job->wait();
+  } else {
+    mapping->update(osdmap);
+  }
+  utime_t end = ceph_clock_now();
+  dout(10) << __func__ << " in " << (end - start) << dendl;
+}
+
 void OSDMonitor::on_active()
 {
   update_logger();
@@ -356,10 +375,7 @@ void OSDMonitor::on_active()
   }
 
   // FIXME: hacky synchronous blocking mapping update
-  utime_t start = ceph_clock_now();
-  mapping.update(osdmap);
-  utime_t end = ceph_clock_now();
-  dout(10) << __func__ << " updated mapping in " << (end - start) << dendl;
+  _calc_mapping(osdmap, mapping.get());
 }
 
 void OSDMonitor::on_shutdown()
@@ -1020,7 +1036,7 @@ void OSDMonitor::maybe_prime_pg_temp()
   } else {
     dout(10) << __func__ << " " << osds.size() << " interesting osds" << dendl;
     for (auto osd : osds) {
-      const vector<pg_t>& pgs = mapping.get_osd_acting_pgs(osd);
+      const vector<pg_t>& pgs = mapping->get_osd_acting_pgs(osd);
       dout(20) << __func__ << " osd." << osd << " " << pgs << dendl;
       for (auto pgid : pgs) {
        if (!pg_map->creating_pgs.count(pgid)) {
@@ -1053,7 +1069,7 @@ void OSDMonitor::prime_pg_temp(
   if (pending_inc.new_pg_temp.count(pgid))
     return;
   vector<int> old_acting;
-  mapping.get(pgid, nullptr, nullptr, &old_acting, nullptr);
+  mapping->get(pgid, nullptr, nullptr, &old_acting, nullptr);
   vector<int> up, acting;
   int up_primary, acting_primary;
   next.pg_to_up_acting_osds(pgid, &up, &up_primary, &acting, &acting_primary);

diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h
index 1e64e05978ab1c11e021db08fa1a64f683dcf114..1cefe7867976ce067d64db5f5f34d7174a50b7e3 100644 (file)
--- a/src/mon/OSDMonitor.h
+++ b/src/mon/OSDMonitor.h
@@ -114,9 +114,11 @@ class OSDMonitor : public PaxosService {
   CephContext *cct;
 public:
   OSDMap osdmap;
-  OSDMapMapping mapping;
+  unique_ptr<OSDMapMapping> mapping;
 
 private:
+  ParallelOSDMapper mapper;
+
   // [leader]
   OSDMap::Incremental pending_inc;
   map<int, bufferlist> pending_metadata;
@@ -146,6 +148,8 @@ private:
     FAST_READ_DEFAULT
   };
 
+  void _calc_mapping(const OSDMap& osdmap, OSDMapMapping *mapping);
+
   // svc
 public:  
   void create_initial();