osd: Twiddle should create a full sized vector for optimized EC

The twiddle function in OSD is designed to force a PG to select new OSDs
in a similar mechanism to the "repeer" CLI command. To be compatible with
Fast EC, it must return a vector which the size of the acting set. However,
it returns a smaller vector.  This can cause the OSD to panic in
pgtemp_undo_primaryfirst.

The twiddle function is used in one place and is only called when a PG goes
into a "pending" state. This only happens when the system hits the maximum
PGs per OSD (see OSD::maybe_wait_for_max_pg). As a result it is hard to hit,
however it was seen on a large test system when testing this scenario.

The fix is similar to the repeer fix put in place to fix tracker 73897. See
commit a3cc500a543d1c2fb9e1d55c144e0a041e3d1f80 for details.

In this case, we restrict the change to just Fast EC, so as to reduce the
testing requirement of the change, which is very hard to hit.

Signed-off-by: Alex Ainscow <aainscow@uk.ibm.com>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index a93cf599d059471886c324bbde4c2628771c7658..151afb15fc825d084e3be7c783317c86048a9d52 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -5515,14 +5515,26 @@ bool OSD::maybe_wait_for_max_pg(const OSDMapRef& osdmap,
 // to re-trigger a peering, we have to twiddle the pg mapping a little bit,
 // see PG::should_restart_peering(). OSDMap::pg_to_up_acting_osds() will turn
 // to up set if pg_temp is empty. so an empty pg_temp won't work.
-static vector<int32_t> twiddle(const vector<int>& acting) {
+static vector<int32_t> twiddle(const vector<int>& acting, const OSDMapRef& osdmap, pg_t pgid) {
+  vector<int32_t> twiddled;
   if (acting.size() > 1) {
-    return {acting[0]};
+    twiddled = {acting[0]};
   } else {
-    vector<int32_t> twiddled(acting.begin(), acting.end());
+    twiddled = vector<int32_t>(acting.begin(), acting.end());
     twiddled.push_back(-1);
-    return twiddled;
   }
+  
+  // Optimized EC does not cope with pg temp with a mismatched size.
+  // Only resize for EC pools with optimizations enabled.
+  const pg_pool_t *pool = osdmap->get_pg_pool(pgid.pool());
+  if (pool && pool->is_erasure() && pool->allows_ecoptimizations()) {
+    unsigned pool_size = pool->get_size();
+    if (twiddled.size() < pool_size) {
+      twiddled.resize(pool_size, CRUSH_ITEM_NONE);
+    }
+  }
+  
+  return twiddled;
 }
 
 void OSD::resume_creating_pg()
@@ -5555,7 +5567,7 @@ void OSD::resume_creating_pg()
       dout(20) << __func__ << " pg " << pg->first << dendl;
       vector<int> acting;
       get_osdmap()->pg_to_up_acting_osds(pg->first.pgid, nullptr, nullptr, &acting, nullptr);
-      service.queue_want_pg_temp(pg->first.pgid, twiddle(acting), true);
+      service.queue_want_pg_temp(pg->first.pgid, twiddle(acting, get_osdmap(), pg->first.pgid), true);
       pg = pending_creates_from_osd.erase(pg);
       do_sub_pg_creates = true;
       spare_pgs--;