osd: fix osd shutdown vs scrub work queue deadlock

Pause new disk_tp items, kick all (possibly) blocked pgs, and _then_
stop the disk_tp.  And make scrub() bail out if the osd is stopping.

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 1430120efea377d3e87a1db84c3bc1958228316a..a6829e6a416773b467a8f15041d5d5b30d12d6d2 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -455,14 +455,30 @@ int OSD::shutdown()
   wait_for_no_ops();
   dout(10) << "no ops" << dendl;
 
-  // stop threads
   recovery_tp.stop();
   dout(10) << "recovery tp stopped" << dendl;
-  disk_tp.stop();
-  dout(10) << "disk tp stopped" << dendl;
   op_tp.stop();
   dout(10) << "op tp stopped" << dendl;
 
+  // pause _new_ disk work first (to avoid racing with thread pool),
+  disk_tp.pause_new();
+  dout(10) << "disk tp paused (new), kicking all pgs" << dendl;
+
+  // then kick all pgs,
+  for (hash_map<pg_t, PG*>::iterator p = pg_map.begin();
+       p != pg_map.end();
+       p++) {
+    dout(20) << " kicking pg " << p->first << dendl;
+    p->second->lock();
+    p->second->kick();
+    p->second->unlock();
+  }
+  dout(20) << " kicked all pgs" << dendl;
+
+  // then stop thread.
+  disk_tp.stop();
+  dout(10) << "disk tp stopped" << dendl;
+
   // tell pgs we're shutting down
   for (hash_map<pg_t, PG*>::iterator p = pg_map.begin();
        p != pg_map.end();

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index e0b57d044df547d953183ef04a886b4507b692e4..62bc245cc5480f44ede747f468bd48827ace1906 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -2137,6 +2137,8 @@ void PG::scrub()
   while (is_write_in_progress()) {
     dout(10) << "scrub  write(s) in progress, waiting" << dendl;
     wait();
+    if (osd->is_stopping())
+      goto out;
   }
 
 
@@ -2158,7 +2160,8 @@ void PG::scrub()
             << " maps, waiting" << dendl;
     wait();
 
-    if (epoch != info.history.same_since) {
+    if (epoch != info.history.same_since ||
+       osd->is_stopping()) {
       dout(10) << "scrub  pg changed, aborting" << dendl;
       goto out;
     }