OSD: OSDMapRef access by multiple threads is unsafe

we update OSD::osdmap in OSD::_committed_osd_maps() which is executed
by objectstore's finisher thread. while PG::sched_scrub() is called
by OSD's sharded work queue's worker thread.and we push the osdmap
updates down to PGs OSD::consume_map() which is in turn called by
OSD::_committed_osd_maps() where osdmap is updated. so it does not big
deal if we are checking a stale CEPH_OSDMAP_NODEEP_SCRUB flag.
also this flag will be updated with the latest osdmap very soon.

Signed-off-by: Kefu Chai <kchai@redhat.com>
Signed-off-by: Zengran Zhang <zhangzengran@sangfor.com.cn>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 0e4e92dd65fbd54882792d5f8b82575c1e626bd9..16f85379e37de2cab935baa2e2a8bd9892823aeb 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -469,7 +469,7 @@ void OSDService::shutdown()
     f->stop();
   }
 
-  osdmap = OSDMapRef();
+  publish_map(OSDMapRef());
   next_osdmap = OSDMapRef();
 }
 
@@ -3954,7 +3954,10 @@ int OSD::shutdown()
   monc->shutdown();
   osd_lock.Unlock();
 
+  map_lock.get_write();
   osdmap = OSDMapRef();
+  map_lock.put_write();
+
   for (auto s : shards) {
     std::lock_guard l(s->osdmap_lock);
     s->shard_osdmap = OSDMapRef();

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 20f48ecee8ffc35b491b42bc334ee8c3b2470997..ffdce25ab225ea4f26b57ab5c91b2a45f473a733 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -4298,7 +4298,7 @@ bool PG::sched_scrub()
   time_for_deep = (time_for_deep || deep_coin_flip);
 
   //NODEEP_SCRUB so ignore time initiated deep-scrub
-  if (osd->osd->get_osdmap()->test_flag(CEPH_OSDMAP_NODEEP_SCRUB) ||
+  if (get_osdmap()->test_flag(CEPH_OSDMAP_NODEEP_SCRUB) ||
       pool.info.has_flag(pg_pool_t::FLAG_NODEEP_SCRUB)) {
     time_for_deep = false;
     nodeep_scrub = true;
@@ -4308,7 +4308,7 @@ bool PG::sched_scrub()
     ceph_assert(!scrubber.must_deep_scrub);
 
     //NOSCRUB so skip regular scrubs
-    if ((osd->osd->get_osdmap()->test_flag(CEPH_OSDMAP_NOSCRUB) ||
+    if ((get_osdmap()->test_flag(CEPH_OSDMAP_NOSCRUB) ||
         pool.info.has_flag(pg_pool_t::FLAG_NOSCRUB)) && !time_for_deep) {
       if (scrubber.reserved) {
         // cancel scrub if it is still in scheduling,