osd: fix watch reconnect race

It's possible to process a watch reconnect op, setting Watch::conn, before
the ms_handle_reset path tries to disconnect the old one.  Since reset()
was blindly disconnecting the "current" connection, we could race and
disconnect the new con instead of the old one.

Fix this by specifying which con to disconnect.

Fixes: http://tracker.ceph.com/issues/15441
Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 7c260ff39aeae99b023af98aad36bb1674af586d..d37bc7d70bd3835eee49a203b9436042a83cb46d 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -4662,7 +4662,7 @@ bool OSD::ms_handle_reset(Connection *con)
   dout(1) << "ms_handle_reset con " << con << " session " << session << dendl;
   if (!session)
     return false;
-  session->wstate.reset();
+  session->wstate.reset(con);
   session->con.reset(NULL);  // break con <-> session ref cycle
   session_handle_reset(session);
   session->put();

diff --git a/src/osd/Watch.cc b/src/osd/Watch.cc
index f4411baf7feee2008b4bc4f5c04bfed4b02ab144..43846c03a16a8e1c3c580d5e20021651fe855508 100644 (file)
--- a/src/osd/Watch.cc
+++ b/src/osd/Watch.cc
@@ -513,7 +513,7 @@ void WatchConState::removeWatch(WatchRef watch)
   watches.erase(watch);
 }
 
-void WatchConState::reset()
+void WatchConState::reset(Connection *con)
 {
   set<WatchRef> _watches;
   {
@@ -526,7 +526,11 @@ void WatchConState::reset()
     boost::intrusive_ptr<ReplicatedPG> pg((*i)->get_pg());
     pg->lock();
     if (!(*i)->is_discarded()) {
-      (*i)->disconnect();
+      if ((*i)->is_connected(con)) {
+       (*i)->disconnect();
+      } else {
+       generic_derr << __func__ << " not still connected to " << (*i) << dendl;
+      }
     }
     pg->unlock();
   }

diff --git a/src/osd/Watch.h b/src/osd/Watch.h
index 6e4ec37a6b51f13c09a780e9b172d9290078377f..c6843ee785de3376a8d8f68842797cc204a8aa92 100644 (file)
--- a/src/osd/Watch.h
+++ b/src/osd/Watch.h
@@ -202,6 +202,9 @@ public:
   bool is_connected() {
     return conn.get() != NULL;
   }
+  bool is_connected(Connection *con) {
+    return conn.get() == con;
+  }
 
   /// NOTE: must be called with pg lock held
   ~Watch();
@@ -290,7 +293,7 @@ public:
     );
 
   /// Called on session reset, disconnects watchers
-  void reset();
+  void reset(Connection *con);
 };
 
 #endif