msgr: inject delays at inconvenient times

Exercise some rare races by injecting delays before taking locks
via the 'ms inject internal delays' option.

Signed-off-by: Sage Weil <sage@inktank.com>
(cherry picked from commit a5d692a7b9b4bec2c27993ca37aa3fec4065292b)

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 576b3b341e4fa851bce3814fdba9b8b0fc03a028..ef1cdcf6b740a4c6c97499282d8c77349968b647 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -109,6 +109,7 @@ OPTION(ms_inject_socket_failures, OPT_U64, 0)
 OPTION(ms_inject_delay_type, OPT_STR, "")          // "osd mds mon client" allowed
 OPTION(ms_inject_delay_max, OPT_DOUBLE, 1)         // seconds
 OPTION(ms_inject_delay_probability, OPT_DOUBLE, 0) // range [0, 1]
+OPTION(ms_inject_internal_delays, OPT_DOUBLE, 0)   // seconds
 
 OPTION(mon_data, OPT_STR, "/var/lib/ceph/mon/$cluster-$id")
 OPTION(mon_initial_members, OPT_STR, "")    // list of initial cluster mon ids; if specified, need majority to form initial quorum and create new cluster

diff --git a/src/include/utime.h b/src/include/utime.h
index 8e7891f1594b91dc3511336155a3d21f41b30560..526dec568aec9eb0a6915ea04a0037ede7db3e50 100644 (file)
--- a/src/include/utime.h
+++ b/src/include/utime.h
@@ -20,6 +20,8 @@
 #include <time.h>
 
 #include "include/types.h"
+
+
 // --------
 // utime_t
 
@@ -133,6 +135,11 @@ public:
     return ts;
   }
 
+  void sleep() {
+    struct timespec ts = { tv.tv_sec, tv.tv_nsec };
+    nanosleep(&ts, &ts);
+  }
+
   // output
   ostream& gmtime(ostream& out) const {
     out.setf(std::ios::right);

diff --git a/src/msg/Pipe.cc b/src/msg/Pipe.cc
index 7edbb4f31703028a424a73d08f6ef27370ab0bbb..ae6234f84ea85e33c0664e01ce9508f8e6f049e0 100644 (file)
--- a/src/msg/Pipe.cc
+++ b/src/msg/Pipe.cc
@@ -639,10 +639,19 @@ int Pipe::accept()
 
  fail_registered:
   ldout(msgr->cct, 10) << "accept fault after register" << dendl;
+
+  if (msgr->cct->_conf->ms_inject_internal_delays) {
+    ldout(msgr->cct, 10) << " sleep for " << msgr->cct->_conf->ms_inject_internal_delays << dendl;
+    utime_t t;
+    t.set_from_double(msgr->cct->_conf->ms_inject_internal_delays);
+    t.sleep();
+  }
+
  fail_unlocked:
   pipe_lock.Lock();
   if (state != STATE_CLOSED) {
     bool queued = is_queued();
+    ldout(msgr->cct, 10) << "  queued = " << (int)queued << dendl;
     if (queued)
       state = policy.server ? STATE_STANDBY : STATE_CONNECTING;
     else if (replaced)
@@ -658,6 +667,14 @@ int Pipe::accept()
 
  shutting_down:
   msgr->lock.Unlock();
+
+  if (msgr->cct->_conf->ms_inject_internal_delays) {
+    ldout(msgr->cct, 10) << " sleep for " << msgr->cct->_conf->ms_inject_internal_delays << dendl;
+    utime_t t;
+    t.set_from_double(msgr->cct->_conf->ms_inject_internal_delays);
+    t.sleep();
+  }
+
   pipe_lock.Lock();
   state = STATE_CLOSED;
   fault();
@@ -865,6 +882,13 @@ int Pipe::connect()
       }
     }
 
+    if (conf->ms_inject_internal_delays) {
+      ldout(msgr->cct, 10) << " sleep for " << msgr->cct->_conf->ms_inject_internal_delays << dendl;
+      utime_t t;
+      t.set_from_double(msgr->cct->_conf->ms_inject_internal_delays);
+      t.sleep();
+    }
+
     pipe_lock.Lock();
     if (state != STATE_CONNECTING) {
       ldout(msgr->cct,0) << "connect got RESETSESSION but no longer connecting" << dendl;
@@ -987,6 +1011,13 @@ int Pipe::connect()
   }
 
  fail:
+  if (conf->ms_inject_internal_delays) {
+    ldout(msgr->cct, 10) << " sleep for " << msgr->cct->_conf->ms_inject_internal_delays << dendl;
+    utime_t t;
+    t.set_from_double(msgr->cct->_conf->ms_inject_internal_delays);
+    t.sleep();
+  }
+
   pipe_lock.Lock();
  fail_locked:
   if (state == STATE_CONNECTING)
@@ -1093,8 +1124,17 @@ void Pipe::fault(bool onread)
 
     stop();
 
-    // ugh
+    // crib locks, blech.  note that Pipe is now STATE_CLOSED and the
+    // rank_pipe entry is ignored by others.
     pipe_lock.Unlock();
+
+    if (conf->ms_inject_internal_delays) {
+      ldout(msgr->cct, 10) << " sleep for " << msgr->cct->_conf->ms_inject_internal_delays << dendl;
+      utime_t t;
+      t.set_from_double(msgr->cct->_conf->ms_inject_internal_delays);
+      t.sleep();
+    }
+
     msgr->lock.Lock();
     pipe_lock.Lock();
     unregister_pipe();