assert: allow assertions inside calls to dout()

We should handle the situation where we assert() while already holding
the dout() lock. At the same time, we want to get the dout lock if we
can, because it makes the logs look nicer. pthread_mutex_trylock solves
the dilemma.

Signed-off-by: Colin McCabe <colin.mccabe@dreamhost.com>

diff --git a/src/common/assert.cc b/src/common/assert.cc
index 60bdef80cf61d514566fc1b8d3a786272dfda5e0..c4a5c744eb0a2e1a89029733786f6adad9f9db55 100644 (file)
--- a/src/common/assert.cc
+++ b/src/common/assert.cc
@@ -12,19 +12,51 @@
  *
  */
 
+#include <errno.h>
 #include <iostream>
+#include <pthread.h>
 #include <sstream>
+#include <time.h>
 
 #include "BackTrace.h"
 #include "common/debug.h"
 #include "config.h"
 #include "include/assert.h"
 
+static bool dout_trylock(void)
+{
+  time_t cur;
+  time_t end_time = time(NULL);
+  end_time += 10;
+
+  do {
+    int ret = pthread_mutex_trylock(&_dout_lock);
+    if (ret == 0) {
+      // We got the mutex.
+      return true;
+    }
+    else if ((ret == EDEADLK) || (ret == EAGAIN)) {
+      // This thread already owns the mutex.
+      // The assertion happened inside a call to dout()?
+      return false;
+    }
+    else if (ret == EINVAL) {
+      // We can never get the mutex.
+      return false;
+    }
+    // Otherwise, keep trying.
+    usleep(1000);
+    cur = time(NULL);
+  } while (cur < end_time);
+
+  // If 10 seconds go by and we can't get the mutex, just bail.
+  return false;
+}
+
 namespace ceph {
   void __ceph_assert_fail(const char *assertion, const char *file, int line, const char *func)
   {
-    // TODO: replace this with a trylock
-    DoutLocker _dout_locker;
+    int got_lock = dout_trylock();
 
     char buf[8096];
     BackTrace *bt = new BackTrace(1);
@@ -44,19 +76,32 @@ namespace ceph {
             "is needed to interpret this.\n");
     dout_emergency(oss.str());
 
+    if (got_lock) {
+      int ret = pthread_mutex_unlock(&_dout_lock);
+      if (ret) {
+       ; // ignored
+      }
+    }
+
     throw FailedAssertion(bt);
   }
 
   void __ceph_assert_warn(const char *assertion, const char *file,
                          int line, const char *func)
   {
-    // TODO: replace this with a trylock
-    DoutLocker _dout_locker;
+    int got_lock = dout_trylock();
 
     char buf[8096];
     snprintf(buf, sizeof(buf),
             "WARNING: assert(%s) at: %s: %d: %s()\n",
             assertion, file, line, func);
     dout_emergency(buf);
+
+    if (got_lock) {
+      int ret = pthread_mutex_unlock(&_dout_lock);
+      if (ret) {
+       ; // ignored
+      }
+    }
   }
 }