Mutex: Instrument Mutex with perfcouter for Lock() wait

Signed-off-by: Samuel Just <sam.just@inktank.com>

diff --git a/src/Makefile.am b/src/Makefile.am
index a0c2c783ce218c7ea89cfd55a6032408a9f237f1..a72df3863d226ee73c51e56671730b32444958af 100644 (file)
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -1163,6 +1163,7 @@ libcommon_files = \
        common/PrebufferedStreambuf.cc \
        common/BackTrace.cc \
        common/perf_counters.cc \
+       common/Mutex.cc \
        common/admin_socket.cc \
        common/admin_socket_client.cc \
        common/escape.c \

diff --git a/src/common/Mutex.cc b/src/common/Mutex.cc
new file mode 100644 (file)
index 0000000..2dc035a
--- /dev/null
+++ b/src/common/Mutex.cc
@@ -0,0 +1,97 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+using namespace std;
+#include <string>
+
+#include "common/Mutex.h"
+#include "common/perf_counters.h"
+#include "common/ceph_context.h"
+#include "common/config.h"
+#include "include/utime.h"
+#include "common/Clock.h"
+
+Mutex::Mutex(const char *n, bool r, bool ld,
+            bool bt,
+            CephContext *cct) :
+  name(n), id(-1), recursive(r), lockdep(ld), backtrace(bt),
+  nlock(0), locked_by(0), cct(cct), logger(0)
+{
+  if (cct) {
+    PerfCountersBuilder b(cct, string("mutex-") + name,
+                         l_mutex_first, l_mutex_last);
+    b.add_fl_avg(l_mutex_wait, "wait");
+    logger = b.create_perf_counters();
+    cct->get_perfcounters_collection()->add(logger);
+    logger->set(l_mutex_wait, 0);
+  }
+  if (recursive) {
+    // Mutexes of type PTHREAD_MUTEX_RECURSIVE do all the same checks as
+    // mutexes of type PTHREAD_MUTEX_ERRORCHECK.
+    pthread_mutexattr_t attr;
+    pthread_mutexattr_init(&attr);
+    pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
+    pthread_mutex_init(&_m,&attr);
+    pthread_mutexattr_destroy(&attr);
+    if (g_lockdep)
+      _register();
+  }
+  else if (lockdep) {
+    // If the mutex type is PTHREAD_MUTEX_ERRORCHECK, then error checking
+    // shall be provided. If a thread attempts to relock a mutex that it
+    // has already locked, an error shall be returned. If a thread
+    // attempts to unlock a mutex that it has not locked or a mutex which
+    // is unlocked, an error shall be returned.
+    pthread_mutexattr_t attr;
+    pthread_mutexattr_init(&attr);
+    pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK);
+    pthread_mutex_init(&_m, &attr);
+    if (g_lockdep)
+      _register();
+  }
+  else {
+    // If the mutex type is PTHREAD_MUTEX_NORMAL, deadlock detection
+    // shall not be provided. Attempting to relock the mutex causes
+    // deadlock. If a thread attempts to unlock a mutex that  it  has not
+    // locked or a mutex which is unlocked, undefined behavior results.
+    pthread_mutex_init(&_m, NULL);
+  }
+}
+
+Mutex::~Mutex() {
+  assert(nlock == 0);
+  pthread_mutex_destroy(&_m);
+  if (cct && logger) {
+    cct->get_perfcounters_collection()->remove(logger);
+    delete logger;
+  }
+}
+
+void Mutex::Lock(bool no_lockdep) {
+  if (lockdep && g_lockdep && !no_lockdep) _will_lock();
+
+  if (TryLock()) {
+    return;
+  }
+
+  utime_t start;
+  if (logger && cct && cct->_conf->mutex_perf_counter)
+    start = ceph_clock_now(cct);
+  int r = pthread_mutex_lock(&_m);
+  if (logger && cct && cct->_conf->mutex_perf_counter)
+    logger->finc(l_mutex_wait,
+                ceph_clock_now(cct) - start);
+  assert(r == 0);
+  if (lockdep && g_lockdep) _locked();
+  _post_lock();
+}

diff --git a/src/common/Mutex.h b/src/common/Mutex.h
index f781f1141226dbe276bdb0e919e176f0a03d9763..06e435d49df3a5ede1bbbd10937fbcda7c6fb7b5 100644 (file)
--- a/src/common/Mutex.h
+++ b/src/common/Mutex.h
@@ -17,11 +17,20 @@
 
 #include "include/assert.h"
 #include "lockdep.h"
+#include "common/ceph_context.h"
 
 #include <pthread.h>
 
 using namespace ceph;
 
+class PerfCounters;
+
+enum {
+  l_mutex_first = 999082,
+  l_mutex_wait,
+  l_mutex_last
+};
+
 class Mutex {
 private:
   const char *name;
@@ -33,6 +42,8 @@ private:
   pthread_mutex_t _m;
   int nlock;
   pthread_t locked_by;
+  CephContext *cct;
+  PerfCounters *logger;
 
   // don't allow copying.
   void operator=(Mutex &M) {}
@@ -52,45 +63,9 @@ private:
   }
 
 public:
-  Mutex(const char *n, bool r = false, bool ld=true, bool bt=false) :
-    name(n), id(-1), recursive(r), lockdep(ld), backtrace(bt), nlock(0), locked_by(0) {
-    if (recursive) {
-      // Mutexes of type PTHREAD_MUTEX_RECURSIVE do all the same checks as
-      // mutexes of type PTHREAD_MUTEX_ERRORCHECK.
-      pthread_mutexattr_t attr;
-      pthread_mutexattr_init(&attr);
-      pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
-      pthread_mutex_init(&_m,&attr);
-      pthread_mutexattr_destroy(&attr);
-      if (g_lockdep)
-       _register();
-    }
-    else if (lockdep) {
-      // If the mutex type is PTHREAD_MUTEX_ERRORCHECK, then error checking
-      // shall be provided. If a thread attempts to relock a mutex that it
-      // has already locked, an error shall be returned. If a thread
-      // attempts to unlock a mutex that it has not locked or a mutex which
-      // is unlocked, an error shall be returned.
-      pthread_mutexattr_t attr;
-      pthread_mutexattr_init(&attr);
-      pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK);
-      pthread_mutex_init(&_m, &attr);
-      if (g_lockdep)
-       _register();
-    }
-    else {
-      // If the mutex type is PTHREAD_MUTEX_NORMAL, deadlock detection
-      // shall not be provided. Attempting to relock the mutex causes
-      // deadlock. If a thread attempts to unlock a mutex that  it  has not
-      // locked or a mutex which is unlocked, undefined behavior results.
-      pthread_mutex_init(&_m, NULL);
-    }
-  }
-  ~Mutex() {
-    assert(nlock == 0);
-    pthread_mutex_destroy(&_m); 
-  }
-
+  Mutex(const char *n, bool r = false, bool ld=true, bool bt=false,
+       CephContext *cct = 0);
+  ~Mutex();
   bool is_locked() const {
     return (nlock > 0);
   }
@@ -107,13 +82,8 @@ public:
     return r == 0;
   }
 
-  void Lock(bool no_lockdep=false) {
-    if (lockdep && g_lockdep && !no_lockdep) _will_lock();
-    int r = pthread_mutex_lock(&_m);
-    assert(r == 0);
-    if (lockdep && g_lockdep) _locked();
-    _post_lock();
-  }
+  void Lock(bool no_lockdep=false);
+
   void _post_lock() {
     if (!recursive) {
       assert(nlock == 0);

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 8354d24107369f698f664df743e2cf794f22c7c3..06b3031d18032fe71836bbd6a371a10bce803289 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -453,6 +453,8 @@ OPTION(rgw_gc_obj_min_wait, OPT_INT, 2 * 3600)    // wait time before object may
 OPTION(rgw_gc_processor_max_time, OPT_INT, 3600)  // total run time for a single gc processor work
 OPTION(rgw_gc_processor_period, OPT_INT, 3600)  // gc processor cycle time
 
+OPTION(mutex_perf_counter, OPT_BOOL, false) // enable/disable mutex perf counter
+
 // This will be set to true when it is safe to start threads.
 // Once it is true, it will never change.
 OPTION(internal_safe_to_start_threads, OPT_BOOL, false)