From: Samuel Just Date: Thu, 11 Oct 2012 01:20:31 +0000 (-0700) Subject: Mutex: Instrument Mutex with perfcouter for Lock() wait X-Git-Tag: v0.55~130^2~41^2~19 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=fe2814e4f6af44ee209ee47c43f18c223f760688;p=ceph.git Mutex: Instrument Mutex with perfcouter for Lock() wait Signed-off-by: Samuel Just --- diff --git a/src/Makefile.am b/src/Makefile.am index a0c2c783ce218..a72df3863d226 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1163,6 +1163,7 @@ libcommon_files = \ common/PrebufferedStreambuf.cc \ common/BackTrace.cc \ common/perf_counters.cc \ + common/Mutex.cc \ common/admin_socket.cc \ common/admin_socket_client.cc \ common/escape.c \ diff --git a/src/common/Mutex.cc b/src/common/Mutex.cc new file mode 100644 index 0000000000000..2dc035a9479d5 --- /dev/null +++ b/src/common/Mutex.cc @@ -0,0 +1,97 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2006 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ +using namespace std; +#include + +#include "common/Mutex.h" +#include "common/perf_counters.h" +#include "common/ceph_context.h" +#include "common/config.h" +#include "include/utime.h" +#include "common/Clock.h" + +Mutex::Mutex(const char *n, bool r, bool ld, + bool bt, + CephContext *cct) : + name(n), id(-1), recursive(r), lockdep(ld), backtrace(bt), + nlock(0), locked_by(0), cct(cct), logger(0) +{ + if (cct) { + PerfCountersBuilder b(cct, string("mutex-") + name, + l_mutex_first, l_mutex_last); + b.add_fl_avg(l_mutex_wait, "wait"); + logger = b.create_perf_counters(); + cct->get_perfcounters_collection()->add(logger); + logger->set(l_mutex_wait, 0); + } + if (recursive) { + // Mutexes of type PTHREAD_MUTEX_RECURSIVE do all the same checks as + // mutexes of type PTHREAD_MUTEX_ERRORCHECK. + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&_m,&attr); + pthread_mutexattr_destroy(&attr); + if (g_lockdep) + _register(); + } + else if (lockdep) { + // If the mutex type is PTHREAD_MUTEX_ERRORCHECK, then error checking + // shall be provided. If a thread attempts to relock a mutex that it + // has already locked, an error shall be returned. If a thread + // attempts to unlock a mutex that it has not locked or a mutex which + // is unlocked, an error shall be returned. + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK); + pthread_mutex_init(&_m, &attr); + if (g_lockdep) + _register(); + } + else { + // If the mutex type is PTHREAD_MUTEX_NORMAL, deadlock detection + // shall not be provided. Attempting to relock the mutex causes + // deadlock. If a thread attempts to unlock a mutex that it has not + // locked or a mutex which is unlocked, undefined behavior results. + pthread_mutex_init(&_m, NULL); + } +} + +Mutex::~Mutex() { + assert(nlock == 0); + pthread_mutex_destroy(&_m); + if (cct && logger) { + cct->get_perfcounters_collection()->remove(logger); + delete logger; + } +} + +void Mutex::Lock(bool no_lockdep) { + if (lockdep && g_lockdep && !no_lockdep) _will_lock(); + + if (TryLock()) { + return; + } + + utime_t start; + if (logger && cct && cct->_conf->mutex_perf_counter) + start = ceph_clock_now(cct); + int r = pthread_mutex_lock(&_m); + if (logger && cct && cct->_conf->mutex_perf_counter) + logger->finc(l_mutex_wait, + ceph_clock_now(cct) - start); + assert(r == 0); + if (lockdep && g_lockdep) _locked(); + _post_lock(); +} diff --git a/src/common/Mutex.h b/src/common/Mutex.h index f781f1141226d..06e435d49df3a 100644 --- a/src/common/Mutex.h +++ b/src/common/Mutex.h @@ -17,11 +17,20 @@ #include "include/assert.h" #include "lockdep.h" +#include "common/ceph_context.h" #include using namespace ceph; +class PerfCounters; + +enum { + l_mutex_first = 999082, + l_mutex_wait, + l_mutex_last +}; + class Mutex { private: const char *name; @@ -33,6 +42,8 @@ private: pthread_mutex_t _m; int nlock; pthread_t locked_by; + CephContext *cct; + PerfCounters *logger; // don't allow copying. void operator=(Mutex &M) {} @@ -52,45 +63,9 @@ private: } public: - Mutex(const char *n, bool r = false, bool ld=true, bool bt=false) : - name(n), id(-1), recursive(r), lockdep(ld), backtrace(bt), nlock(0), locked_by(0) { - if (recursive) { - // Mutexes of type PTHREAD_MUTEX_RECURSIVE do all the same checks as - // mutexes of type PTHREAD_MUTEX_ERRORCHECK. - pthread_mutexattr_t attr; - pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); - pthread_mutex_init(&_m,&attr); - pthread_mutexattr_destroy(&attr); - if (g_lockdep) - _register(); - } - else if (lockdep) { - // If the mutex type is PTHREAD_MUTEX_ERRORCHECK, then error checking - // shall be provided. If a thread attempts to relock a mutex that it - // has already locked, an error shall be returned. If a thread - // attempts to unlock a mutex that it has not locked or a mutex which - // is unlocked, an error shall be returned. - pthread_mutexattr_t attr; - pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK); - pthread_mutex_init(&_m, &attr); - if (g_lockdep) - _register(); - } - else { - // If the mutex type is PTHREAD_MUTEX_NORMAL, deadlock detection - // shall not be provided. Attempting to relock the mutex causes - // deadlock. If a thread attempts to unlock a mutex that it has not - // locked or a mutex which is unlocked, undefined behavior results. - pthread_mutex_init(&_m, NULL); - } - } - ~Mutex() { - assert(nlock == 0); - pthread_mutex_destroy(&_m); - } - + Mutex(const char *n, bool r = false, bool ld=true, bool bt=false, + CephContext *cct = 0); + ~Mutex(); bool is_locked() const { return (nlock > 0); } @@ -107,13 +82,8 @@ public: return r == 0; } - void Lock(bool no_lockdep=false) { - if (lockdep && g_lockdep && !no_lockdep) _will_lock(); - int r = pthread_mutex_lock(&_m); - assert(r == 0); - if (lockdep && g_lockdep) _locked(); - _post_lock(); - } + void Lock(bool no_lockdep=false); + void _post_lock() { if (!recursive) { assert(nlock == 0); diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 8354d24107369..06b3031d18032 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -453,6 +453,8 @@ OPTION(rgw_gc_obj_min_wait, OPT_INT, 2 * 3600) // wait time before object may OPTION(rgw_gc_processor_max_time, OPT_INT, 3600) // total run time for a single gc processor work OPTION(rgw_gc_processor_period, OPT_INT, 3600) // gc processor cycle time +OPTION(mutex_perf_counter, OPT_BOOL, false) // enable/disable mutex perf counter + // This will be set to true when it is safe to start threads. // Once it is true, it will never change. OPTION(internal_safe_to_start_threads, OPT_BOOL, false)