From: Noah Watkins Date: Sun, 21 Jul 2013 01:41:40 +0000 (-0700) Subject: spinlock: add generic spinlock implementation X-Git-Tag: v0.75~19^2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=1fec818f7cb7d70af3800643dbf8164ac8efa45a;p=ceph.git spinlock: add generic spinlock implementation Adds a ceph_spinlock_t implementation that will use pthread_spinlock_t if available, and otherwise reverts to pthread_mutex_t. Note that this spinlock is not intended to be used in process-shared memory. Switches implementation in: ceph_context SimpleMessenger atomic_t Only ceph_context initialized its spinlock with PTHREAD_PROCESS_SHARED. However, there does not appear to be any instance in which CephContext is allocated in shared memory, and thus can use the default private memory space behavior. Signed-off-by: Noah Watkins --- diff --git a/configure.ac b/configure.ac index 3520acf121320..8ed663dc1b83b 100644 --- a/configure.ac +++ b/configure.ac @@ -664,6 +664,18 @@ AC_DEFINE([HAVE_FDATASYNC], 1, [Define to 1 if you have fdatasync.]) AC_MSG_RESULT([no]) ]) +# +# Check for pthread spinlock (depends on ACX_PTHREAD) +# +saved_LIBS="$LIBS" +saved_CFLAGS="$CFLAGS" +LIBS="$PTHREAD_LIBS $LIBS" +CFLAGS="$PTHREAD_CFLAGS $CFLAGS" +AC_CHECK_FUNC([pthread_spin_init], + [AC_DEFINE(HAVE_PTHREAD_SPINLOCK, 1, [Define if you have pthread_spin_init])]) +LIBS="$saved_LIBS" +CFLAGS="$saved_CFLAGS" + # Checks for typedefs, structures, and compiler characteristics. #AC_HEADER_STDBOOL #AC_C_CONST diff --git a/src/common/ceph_context.cc b/src/common/ceph_context.cc index 3116abc3ff6d2..f7c2df5537e7c 100644 --- a/src/common/ceph_context.cc +++ b/src/common/ceph_context.cc @@ -33,6 +33,8 @@ #include #include +#include "include/Spinlock.h" + using ceph::HeartbeatMap; class CephContextServiceThread : public Thread @@ -260,7 +262,7 @@ CephContext::CephContext(uint32_t module_type_) _crypto_none(NULL), _crypto_aes(NULL) { - pthread_spin_init(&_service_thread_lock, PTHREAD_PROCESS_SHARED); + ceph_spin_init(&_service_thread_lock); _log = new ceph::log::Log(&_conf->subsys); _log->start(); @@ -330,7 +332,7 @@ CephContext::~CephContext() _log = NULL; delete _conf; - pthread_spin_destroy(&_service_thread_lock); + ceph_spin_destroy(&_service_thread_lock); delete _crypto_none; delete _crypto_aes; @@ -338,14 +340,14 @@ CephContext::~CephContext() void CephContext::start_service_thread() { - pthread_spin_lock(&_service_thread_lock); + ceph_spin_lock(&_service_thread_lock); if (_service_thread) { - pthread_spin_unlock(&_service_thread_lock); + ceph_spin_unlock(&_service_thread_lock); return; } _service_thread = new CephContextServiceThread(this); _service_thread->create(); - pthread_spin_unlock(&_service_thread_lock); + ceph_spin_unlock(&_service_thread_lock); // make logs flush on_exit() if (_conf->log_flush_on_exit) @@ -363,22 +365,22 @@ void CephContext::start_service_thread() void CephContext::reopen_logs() { - pthread_spin_lock(&_service_thread_lock); + ceph_spin_lock(&_service_thread_lock); if (_service_thread) _service_thread->reopen_logs(); - pthread_spin_unlock(&_service_thread_lock); + ceph_spin_unlock(&_service_thread_lock); } void CephContext::join_service_thread() { - pthread_spin_lock(&_service_thread_lock); + ceph_spin_lock(&_service_thread_lock); CephContextServiceThread *thread = _service_thread; if (!thread) { - pthread_spin_unlock(&_service_thread_lock); + ceph_spin_unlock(&_service_thread_lock); return; } _service_thread = NULL; - pthread_spin_unlock(&_service_thread_lock); + ceph_spin_unlock(&_service_thread_lock); thread->exit_thread(); thread->join(); diff --git a/src/common/ceph_context.h b/src/common/ceph_context.h index 08efeceaa674b..ba6062032e7df 100644 --- a/src/common/ceph_context.h +++ b/src/common/ceph_context.h @@ -21,6 +21,7 @@ #include "include/buffer.h" #include "include/atomic.h" #include "common/cmdparse.h" +#include "include/Spinlock.h" class AdminSocket; class CephContextServiceThread; @@ -126,7 +127,7 @@ private: AdminSocket *_admin_socket; /* lock which protects service thread creation, destruction, etc. */ - pthread_spinlock_t _service_thread_lock; + ceph_spinlock_t _service_thread_lock; /* The collection of profiling loggers associated with this context */ PerfCountersCollection *_perf_counters_collection; diff --git a/src/include/Spinlock.h b/src/include/Spinlock.h index 6154ae1124b5a..d0584c5953dee 100644 --- a/src/include/Spinlock.h +++ b/src/include/Spinlock.h @@ -16,17 +16,73 @@ #ifndef CEPH_SPINLOCK_H #define CEPH_SPINLOCK_H +#include "acconfig.h" + #include +typedef struct { +#ifdef HAVE_PTHREAD_SPINLOCK + pthread_spinlock_t lock; +#else + pthread_mutex_t lock; +#endif +} ceph_spinlock_t; + +#ifdef HAVE_PTHREAD_SPINLOCK + +static inline int ceph_spin_init(ceph_spinlock_t *l) +{ + return pthread_spin_init(&l->lock, PTHREAD_PROCESS_PRIVATE); +} + +static inline int ceph_spin_destroy(ceph_spinlock_t *l) +{ + return pthread_spin_destroy(&l->lock); +} + +static inline int ceph_spin_lock(ceph_spinlock_t *l) +{ + return pthread_spin_lock(&l->lock); +} + +static inline int ceph_spin_unlock(ceph_spinlock_t *l) +{ + return pthread_spin_unlock(&l->lock); +} + +#else /* !HAVE_PTHREAD_SPINLOCK */ + +static inline int ceph_spin_init(ceph_spinlock_t *l) +{ + return pthread_mutex_init(&l->lock, NULL); +} + +static inline int ceph_spin_destroy(ceph_spinlock_t *l) +{ + return pthread_mutex_destroy(&l->lock); +} + +static inline int ceph_spin_lock(ceph_spinlock_t *l) +{ + return pthread_mutex_lock(&l->lock); +} + +static inline int ceph_spin_unlock(ceph_spinlock_t *l) +{ + return pthread_mutex_unlock(&l->lock); +} + +#endif + class Spinlock { - mutable pthread_spinlock_t _lock; + mutable ceph_spinlock_t _lock; public: Spinlock() { - pthread_spin_init(&_lock, PTHREAD_PROCESS_PRIVATE); + ceph_spin_init(&_lock); } ~Spinlock() { - pthread_spin_destroy(&_lock); + ceph_spin_destroy(&_lock); } // don't allow copying. @@ -35,11 +91,11 @@ public: /// acquire spinlock void lock() const { - pthread_spin_lock(&_lock); + ceph_spin_lock(&_lock); } /// release spinlock void unlock() const { - pthread_spin_unlock(&_lock); + ceph_spin_unlock(&_lock); } class Locker { diff --git a/src/include/atomic.h b/src/include/atomic.h index 3ecbd28737634..537fa989cd3f6 100644 --- a/src/include/atomic.h +++ b/src/include/atomic.h @@ -67,53 +67,52 @@ namespace ceph { /* * crappy slow implementation that uses a pthreads spinlock. */ -#include -#include "include/assert.h" +#include "include/Spinlock.h" namespace ceph { class atomic_t { - mutable pthread_spinlock_t lock; + mutable ceph_spinlock_t lock; signed long val; public: atomic_t(int i=0) : val(i) { - pthread_spin_init(&lock, PTHREAD_PROCESS_PRIVATE); + ceph_spin_init(&lock); } ~atomic_t() { - pthread_spin_destroy(&lock); + ceph_spin_destroy(&lock); } void set(size_t v) { - pthread_spin_lock(&lock); + ceph_spin_lock(&lock); val = v; - pthread_spin_unlock(&lock); + ceph_spin_unlock(&lock); } int inc() { - pthread_spin_lock(&lock); + ceph_spin_lock(&lock); int r = ++val; - pthread_spin_unlock(&lock); + ceph_spin_unlock(&lock); return r; } int dec() { - pthread_spin_lock(&lock); + ceph_spin_lock(&lock); int r = --val; - pthread_spin_unlock(&lock); + ceph_spin_unlock(&lock); return r; } void add(int d) { - pthread_spin_lock(&lock); + ceph_spin_lock(&lock); val += d; - pthread_spin_unlock(&lock); + ceph_spin_unlock(&lock); } void sub(int d) { - pthread_spin_lock(&lock); + ceph_spin_lock(&lock); val -= d; - pthread_spin_unlock(&lock); + ceph_spin_unlock(&lock); } int read() const { signed long ret; - pthread_spin_lock(&lock); + ceph_spin_lock(&lock); ret = val; - pthread_spin_unlock(&lock); + ceph_spin_unlock(&lock); return ret; } private: diff --git a/src/msg/SimpleMessenger.cc b/src/msg/SimpleMessenger.cc index 441ed432af00b..a80b39c71ea85 100644 --- a/src/msg/SimpleMessenger.cc +++ b/src/msg/SimpleMessenger.cc @@ -23,6 +23,7 @@ #include "common/Timer.h" #include "common/errno.h" #include "auth/Crypto.h" +#include "include/Spinlock.h" #define dout_subsys ceph_subsys_ms #undef dout_prefix @@ -53,7 +54,7 @@ SimpleMessenger::SimpleMessenger(CephContext *cct, entity_name_t name, timeout(0), local_connection(new Connection(this)) { - pthread_spin_init(&global_seq_lock, PTHREAD_PROCESS_PRIVATE); + ceph_spin_init(&global_seq_lock); init_local_connection(); } diff --git a/src/msg/SimpleMessenger.h b/src/msg/SimpleMessenger.h index 6860c6c21a3a7..6c29999344959 100644 --- a/src/msg/SimpleMessenger.h +++ b/src/msg/SimpleMessenger.h @@ -38,6 +38,7 @@ using namespace __gnu_cxx; #include "Pipe.h" #include "Accepter.h" +#include "include/Spinlock.h" /* * This class handles transmission and reception of messages. Generally @@ -308,7 +309,7 @@ private: /// counter for the global seq our connection protocol uses __u32 global_seq; /// lock to protect the global_seq - pthread_spinlock_t global_seq_lock; + ceph_spinlock_t global_seq_lock; /** * hash map of addresses to Pipes @@ -388,11 +389,11 @@ public: * @return a global sequence ID that nobody else has seen. */ __u32 get_global_seq(__u32 old=0) { - pthread_spin_lock(&global_seq_lock); + ceph_spin_lock(&global_seq_lock); if (old > global_seq) global_seq = old; __u32 ret = ++global_seq; - pthread_spin_unlock(&global_seq_lock); + ceph_spin_unlock(&global_seq_lock); return ret; } /**