From 8845749bf0d9542fa8d6ffa1f90c4eeb039cbcd0 Mon Sep 17 00:00:00 2001 From: Colin Patrick McCabe Date: Wed, 6 Jul 2011 11:07:29 -0700 Subject: [PATCH] systest: add cross process semaphores Signed-off-by: Colin McCabe --- src/Makefile.am | 2 + src/test/system/cross_process_sem.cc | 86 ++++++++++++++++++++++++++ src/test/system/cross_process_sem.h | 35 +++++++++++ src/test/system/rados_list_parallel.cc | 83 +++++++++++++------------ 4 files changed, 167 insertions(+), 39 deletions(-) create mode 100644 src/test/system/cross_process_sem.cc create mode 100644 src/test/system/cross_process_sem.h diff --git a/src/Makefile.am b/src/Makefile.am index a74569d6c6b98..4b836749b77b5 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -377,6 +377,7 @@ endif ## System tests libsystest_la_SOURCES = \ + test/system/cross_process_sem.cc \ test/system/systest_runnable.cc \ test/system/systest_settings.cc libsystest_la_LIBADD = libglobal.la @@ -1162,6 +1163,7 @@ noinst_HEADERS = \ test/osd/RadosModel.h\ global/pidfile.h\ common/sync_filesystem.h \ + test/system/cross_process_sem.h \ test/system/systest_runnable.h \ test/system/systest_settings.h diff --git a/src/test/system/cross_process_sem.cc b/src/test/system/cross_process_sem.cc new file mode 100644 index 0000000000000..108ea21fc776d --- /dev/null +++ b/src/test/system/cross_process_sem.cc @@ -0,0 +1,86 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* +* Ceph - scalable distributed file system +* +* Copyright (C) 2011 New Dream Network +* +* This is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License version 2.1, as published by the Free Software +* Foundation. See file COPYING. +* +*/ + +#include "cross_process_sem.h" + +#include +#include +#include +#include + +/* We put our cross-process semaphore into a page of memory mapped with mmap. */ +struct cross_process_sem_data_t +{ + sem_t sem; +}; + +/* A factory function is a good choice here because we want to be able to + * return an error code. It does force heap allocation, but that is the + * easiest way to use synchronization primitives anyway. Most programmers don't + * care about destroying semaphores before the process finishes. It's pretty + * difficult to get it right and there is usually no benefit. + */ +int CrossProcessSem:: +create(int initial_val, CrossProcessSem** res) +{ + struct cross_process_sem_data_t *data = static_cast < cross_process_sem_data_t*> ( + mmap(NULL, sizeof(struct cross_process_sem_data_t), + PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, 0, 0)); + if (data == MAP_FAILED) { + int err = errno; + return err; + } + int ret = sem_init(&data->sem, 1, initial_val); + if (ret) { + return ret; + } + *res = new CrossProcessSem(data); + return 0; +} + +CrossProcessSem:: +~CrossProcessSem() +{ + munmap(m_data, sizeof(struct cross_process_sem_data_t)); + m_data = NULL; +} + +void CrossProcessSem:: +wait() +{ + while(true) { + int ret = sem_wait(&m_data->sem); + if (ret == 0) + return; + int err = errno; + if (err == -EINTR) + continue; + abort(); + } +} + +void CrossProcessSem:: +post() +{ + int ret = sem_post(&m_data->sem); + if (ret == -1) { + abort(); + } +} + +CrossProcessSem:: +CrossProcessSem(struct cross_process_sem_data_t *data) + : m_data(data) +{ +} diff --git a/src/test/system/cross_process_sem.h b/src/test/system/cross_process_sem.h new file mode 100644 index 0000000000000..3f45f6b68800d --- /dev/null +++ b/src/test/system/cross_process_sem.h @@ -0,0 +1,35 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* +* Ceph - scalable distributed file system +* +* Copyright (C) 2011 New Dream Network +* +* This is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License version 2.1, as published by the Free Software +* Foundation. See file COPYING. +* +*/ + +struct cross_process_sem_data_t; + +class CrossProcessSem +{ +public: + static int create(int initial_val, CrossProcessSem** ret); + ~CrossProcessSem(); + + /* Initialize the semaphore. Must be called before any operations */ + int init(); + + /* Semaphore wait */ + void wait(); + + /* Semaphore post */ + void post(); + +private: + CrossProcessSem(struct cross_process_sem_data_t *data); + struct cross_process_sem_data_t *m_data; +}; diff --git a/src/test/system/rados_list_parallel.cc b/src/test/system/rados_list_parallel.cc index 40264f25804cd..78af6ae84a015 100644 --- a/src/test/system/rados_list_parallel.cc +++ b/src/test/system/rados_list_parallel.cc @@ -3,7 +3,7 @@ /* * Ceph - scalable distributed file system * -* Copyright (C) 2004-2006 Sage Weil +* Copyright (C) 2011 New Dream Network * * This is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -14,6 +14,7 @@ #include "include/rados/librados.h" #include "systest_runnable.h" +#include "cross_process_sem.h" #include #include @@ -43,8 +44,8 @@ static std::string get_random_buf(void) return oss.str(); } -sem_t pool_setup_sem; -sem_t modify_sem; +static CrossProcessSem *pool_setup_sem = NULL; +static CrossProcessSem *modify_sem = NULL; class RadosCreateBigPoolR : public SysTestRunnable { @@ -87,8 +88,8 @@ public: } } printf("%s: finishing.\n", get_id_str()); - sem_post(&pool_setup_sem); - sem_post(&pool_setup_sem); + pool_setup_sem->post(); + pool_setup_sem->post(); rados_ioctx_destroy(cl); return 0; } @@ -112,39 +113,39 @@ public: RETURN_IF_NONZERO(rados_create(&cl, NULL)); RETURN_IF_NONZERO(rados_conf_read_file(cl, NULL)); RETURN_IF_NONZERO(rados_connect(cl)); - sem_wait(&pool_setup_sem); + pool_setup_sem->wait(); rados_ioctx_t io_ctx; RETURN_IF_NONZERO(rados_ioctx_create(cl, "foo", &io_ctx)); - int ret, saw = 0; - const char *obj_name; - char tmp[RLP_OBJECT_SZ_MAX]; - rados_list_ctx_t h; - RETURN_IF_NONZERO(rados_objects_list_open(io_ctx, &h)); - printf("%s: listing objects.\n", get_id_str()); - while (true) { - ret = rados_objects_list_next(h, &obj_name); - if (ret == -ENOENT) { - break; - } - else if (ret != 0) { - printf("%s: rados_objects_list_next error: %d\n", get_id_str(), ret); - return ret; - } - printf("%s: listed an object!\n", get_id_str()); - int len = strlen(obj_name); - if (len > RLP_OBJECT_SZ_MAX) - len = RLP_OBJECT_SZ_MAX; - memcpy(tmp, obj_name, strlen(obj_name)); - printf("%s: listing object '%s'\n", get_id_str(), obj_name); - ++saw; - if (saw == RLP_NUM_OBJECTS / 2) - sem_wait(&modify_sem); - } - rados_objects_list_close(h); - - printf("%s: saw %d objects\n", get_id_str(), saw); +// int ret, saw = 0; +// const char *obj_name; +// char tmp[RLP_OBJECT_SZ_MAX]; +// rados_list_ctx_t h; +// RETURN_IF_NONZERO(rados_objects_list_open(io_ctx, &h)); +// printf("%s: listing objects.\n", get_id_str()); +// while (true) { +// ret = rados_objects_list_next(h, &obj_name); +// if (ret == -ENOENT) { +// break; +// } +// else if (ret != 0) { +// printf("%s: rados_objects_list_next error: %d\n", get_id_str(), ret); +// return ret; +// } +// printf("%s: listed an object!\n", get_id_str()); +// int len = strlen(obj_name); +// if (len > RLP_OBJECT_SZ_MAX) +// len = RLP_OBJECT_SZ_MAX; +// memcpy(tmp, obj_name, strlen(obj_name)); +// printf("%s: listing object '%s'\n", get_id_str(), obj_name); +// ++saw; +//// if (saw == RLP_NUM_OBJECTS / 2) +//// modify_sem->wait(); +// } +// rados_objects_list_close(h); + + //printf("%s: saw %d objects\n", get_id_str(), saw); rados_ioctx_destroy(cl); @@ -171,7 +172,7 @@ public: RETURN_IF_NONZERO(rados_create(&cl, NULL)); RETURN_IF_NONZERO(rados_conf_read_file(cl, NULL)); RETURN_IF_NONZERO(rados_connect(cl)); - sem_wait(&pool_setup_sem); + pool_setup_sem->wait(); rados_ioctx_t io_ctx; RETURN_IF_NONZERO(rados_ioctx_create(cl, "foo", &io_ctx)); @@ -197,7 +198,7 @@ public: } ++removed; if (removed == RLP_NUM_OBJECTS / 2) - sem_post(&modify_sem); + modify_sem->post(); } printf("%s: removed %d objects\n", get_id_str(), removed); @@ -208,11 +209,15 @@ public: } }; +const char *get_id_str() +{ + return "main"; +} + int main(int argc, const char **argv) { - sem_init(&pool_setup_sem, 1, 0); - //sem_init(&modify_sem, 1, 0); - sem_init(&modify_sem, 1, 1); + RETURN_IF_NONZERO(CrossProcessSem::create(0, &pool_setup_sem)); + RETURN_IF_NONZERO(CrossProcessSem::create(1, &modify_sem)); RadosCreateBigPoolR r1; RadosListObjectsR r2; -- 2.39.5