From: Sage Weil Date: Fri, 4 Dec 2009 18:03:49 +0000 (-0800) Subject: rados: move librados.h, rados_bencher.h X-Git-Tag: v0.18~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=a08d9619d5e19a39eaf425345a459d0e1f8f64aa;p=ceph.git rados: move librados.h, rados_bencher.h Also include rados_bencher.h in Makefile.am. --- diff --git a/src/Makefile.am b/src/Makefile.am index 2f3cd824d1f3..77b7c15018f0 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -269,7 +269,7 @@ install-data-local: mkdir -p $(DESTDIR)$(includedir)/ceph $(install_sh_SCRIPT) -m 0644 client/libceph.h $(DESTDIR)$(includedir)/ceph/libceph.h mkdir -p $(DESTDIR)$(includedir)/rados - $(install_sh_SCRIPT) -m 0644 include/librados.h $(DESTDIR)$(includedir)/rados/librados.h + $(install_sh_SCRIPT) -m 0644 osdc/librados.h $(DESTDIR)$(includedir)/rados/librados.h mkdir -p $(DESTDIR)$(includedir)/crush $(install_sh_SCRIPT) -m 0644 crush/hash.h $(DESTDIR)$(includedir)/crush/hash.h $(install_sh_SCRIPT) -m 0644 crush/crush.h $(DESTDIR)$(includedir)/crush/crush.h @@ -507,7 +507,6 @@ noinst_HEADERS = \ include/intarith.h\ include/interval_set.h\ include/inttypes.h\ - include/librados.h\ include/lru.h\ include/msgr.h\ include/nstring.h\ @@ -709,6 +708,8 @@ noinst_HEADERS = \ osd/RAID4PG.h\ osd/ReplicatedPG.h\ osd/osd_types.h\ + osdc/librados.h\ + osdc/rados_bencher.h\ osdc/Blinker.h\ osdc/Filer.h\ osdc/Journaler.h\ diff --git a/src/include/librados.h b/src/include/librados.h deleted file mode 100644 index ecce9ec04e98..000000000000 --- a/src/include/librados.h +++ /dev/null @@ -1,161 +0,0 @@ -#ifndef __LIBRADOS_H -#define __LIBRADOS_H - -#ifdef __cplusplus - -#include "include/types.h" - -extern "C" { -#endif - -#include -#include -#include -#include - -#include "msgr.h" -#include "rados.h" - -/* initialization */ -int rados_initialize(int argc, const char **argv); /* arguments are optional */ -void rados_deinitialize(); - -typedef void *rados_list_ctx_t; - -/* pools */ -typedef void *rados_pool_t; -typedef long long unsigned rados_snap_t; - -struct rados_pool_stat_t { - long long unsigned num_bytes; // in bytes - long long unsigned num_kb; // in KB - long long unsigned num_objects; - long long unsigned num_object_clones; - long long unsigned num_object_copies; // num_objects * num_replicas - long long unsigned num_objects_missing_on_primary; - long long unsigned num_objects_degraded; - long long unsigned num_rd, num_rd_kb,num_wr, num_wr_kb; -}; - -struct rados_statfs_t { - __u64 kb, kb_used, kb_avail; - __u64 num_objects; -}; - -int rados_open_pool(const char *name, rados_pool_t *pool); -int rados_close_pool(rados_pool_t pool); -void rados_set_snap(rados_pool_t pool, rados_snap_t snap); - /* After creating a new rados_list_ctx_t, call this to initialize it*/ -void rados_pool_init_ctx(rados_list_ctx_t *ctx); - /* Once you've finished with a rados_list_ctx_t, call before you dump it*/ -void rados_pool_close_ctx(rados_list_ctx_t *ctx); - /* Given a rados_list_ctx_t and its pool, get the next object in sequence*/ -int rados_pool_list_next(rados_pool_t pool, const char **entry, rados_list_ctx_t *ctx); - -/* snapshots */ -int rados_snap_create(const rados_pool_t pool, const char *snapname); -int rados_snap_remove(const rados_pool_t pool, const char *snapname); -int rados_snap_list(rados_pool_t pool, rados_snap_t *snaps, int maxlen); -int rados_snap_get_name(rados_pool_t pool, rados_snap_t id, char *name, int maxlen); - -/* read/write objects */ -int rados_write(rados_pool_t pool, const char *oid, off_t off, const char *buf, size_t len); -int rados_read(rados_pool_t pool, const char *oid, off_t off, char *buf, size_t len); -int rados_remove(rados_pool_t pool, const char *oid); -int rados_getxattr(rados_pool_t pool, const char *o, const char *name, char *buf, size_t len); -int rados_setxattr(rados_pool_t pool, const char *o, const char *name, const char *buf, size_t len); -int rados_stat(rados_pool_t pool, const char *o, __u64 *psize, time_t *pmtime); -int rados_exec(rados_pool_t pool, const char *oid, const char *cls, const char *method, - const char *in_buf, size_t in_len, char *buf, size_t out_len); - -/* async io */ -typedef void *rados_completion_t; -typedef void (*rados_callback_t)(rados_completion_t cb, void *arg); - -int rados_aio_set_callback(rados_completion_t c, rados_callback_t, void *arg); -int rados_aio_wait_for_complete(rados_completion_t c); -int rados_aio_wait_for_safe(rados_completion_t c); -int rados_aio_is_complete(rados_completion_t c); -int rados_aio_is_safe(rados_completion_t c); -int rados_aio_get_return_value(rados_completion_t c); -void rados_aio_release(rados_completion_t c); - -int rados_aio_write(rados_pool_t pool, const char *oid, off_t off, const char *buf, size_t len, rados_completion_t *completion); -int rados_aio_read(rados_pool_t pool, const char *oid, off_t off, char *buf, size_t len, rados_completion_t *completion); - -#ifdef __cplusplus -} - -class RadosClient; - -class Rados -{ - RadosClient *client; -public: - Rados(); - ~Rados(); - int initialize(int argc, const char *argv[]); - void shutdown(); - - int open_pool(const char *name, rados_pool_t *pool); - int close_pool(rados_pool_t pool); - - void set_snap(rados_pool_t pool, snapid_t seq); - - int create(rados_pool_t pool, const object_t& oid, bool exclusive); - - int write(rados_pool_t pool, const object_t& oid, off_t off, bufferlist& bl, size_t len); - int write_full(rados_pool_t pool, const object_t& oid, bufferlist& bl); - int read(rados_pool_t pool, const object_t& oid, off_t off, bufferlist& bl, size_t len); - int remove(rados_pool_t pool, const object_t& oid); - - int getxattr(rados_pool_t pool, const object_t& oid, const char *name, bufferlist& bl); - int setxattr(rados_pool_t pool, const object_t& oid, const char *name, bufferlist& bl); - int getxattrs(rados_pool_t pool, const object_t& oid, map& attrset); - int stat(rados_pool_t pool, const object_t& oid, __u64 *psize, time_t *pmtime); - - int exec(rados_pool_t pool, const object_t& oid, const char *cls, const char *method, - bufferlist& inbl, bufferlist& outbl); - - struct ListCtx { - void *ctx; - ListCtx() : ctx(NULL) {} - }; - - int list(rados_pool_t pool, int max, std::list& entries, Rados::ListCtx& ctx); - int list_pools(std::vector& v); - int get_pool_stats(std::vector& v, - std::map& stats); - int get_fs_stats(rados_statfs_t& result); - - int create_pool(const char *name); - - int snap_create(const rados_pool_t pool, const char *snapname); - int snap_remove(const rados_pool_t pool, const char *snapname); - int snap_list(rados_pool_t pool, vector *snaps); - int snap_get_name(rados_pool_t pool, rados_snap_t snap, std::string *name); - int snap_get_stamp(rados_pool_t pool, rados_snap_t snap, time_t *t); - int snap_lookup(rados_pool_t, const char *snapname, rados_snap_t *snapid); - - // -- aio -- - struct AioCompletion { - void *pc; - AioCompletion(void *_pc) : pc(_pc) {} - int set_callback(rados_callback_t cb, void *cba); - int wait_for_complete(); - int wait_for_safe(); - bool is_complete(); - bool is_safe(); - int get_return_value(); - void release(); - }; - - int aio_read(rados_pool_t pool, const object_t& oid, off_t off, bufferlist *pbl, size_t len, - AioCompletion **pc); - int aio_write(rados_pool_t pool, const object_t& oid, off_t off, const bufferlist& bl, size_t len, - AioCompletion **pc); - -}; -#endif - -#endif diff --git a/src/librados.cc b/src/librados.cc index 4702e4de49d5..36a3f5e579ae 100644 --- a/src/librados.cc +++ b/src/librados.cc @@ -37,7 +37,7 @@ using namespace std; #include "osdc/Objecter.h" -#include "include/librados.h" +#include "osdc/librados.h" #define RADOS_LIST_MAX_ENTRIES 1024 #define DOUT_SUBSYS rados @@ -1343,8 +1343,6 @@ static int rados_initialized = 0; static RadosClient *radosp; -#include "include/librados.h" - extern "C" int rados_initialize(int argc, const char **argv) { int ret = 0; diff --git a/src/osdc/librados.h b/src/osdc/librados.h new file mode 100644 index 000000000000..ee998c7e9ae8 --- /dev/null +++ b/src/osdc/librados.h @@ -0,0 +1,161 @@ +#ifndef __LIBRADOS_H +#define __LIBRADOS_H + +#ifdef __cplusplus + +#include "include/types.h" + +extern "C" { +#endif + +#include +#include +#include +#include + +#include "include/msgr.h" +#include "include/rados.h" + +/* initialization */ +int rados_initialize(int argc, const char **argv); /* arguments are optional */ +void rados_deinitialize(); + +typedef void *rados_list_ctx_t; + +/* pools */ +typedef void *rados_pool_t; +typedef long long unsigned rados_snap_t; + +struct rados_pool_stat_t { + long long unsigned num_bytes; // in bytes + long long unsigned num_kb; // in KB + long long unsigned num_objects; + long long unsigned num_object_clones; + long long unsigned num_object_copies; // num_objects * num_replicas + long long unsigned num_objects_missing_on_primary; + long long unsigned num_objects_degraded; + long long unsigned num_rd, num_rd_kb,num_wr, num_wr_kb; +}; + +struct rados_statfs_t { + __u64 kb, kb_used, kb_avail; + __u64 num_objects; +}; + +int rados_open_pool(const char *name, rados_pool_t *pool); +int rados_close_pool(rados_pool_t pool); +void rados_set_snap(rados_pool_t pool, rados_snap_t snap); + /* After creating a new rados_list_ctx_t, call this to initialize it*/ +void rados_pool_init_ctx(rados_list_ctx_t *ctx); + /* Once you've finished with a rados_list_ctx_t, call before you dump it*/ +void rados_pool_close_ctx(rados_list_ctx_t *ctx); + /* Given a rados_list_ctx_t and its pool, get the next object in sequence*/ +int rados_pool_list_next(rados_pool_t pool, const char **entry, rados_list_ctx_t *ctx); + +/* snapshots */ +int rados_snap_create(const rados_pool_t pool, const char *snapname); +int rados_snap_remove(const rados_pool_t pool, const char *snapname); +int rados_snap_list(rados_pool_t pool, rados_snap_t *snaps, int maxlen); +int rados_snap_get_name(rados_pool_t pool, rados_snap_t id, char *name, int maxlen); + +/* read/write objects */ +int rados_write(rados_pool_t pool, const char *oid, off_t off, const char *buf, size_t len); +int rados_read(rados_pool_t pool, const char *oid, off_t off, char *buf, size_t len); +int rados_remove(rados_pool_t pool, const char *oid); +int rados_getxattr(rados_pool_t pool, const char *o, const char *name, char *buf, size_t len); +int rados_setxattr(rados_pool_t pool, const char *o, const char *name, const char *buf, size_t len); +int rados_stat(rados_pool_t pool, const char *o, __u64 *psize, time_t *pmtime); +int rados_exec(rados_pool_t pool, const char *oid, const char *cls, const char *method, + const char *in_buf, size_t in_len, char *buf, size_t out_len); + +/* async io */ +typedef void *rados_completion_t; +typedef void (*rados_callback_t)(rados_completion_t cb, void *arg); + +int rados_aio_set_callback(rados_completion_t c, rados_callback_t, void *arg); +int rados_aio_wait_for_complete(rados_completion_t c); +int rados_aio_wait_for_safe(rados_completion_t c); +int rados_aio_is_complete(rados_completion_t c); +int rados_aio_is_safe(rados_completion_t c); +int rados_aio_get_return_value(rados_completion_t c); +void rados_aio_release(rados_completion_t c); + +int rados_aio_write(rados_pool_t pool, const char *oid, off_t off, const char *buf, size_t len, rados_completion_t *completion); +int rados_aio_read(rados_pool_t pool, const char *oid, off_t off, char *buf, size_t len, rados_completion_t *completion); + +#ifdef __cplusplus +} + +class RadosClient; + +class Rados +{ + RadosClient *client; +public: + Rados(); + ~Rados(); + int initialize(int argc, const char *argv[]); + void shutdown(); + + int open_pool(const char *name, rados_pool_t *pool); + int close_pool(rados_pool_t pool); + + void set_snap(rados_pool_t pool, snapid_t seq); + + int create(rados_pool_t pool, const object_t& oid, bool exclusive); + + int write(rados_pool_t pool, const object_t& oid, off_t off, bufferlist& bl, size_t len); + int write_full(rados_pool_t pool, const object_t& oid, bufferlist& bl); + int read(rados_pool_t pool, const object_t& oid, off_t off, bufferlist& bl, size_t len); + int remove(rados_pool_t pool, const object_t& oid); + + int getxattr(rados_pool_t pool, const object_t& oid, const char *name, bufferlist& bl); + int setxattr(rados_pool_t pool, const object_t& oid, const char *name, bufferlist& bl); + int getxattrs(rados_pool_t pool, const object_t& oid, map& attrset); + int stat(rados_pool_t pool, const object_t& oid, __u64 *psize, time_t *pmtime); + + int exec(rados_pool_t pool, const object_t& oid, const char *cls, const char *method, + bufferlist& inbl, bufferlist& outbl); + + struct ListCtx { + void *ctx; + ListCtx() : ctx(NULL) {} + }; + + int list(rados_pool_t pool, int max, std::list& entries, Rados::ListCtx& ctx); + int list_pools(std::vector& v); + int get_pool_stats(std::vector& v, + std::map& stats); + int get_fs_stats(rados_statfs_t& result); + + int create_pool(const char *name); + + int snap_create(const rados_pool_t pool, const char *snapname); + int snap_remove(const rados_pool_t pool, const char *snapname); + int snap_list(rados_pool_t pool, vector *snaps); + int snap_get_name(rados_pool_t pool, rados_snap_t snap, std::string *name); + int snap_get_stamp(rados_pool_t pool, rados_snap_t snap, time_t *t); + int snap_lookup(rados_pool_t, const char *snapname, rados_snap_t *snapid); + + // -- aio -- + struct AioCompletion { + void *pc; + AioCompletion(void *_pc) : pc(_pc) {} + int set_callback(rados_callback_t cb, void *cba); + int wait_for_complete(); + int wait_for_safe(); + bool is_complete(); + bool is_safe(); + int get_return_value(); + void release(); + }; + + int aio_read(rados_pool_t pool, const object_t& oid, off_t off, bufferlist *pbl, size_t len, + AioCompletion **pc); + int aio_write(rados_pool_t pool, const object_t& oid, off_t off, const bufferlist& bl, size_t len, + AioCompletion **pc); + +}; +#endif + +#endif diff --git a/src/osdc/rados_bencher.h b/src/osdc/rados_bencher.h new file mode 100644 index 000000000000..bc85d8f4446e --- /dev/null +++ b/src/osdc/rados_bencher.h @@ -0,0 +1,463 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2009 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ +#include "osdc/librados.h" +#include "config.h" +#include "common/common_init.h" +#include "common/Cond.h" +#include +#include + +#include +#include +#include + +Mutex dataLock("data mutex"); + +struct bench_data { + bool done; //is the benchmark is done + int object_size; //the size of the objects + int trans_size; //size of the write/read to perform + // same as object_size for write tests + int in_flight; //number of reads/writes being waited on + int started; + int finished; + double min_latency; + double max_latency; + double avg_latency; + utime_t cur_latency; //latency of last completed transaction + utime_t start_time; //start time for benchmark + char *object_contents; //pointer to the contents written to each object +}; + +int write_bench(Rados& rados, rados_pool_t pool, + int secondsToRun, int concurrentios, bench_data *data); +int seq_read_bench(Rados& rados, rados_pool_t pool, + int concurrentios, bench_data *data, int verify); +void *status_printer(void * data_store); +void sanitize_object_contents(bench_data *data, int length); + +int aio_bench(Rados& rados, rados_pool_t pool, int secondsToRun, + int concurrentios, int writeSize, int sequentialTest) { + + char* contentsChars = new char[writeSize]; + int r = 0; + + dataLock.Lock(); + bench_data *data = new bench_data(); + data->done = false; + data->object_size = writeSize; + data->trans_size = writeSize; //just for now + data->in_flight = 0; + data->started = 0; + data->finished = 0; + data->min_latency = 9999.0; // this better be higher than initial latency! + data->max_latency = 0; + data->avg_latency = 0; + data->object_contents = contentsChars; + dataLock.Unlock(); + + + //fill in contentsChars deterministically so we can check returns + sanitize_object_contents(data, writeSize); + //set up the pool + cout << "open pool result = " << rados.open_pool("data",&pool) << " pool = " << pool << std::endl; + + r = write_bench(rados, pool, secondsToRun, concurrentios, data); + if (r != 0) goto out; + + //check objects for consistency if requested + if (sequentialTest) { + r = seq_read_bench(rados, pool, concurrentios, data, 1); + } + + out: + delete contentsChars; + delete data; + return r; +} + +int write_bench(Rados& rados, rados_pool_t pool, + int secondsToRun, int concurrentios, bench_data *data) { + cout << "Maintaining " << concurrentios << " concurrent writes of " + << data->object_size << " bytes for at least " + << secondsToRun << " seconds." << std::endl; + + Rados::AioCompletion* completions[concurrentios]; + char* name[concurrentios]; + bufferlist* contents[concurrentios]; + double total_latency = 0; + utime_t start_times[concurrentios]; + utime_t stopTime; + int r = 0; + + //set up writes so I can start them together + for (int i = 0; iobject_contents, data->object_size, "I'm the %dth object!", i); + contents[i]->append(data->object_contents, data->object_size); + } + + pthread_t print_thread; + + pthread_create(&print_thread, NULL, status_printer, (void *)data); + dataLock.Lock(); + data->start_time = g_clock.now(); + dataLock.Unlock(); + for (int i = 0; iobject_size, &completions[i]); + if (r < 0) { //naughty, doesn't clean up heap + dataLock.Unlock(); + return -5; //EIO + } + dataLock.Lock(); + ++data->started; + ++data->in_flight; + dataLock.Unlock(); + } + + //keep on adding new writes as old ones complete until we've passed minimum time + int slot; + bufferlist* newContents; + char* newName; + utime_t runtime; + + utime_t timePassed = g_clock.now() - data->start_time; + //don't need locking for reads because other thread doesn't write + + runtime.set_from_double(secondsToRun); + stopTime = data->start_time + runtime; + while( g_clock.now() < stopTime ) { + slot = data->finished % concurrentios; + //create new contents and name on the heap, and fill them + newContents = new bufferlist(); + newName = new char[128]; + snprintf(newName, 128, "Object %d", data->started); + snprintf(data->object_contents, data->object_size, "I'm the %dth object!", data->started); + newContents->append(data->object_contents, data->object_size); + completions[slot]->wait_for_safe(); + dataLock.Lock(); + r = completions[slot]->get_return_value(); + if (r != 0) { + dataLock.Unlock(); + return r; + } + data->cur_latency = g_clock.now() - start_times[slot]; + total_latency += data->cur_latency; + if( data->cur_latency > data->max_latency) data->max_latency = data->cur_latency; + if (data->cur_latency < data->min_latency) data->min_latency = data->cur_latency; + ++data->finished; + data->avg_latency = total_latency / data->finished; + --data->in_flight; + dataLock.Unlock(); + completions[slot]->release(); + timePassed = g_clock.now() - data->start_time; + + //write new stuff to rados, then delete old stuff + //and save locations of new stuff for later deletion + start_times[slot] = g_clock.now(); + r = rados.aio_write(pool, newName, 0, *newContents, data->object_size, &completions[slot]); + if (r < 0) //naughty; doesn't clean up heap space. + return r; + dataLock.Lock(); + ++data->started; + ++data->in_flight; + dataLock.Unlock(); + delete name[slot]; + delete contents[slot]; + name[slot] = newName; + contents[slot] = newContents; + } + + while (data->finished < data->started) { + slot = data->finished % concurrentios; + completions[slot]->wait_for_safe(); + dataLock.Lock(); + r = completions[slot]->get_return_value(); + if (r != 0) { + dataLock.Unlock(); + return r; + } + data->cur_latency = g_clock.now() - start_times[slot]; + total_latency += data->cur_latency; + if (data->cur_latency > data->max_latency) data->max_latency = data->cur_latency; + if (data->cur_latency < data->min_latency) data->min_latency = data->cur_latency; + ++data->finished; + data->avg_latency = total_latency / data->finished; + --data->in_flight; + dataLock.Unlock(); + completions[slot]-> release(); + delete name[slot]; + delete contents[slot]; + } + + timePassed = g_clock.now() - data->start_time; + dataLock.Lock(); + data->done = true; + dataLock.Unlock(); + + pthread_join(print_thread, NULL); + + double bandwidth; + bandwidth = ((double)data->finished)*((double)data->object_size)/(double)timePassed; + bandwidth = bandwidth/(1024*1024); // we want it in MB/sec + char bw[20]; + sprintf(bw, "%.3lf \n", bandwidth); + + cout << "Total time run: " << timePassed << std::endl + << "Total writes made: " << data->finished << std::endl + << "Write size: " << data->object_size << std::endl + << "Bandwidth (MB/sec): " << bw << std::endl + << "Average Latency: " << data->avg_latency << std::endl + << "Max latency: " << data->max_latency << std::endl + << "Min latency: " << data->min_latency << std::endl; + return 0; +} + +int seq_read_bench(Rados& rados, rados_pool_t pool, + int concurrentios, bench_data *write_data, int verify) { + bench_data *data = new bench_data(); + data->done = false; + data->object_size = write_data->object_size; + data->trans_size = data->object_size; + data->in_flight= 0; + data->started = 0; + data->finished = 0; + data->min_latency = 9999.0; + data->max_latency = 0; + data->avg_latency = 0; + data->object_contents = write_data->object_contents; + + Rados::AioCompletion* completions[concurrentios]; + char* name[concurrentios]; + bufferlist* contents[concurrentios]; + int errors = 0; + utime_t start_time; + utime_t start_times[concurrentios]; + double total_latency = 0; + int r = 0; + sanitize_object_contents(data, 128); //clean it up once; subsequent + //changes will be safe because string length monotonically increases + + //set up initial reads + for (int i = 0; i < concurrentios; ++i) { + name[i] = new char[128]; + snprintf(name[i], 128, "Object %d", i); + contents[i] = new bufferlist(); + } + + pthread_t print_thread; + pthread_create(&print_thread, NULL, status_printer, (void *)data); + + dataLock.Lock(); + data->start_time = g_clock.now(); + //start initial reads + for (int i = 0; i < concurrentios; ++i) { + start_times[i] = g_clock.now(); + r = rados.aio_read(pool, name[i], 0, contents[i], data->object_size, &completions[i]); + if (r < 0) { //naughty, doesn't clean up heap -- oh, or handle the print thread! + cerr << "r = " << r << std::endl; + dataLock.Unlock(); + return -5; //EIO + } + ++data->started; + ++data->in_flight; + } + dataLock.Unlock(); + + //keep on adding new reads as old ones complete + int slot; + char* newName; + utime_t runtime; + bufferlist *cur_contents; + + for (int i = 0; i < write_data->finished - concurrentios; ++i) { + slot = data->finished % concurrentios; + newName = new char[128]; + snprintf(newName, 128, "Object %d", data->started); + completions[slot]->wait_for_complete(); + dataLock.Lock(); + r = completions[slot]->get_return_value(); + if (r != 0) { + cerr << "read got " << r << std::endl; + dataLock.Unlock(); + return r; + } + data->cur_latency = g_clock.now() - start_times[slot]; + total_latency += data->cur_latency; + if( data->cur_latency > data->max_latency) data->max_latency = data->cur_latency; + if (data->cur_latency < data->min_latency) data->min_latency = data->cur_latency; + ++data->finished; + data->avg_latency = total_latency / data->finished; + --data->in_flight; + dataLock.Unlock(); + completions[slot]->release(); + cur_contents = contents[slot]; + + //start new read and check data if requested + start_times[slot] = g_clock.now(); + contents[slot] = new bufferlist(); + r = rados.aio_read(pool, newName, 0, contents[slot], data->object_size, &completions[slot]); + if (r < 0) + return r; + dataLock.Lock(); + ++data->started; + ++data->in_flight; + dataLock.Unlock(); + if (verify) { + dataLock.Lock(); + snprintf(data->object_contents, data->object_size, "I'm the %dth object!", i); + dataLock.Unlock(); + if (memcmp(data->object_contents, cur_contents->c_str(), data->object_size) != 0) { + cerr << name[slot] << " is not correct!"; + ++errors; + } + } + delete name[slot]; + name[slot] = newName; + delete cur_contents; + } + + //wait for final reads to complete + while (data->finished < data->started) { + slot = data->finished % concurrentios; + completions[slot]->wait_for_complete(); + dataLock.Lock(); + r = completions[slot]->get_return_value(); + if (r != 0) { + cerr << "read got " << r << std::endl; + dataLock.Unlock(); + return r; + } + data->cur_latency = g_clock.now() - start_times[slot]; + total_latency += data->cur_latency; + if (data->cur_latency > data->max_latency) data->max_latency = data->cur_latency; + if (data->cur_latency < data->min_latency) data->min_latency = data->cur_latency; + ++data->finished; + data->avg_latency = total_latency / data->finished; + --data->in_flight; + dataLock.Unlock(); + completions[slot]-> release(); + if (verify) { + dataLock.Lock(); + snprintf(data->object_contents, data->object_size, "I'm the %dth object!", data->finished-1); + dataLock.Unlock(); + if (memcmp(data->object_contents, contents[slot]->c_str(), data->object_size) != 0) { + cerr << name[slot] << " is not correct!" << std::endl; + ++errors; + } + } + delete name[slot]; + delete contents[slot]; + } + + runtime = g_clock.now() - data->start_time; + dataLock.Lock(); + data->done = true; + dataLock.Unlock(); + + pthread_join(print_thread, NULL); + + double bandwidth; + bandwidth = ((double)data->finished)*((double)data->object_size)/(double)runtime; + bandwidth = bandwidth/(1024*1024); // we want it in MB/sec + char bw[20]; + sprintf(bw, "%.3lf \n", bandwidth); + + cout << "Total time run: " << runtime << std::endl + << "Total reads made: " << data->finished << std::endl + << "Read size: " << data->object_size << std::endl + << "Bandwidth (MB/sec): " << bw << std::endl + << "Average Latency: " << data->avg_latency << std::endl + << "Max latency: " << data->max_latency << std::endl + << "Min latency: " << data->min_latency << std::endl; + + delete data; + return 0; +} + + + +void *status_printer(void * data_store) { + bench_data *data = (bench_data *) data_store; + Cond cond; + int i = 0; + int previous_writes = 0; + int cycleSinceChange = 0; + double avg_bandwidth; + double bandwidth; + utime_t ONE_SECOND; + ONE_SECOND.set_from_double(1.0); + dataLock.Lock(); + while(!data->done) { + if (i % 20 == 0) { + if (i > 0) + cout << "min lat: " << data->min_latency + << " max lat: " << data->max_latency + << " avg lat: " << data->avg_latency << std::endl; + //I'm naughty and don't reset the fill + cout << setfill(' ') + << setw(5) << "sec" + << setw(8) << "Cur ops" + << setw(10) << "started" + << setw(10) << "finished" + << setw(10) << "avg MB/s" + << setw(10) << "cur MB/s" + << setw(10) << "last lat" + << setw(10) << "avg lat" << std::endl; + } + bandwidth = (double)(data->finished - previous_writes) + * (data->trans_size) + / (1024*1024) + / cycleSinceChange; + avg_bandwidth = (double) (data->trans_size) * (data->finished) + / (double)(g_clock.now() - data->start_time) / (1024*1024); + if (previous_writes != data->finished) { + previous_writes = data->finished; + cycleSinceChange = 0; + cout << setfill(' ') + << setw(5) << i + << setw(8) << data->in_flight + << setw(10) << data->started + << setw(10) << data->finished + << setw(10) << avg_bandwidth + << setw(10) << bandwidth + << setw(10) << (double)data->cur_latency + << setw(10) << data->avg_latency << std::endl; + } + else { + cout << setfill(' ') + << setw(5) << i + << setw(8) << data->in_flight + << setw(10) << data->started + << setw(10) << data->finished + << setw(10) << avg_bandwidth + << setw(10) << '0' + << setw(10) << '-' + << setw(10) << data->avg_latency << std::endl; + } + ++i; + ++cycleSinceChange; + cond.WaitInterval(dataLock, ONE_SECOND); + } + dataLock.Unlock(); + return NULL; +} + +inline void sanitize_object_contents (bench_data *data, int length) { + for (int i = 0; i < length; ++i) { + data->object_contents[i] = i % sizeof(char); + } +} diff --git a/src/rados.cc b/src/rados.cc index 74ea7368bc7e..03a3435159b1 100644 --- a/src/rados.cc +++ b/src/rados.cc @@ -12,8 +12,9 @@ * */ -#include "rados_bencher.h" -#include "include/librados.h" +#include "osdc/rados_bencher.h" +#include "osdc/librados.h" + #include "config.h" #include "common/common_init.h" #include "common/Cond.h" diff --git a/src/rados_bencher.h b/src/rados_bencher.h deleted file mode 100644 index c8ee1980a7d3..000000000000 --- a/src/rados_bencher.h +++ /dev/null @@ -1,463 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2009 Sage Weil - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - * - */ -#include "include/librados.h" -#include "config.h" -#include "common/common_init.h" -#include "common/Cond.h" -#include -#include - -#include -#include -#include - -Mutex dataLock("data mutex"); - -struct bench_data { - bool done; //is the benchmark is done - int object_size; //the size of the objects - int trans_size; //size of the write/read to perform - // same as object_size for write tests - int in_flight; //number of reads/writes being waited on - int started; - int finished; - double min_latency; - double max_latency; - double avg_latency; - utime_t cur_latency; //latency of last completed transaction - utime_t start_time; //start time for benchmark - char *object_contents; //pointer to the contents written to each object -}; - -int write_bench(Rados& rados, rados_pool_t pool, - int secondsToRun, int concurrentios, bench_data *data); -int seq_read_bench(Rados& rados, rados_pool_t pool, - int concurrentios, bench_data *data, int verify); -void *status_printer(void * data_store); -void sanitize_object_contents(bench_data *data, int length); - -int aio_bench(Rados& rados, rados_pool_t pool, int secondsToRun, - int concurrentios, int writeSize, int sequentialTest) { - - char* contentsChars = new char[writeSize]; - int r = 0; - - dataLock.Lock(); - bench_data *data = new bench_data(); - data->done = false; - data->object_size = writeSize; - data->trans_size = writeSize; //just for now - data->in_flight = 0; - data->started = 0; - data->finished = 0; - data->min_latency = 9999.0; // this better be higher than initial latency! - data->max_latency = 0; - data->avg_latency = 0; - data->object_contents = contentsChars; - dataLock.Unlock(); - - - //fill in contentsChars deterministically so we can check returns - sanitize_object_contents(data, writeSize); - //set up the pool - cout << "open pool result = " << rados.open_pool("data",&pool) << " pool = " << pool << std::endl; - - r = write_bench(rados, pool, secondsToRun, concurrentios, data); - if (r != 0) goto out; - - //check objects for consistency if requested - if (sequentialTest) { - r = seq_read_bench(rados, pool, concurrentios, data, 1); - } - - out: - delete contentsChars; - delete data; - return r; -} - -int write_bench(Rados& rados, rados_pool_t pool, - int secondsToRun, int concurrentios, bench_data *data) { - cout << "Maintaining " << concurrentios << " concurrent writes of " - << data->object_size << " bytes for at least " - << secondsToRun << " seconds." << std::endl; - - Rados::AioCompletion* completions[concurrentios]; - char* name[concurrentios]; - bufferlist* contents[concurrentios]; - double total_latency = 0; - utime_t start_times[concurrentios]; - utime_t stopTime; - int r = 0; - - //set up writes so I can start them together - for (int i = 0; iobject_contents, data->object_size, "I'm the %dth object!", i); - contents[i]->append(data->object_contents, data->object_size); - } - - pthread_t print_thread; - - pthread_create(&print_thread, NULL, status_printer, (void *)data); - dataLock.Lock(); - data->start_time = g_clock.now(); - dataLock.Unlock(); - for (int i = 0; iobject_size, &completions[i]); - if (r < 0) { //naughty, doesn't clean up heap - dataLock.Unlock(); - return -5; //EIO - } - dataLock.Lock(); - ++data->started; - ++data->in_flight; - dataLock.Unlock(); - } - - //keep on adding new writes as old ones complete until we've passed minimum time - int slot; - bufferlist* newContents; - char* newName; - utime_t runtime; - - utime_t timePassed = g_clock.now() - data->start_time; - //don't need locking for reads because other thread doesn't write - - runtime.set_from_double(secondsToRun); - stopTime = data->start_time + runtime; - while( g_clock.now() < stopTime ) { - slot = data->finished % concurrentios; - //create new contents and name on the heap, and fill them - newContents = new bufferlist(); - newName = new char[128]; - snprintf(newName, 128, "Object %d", data->started); - snprintf(data->object_contents, data->object_size, "I'm the %dth object!", data->started); - newContents->append(data->object_contents, data->object_size); - completions[slot]->wait_for_safe(); - dataLock.Lock(); - r = completions[slot]->get_return_value(); - if (r != 0) { - dataLock.Unlock(); - return r; - } - data->cur_latency = g_clock.now() - start_times[slot]; - total_latency += data->cur_latency; - if( data->cur_latency > data->max_latency) data->max_latency = data->cur_latency; - if (data->cur_latency < data->min_latency) data->min_latency = data->cur_latency; - ++data->finished; - data->avg_latency = total_latency / data->finished; - --data->in_flight; - dataLock.Unlock(); - completions[slot]->release(); - timePassed = g_clock.now() - data->start_time; - - //write new stuff to rados, then delete old stuff - //and save locations of new stuff for later deletion - start_times[slot] = g_clock.now(); - r = rados.aio_write(pool, newName, 0, *newContents, data->object_size, &completions[slot]); - if (r < 0) //naughty; doesn't clean up heap space. - return r; - dataLock.Lock(); - ++data->started; - ++data->in_flight; - dataLock.Unlock(); - delete name[slot]; - delete contents[slot]; - name[slot] = newName; - contents[slot] = newContents; - } - - while (data->finished < data->started) { - slot = data->finished % concurrentios; - completions[slot]->wait_for_safe(); - dataLock.Lock(); - r = completions[slot]->get_return_value(); - if (r != 0) { - dataLock.Unlock(); - return r; - } - data->cur_latency = g_clock.now() - start_times[slot]; - total_latency += data->cur_latency; - if (data->cur_latency > data->max_latency) data->max_latency = data->cur_latency; - if (data->cur_latency < data->min_latency) data->min_latency = data->cur_latency; - ++data->finished; - data->avg_latency = total_latency / data->finished; - --data->in_flight; - dataLock.Unlock(); - completions[slot]-> release(); - delete name[slot]; - delete contents[slot]; - } - - timePassed = g_clock.now() - data->start_time; - dataLock.Lock(); - data->done = true; - dataLock.Unlock(); - - pthread_join(print_thread, NULL); - - double bandwidth; - bandwidth = ((double)data->finished)*((double)data->object_size)/(double)timePassed; - bandwidth = bandwidth/(1024*1024); // we want it in MB/sec - char bw[20]; - sprintf(bw, "%.3lf \n", bandwidth); - - cout << "Total time run: " << timePassed << std::endl - << "Total writes made: " << data->finished << std::endl - << "Write size: " << data->object_size << std::endl - << "Bandwidth (MB/sec): " << bw << std::endl - << "Average Latency: " << data->avg_latency << std::endl - << "Max latency: " << data->max_latency << std::endl - << "Min latency: " << data->min_latency << std::endl; - return 0; -} - -int seq_read_bench(Rados& rados, rados_pool_t pool, - int concurrentios, bench_data *write_data, int verify) { - bench_data *data = new bench_data(); - data->done = false; - data->object_size = write_data->object_size; - data->trans_size = data->object_size; - data->in_flight= 0; - data->started = 0; - data->finished = 0; - data->min_latency = 9999.0; - data->max_latency = 0; - data->avg_latency = 0; - data->object_contents = write_data->object_contents; - - Rados::AioCompletion* completions[concurrentios]; - char* name[concurrentios]; - bufferlist* contents[concurrentios]; - int errors = 0; - utime_t start_time; - utime_t start_times[concurrentios]; - double total_latency = 0; - int r = 0; - sanitize_object_contents(data, 128); //clean it up once; subsequent - //changes will be safe because string length monotonically increases - - //set up initial reads - for (int i = 0; i < concurrentios; ++i) { - name[i] = new char[128]; - snprintf(name[i], 128, "Object %d", i); - contents[i] = new bufferlist(); - } - - pthread_t print_thread; - pthread_create(&print_thread, NULL, status_printer, (void *)data); - - dataLock.Lock(); - data->start_time = g_clock.now(); - //start initial reads - for (int i = 0; i < concurrentios; ++i) { - start_times[i] = g_clock.now(); - r = rados.aio_read(pool, name[i], 0, contents[i], data->object_size, &completions[i]); - if (r < 0) { //naughty, doesn't clean up heap -- oh, or handle the print thread! - cerr << "r = " << r << std::endl; - dataLock.Unlock(); - return -5; //EIO - } - ++data->started; - ++data->in_flight; - } - dataLock.Unlock(); - - //keep on adding new reads as old ones complete - int slot; - char* newName; - utime_t runtime; - bufferlist *cur_contents; - - for (int i = 0; i < write_data->finished - concurrentios; ++i) { - slot = data->finished % concurrentios; - newName = new char[128]; - snprintf(newName, 128, "Object %d", data->started); - completions[slot]->wait_for_complete(); - dataLock.Lock(); - r = completions[slot]->get_return_value(); - if (r != 0) { - cerr << "read got " << r << std::endl; - dataLock.Unlock(); - return r; - } - data->cur_latency = g_clock.now() - start_times[slot]; - total_latency += data->cur_latency; - if( data->cur_latency > data->max_latency) data->max_latency = data->cur_latency; - if (data->cur_latency < data->min_latency) data->min_latency = data->cur_latency; - ++data->finished; - data->avg_latency = total_latency / data->finished; - --data->in_flight; - dataLock.Unlock(); - completions[slot]->release(); - cur_contents = contents[slot]; - - //start new read and check data if requested - start_times[slot] = g_clock.now(); - contents[slot] = new bufferlist(); - r = rados.aio_read(pool, newName, 0, contents[slot], data->object_size, &completions[slot]); - if (r < 0) - return r; - dataLock.Lock(); - ++data->started; - ++data->in_flight; - dataLock.Unlock(); - if (verify) { - dataLock.Lock(); - snprintf(data->object_contents, data->object_size, "I'm the %dth object!", i); - dataLock.Unlock(); - if (memcmp(data->object_contents, cur_contents->c_str(), data->object_size) != 0) { - cerr << name[slot] << " is not correct!"; - ++errors; - } - } - delete name[slot]; - name[slot] = newName; - delete cur_contents; - } - - //wait for final reads to complete - while (data->finished < data->started) { - slot = data->finished % concurrentios; - completions[slot]->wait_for_complete(); - dataLock.Lock(); - r = completions[slot]->get_return_value(); - if (r != 0) { - cerr << "read got " << r << std::endl; - dataLock.Unlock(); - return r; - } - data->cur_latency = g_clock.now() - start_times[slot]; - total_latency += data->cur_latency; - if (data->cur_latency > data->max_latency) data->max_latency = data->cur_latency; - if (data->cur_latency < data->min_latency) data->min_latency = data->cur_latency; - ++data->finished; - data->avg_latency = total_latency / data->finished; - --data->in_flight; - dataLock.Unlock(); - completions[slot]-> release(); - if (verify) { - dataLock.Lock(); - snprintf(data->object_contents, data->object_size, "I'm the %dth object!", data->finished-1); - dataLock.Unlock(); - if (memcmp(data->object_contents, contents[slot]->c_str(), data->object_size) != 0) { - cerr << name[slot] << " is not correct!" << std::endl; - ++errors; - } - } - delete name[slot]; - delete contents[slot]; - } - - runtime = g_clock.now() - data->start_time; - dataLock.Lock(); - data->done = true; - dataLock.Unlock(); - - pthread_join(print_thread, NULL); - - double bandwidth; - bandwidth = ((double)data->finished)*((double)data->object_size)/(double)runtime; - bandwidth = bandwidth/(1024*1024); // we want it in MB/sec - char bw[20]; - sprintf(bw, "%.3lf \n", bandwidth); - - cout << "Total time run: " << runtime << std::endl - << "Total reads made: " << data->finished << std::endl - << "Read size: " << data->object_size << std::endl - << "Bandwidth (MB/sec): " << bw << std::endl - << "Average Latency: " << data->avg_latency << std::endl - << "Max latency: " << data->max_latency << std::endl - << "Min latency: " << data->min_latency << std::endl; - - delete data; - return 0; -} - - - -void *status_printer(void * data_store) { - bench_data *data = (bench_data *) data_store; - Cond cond; - int i = 0; - int previous_writes = 0; - int cycleSinceChange = 0; - double avg_bandwidth; - double bandwidth; - utime_t ONE_SECOND; - ONE_SECOND.set_from_double(1.0); - dataLock.Lock(); - while(!data->done) { - if (i % 20 == 0) { - if (i > 0) - cout << "min lat: " << data->min_latency - << " max lat: " << data->max_latency - << " avg lat: " << data->avg_latency << std::endl; - //I'm naughty and don't reset the fill - cout << setfill(' ') - << setw(5) << "sec" - << setw(8) << "Cur ops" - << setw(10) << "started" - << setw(10) << "finished" - << setw(10) << "avg MB/s" - << setw(10) << "cur MB/s" - << setw(10) << "last lat" - << setw(10) << "avg lat" << std::endl; - } - bandwidth = (double)(data->finished - previous_writes) - * (data->trans_size) - / (1024*1024) - / cycleSinceChange; - avg_bandwidth = (double) (data->trans_size) * (data->finished) - / (double)(g_clock.now() - data->start_time) / (1024*1024); - if (previous_writes != data->finished) { - previous_writes = data->finished; - cycleSinceChange = 0; - cout << setfill(' ') - << setw(5) << i - << setw(8) << data->in_flight - << setw(10) << data->started - << setw(10) << data->finished - << setw(10) << avg_bandwidth - << setw(10) << bandwidth - << setw(10) << (double)data->cur_latency - << setw(10) << data->avg_latency << std::endl; - } - else { - cout << setfill(' ') - << setw(5) << i - << setw(8) << data->in_flight - << setw(10) << data->started - << setw(10) << data->finished - << setw(10) << avg_bandwidth - << setw(10) << '0' - << setw(10) << '-' - << setw(10) << data->avg_latency << std::endl; - } - ++i; - ++cycleSinceChange; - cond.WaitInterval(dataLock, ONE_SECOND); - } - dataLock.Unlock(); - return NULL; -} - -inline void sanitize_object_contents (bench_data *data, int length) { - for (int i = 0; i < length; ++i) { - data->object_contents[i] = i % sizeof(char); - } -} diff --git a/src/radosacl.cc b/src/radosacl.cc index 199c9ed886b7..a6c2611f8e30 100644 --- a/src/radosacl.cc +++ b/src/radosacl.cc @@ -12,7 +12,7 @@ * */ -#include "include/librados.h" +#include "osdc/librados.h" #include diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 1068c746d135..f94d2ab06216 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -4,7 +4,7 @@ #include "rgw_access.h" #include "rgw_rados.h" -#include "include/librados.h" +#include "osdc/librados.h" #include #include diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 1f7e6b593d54..c8b78b037718 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -1,7 +1,7 @@ #ifndef __RGWRADOS_H #define __RGWRADOS_H -#include "include/librados.h" +#include "osdc/librados.h" #include "rgw_access.h" diff --git a/src/testrados.c b/src/testrados.c index 6bc58c8f7e46..9db257e53820 100644 --- a/src/testrados.c +++ b/src/testrados.c @@ -12,7 +12,7 @@ * */ -#include "include/librados.h" +#include "osdc/librados.h" #include #include diff --git a/src/testradospp.cc b/src/testradospp.cc index ea14741b98d4..b50ec9dca5bd 100644 --- a/src/testradospp.cc +++ b/src/testradospp.cc @@ -12,7 +12,7 @@ * */ -#include "include/librados.h" +#include "osdc/librados.h" #include