Also include rados_bencher.h in Makefile.am.
mkdir -p $(DESTDIR)$(includedir)/ceph
$(install_sh_SCRIPT) -m 0644 client/libceph.h $(DESTDIR)$(includedir)/ceph/libceph.h
mkdir -p $(DESTDIR)$(includedir)/rados
- $(install_sh_SCRIPT) -m 0644 include/librados.h $(DESTDIR)$(includedir)/rados/librados.h
+ $(install_sh_SCRIPT) -m 0644 osdc/librados.h $(DESTDIR)$(includedir)/rados/librados.h
mkdir -p $(DESTDIR)$(includedir)/crush
$(install_sh_SCRIPT) -m 0644 crush/hash.h $(DESTDIR)$(includedir)/crush/hash.h
$(install_sh_SCRIPT) -m 0644 crush/crush.h $(DESTDIR)$(includedir)/crush/crush.h
include/intarith.h\
include/interval_set.h\
include/inttypes.h\
- include/librados.h\
include/lru.h\
include/msgr.h\
include/nstring.h\
osd/RAID4PG.h\
osd/ReplicatedPG.h\
osd/osd_types.h\
+ osdc/librados.h\
+ osdc/rados_bencher.h\
osdc/Blinker.h\
osdc/Filer.h\
osdc/Journaler.h\
+++ /dev/null
-#ifndef __LIBRADOS_H
-#define __LIBRADOS_H
-
-#ifdef __cplusplus
-
-#include "include/types.h"
-
-extern "C" {
-#endif
-
-#include <netinet/in.h>
-#include <linux/types.h>
-#include <string.h>
-#include <stdbool.h>
-
-#include "msgr.h"
-#include "rados.h"
-
-/* initialization */
-int rados_initialize(int argc, const char **argv); /* arguments are optional */
-void rados_deinitialize();
-
-typedef void *rados_list_ctx_t;
-
-/* pools */
-typedef void *rados_pool_t;
-typedef long long unsigned rados_snap_t;
-
-struct rados_pool_stat_t {
- long long unsigned num_bytes; // in bytes
- long long unsigned num_kb; // in KB
- long long unsigned num_objects;
- long long unsigned num_object_clones;
- long long unsigned num_object_copies; // num_objects * num_replicas
- long long unsigned num_objects_missing_on_primary;
- long long unsigned num_objects_degraded;
- long long unsigned num_rd, num_rd_kb,num_wr, num_wr_kb;
-};
-
-struct rados_statfs_t {
- __u64 kb, kb_used, kb_avail;
- __u64 num_objects;
-};
-
-int rados_open_pool(const char *name, rados_pool_t *pool);
-int rados_close_pool(rados_pool_t pool);
-void rados_set_snap(rados_pool_t pool, rados_snap_t snap);
- /* After creating a new rados_list_ctx_t, call this to initialize it*/
-void rados_pool_init_ctx(rados_list_ctx_t *ctx);
- /* Once you've finished with a rados_list_ctx_t, call before you dump it*/
-void rados_pool_close_ctx(rados_list_ctx_t *ctx);
- /* Given a rados_list_ctx_t and its pool, get the next object in sequence*/
-int rados_pool_list_next(rados_pool_t pool, const char **entry, rados_list_ctx_t *ctx);
-
-/* snapshots */
-int rados_snap_create(const rados_pool_t pool, const char *snapname);
-int rados_snap_remove(const rados_pool_t pool, const char *snapname);
-int rados_snap_list(rados_pool_t pool, rados_snap_t *snaps, int maxlen);
-int rados_snap_get_name(rados_pool_t pool, rados_snap_t id, char *name, int maxlen);
-
-/* read/write objects */
-int rados_write(rados_pool_t pool, const char *oid, off_t off, const char *buf, size_t len);
-int rados_read(rados_pool_t pool, const char *oid, off_t off, char *buf, size_t len);
-int rados_remove(rados_pool_t pool, const char *oid);
-int rados_getxattr(rados_pool_t pool, const char *o, const char *name, char *buf, size_t len);
-int rados_setxattr(rados_pool_t pool, const char *o, const char *name, const char *buf, size_t len);
-int rados_stat(rados_pool_t pool, const char *o, __u64 *psize, time_t *pmtime);
-int rados_exec(rados_pool_t pool, const char *oid, const char *cls, const char *method,
- const char *in_buf, size_t in_len, char *buf, size_t out_len);
-
-/* async io */
-typedef void *rados_completion_t;
-typedef void (*rados_callback_t)(rados_completion_t cb, void *arg);
-
-int rados_aio_set_callback(rados_completion_t c, rados_callback_t, void *arg);
-int rados_aio_wait_for_complete(rados_completion_t c);
-int rados_aio_wait_for_safe(rados_completion_t c);
-int rados_aio_is_complete(rados_completion_t c);
-int rados_aio_is_safe(rados_completion_t c);
-int rados_aio_get_return_value(rados_completion_t c);
-void rados_aio_release(rados_completion_t c);
-
-int rados_aio_write(rados_pool_t pool, const char *oid, off_t off, const char *buf, size_t len, rados_completion_t *completion);
-int rados_aio_read(rados_pool_t pool, const char *oid, off_t off, char *buf, size_t len, rados_completion_t *completion);
-
-#ifdef __cplusplus
-}
-
-class RadosClient;
-
-class Rados
-{
- RadosClient *client;
-public:
- Rados();
- ~Rados();
- int initialize(int argc, const char *argv[]);
- void shutdown();
-
- int open_pool(const char *name, rados_pool_t *pool);
- int close_pool(rados_pool_t pool);
-
- void set_snap(rados_pool_t pool, snapid_t seq);
-
- int create(rados_pool_t pool, const object_t& oid, bool exclusive);
-
- int write(rados_pool_t pool, const object_t& oid, off_t off, bufferlist& bl, size_t len);
- int write_full(rados_pool_t pool, const object_t& oid, bufferlist& bl);
- int read(rados_pool_t pool, const object_t& oid, off_t off, bufferlist& bl, size_t len);
- int remove(rados_pool_t pool, const object_t& oid);
-
- int getxattr(rados_pool_t pool, const object_t& oid, const char *name, bufferlist& bl);
- int setxattr(rados_pool_t pool, const object_t& oid, const char *name, bufferlist& bl);
- int getxattrs(rados_pool_t pool, const object_t& oid, map<nstring, bufferlist>& attrset);
- int stat(rados_pool_t pool, const object_t& oid, __u64 *psize, time_t *pmtime);
-
- int exec(rados_pool_t pool, const object_t& oid, const char *cls, const char *method,
- bufferlist& inbl, bufferlist& outbl);
-
- struct ListCtx {
- void *ctx;
- ListCtx() : ctx(NULL) {}
- };
-
- int list(rados_pool_t pool, int max, std::list<object_t>& entries, Rados::ListCtx& ctx);
- int list_pools(std::vector<std::string>& v);
- int get_pool_stats(std::vector<std::string>& v,
- std::map<std::string,rados_pool_stat_t>& stats);
- int get_fs_stats(rados_statfs_t& result);
-
- int create_pool(const char *name);
-
- int snap_create(const rados_pool_t pool, const char *snapname);
- int snap_remove(const rados_pool_t pool, const char *snapname);
- int snap_list(rados_pool_t pool, vector<rados_snap_t> *snaps);
- int snap_get_name(rados_pool_t pool, rados_snap_t snap, std::string *name);
- int snap_get_stamp(rados_pool_t pool, rados_snap_t snap, time_t *t);
- int snap_lookup(rados_pool_t, const char *snapname, rados_snap_t *snapid);
-
- // -- aio --
- struct AioCompletion {
- void *pc;
- AioCompletion(void *_pc) : pc(_pc) {}
- int set_callback(rados_callback_t cb, void *cba);
- int wait_for_complete();
- int wait_for_safe();
- bool is_complete();
- bool is_safe();
- int get_return_value();
- void release();
- };
-
- int aio_read(rados_pool_t pool, const object_t& oid, off_t off, bufferlist *pbl, size_t len,
- AioCompletion **pc);
- int aio_write(rados_pool_t pool, const object_t& oid, off_t off, const bufferlist& bl, size_t len,
- AioCompletion **pc);
-
-};
-#endif
-
-#endif
#include "osdc/Objecter.h"
-#include "include/librados.h"
+#include "osdc/librados.h"
#define RADOS_LIST_MAX_ENTRIES 1024
#define DOUT_SUBSYS rados
static RadosClient *radosp;
-#include "include/librados.h"
-
extern "C" int rados_initialize(int argc, const char **argv)
{
int ret = 0;
--- /dev/null
+#ifndef __LIBRADOS_H
+#define __LIBRADOS_H
+
+#ifdef __cplusplus
+
+#include "include/types.h"
+
+extern "C" {
+#endif
+
+#include <netinet/in.h>
+#include <linux/types.h>
+#include <string.h>
+#include <stdbool.h>
+
+#include "include/msgr.h"
+#include "include/rados.h"
+
+/* initialization */
+int rados_initialize(int argc, const char **argv); /* arguments are optional */
+void rados_deinitialize();
+
+typedef void *rados_list_ctx_t;
+
+/* pools */
+typedef void *rados_pool_t;
+typedef long long unsigned rados_snap_t;
+
+struct rados_pool_stat_t {
+ long long unsigned num_bytes; // in bytes
+ long long unsigned num_kb; // in KB
+ long long unsigned num_objects;
+ long long unsigned num_object_clones;
+ long long unsigned num_object_copies; // num_objects * num_replicas
+ long long unsigned num_objects_missing_on_primary;
+ long long unsigned num_objects_degraded;
+ long long unsigned num_rd, num_rd_kb,num_wr, num_wr_kb;
+};
+
+struct rados_statfs_t {
+ __u64 kb, kb_used, kb_avail;
+ __u64 num_objects;
+};
+
+int rados_open_pool(const char *name, rados_pool_t *pool);
+int rados_close_pool(rados_pool_t pool);
+void rados_set_snap(rados_pool_t pool, rados_snap_t snap);
+ /* After creating a new rados_list_ctx_t, call this to initialize it*/
+void rados_pool_init_ctx(rados_list_ctx_t *ctx);
+ /* Once you've finished with a rados_list_ctx_t, call before you dump it*/
+void rados_pool_close_ctx(rados_list_ctx_t *ctx);
+ /* Given a rados_list_ctx_t and its pool, get the next object in sequence*/
+int rados_pool_list_next(rados_pool_t pool, const char **entry, rados_list_ctx_t *ctx);
+
+/* snapshots */
+int rados_snap_create(const rados_pool_t pool, const char *snapname);
+int rados_snap_remove(const rados_pool_t pool, const char *snapname);
+int rados_snap_list(rados_pool_t pool, rados_snap_t *snaps, int maxlen);
+int rados_snap_get_name(rados_pool_t pool, rados_snap_t id, char *name, int maxlen);
+
+/* read/write objects */
+int rados_write(rados_pool_t pool, const char *oid, off_t off, const char *buf, size_t len);
+int rados_read(rados_pool_t pool, const char *oid, off_t off, char *buf, size_t len);
+int rados_remove(rados_pool_t pool, const char *oid);
+int rados_getxattr(rados_pool_t pool, const char *o, const char *name, char *buf, size_t len);
+int rados_setxattr(rados_pool_t pool, const char *o, const char *name, const char *buf, size_t len);
+int rados_stat(rados_pool_t pool, const char *o, __u64 *psize, time_t *pmtime);
+int rados_exec(rados_pool_t pool, const char *oid, const char *cls, const char *method,
+ const char *in_buf, size_t in_len, char *buf, size_t out_len);
+
+/* async io */
+typedef void *rados_completion_t;
+typedef void (*rados_callback_t)(rados_completion_t cb, void *arg);
+
+int rados_aio_set_callback(rados_completion_t c, rados_callback_t, void *arg);
+int rados_aio_wait_for_complete(rados_completion_t c);
+int rados_aio_wait_for_safe(rados_completion_t c);
+int rados_aio_is_complete(rados_completion_t c);
+int rados_aio_is_safe(rados_completion_t c);
+int rados_aio_get_return_value(rados_completion_t c);
+void rados_aio_release(rados_completion_t c);
+
+int rados_aio_write(rados_pool_t pool, const char *oid, off_t off, const char *buf, size_t len, rados_completion_t *completion);
+int rados_aio_read(rados_pool_t pool, const char *oid, off_t off, char *buf, size_t len, rados_completion_t *completion);
+
+#ifdef __cplusplus
+}
+
+class RadosClient;
+
+class Rados
+{
+ RadosClient *client;
+public:
+ Rados();
+ ~Rados();
+ int initialize(int argc, const char *argv[]);
+ void shutdown();
+
+ int open_pool(const char *name, rados_pool_t *pool);
+ int close_pool(rados_pool_t pool);
+
+ void set_snap(rados_pool_t pool, snapid_t seq);
+
+ int create(rados_pool_t pool, const object_t& oid, bool exclusive);
+
+ int write(rados_pool_t pool, const object_t& oid, off_t off, bufferlist& bl, size_t len);
+ int write_full(rados_pool_t pool, const object_t& oid, bufferlist& bl);
+ int read(rados_pool_t pool, const object_t& oid, off_t off, bufferlist& bl, size_t len);
+ int remove(rados_pool_t pool, const object_t& oid);
+
+ int getxattr(rados_pool_t pool, const object_t& oid, const char *name, bufferlist& bl);
+ int setxattr(rados_pool_t pool, const object_t& oid, const char *name, bufferlist& bl);
+ int getxattrs(rados_pool_t pool, const object_t& oid, map<nstring, bufferlist>& attrset);
+ int stat(rados_pool_t pool, const object_t& oid, __u64 *psize, time_t *pmtime);
+
+ int exec(rados_pool_t pool, const object_t& oid, const char *cls, const char *method,
+ bufferlist& inbl, bufferlist& outbl);
+
+ struct ListCtx {
+ void *ctx;
+ ListCtx() : ctx(NULL) {}
+ };
+
+ int list(rados_pool_t pool, int max, std::list<object_t>& entries, Rados::ListCtx& ctx);
+ int list_pools(std::vector<std::string>& v);
+ int get_pool_stats(std::vector<std::string>& v,
+ std::map<std::string,rados_pool_stat_t>& stats);
+ int get_fs_stats(rados_statfs_t& result);
+
+ int create_pool(const char *name);
+
+ int snap_create(const rados_pool_t pool, const char *snapname);
+ int snap_remove(const rados_pool_t pool, const char *snapname);
+ int snap_list(rados_pool_t pool, vector<rados_snap_t> *snaps);
+ int snap_get_name(rados_pool_t pool, rados_snap_t snap, std::string *name);
+ int snap_get_stamp(rados_pool_t pool, rados_snap_t snap, time_t *t);
+ int snap_lookup(rados_pool_t, const char *snapname, rados_snap_t *snapid);
+
+ // -- aio --
+ struct AioCompletion {
+ void *pc;
+ AioCompletion(void *_pc) : pc(_pc) {}
+ int set_callback(rados_callback_t cb, void *cba);
+ int wait_for_complete();
+ int wait_for_safe();
+ bool is_complete();
+ bool is_safe();
+ int get_return_value();
+ void release();
+ };
+
+ int aio_read(rados_pool_t pool, const object_t& oid, off_t off, bufferlist *pbl, size_t len,
+ AioCompletion **pc);
+ int aio_write(rados_pool_t pool, const object_t& oid, off_t off, const bufferlist& bl, size_t len,
+ AioCompletion **pc);
+
+};
+#endif
+
+#endif
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2009 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include "osdc/librados.h"
+#include "config.h"
+#include "common/common_init.h"
+#include "common/Cond.h"
+#include <iostream>
+#include <fstream>
+
+#include <stdlib.h>
+#include <time.h>
+#include <sstream>
+
+Mutex dataLock("data mutex");
+
+struct bench_data {
+ bool done; //is the benchmark is done
+ int object_size; //the size of the objects
+ int trans_size; //size of the write/read to perform
+ // same as object_size for write tests
+ int in_flight; //number of reads/writes being waited on
+ int started;
+ int finished;
+ double min_latency;
+ double max_latency;
+ double avg_latency;
+ utime_t cur_latency; //latency of last completed transaction
+ utime_t start_time; //start time for benchmark
+ char *object_contents; //pointer to the contents written to each object
+};
+
+int write_bench(Rados& rados, rados_pool_t pool,
+ int secondsToRun, int concurrentios, bench_data *data);
+int seq_read_bench(Rados& rados, rados_pool_t pool,
+ int concurrentios, bench_data *data, int verify);
+void *status_printer(void * data_store);
+void sanitize_object_contents(bench_data *data, int length);
+
+int aio_bench(Rados& rados, rados_pool_t pool, int secondsToRun,
+ int concurrentios, int writeSize, int sequentialTest) {
+
+ char* contentsChars = new char[writeSize];
+ int r = 0;
+
+ dataLock.Lock();
+ bench_data *data = new bench_data();
+ data->done = false;
+ data->object_size = writeSize;
+ data->trans_size = writeSize; //just for now
+ data->in_flight = 0;
+ data->started = 0;
+ data->finished = 0;
+ data->min_latency = 9999.0; // this better be higher than initial latency!
+ data->max_latency = 0;
+ data->avg_latency = 0;
+ data->object_contents = contentsChars;
+ dataLock.Unlock();
+
+
+ //fill in contentsChars deterministically so we can check returns
+ sanitize_object_contents(data, writeSize);
+ //set up the pool
+ cout << "open pool result = " << rados.open_pool("data",&pool) << " pool = " << pool << std::endl;
+
+ r = write_bench(rados, pool, secondsToRun, concurrentios, data);
+ if (r != 0) goto out;
+
+ //check objects for consistency if requested
+ if (sequentialTest) {
+ r = seq_read_bench(rados, pool, concurrentios, data, 1);
+ }
+
+ out:
+ delete contentsChars;
+ delete data;
+ return r;
+}
+
+int write_bench(Rados& rados, rados_pool_t pool,
+ int secondsToRun, int concurrentios, bench_data *data) {
+ cout << "Maintaining " << concurrentios << " concurrent writes of "
+ << data->object_size << " bytes for at least "
+ << secondsToRun << " seconds." << std::endl;
+
+ Rados::AioCompletion* completions[concurrentios];
+ char* name[concurrentios];
+ bufferlist* contents[concurrentios];
+ double total_latency = 0;
+ utime_t start_times[concurrentios];
+ utime_t stopTime;
+ int r = 0;
+
+ //set up writes so I can start them together
+ for (int i = 0; i<concurrentios; ++i) {
+ name[i] = new char[128];
+ contents[i] = new bufferlist();
+ snprintf(name[i], 128, "Object %d", i);
+ snprintf(data->object_contents, data->object_size, "I'm the %dth object!", i);
+ contents[i]->append(data->object_contents, data->object_size);
+ }
+
+ pthread_t print_thread;
+
+ pthread_create(&print_thread, NULL, status_printer, (void *)data);
+ dataLock.Lock();
+ data->start_time = g_clock.now();
+ dataLock.Unlock();
+ for (int i = 0; i<concurrentios; ++i) {
+ start_times[i] = g_clock.now();
+ r = rados.aio_write(pool, name[i], 0, *contents[i], data->object_size, &completions[i]);
+ if (r < 0) { //naughty, doesn't clean up heap
+ dataLock.Unlock();
+ return -5; //EIO
+ }
+ dataLock.Lock();
+ ++data->started;
+ ++data->in_flight;
+ dataLock.Unlock();
+ }
+
+ //keep on adding new writes as old ones complete until we've passed minimum time
+ int slot;
+ bufferlist* newContents;
+ char* newName;
+ utime_t runtime;
+
+ utime_t timePassed = g_clock.now() - data->start_time;
+ //don't need locking for reads because other thread doesn't write
+
+ runtime.set_from_double(secondsToRun);
+ stopTime = data->start_time + runtime;
+ while( g_clock.now() < stopTime ) {
+ slot = data->finished % concurrentios;
+ //create new contents and name on the heap, and fill them
+ newContents = new bufferlist();
+ newName = new char[128];
+ snprintf(newName, 128, "Object %d", data->started);
+ snprintf(data->object_contents, data->object_size, "I'm the %dth object!", data->started);
+ newContents->append(data->object_contents, data->object_size);
+ completions[slot]->wait_for_safe();
+ dataLock.Lock();
+ r = completions[slot]->get_return_value();
+ if (r != 0) {
+ dataLock.Unlock();
+ return r;
+ }
+ data->cur_latency = g_clock.now() - start_times[slot];
+ total_latency += data->cur_latency;
+ if( data->cur_latency > data->max_latency) data->max_latency = data->cur_latency;
+ if (data->cur_latency < data->min_latency) data->min_latency = data->cur_latency;
+ ++data->finished;
+ data->avg_latency = total_latency / data->finished;
+ --data->in_flight;
+ dataLock.Unlock();
+ completions[slot]->release();
+ timePassed = g_clock.now() - data->start_time;
+
+ //write new stuff to rados, then delete old stuff
+ //and save locations of new stuff for later deletion
+ start_times[slot] = g_clock.now();
+ r = rados.aio_write(pool, newName, 0, *newContents, data->object_size, &completions[slot]);
+ if (r < 0) //naughty; doesn't clean up heap space.
+ return r;
+ dataLock.Lock();
+ ++data->started;
+ ++data->in_flight;
+ dataLock.Unlock();
+ delete name[slot];
+ delete contents[slot];
+ name[slot] = newName;
+ contents[slot] = newContents;
+ }
+
+ while (data->finished < data->started) {
+ slot = data->finished % concurrentios;
+ completions[slot]->wait_for_safe();
+ dataLock.Lock();
+ r = completions[slot]->get_return_value();
+ if (r != 0) {
+ dataLock.Unlock();
+ return r;
+ }
+ data->cur_latency = g_clock.now() - start_times[slot];
+ total_latency += data->cur_latency;
+ if (data->cur_latency > data->max_latency) data->max_latency = data->cur_latency;
+ if (data->cur_latency < data->min_latency) data->min_latency = data->cur_latency;
+ ++data->finished;
+ data->avg_latency = total_latency / data->finished;
+ --data->in_flight;
+ dataLock.Unlock();
+ completions[slot]-> release();
+ delete name[slot];
+ delete contents[slot];
+ }
+
+ timePassed = g_clock.now() - data->start_time;
+ dataLock.Lock();
+ data->done = true;
+ dataLock.Unlock();
+
+ pthread_join(print_thread, NULL);
+
+ double bandwidth;
+ bandwidth = ((double)data->finished)*((double)data->object_size)/(double)timePassed;
+ bandwidth = bandwidth/(1024*1024); // we want it in MB/sec
+ char bw[20];
+ sprintf(bw, "%.3lf \n", bandwidth);
+
+ cout << "Total time run: " << timePassed << std::endl
+ << "Total writes made: " << data->finished << std::endl
+ << "Write size: " << data->object_size << std::endl
+ << "Bandwidth (MB/sec): " << bw << std::endl
+ << "Average Latency: " << data->avg_latency << std::endl
+ << "Max latency: " << data->max_latency << std::endl
+ << "Min latency: " << data->min_latency << std::endl;
+ return 0;
+}
+
+int seq_read_bench(Rados& rados, rados_pool_t pool,
+ int concurrentios, bench_data *write_data, int verify) {
+ bench_data *data = new bench_data();
+ data->done = false;
+ data->object_size = write_data->object_size;
+ data->trans_size = data->object_size;
+ data->in_flight= 0;
+ data->started = 0;
+ data->finished = 0;
+ data->min_latency = 9999.0;
+ data->max_latency = 0;
+ data->avg_latency = 0;
+ data->object_contents = write_data->object_contents;
+
+ Rados::AioCompletion* completions[concurrentios];
+ char* name[concurrentios];
+ bufferlist* contents[concurrentios];
+ int errors = 0;
+ utime_t start_time;
+ utime_t start_times[concurrentios];
+ double total_latency = 0;
+ int r = 0;
+ sanitize_object_contents(data, 128); //clean it up once; subsequent
+ //changes will be safe because string length monotonically increases
+
+ //set up initial reads
+ for (int i = 0; i < concurrentios; ++i) {
+ name[i] = new char[128];
+ snprintf(name[i], 128, "Object %d", i);
+ contents[i] = new bufferlist();
+ }
+
+ pthread_t print_thread;
+ pthread_create(&print_thread, NULL, status_printer, (void *)data);
+
+ dataLock.Lock();
+ data->start_time = g_clock.now();
+ //start initial reads
+ for (int i = 0; i < concurrentios; ++i) {
+ start_times[i] = g_clock.now();
+ r = rados.aio_read(pool, name[i], 0, contents[i], data->object_size, &completions[i]);
+ if (r < 0) { //naughty, doesn't clean up heap -- oh, or handle the print thread!
+ cerr << "r = " << r << std::endl;
+ dataLock.Unlock();
+ return -5; //EIO
+ }
+ ++data->started;
+ ++data->in_flight;
+ }
+ dataLock.Unlock();
+
+ //keep on adding new reads as old ones complete
+ int slot;
+ char* newName;
+ utime_t runtime;
+ bufferlist *cur_contents;
+
+ for (int i = 0; i < write_data->finished - concurrentios; ++i) {
+ slot = data->finished % concurrentios;
+ newName = new char[128];
+ snprintf(newName, 128, "Object %d", data->started);
+ completions[slot]->wait_for_complete();
+ dataLock.Lock();
+ r = completions[slot]->get_return_value();
+ if (r != 0) {
+ cerr << "read got " << r << std::endl;
+ dataLock.Unlock();
+ return r;
+ }
+ data->cur_latency = g_clock.now() - start_times[slot];
+ total_latency += data->cur_latency;
+ if( data->cur_latency > data->max_latency) data->max_latency = data->cur_latency;
+ if (data->cur_latency < data->min_latency) data->min_latency = data->cur_latency;
+ ++data->finished;
+ data->avg_latency = total_latency / data->finished;
+ --data->in_flight;
+ dataLock.Unlock();
+ completions[slot]->release();
+ cur_contents = contents[slot];
+
+ //start new read and check data if requested
+ start_times[slot] = g_clock.now();
+ contents[slot] = new bufferlist();
+ r = rados.aio_read(pool, newName, 0, contents[slot], data->object_size, &completions[slot]);
+ if (r < 0)
+ return r;
+ dataLock.Lock();
+ ++data->started;
+ ++data->in_flight;
+ dataLock.Unlock();
+ if (verify) {
+ dataLock.Lock();
+ snprintf(data->object_contents, data->object_size, "I'm the %dth object!", i);
+ dataLock.Unlock();
+ if (memcmp(data->object_contents, cur_contents->c_str(), data->object_size) != 0) {
+ cerr << name[slot] << " is not correct!";
+ ++errors;
+ }
+ }
+ delete name[slot];
+ name[slot] = newName;
+ delete cur_contents;
+ }
+
+ //wait for final reads to complete
+ while (data->finished < data->started) {
+ slot = data->finished % concurrentios;
+ completions[slot]->wait_for_complete();
+ dataLock.Lock();
+ r = completions[slot]->get_return_value();
+ if (r != 0) {
+ cerr << "read got " << r << std::endl;
+ dataLock.Unlock();
+ return r;
+ }
+ data->cur_latency = g_clock.now() - start_times[slot];
+ total_latency += data->cur_latency;
+ if (data->cur_latency > data->max_latency) data->max_latency = data->cur_latency;
+ if (data->cur_latency < data->min_latency) data->min_latency = data->cur_latency;
+ ++data->finished;
+ data->avg_latency = total_latency / data->finished;
+ --data->in_flight;
+ dataLock.Unlock();
+ completions[slot]-> release();
+ if (verify) {
+ dataLock.Lock();
+ snprintf(data->object_contents, data->object_size, "I'm the %dth object!", data->finished-1);
+ dataLock.Unlock();
+ if (memcmp(data->object_contents, contents[slot]->c_str(), data->object_size) != 0) {
+ cerr << name[slot] << " is not correct!" << std::endl;
+ ++errors;
+ }
+ }
+ delete name[slot];
+ delete contents[slot];
+ }
+
+ runtime = g_clock.now() - data->start_time;
+ dataLock.Lock();
+ data->done = true;
+ dataLock.Unlock();
+
+ pthread_join(print_thread, NULL);
+
+ double bandwidth;
+ bandwidth = ((double)data->finished)*((double)data->object_size)/(double)runtime;
+ bandwidth = bandwidth/(1024*1024); // we want it in MB/sec
+ char bw[20];
+ sprintf(bw, "%.3lf \n", bandwidth);
+
+ cout << "Total time run: " << runtime << std::endl
+ << "Total reads made: " << data->finished << std::endl
+ << "Read size: " << data->object_size << std::endl
+ << "Bandwidth (MB/sec): " << bw << std::endl
+ << "Average Latency: " << data->avg_latency << std::endl
+ << "Max latency: " << data->max_latency << std::endl
+ << "Min latency: " << data->min_latency << std::endl;
+
+ delete data;
+ return 0;
+}
+
+
+
+void *status_printer(void * data_store) {
+ bench_data *data = (bench_data *) data_store;
+ Cond cond;
+ int i = 0;
+ int previous_writes = 0;
+ int cycleSinceChange = 0;
+ double avg_bandwidth;
+ double bandwidth;
+ utime_t ONE_SECOND;
+ ONE_SECOND.set_from_double(1.0);
+ dataLock.Lock();
+ while(!data->done) {
+ if (i % 20 == 0) {
+ if (i > 0)
+ cout << "min lat: " << data->min_latency
+ << " max lat: " << data->max_latency
+ << " avg lat: " << data->avg_latency << std::endl;
+ //I'm naughty and don't reset the fill
+ cout << setfill(' ')
+ << setw(5) << "sec"
+ << setw(8) << "Cur ops"
+ << setw(10) << "started"
+ << setw(10) << "finished"
+ << setw(10) << "avg MB/s"
+ << setw(10) << "cur MB/s"
+ << setw(10) << "last lat"
+ << setw(10) << "avg lat" << std::endl;
+ }
+ bandwidth = (double)(data->finished - previous_writes)
+ * (data->trans_size)
+ / (1024*1024)
+ / cycleSinceChange;
+ avg_bandwidth = (double) (data->trans_size) * (data->finished)
+ / (double)(g_clock.now() - data->start_time) / (1024*1024);
+ if (previous_writes != data->finished) {
+ previous_writes = data->finished;
+ cycleSinceChange = 0;
+ cout << setfill(' ')
+ << setw(5) << i
+ << setw(8) << data->in_flight
+ << setw(10) << data->started
+ << setw(10) << data->finished
+ << setw(10) << avg_bandwidth
+ << setw(10) << bandwidth
+ << setw(10) << (double)data->cur_latency
+ << setw(10) << data->avg_latency << std::endl;
+ }
+ else {
+ cout << setfill(' ')
+ << setw(5) << i
+ << setw(8) << data->in_flight
+ << setw(10) << data->started
+ << setw(10) << data->finished
+ << setw(10) << avg_bandwidth
+ << setw(10) << '0'
+ << setw(10) << '-'
+ << setw(10) << data->avg_latency << std::endl;
+ }
+ ++i;
+ ++cycleSinceChange;
+ cond.WaitInterval(dataLock, ONE_SECOND);
+ }
+ dataLock.Unlock();
+ return NULL;
+}
+
+inline void sanitize_object_contents (bench_data *data, int length) {
+ for (int i = 0; i < length; ++i) {
+ data->object_contents[i] = i % sizeof(char);
+ }
+}
*
*/
-#include "rados_bencher.h"
-#include "include/librados.h"
+#include "osdc/rados_bencher.h"
+#include "osdc/librados.h"
+
#include "config.h"
#include "common/common_init.h"
#include "common/Cond.h"
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2009 Sage Weil <sage@newdream.net>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation. See file COPYING.
- *
- */
-#include "include/librados.h"
-#include "config.h"
-#include "common/common_init.h"
-#include "common/Cond.h"
-#include <iostream>
-#include <fstream>
-
-#include <stdlib.h>
-#include <time.h>
-#include <sstream>
-
-Mutex dataLock("data mutex");
-
-struct bench_data {
- bool done; //is the benchmark is done
- int object_size; //the size of the objects
- int trans_size; //size of the write/read to perform
- // same as object_size for write tests
- int in_flight; //number of reads/writes being waited on
- int started;
- int finished;
- double min_latency;
- double max_latency;
- double avg_latency;
- utime_t cur_latency; //latency of last completed transaction
- utime_t start_time; //start time for benchmark
- char *object_contents; //pointer to the contents written to each object
-};
-
-int write_bench(Rados& rados, rados_pool_t pool,
- int secondsToRun, int concurrentios, bench_data *data);
-int seq_read_bench(Rados& rados, rados_pool_t pool,
- int concurrentios, bench_data *data, int verify);
-void *status_printer(void * data_store);
-void sanitize_object_contents(bench_data *data, int length);
-
-int aio_bench(Rados& rados, rados_pool_t pool, int secondsToRun,
- int concurrentios, int writeSize, int sequentialTest) {
-
- char* contentsChars = new char[writeSize];
- int r = 0;
-
- dataLock.Lock();
- bench_data *data = new bench_data();
- data->done = false;
- data->object_size = writeSize;
- data->trans_size = writeSize; //just for now
- data->in_flight = 0;
- data->started = 0;
- data->finished = 0;
- data->min_latency = 9999.0; // this better be higher than initial latency!
- data->max_latency = 0;
- data->avg_latency = 0;
- data->object_contents = contentsChars;
- dataLock.Unlock();
-
-
- //fill in contentsChars deterministically so we can check returns
- sanitize_object_contents(data, writeSize);
- //set up the pool
- cout << "open pool result = " << rados.open_pool("data",&pool) << " pool = " << pool << std::endl;
-
- r = write_bench(rados, pool, secondsToRun, concurrentios, data);
- if (r != 0) goto out;
-
- //check objects for consistency if requested
- if (sequentialTest) {
- r = seq_read_bench(rados, pool, concurrentios, data, 1);
- }
-
- out:
- delete contentsChars;
- delete data;
- return r;
-}
-
-int write_bench(Rados& rados, rados_pool_t pool,
- int secondsToRun, int concurrentios, bench_data *data) {
- cout << "Maintaining " << concurrentios << " concurrent writes of "
- << data->object_size << " bytes for at least "
- << secondsToRun << " seconds." << std::endl;
-
- Rados::AioCompletion* completions[concurrentios];
- char* name[concurrentios];
- bufferlist* contents[concurrentios];
- double total_latency = 0;
- utime_t start_times[concurrentios];
- utime_t stopTime;
- int r = 0;
-
- //set up writes so I can start them together
- for (int i = 0; i<concurrentios; ++i) {
- name[i] = new char[128];
- contents[i] = new bufferlist();
- snprintf(name[i], 128, "Object %d", i);
- snprintf(data->object_contents, data->object_size, "I'm the %dth object!", i);
- contents[i]->append(data->object_contents, data->object_size);
- }
-
- pthread_t print_thread;
-
- pthread_create(&print_thread, NULL, status_printer, (void *)data);
- dataLock.Lock();
- data->start_time = g_clock.now();
- dataLock.Unlock();
- for (int i = 0; i<concurrentios; ++i) {
- start_times[i] = g_clock.now();
- r = rados.aio_write(pool, name[i], 0, *contents[i], data->object_size, &completions[i]);
- if (r < 0) { //naughty, doesn't clean up heap
- dataLock.Unlock();
- return -5; //EIO
- }
- dataLock.Lock();
- ++data->started;
- ++data->in_flight;
- dataLock.Unlock();
- }
-
- //keep on adding new writes as old ones complete until we've passed minimum time
- int slot;
- bufferlist* newContents;
- char* newName;
- utime_t runtime;
-
- utime_t timePassed = g_clock.now() - data->start_time;
- //don't need locking for reads because other thread doesn't write
-
- runtime.set_from_double(secondsToRun);
- stopTime = data->start_time + runtime;
- while( g_clock.now() < stopTime ) {
- slot = data->finished % concurrentios;
- //create new contents and name on the heap, and fill them
- newContents = new bufferlist();
- newName = new char[128];
- snprintf(newName, 128, "Object %d", data->started);
- snprintf(data->object_contents, data->object_size, "I'm the %dth object!", data->started);
- newContents->append(data->object_contents, data->object_size);
- completions[slot]->wait_for_safe();
- dataLock.Lock();
- r = completions[slot]->get_return_value();
- if (r != 0) {
- dataLock.Unlock();
- return r;
- }
- data->cur_latency = g_clock.now() - start_times[slot];
- total_latency += data->cur_latency;
- if( data->cur_latency > data->max_latency) data->max_latency = data->cur_latency;
- if (data->cur_latency < data->min_latency) data->min_latency = data->cur_latency;
- ++data->finished;
- data->avg_latency = total_latency / data->finished;
- --data->in_flight;
- dataLock.Unlock();
- completions[slot]->release();
- timePassed = g_clock.now() - data->start_time;
-
- //write new stuff to rados, then delete old stuff
- //and save locations of new stuff for later deletion
- start_times[slot] = g_clock.now();
- r = rados.aio_write(pool, newName, 0, *newContents, data->object_size, &completions[slot]);
- if (r < 0) //naughty; doesn't clean up heap space.
- return r;
- dataLock.Lock();
- ++data->started;
- ++data->in_flight;
- dataLock.Unlock();
- delete name[slot];
- delete contents[slot];
- name[slot] = newName;
- contents[slot] = newContents;
- }
-
- while (data->finished < data->started) {
- slot = data->finished % concurrentios;
- completions[slot]->wait_for_safe();
- dataLock.Lock();
- r = completions[slot]->get_return_value();
- if (r != 0) {
- dataLock.Unlock();
- return r;
- }
- data->cur_latency = g_clock.now() - start_times[slot];
- total_latency += data->cur_latency;
- if (data->cur_latency > data->max_latency) data->max_latency = data->cur_latency;
- if (data->cur_latency < data->min_latency) data->min_latency = data->cur_latency;
- ++data->finished;
- data->avg_latency = total_latency / data->finished;
- --data->in_flight;
- dataLock.Unlock();
- completions[slot]-> release();
- delete name[slot];
- delete contents[slot];
- }
-
- timePassed = g_clock.now() - data->start_time;
- dataLock.Lock();
- data->done = true;
- dataLock.Unlock();
-
- pthread_join(print_thread, NULL);
-
- double bandwidth;
- bandwidth = ((double)data->finished)*((double)data->object_size)/(double)timePassed;
- bandwidth = bandwidth/(1024*1024); // we want it in MB/sec
- char bw[20];
- sprintf(bw, "%.3lf \n", bandwidth);
-
- cout << "Total time run: " << timePassed << std::endl
- << "Total writes made: " << data->finished << std::endl
- << "Write size: " << data->object_size << std::endl
- << "Bandwidth (MB/sec): " << bw << std::endl
- << "Average Latency: " << data->avg_latency << std::endl
- << "Max latency: " << data->max_latency << std::endl
- << "Min latency: " << data->min_latency << std::endl;
- return 0;
-}
-
-int seq_read_bench(Rados& rados, rados_pool_t pool,
- int concurrentios, bench_data *write_data, int verify) {
- bench_data *data = new bench_data();
- data->done = false;
- data->object_size = write_data->object_size;
- data->trans_size = data->object_size;
- data->in_flight= 0;
- data->started = 0;
- data->finished = 0;
- data->min_latency = 9999.0;
- data->max_latency = 0;
- data->avg_latency = 0;
- data->object_contents = write_data->object_contents;
-
- Rados::AioCompletion* completions[concurrentios];
- char* name[concurrentios];
- bufferlist* contents[concurrentios];
- int errors = 0;
- utime_t start_time;
- utime_t start_times[concurrentios];
- double total_latency = 0;
- int r = 0;
- sanitize_object_contents(data, 128); //clean it up once; subsequent
- //changes will be safe because string length monotonically increases
-
- //set up initial reads
- for (int i = 0; i < concurrentios; ++i) {
- name[i] = new char[128];
- snprintf(name[i], 128, "Object %d", i);
- contents[i] = new bufferlist();
- }
-
- pthread_t print_thread;
- pthread_create(&print_thread, NULL, status_printer, (void *)data);
-
- dataLock.Lock();
- data->start_time = g_clock.now();
- //start initial reads
- for (int i = 0; i < concurrentios; ++i) {
- start_times[i] = g_clock.now();
- r = rados.aio_read(pool, name[i], 0, contents[i], data->object_size, &completions[i]);
- if (r < 0) { //naughty, doesn't clean up heap -- oh, or handle the print thread!
- cerr << "r = " << r << std::endl;
- dataLock.Unlock();
- return -5; //EIO
- }
- ++data->started;
- ++data->in_flight;
- }
- dataLock.Unlock();
-
- //keep on adding new reads as old ones complete
- int slot;
- char* newName;
- utime_t runtime;
- bufferlist *cur_contents;
-
- for (int i = 0; i < write_data->finished - concurrentios; ++i) {
- slot = data->finished % concurrentios;
- newName = new char[128];
- snprintf(newName, 128, "Object %d", data->started);
- completions[slot]->wait_for_complete();
- dataLock.Lock();
- r = completions[slot]->get_return_value();
- if (r != 0) {
- cerr << "read got " << r << std::endl;
- dataLock.Unlock();
- return r;
- }
- data->cur_latency = g_clock.now() - start_times[slot];
- total_latency += data->cur_latency;
- if( data->cur_latency > data->max_latency) data->max_latency = data->cur_latency;
- if (data->cur_latency < data->min_latency) data->min_latency = data->cur_latency;
- ++data->finished;
- data->avg_latency = total_latency / data->finished;
- --data->in_flight;
- dataLock.Unlock();
- completions[slot]->release();
- cur_contents = contents[slot];
-
- //start new read and check data if requested
- start_times[slot] = g_clock.now();
- contents[slot] = new bufferlist();
- r = rados.aio_read(pool, newName, 0, contents[slot], data->object_size, &completions[slot]);
- if (r < 0)
- return r;
- dataLock.Lock();
- ++data->started;
- ++data->in_flight;
- dataLock.Unlock();
- if (verify) {
- dataLock.Lock();
- snprintf(data->object_contents, data->object_size, "I'm the %dth object!", i);
- dataLock.Unlock();
- if (memcmp(data->object_contents, cur_contents->c_str(), data->object_size) != 0) {
- cerr << name[slot] << " is not correct!";
- ++errors;
- }
- }
- delete name[slot];
- name[slot] = newName;
- delete cur_contents;
- }
-
- //wait for final reads to complete
- while (data->finished < data->started) {
- slot = data->finished % concurrentios;
- completions[slot]->wait_for_complete();
- dataLock.Lock();
- r = completions[slot]->get_return_value();
- if (r != 0) {
- cerr << "read got " << r << std::endl;
- dataLock.Unlock();
- return r;
- }
- data->cur_latency = g_clock.now() - start_times[slot];
- total_latency += data->cur_latency;
- if (data->cur_latency > data->max_latency) data->max_latency = data->cur_latency;
- if (data->cur_latency < data->min_latency) data->min_latency = data->cur_latency;
- ++data->finished;
- data->avg_latency = total_latency / data->finished;
- --data->in_flight;
- dataLock.Unlock();
- completions[slot]-> release();
- if (verify) {
- dataLock.Lock();
- snprintf(data->object_contents, data->object_size, "I'm the %dth object!", data->finished-1);
- dataLock.Unlock();
- if (memcmp(data->object_contents, contents[slot]->c_str(), data->object_size) != 0) {
- cerr << name[slot] << " is not correct!" << std::endl;
- ++errors;
- }
- }
- delete name[slot];
- delete contents[slot];
- }
-
- runtime = g_clock.now() - data->start_time;
- dataLock.Lock();
- data->done = true;
- dataLock.Unlock();
-
- pthread_join(print_thread, NULL);
-
- double bandwidth;
- bandwidth = ((double)data->finished)*((double)data->object_size)/(double)runtime;
- bandwidth = bandwidth/(1024*1024); // we want it in MB/sec
- char bw[20];
- sprintf(bw, "%.3lf \n", bandwidth);
-
- cout << "Total time run: " << runtime << std::endl
- << "Total reads made: " << data->finished << std::endl
- << "Read size: " << data->object_size << std::endl
- << "Bandwidth (MB/sec): " << bw << std::endl
- << "Average Latency: " << data->avg_latency << std::endl
- << "Max latency: " << data->max_latency << std::endl
- << "Min latency: " << data->min_latency << std::endl;
-
- delete data;
- return 0;
-}
-
-
-
-void *status_printer(void * data_store) {
- bench_data *data = (bench_data *) data_store;
- Cond cond;
- int i = 0;
- int previous_writes = 0;
- int cycleSinceChange = 0;
- double avg_bandwidth;
- double bandwidth;
- utime_t ONE_SECOND;
- ONE_SECOND.set_from_double(1.0);
- dataLock.Lock();
- while(!data->done) {
- if (i % 20 == 0) {
- if (i > 0)
- cout << "min lat: " << data->min_latency
- << " max lat: " << data->max_latency
- << " avg lat: " << data->avg_latency << std::endl;
- //I'm naughty and don't reset the fill
- cout << setfill(' ')
- << setw(5) << "sec"
- << setw(8) << "Cur ops"
- << setw(10) << "started"
- << setw(10) << "finished"
- << setw(10) << "avg MB/s"
- << setw(10) << "cur MB/s"
- << setw(10) << "last lat"
- << setw(10) << "avg lat" << std::endl;
- }
- bandwidth = (double)(data->finished - previous_writes)
- * (data->trans_size)
- / (1024*1024)
- / cycleSinceChange;
- avg_bandwidth = (double) (data->trans_size) * (data->finished)
- / (double)(g_clock.now() - data->start_time) / (1024*1024);
- if (previous_writes != data->finished) {
- previous_writes = data->finished;
- cycleSinceChange = 0;
- cout << setfill(' ')
- << setw(5) << i
- << setw(8) << data->in_flight
- << setw(10) << data->started
- << setw(10) << data->finished
- << setw(10) << avg_bandwidth
- << setw(10) << bandwidth
- << setw(10) << (double)data->cur_latency
- << setw(10) << data->avg_latency << std::endl;
- }
- else {
- cout << setfill(' ')
- << setw(5) << i
- << setw(8) << data->in_flight
- << setw(10) << data->started
- << setw(10) << data->finished
- << setw(10) << avg_bandwidth
- << setw(10) << '0'
- << setw(10) << '-'
- << setw(10) << data->avg_latency << std::endl;
- }
- ++i;
- ++cycleSinceChange;
- cond.WaitInterval(dataLock, ONE_SECOND);
- }
- dataLock.Unlock();
- return NULL;
-}
-
-inline void sanitize_object_contents (bench_data *data, int length) {
- for (int i = 0; i < length; ++i) {
- data->object_contents[i] = i % sizeof(char);
- }
-}
*
*/
-#include "include/librados.h"
+#include "osdc/librados.h"
#include <iostream>
#include "rgw_access.h"
#include "rgw_rados.h"
-#include "include/librados.h"
+#include "osdc/librados.h"
#include <string>
#include <iostream>
#ifndef __RGWRADOS_H
#define __RGWRADOS_H
-#include "include/librados.h"
+#include "osdc/librados.h"
#include "rgw_access.h"
*
*/
-#include "include/librados.h"
+#include "osdc/librados.h"
#include <stdio.h>
#include <stdlib.h>
*
*/
-#include "include/librados.h"
+#include "osdc/librados.h"
#include <iostream>
projects / ceph.git / commitdiff