rbd: implement bench-write command

We may want to change the command line args around later to make this
more friendly.

Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/doc/man/8/rbd.rst b/doc/man/8/rbd.rst
index ac923f0fb8f981c39586d2d611988388dbbcdc96..b51ad94d47b61d53cb3276c381547c765fd7d396 100644 (file)
--- a/doc/man/8/rbd.rst
+++ b/doc/man/8/rbd.rst
@@ -219,6 +219,10 @@ Commands
   Release a lock on an image. The lock id and locker are
   as output by lock ls.
 
+:command:`bench-write` [*image-name*] --io-size [*io-size-in-bytes*] --io-threads [*num-ios-in-flight*] --io-total [*total-bytes-to-write*]
+  Generate a series of sequential writes to the image and measure the
+  write throughput and latency.
+
 Image name
 ==========
 

diff --git a/src/rbd.cc b/src/rbd.cc
index d9f9f8cf933db0a499dedefa1e37e63b239d8903..de30510eb1c8ad34eb2a6b1a1c7692e3ca1163e3 100644 (file)
--- a/src/rbd.cc
+++ b/src/rbd.cc
@@ -104,6 +104,7 @@ void usage()
 "  lock list <image-name>                      show locks held on an image\n"
 "  lock add <image-name> <id> [--shared <tag>] take a lock called id on an image\n"
 "  lock remove <image-name> <id> <locker>      release a lock on an image\n"
+"  bench-write <image-name> --io-size <bytes> --io-threads <num> --io-total <bytes>\n"
 "\n"
 "<image-name>, <snap-name> are [pool/]name[@snap], or you may specify\n"
 "individual pieces of names with -p/--pool, --image, and/or --snap.\n"
@@ -557,6 +558,103 @@ static int do_lock_remove(librbd::Image& image, const char *client,
   return image.break_lock(client, cookie);
 }
 
+static void rbd_bencher_completion(void *c, void *pc);
+
+struct rbd_bencher;
+
+struct rbd_bencher {
+  librbd::Image *image;
+  Mutex lock;
+  Cond cond;
+  int in_flight;
+
+  rbd_bencher(librbd::Image *i)
+    : image(i),
+      lock("rbd_bencher::lock"),
+      in_flight(0)
+  { }
+
+  bool start_write(int max, uint64_t off, uint64_t len, bufferlist& bl) {
+    Mutex::Locker l(lock);
+    if (in_flight >= max)
+      return false;
+    in_flight++;
+    librbd::RBD::AioCompletion *c = new librbd::RBD::AioCompletion((void *)this, rbd_bencher_completion);
+    image->aio_write(off, len, bl, c);
+    //cout << "start " << c << " at " << off << "~" << len << std::endl;
+    return true;
+  }
+
+  void wait_for(int max) {
+    Mutex::Locker l(lock);
+    while (in_flight > max) {
+      utime_t dur;
+      dur.set_from_double(.2);
+      cond.WaitInterval(g_ceph_context, lock, dur);
+    }
+  }
+
+};
+
+void rbd_bencher_completion(void *vc, void *pc)
+{
+  librbd::RBD::AioCompletion *c = (librbd::RBD::AioCompletion *)vc;
+  rbd_bencher *b = (rbd_bencher *)pc;
+  //cout << "complete " << c << std::endl;
+  b->lock.Lock();
+  b->in_flight--;
+  b->cond.Signal();
+  b->lock.Unlock();
+  c->release();
+}
+
+static int do_bench_write(librbd::Image& image, uint64_t io_size, uint64_t io_threads, uint64_t io_bytes)
+{
+  rbd_bencher b(&image);
+
+  cout << "bench-write "
+       << " io_size " << io_size
+       << " io_threads " << io_threads
+       << " bytes " << io_bytes
+       << std::endl;
+
+  bufferptr bp(io_size);
+  bp.zero();
+  bufferlist bl;
+  bl.push_back(bp);
+
+  utime_t start = ceph_clock_now(NULL);
+  utime_t last;
+  unsigned ios = 0;
+
+  printf("  SEC       OPS   OPS/SEC   BYTES/SEC\n");
+  uint64_t off;
+  for (off = 0; off < io_bytes; off += io_size) {
+    b.wait_for(io_threads - 1);
+    while (b.start_write(io_threads, off, io_size, bl))
+      ios++;
+
+    utime_t now = ceph_clock_now(NULL);
+    utime_t elapsed = now - start;
+    if (elapsed.sec() != last.sec()) {
+      printf("%5d  %8d  %8.2lf  %8.2lf\n", (int)elapsed, (int)(ios - io_threads),
+            (double)(ios - io_threads) / elapsed,
+            (double)(off - io_threads * io_size) / elapsed);
+      last = elapsed;
+    }
+  }
+  b.wait_for(0);
+
+  utime_t now = ceph_clock_now(NULL);
+  double elapsed = now - start;
+
+  printf("elapsed: %5d  ops: %8d  ops/sec: %8.2lf  bytes/sec: %8.2lf\n", (int)elapsed, ios,
+            (double)ios / elapsed,
+            (double)off / elapsed);
+
+  return 0;
+}
+
 struct ExportContext {
   int fd;
   MyProgressContext pc;
@@ -1191,6 +1289,7 @@ enum {
   OPT_LOCK_LIST,
   OPT_LOCK_ADD,
   OPT_LOCK_REMOVE,
+  OPT_BENCH_WRITE,
 };
 
 static int get_cmd(const char *cmd, bool snapcmd, bool lockcmd)
@@ -1231,6 +1330,8 @@ static int get_cmd(const char *cmd, bool snapcmd, bool lockcmd)
       return OPT_SHOWMAPPED;
     if (strcmp(cmd, "unmap") == 0)
       return OPT_UNMAP;
+    if (strcmp(cmd, "bench-write") == 0)
+      return OPT_BENCH_WRITE;
   } else if (snapcmd) {
     if (strcmp(cmd, "create") == 0 ||
         strcmp(cmd, "add") == 0)
@@ -1311,6 +1412,7 @@ int main(int argc, const char **argv)
     *lock_tag = NULL;
   bool lflag = false;
   long long stripe_unit = 0, stripe_count = 0;
+  long long bench_io_size = 4096, bench_io_threads = 16, bench_bytes = 1 << 30;
 
   std::string val;
   std::ostringstream err;
@@ -1362,6 +1464,10 @@ int main(int argc, const char **argv)
        cerr << "rbd: " << err.str() << std::endl;
        return EXIT_FAILURE;
       }
+    } else if (ceph_argparse_withlonglong(args, i, &bench_io_size, &err, "--io-size", (char*)NULL)) {
+    } else if (ceph_argparse_withlonglong(args, i, &bench_io_threads, &err, "--io-threads", (char*)NULL)) {
+    } else if (ceph_argparse_withlonglong(args, i, &bench_bytes, &err, "--io-total", (char*)NULL)) {
+    } else if (ceph_argparse_withlonglong(args, i, &stripe_count, &err, "--stripe-count", (char*)NULL)) {
     } else if (ceph_argparse_witharg(args, i, &val, "--path", (char*)NULL)) {
       path = strdup(val.c_str());
     } else if (ceph_argparse_witharg(args, i, &val, "--dest", (char*)NULL)) {
@@ -1434,6 +1540,7 @@ if (!set_conf_param(v, p1, p2, p3)) { \
       case OPT_SNAP_UNPROTECT:
       case OPT_WATCH:
       case OPT_MAP:
+      case OPT_BENCH_WRITE:
       case OPT_LOCK_LIST:
        SET_CONF_PARAM(v, &imgname, NULL, NULL);
        break;
@@ -1607,7 +1714,7 @@ if (!set_conf_param(v, p1, p2, p3)) { \
        opt_cmd == OPT_WATCH || opt_cmd == OPT_COPY ||
        opt_cmd == OPT_FLATTEN || opt_cmd == OPT_CHILDREN ||
        opt_cmd == OPT_LOCK_LIST || opt_cmd == OPT_LOCK_ADD ||
-       opt_cmd == OPT_LOCK_REMOVE)) {
+       opt_cmd == OPT_LOCK_REMOVE || opt_cmd == OPT_BENCH_WRITE)) {
     r = rbd.open(io_ctx, image, imgname);
     if (r < 0) {
       cerr << "rbd: error opening image " << imgname << ": "
@@ -1941,6 +2048,14 @@ if (!set_conf_param(v, p1, p2, p3)) { \
       return EXIT_FAILURE;
     }
     break;
+
+  case OPT_BENCH_WRITE:
+    r = do_bench_write(image, bench_io_size, bench_io_threads, bench_bytes);
+    if (r < 0) {
+      cerr << "bench-write failed: " << cpp_strerror(-r) << std::endl;
+      return EXIT_FAILURE;
+    }
+    break;
   }
 
   return 0;

diff --git a/src/test/cli/rbd/help.t b/src/test/cli/rbd/help.t
index ce5365ab9167403063e11b9366f3f7ade428e2a3..3e81b3d9c6e9687ed47744368698d5ecbab23b13 100644 (file)
--- a/src/test/cli/rbd/help.t
+++ b/src/test/cli/rbd/help.t
@@ -37,6 +37,7 @@
     lock list <image-name>                      show locks held on an image
     lock add <image-name> <id> [--shared <tag>] take a lock called id on an image
     lock remove <image-name> <id> <locker>      release a lock on an image
+    bench-write <image-name> --io-size <bytes> --io-threads <num> --io-total <bytes>
   
   <image-name>, <snap-name> are [pool/]name[@snap], or you may specify
   individual pieces of names with -p/--pool, --image, and/or --snap.