tools: Don't delete, recreate and re-fill buffers in rados bench.

Fixes the high CPU usage and corrects rados bench scores on fast SSDs
and ramdisks/memstore.
For bench run on SSD, on Intel(R) Xeon(R) CPU E5-2640 v2 @ 2.00GHz
before this patch, times are:
write: real 5m0.169s, user 2m33.565s, sys 4m39.791s
seq: real 4m28.642s, user 1m35.250s, sys 6m42.948s
rand: real 5m0.258s, user 1m19.656s, sys 6m47.145s

After this patch:
write: real 5m1.162s, user 0m27.788s, sys 3m11.707s
seq: real 5m1.149s, user 2m23.278s, sys 4m14.427s
rand: real 5m1.021s, user 2m30.514s, sys 4m20.347s

Bench run: rados -p ssd bench 300 write|seq|read --no-cleanup

Note the increase in user time cpu on seq/read tests,
along with decreased sys cpu time; this is because there's
additional memcmp() that compares read objects with expected
contents. With less time spent memory juggling, more time is
spent performing more reads per second, increasing memcmp call
count and increasing amount of user cpu time used.

Signed-off-by: Piotr Dałek <piotr.dalek@ts.fujitsu.com>

diff --git a/src/common/obj_bencher.cc b/src/common/obj_bencher.cc
index 128a544a54422f6c741464c5f6d3d3b83bd4c698..f39f7a355683e9fef2004bfb2b6d8bc82a1bc415 100644 (file)
--- a/src/common/obj_bencher.cc
+++ b/src/common/obj_bencher.cc
@@ -388,10 +388,12 @@ int ObjBencher::write_bench(int secondsToRun,
     }
     lock.Unlock();
     //create new contents and name on the heap, and fill them
-    newContents = new bufferlist();
     newName = generate_object_name(data.started);
-    snprintf(data.object_contents, data.object_size, "I'm the %16dth object!", data.started);
-    newContents->append(data.object_contents, data.object_size);
+    newContents = contents[slot];
+    snprintf(newContents->c_str(), data.object_size, "I'm the %16dth object!", data.started);
+    // we wrote to buffer, going around internal crc cache, so invalidate it now.
+    newContents->invalidate_crc();
+
     completion_wait(slot);
     lock.Lock();
     r = completion_ret(slot);
@@ -411,8 +413,7 @@ int ObjBencher::write_bench(int secondsToRun,
     release_completion(slot);
     timePassed = ceph_clock_now(cct) - data.start_time;
 
-    //write new stuff to backend, then delete old stuff
-    //and save locations of new stuff for later deletion
+    //write new stuff to backend
     start_times[slot] = ceph_clock_now(cct);
     r = create_completion(slot, _aio_cb, &lc);
     if (r < 0)
@@ -421,10 +422,7 @@ int ObjBencher::write_bench(int secondsToRun,
     if (r < 0) {//naughty; doesn't clean up heap space.
       goto ERR;
     }
-    delete contents[slot];
     name[slot] = newName;
-    contents[slot] = newContents;
-    newContents = 0;
     lock.Lock();
     ++data.started;
     ++data.in_flight;
@@ -451,6 +449,7 @@ int ObjBencher::write_bench(int secondsToRun,
     lock.Unlock();
     release_completion(slot);
     delete contents[slot];
+    contents[slot] = 0;
   }
 
   timePassed = ceph_clock_now(cct) - data.start_time;
@@ -489,6 +488,9 @@ int ObjBencher::write_bench(int secondsToRun,
   sync_write(run_name_meta, b_write, sizeof(int)*3);
 
   completions_done();
+  for (int i = 0; i < concurrentios; i++)
+      if (contents[i])
+          delete contents[i];
 
   return 0;
 
@@ -497,7 +499,9 @@ int ObjBencher::write_bench(int secondsToRun,
   data.done = 1;
   lock.Unlock();
   pthread_join(print_thread, NULL);
-  delete newContents;
+  for (int i = 0; i < concurrentios; i++)
+      if (contents[i])
+          delete contents[i];
   return r;
 }
 
@@ -606,9 +610,11 @@ int ObjBencher::seq_read_bench(int seconds_to_run, int num_objects, int concurre
     release_completion(slot);
     cur_contents = contents[slot];
 
+    // invalidate internal crc cache
+    cur_contents->invalidate_crc();
+
     //start new read and check data if requested
     start_times[slot] = ceph_clock_now(cct);
-    contents[slot] = new bufferlist();
     create_completion(slot, _aio_cb, (void *)&lc);
     r = aio_read(newName, slot, contents[slot], data.object_size);
     if (r < 0) {
@@ -624,7 +630,6 @@ int ObjBencher::seq_read_bench(int seconds_to_run, int num_objects, int concurre
       ++errors;
     }
     name[slot] = newName;
-    delete cur_contents;
   }
 
   //wait for final reads to complete
@@ -799,9 +804,11 @@ int ObjBencher::rand_read_bench(int seconds_to_run, int num_objects, int concurr
     release_completion(slot);
     cur_contents = contents[slot];
 
+    // invalidate internal crc cache
+    cur_contents->invalidate_crc();
+
     //start new read and check data if requested
     start_times[slot] = ceph_clock_now(g_ceph_context);
-    contents[slot] = new bufferlist();
     create_completion(slot, _aio_cb, (void *)&lc);
     r = aio_read(newName, slot, contents[slot], data.object_size);
     if (r < 0) {
@@ -817,7 +824,6 @@ int ObjBencher::rand_read_bench(int seconds_to_run, int num_objects, int concurr
       ++errors;
     }
     name[slot] = newName;
-    delete cur_contents;
   }
 
   //wait for final reads to complete