librbd: fix AioCompletion race condition

If block completions finished before all of them where scheduled, the
pending_count would reach 0, and the AioCompletion would be destroyed
since the refcount would also reach 0. To fix this, hold a reference
to the AioCompletion while scheduling more block completions.

This caused a crash like this when running qemu:

1: ceph::__ceph_assert_fail (assertion=<value optimized out>, file=<value optimized out>, line=<value optimized out>, func=<value optimized out>) at common/assert.cc:86
2: Lock (ictx=0x7f83d0001f00, off=5368705024, len=0, buf=0x2fac000 "", c=0x2cebe00) at common/Mutex.h:118
3: add_block_completion (ictx=0x7f83d0001f00, off=5368705024, len=0, buf=0x2fac000 "", c=0x2cebe00) at librbd.cc:159
4: librbd::aio_read (ictx=0x7f83d0001f00, off=5368705024, len=0, buf=0x2fac000 "", c=0x2cebe00) at librbd.cc:1368
5: rbd_aio_rw_vector (bs=<value optimized out>, sector_num=10485752, qiov=<value optimized out>, nb_sectors=<value optimized out>, cb=<value optimized out>, opaque=<value optimized out>, write=0) at block/rbd.c:626
6: qemu_rbd_aio_readv (bs=<value optimized out>, sector_num=<value optimized out>, qiov=<value optimized out>, nb_sectors=<value optimized out>, cb=<value optimized out>, opaque=<value optimized out>) at block/rbd.c:649
7: bdrv_aio_readv (bs=0x2ce92e0, sector_num=10485752, qiov=0x2de08f0, nb_sectors=8, cb=<value optimized out>, opaque=<value optimized out>)

Fixes: #998
Signed-off-by: Josh Durgin <josh.durgin@dreamhost.com>

diff --git a/src/librbd.cc b/src/librbd.cc
index d1dcfafc5e807aa383323f4863a122e9c09bf525..30aec97c4d8969f90dd9d33ff39b648ae742b104 100644 (file)
--- a/src/librbd.cc
+++ b/src/librbd.cc
@@ -1303,6 +1303,7 @@ int aio_write(ImageCtx *ictx, uint64_t off, size_t len, const char *buf,
   if (r < 0)
     return r;
 
+  c->get();
   for (uint64_t i = start_block; i <= end_block; i++) {
     bufferlist bl;
     string oid = get_block_oid(&ictx->header, i);
@@ -1320,8 +1321,9 @@ int aio_write(ImageCtx *ictx, uint64_t off, size_t len, const char *buf,
     total_write += write_len;
     left -= write_len;
   }
-  return 0;
+  r = 0;
 done:
+  c->put();
   /* FIXME: cleanup all the allocated stuff */
   return r;
 }
@@ -1355,6 +1357,7 @@ int aio_read(ImageCtx *ictx, uint64_t off, size_t len,
   uint64_t block_size = get_block_size(&ictx->header);
   uint64_t left = len;
 
+  c->get();
   for (uint64_t i = start_block; i <= end_block; i++) {
     bufferlist bl;
     string oid = get_block_oid(&ictx->header, i);
@@ -1384,6 +1387,7 @@ int aio_read(ImageCtx *ictx, uint64_t off, size_t len,
   }
   ret = total_read;
 done:
+  c->put();
   return ret;
 }