From: Jason Dillaman <dillaman@redhat.com>
Date: Tue, 14 Jul 2020 21:38:56 +0000 (-0400)
Subject: librbd: managed_lock::BreakRequest needs a reference to AsioEngine
X-Git-Tag: wip-pdonnell-testing-20200918.022351~622^2~2
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=c87c669a7802c1d97409cd6c96d3d3b27c2e1330;p=ceph-ci.git

librbd: managed_lock::BreakRequest needs a reference to AsioEngine

The current usage of the asio::ContextWQ to similate an asynchronous blocklist
API call is resulting in deadlock in the rbd-mirror HA tests when multiple
blocklists are occurring concurrently. The next commit will switch to use the
neorados async MON command API (since librados doesn't offer one).

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
---

diff --git a/src/librbd/ExclusiveLock.cc b/src/librbd/ExclusiveLock.cc
index ebd3198c3d6..bdf67238df3 100644
--- a/src/librbd/ExclusiveLock.cc
+++ b/src/librbd/ExclusiveLock.cc
@@ -31,7 +31,7 @@ using ML = ManagedLock<I>;
 template <typename I>
 ExclusiveLock<I>::ExclusiveLock(I &image_ctx)
   : RefCountedObject(image_ctx.cct),
-    ML<I>(image_ctx.md_ctx, image_ctx.op_work_queue, image_ctx.header_oid,
+    ML<I>(image_ctx.md_ctx, *image_ctx.asio_engine, image_ctx.header_oid,
           image_ctx.image_watcher, managed_lock::EXCLUSIVE,
           image_ctx.config.template get_val<bool>("rbd_blacklist_on_break_lock"),
           image_ctx.config.template get_val<uint64_t>("rbd_blacklist_expire_seconds")),
diff --git a/src/librbd/ManagedLock.cc b/src/librbd/ManagedLock.cc
index 27514a9a09f..bc7245eb010 100644
--- a/src/librbd/ManagedLock.cc
+++ b/src/librbd/ManagedLock.cc
@@ -2,6 +2,9 @@
 // vim: ts=8 sw=2 smarttab
 
 #include "librbd/ManagedLock.h"
+#include "librbd/AsioEngine.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/Watcher.h"
 #include "librbd/asio/ContextWQ.h"
 #include "librbd/managed_lock/AcquireRequest.h"
 #include "librbd/managed_lock/BreakRequest.h"
@@ -10,8 +13,6 @@
 #include "librbd/managed_lock/ReacquireRequest.h"
 #include "librbd/managed_lock/Types.h"
 #include "librbd/managed_lock/Utils.h"
-#include "librbd/Watcher.h"
-#include "librbd/ImageCtx.h"
 #include "cls/lock/cls_lock_client.h"
 #include "common/dout.h"
 #include "common/errno.h"
@@ -63,13 +64,14 @@ using managed_lock::util::decode_lock_cookie;
 using managed_lock::util::encode_lock_cookie;
 
 template <typename I>
-ManagedLock<I>::ManagedLock(librados::IoCtx &ioctx, asio::ContextWQ *work_queue,
+ManagedLock<I>::ManagedLock(librados::IoCtx &ioctx, AsioEngine& asio_engine,
                             const string& oid, Watcher *watcher, Mode mode,
                             bool blacklist_on_break_lock,
                             uint32_t blacklist_expire_seconds)
   : m_lock(ceph::make_mutex(unique_lock_name("librbd::ManagedLock<I>::m_lock", this))),
     m_ioctx(ioctx), m_cct(reinterpret_cast<CephContext *>(ioctx.cct())),
-    m_work_queue(work_queue),
+    m_asio_engine(asio_engine),
+    m_work_queue(asio_engine.get_work_queue()),
     m_oid(oid),
     m_watcher(watcher),
     m_mode(mode),
@@ -267,7 +269,7 @@ void ManagedLock<I>::break_lock(const managed_lock::Locker &locker,
     } else {
       on_finish = new C_Tracked(m_async_op_tracker, on_finish);
       auto req = managed_lock::BreakRequest<I>::create(
-        m_ioctx, m_work_queue, m_oid, locker, m_mode == EXCLUSIVE,
+        m_ioctx, m_asio_engine, m_oid, locker, m_mode == EXCLUSIVE,
         m_blacklist_on_break_lock, m_blacklist_expire_seconds, force_break_lock,
         on_finish);
       req->send();
@@ -509,7 +511,7 @@ void ManagedLock<I>::handle_pre_acquire_lock(int r) {
 
   using managed_lock::AcquireRequest;
   AcquireRequest<I>* req = AcquireRequest<I>::create(
-    m_ioctx, m_watcher, m_work_queue, m_oid, m_cookie, m_mode == EXCLUSIVE,
+    m_ioctx, m_watcher, m_asio_engine, m_oid, m_cookie, m_mode == EXCLUSIVE,
     m_blacklist_on_break_lock, m_blacklist_expire_seconds,
     create_context_callback<
         ManagedLock<I>, &ManagedLock<I>::handle_acquire_lock>(this));
diff --git a/src/librbd/ManagedLock.h b/src/librbd/ManagedLock.h
index 9bf38ec3026..e1e95fe4302 100644
--- a/src/librbd/ManagedLock.h
+++ b/src/librbd/ManagedLock.h
@@ -17,8 +17,8 @@
 
 namespace librbd {
 
+struct AsioEngine;
 struct ImageCtx;
-
 namespace asio { struct ContextWQ; }
 namespace managed_lock { struct Locker; }
 
@@ -30,19 +30,19 @@ private:
 
 public:
   static ManagedLock *create(librados::IoCtx& ioctx,
-                             asio::ContextWQ *work_queue,
+                             AsioEngine& asio_engine,
                              const std::string& oid, Watcher *watcher,
                              managed_lock::Mode mode,
                              bool blacklist_on_break_lock,
                              uint32_t blacklist_expire_seconds) {
-    return new ManagedLock(ioctx, work_queue, oid, watcher, mode,
+    return new ManagedLock(ioctx, asio_engine, oid, watcher, mode,
                            blacklist_on_break_lock, blacklist_expire_seconds);
   }
   void destroy() {
     delete this;
   }
 
-  ManagedLock(librados::IoCtx& ioctx, asio::ContextWQ *work_queue,
+  ManagedLock(librados::IoCtx& ioctx, AsioEngine& asio_engine,
               const std::string& oid, Watcher *watcher,
               managed_lock::Mode mode, bool blacklist_on_break_lock,
               uint32_t blacklist_expire_seconds);
@@ -211,7 +211,8 @@ private:
 
   librados::IoCtx& m_ioctx;
   CephContext *m_cct;
-  asio::ContextWQ *m_work_queue;
+  AsioEngine& m_asio_engine;
+  asio::ContextWQ* m_work_queue;
   std::string m_oid;
   Watcher *m_watcher;
   managed_lock::Mode m_mode;
diff --git a/src/librbd/managed_lock/AcquireRequest.cc b/src/librbd/managed_lock/AcquireRequest.cc
index c0078b966aa..f869938ad0b 100644
--- a/src/librbd/managed_lock/AcquireRequest.cc
+++ b/src/librbd/managed_lock/AcquireRequest.cc
@@ -8,6 +8,7 @@
 #include "common/dout.h"
 #include "common/errno.h"
 #include "include/stringify.h"
+#include "librbd/AsioEngine.h"
 #include "librbd/ImageCtx.h"
 #include "librbd/Utils.h"
 #include "librbd/asio/ContextWQ.h"
@@ -33,21 +34,21 @@ namespace managed_lock {
 template <typename I>
 AcquireRequest<I>* AcquireRequest<I>::create(librados::IoCtx& ioctx,
                                              Watcher *watcher,
-                                             asio::ContextWQ *work_queue,
+                                             AsioEngine& asio_engine,
                                              const string& oid,
                                              const string& cookie,
                                              bool exclusive,
 					     bool blacklist_on_break_lock,
 					     uint32_t blacklist_expire_seconds,
                                              Context *on_finish) {
-    return new AcquireRequest(ioctx, watcher, work_queue, oid, cookie,
+    return new AcquireRequest(ioctx, watcher, asio_engine, oid, cookie,
                               exclusive, blacklist_on_break_lock,
                               blacklist_expire_seconds, on_finish);
 }
 
 template <typename I>
 AcquireRequest<I>::AcquireRequest(librados::IoCtx& ioctx, Watcher *watcher,
-                                  asio::ContextWQ *work_queue,
+                                  AsioEngine& asio_engine,
                                   const string& oid,
                                   const string& cookie, bool exclusive,
                                   bool blacklist_on_break_lock,
@@ -55,11 +56,12 @@ AcquireRequest<I>::AcquireRequest(librados::IoCtx& ioctx, Watcher *watcher,
                                   Context *on_finish)
   : m_ioctx(ioctx), m_watcher(watcher),
     m_cct(reinterpret_cast<CephContext *>(m_ioctx.cct())),
-    m_work_queue(work_queue), m_oid(oid), m_cookie(cookie),
+    m_asio_engine(asio_engine), m_oid(oid), m_cookie(cookie),
     m_exclusive(exclusive),
     m_blacklist_on_break_lock(blacklist_on_break_lock),
     m_blacklist_expire_seconds(blacklist_expire_seconds),
-    m_on_finish(new C_AsyncCallback<asio::ContextWQ>(work_queue, on_finish)) {
+    m_on_finish(new C_AsyncCallback<asio::ContextWQ>(
+      asio_engine.get_work_queue(), on_finish)) {
 }
 
 template <typename I>
@@ -147,7 +149,7 @@ void AcquireRequest<I>::send_break_lock() {
   Context *ctx = create_context_callback<
     AcquireRequest<I>, &AcquireRequest<I>::handle_break_lock>(this);
   auto req = BreakRequest<I>::create(
-    m_ioctx, m_work_queue, m_oid, m_locker, m_exclusive,
+    m_ioctx, m_asio_engine, m_oid, m_locker, m_exclusive,
     m_blacklist_on_break_lock, m_blacklist_expire_seconds, false, ctx);
   req->send();
 }
diff --git a/src/librbd/managed_lock/AcquireRequest.h b/src/librbd/managed_lock/AcquireRequest.h
index 56e85bfb733..094cd55b4ce 100644
--- a/src/librbd/managed_lock/AcquireRequest.h
+++ b/src/librbd/managed_lock/AcquireRequest.h
@@ -16,8 +16,8 @@ class Context;
 
 namespace librbd {
 
+class AsioEngine;
 class Watcher;
-namespace asio { struct ContextWQ; }
 
 namespace managed_lock {
 
@@ -29,7 +29,7 @@ private:
 
 public:
   static AcquireRequest* create(librados::IoCtx& ioctx, Watcher *watcher,
-                                asio::ContextWQ *work_queue,
+                                AsioEngine& asio_engine,
                                 const std::string& oid,
                                 const std::string& cookie,
                                 bool exclusive,
@@ -64,7 +64,7 @@ private:
    */
 
   AcquireRequest(librados::IoCtx& ioctx, Watcher *watcher,
-                 asio::ContextWQ *work_queue, const std::string& oid,
+                 AsioEngine& asio_engine, const std::string& oid,
                  const std::string& cookie, bool exclusive,
                  bool blacklist_on_break_lock,
                  uint32_t blacklist_expire_seconds, Context *on_finish);
@@ -72,7 +72,7 @@ private:
   librados::IoCtx& m_ioctx;
   Watcher *m_watcher;
   CephContext *m_cct;
-  asio::ContextWQ *m_work_queue;
+  AsioEngine& m_asio_engine;
   std::string m_oid;
   std::string m_cookie;
   bool m_exclusive;
diff --git a/src/librbd/managed_lock/BreakRequest.cc b/src/librbd/managed_lock/BreakRequest.cc
index d007380e3ae..62605704890 100644
--- a/src/librbd/managed_lock/BreakRequest.cc
+++ b/src/librbd/managed_lock/BreakRequest.cc
@@ -4,9 +4,11 @@
 #include "librbd/managed_lock/BreakRequest.h"
 #include "common/dout.h"
 #include "common/errno.h"
+#include "include/neorados/RADOS.hpp"
 #include "include/stringify.h"
 #include "cls/lock/cls_lock_client.h"
 #include "cls/lock/cls_lock_types.h"
+#include "librbd/AsioEngine.h"
 #include "librbd/ImageCtx.h"
 #include "librbd/Utils.h"
 #include "librbd/asio/ContextWQ.h"
@@ -48,13 +50,13 @@ struct C_BlacklistClient : public Context {
 
 template <typename I>
 BreakRequest<I>::BreakRequest(librados::IoCtx& ioctx,
-                              asio::ContextWQ *work_queue,
+                              AsioEngine& asio_engine,
                               const std::string& oid, const Locker &locker,
                               bool exclusive, bool blacklist_locker,
                               uint32_t blacklist_expire_seconds,
                               bool force_break_lock, Context *on_finish)
   : m_ioctx(ioctx), m_cct(reinterpret_cast<CephContext *>(m_ioctx.cct())),
-    m_work_queue(work_queue), m_oid(oid), m_locker(locker),
+    m_asio_engine(asio_engine), m_oid(oid), m_locker(locker),
     m_exclusive(exclusive), m_blacklist_locker(blacklist_locker),
     m_blacklist_expire_seconds(blacklist_expire_seconds),
     m_force_break_lock(force_break_lock), m_on_finish(on_finish) {
@@ -175,9 +177,9 @@ void BreakRequest<I>::send_blacklist() {
   using klass = BreakRequest<I>;
   Context *ctx = create_context_callback<klass, &klass::handle_blacklist>(
     this);
-  m_work_queue->queue(new C_BlacklistClient(m_ioctx, m_locker.address,
-                                            m_blacklist_expire_seconds, ctx),
-                      0);
+  m_asio_engine.get_work_queue()->queue(
+    new C_BlacklistClient(m_ioctx, m_locker.address,
+                          m_blacklist_expire_seconds, ctx), 0);
 }
 
 template <typename I>
diff --git a/src/librbd/managed_lock/BreakRequest.h b/src/librbd/managed_lock/BreakRequest.h
index 50bc0b0cb85..640c0146e07 100644
--- a/src/librbd/managed_lock/BreakRequest.h
+++ b/src/librbd/managed_lock/BreakRequest.h
@@ -19,6 +19,7 @@ class obj_watch_t;
 
 namespace librbd {
 
+class AsioEngine;
 class ImageCtx;
 template <typename> class Journal;
 namespace asio { struct ContextWQ; }
@@ -29,12 +30,12 @@ template <typename ImageCtxT = ImageCtx>
 class BreakRequest {
 public:
   static BreakRequest* create(librados::IoCtx& ioctx,
-                              asio::ContextWQ *work_queue,
+                              AsioEngine& asio_engine,
                               const std::string& oid, const Locker &locker,
                               bool exclusive, bool blacklist_locker,
                               uint32_t blacklist_expire_seconds,
                               bool force_break_lock, Context *on_finish) {
-    return new BreakRequest(ioctx, work_queue, oid, locker, exclusive,
+    return new BreakRequest(ioctx, asio_engine, oid, locker, exclusive,
                             blacklist_locker, blacklist_expire_seconds,
                             force_break_lock, on_finish);
   }
@@ -67,7 +68,7 @@ private:
 
   librados::IoCtx &m_ioctx;
   CephContext *m_cct;
-  asio::ContextWQ *m_work_queue;
+  AsioEngine& m_asio_engine;
   std::string m_oid;
   Locker m_locker;
   bool m_exclusive;
@@ -83,7 +84,7 @@ private:
 
   Locker m_refreshed_locker;
 
-  BreakRequest(librados::IoCtx& ioctx, asio::ContextWQ *work_queue,
+  BreakRequest(librados::IoCtx& ioctx, AsioEngine& asio_engine,
                const std::string& oid, const Locker &locker,
                bool exclusive, bool blacklist_locker,
                uint32_t blacklist_expire_seconds, bool force_break_lock,
diff --git a/src/test/librbd/managed_lock/test_mock_AcquireRequest.cc b/src/test/librbd/managed_lock/test_mock_AcquireRequest.cc
index 92b7b61a0e4..4edd448112d 100644
--- a/src/test/librbd/managed_lock/test_mock_AcquireRequest.cc
+++ b/src/test/librbd/managed_lock/test_mock_AcquireRequest.cc
@@ -29,7 +29,7 @@ struct BreakRequest<librbd::MockImageCtx> {
   Context *on_finish = nullptr;
   static BreakRequest *s_instance;
   static BreakRequest* create(librados::IoCtx& ioctx,
-                              asio::ContextWQ *work_queue,
+                              AsioEngine& asio_engine,
                               const std::string& oid, const Locker &locker,
                               bool exclusive, bool blacklist_locker,
                               uint32_t blacklist_expire_seconds,
@@ -156,8 +156,8 @@ TEST_F(TestMockManagedLockAcquireRequest, SuccessExclusive) {
 
   C_SaferCond ctx;
   MockAcquireRequest *req = MockAcquireRequest::create(mock_image_ctx.md_ctx,
-     mock_image_ctx.image_watcher, ictx->op_work_queue, mock_image_ctx.header_oid,
-     TEST_COOKIE, true, true, 0, &ctx);
+    mock_image_ctx.image_watcher, *ictx->asio_engine, mock_image_ctx.header_oid,
+    TEST_COOKIE, true, true, 0, &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
 }
@@ -176,8 +176,8 @@ TEST_F(TestMockManagedLockAcquireRequest, SuccessShared) {
 
   C_SaferCond ctx;
   MockAcquireRequest *req = MockAcquireRequest::create(mock_image_ctx.md_ctx,
-     mock_image_ctx.image_watcher, ictx->op_work_queue, mock_image_ctx.header_oid,
-     TEST_COOKIE, false, true, 0, &ctx);
+    mock_image_ctx.image_watcher, *ictx->asio_engine, mock_image_ctx.header_oid,
+    TEST_COOKIE, false, true, 0, &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
 }
@@ -201,8 +201,8 @@ TEST_F(TestMockManagedLockAcquireRequest, LockBusy) {
 
   C_SaferCond ctx;
   MockAcquireRequest *req = MockAcquireRequest::create(mock_image_ctx.md_ctx,
-     mock_image_ctx.image_watcher, ictx->op_work_queue, mock_image_ctx.header_oid,
-     TEST_COOKIE, true, true, 0, &ctx);
+    mock_image_ctx.image_watcher, *ictx->asio_engine, mock_image_ctx.header_oid,
+    TEST_COOKIE, true, true, 0, &ctx);
   req->send();
   ASSERT_EQ(-ENOENT, ctx.wait());
 }
@@ -219,8 +219,8 @@ TEST_F(TestMockManagedLockAcquireRequest, GetLockInfoError) {
 
   C_SaferCond ctx;
   MockAcquireRequest *req = MockAcquireRequest::create(mock_image_ctx.md_ctx,
-     mock_image_ctx.image_watcher, ictx->op_work_queue, mock_image_ctx.header_oid,
-     TEST_COOKIE, true, true, 0, &ctx);
+    mock_image_ctx.image_watcher, *ictx->asio_engine, mock_image_ctx.header_oid,
+    TEST_COOKIE, true, true, 0, &ctx);
   req->send();
   ASSERT_EQ(-EINVAL, ctx.wait());
 }
@@ -238,8 +238,8 @@ TEST_F(TestMockManagedLockAcquireRequest, GetLockInfoEmpty) {
 
   C_SaferCond ctx;
   MockAcquireRequest *req = MockAcquireRequest::create(mock_image_ctx.md_ctx,
-     mock_image_ctx.image_watcher, ictx->op_work_queue, mock_image_ctx.header_oid,
-     TEST_COOKIE, true, true, 0, &ctx);
+    mock_image_ctx.image_watcher, *ictx->asio_engine, mock_image_ctx.header_oid,
+    TEST_COOKIE, true, true, 0, &ctx);
   req->send();
   ASSERT_EQ(-EINVAL, ctx.wait());
 }
@@ -261,8 +261,8 @@ TEST_F(TestMockManagedLockAcquireRequest, BreakLockError) {
 
   C_SaferCond ctx;
   MockAcquireRequest *req = MockAcquireRequest::create(mock_image_ctx.md_ctx,
-     mock_image_ctx.image_watcher, ictx->op_work_queue, mock_image_ctx.header_oid,
-     TEST_COOKIE, true, true, 0, &ctx);
+    mock_image_ctx.image_watcher, *ictx->asio_engine, mock_image_ctx.header_oid,
+    TEST_COOKIE, true, true, 0, &ctx);
   req->send();
   ASSERT_EQ(-EINVAL, ctx.wait());
 }
diff --git a/src/test/librbd/managed_lock/test_mock_BreakRequest.cc b/src/test/librbd/managed_lock/test_mock_BreakRequest.cc
index 5a9940e49e4..e9ac8be7e9e 100644
--- a/src/test/librbd/managed_lock/test_mock_BreakRequest.cc
+++ b/src/test/librbd/managed_lock/test_mock_BreakRequest.cc
@@ -147,7 +147,7 @@ TEST_F(TestMockManagedLockBreakRequest, DeadLockOwner) {
   C_SaferCond ctx;
   Locker locker{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123};
   MockBreakRequest *req = MockBreakRequest::create(
-      mock_image_ctx.md_ctx, ictx->op_work_queue, mock_image_ctx.header_oid,
+      mock_image_ctx.md_ctx, *ictx->asio_engine, mock_image_ctx.header_oid,
       locker, true, true, 0, false, &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
@@ -176,7 +176,7 @@ TEST_F(TestMockManagedLockBreakRequest, ForceBreak) {
   C_SaferCond ctx;
   Locker locker{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123};
   MockBreakRequest *req = MockBreakRequest::create(
-      mock_image_ctx.md_ctx, ictx->op_work_queue, mock_image_ctx.header_oid,
+      mock_image_ctx.md_ctx, *ictx->asio_engine, mock_image_ctx.header_oid,
       locker, true, true, 0, true, &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
@@ -197,7 +197,7 @@ TEST_F(TestMockManagedLockBreakRequest, GetWatchersError) {
   C_SaferCond ctx;
   Locker locker{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123};
   MockBreakRequest *req = MockBreakRequest::create(
-      mock_image_ctx.md_ctx, ictx->op_work_queue, mock_image_ctx.header_oid,
+      mock_image_ctx.md_ctx, *ictx->asio_engine, mock_image_ctx.header_oid,
       locker, true, true, 0, false, &ctx);
   req->send();
   ASSERT_EQ(-EINVAL, ctx.wait());
@@ -218,7 +218,7 @@ TEST_F(TestMockManagedLockBreakRequest, GetWatchersAlive) {
   C_SaferCond ctx;
   Locker locker{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123};
   MockBreakRequest *req = MockBreakRequest::create(
-      mock_image_ctx.md_ctx, ictx->op_work_queue, mock_image_ctx.header_oid,
+      mock_image_ctx.md_ctx, *ictx->asio_engine, mock_image_ctx.header_oid,
       locker, true, true, 0, false, &ctx);
   req->send();
   ASSERT_EQ(-EAGAIN, ctx.wait());
@@ -244,7 +244,7 @@ TEST_F(TestMockManagedLockBreakRequest, GetLockerUpdated) {
   C_SaferCond ctx;
   Locker locker{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123};
   MockBreakRequest *req = MockBreakRequest::create(
-      mock_image_ctx.md_ctx, ictx->op_work_queue, mock_image_ctx.header_oid,
+      mock_image_ctx.md_ctx, *ictx->asio_engine, mock_image_ctx.header_oid,
       locker, true, false, 0, false, &ctx);
   req->send();
   ASSERT_EQ(-EAGAIN, ctx.wait());
@@ -270,7 +270,7 @@ TEST_F(TestMockManagedLockBreakRequest, GetLockerBusy) {
   C_SaferCond ctx;
   Locker locker{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123};
   MockBreakRequest *req = MockBreakRequest::create(
-      mock_image_ctx.md_ctx, ictx->op_work_queue, mock_image_ctx.header_oid,
+      mock_image_ctx.md_ctx, *ictx->asio_engine, mock_image_ctx.header_oid,
       locker, true, false, 0, false, &ctx);
   req->send();
   ASSERT_EQ(-EAGAIN, ctx.wait());
@@ -296,7 +296,7 @@ TEST_F(TestMockManagedLockBreakRequest, GetLockerMissing) {
   C_SaferCond ctx;
   Locker locker{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123};
   MockBreakRequest *req = MockBreakRequest::create(
-      mock_image_ctx.md_ctx, ictx->op_work_queue, mock_image_ctx.header_oid,
+      mock_image_ctx.md_ctx, *ictx->asio_engine, mock_image_ctx.header_oid,
       locker, true, false, 0, false, &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
@@ -320,7 +320,7 @@ TEST_F(TestMockManagedLockBreakRequest, GetLockerError) {
   C_SaferCond ctx;
   Locker locker{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123};
   MockBreakRequest *req = MockBreakRequest::create(
-      mock_image_ctx.md_ctx, ictx->op_work_queue, mock_image_ctx.header_oid,
+      mock_image_ctx.md_ctx, *ictx->asio_engine, mock_image_ctx.header_oid,
       locker, true, false, 0, false, &ctx);
   req->send();
   ASSERT_EQ(-EINVAL, ctx.wait());
@@ -348,7 +348,7 @@ TEST_F(TestMockManagedLockBreakRequest, BlacklistDisabled) {
   C_SaferCond ctx;
   Locker locker{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123};
   MockBreakRequest *req = MockBreakRequest::create(
-      mock_image_ctx.md_ctx, ictx->op_work_queue, mock_image_ctx.header_oid,
+      mock_image_ctx.md_ctx, *ictx->asio_engine, mock_image_ctx.header_oid,
       locker, true, false, 0, false, &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
@@ -376,7 +376,7 @@ TEST_F(TestMockManagedLockBreakRequest, BlacklistSelf) {
   C_SaferCond ctx;
   Locker locker{entity_name_t::CLIENT(456), "auto 123", "1.2.3.4:0/0", 123};
   MockBreakRequest *req = MockBreakRequest::create(
-      mock_image_ctx.md_ctx, ictx->op_work_queue, mock_image_ctx.header_oid,
+      mock_image_ctx.md_ctx, *ictx->asio_engine, mock_image_ctx.header_oid,
       locker, true, true, 0, false, &ctx);
   req->send();
   ASSERT_EQ(-EINVAL, ctx.wait());
@@ -404,7 +404,7 @@ TEST_F(TestMockManagedLockBreakRequest, BlacklistError) {
   C_SaferCond ctx;
   Locker locker{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123};
   MockBreakRequest *req = MockBreakRequest::create(
-      mock_image_ctx.md_ctx, ictx->op_work_queue, mock_image_ctx.header_oid,
+      mock_image_ctx.md_ctx, *ictx->asio_engine, mock_image_ctx.header_oid,
       locker, true, true, 0, false, &ctx);
   req->send();
   ASSERT_EQ(-EINVAL, ctx.wait());
@@ -433,7 +433,7 @@ TEST_F(TestMockManagedLockBreakRequest, BreakLockMissing) {
   C_SaferCond ctx;
   Locker locker{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123};
   MockBreakRequest *req = MockBreakRequest::create(
-      mock_image_ctx.md_ctx, ictx->op_work_queue, mock_image_ctx.header_oid,
+      mock_image_ctx.md_ctx, *ictx->asio_engine, mock_image_ctx.header_oid,
       locker, true, true, 0, false, &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
@@ -462,7 +462,7 @@ TEST_F(TestMockManagedLockBreakRequest, BreakLockError) {
   C_SaferCond ctx;
   Locker locker{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123};
   MockBreakRequest *req = MockBreakRequest::create(
-      mock_image_ctx.md_ctx, ictx->op_work_queue, mock_image_ctx.header_oid,
+      mock_image_ctx.md_ctx, *ictx->asio_engine, mock_image_ctx.header_oid,
       locker, true, true, 0, false, &ctx);
   req->send();
   ASSERT_EQ(-EINVAL, ctx.wait());
diff --git a/src/test/librbd/test_mock_ExclusiveLock.cc b/src/test/librbd/test_mock_ExclusiveLock.cc
index 34ae8c5a6ef..c0693f78a1d 100644
--- a/src/test/librbd/test_mock_ExclusiveLock.cc
+++ b/src/test/librbd/test_mock_ExclusiveLock.cc
@@ -40,7 +40,7 @@ struct Traits<MockExclusiveLockImageCtx> {
 
 template <>
 struct ManagedLock<MockExclusiveLockImageCtx> {
-  ManagedLock(librados::IoCtx& ioctx, asio::ContextWQ *work_queue,
+  ManagedLock(librados::IoCtx& ioctx, AsioEngine& asio_engine,
               const std::string& oid, librbd::MockImageWatcher *watcher,
               managed_lock::Mode  mode, bool blacklist_on_break_lock,
               uint32_t blacklist_expire_seconds)
diff --git a/src/test/librbd/test_mock_ManagedLock.cc b/src/test/librbd/test_mock_ManagedLock.cc
index bfad7422405..89fd9d20616 100644
--- a/src/test/librbd/test_mock_ManagedLock.cc
+++ b/src/test/librbd/test_mock_ManagedLock.cc
@@ -27,11 +27,11 @@ struct Traits<MockManagedLockImageCtx> {
 }
 
 struct MockMockManagedLock : public ManagedLock<MockManagedLockImageCtx> {
-  MockMockManagedLock(librados::IoCtx& ioctx, asio::ContextWQ *work_queue,
+  MockMockManagedLock(librados::IoCtx& ioctx, AsioEngine& asio_engine,
                  const std::string& oid, librbd::MockImageWatcher *watcher,
-                 managed_lock::Mode  mode, bool blacklist_on_break_lock, 
+                 managed_lock::Mode  mode, bool blacklist_on_break_lock,
                  uint32_t blacklist_expire_seconds)
-    : ManagedLock<MockManagedLockImageCtx>(ioctx, work_queue, oid, watcher, 
+    : ManagedLock<MockManagedLockImageCtx>(ioctx, asio_engine, oid, watcher,
       librbd::managed_lock::EXCLUSIVE, true, 0) {
   };
   virtual ~MockMockManagedLock() = default;
@@ -50,8 +50,8 @@ struct BaseRequest {
   Context *on_finish = nullptr;
 
   static T* create(librados::IoCtx& ioctx, MockImageWatcher *watcher,
-                   asio::ContextWQ *work_queue, const std::string& oid,
-                   const std::string& cookie, Context *on_finish) {
+                   const std::string& oid, const std::string& cookie,
+                   Context *on_finish) {
     ceph_assert(!s_requests.empty());
     T* req = s_requests.front();
     req->on_finish = on_finish;
@@ -68,16 +68,17 @@ template<typename T>
 std::list<T *> BaseRequest<T>::s_requests;
 
 template <>
-struct AcquireRequest<MockManagedLockImageCtx> : public BaseRequest<AcquireRequest<MockManagedLockImageCtx> > {
+struct AcquireRequest<MockManagedLockImageCtx>
+  : public BaseRequest<AcquireRequest<MockManagedLockImageCtx> > {
   static AcquireRequest* create(librados::IoCtx& ioctx,
 				MockImageWatcher *watcher,
-                                asio::ContextWQ *work_queue,
+                                AsioEngine& asio_engine,
                                 const std::string& oid,
                                 const std::string& cookie,
                                 bool exclusive, bool blacklist_on_break_lock,
                                 uint32_t blacklist_expire_seconds,
                                 Context *on_finish) {
-    return BaseRequest::create(ioctx, watcher, work_queue, oid, cookie, on_finish);
+    return BaseRequest::create(ioctx, watcher, oid, cookie, on_finish);
   }
 
   MOCK_METHOD0(send, void());
@@ -88,7 +89,7 @@ struct ReacquireRequest<MockManagedLockImageCtx> : public BaseRequest<ReacquireR
   static ReacquireRequest* create(librados::IoCtx &ioctx, const std::string& oid,
                                 const string& old_cookie, const std::string& new_cookie,
                                 bool exclusive, Context *on_finish) {
-    return BaseRequest::create(ioctx, nullptr, nullptr, oid, new_cookie,
+    return BaseRequest::create(ioctx, nullptr, oid, new_cookie,
                                on_finish);
   }
 
@@ -101,8 +102,7 @@ struct ReleaseRequest<MockManagedLockImageCtx> : public BaseRequest<ReleaseReque
                                 asio::ContextWQ *work_queue,
                                 const std::string& oid,
                                 const std::string& cookie, Context *on_finish) {
-    return BaseRequest::create(ioctx, watcher, work_queue, oid, cookie,
-                               on_finish);
+    return BaseRequest::create(ioctx, watcher, oid, cookie, on_finish);
   }
   MOCK_METHOD0(send, void());
 };
@@ -123,7 +123,7 @@ struct GetLockerRequest<MockManagedLockImageCtx> {
 template <>
 struct BreakRequest<MockManagedLockImageCtx> {
   static BreakRequest* create(librados::IoCtx& ioctx,
-                              asio::ContextWQ *work_queue,
+                              AsioEngine& asio_engine,
                               const std::string& oid, const Locker &locker,
                               bool exclusive, bool blacklist_locker,
                               uint32_t blacklist_expire_seconds,
@@ -211,7 +211,7 @@ public:
                   .WillOnce(CompleteContext(0, (asio::ContextWQ *)nullptr));
   }
 
-  void expect_post_reacquired_lock_handler(MockImageWatcher& watcher, 
+  void expect_post_reacquired_lock_handler(MockImageWatcher& watcher,
                         MockMockManagedLock &managed_lock, uint64_t &client_id) {
     expect_get_watch_handle(watcher);
     EXPECT_CALL(managed_lock, post_reacquire_lock_handler(_, _))
@@ -271,7 +271,7 @@ TEST_F(TestMockManagedLock, StateTransitions) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockManagedLockImageCtx mock_image_ctx(*ictx);
-  MockManagedLock managed_lock(ictx->md_ctx, ictx->op_work_queue,
+  MockManagedLock managed_lock(ictx->md_ctx, *ictx->asio_engine,
                                ictx->header_oid, mock_image_ctx.image_watcher,
                                librbd::managed_lock::EXCLUSIVE, true, 0);
   InSequence seq;
@@ -302,7 +302,7 @@ TEST_F(TestMockManagedLock, AcquireLockLockedState) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockManagedLockImageCtx mock_image_ctx(*ictx);
-  MockManagedLock managed_lock(ictx->md_ctx, ictx->op_work_queue,
+  MockManagedLock managed_lock(ictx->md_ctx, *ictx->asio_engine,
                                ictx->header_oid, mock_image_ctx.image_watcher,
                                librbd::managed_lock::EXCLUSIVE, true, 0);
   InSequence seq;
@@ -322,7 +322,7 @@ TEST_F(TestMockManagedLock, AcquireLockAlreadyLocked) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockManagedLockImageCtx mock_image_ctx(*ictx);
-  MockManagedLock managed_lock(ictx->md_ctx, ictx->op_work_queue,
+  MockManagedLock managed_lock(ictx->md_ctx, *ictx->asio_engine,
                                ictx->header_oid, mock_image_ctx.image_watcher,
                                librbd::managed_lock::EXCLUSIVE, true, 0);
   InSequence seq;
@@ -340,7 +340,7 @@ TEST_F(TestMockManagedLock, AcquireLockBusy) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockManagedLockImageCtx mock_image_ctx(*ictx);
-  MockManagedLock managed_lock(ictx->md_ctx, ictx->op_work_queue,
+  MockManagedLock managed_lock(ictx->md_ctx, *ictx->asio_engine,
                                ictx->header_oid, mock_image_ctx.image_watcher,
                                librbd::managed_lock::EXCLUSIVE, true, 0);
   InSequence seq;
@@ -358,7 +358,7 @@ TEST_F(TestMockManagedLock, AcquireLockError) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockManagedLockImageCtx mock_image_ctx(*ictx);
-  MockManagedLock managed_lock(ictx->md_ctx, ictx->op_work_queue,
+  MockManagedLock managed_lock(ictx->md_ctx, *ictx->asio_engine,
                                ictx->header_oid, mock_image_ctx.image_watcher,
                                librbd::managed_lock::EXCLUSIVE, true, 0);
   InSequence seq;
@@ -377,7 +377,7 @@ TEST_F(TestMockManagedLock, AcquireLockBlacklist) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockManagedLockImageCtx mock_image_ctx(*ictx);
-  MockManagedLock managed_lock(ictx->md_ctx, ictx->op_work_queue,
+  MockManagedLock managed_lock(ictx->md_ctx, *ictx->asio_engine,
                                ictx->header_oid, mock_image_ctx.image_watcher,
                                librbd::managed_lock::EXCLUSIVE, true, 0);
   InSequence seq;
@@ -396,7 +396,7 @@ TEST_F(TestMockManagedLock, ReleaseLockUnlockedState) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockManagedLockImageCtx mock_image_ctx(*ictx);
-  MockManagedLock managed_lock(ictx->md_ctx, ictx->op_work_queue,
+  MockManagedLock managed_lock(ictx->md_ctx, *ictx->asio_engine,
                                ictx->header_oid, mock_image_ctx.image_watcher,
                                librbd::managed_lock::EXCLUSIVE, true, 0);
   InSequence seq;
@@ -411,7 +411,7 @@ TEST_F(TestMockManagedLock, ReleaseLockBlacklist) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockManagedLockImageCtx mock_image_ctx(*ictx);
-  MockMockManagedLock managed_lock(ictx->md_ctx, ictx->op_work_queue,
+  MockMockManagedLock managed_lock(ictx->md_ctx, *ictx->asio_engine,
                                    ictx->header_oid, mock_image_ctx.image_watcher,
                                    librbd::managed_lock::EXCLUSIVE, true, 0);
   InSequence seq;
@@ -433,7 +433,7 @@ TEST_F(TestMockManagedLock, ReleaseLockError) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockManagedLockImageCtx mock_image_ctx(*ictx);
-  MockManagedLock managed_lock(ictx->md_ctx, ictx->op_work_queue,
+  MockManagedLock managed_lock(ictx->md_ctx, *ictx->asio_engine,
                                ictx->header_oid, mock_image_ctx.image_watcher,
                                librbd::managed_lock::EXCLUSIVE, true, 0);
   InSequence seq;
@@ -459,7 +459,7 @@ TEST_F(TestMockManagedLock, ConcurrentRequests) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockManagedLockImageCtx mock_image_ctx(*ictx);
-  MockManagedLock managed_lock(ictx->md_ctx, ictx->op_work_queue,
+  MockManagedLock managed_lock(ictx->md_ctx, *ictx->asio_engine,
                                ictx->header_oid, mock_image_ctx.image_watcher,
                                librbd::managed_lock::EXCLUSIVE, true, 0);
   InSequence seq;
@@ -516,7 +516,7 @@ TEST_F(TestMockManagedLock, ReacquireLock) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockManagedLockImageCtx mock_image_ctx(*ictx);
-  MockManagedLock managed_lock(ictx->md_ctx, ictx->op_work_queue,
+  MockManagedLock managed_lock(ictx->md_ctx, *ictx->asio_engine,
                                ictx->header_oid, mock_image_ctx.image_watcher,
                                librbd::managed_lock::EXCLUSIVE, true, 0);
   InSequence seq;
@@ -544,7 +544,7 @@ TEST_F(TestMockManagedLock, AttemptReacquireBlacklistedLock) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockManagedLockImageCtx mock_image_ctx(*ictx);
-  MockManagedLock managed_lock(ictx->md_ctx, ictx->op_work_queue,
+  MockManagedLock managed_lock(ictx->md_ctx, *ictx->asio_engine,
                                ictx->header_oid, mock_image_ctx.image_watcher,
                                librbd::managed_lock::EXCLUSIVE, true, 0);
   InSequence seq;
@@ -573,7 +573,7 @@ TEST_F(TestMockManagedLock, ReacquireBlacklistedLock) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockManagedLockImageCtx mock_image_ctx(*ictx);
-  MockManagedLock managed_lock(ictx->md_ctx, ictx->op_work_queue,
+  MockManagedLock managed_lock(ictx->md_ctx, *ictx->asio_engine,
                                ictx->header_oid, mock_image_ctx.image_watcher,
                                librbd::managed_lock::EXCLUSIVE, true, 0);
   InSequence seq;
@@ -610,7 +610,7 @@ TEST_F(TestMockManagedLock, ReacquireLockError) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockManagedLockImageCtx mock_image_ctx(*ictx);
-  MockManagedLock managed_lock(ictx->md_ctx, ictx->op_work_queue,
+  MockManagedLock managed_lock(ictx->md_ctx, *ictx->asio_engine,
                                ictx->header_oid, mock_image_ctx.image_watcher,
                                librbd::managed_lock::EXCLUSIVE, true, 0);
   InSequence seq;
@@ -645,7 +645,7 @@ TEST_F(TestMockManagedLock, ReacquireWithSameCookie) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockManagedLockImageCtx mock_image_ctx(*ictx);
-  MockMockManagedLock managed_lock(ictx->md_ctx, ictx->op_work_queue,
+  MockMockManagedLock managed_lock(ictx->md_ctx, *ictx->asio_engine,
                                ictx->header_oid, mock_image_ctx.image_watcher,
                                librbd::managed_lock::EXCLUSIVE, true, 0);
   InSequence seq;
@@ -675,7 +675,7 @@ TEST_F(TestMockManagedLock, ShutDownWhileWaiting) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockManagedLockImageCtx mock_image_ctx(*ictx);
-  MockMockManagedLock managed_lock(ictx->md_ctx, ictx->op_work_queue,
+  MockMockManagedLock managed_lock(ictx->md_ctx, *ictx->asio_engine,
                                ictx->header_oid, mock_image_ctx.image_watcher,
                                librbd::managed_lock::EXCLUSIVE, true, 0);
 
diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc
index 540a5a94fb7..cb21fc72396 100644
--- a/src/test/rbd_mirror/test_ImageReplayer.cc
+++ b/src/test/rbd_mirror/test_ImageReplayer.cc
@@ -176,7 +176,7 @@ public:
         cct, "rbd_mirror_concurrent_image_syncs"));
 
     m_instance_watcher = InstanceWatcher<>::create(
-        m_local_ioctx, m_threads->work_queue, nullptr,
+        m_local_ioctx, *m_threads->asio_engine, nullptr,
         m_image_sync_throttler.get());
     m_instance_watcher->handle_acquire_leader();
 
diff --git a/src/test/rbd_mirror/test_ImageSync.cc b/src/test/rbd_mirror/test_ImageSync.cc
index 6b9b2f7d407..646534ccbd7 100644
--- a/src/test/rbd_mirror/test_ImageSync.cc
+++ b/src/test/rbd_mirror/test_ImageSync.cc
@@ -80,7 +80,7 @@ public:
         cct, "rbd_mirror_concurrent_image_syncs");
 
     m_instance_watcher = rbd::mirror::InstanceWatcher<>::create(
-        m_local_io_ctx, m_threads->work_queue, nullptr, m_image_sync_throttler);
+      m_local_io_ctx, *m_threads->asio_engine, nullptr, m_image_sync_throttler);
     m_instance_watcher->handle_acquire_leader();
 
     ContextWQ* context_wq;
diff --git a/src/test/rbd_mirror/test_InstanceWatcher.cc b/src/test/rbd_mirror/test_InstanceWatcher.cc
index ba19dc1f41e..6b8176d8a92 100644
--- a/src/test/rbd_mirror/test_InstanceWatcher.cc
+++ b/src/test/rbd_mirror/test_InstanceWatcher.cc
@@ -44,7 +44,7 @@ public:
 
 TEST_F(TestInstanceWatcher, InitShutdown)
 {
-  InstanceWatcher<> instance_watcher(m_local_io_ctx, m_threads->work_queue,
+  InstanceWatcher<> instance_watcher(m_local_io_ctx, *m_threads->asio_engine,
                                      nullptr, nullptr, m_instance_id);
   std::vector<std::string> instance_ids;
   get_instances(&instance_ids);
@@ -93,7 +93,7 @@ TEST_F(TestInstanceWatcher, Remove)
   librados::IoCtx io_ctx;
   ASSERT_EQ("", connect_cluster_pp(cluster));
   ASSERT_EQ(0, cluster.ioctx_create(_local_pool_name.c_str(), io_ctx));
-  InstanceWatcher<> instance_watcher(m_local_io_ctx, m_threads->work_queue,
+  InstanceWatcher<> instance_watcher(m_local_io_ctx, *m_threads->asio_engine,
                                      nullptr, nullptr, "instance_id");
   // Init
   ASSERT_EQ(0, instance_watcher.init());
@@ -109,7 +109,7 @@ TEST_F(TestInstanceWatcher, Remove)
 
   // Remove
   C_SaferCond on_remove;
-  InstanceWatcher<>::remove_instance(m_local_io_ctx, m_threads->work_queue,
+  InstanceWatcher<>::remove_instance(m_local_io_ctx, *m_threads->asio_engine,
                                      "instance_id", &on_remove);
   ASSERT_EQ(0, on_remove.wait());
 
@@ -126,7 +126,7 @@ TEST_F(TestInstanceWatcher, Remove)
 
   // Remove NOENT
   C_SaferCond on_remove_noent;
-  InstanceWatcher<>::remove_instance(m_local_io_ctx, m_threads->work_queue,
+  InstanceWatcher<>::remove_instance(m_local_io_ctx, *m_threads->asio_engine,
                                      instance_id, &on_remove_noent);
   ASSERT_EQ(0, on_remove_noent.wait());
 }
diff --git a/src/test/rbd_mirror/test_mock_InstanceWatcher.cc b/src/test/rbd_mirror/test_mock_InstanceWatcher.cc
index 7920edde48c..9aadc33423a 100644
--- a/src/test/rbd_mirror/test_mock_InstanceWatcher.cc
+++ b/src/test/rbd_mirror/test_mock_InstanceWatcher.cc
@@ -29,7 +29,7 @@ struct ManagedLock<MockTestImageCtx> {
   static ManagedLock* s_instance;
 
   static ManagedLock *create(librados::IoCtx& ioctx,
-                             librbd::asio::ContextWQ *work_queue,
+                             librbd::AsioEngine& asio_engine,
                              const std::string& oid, librbd::Watcher *watcher,
                              managed_lock::Mode  mode,
                              bool blacklist_on_break_lock,
@@ -67,10 +67,11 @@ struct Threads<librbd::MockTestImageCtx> {
   ceph::mutex &timer_lock;
   SafeTimer *timer;
   librbd::asio::ContextWQ *work_queue;
+  librbd::AsioEngine* asio_engine;
 
   Threads(Threads<librbd::ImageCtx> *threads)
     : timer_lock(threads->timer_lock), timer(threads->timer),
-      work_queue(threads->work_queue) {
+      work_queue(threads->work_queue), asio_engine(threads->asio_engine) {
   }
 };
 
@@ -219,7 +220,7 @@ TEST_F(TestMockInstanceWatcher, InitShutdown) {
   librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx));
 
   auto instance_watcher = new MockInstanceWatcher(
-      m_local_io_ctx, m_mock_threads->work_queue, nullptr, nullptr,
+      m_local_io_ctx, *m_mock_threads->asio_engine, nullptr, nullptr,
       m_instance_id);
   InSequence seq;
 
@@ -244,7 +245,7 @@ TEST_F(TestMockInstanceWatcher, InitError) {
   librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx));
 
   auto instance_watcher = new MockInstanceWatcher(
-      m_local_io_ctx, m_mock_threads->work_queue, nullptr, nullptr,
+      m_local_io_ctx, *m_mock_threads->asio_engine, nullptr, nullptr,
       m_instance_id);
   InSequence seq;
 
@@ -265,7 +266,7 @@ TEST_F(TestMockInstanceWatcher, ShutdownError) {
   librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx));
 
   auto instance_watcher = new MockInstanceWatcher(
-      m_local_io_ctx, m_mock_threads->work_queue, nullptr, nullptr,
+      m_local_io_ctx, *m_mock_threads->asio_engine, nullptr, nullptr,
       m_instance_id);
   InSequence seq;
 
@@ -302,7 +303,7 @@ TEST_F(TestMockInstanceWatcher, Remove) {
 
   C_SaferCond on_remove;
   MockInstanceWatcher::remove_instance(m_local_io_ctx,
-                                       m_mock_threads->work_queue,
+                                       *m_mock_threads->asio_engine,
                                        "instance_id", &on_remove);
   ASSERT_EQ(0, on_remove.wait());
   ASSERT_EQ(0, on_destroy.wait());
@@ -321,7 +322,7 @@ TEST_F(TestMockInstanceWatcher, RemoveNoent) {
 
   C_SaferCond on_remove;
   MockInstanceWatcher::remove_instance(m_local_io_ctx,
-                                       m_mock_threads->work_queue,
+                                       *m_mock_threads->asio_engine,
                                        "instance_id", &on_remove);
   ASSERT_EQ(0, on_remove.wait());
   ASSERT_EQ(0, on_destroy.wait());
@@ -335,7 +336,7 @@ TEST_F(TestMockInstanceWatcher, ImageAcquireRelease) {
   librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1));
   MockInstanceReplayer mock_instance_replayer1;
   auto instance_watcher1 = MockInstanceWatcher::create(
-      io_ctx1, m_mock_threads->work_queue, &mock_instance_replayer1, nullptr);
+      io_ctx1, *m_mock_threads->asio_engine, &mock_instance_replayer1, nullptr);
 
   librados::Rados cluster;
   librados::IoCtx io_ctx2;
@@ -345,7 +346,7 @@ TEST_F(TestMockInstanceWatcher, ImageAcquireRelease) {
   librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2));
   MockInstanceReplayer mock_instance_replayer2;
   auto instance_watcher2 = MockInstanceWatcher::create(
-    io_ctx2, m_mock_threads->work_queue, &mock_instance_replayer2, nullptr);
+    io_ctx2, *m_mock_threads->asio_engine, &mock_instance_replayer2, nullptr);
 
   InSequence seq;
 
@@ -418,7 +419,7 @@ TEST_F(TestMockInstanceWatcher, PeerImageRemoved) {
   librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1));
   MockInstanceReplayer mock_instance_replayer1;
   auto instance_watcher1 = MockInstanceWatcher::create(
-      io_ctx1, m_mock_threads->work_queue, &mock_instance_replayer1, nullptr);
+      io_ctx1, *m_mock_threads->asio_engine, &mock_instance_replayer1, nullptr);
 
   librados::Rados cluster;
   librados::IoCtx io_ctx2;
@@ -428,7 +429,7 @@ TEST_F(TestMockInstanceWatcher, PeerImageRemoved) {
   librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2));
   MockInstanceReplayer mock_instance_replayer2;
   auto instance_watcher2 = MockInstanceWatcher::create(
-    io_ctx2, m_mock_threads->work_queue, &mock_instance_replayer2, nullptr);
+    io_ctx2, *m_mock_threads->asio_engine, &mock_instance_replayer2, nullptr);
 
   InSequence seq;
 
@@ -484,7 +485,7 @@ TEST_F(TestMockInstanceWatcher, ImageAcquireReleaseCancel) {
   librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx));
 
   auto instance_watcher = new MockInstanceWatcher(
-      m_local_io_ctx, m_mock_threads->work_queue, nullptr, nullptr,
+      m_local_io_ctx, *m_mock_threads->asio_engine, nullptr, nullptr,
       m_instance_id);
   InSequence seq;
 
@@ -552,7 +553,7 @@ TEST_F(TestMockInstanceWatcher, PeerImageAcquireWatchDNE) {
 
   MockInstanceReplayer mock_instance_replayer;
   auto instance_watcher = new MockInstanceWatcher(
-      m_local_io_ctx, m_mock_threads->work_queue, &mock_instance_replayer,
+      m_local_io_ctx, *m_mock_threads->asio_engine, &mock_instance_replayer,
       nullptr, m_instance_id);
   InSequence seq;
 
@@ -584,7 +585,7 @@ TEST_F(TestMockInstanceWatcher, PeerImageReleaseWatchDNE) {
 
   MockInstanceReplayer mock_instance_replayer;
   auto instance_watcher = new MockInstanceWatcher(
-      m_local_io_ctx, m_mock_threads->work_queue, &mock_instance_replayer,
+      m_local_io_ctx, *m_mock_threads->asio_engine, &mock_instance_replayer,
       nullptr, m_instance_id);
   InSequence seq;
 
@@ -615,7 +616,7 @@ TEST_F(TestMockInstanceWatcher, PeerImageRemovedCancel) {
   librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx));
 
   auto instance_watcher = new MockInstanceWatcher(
-      m_local_io_ctx, m_mock_threads->work_queue, nullptr, nullptr,
+      m_local_io_ctx, *m_mock_threads->asio_engine, nullptr, nullptr,
       m_instance_id);
   InSequence seq;
 
@@ -679,7 +680,7 @@ public:
     librados::IoCtx& io_ctx1 = m_local_io_ctx;
     librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1));
     instance_watcher1 = MockInstanceWatcher::create(io_ctx1,
-                                                    m_mock_threads->work_queue,
+                                                    *m_mock_threads->asio_engine,
                                                     nullptr,
                                                     &mock_image_sync_throttler);
     EXPECT_EQ("", connect_cluster_pp(cluster));
@@ -687,7 +688,7 @@ public:
     instance_id2 = stringify(io_ctx2.get_instance_id());
     librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2));
     instance_watcher2 = MockInstanceWatcher::create(io_ctx2,
-                                                    m_mock_threads->work_queue,
+                                                    *m_mock_threads->asio_engine,
                                                     nullptr,
                                                     &mock_image_sync_throttler);
     InSequence seq;
diff --git a/src/test/rbd_mirror/test_mock_LeaderWatcher.cc b/src/test/rbd_mirror/test_mock_LeaderWatcher.cc
index 9365a9b4314..06ac8f69acb 100644
--- a/src/test/rbd_mirror/test_mock_LeaderWatcher.cc
+++ b/src/test/rbd_mirror/test_mock_LeaderWatcher.cc
@@ -1,6 +1,7 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#include "librbd/AsioEngine.h"
 #include "librbd/Utils.h"
 #include "test/librbd/mock/MockImageCtx.h"
 #include "test/rbd_mirror/test_mock_fixture.h"
@@ -60,11 +61,11 @@ MockManagedLock *MockManagedLock::s_instance = nullptr;
 
 template <>
 struct ManagedLock<MockTestImageCtx> {
-  ManagedLock(librados::IoCtx& ioctx, librbd::asio::ContextWQ *work_queue,
+  ManagedLock(librados::IoCtx& ioctx, librbd::AsioEngine& asio_engine,
               const std::string& oid, librbd::Watcher *watcher,
               managed_lock::Mode  mode, bool blacklist_on_break_lock,
               uint32_t blacklist_expire_seconds)
-    : m_work_queue(work_queue) {
+    : m_work_queue(asio_engine.get_work_queue()) {
     MockManagedLock::get_instance().construct();
   }
 
@@ -185,10 +186,11 @@ struct Threads<librbd::MockTestImageCtx> {
   ceph::mutex &timer_lock;
   SafeTimer *timer;
   librbd::asio::ContextWQ *work_queue;
+  librbd::AsioEngine* asio_engine;
 
   Threads(Threads<librbd::ImageCtx> *threads)
     : timer_lock(threads->timer_lock), timer(threads->timer),
-      work_queue(threads->work_queue) {
+      work_queue(threads->work_queue), asio_engine(threads->asio_engine) {
   }
 };
 
diff --git a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc
index 0470fb5efd8..3b262b77528 100644
--- a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc
+++ b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc
@@ -132,7 +132,7 @@ struct InstanceWatcher<librbd::MockTestImageCtx> {
   static InstanceWatcher* s_instance;
 
   static InstanceWatcher* create(
-      librados::IoCtx &ioctx, librbd::asio::ContextWQ* work_queue,
+      librados::IoCtx &ioctx, librbd::AsioEngine& asio_engine,
       InstanceReplayer<librbd::MockTestImageCtx>* instance_replayer,
       Throttler<librbd::MockTestImageCtx> *image_sync_throttler) {
     ceph_assert(s_instance != nullptr);
@@ -250,10 +250,11 @@ struct Threads<librbd::MockTestImageCtx> {
   ceph::mutex &timer_lock;
   SafeTimer *timer;
   librbd::asio::ContextWQ *work_queue;
+  librbd::AsioEngine* asio_engine;
 
   Threads(Threads<librbd::ImageCtx> *threads)
     : timer_lock(threads->timer_lock), timer(threads->timer),
-      work_queue(threads->work_queue) {
+      work_queue(threads->work_queue), asio_engine(threads->asio_engine) {
   }
 };
 
diff --git a/src/tools/rbd_mirror/InstanceWatcher.cc b/src/tools/rbd_mirror/InstanceWatcher.cc
index 2ebce31e82e..f7c31013743 100644
--- a/src/tools/rbd_mirror/InstanceWatcher.cc
+++ b/src/tools/rbd_mirror/InstanceWatcher.cc
@@ -6,6 +6,7 @@
 #include "common/debug.h"
 #include "common/errno.h"
 #include "cls/rbd/cls_rbd_client.h"
+#include "librbd/AsioEngine.h"
 #include "librbd/ManagedLock.h"
 #include "librbd/Utils.h"
 #include "librbd/asio/ContextWQ.h"
@@ -59,9 +60,9 @@ struct C_RemoveInstanceRequest : public Context {
   Context *on_finish;
 
   C_RemoveInstanceRequest(librados::IoCtx &io_ctx,
-                          librbd::asio::ContextWQ *work_queue,
+                          librbd::AsioEngine& asio_engine,
                           const std::string &instance_id, Context *on_finish)
-    : instance_watcher(io_ctx, work_queue, nullptr, nullptr, instance_id),
+    : instance_watcher(io_ctx, asio_engine, nullptr, nullptr, instance_id),
       on_finish(on_finish) {
   }
 
@@ -303,37 +304,38 @@ void InstanceWatcher<I>::get_instances(librados::IoCtx &io_ctx,
 
 template <typename I>
 void InstanceWatcher<I>::remove_instance(librados::IoCtx &io_ctx,
-                                         librbd::asio::ContextWQ *work_queue,
+                                         librbd::AsioEngine& asio_engine,
                                          const std::string &instance_id,
                                          Context *on_finish) {
-  auto req = new C_RemoveInstanceRequest<I>(io_ctx, work_queue, instance_id,
+  auto req = new C_RemoveInstanceRequest<I>(io_ctx, asio_engine, instance_id,
                                             on_finish);
   req->send();
 }
 
 template <typename I>
 InstanceWatcher<I> *InstanceWatcher<I>::create(
-    librados::IoCtx &io_ctx, librbd::asio::ContextWQ *work_queue,
+    librados::IoCtx &io_ctx, librbd::AsioEngine& asio_engine,
     InstanceReplayer<I> *instance_replayer,
     Throttler<I> *image_sync_throttler) {
-  return new InstanceWatcher<I>(io_ctx, work_queue, instance_replayer,
+  return new InstanceWatcher<I>(io_ctx, asio_engine, instance_replayer,
                                 image_sync_throttler,
                                 stringify(io_ctx.get_instance_id()));
 }
 
 template <typename I>
 InstanceWatcher<I>::InstanceWatcher(librados::IoCtx &io_ctx,
-                                    librbd::asio::ContextWQ *work_queue,
+                                    librbd::AsioEngine& asio_engine,
                                     InstanceReplayer<I> *instance_replayer,
                                     Throttler<I> *image_sync_throttler,
                                     const std::string &instance_id)
-  : Watcher(io_ctx, work_queue, RBD_MIRROR_INSTANCE_PREFIX + instance_id),
+  : Watcher(io_ctx, asio_engine.get_work_queue(),
+            RBD_MIRROR_INSTANCE_PREFIX + instance_id),
     m_instance_replayer(instance_replayer),
     m_image_sync_throttler(image_sync_throttler), m_instance_id(instance_id),
     m_lock(ceph::make_mutex(
       unique_lock_name("rbd::mirror::InstanceWatcher::m_lock", this))),
     m_instance_lock(librbd::ManagedLock<I>::create(
-      m_ioctx, m_work_queue, m_oid, this, librbd::managed_lock::EXCLUSIVE, true,
+      m_ioctx, asio_engine, m_oid, this, librbd::managed_lock::EXCLUSIVE, true,
       m_cct->_conf.get_val<uint64_t>("rbd_blacklist_expire_seconds"))) {
 }
 
diff --git a/src/tools/rbd_mirror/InstanceWatcher.h b/src/tools/rbd_mirror/InstanceWatcher.h
index c6d983c7e80..08e40b40bf1 100644
--- a/src/tools/rbd_mirror/InstanceWatcher.h
+++ b/src/tools/rbd_mirror/InstanceWatcher.h
@@ -17,9 +17,9 @@
 
 namespace librbd {
 
+class AsioEngine;
 class ImageCtx;
 template <typename> class ManagedLock;
-namespace asio { struct ContextWQ; }
 
 } // namespace librbd
 
@@ -38,19 +38,19 @@ public:
                             std::vector<std::string> *instance_ids,
                             Context *on_finish);
   static void remove_instance(librados::IoCtx &io_ctx,
-                              librbd::asio::ContextWQ *work_queue,
+                              librbd::AsioEngine& asio_engine,
                               const std::string &instance_id,
                               Context *on_finish);
 
   static InstanceWatcher *create(
-    librados::IoCtx &io_ctx, librbd::asio::ContextWQ *work_queue,
+    librados::IoCtx &io_ctx, librbd::AsioEngine& asio_engine,
     InstanceReplayer<ImageCtxT> *instance_replayer,
     Throttler<ImageCtxT> *image_sync_throttler);
   void destroy() {
     delete this;
   }
 
-  InstanceWatcher(librados::IoCtx &io_ctx, librbd::asio::ContextWQ *work_queue,
+  InstanceWatcher(librados::IoCtx &io_ctx, librbd::AsioEngine& asio_engine,
                   InstanceReplayer<ImageCtxT> *instance_replayer,
                   Throttler<ImageCtxT> *image_sync_throttler,
                   const std::string &instance_id);
diff --git a/src/tools/rbd_mirror/Instances.cc b/src/tools/rbd_mirror/Instances.cc
index d5ac0614f0e..4b59365cd43 100644
--- a/src/tools/rbd_mirror/Instances.cc
+++ b/src/tools/rbd_mirror/Instances.cc
@@ -262,7 +262,7 @@ void Instances<I>::remove_instances(const Instances<I>::clock_t::time_point& tim
 
   auto gather_ctx = new C_Gather(m_cct, ctx);
   for (auto& instance_id : instance_ids) {
-    InstanceWatcher<I>::remove_instance(m_ioctx, m_threads->work_queue,
+    InstanceWatcher<I>::remove_instance(m_ioctx, *m_threads->asio_engine,
                                         instance_id, gather_ctx->new_sub());
   }
 
diff --git a/src/tools/rbd_mirror/LeaderWatcher.cc b/src/tools/rbd_mirror/LeaderWatcher.cc
index 844d7c7811d..c6eed29cf54 100644
--- a/src/tools/rbd_mirror/LeaderWatcher.cc
+++ b/src/tools/rbd_mirror/LeaderWatcher.cc
@@ -36,8 +36,8 @@ LeaderWatcher<I>::LeaderWatcher(Threads<I> *threads, librados::IoCtx &io_ctx,
 			    io_ctx.get_pool_name())),
     m_notifier_id(librados::Rados(io_ctx).get_instance_id()),
     m_instance_id(stringify(m_notifier_id)),
-    m_leader_lock(new LeaderLock(m_ioctx, m_work_queue, m_oid, this, true,
-                                 m_cct->_conf.get_val<uint64_t>(
+    m_leader_lock(new LeaderLock(m_ioctx, *m_threads->asio_engine, m_oid, this,
+                                 true, m_cct->_conf.get_val<uint64_t>(
                                    "rbd_blacklist_expire_seconds"))) {
 }
 
diff --git a/src/tools/rbd_mirror/LeaderWatcher.h b/src/tools/rbd_mirror/LeaderWatcher.h
index 60dbd20bd99..223cf3e20c8 100644
--- a/src/tools/rbd_mirror/LeaderWatcher.h
+++ b/src/tools/rbd_mirror/LeaderWatcher.h
@@ -119,12 +119,13 @@ private:
   public:
     typedef librbd::ManagedLock<ImageCtxT> Parent;
 
-    LeaderLock(librados::IoCtx& ioctx, librbd::asio::ContextWQ *work_queue,
+    LeaderLock(librados::IoCtx& ioctx, librbd::AsioEngine& asio_engine,
                const std::string& oid, LeaderWatcher *watcher,
                bool blacklist_on_break_lock,
                uint32_t blacklist_expire_seconds)
-      : Parent(ioctx, work_queue, oid, watcher, librbd::managed_lock::EXCLUSIVE,
-               blacklist_on_break_lock, blacklist_expire_seconds),
+      : Parent(ioctx, asio_engine, oid, watcher,
+               librbd::managed_lock::EXCLUSIVE, blacklist_on_break_lock,
+               blacklist_expire_seconds),
         watcher(watcher) {
     }
 
diff --git a/src/tools/rbd_mirror/NamespaceReplayer.cc b/src/tools/rbd_mirror/NamespaceReplayer.cc
index 10382c914b6..570a829f823 100644
--- a/src/tools/rbd_mirror/NamespaceReplayer.cc
+++ b/src/tools/rbd_mirror/NamespaceReplayer.cc
@@ -386,7 +386,7 @@ void NamespaceReplayer<I>::init_instance_watcher() {
   ceph_assert(!m_instance_watcher);
 
   m_instance_watcher.reset(InstanceWatcher<I>::create(
-      m_local_io_ctx, m_threads->work_queue, m_instance_replayer.get(),
+      m_local_io_ctx, *m_threads->asio_engine, m_instance_replayer.get(),
       m_image_sync_throttler));
   auto ctx = create_context_callback<NamespaceReplayer<I>,
       &NamespaceReplayer<I>::handle_init_instance_watcher>(this);
diff --git a/src/tools/rbd_mirror/Threads.h b/src/tools/rbd_mirror/Threads.h
index 6cbac09bc82..91c923ab4a0 100644
--- a/src/tools/rbd_mirror/Threads.h
+++ b/src/tools/rbd_mirror/Threads.h
@@ -23,10 +23,8 @@ namespace mirror {
 
 template <typename ImageCtxT = librbd::ImageCtx>
 class Threads {
-private:
-  librbd::AsioEngine* asio_engine = nullptr;
-
 public:
+  librbd::AsioEngine* asio_engine = nullptr;
   librbd::asio::ContextWQ* work_queue = nullptr;
 
   SafeTimer *timer = nullptr;