From: Yuan Lu <yuan.y.lu@intel.com>
Date: Thu, 11 Jun 2020 11:11:54 +0000 (+0800)
Subject: librbd: add InitRequest and ShutdownRequest state machine
X-Git-Tag: wip-pdonnell-testing-20200918.022351~471^2~1
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=99fdc0a3ce999da9cd4d4060b28278520174e01d;p=ceph-ci.git

librbd: add InitRequest and ShutdownRequest state machine

Signed-off-by: Peterson, Scott <scott.d.peterson@intel.com>
Signed-off-by: Li, Xiaoyan <xiaoyan.li@intel.com>
Signed-off-by: Lu, Yuan <yuan.y.lu@intel.com>
Signed-off-by: Chamarthy, Mahati <mahati.chamarthy@intel.com>
---

diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt
index c9f6146ec98..c0d0120ce3d 100644
--- a/src/librbd/CMakeLists.txt
+++ b/src/librbd/CMakeLists.txt
@@ -194,11 +194,13 @@ if(WITH_RBD_RWL)
   set(librbd_internal_srcs
     ${librbd_internal_srcs}
     cache/rwl/ImageCacheState.cc
+    cache/rwl/InitRequest.cc
     cache/rwl/LogEntry.cc
     cache/rwl/LogMap.cc
     cache/rwl/LogOperation.cc
     cache/rwl/ReadRequest.cc
     cache/rwl/Request.cc
+    cache/rwl/ShutdownRequest.cc
     cache/rwl/SyncPoint.cc
     cache/rwl/Types.cc
     cache/ReplicatedWriteLog.cc)
diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h
index c8c3caf97ab..26d993e4c25 100644
--- a/src/librbd/ImageCtx.h
+++ b/src/librbd/ImageCtx.h
@@ -57,7 +57,12 @@ namespace librbd {
   template <typename> class PluginRegistry;
 
   namespace asio { struct ContextWQ; }
-  namespace cache { template <typename> class ImageCache; }
+  namespace cache {
+  template <typename> class ImageCache;
+  namespace rwl {
+  template <typename> class ImageCacheState;
+  } // namespace rwl
+  } // namespace cache
   namespace exclusive_lock { struct Policy; }
   namespace io {
   class AioCompletion;
@@ -177,6 +182,7 @@ namespace librbd {
     file_layout_t layout;
 
     cache::ImageCache<ImageCtx> *image_cache = nullptr;
+    cache::rwl::ImageCacheState<ImageCtx> *cache_state = nullptr;
 
     Readahead readahead;
     std::atomic<uint64_t> total_bytes_read = {0};
diff --git a/src/librbd/cache/ReplicatedWriteLog.cc b/src/librbd/cache/ReplicatedWriteLog.cc
index bcc434a74c7..d5763a75e66 100644
--- a/src/librbd/cache/ReplicatedWriteLog.cc
+++ b/src/librbd/cache/ReplicatedWriteLog.cc
@@ -1156,7 +1156,12 @@ void ReplicatedWriteLog<I>::aio_compare_and_write(Extents &&image_extents,
                                      << "cw_req=" << cw_req << dendl;
 
           /* Compare read_bl to cmp_bl to determine if this will produce a write */
-          if (cw_req->cmp_bl.contents_equal(cw_req->read_bl)) {
+          buffer::list aligned_read_bl;
+          if (cw_req->cmp_bl.length() < cw_req->read_bl.length()) {
+            aligned_read_bl.substr_of(cw_req->read_bl, 0, cw_req->cmp_bl.length());
+          }
+          if (cw_req->cmp_bl.contents_equal(cw_req->read_bl) ||
+              cw_req->cmp_bl.contents_equal(aligned_read_bl)) {
             /* Compare phase succeeds. Begin write */
             ldout(m_image_ctx.cct, 5) << " cw_req=" << cw_req << " compare matched" << dendl;
             cw_req->compare_succeeded = true;
@@ -1169,7 +1174,6 @@ void ReplicatedWriteLog<I>::aio_compare_and_write(Extents &&image_extents,
             /* Compare phase fails. Comp-and write ends now. */
             ldout(m_image_ctx.cct, 15) << " cw_req=" << cw_req << " compare failed" << dendl;
             /* Bufferlist doesn't tell us where they differed, so we'll have to determine that here */
-            ceph_assert(cw_req->read_bl.length() == cw_req->cmp_bl.length());
             uint64_t bl_index = 0;
             for (bl_index = 0; bl_index < cw_req->cmp_bl.length(); bl_index++) {
               if (cw_req->cmp_bl[bl_index] != cw_req->read_bl[bl_index]) {
@@ -2759,3 +2763,6 @@ bool ReplicatedWriteLog<I>::retire_entries(const unsigned long int frees_per_tx)
 
 template class librbd::cache::ReplicatedWriteLog<librbd::ImageCtx>;
 template class librbd::cache::ImageCache<librbd::ImageCtx>;
+template void librbd::cache::ReplicatedWriteLog<librbd::ImageCtx>:: \
+  flush_pmem_buffer(std::vector<std::shared_ptr< \
+    librbd::cache::rwl::GenericLogOperation>>&);
diff --git a/src/librbd/cache/Types.h b/src/librbd/cache/Types.h
index b0852e74959..2b08b7b4676 100644
--- a/src/librbd/cache/Types.h
+++ b/src/librbd/cache/Types.h
@@ -5,6 +5,7 @@
 #define CEPH_LIBRBD_CACHE_TYPES_H
 
 #include <list>
+#include <string>
 
 class Context;
 
@@ -17,6 +18,8 @@ enum ImageCacheType {
 
 typedef std::list<Context *> Contexts;
 
+const std::string IMAGE_CACHE_STATE = ".librbd/image_cache_state";
+
 } // namespace cache
 } // namespace librbd
 
diff --git a/src/librbd/cache/Utils.h b/src/librbd/cache/Utils.h
new file mode 100644
index 00000000000..aa08bcba1c6
--- /dev/null
+++ b/src/librbd/cache/Utils.h
@@ -0,0 +1,21 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_CACHE_UTILS_H
+#define CEPH_LIBRBD_CACHE_UTILS_H
+
+namespace librbd {
+
+namespace cache {
+namespace util {
+
+template <typename T>
+bool is_rwl_enabled(T& image_ctx) {
+  return image_ctx.config.template get_val<bool>("rbd_rwl_enabled");
+}
+
+} // namespace util
+} // namespace cache
+} // namespace librbd
+
+#endif // CEPH_LIBRBD_CACHE_UTILS_H
diff --git a/src/librbd/cache/rwl/ImageCacheState.cc b/src/librbd/cache/rwl/ImageCacheState.cc
index 7a449733db1..5c491b190d2 100644
--- a/src/librbd/cache/rwl/ImageCacheState.cc
+++ b/src/librbd/cache/rwl/ImageCacheState.cc
@@ -1,6 +1,7 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#include "librbd/cache/Types.h"
 #include "librbd/cache/rwl/ImageCacheState.h"
 #include "librbd/ImageCtx.h"
 #include "librbd/Operations.h"
@@ -12,19 +13,28 @@
 #undef dout_subsys
 #define dout_subsys ceph_subsys_rbd_rwl
 #undef dout_prefix
-#define dout_prefix *_dout << "librbd::cache::rwl::ImageCacheState: " << this << " " \
+#define dout_prefix *_dout << "librbd::cache::rwl::ImageCacheState: " \
                            <<  __func__ << ": "
 
 namespace librbd {
 namespace cache {
 namespace rwl {
 
-template <typename I>
-const std::string ImageCacheState<I>::image_cache_state = ".librbd/image_cache_state";
+namespace {
+bool get_json_format(const std::string& s, JSONFormattable *f) {
+  JSONParser p;
+  bool success = p.parse(s.c_str(), s.size());
+  if (success) {
+    decode_json_obj(*f, &p);
+  }
+  return success;
+}
+} // namespace
 
 template <typename I>
 ImageCacheState<I>::ImageCacheState(I *image_ctx) : m_image_ctx(image_ctx) {
-  ldout(image_ctx->cct, 20) << "Initialize RWL cache state with config data. " << dendl;
+  ldout(image_ctx->cct, 20) << "Initialize RWL cache state with config data. "
+                            << dendl;
 
   ConfigProxy &config = image_ctx->config;
   host = ceph_get_short_hostname();
@@ -34,8 +44,10 @@ ImageCacheState<I>::ImageCacheState(I *image_ctx) : m_image_ctx(image_ctx) {
 }
 
 template <typename I>
-ImageCacheState<I>::ImageCacheState(I *image_ctx, JSONFormattable &f) : m_image_ctx(image_ctx) {
-  ldout(image_ctx->cct, 20) << "Initialize RWL cache state with data from server side" << dendl;
+ImageCacheState<I>::ImageCacheState(
+    I *image_ctx, JSONFormattable &f) : m_image_ctx(image_ctx) {
+  ldout(image_ctx->cct, 20) << "Initialize RWL cache state with data from "
+                            << "server side"<< dendl;
 
   present = (bool)f["present"];
   empty = (bool)f["empty"];
@@ -56,20 +68,22 @@ template <typename I>
 void ImageCacheState<I>::write_image_cache_state(Context *on_finish) {
   std::shared_lock owner_lock{m_image_ctx->owner_lock};
   JSONFormattable f;
-  ::encode_json(image_cache_state.c_str(), *this, &f);
+  ::encode_json(IMAGE_CACHE_STATE.c_str(), *this, &f);
   std::ostringstream oss;
   f.flush(oss);
   std::string image_state_json = oss.str();
 
-  ldout(m_image_ctx->cct, 20) << __func__ << " Store state: " << image_state_json << dendl;
-  m_image_ctx->operations->execute_metadata_set(image_cache_state.c_str(), image_state_json, on_finish);
+  ldout(m_image_ctx->cct, 20) << __func__ << " Store state: "
+                              << image_state_json << dendl;
+  m_image_ctx->operations->execute_metadata_set(IMAGE_CACHE_STATE,
+                                                image_state_json, on_finish);
 }
 
 template <typename I>
 void ImageCacheState<I>::clear_image_cache_state(Context *on_finish) {
   std::shared_lock owner_lock{m_image_ctx->owner_lock};
   ldout(m_image_ctx->cct, 20) << __func__ << " Remove state: " << dendl;
-  m_image_ctx->operations->execute_metadata_remove(image_cache_state.c_str(), on_finish);
+  m_image_ctx->operations->execute_metadata_remove(IMAGE_CACHE_STATE, on_finish);
 }
 
 template <typename I>
@@ -83,6 +97,81 @@ void ImageCacheState<I>::dump(ceph::Formatter *f) const {
   ::encode_json("rwl_size", size, f);
 }
 
+template <typename I>
+ImageCacheState<I>* ImageCacheState<I>::get_image_cache_state(
+    I* image_ctx, int &r) {
+  std::string cache_state_str;
+  ImageCacheState<I>* cache_state = nullptr;
+  ldout(image_ctx->cct, 20) << "image_cache_state:" << cache_state_str << dendl;
+
+  r = 0;
+  bool dirty_cache = image_ctx->test_features(RBD_FEATURE_DIRTY_CACHE);
+  if (dirty_cache) {
+    cls_client::metadata_get(&image_ctx->md_ctx, image_ctx->header_oid,
+                             IMAGE_CACHE_STATE, &cache_state_str);
+  }
+
+  bool rwl_enabled = image_ctx->config.template get_val<bool>("rbd_rwl_enabled");
+  bool cache_desired = rwl_enabled;
+  cache_desired &= !image_ctx->read_only;
+  cache_desired &= !image_ctx->test_features(RBD_FEATURE_MIGRATING);
+  cache_desired &= !image_ctx->test_features(RBD_FEATURE_JOURNALING);
+  cache_desired &= !image_ctx->old_format;
+
+  if (!dirty_cache && !cache_desired) {
+    ldout(image_ctx->cct, 20) << "Do not desire to use image cache." << dendl;
+  } else if (dirty_cache && !cache_desired) {
+    lderr(image_ctx->cct) << "There's a dirty cache, but RWL cache is disabled."
+                          << dendl;
+    r = -EINVAL;
+  }else if ((!dirty_cache || cache_state_str.empty()) && cache_desired) {
+    cache_state = new ImageCacheState<I>(image_ctx);
+  } else {
+    ceph_assert(!cache_state_str.empty());
+    JSONFormattable f;
+    bool success = get_json_format(cache_state_str, &f);
+    if (!success) {
+      lderr(image_ctx->cct) << "Failed to parse cache state: "
+                            << cache_state_str << dendl;
+      r = -EINVAL;
+      return nullptr;
+    }
+
+    bool cache_exists = (bool)f["present"];
+    int cache_type = (int)f["cache_type"];
+
+    switch (cache_type) {
+      case IMAGE_CACHE_TYPE_RWL:
+        if (!cache_exists) {
+          cache_state = new ImageCacheState<I>(image_ctx);
+        } else {
+          cache_state = new ImageCacheState<I>(image_ctx, f);
+        }
+        break;
+      default:
+	r = -EINVAL;
+    }
+  }
+  return cache_state;
+}
+
+template <typename I>
+bool ImageCacheState<I>::is_valid() {
+  if (this->present &&
+      (host.compare(ceph_get_short_hostname()) != 0)) {
+    auto cleanstring = "dirty";
+    if (this->clean) {
+      cleanstring = "clean";
+    }
+    lderr(m_image_ctx->cct) << "An image cache (RWL) remains on another host "
+                            << host << " which is " << cleanstring
+                            << ". Flush/close the image there to remove the "
+                            << "image cache" << dendl;
+    return false;
+  }
+  return true;
+}
+
 } // namespace rwl
 } // namespace cache
 } // namespace librbd
diff --git a/src/librbd/cache/rwl/ImageCacheState.h b/src/librbd/cache/rwl/ImageCacheState.h
index 0bab76c8f03..f2c82cfefdd 100644
--- a/src/librbd/cache/rwl/ImageCacheState.h
+++ b/src/librbd/cache/rwl/ImageCacheState.h
@@ -25,7 +25,6 @@ public:
   bool present = true;
   bool empty = true;
   bool clean = true;
-  static const std::string image_cache_state;
   std::string host;
   std::string path;
   uint64_t size;
@@ -47,6 +46,10 @@ public:
   void clear_image_cache_state(Context *on_finish);
 
   void dump(ceph::Formatter *f) const;
+
+  static void get_image_cache_state(ImageCtxT* image_ctx, Context *on_finish);
+
+  bool is_valid();
 };
 
 } // namespace rwl
diff --git a/src/librbd/cache/rwl/InitRequest.cc b/src/librbd/cache/rwl/InitRequest.cc
new file mode 100644
index 00000000000..ab5b714c320
--- /dev/null
+++ b/src/librbd/cache/rwl/InitRequest.cc
@@ -0,0 +1,171 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "librbd/cache/rwl/InitRequest.h"
+#include "librbd/Utils.h"
+#include "common/dout.h"
+#include "common/errno.h"
+#include "librbd/asio/ContextWQ.h"
+
+#if defined(WITH_RBD_RWL)
+#include "librbd/cache/rwl/ImageCacheState.h"
+#include "librbd/cache/ReplicatedWriteLog.h"
+#endif // WITH_RBD_RWL
+
+#include "librbd/cache/Utils.h"
+#include "librbd/ImageCtx.h"
+
+#define dout_subsys ceph_subsys_rbd_rwl
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::cache::rwl:InitRequest " \
+                           << this << " " << __func__ << ": "
+
+namespace librbd {
+namespace cache {
+namespace rwl {
+
+using librbd::util::create_async_context_callback;
+using librbd::util::create_context_callback;
+
+template <typename I>
+InitRequest<I>* InitRequest<I>::create(I &image_ctx,
+                                       Context *on_finish) {
+  return new InitRequest(image_ctx, on_finish);
+}
+
+template <typename I>
+InitRequest<I>::InitRequest(I &image_ctx, Context *on_finish)
+  : m_image_ctx(image_ctx),
+    m_on_finish(create_async_context_callback(image_ctx, on_finish)),
+    m_error_result(0) {
+}
+
+template <typename I>
+void InitRequest<I>::send() {
+#if defined(WITH_RBD_RWL)
+  get_image_cache_state();
+#else
+  finish();
+#endif // WITH_RBD_RWL
+}
+
+#if defined(WITH_RBD_RWL)
+template <typename I>
+void InitRequest<I>::get_image_cache_state() {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 10) << dendl;
+
+  int r;
+  auto cache_state = ImageCacheState<I>::get_image_cache_state(&m_image_ctx, r);
+
+  if (r < 0 || !cache_state) {
+    save_result(r);
+    finish();
+    return;
+  } else if (!cache_state->is_valid()) {
+    delete cache_state;
+    cache_state = nullptr;
+    lderr(cct) << "failed to get image cache state: " << cpp_strerror(r)
+               << dendl;
+    save_result(-ENOENT);
+    finish();
+    return;
+  }
+
+  auto cache_type = cache_state->get_image_cache_type();
+  switch(cache_type) {
+    case cache::IMAGE_CACHE_TYPE_RWL:
+      m_image_ctx.image_cache =
+        new librbd::cache::ReplicatedWriteLog<I>(m_image_ctx,
+                                                 cache_state);
+      break;
+    default:
+      delete cache_state;
+      cache_state = nullptr;
+      save_result(-ENOENT);
+      finish();
+      return;
+  }
+
+  init_image_cache();
+}
+
+template <typename I>
+void InitRequest<I>::init_image_cache() {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 10) << dendl;
+
+  using klass = InitRequest<I>;
+  Context *ctx = create_context_callback<klass, &klass::handle_init_image_cache>(
+    this);
+  m_image_ctx.image_cache->init(ctx);
+}
+
+template <typename I>
+void InitRequest<I>::handle_init_image_cache(int r) {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 10) << dendl;
+
+  if (r < 0) {
+    lderr(cct) << "failed to init image cache: " << cpp_strerror(r)
+               << dendl;
+    delete m_image_ctx.image_cache;
+    m_image_ctx.image_cache = nullptr;
+    save_result(r);
+    finish();
+    return;
+  }
+  set_feature_bit();
+}
+
+template <typename I>
+void InitRequest<I>::set_feature_bit() {
+  CephContext *cct = m_image_ctx.cct;
+
+  uint64_t new_features = m_image_ctx.features | RBD_FEATURE_DIRTY_CACHE;
+  uint64_t features_mask = RBD_FEATURE_DIRTY_CACHE;
+  ldout(cct, 10) << "old_features=" << m_image_ctx.features
+                 << ", new_features=" << new_features
+                 << ", features_mask=" << features_mask
+                 << dendl;
+
+  int r = librbd::cls_client::set_features(&m_image_ctx.md_ctx,
+                                           m_image_ctx.header_oid,
+                                           new_features, features_mask);
+  m_image_ctx.features |= RBD_FEATURE_DIRTY_CACHE;
+  using klass = InitRequest<I>;
+  Context *ctx = create_context_callback<klass, &klass::handle_set_feature_bit>(
+    this);
+  ctx->complete(r);
+}
+
+template <typename I>
+void InitRequest<I>::handle_set_feature_bit(int r) {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 10) << "r=" << r << dendl;
+
+  if (r < 0) {
+    lderr(cct) << "failed to set feature bit: " << cpp_strerror(r)
+               << dendl;
+    save_result(r);
+  } else if (m_image_ctx.discard_granularity_bytes) {
+    ldout(cct, 1) << "RWL image cache is enabled and "
+                  << "set discard_granularity_bytes = 0." << dendl;
+    m_image_ctx.discard_granularity_bytes = 0;
+  }
+  finish();
+}
+
+#endif // WITH_RBD_RWL
+
+template <typename I>
+void InitRequest<I>::finish() {
+  m_on_finish->complete(m_error_result);
+  delete this;
+}
+
+} // namespace rwl
+} // namespace cache
+} // namespace librbd
+
+template class librbd::cache::rwl::InitRequest<librbd::ImageCtx>;
diff --git a/src/librbd/cache/rwl/InitRequest.h b/src/librbd/cache/rwl/InitRequest.h
new file mode 100644
index 00000000000..9d18e678354
--- /dev/null
+++ b/src/librbd/cache/rwl/InitRequest.h
@@ -0,0 +1,82 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_CACHE_RWL_INIT_REQUEST_H
+#define CEPH_LIBRBD_CACHE_RWL_INIT_REQUEST_H
+
+class Context;
+
+namespace librbd {
+
+class ImageCtx;
+
+namespace cache {
+namespace rwl {
+
+template<typename>
+class ImageCacheState;
+
+template <typename ImageCtxT = ImageCtx>
+class InitRequest {
+public:
+  static InitRequest* create(ImageCtxT &image_ctx, Context *on_finish);
+
+  void send();
+
+private:
+
+  /**
+   * @verbatim
+   *
+   * Init request goes through the following state machine:
+   *
+   * <start>
+   *    |
+   *    v
+   * GET_IMAGE_CACHE_STATE
+   *    |
+   *    v
+   * INIT_IMAGE_CACHE
+   *    |
+   *    v
+   * SET_FEATURE_BIT
+   *    |
+   *    v
+   * <finish>
+   *
+   * @endverbatim
+   */
+
+  InitRequest(ImageCtxT &image_ctx, Context *on_finish);
+
+  ImageCtxT &m_image_ctx;
+  Context *m_on_finish;
+
+  int m_error_result;
+
+  bool is_rwl_enabled();
+
+  void get_image_cache_state();
+
+  void init_image_cache();
+  void handle_init_image_cache(int r);
+
+  void set_feature_bit();
+  void handle_set_feature_bit(int r);
+
+  void finish();
+
+  void save_result(int result) {
+    if (m_error_result == 0 && result < 0) {
+      m_error_result = result;
+    }
+  }
+};
+
+} // namespace rwl
+} // namespace cache
+} // namespace librbd
+
+extern template class librbd::cache::rwl::InitRequest<librbd::ImageCtx>;
+
+#endif // CEPH_LIBRBD_CACHE_RWL_INIT_REQUEST_H
diff --git a/src/librbd/cache/rwl/LogMap.cc b/src/librbd/cache/rwl/LogMap.cc
index 234113dee7e..e432da7b671 100644
--- a/src/librbd/cache/rwl/LogMap.cc
+++ b/src/librbd/cache/rwl/LogMap.cc
@@ -4,6 +4,7 @@
 #include "LogMap.h"
 #include "include/ceph_assert.h"
 #include "librbd/Utils.h"
+#include "librbd/cache/rwl/LogEntry.h"
 
 namespace librbd {
 namespace cache {
@@ -273,3 +274,5 @@ bool LogMap<T>::LogMapEntryCompare::operator()(const LogMapEntry<T> &lhs,
 } //namespace rwl
 } //namespace cache
 } //namespace librbd
+
+template class librbd::cache::rwl::LogMap<librbd::cache::rwl::GenericWriteLogEntry>;
diff --git a/src/librbd/cache/rwl/Request.cc b/src/librbd/cache/rwl/Request.cc
index 1a3d8e9b2e4..299dc18813e 100644
--- a/src/librbd/cache/rwl/Request.cc
+++ b/src/librbd/cache/rwl/Request.cc
@@ -4,6 +4,7 @@
 #include "Request.h"
 #include "librbd/BlockGuard.h"
 #include "librbd/cache/rwl/LogEntry.h"
+#include "librbd/cache/ReplicatedWriteLog.h"
 
 #define dout_subsys ceph_subsys_rbd_rwl
 #undef dout_prefix
@@ -623,3 +624,10 @@ std::ostream &operator<<(std::ostream &os,
 } // namespace rwl
 } // namespace cache
 } // namespace librbd
+
+template class librbd::cache::rwl::C_BlockIORequest<librbd::cache::ReplicatedWriteLog<librbd::ImageCtx> >;
+template class librbd::cache::rwl::C_WriteRequest<librbd::cache::ReplicatedWriteLog<librbd::ImageCtx> >;
+template class librbd::cache::rwl::C_FlushRequest<librbd::cache::ReplicatedWriteLog<librbd::ImageCtx> >;
+template class librbd::cache::rwl::C_DiscardRequest<librbd::cache::ReplicatedWriteLog<librbd::ImageCtx> >;
+template class librbd::cache::rwl::C_WriteSameRequest<librbd::cache::ReplicatedWriteLog<librbd::ImageCtx> >;
+template class librbd::cache::rwl::C_CompAndWriteRequest<librbd::cache::ReplicatedWriteLog<librbd::ImageCtx> >;
diff --git a/src/librbd/cache/rwl/ShutdownRequest.cc b/src/librbd/cache/rwl/ShutdownRequest.cc
new file mode 100644
index 00000000000..8259662ae97
--- /dev/null
+++ b/src/librbd/cache/rwl/ShutdownRequest.cc
@@ -0,0 +1,151 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "librbd/cache/rwl/ShutdownRequest.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/Utils.h"
+#include "common/dout.h"
+#include "common/errno.h"
+#include "librbd/Operations.h"
+#include "librbd/asio/ContextWQ.h"
+#include "librbd/cache/ImageCache.h"
+#include "librbd/cache/Types.h"
+
+
+#define dout_subsys ceph_subsys_rbd_rwl
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::cache::rwl:ShutdownRequest: " \
+                           << this << " " << __func__ << ": "
+
+namespace librbd {
+namespace cache {
+namespace rwl {
+
+using librbd::util::create_async_context_callback;
+using librbd::util::create_context_callback;
+
+template <typename I>
+ShutdownRequest<I>* ShutdownRequest<I>::create(I &image_ctx,
+                                       Context *on_finish) {
+  return new ShutdownRequest(image_ctx, on_finish);
+}
+
+template <typename I>
+ShutdownRequest<I>::ShutdownRequest(I &image_ctx, Context *on_finish)
+  : m_image_ctx(image_ctx),
+    m_on_finish(create_async_context_callback(image_ctx, on_finish)),
+    m_error_result(0) {
+}
+
+template <typename I>
+void ShutdownRequest<I>::send() {
+  send_shutdown_image_cache();
+}
+
+template <typename I>
+void ShutdownRequest<I>::send_shutdown_image_cache() {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 10) << dendl;
+
+  if (m_image_ctx.image_cache == nullptr) {
+    finish();
+    return;
+  }
+
+  using klass = ShutdownRequest<I>;
+  Context *ctx = create_context_callback<klass, &klass::handle_shutdown_image_cache>(
+    this);
+
+  m_image_ctx.image_cache->shut_down(ctx);
+}
+
+template <typename I>
+void ShutdownRequest<I>::handle_shutdown_image_cache(int r) {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 10) << dendl;
+
+  if (r < 0) {
+    lderr(cct) << "failed to shut down the image cache: " << cpp_strerror(r)
+               << dendl;
+    save_result(r);
+    finish();
+    return;
+  } else {
+    delete m_image_ctx.image_cache;
+    m_image_ctx.image_cache = nullptr;
+  }
+  send_remove_feature_bit();
+}
+
+template <typename I>
+void ShutdownRequest<I>::send_remove_feature_bit() {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 10) << dendl;
+
+  uint64_t new_features = m_image_ctx.features & ~RBD_FEATURE_DIRTY_CACHE;
+  uint64_t features_mask = RBD_FEATURE_DIRTY_CACHE;
+  ldout(cct, 10) << "old_features=" << m_image_ctx.features
+                 << ", new_features=" << new_features
+                 << ", features_mask=" << features_mask
+                 << dendl;
+
+  int r = librbd::cls_client::set_features(&m_image_ctx.md_ctx, m_image_ctx.header_oid,
+                                           new_features, features_mask);
+  m_image_ctx.features &= ~RBD_FEATURE_DIRTY_CACHE;
+  using klass = ShutdownRequest<I>;
+  Context *ctx = create_context_callback<klass, &klass::handle_remove_feature_bit>(
+    this);
+  ctx->complete(r);
+}
+
+template <typename I>
+void ShutdownRequest<I>::handle_remove_feature_bit(int r) {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 10) << dendl;
+
+  if (r < 0) {
+    lderr(cct) << "failed to remove the feature bit: " << cpp_strerror(r)
+               << dendl;
+    save_result(r);
+    finish();
+    return;
+  }
+  send_remove_image_cache_state();
+}
+
+template <typename I>
+void ShutdownRequest<I>::send_remove_image_cache_state() {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 10) << dendl;
+
+  using klass = ShutdownRequest<I>;
+  Context *ctx = create_context_callback<klass, &klass::handle_remove_image_cache_state>(
+    this);
+  std::shared_lock owner_lock{m_image_ctx.owner_lock};
+  m_image_ctx.operations->execute_metadata_remove(IMAGE_CACHE_STATE, ctx);
+}
+
+template <typename I>
+void ShutdownRequest<I>::handle_remove_image_cache_state(int r) {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 10) << dendl;
+
+  if (r < 0) {
+    lderr(cct) << "failed to remove the image cache state: " << cpp_strerror(r)
+               << dendl;
+    save_result(r);
+  }
+  finish();
+}
+
+template <typename I>
+void ShutdownRequest<I>::finish() {
+  m_on_finish->complete(m_error_result);
+  delete this;
+}
+
+} // namespace rwl
+} // namespace cache
+} // namespace librbd
+
+template class librbd::cache::rwl::ShutdownRequest<librbd::ImageCtx>;
diff --git a/src/librbd/cache/rwl/ShutdownRequest.h b/src/librbd/cache/rwl/ShutdownRequest.h
new file mode 100644
index 00000000000..635527f63c7
--- /dev/null
+++ b/src/librbd/cache/rwl/ShutdownRequest.h
@@ -0,0 +1,81 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_CACHE_RWL_SHUTDOWN_REQUEST_H
+#define CEPH_LIBRBD_CACHE_RWL_SHUTDOWN_REQUEST_H
+
+class Context;
+
+namespace librbd {
+
+class ImageCtx;
+
+namespace cache {
+namespace rwl {
+
+template<typename>
+class ImageCacheState;
+
+template <typename ImageCtxT = ImageCtx>
+class ShutdownRequest {
+public:
+  static ShutdownRequest* create(ImageCtxT &image_ctx, Context *on_finish);
+
+  void send();
+
+private:
+
+  /**
+   * @verbatim
+   *
+   * Shutdown request goes through the following state machine:
+   *
+   * <start>
+   *    |
+   *    v
+   * SHUTDOWN_IMAGE_CACHE
+   *    |
+   *    v
+   * REMOVE_IMAGE_FEATURE_BIT
+   *    |
+   *    v
+   * REMOVE_IMAGE_CACHE_STATE
+   *    |
+   *    v
+   * <finish>
+   *
+   * @endverbatim
+   */
+
+  ShutdownRequest(ImageCtxT &image_ctx, Context *on_finish);
+
+  ImageCtxT &m_image_ctx;
+  Context *m_on_finish;
+
+  int m_error_result;
+
+  void send_shutdown_image_cache();
+  void handle_shutdown_image_cache(int r);
+
+  void send_remove_feature_bit();
+  void handle_remove_feature_bit(int r);
+
+  void send_remove_image_cache_state();
+  void handle_remove_image_cache_state(int r);
+
+  void finish();
+
+  void save_result(int result) {
+    if (m_error_result == 0 && result < 0) {
+      m_error_result = result;
+    }
+  }
+};
+
+} // namespace rwl
+} // namespace cache
+} // namespace librbd
+
+extern template class librbd::cache::rwl::ShutdownRequest<librbd::ImageCtx>;
+
+#endif // CEPH_LIBRBD_CACHE_RWL_SHUTDOWN_REQUEST_H
diff --git a/src/librbd/cache/rwl/Types.cc b/src/librbd/cache/rwl/Types.cc
index 9df800d3c9b..7f46c070410 100644
--- a/src/librbd/cache/rwl/Types.cc
+++ b/src/librbd/cache/rwl/Types.cc
@@ -94,15 +94,6 @@ ExtentsSummary<ExtentsType>::ExtentsSummary(const ExtentsType &extents)
   }
 }
 
-template <typename T>
-std::ostream &operator<<(std::ostream &os,
-                         const ExtentsSummary<T> &s) {
-  os << "total_bytes=" << s.total_bytes << ", "
-     << "first_image_byte=" << s.first_image_byte << ", "
-     << "last_image_byte=" << s.last_image_byte << "";
-  return os;
-}
-
 io::Extent whole_volume_extent() {
   return io::Extent({0, std::numeric_limits<uint64_t>::max()});
 }
@@ -126,3 +117,5 @@ Context * override_ctx(int r, Context *ctx) {
 } // namespace rwl
 } // namespace cache
 } // namespace librbd
+
+template class librbd::cache::rwl::ExtentsSummary<librbd::io::Extents>;
diff --git a/src/librbd/cache/rwl/Types.h b/src/librbd/cache/rwl/Types.h
index 2fc98b055e6..61b77862193 100644
--- a/src/librbd/cache/rwl/Types.h
+++ b/src/librbd/cache/rwl/Types.h
@@ -275,14 +275,18 @@ public:
   uint64_t first_image_byte;
   uint64_t last_image_byte;
   explicit ExtentsSummary(const ExtentsType &extents);
-  template <typename U>
   friend std::ostream &operator<<(std::ostream &os,
-                                  const ExtentsSummary<U> &s);
+                                  const ExtentsSummary &s) {
+    os << "total_bytes=" << s.total_bytes << ", "
+       << "first_image_byte=" << s.first_image_byte << ", "
+       << "last_image_byte=" << s.last_image_byte << "";
+    return os;
+  }
   BlockExtent block_extent() {
     return BlockExtent(first_image_byte, last_image_byte);
   }
   io::Extent image_extent() {
-    return image_extent(block_extent());
+    return librbd::cache::rwl::image_extent(block_extent());
   }
 };
 
diff --git a/src/test/librbd/cache/test_mock_ReplicatedWriteLog.cc b/src/test/librbd/cache/test_mock_ReplicatedWriteLog.cc
index 5d541c6e113..e7bc7c66e49 100644
--- a/src/test/librbd/cache/test_mock_ReplicatedWriteLog.cc
+++ b/src/test/librbd/cache/test_mock_ReplicatedWriteLog.cc
@@ -41,14 +41,7 @@ inline ImageCtx *get_image_ctx(MockImageCtx *image_ctx) {
 // template definitions
 #include "librbd/cache/ImageWriteback.cc"
 #include "librbd/cache/rwl/ImageCacheState.cc"
-#include "librbd/cache/rwl/SyncPoint.cc"
 #include "librbd/cache/rwl/Request.cc"
-#include "librbd/cache/rwl/Types.cc"
-#include "librbd/cache/rwl/LogOperation.cc"
-#include "librbd/cache/rwl/LogMap.cc"
-
-template class librbd::cache::ImageWriteback<librbd::MockImageCtx>;
-template class librbd::cache::rwl::ImageCacheState<librbd::MockImageCtx>;
 
 namespace librbd {
 namespace cache {