From 3d5b89e85b07e170ebbc17dc2c097d7c120cfbb9 Mon Sep 17 00:00:00 2001 From: lixiaoy1 Date: Fri, 11 Sep 2020 10:24:56 -0400 Subject: [PATCH] librbd/cache: discard writeback cache Add an interface in ImageCache to discard writeback cache. Signed-off-by: Li, Xiaoyan --- src/librbd/CMakeLists.txt | 4 + src/librbd/cache/Utils.cc | 24 ++++ src/librbd/cache/Utils.h | 7 + src/librbd/cache/pwl/DiscardRequest.cc | 167 ++++++++++++++++++++++++ src/librbd/cache/pwl/DiscardRequest.h | 86 ++++++++++++ src/librbd/cache/pwl/ImageCacheState.cc | 20 ++- src/librbd/cache/pwl/ImageCacheState.h | 5 +- src/librbd/cache/pwl/InitRequest.cc | 2 +- 8 files changed, 312 insertions(+), 3 deletions(-) create mode 100644 src/librbd/cache/Utils.cc create mode 100644 src/librbd/cache/pwl/DiscardRequest.cc create mode 100644 src/librbd/cache/pwl/DiscardRequest.h diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index c0aae05bd11ec..eafbef9430f1b 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -42,8 +42,10 @@ set(librbd_internal_srcs cache/ImageWriteback.cc cache/ObjectCacherObjectDispatch.cc cache/ObjectCacherWriteback.cc + cache/pwl/DiscardRequest.cc cache/pwl/InitRequest.cc cache/pwl/ShutdownRequest.cc + cache/Utils.cc cache/WriteAroundObjectDispatch.cc crypto/BlockCrypto.cc crypto/CryptoContextPool.cc @@ -222,6 +224,8 @@ target_include_directories(rbd_internal PRIVATE ${OPENSSL_INCLUDE_DIR}) if(WITH_RBD_RWL) target_link_libraries(rbd_internal PUBLIC blk) + target_link_libraries(rbd_internal PRIVATE + StdFilesystem::filesystem) endif() add_custom_target(librbd_plugins) diff --git a/src/librbd/cache/Utils.cc b/src/librbd/cache/Utils.cc new file mode 100644 index 0000000000000..c65c57551a5ba --- /dev/null +++ b/src/librbd/cache/Utils.cc @@ -0,0 +1,24 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/cache/pwl/DiscardRequest.h" +#include "librbd/cache/Utils.h" +#include "include/Context.h" + +namespace librbd { +namespace cache { +namespace util { + +template +void discard_cache(I &image_ctx, Context *ctx) { + cache::pwl::DiscardRequest *req = cache::pwl::DiscardRequest::create( + image_ctx, ctx); + req->send(); +} + +} // namespace util +} // namespace cache +} // namespace librbd + +template void librbd::cache::util::discard_cache( + librbd::ImageCtx &image_ctx, Context *ctx); diff --git a/src/librbd/cache/Utils.h b/src/librbd/cache/Utils.h index 827dcc5daa6a5..67c0b6fbc7358 100644 --- a/src/librbd/cache/Utils.h +++ b/src/librbd/cache/Utils.h @@ -6,8 +6,12 @@ #include "acconfig.h" +class Context; + namespace librbd { +struct ImageCtx; + namespace cache { namespace util { @@ -20,6 +24,9 @@ bool is_pwl_enabled(T& image_ctx) { #endif // WITH_RBD_RWL } +template +void discard_cache(T &image_ctx, Context *ctx); + } // namespace util } // namespace cache } // namespace librbd diff --git a/src/librbd/cache/pwl/DiscardRequest.cc b/src/librbd/cache/pwl/DiscardRequest.cc new file mode 100644 index 0000000000000..32a0077d4e783 --- /dev/null +++ b/src/librbd/cache/pwl/DiscardRequest.cc @@ -0,0 +1,167 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "common/dout.h" +#include "common/errno.h" +#include "common/hostname.h" +#include "librbd/asio/ContextWQ.h" +#include "librbd/cache/pwl/DiscardRequest.h" + +#if defined(WITH_RBD_RWL) +#if __has_include() +#include +namespace fs = std::filesystem; +#elif __has_include() +#include +namespace fs = std::experimental::filesystem; +#endif + +#include "librbd/cache/pwl/ImageCacheState.h" +#endif // WITH_RBD_RWL + +#include "librbd/cache/Types.h" +#include "librbd/io/ImageDispatcherInterface.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" + + +#define dout_subsys ceph_subsys_rbd_pwl +#undef dout_prefix +#define dout_prefix *_dout << "librbd::cache::pwl:DiscardRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace cache { +namespace pwl { + +using librbd::util::create_async_context_callback; +using librbd::util::create_context_callback; + +template +DiscardRequest* DiscardRequest::create( + I &image_ctx, + Context *on_finish) { + return new DiscardRequest(image_ctx, on_finish); +} + +template +DiscardRequest::DiscardRequest( + I &image_ctx, + Context *on_finish) + : m_image_ctx(image_ctx), + m_on_finish(create_async_context_callback(image_ctx, on_finish)), + m_error_result(0) { +} + +template +void DiscardRequest::send() { +#if defined(WITH_RBD_RWL) + delete_image_cache_file(); +#else + finish(); +#endif +} + +template +void DiscardRequest::delete_image_cache_file() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << dendl; + + m_cache_state = ImageCacheState::get_image_cache_state(&m_image_ctx); + if (!m_cache_state) { + remove_feature_bit(); + return; + } + if (m_cache_state->present && + !m_cache_state->host.compare(ceph_get_short_hostname()) && + fs::exists(m_cache_state->path)) { + fs::remove(m_cache_state->path); + } + + remove_image_cache_state(); +} + +template +void DiscardRequest::remove_image_cache_state() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << dendl; + + using klass = DiscardRequest; + Context *ctx = create_context_callback( + this); + + m_cache_state->clear_image_cache_state(ctx); +} + +template +void DiscardRequest::handle_remove_image_cache_state(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << dendl; + + if (r < 0) { + lderr(cct) << "failed to remove the image cache state: " << cpp_strerror(r) + << dendl; + save_result(r); + finish(); + return; + } + + remove_feature_bit(); +} + +template +void DiscardRequest::remove_feature_bit() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << dendl; + + if (!(m_image_ctx.features &&RBD_FEATURE_DIRTY_CACHE)) { + finish(); + return; + } + uint64_t new_features = m_image_ctx.features & ~RBD_FEATURE_DIRTY_CACHE; + uint64_t features_mask = RBD_FEATURE_DIRTY_CACHE; + ldout(cct, 10) << "old_features=" << m_image_ctx.features + << ", new_features=" << new_features + << ", features_mask=" << features_mask + << dendl; + + int r = librbd::cls_client::set_features(&m_image_ctx.md_ctx, m_image_ctx.header_oid, + new_features, features_mask); + m_image_ctx.features &= ~RBD_FEATURE_DIRTY_CACHE; + using klass = DiscardRequest; + Context *ctx = create_context_callback( + this); + ctx->complete(r); +} + +template +void DiscardRequest::handle_remove_feature_bit(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << dendl; + + if (r < 0) { + lderr(cct) << "failed to remove the feature bit: " << cpp_strerror(r) + << dendl; + save_result(r); + } + finish(); +} + +template +void DiscardRequest::finish() { +#if defined(WITH_RBD_RWL) + if (m_cache_state) { + delete m_cache_state; + m_cache_state = nullptr; + } +#endif // WITH_RBD_RWL + + m_on_finish->complete(m_error_result); + delete this; +} + +} // namespace pwl +} // namespace cache +} // namespace librbd + +template class librbd::cache::pwl::DiscardRequest; diff --git a/src/librbd/cache/pwl/DiscardRequest.h b/src/librbd/cache/pwl/DiscardRequest.h new file mode 100644 index 0000000000000..6edd194b7d9c5 --- /dev/null +++ b/src/librbd/cache/pwl/DiscardRequest.h @@ -0,0 +1,86 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_CACHE_PWL_SHUTDOWN_REQUEST_H +#define CEPH_LIBRBD_CACHE_PWL_SHUTDOWN_REQUEST_H + +class Context; + +namespace librbd { + +class ImageCtx; + +namespace cache { + +namespace pwl { + +template +class ImageCacheState; + +template +class DiscardRequest { +public: + static DiscardRequest* create( + ImageCtxT &image_ctx, + Context *on_finish); + + void send(); + +private: + + /** + * @verbatim + * + * Shutdown request goes through the following state machine: + * + * + * | + * v + * REMOVE_IMAGE_CACHE_FILE + * | + * v + * REMOVE_IMAGE_CACHE_STATE + * | + * v + * REMOVE_IMAGE_FEATURE_BIT + * | + * v + * + * + * @endverbatim + */ + + DiscardRequest(ImageCtxT &image_ctx, + Context *on_finish); + + ImageCtxT &m_image_ctx; + ImageCacheState* m_cache_state; + Context *m_on_finish; + + int m_error_result; + + void delete_image_cache_file(); + + void remove_image_cache_state(); + void handle_remove_image_cache_state(int r); + + void remove_feature_bit(); + void handle_remove_feature_bit(int r); + + void finish(); + + void save_result(int result) { + if (m_error_result == 0 && result < 0) { + m_error_result = result; + } + } + +}; + +} // namespace pwl +} // namespace cache +} // namespace librbd + +extern template class librbd::cache::pwl::DiscardRequest; + +#endif // CEPH_LIBRBD_CACHE_PWL_SHUTDOWN_REQUEST_H diff --git a/src/librbd/cache/pwl/ImageCacheState.cc b/src/librbd/cache/pwl/ImageCacheState.cc index b36896a6915dc..dba7b971d226e 100644 --- a/src/librbd/cache/pwl/ImageCacheState.cc +++ b/src/librbd/cache/pwl/ImageCacheState.cc @@ -96,7 +96,7 @@ void ImageCacheState::dump(ceph::Formatter *f) const { } template -ImageCacheState* ImageCacheState::get_image_cache_state( +ImageCacheState* ImageCacheState::create_image_cache_state( I* image_ctx, int &r) { std::string cache_state_str; ImageCacheState* cache_state = nullptr; @@ -153,6 +153,24 @@ ImageCacheState* ImageCacheState::get_image_cache_state( return cache_state; } +template +ImageCacheState* ImageCacheState::get_image_cache_state(I* image_ctx) { + ImageCacheState* cache_state = nullptr; + string cache_state_str; + cls_client::metadata_get(&image_ctx->md_ctx, image_ctx->header_oid, + IMAGE_CACHE_STATE, &cache_state_str); + if (!cache_state_str.empty()) { + JSONFormattable f; + bool success = get_json_format(cache_state_str, &f); + if (!success) { + cache_state = new ImageCacheState(image_ctx); + } else { + cache_state = new ImageCacheState(image_ctx, f); + } + } + return cache_state; +} + template bool ImageCacheState::is_valid() { if (this->present && diff --git a/src/librbd/cache/pwl/ImageCacheState.h b/src/librbd/cache/pwl/ImageCacheState.h index bdc8cb080ff65..453316e866136 100644 --- a/src/librbd/cache/pwl/ImageCacheState.h +++ b/src/librbd/cache/pwl/ImageCacheState.h @@ -47,8 +47,11 @@ public: void dump(ceph::Formatter *f) const; + static ImageCacheState* create_image_cache_state( + ImageCtxT* image_ctx, int &r); + static ImageCacheState* get_image_cache_state( - ImageCtxT* image_ctx, int &r); + ImageCtxT* image_ctx); bool is_valid(); }; diff --git a/src/librbd/cache/pwl/InitRequest.cc b/src/librbd/cache/pwl/InitRequest.cc index 1f696d518bd33..bcf66ede3efb9 100644 --- a/src/librbd/cache/pwl/InitRequest.cc +++ b/src/librbd/cache/pwl/InitRequest.cc @@ -58,7 +58,7 @@ void InitRequest::get_image_cache_state() { ldout(cct, 10) << dendl; int r; - auto cache_state = ImageCacheState::get_image_cache_state(&m_image_ctx, r); + auto cache_state = ImageCacheState::create_image_cache_state(&m_image_ctx, r); if (r < 0 || !cache_state) { save_result(r); -- 2.39.5