- ``rbd_persistent_cache_size`` The cache size per image. The minimum cache
size is 1 GB.
-- ``rbd_persistent_cache_log_periodic_stats`` This is a debug option. It is
- used to emit periodic perf stats to the debug log if ``debug rbd pwl`` is
- set to ``1`` or higher.
-
The above configurations can be set per-host, per-pool, per-image etc. Eg, to
set per-host, add the overrides to the appropriate `section`_ in the host's
``ceph.conf`` file. To set per-pool, per-image, etc, please refer to the
- disabled
- rwl
- ssd
-- name: rbd_persistent_cache_log_periodic_stats
- type: bool
- level: advanced
- desc: emit periodic perf stats to debug log
- default: false
- services:
- - rbd
- name: rbd_persistent_cache_size
type: uint
level: advanced
template <typename I>
void AbstractWriteLog<I>::periodic_stats() {
std::lock_guard locker(m_lock);
- ldout(m_image_ctx.cct, 1) << "STATS: m_log_entries=" << m_log_entries.size()
+ m_cache_state->allocated_bytes = m_bytes_allocated;
+ m_cache_state->cached_bytes = m_bytes_cached;
+ m_cache_state->dirty_bytes = m_bytes_dirty;
+ m_cache_state->free_bytes = m_bytes_allocated_cap - m_bytes_allocated;
+ m_cache_state->hits_full = m_perfcounter->get(l_librbd_pwl_rd_hit_req);
+ m_cache_state->hits_partial = m_perfcounter->get(l_librbd_pwl_rd_part_hit_req);
+ m_cache_state->misses = m_perfcounter->get(l_librbd_pwl_rd_req) -
+ m_cache_state->hits_full - m_cache_state->hits_partial;
+ m_cache_state->hit_bytes = m_perfcounter->get(l_librbd_pwl_rd_hit_bytes);
+ m_cache_state->miss_bytes = m_perfcounter->get(l_librbd_pwl_rd_bytes) -
+ m_cache_state->hit_bytes;
+ update_image_cache_state();
+ ldout(m_image_ctx.cct, 5) << "STATS: m_log_entries=" << m_log_entries.size()
<< ", m_dirty_log_entries=" << m_dirty_log_entries.size()
<< ", m_free_log_entries=" << m_free_log_entries
<< ", m_bytes_allocated=" << m_bytes_allocated
template <typename I>
void AbstractWriteLog<I>::arm_periodic_stats() {
ceph_assert(ceph_mutex_is_locked(*m_timer_lock));
- if (m_periodic_stats_enabled) {
- m_timer_ctx = new LambdaContext(
- [this](int r) {
- /* m_timer_lock is held */
- periodic_stats();
- arm_periodic_stats();
- });
- m_timer->add_event_after(LOG_STATS_INTERVAL_SECONDS, m_timer_ctx);
- }
+ m_timer_ctx = new LambdaContext([this](int r) {
+ /* m_timer_lock is held */
+ periodic_stats();
+ arm_periodic_stats();
+ });
+ m_timer->add_event_after(LOG_STATS_INTERVAL_SECONDS, m_timer_ctx);
}
template <typename I>
// Start the thread
m_thread_pool.start();
- m_periodic_stats_enabled = m_cache_state->log_periodic_stats;
/* Do these after we drop lock */
later.add(new LambdaContext([this](int r) {
- if (m_periodic_stats_enabled) {
/* Log stats for the first time */
periodic_stats();
/* Arm periodic stats logging for the first time */
std::lock_guard timer_locker(*m_timer_lock);
arm_periodic_stats();
- }
- }));
+ }));
m_image_ctx.op_work_queue->queue(on_finish, 0);
}
[this, ctx](int r) {
ldout(m_image_ctx.cct, 6) << "image cache cleaned" << dendl;
Context *next_ctx = override_ctx(r, ctx);
- bool periodic_stats_enabled = m_periodic_stats_enabled;
- m_periodic_stats_enabled = false;
-
- if (periodic_stats_enabled) {
- /* Log stats one last time if they were enabled */
- periodic_stats();
- }
+ periodic_stats();
{
std::lock_guard locker(m_lock);
check_image_cache_state_clean();
/* Flush all writes to OSDs (unless disabled) and wait for all
in-progress flush writes to complete */
ldout(m_image_ctx.cct, 6) << "flushing" << dendl;
- if (m_periodic_stats_enabled) {
- periodic_stats();
- }
+ periodic_stats();
}
flush_dirty_entries(next_ctx);
});
/* Throttle writes concurrently allocating & replicating */
unsigned int m_free_lanes = pwl::MAX_CONCURRENT_WRITES;
- /* Initialized from config, then set false during shutdown */
- std::atomic<bool> m_periodic_stats_enabled = {false};
SafeTimer *m_timer = nullptr; /* Used with m_timer_lock */
mutable ceph::mutex *m_timer_lock = nullptr; /* Used with and by m_timer */
Context *m_timer_ctx = nullptr;
#include "librbd/ImageCtx.h"
#include "librbd/Operations.h"
#include "common/config_proxy.h"
-#include "common/ceph_json.h"
#include "common/environment.h"
#include "common/hostname.h"
#include "librbd/plugin/Api.h"
using namespace std;
namespace {
-bool get_json_format(const std::string& s, JSONFormattable *f) {
- JSONParser p;
- bool success = p.parse(s.c_str(), s.size());
- if (success) {
- decode_json_obj(*f, &p);
+bool get_json_format(const std::string& s, json_spirit::mObject *o) {
+ json_spirit::mValue json_root;
+ if (!json_spirit::read(s.c_str(), json_root)) {
+ return false;
+ } else {
+ auto cache_state_root = json_root.get_obj()["persistent_cache"];
+ *o = cache_state_root.get_obj();
}
- return success;
+ return true;
}
} // namespace
<< dendl;
ConfigProxy &config = image_ctx->config;
- log_periodic_stats = config.get_val<bool>("rbd_persistent_cache_log_periodic_stats");
- cache_type = config.get_val<std::string>("rbd_persistent_cache_mode");
+ mode = config.get_val<std::string>("rbd_persistent_cache_mode");
}
template <typename I>
ImageCacheState<I>::ImageCacheState(
- I *image_ctx, JSONFormattable &f, plugin::Api<I>& plugin_api) :
+ I *image_ctx, json_spirit::mObject &o, plugin::Api<I>& plugin_api) :
m_image_ctx(image_ctx), m_plugin_api(plugin_api) {
ldout(image_ctx->cct, 20) << "Initialize RWL cache state with data from "
<< "server side"<< dendl;
- present = (bool)f["present"];
- empty = (bool)f["empty"];
- clean = (bool)f["clean"];
- cache_type = f["cache_type"];
- host = f["pwl_host"];
- path = f["pwl_path"];
- uint64_t pwl_size;
- std::istringstream iss(f["pwl_size"]);
- iss >> pwl_size;
- size = pwl_size;
-
- // Others from config
- ConfigProxy &config = image_ctx->config;
- log_periodic_stats = config.get_val<bool>("rbd_persistent_cache_log_periodic_stats");
+ try {
+ present = o["present"].get_bool();
+ empty = o["empty"].get_bool();
+ clean = o["clean"].get_bool();
+ host = o["host"].get_str();
+ path = o["path"].get_str();
+ mode = o["mode"].get_str();
+ size = o["size"].get_uint64();
+ } catch (std::runtime_error& e) {
+ lderr(image_ctx->cct) << "failed to parse cache state: " << e.what()
+ << dendl;
+ }
}
template <typename I>
void ImageCacheState<I>::write_image_cache_state(Context *on_finish) {
+ stats_timestamp = ceph_clock_now();
std::shared_lock owner_lock{m_image_ctx->owner_lock};
- JSONFormattable f;
- ::encode_json(IMAGE_CACHE_STATE.c_str(), *this, &f);
- std::ostringstream oss;
- f.flush(oss);
- std::string image_state_json = oss.str();
+ json_spirit::mObject o;
+ o["present"] = present;
+ o["empty"] = empty;
+ o["clean"] = clean;
+ o["host"] = host;
+ o["path"] = path;
+ o["mode"] = mode;
+ o["size"] = size;
+ o["stats_timestamp"] = stats_timestamp.sec();
+ o["allocated_bytes"] = allocated_bytes;
+ o["cached_bytes"] = cached_bytes;
+ o["dirty_bytes"] = dirty_bytes;
+ o["free_bytes"] = free_bytes;
+ o["hits_full"] = hits_full;
+ o["hits_partial"] = hits_partial;
+ o["misses"] = misses;
+ o["hit_bytes"] = hit_bytes;
+ o["miss_bytes"] = miss_bytes;
+ json_spirit::mObject cache_state_obj;
+ cache_state_obj["persistent_cache"] = o;
+ stringstream json_string;
+ json_spirit::write(cache_state_obj, json_string);
+ std::string image_state_json = json_string.str();
ldout(m_image_ctx->cct, 20) << __func__ << " Store state: "
<< image_state_json << dendl;
m_image_ctx, IMAGE_CACHE_STATE, on_finish);
}
-template <typename I>
-void ImageCacheState<I>::dump(ceph::Formatter *f) const {
- ::encode_json("present", present, f);
- ::encode_json("empty", empty, f);
- ::encode_json("clean", clean, f);
- ::encode_json("cache_type", cache_type, f);
- ::encode_json("pwl_host", host, f);
- ::encode_json("pwl_path", path, f);
- ::encode_json("pwl_size", size, f);
-}
-
template <typename I>
ImageCacheState<I>* ImageCacheState<I>::create_image_cache_state(
I* image_ctx, plugin::Api<I>& plugin_api, int &r) {
cache_state = new ImageCacheState<I>(image_ctx, plugin_api);
} else {
ceph_assert(!cache_state_str.empty());
- JSONFormattable f;
- bool success = get_json_format(cache_state_str, &f);
+ json_spirit::mObject o;
+ bool success = get_json_format(cache_state_str, &o);
if (!success) {
- lderr(image_ctx->cct) << "Failed to parse cache state: "
+ lderr(image_ctx->cct) << "failed to parse cache state: "
<< cache_state_str << dendl;
r = -EINVAL;
return nullptr;
}
- bool cache_exists = (bool)f["present"];
+ bool cache_exists = o["present"].get_bool();
if (!cache_exists) {
cache_state = new ImageCacheState<I>(image_ctx, plugin_api);
} else {
- cache_state = new ImageCacheState<I>(image_ctx, f, plugin_api);
+ cache_state = new ImageCacheState<I>(image_ctx, o, plugin_api);
}
}
return cache_state;
cls_client::metadata_get(&image_ctx->md_ctx, image_ctx->header_oid,
IMAGE_CACHE_STATE, &cache_state_str);
if (!cache_state_str.empty()) {
- JSONFormattable f;
- bool success = get_json_format(cache_state_str, &f);
+ json_spirit::mObject o;
+ bool success = get_json_format(cache_state_str, &o);
if (!success) {
+ lderr(image_ctx->cct) << "failed to parse cache state" << dendl;
cache_state = new ImageCacheState<I>(image_ctx, plugin_api);
} else {
- cache_state = new ImageCacheState<I>(image_ctx, f, plugin_api);
+ cache_state = new ImageCacheState<I>(image_ctx, o, plugin_api);
}
}
return cache_state;
#ifndef CEPH_LIBRBD_CACHE_RWL_IMAGE_CACHE_STATE_H
#define CEPH_LIBRBD_CACHE_RWL_IMAGE_CACHE_STATE_H
+#include "json_spirit/json_spirit.h"
#include "librbd/ImageCtx.h"
#include "librbd/cache/Types.h"
#include <string>
-class JSONFormattable;
namespace ceph {
class Formatter;
}
bool clean = true;
std::string host;
std::string path;
- std::string cache_type;
+ std::string mode;
uint64_t size = 0;
- bool log_periodic_stats;
+ /* After reloading, the following data does not need to be read,
+ * but recalculated. */
+ utime_t stats_timestamp;
+ uint64_t allocated_bytes = 0;
+ uint64_t cached_bytes = 0;
+ uint64_t dirty_bytes = 0;
+ uint64_t free_bytes = 0;
+ uint64_t hits_full = 0;
+ uint64_t hits_partial = 0;
+ uint64_t misses = 0;
+ uint64_t hit_bytes = 0;
+ uint64_t miss_bytes = 0;
ImageCacheState(ImageCtxT* image_ctx, plugin::Api<ImageCtxT>& plugin_api);
- ImageCacheState(ImageCtxT* image_ctx, JSONFormattable& f,
+ ImageCacheState(ImageCtxT* image_ctx, json_spirit::mObject& f,
plugin::Api<ImageCtxT>& plugin_api);
~ImageCacheState() {}
- ImageCacheType get_image_cache_type() const {
- if (cache_type == "rwl") {
+ ImageCacheType get_image_cache_mode() const {
+ if (mode == "rwl") {
return IMAGE_CACHE_TYPE_RWL;
- } else if (cache_type == "ssd") {
+ } else if (mode == "ssd") {
return IMAGE_CACHE_TYPE_SSD;
}
return IMAGE_CACHE_TYPE_UNKNOWN;
}
-
void write_image_cache_state(Context *on_finish);
void clear_image_cache_state(Context *on_finish);
- void dump(ceph::Formatter *f) const;
-
static ImageCacheState<ImageCtxT>* create_image_cache_state(
ImageCtxT* image_ctx, plugin::Api<ImageCtxT>& plugin_api, int &r);
return;
}
- auto cache_type = cache_state->get_image_cache_type();
- switch(cache_type) {
+ auto mode = cache_state->get_image_cache_mode();
+ switch (mode) {
#ifdef WITH_RBD_RWL
case cache::IMAGE_CACHE_TYPE_RWL:
m_image_cache =
bool present, bool empty, bool clean,
string host, string path,
uint64_t size) {
- ConfigProxy &config = image_ctx->config;
ASSERT_EQ(present, state.present);
ASSERT_EQ(empty, state.empty);
ASSERT_EQ(clean, state.clean);
ASSERT_EQ(host, state.host);
ASSERT_EQ(path, state.path);
ASSERT_EQ(size, state.size);
- ASSERT_EQ(config.get_val<bool>("rbd_persistent_cache_log_periodic_stats"),
- state.log_periodic_stats);
}
void expect_op_work_queue(MockImageCtx& mock_image_ctx) {
ASSERT_EQ(0, finish_ctx.wait());
}
-static void get_jf(const string& s, JSONFormattable *f)
+static void get_jf(const string& s, json_spirit::mObject *f)
{
- JSONParser p;
- bool result = p.parse(s.c_str(), s.size());
+ json_spirit::mValue json_root;
+ bool result = json_spirit::read(s.c_str(), json_root);
if (!result) {
- cout << "Failed to parse: '" << s << "'" << std::endl;
+ cout << "failed to parse: '" << s << "'" << std::endl;
+ } else {
+ auto cache_state_root = json_root.get_obj()["persistent_cache"];
+ *f = cache_state_root.get_obj();
}
ASSERT_EQ(true, result);
- try {
- decode_json_obj(*f, &p);
- } catch (JSONDecoder::err& e) {
- ASSERT_TRUE(0 == "Failed to decode JSON object");
- }
}
TEST_F(TestMockCacheReplicatedWriteLog, init_state_json_write) {
MockImageCtx mock_image_ctx(*ictx);
- JSONFormattable f;
+ json_spirit::mObject f;
string strf = "{ \"present\": \"1\", \"empty\": \"0\", \"clean\": \"0\", \
\"pwl_host\": \"testhost\", \
\"pwl_path\": \"/tmp\", \
bool present, bool empty, bool clean,
string host, string path,
uint64_t size) {
- ConfigProxy &config = image_ctx->config;
ASSERT_EQ(present, state.present);
ASSERT_EQ(empty, state.empty);
ASSERT_EQ(clean, state.clean);
ASSERT_EQ(host, state.host);
ASSERT_EQ(path, state.path);
ASSERT_EQ(size, state.size);
- ASSERT_EQ(config.get_val<bool>("rbd_persistent_cache_log_periodic_stats"),
- state.log_periodic_stats);
}
void expect_op_work_queue(MockImageCtx& mock_image_ctx) {
ASSERT_EQ(0, finish_ctx.wait());
}
-static void get_jf(const string& s, JSONFormattable *f)
+static void get_jf(const string& s, json_spirit::mObject *f)
{
- JSONParser p;
- bool result = p.parse(s.c_str(), s.size());
+ json_spirit::mValue json_root;
+ bool result = json_spirit::read(s.c_str(), json_root);
if (!result) {
- cout << "Failed to parse: '" << s << "'" << std::endl;
+ cout << "failed to parse: '" << s << "'" << std::endl;
+ } else {
+ auto cache_state_root = json_root.get_obj()["persistent_cache"];
+ *f = cache_state_root.get_obj();
}
ASSERT_EQ(true, result);
- try {
- decode_json_obj(*f, &p);
- } catch (JSONDecoder::err& e) {
- ASSERT_TRUE(0 == "Failed to decode JSON object");
- }
}
TEST_F(TestMockCacheSSDWriteLog, init_state_json_write) {
MockImageCtx mock_image_ctx(*ictx);
- JSONFormattable f;
+ json_spirit::mObject f;
string strf = "{ \"present\": \"1\", \"empty\": \"0\", \"clean\": \"0\", \
\"pwl_host\": \"testhost\", \
\"pwl_path\": \"/tmp\", \
projects / ceph.git / commitdiff