});
}
+cache_stats_t Cache::get_stats(
+ bool report_detail, double seconds) const
+{
+ cache_stats_t ret;
+ lru.get_stats(ret, report_detail, seconds);
+ return ret;
+}
+
+void Cache::LRU::get_stats(
+ cache_stats_t &stats,
+ bool report_detail,
+ double seconds) const
+{
+ LOG_PREFIX(Cache::LRU::get_stats);
+
+ stats.lru_sizes = cache_size_stats_t{current_size, lru.size()};
+ stats.lru_io = overall_io;
+ stats.lru_io.minus(last_overall_io);
+
+ if (report_detail && seconds != 0) {
+ cache_io_stats_t _trans_io = trans_io;
+ _trans_io.minus(last_trans_io);
+
+ cache_io_stats_t other_io = stats.lru_io;
+ other_io.minus(_trans_io);
+
+ counter_by_src_t<counter_by_extent_t<cache_io_stats_t> >
+ _trans_io_by_src_ext = trans_io_by_src_ext;
+ counter_by_src_t<cache_io_stats_t> trans_io_by_src;
+ for (uint8_t _src=0; _src<TRANSACTION_TYPE_MAX; ++_src) {
+ auto src = static_cast<transaction_type_t>(_src);
+ auto& io_by_ext = get_by_src(_trans_io_by_src_ext, src);
+ const auto& last_io_by_ext = get_by_src(last_trans_io_by_src_ext, src);
+ auto& trans_io_per_src = get_by_src(trans_io_by_src, src);
+ for (uint8_t _ext=0; _ext<EXTENT_TYPES_MAX; ++_ext) {
+ auto ext = static_cast<extent_types_t>(_ext);
+ auto& extent_io = get_by_ext(io_by_ext, ext);
+ const auto& last_extent_io = get_by_ext(last_io_by_ext, ext);
+ extent_io.minus(last_extent_io);
+ trans_io_per_src.add(extent_io);
+ }
+ }
+
+ std::ostringstream oss;
+ oss << "\nlru total" << stats.lru_sizes;
+ cache_size_stats_t data_sizes;
+ cache_size_stats_t mdat_sizes;
+ cache_size_stats_t phys_sizes;
+ for (uint8_t _ext=0; _ext<EXTENT_TYPES_MAX; ++_ext) {
+ auto ext = static_cast<extent_types_t>(_ext);
+ const auto extent_sizes = get_by_ext(sizes_by_ext, ext);
+ if (is_data_type(ext)) {
+ data_sizes.add(extent_sizes);
+ } else if (is_logical_metadata_type(ext)) {
+ mdat_sizes.add(extent_sizes);
+ } else if (is_physical_type(ext)) {
+ phys_sizes.add(extent_sizes);
+ }
+ }
+ oss << "\n data" << data_sizes
+ << "\n mdat" << mdat_sizes
+ << "\n phys" << phys_sizes;
+
+ oss << "\nlru io: trans-"
+ << cache_io_stats_printer_t{seconds, _trans_io}
+ << "; other-"
+ << cache_io_stats_printer_t{seconds, other_io};
+ for (uint8_t _src=0; _src<TRANSACTION_TYPE_MAX; ++_src) {
+ auto src = static_cast<transaction_type_t>(_src);
+ const auto& trans_io_per_src = get_by_src(trans_io_by_src, src);
+ if (trans_io_per_src.is_empty()) {
+ continue;
+ }
+ cache_io_stats_t data_io;
+ cache_io_stats_t mdat_io;
+ cache_io_stats_t phys_io;
+ const auto& io_by_ext = get_by_src(_trans_io_by_src_ext, src);
+ for (uint8_t _ext=0; _ext<EXTENT_TYPES_MAX; ++_ext) {
+ auto ext = static_cast<extent_types_t>(_ext);
+ const auto extent_io = get_by_ext(io_by_ext, ext);
+ if (is_data_type(ext)) {
+ data_io.add(extent_io);
+ } else if (is_logical_metadata_type(ext)) {
+ mdat_io.add(extent_io);
+ } else if (is_physical_type(ext)) {
+ phys_io.add(extent_io);
+ }
+ }
+ oss << "\n " << src << ": "
+ << cache_io_stats_printer_t{seconds, trans_io_per_src}
+ << "\n data: "
+ << cache_io_stats_printer_t{seconds, data_io}
+ << "\n mdat: "
+ << cache_io_stats_printer_t{seconds, mdat_io}
+ << "\n phys: "
+ << cache_io_stats_printer_t{seconds, phys_io};
+ }
+
+ INFO("{}", oss.str());
+ }
+
+ last_overall_io = overall_io;
+ last_trans_io = trans_io;
+ last_trans_io_by_src_ext = trans_io_by_src_ext;
+}
+
}
Cache(ExtentPlacementManager &epm);
~Cache();
+ cache_stats_t get_stats(bool report_detail, double seconds) const;
+
/// Creates empty transaction by source
TransactionRef create_transaction(
Transaction::src_t src,
return;
}
if (ext.is_stable_clean() && !ext.is_placeholder()) {
- lru.move_to_top(ext);
+ lru.move_to_top(ext, p_src);
}
}
friend class crimson::os::seastore::backref::BtreeBackrefManager;
friend class crimson::os::seastore::BackrefManager;
+
/**
* lru
*
// current size (bytes)
size_t current_size = 0;
+ counter_by_extent_t<cache_size_stats_t> sizes_by_ext;
+ cache_io_stats_t overall_io;
+ cache_io_stats_t trans_io;
+ counter_by_src_t<counter_by_extent_t<cache_io_stats_t> >
+ trans_io_by_src_ext;
+
+ mutable cache_io_stats_t last_overall_io;
+ mutable cache_io_stats_t last_trans_io;
+ mutable counter_by_src_t<counter_by_extent_t<cache_io_stats_t> >
+ last_trans_io_by_src_ext;
+
CachedExtent::primary_ref_list lru;
- void do_remove_from_lru(CachedExtent &extent) {
+ void do_remove_from_lru(
+ CachedExtent &extent,
+ const Transaction::src_t* p_src) {
assert(extent.is_stable_clean() && !extent.is_placeholder());
assert(extent.primary_ref_list_hook.is_linked());
assert(lru.size() > 0);
- assert(current_size >= extent.get_length());
+ auto extent_length = extent.get_length();
+ assert(current_size >= extent_length);
lru.erase(lru.s_iterator_to(extent));
- current_size -= extent.get_length();
+ current_size -= extent_length;
+ get_by_ext(sizes_by_ext, extent.get_type()).account_out(extent_length);
+ overall_io.account_out(extent_length);
+ if (p_src) {
+ trans_io.account_out(extent_length);
+ get_by_ext(
+ get_by_src(trans_io_by_src_ext, *p_src),
+ extent.get_type()).account_out(extent_length);
+ }
intrusive_ptr_release(&extent);
}
return lru.size();
}
+ void get_stats(
+ cache_stats_t &stats,
+ bool report_detail,
+ double seconds) const;
+
void remove_from_lru(CachedExtent &extent) {
assert(extent.is_stable_clean() && !extent.is_placeholder());
if (extent.primary_ref_list_hook.is_linked()) {
- do_remove_from_lru(extent);
+ do_remove_from_lru(extent, nullptr);
}
}
- void move_to_top(CachedExtent &extent) {
+ void move_to_top(
+ CachedExtent &extent,
+ const Transaction::src_t* p_src) {
assert(extent.is_stable_clean() && !extent.is_placeholder());
+ auto extent_length = extent.get_length();
if (extent.primary_ref_list_hook.is_linked()) {
// present, move to top (back)
assert(lru.size() > 0);
- assert(current_size >= extent.get_length());
+ assert(current_size >= extent_length);
lru.erase(lru.s_iterator_to(extent));
lru.push_back(extent);
} else {
// absent, add to top (back)
- current_size += extent.get_length();
+ current_size += extent_length;
+ get_by_ext(sizes_by_ext, extent.get_type()).account_in(extent_length);
+ overall_io.account_in(extent_length);
+ if (p_src) {
+ trans_io.account_in(extent_length);
+ get_by_ext(
+ get_by_src(trans_io_by_src_ext, *p_src),
+ extent.get_type()).account_in(extent_length);
+ }
intrusive_ptr_add_ref(&extent);
lru.push_back(extent);
// trim to capacity
while (current_size > capacity) {
- do_remove_from_lru(lru.front());
+ do_remove_from_lru(lru.front(), p_src);
}
}
}
LOG_PREFIX(Cache::LRU::clear);
for (auto iter = lru.begin(); iter != lru.end();) {
SUBDEBUG(seastore_cache, "clearing {}", *iter);
- do_remove_from_lru(*(iter++));
+ do_remove_from_lru(*(iter++), nullptr);
}
}
ceph_assert(seastar::this_shard_id() == primary_core);
shard_device_stats.resize(seastar::smp::count);
shard_io_stats.resize(seastar::smp::count);
+ shard_cache_stats.resize(seastar::smp::count);
return shard_stores.invoke_on_all([this](const Shard &local_store) {
bool report_detail = false;
double seconds = 0;
local_store.get_device_stats(report_detail, seconds);
shard_io_stats[seastar::this_shard_id()] =
local_store.get_io_stats(report_detail, seconds);
+ shard_cache_stats[seastar::this_shard_id()] =
+ local_store.get_cache_stats(report_detail, seconds);
}).then([this] {
LOG_PREFIX(SeaStore);
auto now = seastar::lowres_clock::now();
<< ") ";
}
INFO("details: {}", oss_pending.str());
+
+ cache_stats_t cache_total = {};
+ for (const auto& s : shard_cache_stats) {
+ cache_total.add(s);
+ }
+ cache_size_stats_t lru_sizes_ps = cache_total.lru_sizes;
+ lru_sizes_ps.size /= seastar::smp::count;
+ lru_sizes_ps.num_extents /= seastar::smp::count;
+ cache_io_stats_t lru_io_ps = cache_total.lru_io;
+ lru_io_ps.in_size /= seastar::smp::count;
+ lru_io_ps.in_num_extents /= seastar::smp::count;
+ lru_io_ps.out_size /= seastar::smp::count;
+ lru_io_ps.out_num_extents /= seastar::smp::count;
+ INFO("cache lru: total{} {}; per-shard: total{} {}",
+ cache_total.lru_sizes,
+ cache_io_stats_printer_t{seconds, cache_total.lru_io},
+ lru_sizes_ps,
+ cache_io_stats_printer_t{seconds, lru_io_ps});
return seastar::now();
});
}
return ret;
}
+cache_stats_t SeaStore::Shard::get_cache_stats(
+ bool report_detail, double seconds) const
+{
+ return transaction_manager->get_cache_stats(
+ report_detail, seconds);
+}
+
std::unique_ptr<SeaStore> make_seastore(
const std::string &device)
{
shard_stats_t get_io_stats(bool report_detail, double seconds) const;
+ cache_stats_t get_cache_stats(bool report_detail, double seconds) const;
+
private:
struct internal_context_t {
CollectionRef ch;
seastar::lowres_clock::time_point::min();
mutable std::vector<device_stats_t> shard_device_stats;
mutable std::vector<shard_stats_t> shard_io_stats;
+ mutable std::vector<cache_stats_t> shard_cache_stats;
};
std::unique_ptr<SeaStore> make_seastore(
return out;
}
+std::ostream& operator<<(std::ostream& out, const cache_io_stats_printer_t& p)
+{
+ constexpr const char* dfmt = "{:.2f}";
+ out << "in("
+ << fmt::format(dfmt, p.stats.get_in_mbs(p.seconds))
+ << "MiB/s,"
+ << fmt::format(dfmt, p.stats.get_in_avg_kb())
+ << "KiB,"
+ << fmt::format(dfmt, p.stats.in_num_extents/p.seconds)
+ << "ps) out("
+ << fmt::format(dfmt, p.stats.get_out_mbs(p.seconds))
+ << "MiB/s,"
+ << fmt::format(dfmt, p.stats.get_out_avg_kb())
+ << "KiB,"
+ << fmt::format(dfmt, p.stats.out_num_extents/p.seconds)
+ << "ps)";
+ return out;
+}
+
+std::ostream& operator<<(std::ostream& out, const cache_size_stats_t& p)
+{
+ constexpr const char* dfmt = "{:.2f}";
+ out << "("
+ << fmt::format(dfmt, p.get_mb())
+ << "MiB,"
+ << fmt::format(dfmt, p.get_avg_kb())
+ << "KiB,"
+ << p.num_extents
+ << ")";
+ return out;
}
+
+} // namespace crimson::os::seastore
return counters_by_ext[index];
}
+template <typename CounterT>
+const CounterT& get_by_ext(
+ const counter_by_extent_t<CounterT>& counters_by_ext,
+ extent_types_t ext) {
+ auto index = static_cast<uint8_t>(ext);
+ assert(index < EXTENT_TYPES_MAX);
+ return counters_by_ext[index];
+}
+
struct grouped_io_stats {
uint64_t num_io = 0;
uint64_t num_io_grouped = 0;
}
};
+struct cache_io_stats_t {
+ uint64_t in_size = 0;
+ uint64_t in_num_extents = 0;
+ uint64_t out_size = 0;
+ uint64_t out_num_extents = 0;
+
+ bool is_empty() const {
+ return in_num_extents == 0 && out_num_extents == 0;
+ }
+
+ double get_in_mbs(double seconds) const {
+ return (in_size>>12)/(seconds*256);
+ }
+
+ double get_in_avg_kb() const {
+ return (in_size>>10)/static_cast<double>(in_num_extents);
+ }
+
+ double get_out_mbs(double seconds) const {
+ return (out_size>>12)/(seconds*256);
+ }
+
+ double get_out_avg_kb() const {
+ return (out_size>>10)/static_cast<double>(out_num_extents);
+ }
+
+ void account_in(extent_len_t size) {
+ in_size += size;
+ ++in_num_extents;
+ }
+
+ void account_out(extent_len_t size) {
+ out_size += size;
+ ++out_num_extents;
+ }
+
+ void minus(const cache_io_stats_t& o) {
+ in_size -= o.in_size;
+ in_num_extents -= o.in_num_extents;
+ out_size -= o.out_size;
+ out_num_extents -= o.out_num_extents;
+ }
+
+ void add(const cache_io_stats_t& o) {
+ in_size += o.in_size;
+ in_num_extents += o.in_num_extents;
+ out_size += o.out_size;
+ out_num_extents += o.out_num_extents;
+ }
+};
+struct cache_io_stats_printer_t {
+ double seconds;
+ const cache_io_stats_t &stats;
+};
+std::ostream& operator<<(std::ostream&, const cache_io_stats_printer_t&);
+
+struct cache_size_stats_t {
+ uint64_t size = 0;
+ uint64_t num_extents = 0;
+
+ double get_mb() const {
+ return (size>>12)/static_cast<double>(256);
+ }
+
+ double get_avg_kb() const {
+ return (size>>10)/static_cast<double>(num_extents);
+ }
+
+ void account_in(extent_len_t sz) {
+ size += sz;
+ ++num_extents;
+ }
+
+ void account_out(extent_len_t sz) {
+ assert(size >= sz);
+ assert(num_extents > 0);
+ size -= sz;
+ --num_extents;
+ }
+
+ void add(const cache_size_stats_t& o) {
+ size += o.size;
+ num_extents += o.num_extents;
+ }
+};
+std::ostream& operator<<(std::ostream&, const cache_size_stats_t&);
+
+struct cache_stats_t {
+ cache_size_stats_t lru_sizes;
+ cache_io_stats_t lru_io;
+
+ void add(const cache_stats_t& o) {
+ lru_sizes.add(o.lru_sizes);
+ lru_io.add(o.lru_io);
+ }
+};
+
}
WRITE_CLASS_DENC_BOUNDED(crimson::os::seastore::seastore_meta_t)
WRITE_CLASS_DENC_BOUNDED(crimson::os::seastore::segment_tail_t)
#if FMT_VERSION >= 90000
+template <> struct fmt::formatter<crimson::os::seastore::cache_io_stats_printer_t> : fmt::ostream_formatter {};
+template <> struct fmt::formatter<crimson::os::seastore::cache_size_stats_t> : fmt::ostream_formatter {};
template <> struct fmt::formatter<crimson::os::seastore::data_category_t> : fmt::ostream_formatter {};
template <> struct fmt::formatter<crimson::os::seastore::delta_info_t> : fmt::ostream_formatter {};
template <> struct fmt::formatter<crimson::os::seastore::device_id_printer_t> : fmt::ostream_formatter {};
return epm->get_device_stats(journal_stats, report_detail, seconds);
}
+ cache_stats_t get_cache_stats(bool report_detail, double seconds) const {
+ return cache->get_stats(report_detail, seconds);
+ }
+
/// Resets transaction
void reset_transaction_preserve_handle(Transaction &t) {
return cache->reset_transaction_preserve_handle(t);
projects / ceph-ci.git / commitdiff