From 4e437c206e53dfc34622e4dc7e81d88f02db0c10 Mon Sep 17 00:00:00 2001 From: Joshua Baergen Date: Thu, 16 Feb 2023 09:51:56 -0700 Subject: [PATCH] os/bluestore/bluefs: Add write op count metrics. There were already several metrics counting bytes written to the various regions of bluefs but nothing counting ops. Also add a sum of bytes written to match the read and new write count metrics. This provides more insight behind the cause of https://tracker.ceph.com/issues/58530. Signed-off-by: Joshua Baergen (cherry picked from commit b4ad3840c513a15a5828418b5a733ed959390845) --- src/os/bluestore/BlueFS.cc | 21 +++++++++++++++++++++ src/os/bluestore/BlueFS.h | 6 ++++++ 2 files changed, 27 insertions(+) diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc index 64fbe1e88039f..9fcce83fb4198 100644 --- a/src/os/bluestore/BlueFS.cc +++ b/src/os/bluestore/BlueFS.cc @@ -248,6 +248,8 @@ void BlueFS::_init_logger() "jlen", PerfCountersBuilder::PRIO_INTERESTING, unit_t(UNIT_BYTES)); b.add_u64_counter(l_bluefs_log_compactions, "log_compactions", "Compactions of the metadata log"); + b.add_u64_counter(l_bluefs_log_write_count, "log_write_count", + "Write op count to the metadata log"); b.add_u64_counter(l_bluefs_logged_bytes, "logged_bytes", "Bytes written to the metadata log", "j", @@ -256,6 +258,10 @@ void BlueFS::_init_logger() "Files written to WAL"); b.add_u64_counter(l_bluefs_files_written_sst, "files_written_sst", "Files written to SSTs"); + b.add_u64_counter(l_bluefs_write_count_wal, "write_count_wal", + "Write op count to WAL"); + b.add_u64_counter(l_bluefs_write_count_sst, "write_count_sst", + "Write op count to SSTs"); b.add_u64_counter(l_bluefs_bytes_written_wal, "bytes_written_wal", "Bytes written to WAL", "walb", @@ -372,6 +378,13 @@ void BlueFS::_init_logger() "Bytes requested in prefetch read mode", NULL, PerfCountersBuilder::PRIO_USEFUL, unit_t(UNIT_BYTES)); + b.add_u64_counter(l_bluefs_write_count, "write_count", + "Write requests processed"); + b.add_u64_counter(l_bluefs_write_disk_count, "write_disk_count", + "Write requests sent to disk"); + b.add_u64_counter(l_bluefs_write_bytes, "write_bytes", + "Bytes written", NULL, + PerfCountersBuilder::PRIO_USEFUL, unit_t(UNIT_BYTES)); b.add_time_avg (l_bluefs_compaction_lat, "compact_lat", "Average bluefs log compaction latency", "c__t", @@ -3095,6 +3108,7 @@ void BlueFS::_flush_and_sync_log_core(int64_t runway) if (realign && realign != super.block_size) bl.append_zero(realign); + logger->inc(l_bluefs_log_write_count, 1); logger->inc(l_bluefs_logged_bytes, bl.length()); if (true) { @@ -3425,11 +3439,16 @@ int BlueFS::_flush_data(FileWriter *h, uint64_t offset, uint64_t length, bool bu h->pos = offset + length; length = bl.length(); + logger->inc(l_bluefs_write_count, 1); + logger->inc(l_bluefs_write_bytes, length); + switch (h->writer_type) { case WRITER_WAL: + logger->inc(l_bluefs_write_count_wal, 1); logger->inc(l_bluefs_bytes_written_wal, length); break; case WRITER_SST: + logger->inc(l_bluefs_write_count_sst, 1); logger->inc(l_bluefs_bytes_written_sst, length); break; } @@ -3441,6 +3460,8 @@ int BlueFS::_flush_data(FileWriter *h, uint64_t offset, uint64_t length, bool bu uint64_t bloff = 0; uint64_t bytes_written_slow = 0; while (length > 0) { + logger->inc(l_bluefs_write_disk_count, 1); + uint64_t x_len = std::min(p->length - x_off, length); bufferlist t; t.substr_of(bl, bloff, x_len); diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h index 736450c5e9939..3c0994114a1ac 100644 --- a/src/os/bluestore/BlueFS.h +++ b/src/os/bluestore/BlueFS.h @@ -31,9 +31,12 @@ enum { l_bluefs_num_files, l_bluefs_log_bytes, l_bluefs_log_compactions, + l_bluefs_log_write_count, l_bluefs_logged_bytes, l_bluefs_files_written_wal, l_bluefs_files_written_sst, + l_bluefs_write_count_wal, + l_bluefs_write_count_sst, l_bluefs_bytes_written_wal, l_bluefs_bytes_written_sst, l_bluefs_bytes_written_slow, @@ -61,6 +64,9 @@ enum { l_bluefs_read_disk_bytes_slow, l_bluefs_read_prefetch_count, l_bluefs_read_prefetch_bytes, + l_bluefs_write_count, + l_bluefs_write_disk_count, + l_bluefs_write_bytes, l_bluefs_compaction_lat, l_bluefs_compaction_lock_lat, l_bluefs_alloc_shared_dev_fallbacks, -- 2.39.5