From 899276a5aa9f07a2faf4e7b659c0ea5c90dbf3b9 Mon Sep 17 00:00:00 2001 From: NitzanMordhai Date: Wed, 28 Jun 2023 09:57:11 +0000 Subject: [PATCH] ceph-dencoder: osd - Add missing types Currently, ceph-dencoder lacks certain osd types, preventing us from accurately checking the ceph corpus for encode-decode mismatches. This pull request aims to address this issue by adding the missing types to ceph-dencoder. To successfully incorporate these types into ceph-dencoder, we need to introduce the necessary `dump` and `generate_test_instances` functions that was missing in some types. These functions are essential for proper encode and decode of the added types. This PR will enhance the functionality of ceph-dencoder by including the missing types, enabling a comprehensive analysis of encode-decode consistency. With the addition of these types, we can ensure the robustness and correctness of the ceph corpus. This update will significantly contribute to improving the overall reliability and accuracy of ceph-dencoder. It allows for a more comprehensive assessment of the encode-decode behavior, leading to enhanced data integrity and stability within the ceph ecosystem. Fixes: https://tracker.ceph.com/issues/61788 Signed-off-by: Nitzan Mordechai --- src/cls/fifo/cls_fifo.cc | 9 +++ src/include/cephfs/types.h | 8 +++ src/mgr/DaemonHealthMetric.h | 12 +++- src/mgr/MetricTypes.h | 8 +++ src/mgr/OSDPerfMetricTypes.cc | 2 +- src/mgr/OSDPerfMetricTypes.h | 76 ++++++++++++++++++++++- src/mon/CreatingPGs.h | 14 ++++- src/mon/MgrMap.h | 53 ++++++++++++++-- src/mon/MonCommand.h | 19 ++++++ src/mon/MonMap.cc | 21 +++++++ src/mon/MonMap.h | 2 + src/mon/PGMap.h | 12 ++++ src/mon/health_check.h | 1 + src/mon/mon_types.h | 40 +++++++++++- src/os/Transaction.h | 4 +- src/os/bluestore/bluefs_types.cc | 9 +++ src/os/bluestore/bluefs_types.h | 1 + src/os/bluestore/bluestore_types.h | 11 +++- src/osd/OSDMap.h | 6 ++ src/osd/SnapMapper.cc | 16 +++++ src/osd/SnapMapper.h | 12 ++++ src/osd/osd_types.cc | 24 ++++++-- src/osd/osd_types.h | 82 ++++++++++++++++++++++++- src/tools/ceph-dencoder/denc_plugin.h | 1 + src/tools/ceph-dencoder/denc_registry.h | 5 +- src/tools/ceph-dencoder/osd_types.h | 63 ++++++++++++++++++- 26 files changed, 488 insertions(+), 23 deletions(-) diff --git a/src/cls/fifo/cls_fifo.cc b/src/cls/fifo/cls_fifo.cc index 85022eeb061..4b02903916d 100644 --- a/src/cls/fifo/cls_fifo.cc +++ b/src/cls/fifo/cls_fifo.cc @@ -14,6 +14,7 @@ #include +#include "common/Formatter.h" #include "include/buffer.h" #include "include/types.h" @@ -53,6 +54,14 @@ struct entry_header { decode(mtime, bl); DECODE_FINISH(bl); } + void dump(ceph::Formatter *f) const { + f->dump_stream("mtime") << mtime; + } + static void generate_test_instances(std::list& ls) { + ls.push_back(new entry_header); + ls.push_back(new entry_header); + ls.back()->mtime = ceph::real_clock::now(); + } }; WRITE_CLASS_ENCODER(entry_header) diff --git a/src/include/cephfs/types.h b/src/include/cephfs/types.h index 2abfe29b08c..068c9ef8199 100644 --- a/src/include/cephfs/types.h +++ b/src/include/cephfs/types.h @@ -199,6 +199,14 @@ struct vinodeno_t { decode(ino, p); decode(snapid, p); } + void dump(ceph::Formatter *f) const { + f->dump_unsigned("ino", ino); + f->dump_unsigned("snapid", snapid); + } + static void generate_test_instances(std::list& ls) { + ls.push_back(new vinodeno_t); + ls.push_back(new vinodeno_t(1, 2)); + } inodeno_t ino; snapid_t snapid; diff --git a/src/mgr/DaemonHealthMetric.h b/src/mgr/DaemonHealthMetric.h index ce0dad2c87e..a6fbce29910 100644 --- a/src/mgr/DaemonHealthMetric.h +++ b/src/mgr/DaemonHealthMetric.h @@ -5,6 +5,7 @@ #include #include +#include "common/Formatter.h" #include "include/denc.h" enum class daemon_metric : uint8_t { @@ -66,7 +67,16 @@ public: denc(v.value.n, p); DENC_FINISH(p); } - + void dump(Formatter *f) const { + f->dump_string("type", get_type_name()); + f->dump_int("n", get_n()); + f->dump_int("n1", get_n1()); + f->dump_int("n2", get_n2()); + } + static void generate_test_instances(std::list& o) { + o.push_back(new DaemonHealthMetric(daemon_metric::SLOW_OPS, 1)); + o.push_back(new DaemonHealthMetric(daemon_metric::PENDING_CREATING_PGS, 1, 2)); + } std::string get_type_name() const { return daemon_metric_name(get_type()); } diff --git a/src/mgr/MetricTypes.h b/src/mgr/MetricTypes.h index 586c470ca56..762564f37ee 100644 --- a/src/mgr/MetricTypes.h +++ b/src/mgr/MetricTypes.h @@ -30,6 +30,14 @@ struct OSDMetricPayload { denc(v.report, p); DENC_FINISH(p); } + + void dump(ceph::Formatter *f) const { + encode_json("report", report, f); + } + + static void generate_test_instances(std::list& ls) { + ls.push_back(new OSDMetricPayload); + } }; struct MDSMetricPayload { diff --git a/src/mgr/OSDPerfMetricTypes.cc b/src/mgr/OSDPerfMetricTypes.cc index bce95e0ae38..e21318cc744 100644 --- a/src/mgr/OSDPerfMetricTypes.cc +++ b/src/mgr/OSDPerfMetricTypes.cc @@ -1,8 +1,8 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab +#include "common/Formatter.h" #include "mgr/OSDPerfMetricTypes.h" - #include using ceph::bufferlist; diff --git a/src/mgr/OSDPerfMetricTypes.h b/src/mgr/OSDPerfMetricTypes.h index 1b5904e13ae..8dd0afd8cb4 100644 --- a/src/mgr/OSDPerfMetricTypes.h +++ b/src/mgr/OSDPerfMetricTypes.h @@ -4,9 +4,9 @@ #ifndef OSD_PERF_METRIC_H_ #define OSD_PERF_METRIC_H_ +#include "common/ceph_json.h" #include "include/denc.h" #include "include/stringify.h" - #include "mgr/Types.h" #include @@ -70,6 +70,23 @@ struct OSDPerfMetricSubKeyDescriptor { denc(v.regex_str, p); DENC_FINISH(p); } + + void dump(ceph::Formatter *f) const { + f->dump_unsigned("type", static_cast(type)); + f->dump_string("regex", regex_str); + } + + static void generate_test_instances(std::list& o) { + o.push_back(new OSDPerfMetricSubKeyDescriptor()); + o.push_back(new OSDPerfMetricSubKeyDescriptor(OSDPerfMetricSubKeyType::CLIENT_ID, ".*")); + o.push_back(new OSDPerfMetricSubKeyDescriptor(OSDPerfMetricSubKeyType::CLIENT_ADDRESS, ".*")); + o.push_back(new OSDPerfMetricSubKeyDescriptor(OSDPerfMetricSubKeyType::POOL_ID, ".*")); + o.push_back(new OSDPerfMetricSubKeyDescriptor(OSDPerfMetricSubKeyType::NAMESPACE, ".*")); + o.push_back(new OSDPerfMetricSubKeyDescriptor(OSDPerfMetricSubKeyType::OSD_ID, ".*")); + o.push_back(new OSDPerfMetricSubKeyDescriptor(OSDPerfMetricSubKeyType::PG_ID, ".*")); + o.push_back(new OSDPerfMetricSubKeyDescriptor(OSDPerfMetricSubKeyType::OBJECT_NAME, ".*")); + o.push_back(new OSDPerfMetricSubKeyDescriptor(OSDPerfMetricSubKeyType::SNAP_ID, ".*")); + } }; WRITE_CLASS_DENC(OSDPerfMetricSubKeyDescriptor) @@ -184,6 +201,23 @@ struct PerformanceCounterDescriptor { DENC_FINISH(p); } + void dump(ceph::Formatter *f) const { + f->dump_unsigned("type", static_cast(type)); + } + + static void generate_test_instances(std::list& o) { + o.push_back(new PerformanceCounterDescriptor()); + o.push_back(new PerformanceCounterDescriptor(PerformanceCounterType::OPS)); + o.push_back(new PerformanceCounterDescriptor(PerformanceCounterType::WRITE_OPS)); + o.push_back(new PerformanceCounterDescriptor(PerformanceCounterType::READ_OPS)); + o.push_back(new PerformanceCounterDescriptor(PerformanceCounterType::BYTES)); + o.push_back(new PerformanceCounterDescriptor(PerformanceCounterType::WRITE_BYTES)); + o.push_back(new PerformanceCounterDescriptor(PerformanceCounterType::READ_BYTES)); + o.push_back(new PerformanceCounterDescriptor(PerformanceCounterType::LATENCY)); + o.push_back(new PerformanceCounterDescriptor(PerformanceCounterType::WRITE_LATENCY)); + o.push_back(new PerformanceCounterDescriptor(PerformanceCounterType::READ_LATENCY)); + } + void pack_counter(const PerformanceCounter &c, ceph::buffer::list *bl) const; void unpack_counter(ceph::buffer::list::const_iterator& bl, PerformanceCounter *c) const; @@ -306,6 +340,28 @@ struct OSDPerfMetricQuery { DENC_FINISH(p); } + void dump(ceph::Formatter *f) const { + encode_json("key_descriptor", key_descriptor, f); + encode_json("performance_counter_descriptors", + performance_counter_descriptors, f); + } + + static void generate_test_instances(std::list &o) { + o.push_back(new OSDPerfMetricQuery()); + o.push_back(new OSDPerfMetricQuery(OSDPerfMetricKeyDescriptor(), + PerformanceCounterDescriptors())); + o.push_back(new OSDPerfMetricQuery(OSDPerfMetricKeyDescriptor(), + PerformanceCounterDescriptors{ + PerformanceCounterType::WRITE_OPS, + PerformanceCounterType::READ_OPS, + PerformanceCounterType::BYTES, + PerformanceCounterType::WRITE_BYTES, + PerformanceCounterType::READ_BYTES, + PerformanceCounterType::LATENCY, + PerformanceCounterType::WRITE_LATENCY, + PerformanceCounterType::READ_LATENCY})); + } + void get_performance_counter_descriptors( PerformanceCounterDescriptors *descriptors) const { *descriptors = performance_counter_descriptors; @@ -353,6 +409,24 @@ struct OSDPerfMetricReport { denc(v.group_packed_performance_counters, p); DENC_FINISH(p); } + + void dump(ceph::Formatter *f) const { + encode_json("performance_counter_descriptors", + performance_counter_descriptors, f); + encode_json("group_packed_performance_counters", + group_packed_performance_counters, f); + } + + static void generate_test_instances(std::list &o) { + o.push_back(new OSDPerfMetricReport); + o.push_back(new OSDPerfMetricReport); + o.back()->performance_counter_descriptors.push_back( + PerformanceCounterDescriptor(PerformanceCounterType::OPS)); + o.back()->performance_counter_descriptors.push_back( + PerformanceCounterDescriptor(PerformanceCounterType::WRITE_OPS)); + o.back()->performance_counter_descriptors.push_back( + PerformanceCounterDescriptor(PerformanceCounterType::READ_OPS)); + } }; WRITE_CLASS_DENC(OSDPerfMetricReport) diff --git a/src/mon/CreatingPGs.h b/src/mon/CreatingPGs.h index 0075f81e7bb..808a22b1567 100644 --- a/src/mon/CreatingPGs.h +++ b/src/mon/CreatingPGs.h @@ -83,8 +83,20 @@ struct creating_pgs_t { f->dump_object("pg_history", history); f->dump_object("past_intervals", past_intervals); } + static void generate_test_instances(std::list& o) { + o.push_back(new pg_create_info); + o.back()->create_epoch = 10; + o.push_back(new pg_create_info); + o.back()->create_epoch = 1; + o.back()->create_stamp = utime_t(2, 3); + o.back()->up = {1, 2, 3}; + o.back()->up_primary = 1; + o.back()->acting = {1, 2, 3}; + o.back()->acting_primary = 1; + } - pg_create_info() {} + pg_create_info() + : create_epoch(0) {} pg_create_info(epoch_t e, utime_t t) : create_epoch(e), create_stamp(t) { diff --git a/src/mon/MgrMap.h b/src/mon/MgrMap.h index f37ed97fd16..0a906cd039f 100644 --- a/src/mon/MgrMap.h +++ b/src/mon/MgrMap.h @@ -20,6 +20,7 @@ #include "msg/msg_types.h" #include "include/encoding.h" #include "include/utime.h" +#include "common/ceph_json.h" #include "common/Formatter.h" #include "common/ceph_releases.h" #include "common/version.h" @@ -74,7 +75,8 @@ public: decode(see_also, p); DECODE_FINISH(p); } - void dump(ceph::Formatter *f) const { + void dump(ceph::Formatter *f) const + { f->dump_string("name", name); f->dump_string("type", Option::type_to_str( static_cast(type))); @@ -102,6 +104,23 @@ public: } f->close_section(); } + static void generate_test_instances(std::list& ls) + { + ls.push_back(new ModuleOption); + ls.push_back(new ModuleOption); + ls.back()->name = "name"; + ls.back()->type = Option::TYPE_STR; + ls.back()->level = Option::LEVEL_ADVANCED; + ls.back()->flags = Option::FLAG_RUNTIME; + ls.back()->default_value = "default_value"; + ls.back()->min = "min"; + ls.back()->max = "max"; + ls.back()->enum_allowed.insert("enum_allowed"); + ls.back()->desc = "desc"; + ls.back()->long_desc = "long_desc"; + ls.back()->tags.insert("tag"); + ls.back()->see_also.insert("see_also"); + } }; class ModuleInfo @@ -139,7 +158,8 @@ public: return (name == rhs.name) && (can_run == rhs.can_run); } - void dump(ceph::Formatter *f) const { + void dump(ceph::Formatter *f) const + { f->open_object_section("module"); f->dump_string("name", name); f->dump_bool("can_run", can_run); @@ -151,6 +171,16 @@ public: f->close_section(); f->close_section(); } + + static void generate_test_instances(std::list& ls) + { + ls.push_back(new ModuleInfo); + ls.push_back(new ModuleInfo); + ls.back()->name = "name"; + ls.back()->can_run = true; + ls.back()->error_string = "error_string"; + ls.back()->module_options["module_option"] = ModuleOption(); + } }; class StandbyInfo @@ -209,6 +239,19 @@ public: } DECODE_FINISH(p); } + void dump(ceph::Formatter *f) const + { + f->dump_unsigned("gid", gid); + f->dump_string("name", name); + encode_json("available_modules", available_modules, f); + f->dump_unsigned("mgr_features", mgr_features); + } + static void generate_test_instances(std::list& ls) + { + ls.push_back(new StandbyInfo(1, "a", {}, 0)); + ls.push_back(new StandbyInfo(2, "b", {}, 0)); + ls.push_back(new StandbyInfo(3, "c", {}, 0)); + } bool have_module(const std::string &module_name) const { @@ -501,7 +544,8 @@ public: DECODE_FINISH(p); } - void dump(ceph::Formatter *f) const { + void dump(ceph::Formatter *f) const + { f->dump_int("epoch", epoch); f->dump_int("active_gid", get_active_gid()); f->dump_string("active_name", get_active_name()); @@ -561,7 +605,8 @@ public: f->close_section(); // active_clients } - static void generate_test_instances(std::list &l) { + static void generate_test_instances(std::list &l) + { l.push_back(new MgrMap); } diff --git a/src/mon/MonCommand.h b/src/mon/MonCommand.h index cb60d3d1706..5caebfc6f13 100644 --- a/src/mon/MonCommand.h +++ b/src/mon/MonCommand.h @@ -14,6 +14,7 @@ #pragma once #include +#include "common/Formatter.h" #include "include/encoding.h" struct MonCommand { @@ -53,6 +54,24 @@ struct MonCommand { DECODE_FINISH(bl); } + void dump(ceph::Formatter *f) const { + f->dump_string("cmdstring", cmdstring); + f->dump_string("helpstring", helpstring); + f->dump_string("module", module); + f->dump_string("req_perms", req_perms); + f->dump_unsigned("flags", flags); + } + + static void generate_test_instances(std::list& ls) { + ls.push_back(new MonCommand); + ls.push_back(new MonCommand); + ls.back()->cmdstring = "foo"; + ls.back()->helpstring = "bar"; + ls.back()->module = "baz"; + ls.back()->req_perms = "quux"; + ls.back()->flags = FLAG_NOFORWARD; + } + /** * Unversioned encoding for use within encode_array. */ diff --git a/src/mon/MonMap.cc b/src/mon/MonMap.cc index bb8a4b19455..6eb37df171a 100644 --- a/src/mon/MonMap.cc +++ b/src/mon/MonMap.cc @@ -21,6 +21,7 @@ #include "include/ceph_features.h" #include "include/addr_parsing.h" #include "common/ceph_argparse.h" +#include "common/ceph_json.h" #include "common/dns_resolve.h" #include "common/errno.h" #include "common/dout.h" @@ -110,6 +111,26 @@ void mon_info_t::print(ostream& out) const << " crush location " << crush_loc; } +void mon_info_t::dump(ceph::Formatter *f) const +{ + f->dump_string("name", name); + f->dump_stream("addr") << public_addrs; + f->dump_int("priority", priority); + f->dump_float("weight", weight); + encode_json("crush_location", crush_loc, f); +} + +void mon_info_t::generate_test_instances(list& ls) +{ + ls.push_back(new mon_info_t); + ls.push_back(new mon_info_t); + ls.back()->name = "noname"; + ls.back()->public_addrs.parse("v1:1.2.3.4:567/890"); + ls.back()->priority = 1; + ls.back()->weight = 1.0; + ls.back()->crush_loc.emplace("root", "default"); + ls.back()->crush_loc.emplace("host", "foo"); +} namespace { struct rank_cmp { bool operator()(const mon_info_t &a, const mon_info_t &b) const { diff --git a/src/mon/MonMap.h b/src/mon/MonMap.h index 7fedd2c1128..5bd72b1d917 100644 --- a/src/mon/MonMap.h +++ b/src/mon/MonMap.h @@ -85,6 +85,8 @@ struct mon_info_t { void encode(ceph::buffer::list& bl, uint64_t features) const; void decode(ceph::buffer::list::const_iterator& p); void print(std::ostream& out) const; + void dump(ceph::Formatter *f) const; + static void generate_test_instances(std::list& ls); }; WRITE_CLASS_ENCODER_FEATURES(mon_info_t) diff --git a/src/mon/PGMap.h b/src/mon/PGMap.h index dbbfce9d52a..2a127b802a8 100644 --- a/src/mon/PGMap.h +++ b/src/mon/PGMap.h @@ -66,6 +66,18 @@ public: decode(up_not_acting, p); decode(primary, p); } + void dump(ceph::Formatter *f) const { + f->dump_int("acting", acting); + f->dump_int("up_not_acting", up_not_acting); + f->dump_int("primary", primary); + } + static void generate_test_instances(std::list& o) { + o.push_back(new pg_count); + o.push_back(new pg_count); + o.back()->acting = 1; + o.back()->up_not_acting = 2; + o.back()->primary = 3; + } }; mempool::pgmap::unordered_map num_pg_by_osd; diff --git a/src/mon/health_check.h b/src/mon/health_check.h index 4e74637f9e5..e6d1d759551 100644 --- a/src/mon/health_check.h +++ b/src/mon/health_check.h @@ -60,6 +60,7 @@ struct health_check_t { static void generate_test_instances(std::list& ls) { ls.push_back(new health_check_t); + ls.back()->severity = HEALTH_WARN; ls.push_back(new health_check_t); ls.back()->severity = HEALTH_ERR; ls.back()->summary = "summarization"; diff --git a/src/mon/mon_types.h b/src/mon/mon_types.h index cce9976f3c3..d23d3ba3ae4 100644 --- a/src/mon/mon_types.h +++ b/src/mon/mon_types.h @@ -110,6 +110,15 @@ struct FeatureMap { f->close_section(); } } + + static void generate_test_instances(std::list& ls) { + ls.push_back(new FeatureMap); + ls.push_back(new FeatureMap); + ls.back()->add(CEPH_ENTITY_TYPE_OSD, CEPH_FEATURE_UID); + ls.back()->add(CEPH_ENTITY_TYPE_OSD, CEPH_FEATURE_NOSRCADDR); + ls.back()->add(CEPH_ENTITY_TYPE_OSD, CEPH_FEATURE_PGID64); + ls.back()->add(CEPH_ENTITY_TYPE_OSD, CEPH_FEATURE_INCSUBOSDMAP); + } }; WRITE_CLASS_ENCODER(FeatureMap) @@ -190,6 +199,20 @@ struct DataStats { store_stats.dump(f); f->close_section(); } + static void generate_test_instances(std::list& ls) { + ls.push_back(new DataStats); + ls.push_back(new DataStats); + ls.back()->fs_stats.byte_total = 1024*1024; + ls.back()->fs_stats.byte_used = 512*1024; + ls.back()->fs_stats.byte_avail = 256*1024; + ls.back()->fs_stats.avail_percent = 50; + ls.back()->last_update = utime_t(); + ls.back()->store_stats.bytes_total = 1024*1024; + ls.back()->store_stats.bytes_sst = 512*1024; + ls.back()->store_stats.bytes_log = 256*1024; + ls.back()->store_stats.bytes_misc = 256*1024; + ls.back()->store_stats.last_update = utime_t(); + } void encode(ceph::buffer::list &bl) const { ENCODE_START(3, 1, bl); @@ -467,6 +490,14 @@ public: decode(features, p); DECODE_FINISH(p); } + + static void generate_test_instances(std::list& ls) { + ls.push_back(new mon_feature_t); + ls.push_back(new mon_feature_t); + ls.back()->features = 1; + ls.push_back(new mon_feature_t); + ls.back()->features = 2; + } }; WRITE_CLASS_ENCODER(mon_feature_t) @@ -642,7 +673,7 @@ inline std::ostream& operator<<(std::ostream& out, const mon_feature_t& f) { struct ProgressEvent { std::string message; ///< event description - float progress; ///< [0..1] + float progress = 0.0f; ///< [0..1] bool add_to_ceph_s; void encode(ceph::buffer::list& bl) const { ENCODE_START(2, 1, bl); @@ -669,6 +700,13 @@ struct ProgressEvent { f->dump_float("progress", progress); f->dump_bool("add_to_ceph_s", add_to_ceph_s); } + static void generate_test_instances(std::list& o) { + o.push_back(new ProgressEvent); + o.push_back(new ProgressEvent); + o.back()->message = "test message"; + o.back()->progress = 0.5; + o.back()->add_to_ceph_s = true; + } }; WRITE_CLASS_ENCODER(ProgressEvent) diff --git a/src/os/Transaction.h b/src/os/Transaction.h index 119d676702e..f349a29c9f1 100644 --- a/src/os/Transaction.h +++ b/src/os/Transaction.h @@ -1299,8 +1299,8 @@ public: void dump(ceph::Formatter *f); static void generate_test_instances(std::list& o); }; -WRITE_CLASS_ENCODER(Transaction) -WRITE_CLASS_ENCODER(Transaction::TransactionData) +WRITE_CLASS_ENCODER(ceph::os::Transaction) +WRITE_CLASS_ENCODER(ceph::os::Transaction::TransactionData) std::ostream& operator<<(std::ostream& out, const Transaction& tx); diff --git a/src/os/bluestore/bluefs_types.cc b/src/os/bluestore/bluefs_types.cc index 70c8a4fbf1c..ff9c96ab660 100644 --- a/src/os/bluestore/bluefs_types.cc +++ b/src/os/bluestore/bluefs_types.cc @@ -64,6 +64,15 @@ void bluefs_layout_t::dump(Formatter *f) const f->dump_stream("dedicated_wal") << dedicated_wal; } +void bluefs_layout_t::generate_test_instances(list& ls) +{ + ls.push_back(new bluefs_layout_t); + ls.push_back(new bluefs_layout_t); + ls.back()->shared_bdev = 1; + ls.back()->dedicated_db = true; + ls.back()->dedicated_wal = true; +} + // bluefs_super_t void bluefs_super_t::encode(bufferlist& bl) const diff --git a/src/os/bluestore/bluefs_types.h b/src/os/bluestore/bluefs_types.h index b0ce7c5c9d3..6516f404e12 100644 --- a/src/os/bluestore/bluefs_types.h +++ b/src/os/bluestore/bluefs_types.h @@ -205,6 +205,7 @@ struct bluefs_layout_t { void encode(ceph::buffer::list& bl) const; void decode(ceph::buffer::list::const_iterator& p); void dump(ceph::Formatter *f) const; + static void generate_test_instances(std::list& ls); }; WRITE_CLASS_ENCODER(bluefs_layout_t) diff --git a/src/os/bluestore/bluestore_types.h b/src/os/bluestore/bluestore_types.h index 66d4189ab6d..500a78a143f 100644 --- a/src/os/bluestore/bluestore_types.h +++ b/src/os/bluestore/bluestore_types.h @@ -158,6 +158,14 @@ struct bluestore_extent_ref_map_t { denc_varint_lowz(v.length, p); denc_varint(v.refs, p); } + void dump(ceph::Formatter *f) const { + f->dump_unsigned("length", length); + f->dump_unsigned("refs", refs); + } + static void generate_test_instances(std::list& o) { + o.push_back(new record_t); + o.push_back(new record_t(123, 456)); + } }; typedef mempool::bluestore_cache_other::map map_t; @@ -229,7 +237,7 @@ struct bluestore_extent_ref_map_t { static void generate_test_instances(std::list& o); }; WRITE_CLASS_DENC(bluestore_extent_ref_map_t) - +WRITE_CLASS_DENC(bluestore_extent_ref_map_t::record_t) std::ostream& operator<<(std::ostream& out, const bluestore_extent_ref_map_t& rm); static inline bool operator==(const bluestore_extent_ref_map_t::record_t& l, @@ -973,6 +981,7 @@ struct bluestore_shared_blob_t { uint64_t sbid; ///> shared blob id bluestore_extent_ref_map_t ref_map; ///< shared blob extents + bluestore_shared_blob_t() : sbid(0) {} bluestore_shared_blob_t(uint64_t _sbid) : sbid(_sbid) {} bluestore_shared_blob_t(uint64_t _sbid, bluestore_extent_ref_map_t&& _ref_map ) diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h index 963039d0213..e37aeafc431 100644 --- a/src/osd/OSDMap.h +++ b/src/osd/OSDMap.h @@ -342,6 +342,12 @@ struct PGTempMap { f->close_section(); } } + static void generate_test_instances(std::list& o) { + o.push_back(new PGTempMap); + o.push_back(new PGTempMap); + o.back()->set(pg_t(1, 2), { 3, 4 }); + o.back()->set(pg_t(2, 3), { 4, 5 }); + } }; WRITE_CLASS_ENCODER(PGTempMap) diff --git a/src/osd/SnapMapper.cc b/src/osd/SnapMapper.cc index e41cd1df08e..e79bc5d450a 100644 --- a/src/osd/SnapMapper.cc +++ b/src/osd/SnapMapper.cc @@ -280,6 +280,22 @@ void SnapMapper::object_snaps::decode(ceph::buffer::list::const_iterator &bl) DECODE_FINISH(bl); } +void SnapMapper::object_snaps::dump(ceph::Formatter *f) const +{ + f->dump_stream("oid") << oid; + f->dump_stream("snaps") << snaps; +} + +void SnapMapper::object_snaps::generate_test_instances( + std::list &o) +{ + o.push_back(new object_snaps); + o.push_back(new object_snaps); + o.back()->oid = hobject_t(sobject_t("name", CEPH_NOSNAP)); + o.back()->snaps.insert(1); + o.back()->snaps.insert(2); +} + bool SnapMapper::check(const hobject_t &hoid) const { if (hoid.match(mask_bits, match)) { diff --git a/src/osd/SnapMapper.h b/src/osd/SnapMapper.h index a28b25970fb..a0a4cd3221b 100644 --- a/src/osd/SnapMapper.h +++ b/src/osd/SnapMapper.h @@ -132,6 +132,8 @@ public: object_snaps() {} void encode(ceph::buffer::list &bl) const; void decode(ceph::buffer::list::const_iterator &bp); + void dump(ceph::Formatter *f) const; + static void generate_test_instances(std::list& o); }; struct Mapping { @@ -152,6 +154,16 @@ public: decode(hoid, bl); DECODE_FINISH(bl); } + void dump(ceph::Formatter *f) const { + f->dump_unsigned("snap", snap); + f->dump_stream("hoid") << hoid; + } + static void generate_test_instances(std::list& o) { + o.push_back(new Mapping); + o.push_back(new Mapping); + o.back()->snap = 1; + o.back()->hoid = hobject_t(object_t("objname"), "key", 123, 456, 0, ""); + } }; static const std::string LEGACY_MAPPING_PREFIX; diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc index 948abeaafc8..acb9777ffe9 100644 --- a/src/osd/osd_types.cc +++ b/src/osd/osd_types.cc @@ -1528,6 +1528,11 @@ void pool_opts_t::decode(ceph::buffer::list::const_iterator& bl) DECODE_FINISH(bl); } +void pool_opts_t::generate_test_instances(std::list& o) +{ + o.push_back(new pool_opts_t); +} + ostream& operator<<(ostream& out, const pool_opts_t& opts) { for (auto i = opt_mapping.begin(); i != opt_mapping.end(); ++i) { @@ -3614,6 +3619,7 @@ void pg_info_t::decode(ceph::buffer::list::const_iterator &bl) void pg_info_t::dump(Formatter *f) const { f->dump_stream("pgid") << pgid; + f->dump_stream("shared") << pgid.shard; f->dump_stream("last_update") << last_update; f->dump_stream("last_complete") << last_complete; f->dump_stream("log_tail") << log_tail; @@ -3714,10 +3720,11 @@ void pg_notify_t::dump(Formatter *f) const void pg_notify_t::generate_test_instances(list& o) { + o.push_back(new pg_notify_t); o.push_back(new pg_notify_t(shard_id_t(3), shard_id_t::NO_SHARD, 1, 1, - pg_info_t(), PastIntervals())); - o.push_back(new pg_notify_t(shard_id_t(0), shard_id_t(0), 3, 10, - pg_info_t(), PastIntervals())); + pg_info_t(spg_t(pg_t(0,10), shard_id_t(-1))), PastIntervals())); + o.push_back(new pg_notify_t(shard_id_t(0), shard_id_t(2), 3, 10, + pg_info_t(spg_t(pg_t(10,10), shard_id_t(2))), PastIntervals())); } ostream &operator<<(ostream &lhs, const pg_notify_t ¬ify) @@ -3812,9 +3819,6 @@ void PastIntervals::pg_interval_t::generate_test_instances(list& o.back()->maybe_went_rw = true; } -WRITE_CLASS_ENCODER(PastIntervals::pg_interval_t) - - /** * pi_compact_rep * @@ -6059,6 +6063,14 @@ void chunk_info_t::dump(Formatter *f) const f->dump_unsigned("flags", flags); } +void chunk_info_t::generate_test_instances(std::list& o) +{ + o.push_back(new chunk_info_t); + o.push_back(new chunk_info_t); + o.back()->length = 123; + o.back()->oid = hobject_t(object_t("foo"), "", 123, 456, -1, ""); + o.back()->flags = cflag_t::FLAG_DIRTY; +} bool chunk_info_t::operator==(const chunk_info_t& cit) const { diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h index 8b86b0a3635..11c8e7a3b9b 100644 --- a/src/osd/osd_types.h +++ b/src/osd/osd_types.h @@ -196,6 +196,11 @@ struct pg_shard_t { f->dump_unsigned("shard", shard); } } + static void generate_test_instances(std::list& o) { + o.push_back(new pg_shard_t); + o.push_back(new pg_shard_t(1)); + o.push_back(new pg_shard_t(1, shard_id_t(2))); + } auto operator<=>(const pg_shard_t&) const = default; }; WRITE_CLASS_ENCODER(pg_shard_t) @@ -596,7 +601,14 @@ struct spg_t { decode(shard, bl); DECODE_FINISH(bl); } - + void dump(ceph::Formatter *f) const { + f->dump_stream("pgid") << pgid; + f->dump_unsigned("shard", shard); + } + static void generate_test_instances(std::list& o) { + o.push_back(new spg_t); + o.push_back(new spg_t(pg_t(1, 2), shard_id_t(3))); + } ghobject_t make_temp_ghobject(const std::string& name) const { return ghobject_t( hobject_t(object_t(name), "", CEPH_NOSNAP, @@ -922,6 +934,14 @@ public: auto p = std::cbegin(bl); decode(p); } + void dump(ceph::Formatter *f) const { + f->dump_unsigned("version", version); + f->dump_unsigned("epoch", epoch); + } + static void generate_test_instances(std::list& o) { + o.push_back(new eversion_t); + o.push_back(new eversion_t(1, 2)); + } }; WRITE_CLASS_ENCODER(eversion_t) @@ -1143,6 +1163,7 @@ public: void dump(ceph::Formatter *f) const; void encode(ceph::buffer::list &bl, uint64_t features) const; void decode(ceph::buffer::list::const_iterator &bl); + static void generate_test_instances(std::list& o); private: typedef std::map opts_t; @@ -1188,6 +1209,16 @@ struct pg_merge_meta_t { f->dump_stream("source_version") << source_version; f->dump_stream("target_version") << target_version; } + static void generate_test_instances(std::list& o) { + o.push_back(new pg_merge_meta_t); + o.push_back(new pg_merge_meta_t); + o.back()->source_pgid = pg_t(1,2); + o.back()->ready_epoch = 1; + o.back()->last_epoch_started = 2; + o.back()->last_epoch_clean = 3; + o.back()->source_version = eversion_t(4,5); + o.back()->target_version = eversion_t(6,7); + } }; WRITE_CLASS_ENCODER(pg_merge_meta_t) @@ -3187,6 +3218,46 @@ struct pg_fast_info_t { decode(stats.stats.sum.num_objects_dirty, p); DECODE_FINISH(p); } + void dump(ceph::Formatter *f) const { + f->dump_stream("last_update") << last_update; + f->dump_stream("last_complete") << last_complete; + f->dump_stream("last_user_version") << last_user_version; + f->open_object_section("stats"); + f->dump_stream("version") << stats.version; + f->dump_unsigned("reported_seq", stats.reported_seq); + f->dump_stream("last_fresh") << stats.last_fresh; + f->dump_stream("last_active") << stats.last_active; + f->dump_stream("last_peered") << stats.last_peered; + f->dump_stream("last_clean") << stats.last_clean; + f->dump_stream("last_unstale") << stats.last_unstale; + f->dump_stream("last_undegraded") << stats.last_undegraded; + f->dump_stream("last_fullsized") << stats.last_fullsized; + f->dump_unsigned("log_size", stats.log_size); + f->dump_unsigned("ondisk_log_size", stats.log_size); + f->dump_unsigned("num_bytes", stats.stats.sum.num_bytes); + f->dump_unsigned("num_objects", stats.stats.sum.num_objects); + f->dump_unsigned("num_object_copies", stats.stats.sum.num_object_copies); + f->dump_unsigned("num_rd", stats.stats.sum.num_rd); + f->dump_unsigned("num_rd_kb", stats.stats.sum.num_rd_kb); + f->dump_unsigned("num_wr", stats.stats.sum.num_wr); + f->dump_unsigned("num_wr_kb", stats.stats.sum.num_wr_kb); + f->dump_unsigned("num_objects_dirty", stats.stats.sum.num_objects_dirty); + f->close_section(); + } + static void generate_test_instances(std::list& o) { + o.push_back(new pg_fast_info_t); + o.push_back(new pg_fast_info_t); + o.back()->last_update = eversion_t(1, 2); + o.back()->last_complete = eversion_t(3, 4); + o.back()->last_user_version = version_t(5); + o.back()->stats.version = eversion_t(7, 8); + o.back()->stats.reported_seq = 9; + o.back()->stats.last_fresh = utime_t(10, 0); + o.back()->stats.last_active = utime_t(11, 0); + o.back()->stats.last_peered = utime_t(12, 0); + o.back()->stats.last_clean = utime_t(13, 0); + o.back()->stats.last_unstale = utime_t(14, 0); + } }; WRITE_CLASS_ENCODER(pg_fast_info_t) @@ -3548,6 +3619,7 @@ public: } }; WRITE_CLASS_ENCODER(PastIntervals) +WRITE_CLASS_ENCODER(PastIntervals::pg_interval_t) std::ostream& operator<<(std::ostream& out, const PastIntervals::pg_interval_t& i); std::ostream& operator<<(std::ostream& out, const PastIntervals &i); @@ -4209,6 +4281,13 @@ struct pg_log_op_return_item_t { f->dump_int("rval", rval); f->dump_unsigned("bl_length", bl.length()); } + static void generate_test_instances(std::list& o) { + o.push_back(new pg_log_op_return_item_t); + o.back()->rval = 0; + o.push_back(new pg_log_op_return_item_t); + o.back()->rval = 1; + o.back()->bl.append("asdf"); + } friend bool operator==(const pg_log_op_return_item_t& lhs, const pg_log_op_return_item_t& rhs) { return lhs.rval == rhs.rval && @@ -5727,6 +5806,7 @@ struct chunk_info_t { void encode(ceph::buffer::list &bl) const; void decode(ceph::buffer::list::const_iterator &bl); void dump(ceph::Formatter *f) const; + static void generate_test_instances(std::list& ls); friend std::ostream& operator<<(std::ostream& out, const chunk_info_t& ci); bool operator==(const chunk_info_t& cit) const; bool operator!=(const chunk_info_t& cit) const { diff --git a/src/tools/ceph-dencoder/denc_plugin.h b/src/tools/ceph-dencoder/denc_plugin.h index a203551eadc..e43ad26e174 100644 --- a/src/tools/ceph-dencoder/denc_plugin.h +++ b/src/tools/ceph-dencoder/denc_plugin.h @@ -66,6 +66,7 @@ private: }; #define TYPE(t) plugin->emplace>(#t, false, false); +#define TYPE_VARARGS(t, ...) plugin->emplace>(#t, false, false, ##__VA_ARGS__); #define TYPE_STRAYDATA(t) plugin->emplace>(#t, true, false); #define TYPE_NONDETERMINISTIC(t) plugin->emplace>(#t, false, true); #define TYPE_FEATUREFUL(t) plugin->emplace>(#t, false, false); diff --git a/src/tools/ceph-dencoder/denc_registry.h b/src/tools/ceph-dencoder/denc_registry.h index aad52cbf710..61acc833b90 100644 --- a/src/tools/ceph-dencoder/denc_registry.h +++ b/src/tools/ceph-dencoder/denc_registry.h @@ -47,8 +47,9 @@ protected: bool nondeterministic; public: - DencoderBase(bool stray_okay, bool nondeterministic) - : m_object(new T), + template + DencoderBase(bool stray_okay, bool nondeterministic, Args&&... args) + : m_object(new T(std::forward(args)...)), stray_okay(stray_okay), nondeterministic(nondeterministic) {} ~DencoderBase() override { diff --git a/src/tools/ceph-dencoder/osd_types.h b/src/tools/ceph-dencoder/osd_types.h index 2dcaeb92827..749233e3b7d 100644 --- a/src/tools/ceph-dencoder/osd_types.h +++ b/src/tools/ceph-dencoder/osd_types.h @@ -3,6 +3,7 @@ TYPE(osd_info_t) TYPE_FEATUREFUL(osd_xinfo_t) TYPE_FEATUREFUL_NOCOPY(OSDMap) TYPE_FEATUREFUL_STRAYDATA(OSDMap::Incremental) +TYPE(PGTempMap) #include "osd/osd_types.h" TYPE(osd_reqid_t) @@ -24,12 +25,12 @@ TYPE(pg_hit_set_history_t) TYPE(pg_history_t) TYPE(pg_info_t) TYPE(PastIntervals) +TYPE(PastIntervals::pg_interval_t) TYPE_FEATUREFUL(pg_query_t) TYPE(ObjectModDesc) TYPE(pg_log_entry_t) TYPE(pg_log_dup_t) TYPE(pg_log_t) -TYPE_FEATUREFUL(pg_missing_item) TYPE_FEATUREFUL(pg_missing_t) TYPE(pg_nls_response_t) TYPE(pg_ls_response_t) @@ -54,6 +55,22 @@ TYPE_FEATUREFUL(obj_list_watch_response_t) TYPE(clone_info) TYPE(obj_list_snap_response_t) TYPE(pool_pg_num_history_t) +TYPE(store_statfs_t) +TYPE(spg_t) +TYPE(ObjectCleanRegions) +TYPE(pg_notify_t) +TYPE(pg_merge_meta_t) +TYPE(pg_shard_t) +TYPE(pg_fast_info_t) +TYPE(pg_lease_ack_t) +TYPE(pg_lease_t) +TYPE(pg_log_op_return_item_t) +TYPE(chunk_info_t) +TYPE_FEATUREFUL(pool_opts_t) +TYPE_FEATUREFUL(pg_missing_item) +TYPE(eversion_t) +//TYPE(compact_interval_t) declared in .cc +//TYPE(pg_missing_t::item) #include "osd/ECUtil.h" // TYPE(stripe_info_t) non-standard encoding/decoding functions @@ -72,6 +89,13 @@ TYPE(BloomHitSet) TYPE_NONDETERMINISTIC(HitSet) // because some subclasses are TYPE(HitSet::Params) +#include "osd/SnapMapper.h" +TYPE(SnapMapper::Mapping) +TYPE(SnapMapper::object_snaps) + +#include "os/Transaction.h" +TYPE(ceph::os::Transaction) + #include "os/ObjectStore.h" TYPE(ObjectStore::Transaction) @@ -84,6 +108,7 @@ TYPE(bluestore_bdev_label_t) TYPE(bluestore_cnode_t) TYPE(bluestore_compression_header_t) TYPE(bluestore_extent_ref_map_t) +TYPE_FEATUREFUL(bluestore_extent_ref_map_t::record_t) TYPE(bluestore_pextent_t) TYPE(bluestore_blob_use_tracker_t) // TODO: bluestore_blob_t repurposes the "feature" param of encode() for its @@ -92,8 +117,11 @@ TYPE(bluestore_blob_use_tracker_t) // BlueStore::ExtentMap::encode_some(). maybe we can test it using another // approach. // TYPE_FEATUREFUL(bluestore_blob_t) -// TYPE(bluestore_shared_blob_t) there is no encode here +TYPE(bluestore_shared_blob_t) TYPE(bluestore_onode_t) +TYPE(bluestore_onode_t::shard_info) +using shard_info = bluestore_onode_t::shard_info; +TYPE(shard_info) TYPE(bluestore_deferred_op_t) TYPE(bluestore_deferred_transaction_t) // TYPE(bluestore_compression_header_t) there is no encode here @@ -103,6 +131,7 @@ TYPE(bluefs_extent_t) TYPE(bluefs_fnode_t) TYPE(bluefs_super_t) TYPE(bluefs_transaction_t) +TYPE(bluefs_layout_t) #endif #include "mon/AuthMonitor.h" @@ -110,7 +139,9 @@ TYPE_FEATUREFUL(AuthMonitor::Incremental) #include "mon/PGMap.h" TYPE_FEATUREFUL_NONDETERMINISTIC(PGMapDigest) +TYPE(PGMapDigest::pg_count) TYPE_FEATUREFUL_NONDETERMINISTIC(PGMap) +//TYPE(PGMap::Incremental) #include "mon/MonitorDBStore.h" TYPE(MonitorDBStore::Transaction) @@ -118,29 +149,57 @@ TYPE(MonitorDBStore::Op) #include "mon/MonMap.h" TYPE_FEATUREFUL(MonMap) +TYPE_FEATUREFUL(mon_info_t) + +#include "mon/MonCommand.h" +TYPE(MonCommand) #include "mon/MonCap.h" TYPE(MonCap) #include "mon/MgrMap.h" TYPE_FEATUREFUL(MgrMap) +TYPE(MgrMap::ModuleOption) +TYPE(MgrMap::ModuleInfo) +TYPE(MgrMap::StandbyInfo) #include "mon/mon_types.h" TYPE(MonitorDBStoreStats) TYPE(ScrubResult) +TYPE(mon_feature_t) +TYPE_FEATUREFUL(DataStats) +TYPE_FEATUREFUL(ProgressEvent) +TYPE(FeatureMap) #include "mon/CreatingPGs.h" TYPE_FEATUREFUL(creating_pgs_t) +TYPE_FEATUREFUL(creating_pgs_t::pg_create_info) #include "mgr/ServiceMap.h" TYPE_FEATUREFUL(ServiceMap) TYPE_FEATUREFUL(ServiceMap::Service) TYPE_FEATUREFUL(ServiceMap::Daemon) +#include "mgr/DaemonHealthMetric.h" +TYPE(DaemonHealthMetric) + +#include "mgr/MetricTypes.h" +TYPE(OSDMetricPayload) + +#include "mgr/OSDPerfMetricTypes.h" +TYPE(OSDPerfMetricQuery) +TYPE(OSDPerfMetricSubKeyDescriptor) +TYPE(PerformanceCounterDescriptor) +TYPE(OSDPerfMetricReport) + #include "mon/ConnectionTracker.h" TYPE(ConnectionReport); TYPE(ConnectionTracker); +#include "mon/health_check.h" +TYPE(health_check_t) +TYPE(health_check_map_t) + #include "os/DBObjectMap.h" TYPE(DBObjectMap::_Header) TYPE(DBObjectMap::State) -- 2.39.5