From: Sridhar Seshasayee Date: Thu, 17 Feb 2022 11:38:36 +0000 (+0530) Subject: mon, osd: Add objects trimmed to pg dump stats. X-Git-Tag: v17.2.0~40^2~1 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=5051511e0866de035c3680c41bb1f309ca677d02;p=ceph.git mon, osd: Add objects trimmed to pg dump stats. Add a new column, OBJECTS_TRIMMED, to the pg dump stats that shows the number of objects trimmed when a snap is removed. When a pg splits, the stats from the parent pg is copied to the child pg. In such a case, reset objects_trimmed to 0 for the child pg (see PeeringState::split_into()). Otherwise, this will result in incorrect stats to be shown for a child pg after the split operation. Tests: - Librados C and C++ API tests to verify the number of objects trimmed during snaptrim operation. These tests use the self-managed snaps APIs. - Standalone tests to verify objects trimmed using rados pool snaps. Signed-off-by: Sridhar Seshasayee (cherry picked from commit 00249dc0cc69d4c065acbb33543d10cb360930dc) --- diff --git a/qa/standalone/misc/test-snaptrim-stats.sh b/qa/standalone/misc/test-snaptrim-stats.sh new file mode 100755 index 0000000000000..71a6fced0409c --- /dev/null +++ b/qa/standalone/misc/test-snaptrim-stats.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Red Hat +# +# Author: Sridhar Seshasayee +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7124" # git grep '\<7124\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + CEPH_ARGS+="--debug-bluestore 20 " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + +function TEST_snaptrim_stats() { + local dir=$1 + local poolname=test + local OSDS=3 + local PGNUM=8 + local PGPNUM=8 + local objects=10 + local WAIT_FOR_UPDATE=10 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=$OSDS || return 1 + run_mgr $dir x || return 1 + for osd in $(seq 0 $(expr $OSDS - 1)) + do + run_osd $dir $osd --osd_pool_default_pg_autoscale_mode=off || return 1 + done + + # disable scrubs + ceph osd set noscrub || return 1 + ceph osd set nodeep-scrub || return 1 + + # Create a pool + create_pool $poolname $PGNUM $PGPNUM + wait_for_clean || return 1 + poolid=$(ceph osd dump | grep "^pool.*[']${poolname}[']" | awk '{ print $2 }') + + # write a few objects + TESTDATA="testdata.1" + dd if=/dev/urandom of=$TESTDATA bs=4096 count=1 + for i in `seq 1 $objects` + do + rados -p $poolname put obj${i} $TESTDATA + done + rm -f $TESTDATA + + # create a snapshot, clones + SNAP=1 + rados -p $poolname mksnap snap${SNAP} + TESTDATA="testdata.2" + dd if=/dev/urandom of=$TESTDATA bs=4096 count=1 + for i in `seq 1 $objects` + do + rados -p $poolname put obj${i} $TESTDATA + done + rm -f $TESTDATA + + # remove the snapshot, should trigger snaptrim + rados -p $poolname rmsnap snap${SNAP} + + # check for snaptrim stats + wait_for_clean || return 1 + sleep $WAIT_FOR_UPDATE + local objects_trimmed=0 + for i in $(seq 0 $(expr $PGNUM - 1)) + do + local pgid="${poolid}.${i}" + objects_trimmed=$(expr $objects_trimmed + $(ceph pg $pgid query | \ + jq '.info.stats.objects_trimmed')) + done + + test $objects_trimmed -eq $objects || return 1 + + teardown $dir || return 1 +} + +main test-snaptrim-stats "$@" + +# Local Variables: +# compile-command: "cd build ; make -j4 && \ +# ../qa/run-standalone.sh test-snaptrim-stats.sh" +# End: diff --git a/src/mon/PGMap.cc b/src/mon/PGMap.cc index f7e40e207022d..7e2b42ea82adb 100644 --- a/src/mon/PGMap.cc +++ b/src/mon/PGMap.cc @@ -1676,6 +1676,7 @@ void PGMap::dump_pg_stats_plain( tab.define_column("LAST_SCRUB_DURATION", TextTable::LEFT, TextTable::RIGHT); tab.define_column("SCRUB_SCHEDULING", TextTable::LEFT, TextTable::LEFT); tab.define_column("OBJECTS_SCRUBBED", TextTable::LEFT, TextTable::RIGHT); + tab.define_column("OBJECTS_TRIMMED", TextTable::LEFT, TextTable::RIGHT); } for (const auto& [pg, st] : pg_stats) { @@ -1718,6 +1719,7 @@ void PGMap::dump_pg_stats_plain( << st.last_scrub_duration << st.dump_scrub_schedule() << st.objects_scrubbed + << st.objects_trimmed << TextTable::endrow; } } diff --git a/src/osd/PG.h b/src/osd/PG.h index 5946960240570..8c4b5ce650591 100644 --- a/src/osd/PG.h +++ b/src/osd/PG.h @@ -546,6 +546,30 @@ public: uint64_t get_snap_trimq_size() const override { return snap_trimq.size(); } + + static void add_objects_trimmed_count( + int64_t count, pg_stat_t &stats) { + stats.objects_trimmed += count; + } + + void add_objects_trimmed_count(int64_t count) { + recovery_state.update_stats_wo_resched( + [=](auto &history, auto &stats) { + add_objects_trimmed_count(count, stats); + }); + } + + static void reset_objects_trimmed(pg_stat_t &stats) { + stats.objects_trimmed = 0; + } + + void reset_objects_trimmed() { + recovery_state.update_stats_wo_resched( + [=](auto &history, auto &stats) { + reset_objects_trimmed(stats); + }); + } + unsigned get_target_pg_log_entries() const override; void clear_publish_stats() override; diff --git a/src/osd/PeeringState.cc b/src/osd/PeeringState.cc index dd73537799e82..2101449867cb1 100644 --- a/src/osd/PeeringState.cc +++ b/src/osd/PeeringState.cc @@ -3238,6 +3238,7 @@ void PeeringState::split_into( child->info.stats.parent_split_bits = split_bits; info.stats.stats_invalid = true; child->info.stats.stats_invalid = true; + child->info.stats.objects_trimmed = 0; child->info.last_epoch_started = info.last_epoch_started; child->info.last_interval_started = info.last_interval_started; diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc index dc2160356a816..90d9eaef1f4fb 100644 --- a/src/osd/PrimaryLogPG.cc +++ b/src/osd/PrimaryLogPG.cc @@ -4694,6 +4694,8 @@ int PrimaryLogPG::trim_object( PGTransaction *t = ctx->op_t.get(); + int64_t num_objects_before_trim = ctx->delta_stats.num_objects; + if (new_snaps.empty()) { // remove clone dout(10) << coid << " snaps " << old_snaps << " -> " @@ -4881,6 +4883,13 @@ int PrimaryLogPG::trim_object( t->setattrs(head_oid, attrs); } + // Stats reporting - Set number of objects trimmed + if (num_objects_before_trim > ctx->delta_stats.num_objects) { + int64_t num_objects_trimmed = + num_objects_before_trim - ctx->delta_stats.num_objects; + add_objects_trimmed_count(num_objects_trimmed); + } + *ctxp = std::move(ctx); return 0; } @@ -4896,6 +4905,7 @@ void PrimaryLogPG::kick_snap_trim() dout(10) << __func__ << ": nosnaptrim set, not kicking" << dendl; } else { dout(10) << __func__ << ": clean and snaps to trim, kicking" << dendl; + reset_objects_trimmed(); snap_trimmer_machine.process_event(KickTrim()); } } diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc index cbfd9abd369c9..d47c1e1f04756 100644 --- a/src/osd/osd_types.cc +++ b/src/osd/osd_types.cc @@ -2867,6 +2867,7 @@ void pg_stat_t::dump(Formatter *f) const f->dump_int("last_scrub_duration", last_scrub_duration); f->dump_string("scrub_schedule", dump_scrub_schedule()); f->dump_float("scrub_duration", scrub_duration); + f->dump_int("objects_trimmed", objects_trimmed); stats.dump(f); f->open_array_section("up"); for (auto p = up.cbegin(); p != up.cend(); ++p) @@ -2962,7 +2963,7 @@ bool operator==(const pg_scrubbing_status_t& l, const pg_scrubbing_status_t& r) void pg_stat_t::encode(ceph::buffer::list &bl) const { - ENCODE_START(27, 22, bl); + ENCODE_START(28, 22, bl); encode(version, bl); encode(reported_seq, bl); encode(reported_epoch, bl); @@ -3019,6 +3020,7 @@ void pg_stat_t::encode(ceph::buffer::list &bl) const encode(scrub_sched_status.m_is_periodic, bl); encode(objects_scrubbed, bl); encode(scrub_duration, bl); + encode(objects_trimmed, bl); ENCODE_FINISH(bl); } @@ -3027,7 +3029,7 @@ void pg_stat_t::decode(ceph::buffer::list::const_iterator &bl) { bool tmp; uint32_t old_state; - DECODE_START(27, bl); + DECODE_START(28, bl); decode(version, bl); decode(reported_seq, bl); decode(reported_epoch, bl); @@ -3110,6 +3112,9 @@ void pg_stat_t::decode(ceph::buffer::list::const_iterator &bl) decode(objects_scrubbed, bl); decode(scrub_duration, bl); } + if (struct_v >= 28) { + decode(objects_trimmed, bl); + } } DECODE_FINISH(bl); } @@ -3146,6 +3151,7 @@ void pg_stat_t::generate_test_instances(list& o) a.scrub_duration = 0.003; a.snaptrimq_len = 1048576; a.objects_scrubbed = 0; + a.objects_trimmed = 0; list l; object_stat_collection_t::generate_test_instances(l); a.stats = *l.back(); @@ -3222,7 +3228,8 @@ bool operator==(const pg_stat_t& l, const pg_stat_t& r) l.last_scrub_duration == r.last_scrub_duration && l.scrub_sched_status == r.scrub_sched_status && l.objects_scrubbed == r.objects_scrubbed && - l.scrub_duration == r.scrub_duration; + l.scrub_duration == r.scrub_duration && + l.objects_trimmed == r.objects_trimmed; } // -- store_statfs_t -- diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h index ffce9db071e1d..6581f2155a522 100644 --- a/src/osd/osd_types.h +++ b/src/osd/osd_types.h @@ -2271,6 +2271,7 @@ struct pg_stat_t { // snaptrimq.size() is 64bit, but let's be serious - anything over 50k is // absurd already, so cap it to 2^32 and save 4 bytes at the same time uint32_t snaptrimq_len; + int64_t objects_trimmed; pg_scrubbing_status_t scrub_sched_status; @@ -2297,6 +2298,7 @@ struct pg_stat_t { up_primary(-1), acting_primary(-1), snaptrimq_len(0), + objects_trimmed(0), stats_invalid(false), dirty_stats_invalid(false), omap_stats_invalid(false), diff --git a/src/test/librados/CMakeLists.txt b/src/test/librados/CMakeLists.txt index 9242f68ff7e41..fc033766cc47f 100644 --- a/src/test/librados/CMakeLists.txt +++ b/src/test/librados/CMakeLists.txt @@ -140,6 +140,14 @@ add_executable(ceph_test_rados_api_snapshots_pp snapshots_cxx.cc) target_link_libraries(ceph_test_rados_api_snapshots_pp librados ${UNITTEST_LIBS} radostest-cxx) +add_executable(ceph_test_rados_api_snapshots_stats + snapshots_stats.cc) +target_link_libraries(ceph_test_rados_api_snapshots_stats + librados ${UNITTEST_LIBS} radostest) +add_executable(ceph_test_rados_api_snapshots_stats_pp + snapshots_stats_cxx.cc) +target_link_libraries(ceph_test_rados_api_snapshots_stats_pp + librados ${UNITTEST_LIBS} radostest-cxx) add_executable(ceph_test_rados_api_cls_remote_reads cls_remote_reads.cc diff --git a/src/test/librados/snapshots_stats.cc b/src/test/librados/snapshots_stats.cc new file mode 100644 index 0000000000000..6a0fcaec48acc --- /dev/null +++ b/src/test/librados/snapshots_stats.cc @@ -0,0 +1,326 @@ +#include "include/rados.h" +#include "json_spirit/json_spirit.h" +#include "test/librados/test.h" +#include "test/librados/TestCase.h" + +#include +#include +#include "gtest/gtest.h" +#include +#include + +using std::string; + +class LibRadosSnapshotStatsSelfManaged : public RadosTest { +public: + LibRadosSnapshotStatsSelfManaged() {}; + ~LibRadosSnapshotStatsSelfManaged() override {}; +protected: + void SetUp() override { + // disable pg autoscaler for the tests + string c = + "{" + "\"prefix\": \"config set\", " + "\"who\": \"global\", " + "\"name\": \"osd_pool_default_pg_autoscale_mode\", " + "\"value\": \"off\"" + "}"; + char *cmd[1]; + cmd[0] = (char *)c.c_str(); + std::cout << "Setting pg_autoscaler to 'off'" << std::endl; + ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, + 0, NULL, 0)); + + // disable scrubs for the test + c = string("{\"prefix\": \"osd set\",\"key\":\"noscrub\"}"); + cmd[0] = (char *)c.c_str(); + ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0)); + c = string("{\"prefix\": \"osd set\",\"key\":\"nodeep-scrub\"}"); + cmd[0] = (char *)c.c_str(); + ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0)); + + RadosTest::SetUp(); + } + + void TearDown() override { + // re-enable pg autoscaler + string c = + "{" + "\"prefix\": \"config set\", " + "\"who\": \"global\", " + "\"name\": \"osd_pool_default_pg_autoscale_mode\", " + "\"value\": \"on\"" + "}"; + char *cmd[1]; + cmd[0] = (char *)c.c_str(); + std::cout << "Setting pg_autoscaler to 'on'" << std::endl; + ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, + 0, NULL, 0)); + + // re-enable scrubs + c = string("{\"prefix\": \"osd unset\",\"key\":\"noscrub\"}"); + cmd[0] = (char *)c.c_str(); + ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0)); + c = string("{\"prefix\": \"osd unset\",\"key\":\"nodeep-scrub\"}"); + cmd[0] = (char *)c.c_str(); + ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0)); + + RadosTest::TearDown(); + } +}; + +class LibRadosSnapshotStatsSelfManagedEC : public RadosTestEC { +public: + LibRadosSnapshotStatsSelfManagedEC() {}; + ~LibRadosSnapshotStatsSelfManagedEC() override {}; +protected: + void SetUp() override { + // disable pg autoscaler for the tests + string c = + "{" + "\"prefix\": \"config set\", " + "\"who\": \"global\", " + "\"name\": \"osd_pool_default_pg_autoscale_mode\", " + "\"value\": \"off\"" + "}"; + char *cmd[1]; + cmd[0] = (char *)c.c_str(); + std::cout << "Setting pg_autoscaler to 'off'" << std::endl; + ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, + 0, NULL, 0)); + + // disable scrubs for the test + c = string("{\"prefix\": \"osd set\",\"key\":\"noscrub\"}"); + cmd[0] = (char *)c.c_str(); + ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0)); + c = string("{\"prefix\": \"osd set\",\"key\":\"nodeep-scrub\"}"); + cmd[0] = (char *)c.c_str(); + ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0)); + + RadosTestEC::SetUp(); + } + + void TearDown() override { + // re-enable pg autoscaler + string c = + "{" + "\"prefix\": \"config set\", " + "\"who\": \"global\", " + "\"name\": \"osd_pool_default_pg_autoscale_mode\", " + "\"value\": \"on\"" + "}"; + char *cmd[1]; + cmd[0] = (char *)c.c_str(); + std::cout << "Setting pg_autoscaler to 'on'" << std::endl; + ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, + 0, NULL, 0)); + + // re-enable scrubs + c = string("{\"prefix\": \"osd unset\",\"key\":\"noscrub\"}"); + cmd[0] = (char *)c.c_str(); + ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0)); + c = string("{\"prefix\": \"osd unset\",\"key\":\"nodeep-scrub\"}"); + cmd[0] = (char *)c.c_str(); + ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0)); + + RadosTestEC::TearDown(); + } +}; + +int get_objects_trimmed(json_spirit::Object& pg_dump) { + int objs_trimmed = 0; + + // pg_map + json_spirit::Object pgmap; + for (json_spirit::Object::size_type i = 0; i < pg_dump.size(); ++i) { + json_spirit::Pair& p = pg_dump[i]; + if (p.name_ == "pg_map") { + pgmap = p.value_.get_obj(); + break; + } + } + + // pg_stats array + json_spirit::Array pgs; + for (json_spirit::Object::size_type i = 0; i < pgmap.size(); ++i) { + json_spirit::Pair& p = pgmap[i]; + if (p.name_ == "pg_stats") { + pgs = p.value_.get_array(); + break; + } + } + + // snaptrim stats + for (json_spirit::Object::size_type j = 0; j < pgs.size(); ++j) { + json_spirit::Object& pg_stat = pgs[j].get_obj(); + for(json_spirit::Object::size_type k = 0; k < pg_stat.size(); ++k) { + json_spirit::Pair& stats = pg_stat[k]; + if (stats.name_ == "objects_trimmed") { + objs_trimmed += stats.value_.get_int(); + break; + } + } + } + + return objs_trimmed; +} + +const int bufsize = 128; + +TEST_F(LibRadosSnapshotStatsSelfManaged, SnaptrimStats) { + int num_objs = 10; + + // create objects + char buf[bufsize]; + memset(buf, 0xcc, sizeof(buf)); + for (int i = 0; i < num_objs; ++i) { + string obj = string("foo") + std::to_string(i); + ASSERT_EQ(0, rados_write(ioctx, obj.c_str(), buf, sizeof(buf), 0)); + } + + std::vector my_snaps; + for (int snap = 0; snap < 1; ++snap) { + // create a snapshot, clone + std::vector ns(1); + ns.insert(ns.end(), my_snaps.begin(), my_snaps.end()); + my_snaps.swap(ns); + ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps[0])); + ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0], + &my_snaps[0], my_snaps.size())); + char buf2[sizeof(buf)]; + memset(buf2, 0xdd, sizeof(buf2)); + for (int i = 0; i < num_objs; ++i) { + string obj = string("foo") + std::to_string(i); + ASSERT_EQ(0, rados_write(ioctx, obj.c_str(), buf2, sizeof(buf2), 0)); + } + } + + // wait for maps to settle + ASSERT_EQ(0, rados_wait_for_latest_osdmap(cluster)); + + // remove snaps - should trigger snaptrim + rados_ioctx_snap_set_read(ioctx, LIBRADOS_SNAP_HEAD); + for (unsigned snap = 0; snap < my_snaps.size(); ++snap) { + ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps[snap])); + } + + // sleep for few secs for the trim stats to populate + std::cout << "Waiting for snaptrim stats to be generated" << std::endl; + sleep(30); + + // Dump pg stats and determine if snaptrim stats are getting set + int objects_trimmed = 0; + int tries = 0; + do { + char *buf, *st; + size_t buflen, stlen; + string c = string("{\"prefix\": \"pg dump\",\"format\":\"json\"}"); + const char *cmd = c.c_str(); + ASSERT_EQ(0, rados_mon_command(cluster, (const char **)&cmd, 1, "", 0, + &buf, &buflen, &st, &stlen)); + string outstr(buf, buflen); + json_spirit::Value v; + ASSERT_NE(0, json_spirit::read(outstr, v)) << "unable to parse json." + << '\n' << outstr; + + // pg dump object + json_spirit::Object& obj = v.get_obj(); + objects_trimmed = get_objects_trimmed(obj); + if (objects_trimmed < num_objs) { + tries++; + objects_trimmed = 0; + std::cout << "Still waiting for all objects to be trimmed... " < my_snaps; + for (int snap = 0; snap < 1; ++snap) { + // create a snapshot, clone + std::vector ns(1); + ns.insert(ns.end(), my_snaps.begin(), my_snaps.end()); + my_snaps.swap(ns); + ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps[0])); + ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0], + &my_snaps[0], my_snaps.size())); + char *buf2 = (char *)new char[bsize]; + memset(buf2, 0xdd, bsize); + for (int i = 0; i < num_objs; ++i) { + string obj = string("foo") + std::to_string(i); + ASSERT_EQ(0, rados_write(ioctx, obj.c_str(), buf2, bsize, bsize)); + } + delete[] buf2; + } + + // wait for maps to settle + ASSERT_EQ(0, rados_wait_for_latest_osdmap(cluster)); + + // remove snaps - should trigger snaptrim + rados_ioctx_snap_set_read(ioctx, LIBRADOS_SNAP_HEAD); + for (unsigned snap = 0; snap < my_snaps.size(); ++snap) { + ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps[snap])); + } + + // sleep for few secs for the trim stats to populate + std::cout << "Waiting for snaptrim stats to be generated" << std::endl; + sleep(30); + + // Dump pg stats and determine if snaptrim stats are getting set + int objects_trimmed = 0; + int tries = 0; + do { + char *buf, *st; + size_t buflen, stlen; + string c = string("{\"prefix\": \"pg dump\",\"format\":\"json\"}"); + const char *cmd = c.c_str(); + ASSERT_EQ(0, rados_mon_command(cluster, (const char **)&cmd, 1, 0, 0, + &buf, &buflen, &st, &stlen)); + string outstr(buf, buflen); + json_spirit::Value v; + ASSERT_NE(0, json_spirit::read(outstr, v)) << "Unable tp parse json." + << '\n' << outstr; + + // pg dump object + json_spirit::Object& obj = v.get_obj(); + objects_trimmed = get_objects_trimmed(obj); + if (objects_trimmed != num_objs) { + tries++; + objects_trimmed = 0; + std::cout << "Still waiting for all objects to be trimmed... " < +#include +#include +#include + +#include "gtest/gtest.h" + +#include "include/rados.h" +#include "include/rados/librados.hpp" +#include "json_spirit/json_spirit.h" +#include "test/librados/test_cxx.h" +#include "test/librados/testcase_cxx.h" + +using namespace librados; + +using std::string; + +class LibRadosSnapshotStatsSelfManagedPP : public RadosTestPP { +public: + LibRadosSnapshotStatsSelfManagedPP() {}; + ~LibRadosSnapshotStatsSelfManagedPP() override {}; +protected: + void SetUp() override { + // disable pg autoscaler for the tests + string cmd = + "{" + "\"prefix\": \"config set\", " + "\"who\": \"global\", " + "\"name\": \"osd_pool_default_pg_autoscale_mode\", " + "\"value\": \"off\"" + "}"; + std::cout << "Setting pg_autoscaler to 'off'" << std::endl; + bufferlist inbl; + bufferlist outbl; + ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL)); + + // disable scrubs for the test + cmd = "{\"prefix\": \"osd set\",\"key\":\"noscrub\"}"; + ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL)); + cmd = "{\"prefix\": \"osd set\",\"key\":\"nodeep-scrub\"}"; + ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL)); + + RadosTestPP::SetUp(); + } + + void TearDown() override { + // re-enable pg autoscaler + string cmd = + "{" + "\"prefix\": \"config set\", " + "\"who\": \"global\", " + "\"name\": \"osd_pool_default_pg_autoscale_mode\", " + "\"value\": \"on\"" + "}"; + std::cout << "Setting pg_autoscaler to 'on'" << std::endl; + bufferlist inbl; + bufferlist outbl; + ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL)); + + // re-enable scrubs + cmd = "{\"prefix\": \"osd unset\",\"key\":\"noscrub\"}"; + ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL)); + cmd = string("{\"prefix\": \"osd unset\",\"key\":\"nodeep-scrub\"}"); + ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL)); + + RadosTestPP::TearDown(); + } +}; + +class LibRadosSnapshotStatsSelfManagedECPP : public RadosTestECPP { +public: + LibRadosSnapshotStatsSelfManagedECPP() {}; + ~LibRadosSnapshotStatsSelfManagedECPP() override {}; +protected: + void SetUp() override { + // disable pg autoscaler for the tests + string cmd = + "{" + "\"prefix\": \"config set\", " + "\"who\": \"global\", " + "\"name\": \"osd_pool_default_pg_autoscale_mode\", " + "\"value\": \"off\"" + "}"; + std::cout << "Setting pg_autoscaler to 'off'" << std::endl; + bufferlist inbl; + bufferlist outbl; + ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL)); + + // disable scrubs for the test + cmd = string("{\"prefix\": \"osd set\",\"key\":\"noscrub\"}"); + ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL)); + cmd = string("{\"prefix\": \"osd set\",\"key\":\"nodeep-scrub\"}"); + ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL)); + + RadosTestECPP::SetUp(); + } + + void TearDown() override { + // re-enable pg autoscaler + string cmd = + "{" + "\"prefix\": \"config set\", " + "\"who\": \"global\", " + "\"name\": \"osd_pool_default_pg_autoscale_mode\", " + "\"value\": \"on\"" + "}"; + std::cout << "Setting pg_autoscaler to 'on'" << std::endl; + bufferlist inbl; + bufferlist outbl; + ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL)); + + // re-enable scrubs + cmd = string("{\"prefix\": \"osd unset\",\"key\":\"noscrub\"}"); + ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL)); + cmd = string("{\"prefix\": \"osd unset\",\"key\":\"nodeep-scrub\"}"); + ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL)); + + RadosTestECPP::TearDown(); + } +}; + +int get_objects_trimmed(json_spirit::Object& pg_dump) { + int objs_trimmed = 0; + + // pg_map + json_spirit::Object pgmap; + for (json_spirit::Object::size_type i = 0; i < pg_dump.size(); ++i) { + json_spirit::Pair& p = pg_dump[i]; + if (p.name_ == "pg_map") { + pgmap = p.value_.get_obj(); + break; + } + } + + // pg_stats array + json_spirit::Array pgs; + for (json_spirit::Object::size_type i = 0; i < pgmap.size(); ++i) { + json_spirit::Pair& p = pgmap[i]; + if (p.name_ == "pg_stats") { + pgs = p.value_.get_array(); + break; + } + } + + // snaptrim stats + for (json_spirit::Object::size_type j = 0; j < pgs.size(); ++j) { + json_spirit::Object& pg_stat = pgs[j].get_obj(); + for(json_spirit::Object::size_type k = 0; k < pg_stat.size(); ++k) { + json_spirit::Pair& stats = pg_stat[k]; + if (stats.name_ == "objects_trimmed") { + objs_trimmed += stats.value_.get_int(); + break; + } + } + } + + return objs_trimmed; +} +const int bufsize = 128; + +TEST_F(LibRadosSnapshotStatsSelfManagedPP, SnaptrimStatsPP) { + int num_objs = 10; + + // create objects + char buf[bufsize]; + memset(buf, 0xcc, sizeof(buf)); + bufferlist bl; + bl.append(buf, sizeof(buf)); + for (int i = 0; i < num_objs; ++i) { + string obj = string("foo") + std::to_string(i); + ASSERT_EQ(0, ioctx.write(obj, bl, sizeof(buf), 0)); + } + + std::vector my_snaps; + char buf2[sizeof(buf)]; + memset(buf2, 0xdd, sizeof(buf2)); + bufferlist bl2; + bl2.append(buf2, sizeof(buf2)); + for (int snap = 0; snap < 1; ++snap) { + // create a snapshot, clone + std::vector ns(1); + ns.insert(ns.end(), my_snaps.begin(), my_snaps.end()); + my_snaps.swap(ns); + ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0])); + ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps)); + for (int i = 0; i < num_objs; ++i) { + string obj = string("foo") + std::to_string(i); + ASSERT_EQ(0, ioctx.write(obj, bl2, sizeof(buf2), 0)); + } + } + + // wait for maps to settle + cluster.wait_for_latest_osdmap(); + + // remove snaps - should trigger snaptrim + for (unsigned snap = 0; snap < my_snaps.size(); ++snap) { + ioctx.selfmanaged_snap_remove(my_snaps[snap]); + } + + // sleep for few secs for the trim stats to populate + std::cout << "Waiting for snaptrim stats to be generated" << std::endl; + sleep(30); + + // Dump pg stats and determine if snaptrim stats are getting set + int objects_trimmed = 0; + int tries = 0; + do { + string cmd = string("{\"prefix\": \"pg dump\",\"format\":\"json\"}"); + bufferlist inbl; + bufferlist outbl; + ASSERT_EQ(0, cluster.mon_command(cmd, inbl, &outbl, NULL)); + string outstr(outbl.c_str(), outbl.length()); + json_spirit::Value v; + ASSERT_NE(0, json_spirit::read(outstr, v)) << "unable to parse json." << '\n' << outstr; + + // pg_map + json_spirit::Object& obj = v.get_obj(); + objects_trimmed = get_objects_trimmed(obj); + if (objects_trimmed < num_objs) { + tries++; + objects_trimmed = 0; + std::cout << "Still waiting for all objects to be trimmed... " < my_snaps; + char *buf2 = (char *)new char[bsize]; + memset(buf2, 0xdd, bsize); + bufferlist bl2; + bl2.append(buf2, bsize); + for (int snap = 0; snap < 1; ++snap) { + // create a snapshot, clone + std::vector ns(1); + ns.insert(ns.end(), my_snaps.begin(), my_snaps.end()); + my_snaps.swap(ns); + ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0])); + ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps)); + for (int i = 0; i < num_objs; ++i) { + string obj = string("foo") + std::to_string(i); + ASSERT_EQ(0, ioctx.write(obj, bl2, bsize, bsize)); + } + } + + // wait for maps to settle + cluster.wait_for_latest_osdmap(); + + // remove snaps - should trigger snaptrim + for (unsigned snap = 0; snap < my_snaps.size(); ++snap) { + ioctx.selfmanaged_snap_remove(my_snaps[snap]); + } + + // sleep for few secs for the trim stats to populate + std::cout << "Waiting for snaptrim stats to be generated" << std::endl; + sleep(30); + + // Dump pg stats and determine if snaptrim stats are getting set + int objects_trimmed = 0; + int tries = 0; + do { + string cmd = string("{\"prefix\": \"pg dump\",\"format\":\"json\"}"); + bufferlist inbl; + bufferlist outbl; + ASSERT_EQ(0, cluster.mon_command(cmd, inbl, &outbl, NULL)); + string outstr(outbl.c_str(), outbl.length()); + json_spirit::Value v; + ASSERT_NE(0, json_spirit::read(outstr, v)) << "unable to parse json." << '\n' << outstr; + + // pg_map + json_spirit::Object& obj = v.get_obj(); + objects_trimmed = get_objects_trimmed(obj); + if (objects_trimmed < num_objs) { + tries++; + objects_trimmed = 0; + std::cout << "Still waiting for all objects to be trimmed... " <