From ffab13bcd9006c1f961a24b8016df9d1fe06ba1d Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Fri, 24 Sep 2021 23:33:03 +0800 Subject: [PATCH] os/bluestore: list obj which equals to pend otherwise we could have failures like scrub : stat mismatch, got 3/4 objects, 1/2 clones, 3/4 dirty, 3/4 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 49/56 bytes, 0/0 manifest objects, 0/0 hit_set_archive bytes." where the numbers of scrubbed object, clones, dirty and omap are always less than the total number of corresponding numbers, if the PG contains object(s) whose hash happens to be 0xffffffff. in this change, if the calculated hash of the upper bound is greater than the maximum possible number represented by uint32_t, in addition to setting the hash of the upper bound hobj to 0xffffffff, we also set the nspace of hobj of the upper bound to "\xff", so that the upper bound is greater than an hobj whose hash happens to be 0xfffffff. please note, the nspace of "\xff" is not an ascii string, so it's not likely to be less than a real-world nspace of an hobj. with this new *greater* upper bound, we are able to include the previous missing hobj when listing the objects in a PG. so the scrub won't be annoyed when the number of objects does not match. Fixes: https://tracker.ceph.com/issues/52705 Signed-off-by: Mykola Golub Signed-off-by: Kefu Chai --- src/os/bluestore/BlueStore.cc | 26 +++++++++----- src/test/objectstore/store_test.cc | 55 ++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 62e18398d3ea5..fa76dc5ac1edd 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -288,9 +288,13 @@ static const char *_key_decode_shard(const char *key, shard_id_t *pshard) static void get_coll_range(const coll_t& cid, int bits, ghobject_t *temp_start, ghobject_t *temp_end, - ghobject_t *start, ghobject_t *end) + ghobject_t *start, ghobject_t *end, bool legacy) { spg_t pgid; + constexpr uint32_t MAX_HASH = std::numeric_limits::max(); + // use different nspaces due to we use different schemes when encoding + // keys for listing objects + const std::string_view MAX_NSPACE = legacy ? "\x7f" : "\xff"; if (cid.is_pg(&pgid)) { start->shard_id = pgid.shard; *temp_start = *start; @@ -306,19 +310,23 @@ static void get_coll_range(const coll_t& cid, int bits, temp_start->hobj.set_bitwise_key_u32(reverse_hash); uint64_t end_hash = reverse_hash + (1ull << (32 - bits)); - if (end_hash > 0xffffffffull) - end_hash = 0xffffffffull; - - end->hobj.set_bitwise_key_u32(end_hash); - temp_end->hobj.set_bitwise_key_u32(end_hash); + if (end_hash > MAX_HASH) { + // make sure end hobj is even greater than the maximum possible hobj + end->hobj.set_bitwise_key_u32(MAX_HASH); + temp_end->hobj.set_bitwise_key_u32(MAX_HASH); + end->hobj.nspace = MAX_NSPACE; + } else { + end->hobj.set_bitwise_key_u32(end_hash); + temp_end->hobj.set_bitwise_key_u32(end_hash); + } } else { start->shard_id = shard_id_t::NO_SHARD; start->hobj.pool = -1ull; *end = *start; start->hobj.set_bitwise_key_u32(0); - end->hobj.set_bitwise_key_u32(0xffffffff); - + end->hobj.set_bitwise_key_u32(MAX_HASH); + end->hobj.nspace = MAX_NSPACE; // no separate temp section *temp_start = *end; *temp_end = *end; @@ -10826,7 +10834,7 @@ int BlueStore::_collection_list( return 0; } get_coll_range(c->cid, c->cnode.bits, &coll_range_temp_start, - &coll_range_temp_end, &coll_range_start, &coll_range_end); + &coll_range_temp_end, &coll_range_start, &coll_range_end, legacy); dout(20) << __func__ << " range " << coll_range_temp_start << " to " << coll_range_temp_end diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc index a4057ea353dd8..d942eebd784d5 100644 --- a/src/test/objectstore/store_test.cc +++ b/src/test/objectstore/store_test.cc @@ -2933,6 +2933,61 @@ TEST_P(StoreTest, ListEndTest) { } } +TEST_P(StoreTest, List_0xfffffff_Hash_Test_in_meta) { + int r = 0; + coll_t cid; + auto ch = store->create_new_collection(cid); + { + ObjectStore::Transaction t; + t.create_collection(cid, 0); + r = queue_transaction(store, ch, std::move(t)); + ASSERT_EQ(r, 0); + } + { + ObjectStore::Transaction t; + ghobject_t hoid(hobject_t(sobject_t("obj", CEPH_NOSNAP), + "", UINT32_C(0xffffffff), -1, "nspace")); + t.touch(cid, hoid); + r = queue_transaction(store, ch, std::move(t)); + ASSERT_EQ(r, 0); + } + { + vector objects; + r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), INT_MAX, + &objects, nullptr, true); + ASSERT_EQ(r, 0); + ASSERT_EQ(objects.size(), 1); + } +} + +TEST_P(StoreTest, List_0xfffffff_Hash_Test_in_PG) { + int r = 0; + const int64_t poolid = 1; + coll_t cid(spg_t(pg_t(0, poolid), shard_id_t::NO_SHARD)); + auto ch = store->create_new_collection(cid); + { + ObjectStore::Transaction t; + t.create_collection(cid, 0); + r = queue_transaction(store, ch, std::move(t)); + ASSERT_EQ(r, 0); + } + { + ObjectStore::Transaction t; + ghobject_t hoid(hobject_t(sobject_t("obj", CEPH_NOSNAP), + "", UINT32_C(0xffffffff), poolid, "nspace")); + t.touch(cid, hoid); + r = queue_transaction(store, ch, std::move(t)); + ASSERT_EQ(r, 0); + } + { + vector objects; + r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), INT_MAX, + &objects, nullptr, true); + ASSERT_EQ(r, 0); + ASSERT_EQ(objects.size(), 1); + } +} + TEST_P(StoreTest, Sort) { { hobject_t a(sobject_t("a", CEPH_NOSNAP)); -- 2.39.5