From fab880e7ef21b146b5a58037567169c13f5f1b78 Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Fri, 24 Sep 2021 23:33:03 +0800 Subject: [PATCH] os/bluestore: list obj which equals to pend otherwise we could have failures like scrub : stat mismatch, got 3/4 objects, 1/2 clones, 3/4 dirty, 3/4 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 49/56 bytes, 0/0 manifest objects, 0/0 hit_set_archive bytes." where the numbers of scrubbed object, clones, dirty and omap are always less than the total number of corresponding numbers, if the PG contains object(s) whose hash happens to be 0xffffffff. in this change, if the calculated hash of the upper bound is greater than the maximum possible number represented by uint32_t, in addition to setting the hash of the upper bound hobj to 0xffffffff, we also set the nspace of hobj of the upper bound to "\xff", so that the upper bound is greater than an hobj whose hash happens to be 0xfffffff. please note, the nspace of "\xff" is not an ascii string, so it's not likely to be less than a real-world nspace of an hobj. with this new *greater* upper bound, we are able to include the previous missing hobj when listing the objects in a PG. so the scrub won't be annoyed when the number of objects does not match. Fixes: https://tracker.ceph.com/issues/52705 Signed-off-by: Mykola Golub Signed-off-by: Kefu Chai (cherry picked from commit ffab13bcd9006c1f961a24b8016df9d1fe06ba1d) --- src/os/bluestore/BlueStore.cc | 26 +++++++++----- src/test/objectstore/store_test.cc | 55 ++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index cf689fd45107e..232e44d949d92 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -275,9 +275,13 @@ static const char *_key_decode_shard(const char *key, shard_id_t *pshard) static void get_coll_range(const coll_t& cid, int bits, ghobject_t *temp_start, ghobject_t *temp_end, - ghobject_t *start, ghobject_t *end) + ghobject_t *start, ghobject_t *end, bool legacy) { spg_t pgid; + constexpr uint32_t MAX_HASH = std::numeric_limits::max(); + // use different nspaces due to we use different schemes when encoding + // keys for listing objects + const std::string_view MAX_NSPACE = legacy ? "\x7f" : "\xff"; if (cid.is_pg(&pgid)) { start->shard_id = pgid.shard; *temp_start = *start; @@ -293,19 +297,23 @@ static void get_coll_range(const coll_t& cid, int bits, temp_start->hobj.set_bitwise_key_u32(reverse_hash); uint64_t end_hash = reverse_hash + (1ull << (32 - bits)); - if (end_hash > 0xffffffffull) - end_hash = 0xffffffffull; - - end->hobj.set_bitwise_key_u32(end_hash); - temp_end->hobj.set_bitwise_key_u32(end_hash); + if (end_hash > MAX_HASH) { + // make sure end hobj is even greater than the maximum possible hobj + end->hobj.set_bitwise_key_u32(MAX_HASH); + temp_end->hobj.set_bitwise_key_u32(MAX_HASH); + end->hobj.nspace = MAX_NSPACE; + } else { + end->hobj.set_bitwise_key_u32(end_hash); + temp_end->hobj.set_bitwise_key_u32(end_hash); + } } else { start->shard_id = shard_id_t::NO_SHARD; start->hobj.pool = -1ull; *end = *start; start->hobj.set_bitwise_key_u32(0); - end->hobj.set_bitwise_key_u32(0xffffffff); - + end->hobj.set_bitwise_key_u32(MAX_HASH); + end->hobj.nspace = MAX_NSPACE; // no separate temp section *temp_start = *end; *temp_end = *end; @@ -10577,7 +10585,7 @@ int BlueStore::_collection_list( return 0; } get_coll_range(c->cid, c->cnode.bits, &coll_range_temp_start, - &coll_range_temp_end, &coll_range_start, &coll_range_end); + &coll_range_temp_end, &coll_range_start, &coll_range_end, legacy); dout(20) << __func__ << " range " << coll_range_temp_start << " to " << coll_range_temp_end diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc index f4e44e55f7f63..0ef0d75e09b09 100644 --- a/src/test/objectstore/store_test.cc +++ b/src/test/objectstore/store_test.cc @@ -2932,6 +2932,61 @@ TEST_P(StoreTest, ListEndTest) { } } +TEST_P(StoreTest, List_0xfffffff_Hash_Test_in_meta) { + int r = 0; + coll_t cid; + auto ch = store->create_new_collection(cid); + { + ObjectStore::Transaction t; + t.create_collection(cid, 0); + r = queue_transaction(store, ch, std::move(t)); + ASSERT_EQ(r, 0); + } + { + ObjectStore::Transaction t; + ghobject_t hoid(hobject_t(sobject_t("obj", CEPH_NOSNAP), + "", UINT32_C(0xffffffff), -1, "nspace")); + t.touch(cid, hoid); + r = queue_transaction(store, ch, std::move(t)); + ASSERT_EQ(r, 0); + } + { + vector objects; + r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), INT_MAX, + &objects, nullptr, true); + ASSERT_EQ(r, 0); + ASSERT_EQ(objects.size(), 1); + } +} + +TEST_P(StoreTest, List_0xfffffff_Hash_Test_in_PG) { + int r = 0; + const int64_t poolid = 1; + coll_t cid(spg_t(pg_t(0, poolid), shard_id_t::NO_SHARD)); + auto ch = store->create_new_collection(cid); + { + ObjectStore::Transaction t; + t.create_collection(cid, 0); + r = queue_transaction(store, ch, std::move(t)); + ASSERT_EQ(r, 0); + } + { + ObjectStore::Transaction t; + ghobject_t hoid(hobject_t(sobject_t("obj", CEPH_NOSNAP), + "", UINT32_C(0xffffffff), poolid, "nspace")); + t.touch(cid, hoid); + r = queue_transaction(store, ch, std::move(t)); + ASSERT_EQ(r, 0); + } + { + vector objects; + r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), INT_MAX, + &objects, nullptr, true); + ASSERT_EQ(r, 0); + ASSERT_EQ(objects.size(), 1); + } +} + TEST_P(StoreTest, Sort) { { hobject_t a(sobject_t("a", CEPH_NOSNAP)); -- 2.39.5