From: Adam C. Emerson Date: Sat, 21 Nov 2020 06:44:36 +0000 (-0500) Subject: rgw: Factor out tool to deal with different log backing X-Git-Tag: v16.2.2~8^2~9^2~23 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=aede44ac6667c9a1ec7e813b547f8765754d896f;p=ceph.git rgw: Factor out tool to deal with different log backing Read through the shards of a log and find out what kind it is. Also remove a log. Signed-off-by: Adam C. Emerson (cherry picked from commit ed15d03f068c6f6e959f04d9d8f99eac82ebbd29) Signed-off-by: Adam C. Emerson --- diff --git a/src/cls/log/cls_log_types.h b/src/cls/log/cls_log_types.h index c5c00766d815..1746d243e5a1 100644 --- a/src/cls/log/cls_log_types.h +++ b/src/cls/log/cls_log_types.h @@ -65,6 +65,9 @@ inline bool operator ==(const cls_log_header& lhs, const cls_log_header& rhs) { return (lhs.max_marker == rhs.max_marker && lhs.max_time == rhs.max_time); } +inline bool operator !=(const cls_log_header& lhs, const cls_log_header& rhs) { + return !(lhs == rhs); +} WRITE_CLASS_ENCODER(cls_log_header) diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index 44de25895ea2..d3d91d495794 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -141,6 +141,7 @@ set(librgw_common_srcs rgw_tag.cc rgw_tag_s3.cc rgw_tools.cc + rgw_log_backing.cc rgw_user.cc rgw_website.cc rgw_xml.cc diff --git a/src/rgw/rgw_log_backing.cc b/src/rgw/rgw_log_backing.cc new file mode 100644 index 000000000000..63edf972a030 --- /dev/null +++ b/src/rgw/rgw_log_backing.cc @@ -0,0 +1,215 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "cls/log/cls_log_client.h" + +#include "rgw_log_backing.h" +#include "rgw_tools.h" +#include "cls_fifo_legacy.h" + +static constexpr auto dout_subsys = ceph_subsys_rgw; + +enum class shard_check { dne, omap, fifo, corrupt }; +inline std::ostream& operator <<(std::ostream& m, const shard_check& t) { + switch (t) { + case shard_check::dne: + return m << "shard_check::dne"; + case shard_check::omap: + return m << "shard_check::omap"; + case shard_check::fifo: + return m << "shard_check::fifo"; + case shard_check::corrupt: + return m << "shard_check::corrupt"; + } + + return m << "shard_check::UNKNOWN=" << static_cast(t); +} + +namespace { +/// Return the shard type, and a bool to see whether it has entries. +std::pair +probe_shard(librados::IoCtx& ioctx, const std::string& oid, optional_yield y) +{ + auto cct = static_cast(ioctx.cct()); + bool omap = false; + { + librados::ObjectReadOperation op; + cls_log_header header; + cls_log_info(op, &header); + auto r = rgw_rados_operate(ioctx, oid, &op, nullptr, y); + if (r == -ENOENT) { + return { shard_check::dne, {} }; + } + + if (r < 0) { + lderr(cct) << __PRETTY_FUNCTION__ << ":" << __LINE__ + << " error probing for omap: r=" << r + << ", oid=" << oid << dendl; + return { shard_check::corrupt, {} }; + } + if (header != cls_log_header{}) + omap = true; + } + std::unique_ptr fifo; + auto r = rgw::cls::fifo::FIFO::open(ioctx, oid, + &fifo, y, + std::nullopt, true); + if (r < 0 && !(r == -ENOENT || r == -ENODATA)) { + lderr(cct) << __PRETTY_FUNCTION__ << ":" << __LINE__ + << " error probing for fifo: r=" << r + << ", oid=" << oid << dendl; + return { shard_check::corrupt, {} }; + } + if (fifo && omap) { + lderr(cct) << __PRETTY_FUNCTION__ << ":" << __LINE__ + << " fifo and omap found: oid=" << oid << dendl; + return { shard_check::corrupt, {} }; + } + if (fifo) { + bool more = false; + std::vector entries; + r = fifo->list(1, nullopt, &entries, &more, y); + if (r < 0) { + lderr(cct) << __PRETTY_FUNCTION__ << ":" << __LINE__ + << ": unable to list entries: r=" << r + << ", oid=" << oid << dendl; + return { shard_check::corrupt, {} }; + } + return { shard_check::fifo, !entries.empty() }; + } + if (omap) { + std::list entries; + std::string out_marker; + bool truncated = false; + librados::ObjectReadOperation op; + cls_log_list(op, {}, {}, {}, 1, entries, + &out_marker, &truncated); + auto r = rgw_rados_operate(ioctx, oid, &op, nullptr, y); + if (r < 0) { + lderr(cct) << __PRETTY_FUNCTION__ << ":" << __LINE__ + << ": failed to list: r=" << r << ", oid=" << oid << dendl; + return { shard_check::corrupt, {} }; + } + return { shard_check::omap, !entries.empty() }; + } + + // An object exists, but has never had FIFO or cls_log entries written + // to it. Likely just the marker Omap. + return { shard_check::dne, {} }; +} + +tl::expected +handle_dne(librados::IoCtx& ioctx, + log_type def, + std::string oid, + optional_yield y) +{ + auto cct = static_cast(ioctx.cct()); + if (def == log_type::fifo) { + std::unique_ptr fifo; + auto r = rgw::cls::fifo::FIFO::create(ioctx, oid, + &fifo, y, + std::nullopt); + if (r < 0) { + lderr(cct) << __PRETTY_FUNCTION__ << ":" << __LINE__ + << " error creating FIFO: r=" << r + << ", oid=" << oid << dendl; + return tl::unexpected(bs::error_code(-r, bs::system_category())); + } + } + return def; +} +} + +tl::expected +log_backing_type(librados::IoCtx& ioctx, + log_type def, + int shards, + const fu2::unique_function& get_oid, + optional_yield y) +{ + auto cct = static_cast(ioctx.cct()); + auto check = shard_check::dne; + for (int i = 0; i < shards; ++i) { + auto [c, e] = probe_shard(ioctx, get_oid(i), y); + if (c == shard_check::corrupt) + return tl::unexpected(bs::error_code(EIO, bs::system_category())); + if (c == shard_check::dne) continue; + if (check == shard_check::dne) { + check = c; + continue; + } + + if (check != c) { + lderr(cct) << __PRETTY_FUNCTION__ << ":" << __LINE__ + << " clashing types: check=" << check + << ", c=" << c << dendl; + return tl::unexpected(bs::error_code(EIO, bs::system_category())); + } + } + if (check == shard_check::corrupt) { + lderr(cct) << __PRETTY_FUNCTION__ << ":" << __LINE__ + << " should be unreachable!" << dendl; + return tl::unexpected(bs::error_code(EIO, bs::system_category())); + } + + if (check == shard_check::dne) + return handle_dne(ioctx, + def, + get_oid(0), + y); + + return (check == shard_check::fifo ? log_type::fifo : log_type::omap); +} + +bs::error_code log_remove(librados::IoCtx& ioctx, + int shards, + const fu2::unique_function& get_oid, + optional_yield y) +{ + bs::error_code ec; + auto cct = static_cast(ioctx.cct()); + for (int i = 0; i < shards; ++i) { + auto oid = get_oid(i); + rados::cls::fifo::info info; + uint32_t part_header_size = 0, part_entry_overhead = 0; + + auto r = rgw::cls::fifo::get_meta(ioctx, oid, nullopt, &info, + &part_header_size, &part_entry_overhead, + 0, y, true); + if (r == -ENOENT) continue; + if (r == 0 && info.head_part_num > -1) { + for (auto j = info.tail_part_num; j <= info.head_part_num; ++j) { + librados::ObjectWriteOperation op; + op.remove(); + auto part_oid = info.part_oid(j); + auto subr = rgw_rados_operate(ioctx, part_oid, &op, null_yield); + if (subr < 0 && subr != -ENOENT) { + if (!ec) + ec = bs::error_code(-subr, bs::system_category()); + lderr(cct) << __PRETTY_FUNCTION__ << ":" << __LINE__ + << ": failed removing FIFO part: part_oid=" << part_oid + << ", subr=" << subr << dendl; + } + } + } + if (r < 0 && r != -ENODATA) { + if (!ec) + ec = bs::error_code(-r, bs::system_category()); + lderr(cct) << __PRETTY_FUNCTION__ << ":" << __LINE__ + << ": failed checking FIFO part: oid=" << oid + << ", r=" << r << dendl; + } + librados::ObjectWriteOperation op; + op.remove(); + r = rgw_rados_operate(ioctx, oid, &op, null_yield); + if (r < 0 && r != -ENOENT) { + if (!ec) + ec = bs::error_code(-r, bs::system_category()); + lderr(cct) << __PRETTY_FUNCTION__ << ":" << __LINE__ + << ": failed removing shard: oid=" << oid + << ", r=" << r << dendl; + } + } + return ec; +} diff --git a/src/rgw/rgw_log_backing.h b/src/rgw/rgw_log_backing.h new file mode 100644 index 000000000000..d769af48b01f --- /dev/null +++ b/src/rgw/rgw_log_backing.h @@ -0,0 +1,70 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#ifndef CEPH_RGW_LOGBACKING_H +#define CEPH_RGW_LOGBACKING_H + +#include +#include +#include +#include + +#include + +#include + +#include "include/rados/librados.hpp" +#include "include/expected.hpp" +#include "include/function2.hpp" + +#include "common/async/yield_context.h" + +namespace bs = boost::system; + +/// Type of log backing, stored in the mark used in the quick check, +/// and passed to checking functions. +enum class log_type { + omap = 0, + fifo = 1 +}; + +inline std::optional to_log_type(std::string_view s) { + if (strncasecmp(s.data(), "omap", s.length()) == 0) { + return log_type::omap; + } else if (strncasecmp(s.data(), "fifo", s.length()) == 0) { + return log_type::fifo; + } else { + return std::nullopt; + } +} +inline std::ostream& operator <<(std::ostream& m, const log_type& t) { + switch (t) { + case log_type::omap: + return m << "log_type::omap"; + case log_type::fifo: + return m << "log_type::fifo"; + } + + return m << "log_type::UNKNOWN=" << static_cast(t); +} + +/// Look over the shards in a log and determine the type. +tl::expected +log_backing_type(librados::IoCtx& ioctx, + log_type def, + int shards, //< Total number of shards + /// A function taking a shard number and + /// returning an oid. + const fu2::unique_function& get_oid, + optional_yield y); + +/// Remove all log shards and associated parts of fifos. +bs::error_code log_remove(librados::IoCtx& ioctx, + int shards, //< Total number of shards + /// A function taking a shard number and + /// returning an oid. + const fu2::unique_function& get_oid, + optional_yield y); + + +#endif diff --git a/src/test/rgw/CMakeLists.txt b/src/test/rgw/CMakeLists.txt index 7817a42ef9ab..c4aa22db8174 100644 --- a/src/test/rgw/CMakeLists.txt +++ b/src/test/rgw/CMakeLists.txt @@ -213,6 +213,11 @@ add_executable(unittest_cls_fifo_legacy test_cls_fifo_legacy.cc) target_link_libraries(unittest_cls_fifo_legacy radostest-cxx ${UNITTEST_LIBS} ${rgw_libs}) +# unittest_log_backing +add_executable(unittest_log_backing test_log_backing.cc) +target_link_libraries(unittest_log_backing radostest-cxx ${UNITTEST_LIBS} + ${rgw_libs}) + add_executable(unittest_rgw_lua test_rgw_lua.cc) add_ceph_unittest(unittest_rgw_lua) target_link_libraries(unittest_rgw_lua ${rgw_libs} ${LUA_LIBRARIES}) diff --git a/src/test/rgw/test_log_backing.cc b/src/test/rgw/test_log_backing.cc new file mode 100644 index 000000000000..5180d5fc74fe --- /dev/null +++ b/src/test/rgw/test_log_backing.cc @@ -0,0 +1,176 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "rgw_log_backing.h" + +#include +#include +#include + +#undef FMT_HEADER_ONLY +#define FMT_HEADER_ONLY 1 +#include + +#include "include/types.h" +#include "include/rados/librados.hpp" + +#include "test/librados/test_cxx.h" +#include "global/global_context.h" + +#include "cls/log/cls_log_client.h" + +#include "rgw/rgw_tools.h" +#include "rgw/cls_fifo_legacy.h" + +#include "gtest/gtest.h" + +namespace lr = librados; +namespace cb = ceph::buffer; +namespace fifo = rados::cls::fifo; +namespace RCf = rgw::cls::fifo; + +class LogBacking : public testing::Test { +protected: + static constexpr int SHARDS = 3; + const std::string pool_name = get_temp_pool_name(); + lr::Rados rados; + lr::IoCtx ioctx; + + void SetUp() override { + ASSERT_EQ("", create_one_pool_pp(pool_name, rados)); + ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx)); + } + void TearDown() override { + destroy_one_pool_pp(pool_name, rados); + } + + static std::string get_oid(int i) { + return fmt::format("shard.{}", i); + } + + void make_omap() { + for (int i = 0; i < SHARDS; ++i) { + using ceph::encode; + lr::ObjectWriteOperation op; + cb::list bl; + encode(i, bl); + cls_log_add(op, ceph_clock_now(), {}, "meow", bl); + auto r = rgw_rados_operate(ioctx, get_oid(i), &op, null_yield); + ASSERT_GE(r, 0); + } + } + + void add_omap(int i) { + using ceph::encode; + lr::ObjectWriteOperation op; + cb::list bl; + encode(i, bl); + cls_log_add(op, ceph_clock_now(), {}, "meow", bl); + auto r = rgw_rados_operate(ioctx, get_oid(i), &op, null_yield); + ASSERT_GE(r, 0); + } + + void empty_omap() { + for (int i = 0; i < SHARDS; ++i) { + auto oid = get_oid(i); + std::string to_marker; + { + lr::ObjectReadOperation op; + std::list entries; + bool truncated = false; + cls_log_list(op, {}, {}, {}, 1, entries, &to_marker, &truncated); + auto r = rgw_rados_operate(ioctx, oid, &op, nullptr, null_yield); + ASSERT_GE(r, 0); + ASSERT_FALSE(entries.empty()); + } + { + lr::ObjectWriteOperation op; + cls_log_trim(op, {}, {}, {}, to_marker); + auto r = rgw_rados_operate(ioctx, oid, &op, null_yield); + ASSERT_GE(r, 0); + } + { + lr::ObjectReadOperation op; + std::list entries; + bool truncated = false; + cls_log_list(op, {}, {}, {}, 1, entries, &to_marker, &truncated); + auto r = rgw_rados_operate(ioctx, oid, &op, nullptr, null_yield); + ASSERT_GE(r, 0); + ASSERT_TRUE(entries.empty()); + } + } + } + + void make_fifo() + { + for (int i = 0; i < SHARDS; ++i) { + std::unique_ptr fifo; + auto r = RCf::FIFO::create(ioctx, get_oid(i), &fifo, null_yield); + ASSERT_EQ(0, r); + ASSERT_TRUE(fifo); + } + } + + void add_fifo(int i) + { + using ceph::encode; + std::unique_ptr fifo; + auto r = RCf::FIFO::open(ioctx, get_oid(i), &fifo, null_yield); + ASSERT_GE(0, r); + ASSERT_TRUE(fifo); + cb::list bl; + encode(i, bl); + r = fifo->push(bl, null_yield); + ASSERT_GE(0, r); + } + + void assert_empty() { + std::vector result; + lr::ObjectCursor next; + auto r = ioctx.object_list(ioctx.object_list_begin(), ioctx.object_list_end(), + 100, {}, &result, &next); + ASSERT_GE(r, 0); + ASSERT_TRUE(result.empty()); + } +}; + +TEST_F(LogBacking, TestOmap) +{ + make_omap(); + auto stat = log_backing_type(ioctx, log_type::fifo, SHARDS, + get_oid, null_yield); + ASSERT_EQ(log_type::omap, *stat); +} + +TEST_F(LogBacking, TestOmapEmpty) +{ + auto stat = log_backing_type(ioctx, log_type::omap, SHARDS, + get_oid, null_yield); + ASSERT_EQ(log_type::omap, *stat); +} + +TEST_F(LogBacking, TestFIFO) +{ + make_fifo(); + auto stat = log_backing_type(ioctx, log_type::fifo, SHARDS, + get_oid, null_yield); + ASSERT_EQ(log_type::fifo, *stat); +} + +TEST_F(LogBacking, TestFIFOEmpty) +{ + auto stat = log_backing_type(ioctx, log_type::fifo, SHARDS, + get_oid, null_yield); + ASSERT_EQ(log_type::fifo, *stat); +}