From 2b0ec399c2cc0f94b41e4f4967b4d226bb1e3967 Mon Sep 17 00:00:00 2001 From: Adam Emerson Date: Wed, 15 Nov 2023 19:47:22 -0500 Subject: [PATCH] rgw: Add neorados utilities Signed-off-by: Adam Emerson --- src/rgw/driver/rados/rgw_tools.h | 326 ++++++++++++++++++++++++++++++- 1 file changed, 325 insertions(+), 1 deletion(-) diff --git a/src/rgw/driver/rados/rgw_tools.h b/src/rgw/driver/rados/rgw_tools.h index 43470a0c718..f4d3033524e 100644 --- a/src/rgw/driver/rados/rgw_tools.h +++ b/src/rgw/driver/rados/rgw_tools.h @@ -3,17 +3,34 @@ #pragma once +#include #include #include #include +#include +#include + +#include + +#include +#include + +#include + #include "include/types.h" #include "include/ceph_hash.h" +#include "include/neorados/RADOS.hpp" + #include "common/ceph_time.h" +#include "common/dout.h" + +#include "osd/osd_types.h" #include "rgw_common.h" -#include "rgw_sal_fwd.h" +#include "rgw_obj_types.h" +#include "rgw_pool_types.h" class RGWSI_SysObj; @@ -186,3 +203,310 @@ int rgw_list_pool(const DoutPrefixProvider *dpp, std::string& marker, std::vector *oids, bool *is_truncated); + +// Asio's co_compse generates spurious warnings when compiled with +// -O0. the 'mismatched' `operator new` calls directly into the +// matching `operator new`, returning its result. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmismatched-new-delete" +namespace rgw { +struct mostly_omap_t {}; +inline constexpr mostly_omap_t mostly_omap; + +struct create_t {}; +inline constexpr create_t create; + +template CompletionToken> +auto set_mostly_omap(const DoutPrefixProvider* dpp, + neorados::RADOS& rados, std::string name, + CompletionToken&& token) +{ + namespace asio = ::boost::asio; + namespace sys = ::boost::system; + namespace R = ::neorados; + return asio::async_initiate + (asio::experimental::co_composed + ([](auto state, const DoutPrefixProvider* dpp, + R::RADOS& rados, std::string name) -> void { + try { + state.throw_if_cancelled(true); + try { + // set pg_autoscale_bias + auto bias = rados.cct()->_conf.get_val( + "rgw_rados_pool_autoscale_bias"); + std::vector biascommand{ + {fmt::format(R"({{"prefix": "osd pool set", "pool": "{}", )" + R"("var": "pg_autoscale_bias", "val": "{}"}})", + name, bias)} + }; + co_await rados.mon_command(std::move(biascommand), {}, nullptr, + nullptr, asio::deferred); + } catch (const std::exception& e) { + ldpp_dout(dpp, 2) + << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": " + << "mon_command to set " + << "autoscale bias failed with error: " + << e.what() << dendl; + throw; + } + try { + // set recovery_priority + auto p = rados.cct()->_conf.get_val( + "rgw_rados_pool_recovery_priority"); + std::vector recoverycommand{ + {fmt::format(R"({{"prefix": "osd pool set", "pool": "{}", )" + R"("var": "recovery_priority": "{}"}})", + name, p)} + }; + co_await rados.mon_command(std::move(recoverycommand), {}, nullptr, + nullptr, asio::deferred); + } catch (const std::exception& e) { + ldpp_dout(dpp, 2) + << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": " + << "mon_command to set recovery priority failed with error: " + << e.what() << dendl; + throw; + } + co_return sys::error_code{}; + } catch (const sys::system_error& e) { + ldpp_dout(dpp, -1) << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": " + << ": " << e.what() << dendl; + co_return e.code(); + } + }, rados.get_executor()), + token, dpp, std::ref(rados), std::move(name)); +} + +template CompletionToken> +auto create_pool(const DoutPrefixProvider* dpp, neorados::RADOS& rados, + std::string name, CompletionToken&& token) +{ + namespace asio = ::boost::asio; + namespace sys = ::boost::system; + namespace R = ::neorados; + return asio::async_initiate + (asio::experimental::co_composed + ([](auto state, const DoutPrefixProvider* dpp, + R::RADOS& rados, std::string name) -> void { + try { + state.throw_if_cancelled(true); + try { + co_await rados.create_pool(std::string(name), std::nullopt, + asio::deferred); + } catch (const sys::system_error& e) { + if (e.code() == sys::errc::result_out_of_range) { + ldpp_dout(dpp, 0) + << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": " + << "ERROR: RADOS::create_pool failed with " << e.what() + << " (this can be due to a pool or placement group " + << "misconfiguration, e.g. pg_num < pgp_num or " + << "mon_max_pg_per_osd exceeded)" << dendl; + throw; + } else if (e.code() == sys::errc::file_exists) { + co_return sys::error_code{}; + } else { + ldpp_dout(dpp, 2) + << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": " + << "RADOS::create_pool failed with error: " << e.what() << dendl; + throw; + } + } catch (const std::exception& e) { + ldpp_dout(dpp, 2) + << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": " + << "RADOS::create_pool failed with error: " << e.what() << dendl; + throw; + } + co_await rados.enable_application(std::move(name), + pg_pool_t::APPLICATION_NAME_RGW, + false, asio::deferred); + co_return sys::error_code{}; + } catch (const std::system_error& e) { + ldpp_dout(dpp, 2) + << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": " + << "ERROR: " << e.what() << dendl; + co_return e.code(); + } catch (const std::exception& e) { + ldpp_dout(dpp, 2) + << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": " + << "ERROR: " << e.what() << dendl; + co_return sys::error_code{EIO, sys::generic_category()}; + } + }, rados.get_executor()), + token, dpp, std::ref(rados), std::move(name)); +} + +template CompletionToken> +auto create_pool(const DoutPrefixProvider* dpp, neorados::RADOS& rados, + std::string name, mostly_omap_t, CompletionToken&& token) +{ + namespace asio = ::boost::asio; + namespace sys = ::boost::system; + namespace R = ::neorados; + return asio::async_initiate + (asio::experimental::co_composed + ([](auto state, const DoutPrefixProvider* dpp, + R::RADOS& rados, std::string name) -> void { + try { + state.throw_if_cancelled(true); + co_await create_pool(dpp, rados, name, asio::deferred); + co_await set_mostly_omap(dpp, rados, std::move(name), asio::deferred); + co_return sys::error_code{}; + } catch (const std::system_error& e) { + ldpp_dout(dpp, 2) + << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": " + << "ERROR: " << e.what() << dendl; + co_return e.code(); + } catch (const std::exception& e) { + ldpp_dout(dpp, 2) + << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": " + << "ERROR: " << e.what() << dendl; + co_return sys::error_code{EIO, sys::generic_category()}; + } + }, rados.get_executor()), + token, dpp, std::ref(rados), std::move(name)); +} + +template CompletionToken> +auto init_iocontext(const DoutPrefixProvider* dpp, + neorados::RADOS& rados, rgw_pool pool, + CompletionToken&& token) +{ + namespace asio = ::boost::asio; + namespace sys = ::boost::system; + namespace R = ::neorados; + return asio::async_initiate + (asio::experimental::co_composed + ([](auto state, const DoutPrefixProvider* dpp, + R::RADOS& rados, rgw_pool pool) -> void { + try { + state.throw_if_cancelled(true); + R::IOContext ioc; + try { + ioc.set_pool(co_await rados.lookup_pool(pool.name, asio::deferred)); + } catch (const std::exception& e) { + ldpp_dout(dpp, 2) + << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": " + << "RADOS::lookup_pool failed with error: " << e.what() << dendl; + throw; + } + ioc.set_ns(pool.ns); + co_return {sys::error_code{}, std::move(ioc)}; + } catch (const std::system_error& e) { + ldpp_dout(dpp, 2) + << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": " + << "ERROR: " << e.what() << dendl; + co_return {e.code(), R::IOContext{}}; + } catch (const std::exception& e) { + ldpp_dout(dpp, 2) + << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": " + << "ERROR: " << e.what() << dendl; + co_return {sys::error_code{EIO, sys::generic_category()}, + R::IOContext{}}; + } + }, rados.get_executor()), + token, dpp, std::ref(rados), std::move(pool)); +} + +template CompletionToken> +auto init_iocontext(const DoutPrefixProvider* dpp, + neorados::RADOS& rados, rgw_pool pool, + create_t, CompletionToken&& token) +{ + namespace asio = ::boost::asio; + namespace sys = ::boost::system; + namespace R = ::neorados; + return asio::async_initiate + (asio::experimental::co_composed + ([](auto state, const DoutPrefixProvider* dpp, + R::RADOS& rados, rgw_pool pool) -> void { + bool must_create = false; + try { + state.throw_if_cancelled(true); + R::IOContext ioc; + try { + ioc.set_pool(co_await rados.lookup_pool(pool.name, asio::deferred)); + } catch (const sys::system_error& e) { + if (e.code() == sys::errc::no_such_file_or_directory) { + must_create = true; + } else { + ldpp_dout(dpp, 2) + << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": " + << "RADOS::lookup_pool failed with error: " << e.what() << dendl; + throw; + } + } + if (must_create) { + co_await create_pool(dpp, rados, pool.name, asio::deferred); + ioc.set_pool(co_await rados.lookup_pool(pool.name, asio::deferred)); + } + ioc.set_ns(pool.ns); + co_return {sys::error_code{}, std::move(ioc)}; + } catch (const std::system_error& e) { + ldpp_dout(dpp, 2) + << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": " + << "ERROR: " << e.what() << dendl; + co_return {e.code(), R::IOContext{}}; + } catch (const std::exception& e) { + ldpp_dout(dpp, 2) + << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": " + << "ERROR: " << e.what() << dendl; + co_return {sys::error_code{EIO, sys::generic_category()}, + R::IOContext{}}; + } + }, rados.get_executor()), + token, dpp, std::ref(rados), std::move(pool)); +} + +template CompletionToken> +auto init_iocontext(const DoutPrefixProvider* dpp, + neorados::RADOS& rados, rgw_pool pool, + create_t, mostly_omap_t, CompletionToken&& token) +{ + namespace asio = ::boost::asio; + namespace sys = ::boost::system; + namespace R = ::neorados; + return asio::async_initiate + (asio::experimental::co_composed + ([](auto state, const DoutPrefixProvider* dpp, + R::RADOS& rados, rgw_pool pool) -> void { + try { + state.throw_if_cancelled(true); + auto ioc = co_await init_iocontext(dpp, rados, pool, create, + asio::deferred); + co_await set_mostly_omap(dpp, rados, pool.name, asio::deferred); + co_return {sys::error_code{}, std::move(ioc)}; + } catch (const std::system_error& e) { + ldpp_dout(dpp, 2) + << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": " + << "ERROR: " << e.what() << dendl; + co_return {e.code(), R::IOContext{}}; + } catch (const std::exception& e) { + ldpp_dout(dpp, 2) + << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": " + << "ERROR: " << e.what() << dendl; + co_return {sys::error_code{EIO, sys::generic_category()}, + R::IOContext{}}; + } + }, rados.get_executor()), + token, dpp, std::ref(rados), std::move(pool)); +} + +template CompletionToken> +auto init_iocontext(const DoutPrefixProvider* dpp, + neorados::RADOS& rados, rgw_pool pool, + mostly_omap_t, create_t, CompletionToken&& token) +{ + return init_iocontext(dpp, rados, pool, create, mostly_omap, + std::forward(token)); +} +} // namespace rgw +#pragma GCC diagnostic push -- 2.39.5