rgw: Add neorados utilities

Signed-off-by: Adam Emerson <aemerson@redhat.com>

diff --git a/src/rgw/driver/rados/rgw_tools.h b/src/rgw/driver/rados/rgw_tools.h
index 43470a0c718160875c4c588f41938ff13eee0920..f4d3033524ed1cbd62e11ed3e37576b6d29c538e 100644 (file)
--- a/src/rgw/driver/rados/rgw_tools.h
+++ b/src/rgw/driver/rados/rgw_tools.h
@@ -3,17 +3,34 @@
 
 #pragma once
 
+#include <coroutine>
 #include <functional>
 #include <string>
 #include <string_view>
 
+#include <boost/asio/deferred.hpp>
+#include <boost/asio/async_result.hpp>
+
+#include <boost/asio/experimental/co_composed.hpp>
+
+#include <boost/system/error_code.hpp>
+#include <boost/system/errc.hpp>
+
+#include <fmt/format.h>
+
 #include "include/types.h"
 #include "include/ceph_hash.h"
 
+#include "include/neorados/RADOS.hpp"
+
 #include "common/ceph_time.h"
+#include "common/dout.h"
+
+#include "osd/osd_types.h"
 
 #include "rgw_common.h"
-#include "rgw_sal_fwd.h"
+#include "rgw_obj_types.h"
+#include "rgw_pool_types.h"
 
 class RGWSI_SysObj;
 
@@ -186,3 +203,310 @@ int rgw_list_pool(const DoutPrefixProvider *dpp,
                  std::string& marker,
                  std::vector<std::string> *oids,
                  bool *is_truncated);
+
+// Asio's co_compse generates spurious warnings when compiled with
+// -O0. the 'mismatched' `operator new` calls directly into the
+// matching `operator new`, returning its result.
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wmismatched-new-delete"
+namespace rgw {
+struct mostly_omap_t {};
+inline constexpr mostly_omap_t mostly_omap;
+
+struct create_t {};
+inline constexpr create_t create;
+
+template<boost::asio::completion_token_for<
+          void(boost::system::error_code)> CompletionToken>
+auto set_mostly_omap(const DoutPrefixProvider* dpp,
+                    neorados::RADOS& rados, std::string name,
+                    CompletionToken&& token)
+{
+  namespace asio = ::boost::asio;
+  namespace sys = ::boost::system;
+  namespace R = ::neorados;
+  return asio::async_initiate<CompletionToken, void(sys::error_code)>
+    (asio::experimental::co_composed<void(sys::error_code)>
+     ([](auto state, const DoutPrefixProvider* dpp,
+        R::RADOS& rados, std::string name) -> void {
+       try {
+        state.throw_if_cancelled(true);
+        try {
+          // set pg_autoscale_bias
+          auto bias = rados.cct()->_conf.get_val<double>(
+            "rgw_rados_pool_autoscale_bias");
+          std::vector<std::string> biascommand{
+            {fmt::format(R"({{"prefix": "osd pool set", "pool": "{}", )"
+                         R"("var": "pg_autoscale_bias", "val": "{}"}})",
+                         name, bias)}
+          };
+          co_await rados.mon_command(std::move(biascommand), {}, nullptr,
+                                     nullptr, asio::deferred);
+        } catch (const std::exception& e) {
+          ldpp_dout(dpp, 2)
+            << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": "
+            << "mon_command to set "
+            <<  "autoscale bias failed with error: "
+            << e.what() << dendl;
+          throw;
+        }
+        try {
+          // set recovery_priority
+          auto p = rados.cct()->_conf.get_val<uint64_t>(
+            "rgw_rados_pool_recovery_priority");
+          std::vector<std::string> recoverycommand{
+            {fmt::format(R"({{"prefix": "osd pool set", "pool": "{}", )"
+                         R"("var": "recovery_priority": "{}"}})",
+                         name, p)}
+          };
+          co_await rados.mon_command(std::move(recoverycommand), {}, nullptr,
+                                     nullptr, asio::deferred);
+        } catch (const std::exception& e) {
+          ldpp_dout(dpp, 2)
+            << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": "
+            << "mon_command to set recovery priority failed with error: "
+            << e.what() << dendl;
+          throw;
+        }
+        co_return sys::error_code{};
+       } catch (const sys::system_error& e) {
+        ldpp_dout(dpp, -1) << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": "
+                           << ": " << e.what() << dendl;
+        co_return e.code();
+       }
+     }, rados.get_executor()),
+     token, dpp, std::ref(rados), std::move(name));
+}
+
+template<boost::asio::completion_token_for<
+          void(boost::system::error_code)> CompletionToken>
+auto create_pool(const DoutPrefixProvider* dpp, neorados::RADOS& rados,
+                std::string name, CompletionToken&& token)
+{
+  namespace asio = ::boost::asio;
+  namespace sys = ::boost::system;
+  namespace R = ::neorados;
+  return asio::async_initiate<CompletionToken, void(sys::error_code)>
+    (asio::experimental::co_composed<void(sys::error_code)>
+     ([](auto state, const DoutPrefixProvider* dpp,
+        R::RADOS& rados, std::string name) -> void {
+       try {
+        state.throw_if_cancelled(true);
+        try {
+          co_await rados.create_pool(std::string(name), std::nullopt,
+                                     asio::deferred);
+        } catch (const sys::system_error& e) {
+          if (e.code() == sys::errc::result_out_of_range) {
+            ldpp_dout(dpp, 0)
+              << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": "
+              << "ERROR: RADOS::create_pool failed with " << e.what()
+              << " (this can be due to a pool or placement group "
+              << "misconfiguration, e.g. pg_num < pgp_num or "
+              << "mon_max_pg_per_osd exceeded)" << dendl;
+            throw;
+          } else if (e.code() == sys::errc::file_exists) {
+            co_return sys::error_code{};
+          } else {
+            ldpp_dout(dpp, 2)
+              << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": "
+              << "RADOS::create_pool failed with error: " << e.what() << dendl;
+            throw;
+          }
+        } catch (const std::exception& e) {
+          ldpp_dout(dpp, 2)
+            << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": "
+            << "RADOS::create_pool failed with error: " << e.what() << dendl;
+          throw;
+        }
+        co_await rados.enable_application(std::move(name),
+                                          pg_pool_t::APPLICATION_NAME_RGW,
+                                          false, asio::deferred);
+        co_return sys::error_code{};
+       } catch (const std::system_error& e) {
+        ldpp_dout(dpp, 2)
+          << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": "
+          << "ERROR: " << e.what() << dendl;
+        co_return e.code();
+       } catch (const std::exception& e) {
+        ldpp_dout(dpp, 2)
+          << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": "
+          << "ERROR: " << e.what() << dendl;
+        co_return sys::error_code{EIO, sys::generic_category()};
+       }
+     }, rados.get_executor()),
+     token, dpp, std::ref(rados), std::move(name));
+}
+
+template<boost::asio::completion_token_for<
+          void(boost::system::error_code)> CompletionToken>
+auto create_pool(const DoutPrefixProvider* dpp, neorados::RADOS& rados,
+                std::string name, mostly_omap_t, CompletionToken&& token)
+{
+  namespace asio = ::boost::asio;
+  namespace sys = ::boost::system;
+  namespace R = ::neorados;
+  return asio::async_initiate<CompletionToken, void(sys::error_code)>
+    (asio::experimental::co_composed<void(sys::error_code)>
+     ([](auto state, const DoutPrefixProvider* dpp,
+        R::RADOS& rados, std::string name) -> void {
+       try {
+        state.throw_if_cancelled(true);
+        co_await create_pool(dpp, rados, name, asio::deferred);
+        co_await set_mostly_omap(dpp, rados, std::move(name), asio::deferred);
+        co_return sys::error_code{};
+       } catch (const std::system_error& e) {
+        ldpp_dout(dpp, 2)
+          << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": "
+          << "ERROR: " << e.what() << dendl;
+        co_return e.code();
+       } catch (const std::exception& e) {
+        ldpp_dout(dpp, 2)
+          << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": "
+          << "ERROR: " << e.what() << dendl;
+        co_return sys::error_code{EIO, sys::generic_category()};
+       }
+     }, rados.get_executor()),
+     token, dpp, std::ref(rados), std::move(name));
+}
+
+template<boost::asio::completion_token_for<
+          void(boost::system::error_code, neorados::IOContext)> CompletionToken>
+auto init_iocontext(const DoutPrefixProvider* dpp,
+                   neorados::RADOS& rados, rgw_pool pool,
+                   CompletionToken&& token)
+{
+  namespace asio = ::boost::asio;
+  namespace sys = ::boost::system;
+  namespace R = ::neorados;
+  return asio::async_initiate<CompletionToken, void(sys::error_code, R::IOContext)>
+    (asio::experimental::co_composed<void(sys::error_code, R::IOContext)>
+     ([](auto state, const DoutPrefixProvider* dpp,
+        R::RADOS& rados, rgw_pool pool) -> void {
+       try {
+        state.throw_if_cancelled(true);
+        R::IOContext ioc;
+        try {
+          ioc.set_pool(co_await rados.lookup_pool(pool.name, asio::deferred));
+        } catch (const std::exception& e) {
+          ldpp_dout(dpp, 2)
+          << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": "
+            << "RADOS::lookup_pool failed with error: " << e.what() << dendl;
+          throw;
+        }
+        ioc.set_ns(pool.ns);
+        co_return {sys::error_code{}, std::move(ioc)};
+       } catch (const std::system_error& e) {
+        ldpp_dout(dpp, 2)
+          << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": "
+          << "ERROR: " << e.what() << dendl;
+        co_return {e.code(), R::IOContext{}};
+       } catch (const std::exception& e) {
+        ldpp_dout(dpp, 2)
+          << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": "
+          << "ERROR: " << e.what() << dendl;
+        co_return {sys::error_code{EIO, sys::generic_category()},
+                   R::IOContext{}};
+       }
+     }, rados.get_executor()),
+     token, dpp, std::ref(rados), std::move(pool));
+}
+
+template<boost::asio::completion_token_for<
+          void(boost::system::error_code, neorados::IOContext)> CompletionToken>
+auto init_iocontext(const DoutPrefixProvider* dpp,
+                   neorados::RADOS& rados, rgw_pool pool,
+                   create_t, CompletionToken&& token)
+{
+  namespace asio = ::boost::asio;
+  namespace sys = ::boost::system;
+  namespace R = ::neorados;
+  return asio::async_initiate<CompletionToken, void(sys::error_code,
+                                                   R::IOContext)>
+    (asio::experimental::co_composed<void(sys::error_code, R::IOContext)>
+     ([](auto state, const DoutPrefixProvider* dpp,
+        R::RADOS& rados, rgw_pool pool) -> void {
+       bool must_create = false;
+       try {
+        state.throw_if_cancelled(true);
+        R::IOContext ioc;
+        try {
+          ioc.set_pool(co_await rados.lookup_pool(pool.name, asio::deferred));
+        } catch (const sys::system_error& e) {
+          if (e.code() == sys::errc::no_such_file_or_directory) {
+            must_create = true;
+          } else {
+            ldpp_dout(dpp, 2)
+              << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": "
+              << "RADOS::lookup_pool failed with error: " << e.what() << dendl;
+            throw;
+          }
+        }
+        if (must_create) {
+          co_await create_pool(dpp, rados, pool.name, asio::deferred);
+          ioc.set_pool(co_await rados.lookup_pool(pool.name, asio::deferred));
+        }
+        ioc.set_ns(pool.ns);
+        co_return {sys::error_code{}, std::move(ioc)};
+       } catch (const std::system_error& e) {
+        ldpp_dout(dpp, 2)
+          << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": "
+          << "ERROR: " << e.what() << dendl;
+        co_return {e.code(), R::IOContext{}};
+       } catch (const std::exception& e) {
+        ldpp_dout(dpp, 2)
+          << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": "
+          << "ERROR: " << e.what() << dendl;
+        co_return {sys::error_code{EIO, sys::generic_category()},
+                   R::IOContext{}};
+       }
+     }, rados.get_executor()),
+     token, dpp, std::ref(rados), std::move(pool));
+}
+
+template<boost::asio::completion_token_for<
+          void(boost::system::error_code, neorados::IOContext)> CompletionToken>
+auto init_iocontext(const DoutPrefixProvider* dpp,
+                   neorados::RADOS& rados, rgw_pool pool,
+                   create_t, mostly_omap_t, CompletionToken&& token)
+{
+  namespace asio = ::boost::asio;
+  namespace sys = ::boost::system;
+  namespace R = ::neorados;
+  return asio::async_initiate<CompletionToken, void(sys::error_code,
+                                                   R::IOContext)>
+    (asio::experimental::co_composed<void(sys::error_code, R::IOContext)>
+     ([](auto state, const DoutPrefixProvider* dpp,
+        R::RADOS& rados, rgw_pool pool) -> void {
+       try {
+        state.throw_if_cancelled(true);
+        auto ioc = co_await init_iocontext(dpp, rados, pool, create,
+                                           asio::deferred);
+        co_await set_mostly_omap(dpp, rados, pool.name, asio::deferred);
+        co_return {sys::error_code{}, std::move(ioc)};
+       } catch (const std::system_error& e) {
+        ldpp_dout(dpp, 2)
+          << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": "
+          << "ERROR: " << e.what() << dendl;
+        co_return {e.code(), R::IOContext{}};
+       } catch (const std::exception& e) {
+        ldpp_dout(dpp, 2)
+          << __PRETTY_FUNCTION__ << ":" << __LINE__ << ": "
+          << "ERROR: " << e.what() << dendl;
+        co_return {sys::error_code{EIO, sys::generic_category()},
+                   R::IOContext{}};
+       }
+     }, rados.get_executor()),
+     token, dpp, std::ref(rados), std::move(pool));
+}
+
+template<boost::asio::completion_token_for<
+          void(boost::system::error_code, neorados::IOContext)> CompletionToken>
+auto init_iocontext(const DoutPrefixProvider* dpp,
+                   neorados::RADOS& rados, rgw_pool pool,
+                   mostly_omap_t, create_t, CompletionToken&& token)
+{
+  return init_iocontext(dpp, rados, pool, create, mostly_omap,
+                       std::forward<CompletionToken>(token));
+}
+} // namespace rgw
+#pragma GCC diagnostic push