From 10bc82fb00e50472689660f8ea705a1f9135508d Mon Sep 17 00:00:00 2001 From: "Kaleb S. KEITHLEY" Date: Mon, 1 Aug 2022 17:30:32 -0400 Subject: [PATCH] rgw: lua, add support for lua packages to the lua manager Refactor lua manager, adding new {add,remove}_package(), and list_packages() methods; and changing get(), put(), del() methods to {get,put,del}_script(). Renaming class from *LuaScriptManager to *LuaManager to be consistent with dual *script and *package functionality. Note: I considered keeping the existing *LuaScriptManager classes as is and adding *LuaPackageManager classes, but that seemed/seems like overkill to me. Signed-off-by: Kaleb S. KEITHLEY --- src/rgw/rgw_lua.cc | 96 +++++++++----------------------------- src/rgw/rgw_lua.h | 16 ++++--- src/rgw/rgw_main.cc | 29 +++++------- src/rgw/rgw_sal.h | 23 +++++---- src/rgw/rgw_sal_dbstore.cc | 34 +++++++++++++- src/rgw/rgw_sal_dbstore.h | 25 ++++++---- src/rgw/rgw_sal_filter.cc | 36 ++++++++++---- src/rgw/rgw_sal_filter.h | 22 +++++---- src/rgw/rgw_sal_fwd.h | 2 +- src/rgw/rgw_sal_motr.cc | 33 ++++++++++++- src/rgw/rgw_sal_motr.h | 25 ++++++---- src/rgw/rgw_sal_rados.cc | 91 +++++++++++++++++++++++++++++++++--- src/rgw/rgw_sal_rados.h | 19 ++++---- src/rgw/rgw_sal_store.h | 4 +- src/test/CMakeLists.txt | 15 ++++-- 15 files changed, 306 insertions(+), 164 deletions(-) diff --git a/src/rgw/rgw_lua.cc b/src/rgw/rgw_lua.cc index 323cc5ed0f62d..e0f61049b6058 100644 --- a/src/rgw/rgw_lua.cc +++ b/src/rgw/rgw_lua.cc @@ -8,7 +8,6 @@ #ifdef WITH_RADOSGW_LUA_PACKAGES #include #include -#include "rgw_lua_version.h" #endif #define dout_subsys ceph_subsys_rgw @@ -70,32 +69,31 @@ std::string script_oid(context ctx, const std::string& tenant) { int read_script(const DoutPrefixProvider *dpp, rgw::sal::Store* store, const std::string& tenant, optional_yield y, context ctx, std::string& script) { - auto lua_script = store->get_lua_script_manager(); + auto lua_mgr = store->get_lua_manager(); - return lua_script->get(dpp, y, script_oid(ctx, tenant), script); + return lua_mgr->get_script(dpp, y, script_oid(ctx, tenant), script); } int write_script(const DoutPrefixProvider *dpp, rgw::sal::Store* store, const std::string& tenant, optional_yield y, context ctx, const std::string& script) { - auto lua_script = store->get_lua_script_manager(); + auto lua_mgr = store->get_lua_manager(); - return lua_script->put(dpp, y, script_oid(ctx, tenant), script); + return lua_mgr->put_script(dpp, y, script_oid(ctx, tenant), script); } int delete_script(const DoutPrefixProvider *dpp, rgw::sal::Store* store, const std::string& tenant, optional_yield y, context ctx) { - auto lua_script = store->get_lua_script_manager(); + auto lua_mgr = store->get_lua_manager(); - return lua_script->del(dpp, y, script_oid(ctx, tenant)); + return lua_mgr->del_script(dpp, y, script_oid(ctx, tenant)); } #ifdef WITH_RADOSGW_LUA_PACKAGES -const std::string PACKAGE_LIST_OBJECT_NAME = "lua_package_allowlist"; - namespace bp = boost::process; -int add_package(const DoutPrefixProvider *dpp, rgw::sal::RadosStore* store, optional_yield y, const std::string& package_name, bool allow_compilation) { +int add_package(const DoutPrefixProvider *dpp, rgw::sal::Store* store, optional_yield y, const std::string& package_name, bool allow_compilation) +{ // verify that luarocks can load this package const auto p = bp::search_path("luarocks"); if (p.empty()) { @@ -122,7 +120,7 @@ int add_package(const DoutPrefixProvider *dpp, rgw::sal::RadosStore* store, opti if (!package_found) { return -EINVAL; } - + //replace previous versions of the package const std::string package_name_no_version = package_name.substr(0, package_name.find(" ")); ret = remove_package(dpp, store, y, package_name_no_version); @@ -130,76 +128,28 @@ int add_package(const DoutPrefixProvider *dpp, rgw::sal::RadosStore* store, opti return ret; } - // add package to list - const bufferlist empty_bl; - std::map new_package{{package_name, empty_bl}}; - librados::ObjectWriteOperation op; - op.omap_set(new_package); - ret = rgw_rados_operate(dpp, *(store->getRados()->get_lc_pool_ctx()), - PACKAGE_LIST_OBJECT_NAME, &op, y); + auto lua_mgr = store->get_lua_manager(); - if (ret < 0) { - return ret; - } - return 0; + return lua_mgr->add_package(dpp, y, package_name); } -int remove_package(const DoutPrefixProvider *dpp, rgw::sal::RadosStore* store, optional_yield y, const std::string& package_name) { - librados::ObjectWriteOperation op; - size_t pos = package_name.find(" "); - if (pos != package_name.npos) { - // remove specfic version of the the package - op.omap_rm_keys(std::set({package_name})); - auto ret = rgw_rados_operate(dpp, *(store->getRados()->get_lc_pool_ctx()), - PACKAGE_LIST_OBJECT_NAME, &op, y); - if (ret < 0) { - return ret; - } - return 0; - } - // otherwise, remove any existing versions of the package - packages_t packages; - auto ret = list_packages(dpp, store, y, packages); - if (ret < 0 && ret != -ENOENT) { - return ret; - } - for(const auto& package : packages) { - const std::string package_no_version = package.substr(0, package.find(" ")); - if (package_no_version.compare(package_name) == 0) { - op.omap_rm_keys(std::set({package})); - ret = rgw_rados_operate(dpp, *(store->getRados()->get_lc_pool_ctx()), - PACKAGE_LIST_OBJECT_NAME, &op, y); - if (ret < 0) { - return ret; - } - } - } - return 0; +int remove_package(const DoutPrefixProvider *dpp, rgw::sal::Store* store, optional_yield y, const std::string& package_name) +{ + auto lua_mgr = store->get_lua_manager(); + + return lua_mgr->remove_package(dpp, y, package_name); } -int list_packages(const DoutPrefixProvider *dpp, rgw::sal::RadosStore* store, optional_yield y, packages_t& packages) { - constexpr auto max_chunk = 1024U; - std::string start_after; - bool more = true; - int rval; - while (more) { - librados::ObjectReadOperation op; - packages_t packages_chunk; - op.omap_get_keys2(start_after, max_chunk, &packages_chunk, &more, &rval); - const auto ret = rgw_rados_operate(dpp, *(store->getRados()->get_lc_pool_ctx()), - PACKAGE_LIST_OBJECT_NAME, &op, nullptr, y); - - if (ret < 0) { - return ret; - } +namespace bp = boost::process; - packages.merge(packages_chunk); - } - - return 0; +int list_packages(const DoutPrefixProvider *dpp, rgw::sal::Store* store, optional_yield y, packages_t& packages) +{ + auto lua_mgr = store->get_lua_manager(); + + return lua_mgr->list_packages(dpp, y, packages); } -int install_packages(const DoutPrefixProvider *dpp, rgw::sal::RadosStore* store, optional_yield y, packages_t& failed_packages, std::string& output) { +int install_packages(const DoutPrefixProvider *dpp, rgw::sal::Store* store, optional_yield y, packages_t& failed_packages, std::string& output) { // luarocks directory cleanup std::error_code ec; const auto& luarocks_path = store->get_luarocks_path(); diff --git a/src/rgw/rgw_lua.h b/src/rgw/rgw_lua.h index 1eb14c605ffd4..0596ee104115c 100644 --- a/src/rgw/rgw_lua.h +++ b/src/rgw/rgw_lua.h @@ -1,7 +1,10 @@ #pragma once #include +#include +#include "rgw_lua_version.h" #include "common/async/yield_context.h" +#include "common/dout.h" #include "rgw_sal_fwd.h" class lua_State; @@ -36,23 +39,22 @@ int read_script(const DoutPrefixProvider *dpp, rgw::sal::Store* store, const std // delete the stored lua script from a context int delete_script(const DoutPrefixProvider *dpp, rgw::sal::Store* store, const std::string& tenant, optional_yield y, context ctx); -#ifdef WITH_RADOSGW_LUA_PACKAGES -#include - using packages_t = std::set; +#ifdef WITH_RADOSGW_LUA_PACKAGES + // add a lua package to the allowlist -int add_package(const DoutPrefixProvider *dpp, rgw::sal::RadosStore* store, optional_yield y, const std::string& package_name, bool allow_compilation); +int add_package(const DoutPrefixProvider *dpp, rgw::sal::Store* store, optional_yield y, const std::string& package_name, bool allow_compilation); // remove a lua package from the allowlist -int remove_package(const DoutPrefixProvider *dpp, rgw::sal::RadosStore* store, optional_yield y, const std::string& package_name); +int remove_package(const DoutPrefixProvider *dpp, rgw::sal::Store* store, optional_yield y, const std::string& package_name); // list lua packages in the allowlist -int list_packages(const DoutPrefixProvider *dpp, rgw::sal::RadosStore* store, optional_yield y, packages_t& packages); +int list_packages(const DoutPrefixProvider *dpp, rgw::sal::Store* store, optional_yield y, packages_t& packages); // install all packages from the allowlist // return the list of packages that failed to install and the output of the install command -int install_packages(const DoutPrefixProvider *dpp, rgw::sal::RadosStore* store, optional_yield y, packages_t& failed_packages, std::string& output); +int install_packages(const DoutPrefixProvider *dpp, rgw::sal::Store* store, optional_yield y, packages_t& failed_packages, std::string& output); #endif } diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc index 6e7509a29df21..6c5ed0e2dea60 100644 --- a/src/rgw/rgw_main.cc +++ b/src/rgw/rgw_main.cc @@ -52,9 +52,7 @@ #endif #include "rgw_asio_frontend.h" #include "rgw_dmclock_scheduler_ctx.h" -#ifdef WITH_RADOSGW_LUA_PACKAGES #include "rgw_lua.h" -#endif #ifdef WITH_RADOSGW_DBSTORE #include "rgw_sal_dbstore.h" #endif @@ -491,20 +489,17 @@ int radosgw_Main(int argc, const char **argv) store->set_luarocks_path(luarocks_path+"/"+g_conf()->name.to_str()); } #ifdef WITH_RADOSGW_LUA_PACKAGES - rgw::sal::RadosStore *rados = dynamic_cast(store); - if (rados) { /* Supported for only RadosStore */ - rgw::lua::packages_t failed_packages; - std::string output; - r = rgw::lua::install_packages(&dp, rados, null_yield, failed_packages, output); - if (r < 0) { - dout(1) << "ERROR: failed to install lua packages from allowlist" << dendl; - } - if (!output.empty()) { - dout(10) << "INFO: lua packages installation output: \n" << output << dendl; - } - for (const auto& p : failed_packages) { - dout(5) << "WARNING: failed to install lua package: " << p << " from allowlist" << dendl; - } + rgw::lua::packages_t failed_packages; + std::string output; + r = rgw::lua::install_packages(&dp, store, null_yield, failed_packages, output); + if (r < 0) { + dout(1) << "WARNING: failed to install lua packages from allowlist" << dendl; + } + if (!output.empty()) { + dout(10) << "INFO: lua packages installation output: \n" << output << dendl; + } + for (const auto& p : failed_packages) { + dout(5) << "WARNING: failed to install lua package: " << p << " from allowlist" << dendl; } #endif @@ -634,7 +629,7 @@ int radosgw_Main(int argc, const char **argv) int fe_count = 0; std::unique_ptr lua_background; - if (rados) { /* Supported for only RadosStore */ + if (store->get_name() == "rados") { /* Supported for only RadosStore */ lua_background = std::make_unique(store, cct.get(), store->get_luarocks_path()); lua_background->start(); } diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index 6b234aa168480..90ccdc165eb08 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -16,6 +16,7 @@ #pragma once #include "rgw_sal_fwd.h" +#include "rgw_lua.h" #include "rgw_user.h" #include "rgw_notify_event_type.h" #include "common/tracer.h" @@ -396,7 +397,7 @@ class Store { /** Get the ID of the current host */ virtual std::string get_host_id() = 0; /** Get a Lua script manager for running lua scripts */ - virtual std::unique_ptr get_lua_script_manager() = 0; + virtual std::unique_ptr get_lua_manager() = 0; /** Get an IAM Role by name etc. */ virtual std::unique_ptr get_role(std::string name, std::string tenant, @@ -1507,20 +1508,26 @@ class Zone { }; /** - * @brief Abstraction of a manager for Lua scripts + * @brief Abstraction of a manager for Lua scripts and packages * - * RGW can load and process Lua scripts. This will handle loading/storing scripts. + * RGW can load and process Lua scripts. This will handle loading/storing scripts; adding, deleting, and listing packages */ -class LuaScriptManager { +class LuaManager { public: - virtual ~LuaScriptManager() = default; + virtual ~LuaManager() = default; /** Get a script named with the given key from the backing store */ - virtual int get(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) = 0; + virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) = 0; /** Put a script named with the given key to the backing store */ - virtual int put(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) = 0; + virtual int put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) = 0; /** Delete a script named with the given key from the backing store */ - virtual int del(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) = 0; + virtual int del_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) = 0; + /** Add a lua package */ + virtual int add_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name) = 0; + /** Remove a lua package */ + virtual int remove_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name) = 0; + /** List lua packages */ + virtual int list_packages(const DoutPrefixProvider* dpp, optional_yield y, rgw::lua::packages_t& packages) = 0; }; /** @} namespace rgw::sal in group RGWSAL */ diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc index 68090d335e839..1ee859215242e 100644 --- a/src/rgw/rgw_sal_dbstore.cc +++ b/src/rgw/rgw_sal_dbstore.cc @@ -602,9 +602,9 @@ namespace rgw::sal { return realm->get_id(); } - std::unique_ptr DBStore::get_lua_script_manager() + std::unique_ptr DBStore::get_lua_manager() { - return std::make_unique(this); + return std::make_unique(this); } int DBObject::get_obj_state(const DoutPrefixProvider* dpp, RGWObjState **pstate, optional_yield y, bool follow_olh) @@ -1981,6 +1981,36 @@ namespace rgw::sal { return ret; } + + int DBLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) + { + return -ENOENT; + } + + int DBLuaManager::put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) + { + return -ENOENT; + } + + int DBLuaManager::del_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) + { + return -ENOENT; + } + + int DBLuaManager::add_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name) + { + return -ENOENT; + } + + int DBLuaManager::remove_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name) + { + return -ENOENT; + } + + int DBLuaManager::list_packages(const DoutPrefixProvider* dpp, optional_yield y, rgw::lua::packages_t& packages) + { + return -ENOENT; + } } // namespace rgw::sal extern "C" { diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h index 038d188dc9fbc..259c8e6caa602 100644 --- a/src/rgw/rgw_sal_dbstore.h +++ b/src/rgw/rgw_sal_dbstore.h @@ -347,18 +347,27 @@ protected: virtual const std::string_view get_tier_type() override { return "rgw"; } }; - class DBLuaScriptManager : public StoreLuaScriptManager { + class DBLuaManager : public StoreLuaManager { DBStore* store; public: - DBLuaScriptManager(DBStore* _s) : store(_s) + DBLuaManager(DBStore* _s) : store(_s) { } - virtual ~DBLuaScriptManager() = default; - - virtual int get(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override { return -ENOENT; } - virtual int put(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) override { return -ENOENT; } - virtual int del(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) override { return -ENOENT; } + virtual ~DBLuaManager() = default; + + /** Get a script named with the given key from the backing store */ + virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override; + /** Put a script named with the given key to the backing store */ + virtual int put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) override; + /** Delete a script named with the given key from the backing store */ + virtual int del_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) override; + /** Add a lua package */ + virtual int add_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name) override; + /** Remove a lua package */ + virtual int remove_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name) override; + /** List lua packages */ + virtual int list_packages(const DoutPrefixProvider* dpp, optional_yield y, rgw::lua::packages_t& packages) override; }; class DBOIDCProvider : public RGWOIDCProvider { @@ -837,7 +846,7 @@ public: virtual const RGWSyncModuleInstanceRef& get_sync_module() { return sync_module; } virtual std::string get_host_id() { return ""; } - virtual std::unique_ptr get_lua_script_manager() override; + virtual std::unique_ptr get_lua_manager() override; virtual std::unique_ptr get_role(std::string name, std::string tenant, std::string path="", diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc index 64cadd9b68dc9..de2aa5f86b6c2 100644 --- a/src/rgw/rgw_sal_filter.cc +++ b/src/rgw/rgw_sal_filter.cc @@ -414,11 +414,11 @@ const RGWSyncModuleInstanceRef& FilterStore::get_sync_module() return next->get_sync_module(); } -std::unique_ptr FilterStore::get_lua_script_manager() +std::unique_ptr FilterStore::get_lua_manager() { - std::unique_ptr nm = next->get_lua_script_manager(); + std::unique_ptr nm = next->get_lua_manager(); - return std::make_unique(std::move(nm)); + return std::make_unique(std::move(nm)); } std::unique_ptr FilterStore::get_role(std::string name, @@ -1303,22 +1303,40 @@ int FilterWriter::complete(size_t accounted_size, const std::string& etag, canceled, y); } -int FilterLuaScriptManager::get(const DoutPrefixProvider* dpp, optional_yield y, +int FilterLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) { - return next->get(dpp, y, key, script); + return next->get_script(dpp, y, key, script); } -int FilterLuaScriptManager::put(const DoutPrefixProvider* dpp, optional_yield y, +int FilterLuaManager::put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) { - return next->put(dpp, y, key, script); + return next->put_script(dpp, y, key, script); } -int FilterLuaScriptManager::del(const DoutPrefixProvider* dpp, optional_yield y, +int FilterLuaManager::del_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) { - return next->del(dpp, y, key); + return next->del_script(dpp, y, key); +} + +int FilterLuaManager::add_package(const DoutPrefixProvider* dpp, optional_yield y, + const std::string& package_name) +{ + return next->add_package(dpp, y, package_name); +} + +int FilterLuaManager::remove_package(const DoutPrefixProvider* dpp, optional_yield y, + const std::string& package_name) +{ + return next->remove_package(dpp, y, package_name); +} + +int FilterLuaManager::list_packages(const DoutPrefixProvider* dpp, optional_yield y, + rgw::lua::packages_t& packages) +{ + return next->list_packages(dpp, y, packages); } } } // namespace rgw::sal diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h index 69d1ce9b98256..88e92a356a8ec 100644 --- a/src/rgw/rgw_sal_filter.h +++ b/src/rgw/rgw_sal_filter.h @@ -251,7 +251,7 @@ public: std::string& metadata_key, optional_yield y) override; virtual const RGWSyncModuleInstanceRef& get_sync_module() override; virtual std::string get_host_id() override { return next->get_host_id(); } - virtual std::unique_ptr get_lua_script_manager() override; + virtual std::unique_ptr get_lua_manager() override; virtual std::unique_ptr get_role(std::string name, std::string tenant, std::string path="", @@ -858,18 +858,20 @@ public: optional_yield y) override; }; -class FilterLuaScriptManager : public LuaScriptManager { +class FilterLuaManager : public LuaManager { protected: - std::unique_ptr next; + std::unique_ptr next; public: - FilterLuaScriptManager(std::unique_ptr _next) : next(std::move(_next)) {} - virtual ~FilterLuaScriptManager() = default; - - virtual int get(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override; - virtual int put(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) override; - virtual int del(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) override; + FilterLuaManager(std::unique_ptr _next) : next(std::move(_next)) {} + virtual ~FilterLuaManager() = default; + + virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override; + virtual int put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) override; + virtual int del_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) override; + virtual int add_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name) override; + virtual int remove_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name) override; + virtual int list_packages(const DoutPrefixProvider* dpp, optional_yield y, rgw::lua::packages_t& packages) override; }; - } } // namespace rgw::sal diff --git a/src/rgw/rgw_sal_fwd.h b/src/rgw/rgw_sal_fwd.h index bb78ed6adbe10..a4fb859a30e95 100644 --- a/src/rgw/rgw_sal_fwd.h +++ b/src/rgw/rgw_sal_fwd.h @@ -30,7 +30,7 @@ namespace rgw { namespace sal { class PlacementTier; class ZoneGroup; class Zone; - class LuaScriptManager; + class LuaManager; struct RGWRoleInfo; } } // namespace rgw::sal diff --git a/src/rgw/rgw_sal_motr.cc b/src/rgw/rgw_sal_motr.cc index 1de0955ae5b13..ba4feef16ac31 100644 --- a/src/rgw/rgw_sal_motr.cc +++ b/src/rgw/rgw_sal_motr.cc @@ -939,9 +939,9 @@ const std::string& MotrZone::get_current_period_id() return current_period->get_id(); } -std::unique_ptr MotrStore::get_lua_script_manager() +std::unique_ptr MotrStore::get_lua_manager() { - return std::make_unique(this); + return std::make_unique(this); } int MotrObject::get_obj_state(const DoutPrefixProvider* dpp, RGWObjState **_state, optional_yield y, bool follow_olh) @@ -3481,6 +3481,35 @@ int MotrStore::init_metadata_cache(const DoutPrefixProvider *dpp, return 0; } + int MotrLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) + { + return -ENOENT; + } + + int MotrLuaManager::put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) + { + return -ENOENT; + } + + int MotrLuaManager::del_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) + { + return -ENOENT; + } + + int MotrLuaManager::add_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name) + { + return -ENOENT; + } + + int MotrLuaManager::remove_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name) + { + return -ENOENT; + } + + int MotrLuaManager::list_packages(const DoutPrefixProvider* dpp, optional_yield y, rgw::lua::packages_t& packages) + { + return -ENOENT; + } } // namespace rgw::sal extern "C" { diff --git a/src/rgw/rgw_sal_motr.h b/src/rgw/rgw_sal_motr.h index fae89d8cd48d1..9f35d68f85c63 100644 --- a/src/rgw/rgw_sal_motr.h +++ b/src/rgw/rgw_sal_motr.h @@ -467,18 +467,27 @@ class MotrZone : public StoreZone { friend class MotrStore; }; -class MotrLuaScriptManager : public StoreLuaScriptManager { +class MotrLuaManager : public StoreLuaManager { MotrStore* store; public: - MotrLuaScriptManager(MotrStore* _s) : store(_s) + MotrLuaManager(MotrStore* _s) : store(_s) { } - virtual ~MotrLuaScriptManager() = default; - - virtual int get(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override { return -ENOENT; } - virtual int put(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) override { return -ENOENT; } - virtual int del(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) override { return -ENOENT; } + virtual ~MotrLuaManager() = default; + + /** Get a script named with the given key from the backing store */ + virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override; + /** Put a script named with the given key to the backing store */ + virtual int put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) override; + /** Delete a script named with the given key from the backing store */ + virtual int del_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) override; + /** Add a lua package */ + virtual int add_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name) override; + /** Remove a lua package */ + virtual int remove_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name) override; + /** List lua packages */ + virtual int list_packages(const DoutPrefixProvider* dpp, optional_yield y, rgw::lua::packages_t& packages) override; }; class MotrOIDCProvider : public RGWOIDCProvider { @@ -969,7 +978,7 @@ class MotrStore : public StoreStore { virtual const RGWSyncModuleInstanceRef& get_sync_module() { return sync_module; } virtual std::string get_host_id() { return ""; } - virtual std::unique_ptr get_lua_script_manager() override; + virtual std::unique_ptr get_lua_manager() override; virtual std::unique_ptr get_role(std::string name, std::string tenant, std::string path="", diff --git a/src/rgw/rgw_sal_rados.cc b/src/rgw/rgw_sal_rados.cc index d3b53ecc7293a..0955366f6d017 100644 --- a/src/rgw/rgw_sal_rados.cc +++ b/src/rgw/rgw_sal_rados.cc @@ -16,9 +16,11 @@ #include #include #include +#include #include #include #include +#include #include "common/Clock.h" #include "common/errno.h" @@ -1364,9 +1366,9 @@ void RadosStore::finalize(void) rados->finalize(); } -std::unique_ptr RadosStore::get_lua_script_manager() +std::unique_ptr RadosStore::get_lua_manager() { - return std::make_unique(this); + return std::make_unique(this); } std::unique_ptr RadosStore::get_role(std::string name, @@ -3007,12 +3009,12 @@ const std::string_view RadosZone::get_tier_type() return store->svc()->zone->get_zone().tier_type; } -RadosLuaScriptManager::RadosLuaScriptManager(RadosStore* _s) : store(_s) +RadosLuaManager::RadosLuaManager(RadosStore* _s) : store(_s) { pool = store->svc()->zone->get_zone_params().log_pool; } -int RadosLuaScriptManager::get(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) +int RadosLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) { bufferlist bl; @@ -3031,7 +3033,7 @@ int RadosLuaScriptManager::get(const DoutPrefixProvider* dpp, optional_yield y, return 0; } -int RadosLuaScriptManager::put(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) +int RadosLuaManager::put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) { bufferlist bl; ceph::encode(script, bl); @@ -3044,7 +3046,7 @@ int RadosLuaScriptManager::put(const DoutPrefixProvider* dpp, optional_yield y, return 0; } -int RadosLuaScriptManager::del(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) +int RadosLuaManager::del_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) { int r = rgw_delete_system_obj(dpp, store->svc()->sysobj, pool, key, nullptr, y); if (r < 0 && r != -ENOENT) { @@ -3054,6 +3056,83 @@ int RadosLuaScriptManager::del(const DoutPrefixProvider* dpp, optional_yield y, return 0; } +#ifdef WITH_RADOSGW_LUA_PACKAGES +const std::string PACKAGE_LIST_OBJECT_NAME = "lua_package_allowlist"; + +int RadosLuaManager::add_package(const DoutPrefixProvider *dpp, optional_yield y, const std::string& package_name) +{ + // add package to list + const bufferlist empty_bl; + std::map new_package{{package_name, empty_bl}}; + librados::ObjectWriteOperation op; + op.omap_set(new_package); + auto ret = rgw_rados_operate(dpp, *(store->getRados()->get_lc_pool_ctx()), + PACKAGE_LIST_OBJECT_NAME, &op, y); + + if (ret < 0) { + return ret; + } + return 0; +} + +int RadosLuaManager::remove_package(const DoutPrefixProvider *dpp, optional_yield y, const std::string& package_name) +{ + librados::ObjectWriteOperation op; + size_t pos = package_name.find(" "); + if (pos != package_name.npos) { + // remove specfic version of the the package + op.omap_rm_keys(std::set({package_name})); + auto ret = rgw_rados_operate(dpp, *(store->getRados()->get_lc_pool_ctx()), + PACKAGE_LIST_OBJECT_NAME, &op, y); + if (ret < 0) { + return ret; + } + return 0; + } + // otherwise, remove any existing versions of the package + rgw::lua::packages_t packages; + auto ret = list_packages(dpp, y, packages); + if (ret < 0 && ret != -ENOENT) { + return ret; + } + for(const auto& package : packages) { + const std::string package_no_version = package.substr(0, package.find(" ")); + if (package_no_version.compare(package_name) == 0) { + op.omap_rm_keys(std::set({package})); + ret = rgw_rados_operate(dpp, *(store->getRados()->get_lc_pool_ctx()), + PACKAGE_LIST_OBJECT_NAME, &op, y); + if (ret < 0) { + return ret; + } + } + } + return 0; +} + +int RadosLuaManager::list_packages(const DoutPrefixProvider *dpp, optional_yield y, rgw::lua::packages_t& packages) +{ + constexpr auto max_chunk = 1024U; + std::string start_after; + bool more = true; + int rval; + while (more) { + librados::ObjectReadOperation op; + rgw::lua::packages_t packages_chunk; + op.omap_get_keys2(start_after, max_chunk, &packages_chunk, &more, &rval); + const auto ret = rgw_rados_operate(dpp, *(store->getRados()->get_lc_pool_ctx()), + PACKAGE_LIST_OBJECT_NAME, &op, nullptr, y); + + if (ret < 0) { + return ret; + } + + packages.merge(packages_chunk); + } + + return 0; +} +#endif + int RadosOIDCProvider::store_url(const DoutPrefixProvider *dpp, const std::string& url, bool exclusive, optional_yield y) { auto sysobj = store->svc()->sysobj; diff --git a/src/rgw/rgw_sal_rados.h b/src/rgw/rgw_sal_rados.h index 8f72c70429c7c..128aead8802a7 100644 --- a/src/rgw/rgw_sal_rados.h +++ b/src/rgw/rgw_sal_rados.h @@ -195,7 +195,7 @@ class RadosStore : public StoreStore { virtual int meta_remove(const DoutPrefixProvider* dpp, std::string& metadata_key, optional_yield y) override; virtual const RGWSyncModuleInstanceRef& get_sync_module() { return rados->get_sync_module(); } virtual std::string get_host_id() { return rados->host_id; } - virtual std::unique_ptr get_lua_script_manager() override; + virtual std::unique_ptr get_lua_manager() override; virtual std::unique_ptr get_role(std::string name, std::string tenant, std::string path="", @@ -878,17 +878,20 @@ public: optional_yield y) override; }; -class RadosLuaScriptManager : public StoreLuaScriptManager { +class RadosLuaManager : public StoreLuaManager { RadosStore* store; rgw_pool pool; public: - RadosLuaScriptManager(RadosStore* _s); - virtual ~RadosLuaScriptManager() = default; - - virtual int get(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override; - virtual int put(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) override; - virtual int del(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) override; + RadosLuaManager(RadosStore* _s); + virtual ~RadosLuaManager() = default; + + virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script); + virtual int put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script); + virtual int del_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key); + virtual int add_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name); + virtual int remove_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name); + virtual int list_packages(const DoutPrefixProvider* dpp, optional_yield y, rgw::lua::packages_t& packages); }; class RadosOIDCProvider : public RGWOIDCProvider { diff --git a/src/rgw/rgw_sal_store.h b/src/rgw/rgw_sal_store.h index 0390922c35793..214c83fe9c7e3 100644 --- a/src/rgw/rgw_sal_store.h +++ b/src/rgw/rgw_sal_store.h @@ -404,9 +404,9 @@ class StoreZone : public Zone { virtual ~StoreZone() = default; }; -class StoreLuaScriptManager : public LuaScriptManager { +class StoreLuaManager : public LuaManager { public: - virtual ~StoreLuaScriptManager() = default; + virtual ~StoreLuaManager() = default; }; } } // namespace rgw::sal diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index ec02efc7e3b78..f07b22fe8fbd0 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -316,13 +316,16 @@ install(TARGETS ceph_test_librgw_file_gp DESTINATION ${CMAKE_INSTALL_BINDIR}) add_executable(ceph_test_librgw_file_nfsns librgw_file_nfsns.cc ) -target_include_directories(ceph_test_librgw_file_nfsns SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw") +target_include_directories(ceph_test_librgw_file_nfsns + PUBLIC "${LUA_INCLUDE_DIR}" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw") target_link_libraries(ceph_test_librgw_file_nfsns rgw librados ceph-common ${UNITTEST_LIBS} ${EXTRALIBS} + ${LUA_LIBRARIES} ) target_link_libraries(ceph_test_librgw_file_nfsns spawn) install(TARGETS ceph_test_librgw_file_nfsns DESTINATION ${CMAKE_INSTALL_BINDIR}) @@ -344,13 +347,16 @@ install(TARGETS ceph_test_librgw_file_aw DESTINATION ${CMAKE_INSTALL_BINDIR}) add_executable(ceph_test_librgw_file_marker librgw_file_marker.cc ) -target_include_directories(ceph_test_librgw_file_marker SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw") +target_include_directories(ceph_test_librgw_file_marker + PUBLIC "${LUA_INCLUDE_DIR}" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw") target_link_libraries(ceph_test_librgw_file_marker rgw librados ceph-common ${UNITTEST_LIBS} ${EXTRALIBS} + ${LUA_LIBRARIES} ) target_link_libraries(ceph_test_librgw_file_marker spawn) install(TARGETS ceph_test_librgw_file_marker DESTINATION ${CMAKE_INSTALL_BINDIR}) @@ -359,13 +365,16 @@ install(TARGETS ceph_test_librgw_file_marker DESTINATION ${CMAKE_INSTALL_BINDIR} add_executable(ceph_test_librgw_file_xattr librgw_file_xattr.cc ) -target_include_directories(ceph_test_librgw_file_xattr SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw") +target_include_directories(ceph_test_librgw_file_xattr + PUBLIC "${LUA_INCLUDE_DIR}" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw") target_link_libraries(ceph_test_librgw_file_xattr rgw librados ceph-common ${UNITTEST_LIBS} ${EXTRALIBS} + ${LUA_LIBRARIES} ) target_link_libraries(ceph_test_librgw_file_xattr spawn) -- 2.39.5