From cf06c005b5c1da7cbd592a8594ed816995147d4d Mon Sep 17 00:00:00 2001 From: kabicin <37311900+kabicin@users.noreply.github.com> Date: Fri, 6 Mar 2026 19:15:16 -0500 Subject: [PATCH] rgw: support more than one lua script per tenant/context - adds the radosgw-admin script list command - adds an optional --script-name argument to support loading and reading more than one lua script - uses a read-write-modify pattern to update the lua scripts with the RGWObjVersionTracker - uses bufferlist to read/write the script list metadata - updates lua python tests for multi script support - updates script list usage docs - checks the default script in the background context for backward compatibility Signed-off-by: Kirby Chin --- doc/man/8/radosgw-admin.rst | 4 + doc/radosgw/lua-scripting.rst | 17 +- src/rgw/driver/daos/rgw_sal_daos.h | 8 +- src/rgw/driver/motr/rgw_sal_motr.cc | 6 +- src/rgw/driver/motr/rgw_sal_motr.h | 4 +- src/rgw/driver/posix/rgw_sal_posix.cc | 7 +- src/rgw/driver/posix/rgw_sal_posix.h | 3 +- src/rgw/driver/rados/rgw_sal_rados.cc | 106 ++++++++++- src/rgw/driver/rados/rgw_sal_rados.h | 5 +- src/rgw/radosgw-admin/radosgw-admin.cc | 62 ++++++- src/rgw/rgw_lua.cc | 60 ++++++- src/rgw/rgw_lua.h | 18 +- src/rgw/rgw_lua_background.cc | 42 ++++- src/rgw/rgw_lua_background.h | 3 +- src/rgw/rgw_op.cc | 4 +- src/rgw/rgw_process.cc | 131 +++++++++----- src/rgw/rgw_sal.h | 4 +- src/rgw/rgw_sal_dbstore.cc | 7 +- src/rgw/rgw_sal_dbstore.h | 4 +- src/rgw/rgw_sal_filter.cc | 10 +- src/rgw/rgw_sal_filter.h | 3 +- src/test/cli/radosgw-admin/help.t | 2 + src/test/rgw/lua/test_lua.py | 234 +++++++++++++++++++++---- src/test/rgw/test_rgw_lua.cc | 5 +- 24 files changed, 620 insertions(+), 129 deletions(-) diff --git a/doc/man/8/radosgw-admin.rst b/doc/man/8/radosgw-admin.rst index f6e66135f42..fd2182fd853 100644 --- a/doc/man/8/radosgw-admin.rst +++ b/doc/man/8/radosgw-admin.rst @@ -856,6 +856,10 @@ Options Specify a file to read when setting data. +.. option:: --script-name + + Specify a Lua script name (default 'default'). + .. option:: --categories= Comma separated list of categories, used in usage show. diff --git a/doc/radosgw/lua-scripting.rst b/doc/radosgw/lua-scripting.rst index e3cc950022b..f9406a7cfbb 100644 --- a/doc/radosgw/lua-scripting.rst +++ b/doc/radosgw/lua-scripting.rst @@ -22,6 +22,8 @@ All Lua language features can be used in all contexts. An execution of a script in a context can use up to 128K byte of memory. This include all libraries used by Lua, but not the memory which is managed by the RGW itself, and may be accessed from Lua. To change this default value, use the ``rgw_lua_max_memory_per_state`` configuration parameter. Note that the basic overhead of Lua with its standard libraries is ~32K bytes. To disable the limit, use zero. By default, the execution of a Lua script is limited to a maximum runtime of 1000 milliseconds. This limit can be changed using the ``rgw_lua_max_runtime_per_state`` configuration parameter. If a Lua script exceeds this runtime, it will be terminated. To disable the runtime limit, use zero. +Scripts maintain separate local scopes during execution. When a global variable is declared within a script, it remains confined to that script's scope and cannot be accessed by other scripts. The execution order follows lexicographical sorting of script names with the unnamed script running first. CPU and memory limits are enforced per script, except in the ``background`` context, where all scripts share the same resource limits. Multiple scripts can be specified for all contexts except ``getdata`` and ``putdata``. The name ``default`` is reserved for the unnamed script. + .. warning:: Be cautious when modifying the memory limit. If the current memory usage exceeds the newly set limit, all previously stored data in the background state will be lost. @@ -50,27 +52,34 @@ To upload a script: :: - # radosgw-admin script put --infile={lua-file-path} --context={prerequest|postauth|postrequest|background|getdata|putdata} [--tenant={tenant-name}] + # radosgw-admin script put --infile={lua-file-path} --context={prerequest|postauth|postrequest|background|getdata|putdata} [--tenant={tenant-name}] [--script-name={script-name}] * When uploading a script with the ``background`` context, a tenant name should not be specified. +* When uploading a script with the ``getdata`` or ``putdata`` contexts, a script name should not be specified. :: - # cephadm shell radosgw-admin script put --infile=/rootfs/{lua-file-path} --context={prerequest|postrequest|background|getdata|putdata} [--tenant={tenant-name}] + # cephadm shell radosgw-admin script put --infile=/rootfs/{lua-file-path} --context={prerequest|postauth|postrequest|background|getdata|putdata} [--tenant={tenant-name}] [--script-name={script-name}] To print the content of the script to standard output: :: - # radosgw-admin script get --context={preRequest|postAuth|postRequest|background|getdata|putdata} [--tenant={tenant-name}] + # radosgw-admin script get --context={preRequest|postAuth|postRequest|background|getdata|putdata} [--tenant={tenant-name}] [--script-name={script-name}] + + +To list the scripts to standard output: + +:: + # radosgw-admin script list --context={prerequest|postauth|postrequest|background|getdata|putdata} [--tenant={tenant-name}] To remove the script: :: - # radosgw-admin script rm --context={preRequest|postAuth|postRequest|background|getdata|putdata} [--tenant={tenant-name}] + # radosgw-admin script rm --context={preRequest|postAuth|postRequest|background|getdata|putdata} [--tenant={tenant-name}] [--script-name={script-name}] Package Management via CLI diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h index 9eed25b46c1..7cf4611e79b 100644 --- a/src/rgw/driver/daos/rgw_sal_daos.h +++ b/src/rgw/driver/daos/rgw_sal_daos.h @@ -507,12 +507,18 @@ class DaosLuaManager : public StoreLuaManager { DaosLuaManager(DaosStore* _s) : store(_s) {} virtual ~DaosLuaManager() = default; - virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, + virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) override { DAOS_NOT_IMPLEMENTED_LOG(dpp); return -ENOENT; }; + virtual int list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, + std::vector& scripts) override { + DAOS_NOT_IMPLEMENTED_LOG(dpp); + return -ENOENT; + } + virtual int put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) override { diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc index 4cd90d0c7dc..e733706d8bb 100644 --- a/src/rgw/driver/motr/rgw_sal_motr.cc +++ b/src/rgw/driver/motr/rgw_sal_motr.cc @@ -3874,11 +3874,15 @@ int MotrStore::init_metadata_cache(const DoutPrefixProvider *dpp, return 0; } - int MotrLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) + int MotrLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) { return -ENOENT; } + int MotrLuaManager::list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector& scripts) { + return -ENOENT; + } + std::tuple MotrLuaManager::get_script_or_bytecode(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) { diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h index 04a7f0d6cd6..f3c05147738 100644 --- a/src/rgw/driver/motr/rgw_sal_motr.h +++ b/src/rgw/driver/motr/rgw_sal_motr.h @@ -547,7 +547,9 @@ class MotrLuaManager : public StoreLuaManager { virtual ~MotrLuaManager() = default; /** Get a script named with the given key from the backing store */ - virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override; + virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) override; + /** List all scripts named with the given key from the backing store */ + virtual int list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector& scripts) override; /** Get the Lua bytecode if it exists, else script named with the given key from the backing store */ virtual std::tuple get_script_or_bytecode(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) override; diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc index cbf99ef1a30..8b9828ea1cb 100644 --- a/src/rgw/driver/posix/rgw_sal_posix.cc +++ b/src/rgw/driver/posix/rgw_sal_posix.cc @@ -1918,7 +1918,12 @@ RGWBucketSyncPolicyHandlerRef POSIXZone::get_sync_policy_handler() { return nullptr; } -int POSIXLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) +int POSIXLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) +{ + return -ENOENT; +} + +int POSIXLuaManager::list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector& scripts) { return -ENOENT; } diff --git a/src/rgw/driver/posix/rgw_sal_posix.h b/src/rgw/driver/posix/rgw_sal_posix.h index dd63ea77507..ec05d677bd8 100644 --- a/src/rgw/driver/posix/rgw_sal_posix.h +++ b/src/rgw/driver/posix/rgw_sal_posix.h @@ -457,7 +457,8 @@ public: { } virtual ~POSIXLuaManager() = default; - virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override; + virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) override; + virtual int list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector& scripts) override; virtual std::tuple get_script_or_bytecode(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) override; virtual int put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) override; diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc index f70491e080c..b5ced579e6b 100644 --- a/src/rgw/driver/rados/rgw_sal_rados.cc +++ b/src/rgw/driver/rados/rgw_sal_rados.cc @@ -5546,15 +5546,14 @@ int RadosLuaManager::unwatch_script(const DoutPrefixProvider* dpp, const std::st return 0; } -int RadosLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) -{ +int RadosLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) { if (pool.empty()) { ldpp_dout(dpp, 10) << "WARNING: missing pool when reading Lua script " << dendl; return 0; } - bufferlist bl; - int r = rgw_get_system_obj(store->svc()->sysobj, pool, key, bl, nullptr, nullptr, y, dpp); + bufferlist bl; + int r = rgw_get_system_obj(store->svc()->sysobj, pool, key, bl, objv, nullptr, y, dpp); if (r < 0) { return r; } @@ -5563,12 +5562,40 @@ int RadosLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, try { ceph::decode(script, iter); } catch (buffer::error& err) { + ldpp_dout(dpp, 1) << "ERROR: failed to decode Lua script for " + << key << ", error: " << err.what() << dendl; return -EIO; } return 0; } +int RadosLuaManager::list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector& scripts) { + if (pool.empty()) { + ldpp_dout(dpp, 10) << "WARNING: missing pool when reading Lua script " << dendl; + return 0; + } + + // get the script list metadata as a bufferlist + bufferlist bl; + int r = rgw_get_system_obj(store->svc()->sysobj, pool, list_meta_key, bl, objv, nullptr, y, dpp); + if (r < 0) { + ldpp_dout(dpp, 10) << "WARNING: failed to read Lua script metadata for " << list_meta_key << ", err:" << r << dendl; + return r; + } + + // decode the bufferlist + auto iter = bl.cbegin(); + try { + ceph::decode(scripts, iter); + } catch (buffer::error& err) { + ldpp_dout(dpp, 1) << "ERROR: failed to decode Lua script metadata for " + << list_meta_key << ", error: " << err.what() << dendl; + return -EIO; + } + return 0; +} + std::tuple RadosLuaManager::get_script_or_bytecode(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) { @@ -5616,6 +5643,26 @@ std::tuple RadosLuaManager::get_script_or_bytecode(c return std::make_tuple(script, 0); } +int RadosLuaManager::save_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, const std::vector& scripts) { + // create a bufferlist of scripts + bufferlist bl; + ceph::encode(scripts, bl); + + // write the list to the metadata file + int r; + int retries = 10; + do { + r = rgw_put_system_obj(dpp, store->svc()->sysobj, pool, list_meta_key, bl, false, objv, real_time(), y); + retries--; + } while(r == -EBUSY && retries > 0); + + if (r < 0) { + ldpp_dout(dpp, 10) << "WARNING: failed to update Lua script list: " << list_meta_key << ", err:" << r << dendl; + return r; + } + return 0; +} + int RadosLuaManager::put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) { @@ -5623,11 +5670,26 @@ int RadosLuaManager::put_script(const DoutPrefixProvider* dpp, optional_yield y, ldpp_dout(dpp, 10) << "WARNING: missing pool when writing Lua script " << dendl; return 0; } + + // read the script list + RGWObjVersionTracker objv; + std::vector scripts; + std::string script_tenant, script_name; + rgw::lua::context script_context; + rgw::lua::parse_script_oid(key, script_context, script_tenant, script_name); + std::string list_meta_key = rgw::lua::script_list_metadata_oid(script_context, script_tenant); + int r = list_scripts(dpp, y, list_meta_key, &objv, scripts); + if (r < 0 && r != -ENOENT) { + ldpp_dout(dpp, 10) << "WARNING: failed to list Lua scripts for " << list_meta_key << ", err:" << r << dendl; + return r; + } + + // create the script bufferlist bl; ceph::encode(script, bl); - - int r = rgw_put_system_obj(dpp, store->svc()->sysobj, pool, key, bl, false, nullptr, real_time(), y); + r = rgw_put_system_obj(dpp, store->svc()->sysobj, pool, key, bl, false, nullptr, real_time(), y); if (r < 0) { + ldpp_dout(dpp, 10) << "WARNING: failed to write Lua script " << key << ", err:" << r << dendl; return r; } @@ -5636,6 +5698,14 @@ int RadosLuaManager::put_script(const DoutPrefixProvider* dpp, optional_yield y, ldpp_dout(dpp, 10) << "WARNING: failed to send Lua script update notification :" << key << ", err:" << r << dendl; return r; } + + // write the new script list + if (std::find(scripts.begin(), scripts.end(), script_name) == scripts.end()) { + // insert script in lexicographical order + auto i = std::lower_bound(scripts.begin(), scripts.end(), script_name); + scripts.insert(i, script_name); + return save_scripts(dpp, y, list_meta_key, &objv, scripts); + } return 0; } @@ -5645,11 +5715,33 @@ int RadosLuaManager::del_script(const DoutPrefixProvider* dpp, optional_yield y, ldpp_dout(dpp, 10) << "WARNING: missing pool when deleting Lua script " << dendl; return 0; } - int r = rgw_delete_system_obj(dpp, store->svc()->sysobj, pool, key, nullptr, y); + + // read the script list + RGWObjVersionTracker objv; + std::vector scripts; + std::string script_tenant, script_name; + rgw::lua::context script_context; + rgw::lua::parse_script_oid(key, script_context, script_tenant, script_name); + std::string list_meta_key = rgw::lua::script_list_metadata_oid(script_context, script_tenant); + int r = list_scripts(dpp, y, list_meta_key, &objv, scripts); if (r < 0 && r != -ENOENT) { + ldpp_dout(dpp, 10) << "WARNING: failed to list Lua scripts for " << list_meta_key << ", err:" << r << dendl; return r; } + // delete the script + r = rgw_delete_system_obj(dpp, store->svc()->sysobj, pool, key, nullptr, y); + if (r < 0 && r != -ENOENT) { + ldpp_dout(dpp, 10) << "WARNING: failed to delete Lua script " << key << ", err:" << r << dendl; + return r; + } + + // write the new script list + auto it = std::find(scripts.begin(), scripts.end(), script_name); + if (it != scripts.end()) { + scripts.erase(it); + return save_scripts(dpp, y, rgw::lua::script_list_metadata_oid(script_context, script_tenant), &objv, scripts); + } return 0; } diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h index 6dd8e767571..f6adf74e7ea 100644 --- a/src/rgw/driver/rados/rgw_sal_rados.h +++ b/src/rgw/driver/rados/rgw_sal_rados.h @@ -25,6 +25,7 @@ #include "rgw_role.h" #include "rgw_multi.h" #include "rgw_putobj_processor.h" +#include "rgw_lua.h" #include "services/svc_tier_rados.h" #include "cls/lock/cls_lock_client.h" @@ -1269,6 +1270,7 @@ class RadosLuaManager : public StoreLuaManager { int notify_script_update(const DoutPrefixProvider *dpp, const std::string& script_oid, optional_yield y); uint64_t get_watch_handle_for_script(const std::string& script_oid); std::string get_script_for_watch_handle(uint64_t handle); + int save_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, const std::vector& scripts); uint64_t watch_handle = 0; std::map script_watches; @@ -1279,7 +1281,8 @@ public: ~RadosLuaManager() override = default; // To be used by the radosgw-admin process - int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override; + int get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) override; + int list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector& scripts) override; // To be used by the radosgw process std::tuple get_script_or_bytecode(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) override; int put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) override; diff --git a/src/rgw/radosgw-admin/radosgw-admin.cc b/src/rgw/radosgw-admin/radosgw-admin.cc index 03e825af4e5..a60a7cffe11 100644 --- a/src/rgw/radosgw-admin/radosgw-admin.cc +++ b/src/rgw/radosgw-admin/radosgw-admin.cc @@ -354,6 +354,7 @@ void usage() cout << " topic dump dump (in JSON format) all pending bucket notifications of a persistent topic\n"; cout << " script put upload a Lua script to a context\n"; cout << " script get get the Lua script of a context\n"; + cout << " script list list the Lua scripts of a context\n"; cout << " script rm remove the Lua scripts of a context\n"; cout << " script-package add add a Lua package to the scripts allowlist\n"; cout << " script-package rm remove a Lua package from the scripts allowlist\n"; @@ -477,6 +478,7 @@ void usage() cout << " --skip-zero-entries log show only dumps entries that don't have zero value\n"; cout << " in one of the numeric field\n"; cout << " --infile= file to read in when setting data\n"; + cout << " --script-name= name of the Lua script\n"; cout << " --categories= comma separated list of categories, used in usage show\n"; cout << " --caps= list of caps (e.g., \"usage=read, write; user=read\")\n"; cout << " --op-mask= permission of user's operations (e.g., \"read, write, delete, *\")\n"; @@ -929,6 +931,7 @@ enum class OPT { PUBSUB_TOPIC_STATS, PUBSUB_TOPIC_DUMP, SCRIPT_PUT, + SCRIPT_LIST, SCRIPT_GET, SCRIPT_RM, SCRIPT_PACKAGE_ADD, @@ -1197,6 +1200,7 @@ static SimpleCmd::Commands all_cmds = { { "topic stats", OPT::PUBSUB_TOPIC_STATS }, { "topic dump", OPT::PUBSUB_TOPIC_DUMP }, { "script put", OPT::SCRIPT_PUT }, + { "script list", OPT::SCRIPT_LIST }, { "script get", OPT::SCRIPT_GET }, { "script rm", OPT::SCRIPT_RM }, { "script-package add", OPT::SCRIPT_PACKAGE_ADD }, @@ -3722,6 +3726,7 @@ int main(int argc, const char **argv) int check_objects = false; RGWBucketAdminOpState bucket_op; string infile; + string script_name; string metadata_key; RGWObjVersionTracker objv_tracker; string marker; @@ -4243,6 +4248,8 @@ int main(int argc, const char **argv) caps = val; } else if (ceph_argparse_witharg(args, i, &val, "--infile", (char*)NULL)) { infile = val; + } else if (ceph_argparse_witharg(args, i, &val, "--script-name", (char*)NULL)) { + script_name = (val == "default") ? "" : val; } else if (ceph_argparse_witharg(args, i, &val, "--metadata-key", (char*)NULL)) { metadata_key = val; } else if (ceph_argparse_witharg(args, i, &val, "--marker", (char*)NULL)) { @@ -4791,6 +4798,7 @@ int main(int argc, const char **argv) && opt_cmd != OPT::PUBSUB_TOPIC_DUMP && opt_cmd != OPT::SCRIPT_PUT && opt_cmd != OPT::SCRIPT_GET + && opt_cmd != OPT::SCRIPT_LIST && opt_cmd != OPT::SCRIPT_RM && opt_cmd != OPT::ACCOUNT_CREATE && opt_cmd != OPT::ACCOUNT_MODIFY @@ -12245,8 +12253,14 @@ next: cerr << "ERROR: cannot specify tenant in background context" << std::endl; return EINVAL; } + if ((script_ctx == rgw::lua::context::getData || + script_ctx == rgw::lua::context::putData) && + !script_name.empty()) { + cerr << "ERROR: cannot create more than one Lua script in the " << *str_script_ctx << " context; remove the --script-name flag" << std::endl; + return EINVAL; + } auto lua_manager = driver->get_lua_manager(""); - rc = rgw::lua::write_script(dpp(), lua_manager.get(), tenant, null_yield, script_ctx, script); + rc = rgw::lua::write_script(dpp(), lua_manager.get(), null_yield, tenant, script_ctx, script, script_name); if (rc < 0) { cerr << "ERROR: failed to put script. error: " << rc << std::endl; return -rc; @@ -12263,12 +12277,22 @@ next: cerr << "ERROR: invalid script context: " << *str_script_ctx << ". must be one of: " << LUA_CONTEXT_LIST << std::endl; return EINVAL; } + if ((script_ctx == rgw::lua::context::getData || + script_ctx == rgw::lua::context::putData) && + !script_name.empty()) { + cerr << "ERROR: cannot get more than one Lua script in the " << *str_script_ctx << " context; remove the --script-name flag" << std::endl; + return EINVAL; + } auto lua_manager = driver->get_lua_manager(""); std::string script; - const auto rc = rgw::lua::read_script(dpp(), lua_manager.get(), tenant, null_yield, script_ctx, script); + const auto rc = rgw::lua::read_script(dpp(), lua_manager.get(), null_yield, tenant, script_ctx, script, script_name); if (rc == -ENOENT) { - std::cout << "no script exists for context: " << *str_script_ctx << + if (script_name.empty() || script_name == "default") { + std::cout << "no script exists for context: " << *str_script_ctx << (tenant.empty() ? "" : (" in tenant: " + tenant)) << std::endl; + } else { + std::cout << "'" << script_name << "' script does not exist in context: " << *str_script_ctx << (tenant.empty() ? "" : (" in tenant: " + tenant)) << std::endl; + } } else if (rc < 0) { cerr << "ERROR: failed to read script. error: " << rc << std::endl; return -rc; @@ -12276,6 +12300,36 @@ next: std::cout << script << std::endl; } } + + if (opt_cmd == OPT::SCRIPT_LIST) { + if (!str_script_ctx) { + cerr << "ERROR: context was not provided (via --context)" << std::endl; + return EINVAL; + } + const rgw::lua::context script_ctx = rgw::lua::to_context(*str_script_ctx); + if (script_ctx == rgw::lua::context::none) { + cerr << "ERROR: invalid script context: " << *str_script_ctx << ". must be one of: " << LUA_CONTEXT_LIST << std::endl; + return EINVAL; + } + auto lua_manager = driver->get_lua_manager(""); + std::vector scripts; + const auto rc = rgw::lua::list_scripts(dpp(), lua_manager.get(), null_yield, tenant, script_ctx, scripts); + if (rc < 0 && rc != -ENOENT) { + cerr << "ERROR: failed to list scripts. error: " << rc << std::endl; + return -rc; + } else if (scripts.size() == 0) { + std::cout << "no scripts to list for context: " << *str_script_ctx << + (tenant.empty() ? "" : (" in tenant: " + tenant)) << std::endl; + } else { + for (const auto& script : scripts) { + if (script.empty()) { + std::cout << "default" << std::endl; + } else { + std::cout << script << std::endl; + } + } + } + } if (opt_cmd == OPT::SCRIPT_RM) { if (!str_script_ctx) { @@ -12288,7 +12342,7 @@ next: return EINVAL; } auto lua_manager = driver->get_lua_manager(""); - const auto rc = rgw::lua::delete_script(dpp(), lua_manager.get(), tenant, null_yield, script_ctx); + const auto rc = rgw::lua::delete_script(dpp(), lua_manager.get(), null_yield, tenant, script_ctx, script_name); if (rc < 0) { cerr << "ERROR: failed to remove script. error: " << rc << std::endl; return -rc; diff --git a/src/rgw/rgw_lua.cc b/src/rgw/rgw_lua.cc index a51b8d6fee5..96dcd52e39b 100644 --- a/src/rgw/rgw_lua.cc +++ b/src/rgw/rgw_lua.cc @@ -84,31 +84,73 @@ bool verify(const std::string& script, std::string& err_msg) return true; } -std::string script_oid(context ctx, const std::string& tenant) { +// Parses a script oid key into its context, tenant, and name +void parse_script_oid(const std::string& key, context& context, std::string& tenant, std::string& name) { + std::string prefix; + size_t pos1 = key.find('.'); + if (pos1 == std::string::npos || key.substr(0, pos1) != "script") { + return; + } + size_t pos2 = key.find('.', pos1 + 1); + context = to_context(key.substr(pos1 + 1, pos2)); + size_t pos3 = (pos2 != std::string::npos) ? key.find('.', pos2 + 1) : std::string::npos; + if (pos3 == std::string::npos) { + name = ""; + tenant = key.substr(pos2+1); + } else { + name = key.substr(pos3+1); + tenant = key.substr(pos2+1, pos3); + } +} + +// Returns the oid key of a script list object based on its context and tenant +// For example, the script list for the postrequest context and testx tenant +// will return "metadata.script.postrequest.testx" +// Also, the script list for the background context and global tenant +// will return "metadata.script.background." +std::string script_list_metadata_oid(context ctx, const std::string& tenant) { + static const std::string SCRIPT_LIST_METADATA_PREFIX("metadata."); + return SCRIPT_LIST_METADATA_PREFIX + script_oid(ctx, tenant, ""); +} + +// Returns the oid key of a script object based on its context, tenant, and name. +// For example, a default (unnamed) script within the background context and testx tenant +// will return "script.background.testx" +// Also, a script named "myscript" in the prerequest context and global tenant +// will return "script.prerequest..myscript" +std::string script_oid(context ctx, const std::string& tenant, const std::string& name) { static const std::string SCRIPT_OID_PREFIX("script."); + if (!name.empty()) { + return SCRIPT_OID_PREFIX + to_string(ctx) + "." + tenant + "." + name; + } return SCRIPT_OID_PREFIX + to_string(ctx) + "." + tenant; } +int read_script(const DoutPrefixProvider *dpp, sal::LuaManager* manager, optional_yield y, const std::string& tenant, context ctx, std::string& script, const std::string& name) +{ + return manager ? manager->get_script(dpp, y, nullptr, script_oid(ctx, tenant, name), script) : -ENOENT; +} -int read_script(const DoutPrefixProvider *dpp, sal::LuaManager* manager, const std::string& tenant, optional_yield y, context ctx, std::string& script) +int list_scripts(const DoutPrefixProvider *dpp, sal::LuaManager* manager, optional_yield y, const std::string& tenant, context ctx, std::vector& scripts) { - return manager ? manager->get_script(dpp, y, script_oid(ctx, tenant), script) : -ENOENT; + auto list_meta_key = script_list_metadata_oid (ctx, tenant); + return manager ? manager->list_scripts(dpp, y, list_meta_key, nullptr, scripts) : -ENOENT; } std::tuple read_script_or_bytecode(const DoutPrefixProvider *dpp, sal::LuaManager* manager, - const std::string& tenant, optional_yield y, context ctx) + optional_yield y, const std::string& tenant, context ctx, const std::string& name) { - return manager ? manager->get_script_or_bytecode(dpp, y, script_oid(ctx, tenant)) : std::make_tuple("", -ENOENT); + return manager ? manager->get_script_or_bytecode(dpp, y, script_oid(ctx, tenant, name)) : std::make_tuple("", -ENOENT); } -int write_script(const DoutPrefixProvider *dpp, sal::LuaManager* manager, const std::string& tenant, optional_yield y, context ctx, const std::string& script) +int write_script(const DoutPrefixProvider *dpp, sal::LuaManager* manager, optional_yield y, const std::string& tenant, context ctx, const std::string& script, const std::string& name) { - return manager ? manager->put_script(dpp, y, script_oid(ctx, tenant), script) : -ENOENT; + return manager ? manager->put_script(dpp, y, script_oid(ctx, tenant, name), script) : -ENOENT; } -int delete_script(const DoutPrefixProvider *dpp, sal::LuaManager* manager, const std::string& tenant, optional_yield y, context ctx) +int delete_script(const DoutPrefixProvider *dpp, sal::LuaManager* manager, optional_yield y, const std::string& tenant, context ctx, const std::string& name) { - return manager ? manager->del_script(dpp, y, script_oid(ctx, tenant)) : -ENOENT; + return manager ? manager->del_script(dpp, y, script_oid(ctx, tenant, name)) : -ENOENT; } #ifdef WITH_RADOSGW_LUA_PACKAGES diff --git a/src/rgw/rgw_lua.h b/src/rgw/rgw_lua.h index 8a1b9d386e0..321820ed408 100644 --- a/src/rgw/rgw_lua.h +++ b/src/rgw/rgw_lua.h @@ -34,19 +34,29 @@ enum class context { // return "none" if not matched context to_context(const std::string& s); +std::string to_string(context ctx); + // verify a lua script bool verify(const std::string& script, std::string& err_msg); +// lua script oid key helpers +void parse_script_oid(const std::string& key, context& ctx, std::string& tenant, std::string& name); +std::string script_list_metadata_oid(context ctx, const std::string& tenant); +std::string script_oid(context ctx, const std::string& tenant, const std::string& name); + // driver a lua script in a context -int write_script(const DoutPrefixProvider *dpp, rgw::sal::LuaManager* manager, const std::string& tenant, optional_yield y, context ctx, const std::string& script); +int write_script(const DoutPrefixProvider *dpp, rgw::sal::LuaManager* manager, optional_yield y, const std::string& tenant, context ctx, const std::string& script, const std::string& name); // read the stored lua script from a context -int read_script(const DoutPrefixProvider *dpp, rgw::sal::LuaManager* manager, const std::string& tenant, optional_yield y, context ctx, std::string& script); +int read_script(const DoutPrefixProvider *dpp, rgw::sal::LuaManager* manager, optional_yield y, const std::string& tenant, context ctx, std::string& script, const std::string& name); + +// list the stored lua scripts from a context +int list_scripts(const DoutPrefixProvider *dpp, sal::LuaManager* manager, optional_yield y, const std::string&tenant, context ctx, std::vector& scripts); -std::tuple read_script_or_bytecode(const DoutPrefixProvider *dpp, rgw::sal::LuaManager* manager, const std::string& tenant, optional_yield y, context ctx); +std::tuple read_script_or_bytecode(const DoutPrefixProvider *dpp, rgw::sal::LuaManager* manager, optional_yield y, const std::string& tenant, context ctx, const std::string& name); // delete the stored lua script from a context -int delete_script(const DoutPrefixProvider *dpp, rgw::sal::LuaManager* manager, const std::string& tenant, optional_yield y, context ctx); +int delete_script(const DoutPrefixProvider *dpp, rgw::sal::LuaManager* manager, optional_yield y, const std::string& tenant, context ctx, const std::string& name); using packages_t = std::set; diff --git a/src/rgw/rgw_lua_background.cc b/src/rgw/rgw_lua_background.cc index bd01fc35d96..b31092cdcf2 100644 --- a/src/rgw/rgw_lua_background.cc +++ b/src/rgw/rgw_lua_background.cc @@ -106,13 +106,22 @@ void Background::resume(rgw::sal::Driver*) { cond.notify_all(); } -int Background::read_script() { +int Background::list_scripts(std::vector& script_names) { std::unique_lock cond_lock(pause_mutex); if (paused) { return -EAGAIN; } std::string tenant; - return rgw::lua::read_script(&dp, lua_manager, tenant, null_yield, rgw::lua::context::background, rgw_script); + return rgw::lua::list_scripts(&dp, lua_manager, null_yield, tenant, rgw::lua::context::background, script_names); +} + +int Background::read_script(const std::string& name) { + std::unique_lock cond_lock(pause_mutex); + if (paused) { + return -EAGAIN; + } + std::string tenant; + return rgw::lua::read_script(&dp, lua_manager, null_yield, tenant, rgw::lua::context::background, rgw_script, name); } std::unique_ptr Background::initialize_lguard_state() { @@ -172,12 +181,27 @@ void Background::run() { if (!lguard) { return; } - const auto rc = read_script(); - if (rc == -ENOENT || rc == -EAGAIN) { - // either no script or paused, nothing to do - } else if (rc < 0) { - ldpp_dout(dpp, 1) << "WARNING: failed to read background script. error " << rc << dendl; - } else { + + std::vector script_names; + const auto rc = list_scripts(script_names); // load all scripts stored in list metadata + // include the unnamed script for backward compatibility + if (std::find(script_names.begin(), script_names.end(), "") == script_names.end()) { + script_names.insert(script_names.begin(), ""); + } + if (rc < 0 && rc != -ENOENT && rc != -EAGAIN) { + ldpp_dout(dpp, 1) << "WARNING: failed to list background scripts. error " << rc << dendl; + } + + for (const auto& name : script_names) { + const auto rc = read_script(name); + if (rc == -ENOENT || rc == -EAGAIN) { + // either no script or paused, nothing to do + continue; + } + if (rc < 0) { + ldpp_dout(dpp, 1) << "WARNING: failed to read background script. error " << rc << dendl; + continue; + } auto failed = false; auto L = lguard->get(); try { @@ -240,7 +264,7 @@ void Background::process_scripts() { } std::string script; for (const auto& key: updated_scripts) { - int r = lua_manager->get_script(&dp, null_yield, key, script); + int r = lua_manager->get_script(&dp, null_yield, nullptr, key, script); if (r < 0 && r != -ENOENT) { ldpp_dout(&dp, 10) << "ERROR: Failed to get script : " << key << ". r = " << r << dendl; diff --git a/src/rgw/rgw_lua_background.h b/src/rgw/rgw_lua_background.h index 421e64c2e43..9d70284e9cd 100644 --- a/src/rgw/rgw_lua_background.h +++ b/src/rgw/rgw_lua_background.h @@ -162,7 +162,8 @@ private: void run(); std::string rgw_script; - int read_script(); + int read_script(const std::string& script_name); + int list_scripts(std::vector& script_names); std::unique_ptr initialize_lguard_state(); void process_scripts(); diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index d26ea5be29e..55f95e883b7 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -2525,7 +2525,7 @@ int RGWGetObj::get_data_cb(bufferlist& bl, off_t bl_ofs, off_t bl_len) int RGWGetObj::get_lua_filter(std::unique_ptr* filter, RGWGetObj_Filter* cb) { const auto [script, rc] = rgw::lua::read_script_or_bytecode(s, s->penv.lua.manager.get(), - s->bucket_tenant, s->yield, rgw::lua::context::getData); + s->yield, s->bucket_tenant, rgw::lua::context::getData, ""); if (rc == -ENOENT) { // no script, nothing to do return 0; @@ -4549,7 +4549,7 @@ auto RGWPutObj::get_torrent_filter(rgw::sal::DataProcessor* cb) int RGWPutObj::get_lua_filter(std::unique_ptr* filter, rgw::sal::DataProcessor* cb) { const auto [script, rc] = rgw::lua::read_script_or_bytecode(s, s->penv.lua.manager.get(), - s->bucket_tenant, s->yield, rgw::lua::context::putData); + s->yield, s->bucket_tenant, rgw::lua::context::putData, ""); if (rc == -ENOENT) { // no script, nothing to do return 0; diff --git a/src/rgw/rgw_process.cc b/src/rgw/rgw_process.cc index 28214296c50..e4a212e95f6 100644 --- a/src/rgw/rgw_process.cc +++ b/src/rgw/rgw_process.cc @@ -265,26 +265,37 @@ int rgw_process_authenticated(RGWHandler_REST * const handler, bool is_health_request = (op->get_type() == RGW_OP_GET_HEALTH_CHECK); { if (!is_health_request) { - std::string script; - auto rc = rgw::lua::read_script(s, s->penv.lua.manager.get(), - s->bucket_tenant, s->yield, - rgw::lua::context::postAuth, script); - if (rc == -ENOENT) { - // no script, nothing to do - } else if (rc < 0) { - ldpp_dout(op, 5) << - "WARNING: failed to execute post authorization script. " - "error: " << rc << dendl; - } else { - int script_return_code = 0; - rc = rgw::lua::request::execute(s->penv.rest, s->penv.olog.get(), s, op, script, script_return_code); + std::vector script_names; + const auto rc = rgw::lua::list_scripts(s, s->penv.lua.manager.get(), s->yield, + s->bucket_tenant, rgw::lua::context::postAuth, script_names); + // include the unnamed script for backward compatibility + if (std::find(script_names.begin(), script_names.end(), "") == script_names.end()) { + script_names.insert(script_names.begin(), ""); + } + if (rc < 0 && rc != -ENOENT) { + ldpp_dout(op, 5) << "WARNING: failed to list data scripts in tenant " << s->bucket_tenant + << " and context " << rgw::lua::to_string(rgw::lua::context::postAuth) + << ". error " << rc << dendl; + } + + for (const auto& name : script_names) { + auto [lua_script, rc] = rgw::lua::read_script_or_bytecode(s, s->penv.lua.manager.get(), + s->yield, s->bucket_tenant, + rgw::lua::context::postAuth, name); + if (rc == -ENOENT) { + // no script, nothing to do + continue; + } if (rc < 0) { - ldpp_dout(op, 5) << - "WARNING: failed to execute post authorization script. " - "error: " << rc << dendl; + ldpp_dout(op, 5) << "WARNING: failed to read post authorization script in tenant " << s->bucket_tenant + << ". error " << rc << dendl; + continue; } - if (script_return_code == -EPERM) { - return script_return_code; + + rc = rgw::lua::request::execute(s->penv.rest, s->penv.olog.get(), s, op, lua_script); + if (rc < 0) { + ldpp_dout(op, 5) << "WARNING: failed to execute post authorization script in tenant " << s->bucket_tenant + << ". error " << rc << dendl; } } } @@ -419,23 +430,39 @@ int process_request(const RGWProcessEnv& penv, { s->trace_enabled = tracing::rgw::tracer.is_enabled(); if (!is_health_request) { - auto [lua_script, rc] = rgw::lua::read_script_or_bytecode(s, penv.lua.manager.get(), - s->bucket_tenant, s->yield, - rgw::lua::context::preRequest); - if (rc == -ENOENT) { - // no script, nothing to do - } else if (rc < 0) { - ldpp_dout(op, 5) << - "WARNING: failed to execute pre request script. " - "error: " << rc << dendl; - } else { + std::vector script_names; + const auto rc = rgw::lua::list_scripts(s, penv.lua.manager.get(), s->yield, + s->bucket_tenant, rgw::lua::context::preRequest, script_names); + // include the unnamed script for backward compatibility + if (std::find(script_names.begin(), script_names.end(), "") == script_names.end()) { + script_names.insert(script_names.begin(), ""); + } + if (rc < 0 && rc != -ENOENT) { + ldpp_dout(op, 5) << "WARNING: failed to list data scripts in tenant " << s->bucket_tenant + << " and context " << rgw::lua::to_string(rgw::lua::context::preRequest) + << ". error " << rc << dendl; + } + + for (const auto& name : script_names) { + auto [lua_script, rc] = rgw::lua::read_script_or_bytecode(s, penv.lua.manager.get(), + s->yield, s->bucket_tenant, + rgw::lua::context::preRequest, name); + if (rc == -ENOENT) { + // no script, nothing to do + continue; + } + if (rc < 0) { + ldpp_dout(op, 5) << "WARNING: failed to read pre request script in tenant " << s->bucket_tenant + << ". error " << rc << dendl; + continue; + } + int script_return_code = 0; rc = rgw::lua::request::execute(rest, penv.olog.get(), s, op, lua_script, script_return_code); if (rc < 0) { - ldpp_dout(op, 5) << - "WARNING: failed to execute pre request script. " - "error: " << rc << dendl; + ldpp_dout(op, 5) << "WARNING: failed to execute pre request script in tenant " << s->bucket_tenant + << ". error " << rc << dendl; } if (script_return_code == -EPERM) { abort_early(s, op, script_return_code, handler, yield); @@ -474,21 +501,37 @@ done: } } if (!is_health_request) { - auto [lua_script, rc] = rgw::lua::read_script_or_bytecode(s, penv.lua.manager.get(), - s->bucket_tenant, s->yield, - rgw::lua::context::postRequest); - if (rc == -ENOENT) { - // no script, nothing to do - } else if (rc < 0) { - ldpp_dout(op, 5) << - "WARNING: failed to read post request script. " - "error: " << rc << dendl; - } else { + std::vector script_names; + const auto rc = rgw::lua::list_scripts(s, penv.lua.manager.get(), s->yield, + s->bucket_tenant, rgw::lua::context::postRequest, script_names); + // include the unnamed script for backward compatibility + if (std::find(script_names.begin(), script_names.end(), "") == script_names.end()) { + script_names.insert(script_names.begin(), ""); + } + if (rc < 0 && rc != -ENOENT) { + ldpp_dout(op, 5) << "WARNING: failed to list data scripts in tenant " << s->bucket_tenant + << " and context " << rgw::lua::to_string(rgw::lua::context::postRequest) + << ". error " << rc << dendl; + } + + for (const auto& name : script_names) { + auto [lua_script, rc] = rgw::lua::read_script_or_bytecode(s, penv.lua.manager.get(), + s->yield, s->bucket_tenant, + rgw::lua::context::postRequest, name); + if (rc == -ENOENT) { + // no script, nothing to do + continue; + } + if (rc < 0) { + ldpp_dout(op, 5) << "WARNING: failed to read post request script in tenant " << s->bucket_tenant + << ". error " << rc << dendl; + continue; + } + rc = rgw::lua::request::execute(rest, penv.olog.get(), s, op, lua_script); if (rc < 0) { - ldpp_dout(op, 5) << - "WARNING: failed to execute post request script. " - "error: " << rc << dendl; + ldpp_dout(op, 5) << "WARNING: failed to execute post request script in tenant " << s->bucket_tenant + << ". error " << rc << dendl; } } } diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index efdad9d5195..98fe3f8ca45 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -1925,7 +1925,9 @@ public: virtual ~LuaManager() = default; /** Get a script named with the given key from the backing store */ - virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) = 0; + virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) = 0; + /** List all scripts named with the given key from the backing store */ + virtual int list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector& scripts) = 0; /** Get a copy of the lua bytecode if it exists, else the script named with the given key from the backing store */ virtual std::tuple get_script_or_bytecode(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) = 0; /** Put a script named with the given key to the backing store */ diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc index f9be92fda67..5e7d40de9c9 100644 --- a/src/rgw/rgw_sal_dbstore.cc +++ b/src/rgw/rgw_sal_dbstore.cc @@ -2142,7 +2142,12 @@ namespace rgw::sal { return ret; } - int DBLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) + int DBLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) + { + return -ENOENT; + } + + int DBLuaManager::list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector& scripts) { return -ENOENT; } diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h index 262b2b5ae3f..5e7d863c130 100644 --- a/src/rgw/rgw_sal_dbstore.h +++ b/src/rgw/rgw_sal_dbstore.h @@ -329,7 +329,9 @@ protected: virtual ~DBLuaManager() = default; /** Get a script named with the given key from the backing store */ - virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override; + virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) override; + /** List all scripts named with the given key from the backing store */ + virtual int list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector& scripts) override; /** Get a ref to the Lua bytecode if it exists, else the script named with the given key from the backing store */ virtual std::tuple get_script_or_bytecode(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) override; diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc index 8e89d0ac3b3..462157ade04 100644 --- a/src/rgw/rgw_sal_filter.cc +++ b/src/rgw/rgw_sal_filter.cc @@ -1535,10 +1535,16 @@ int FilterWriter::complete(size_t accounted_size, const std::string& etag, canceled, rctx, flags); } -int FilterLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, +int FilterLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) { - return next->get_script(dpp, y, key, script); + return next->get_script(dpp, y, objv, key, script); +} + +int FilterLuaManager::list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, + RGWObjVersionTracker* objv, std::vector& scripts) +{ + return next->list_scripts(dpp, y, list_meta_key, objv, scripts); } std::tuple FilterLuaManager::get_script_or_bytecode(const DoutPrefixProvider* dpp, optional_yield y, diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h index a92da347740..c5c747ca5c1 100644 --- a/src/rgw/rgw_sal_filter.h +++ b/src/rgw/rgw_sal_filter.h @@ -1155,7 +1155,8 @@ public: FilterLuaManager(std::unique_ptr _next) : next(std::move(_next)) {} virtual ~FilterLuaManager() = default; - virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override; + virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) override; + virtual int list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector& scripts) override; virtual std::tuple get_script_or_bytecode(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) override; virtual int put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) override; diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t index 0a691e71c9c..9984a62b7b9 100644 --- a/src/test/cli/radosgw-admin/help.t +++ b/src/test/cli/radosgw-admin/help.t @@ -214,6 +214,7 @@ topic dump dump (in JSON format) all pending bucket notifications of a persistent topic script put upload a Lua script to a context script get get the Lua script of a context + script list list the Lua scripts of a context script rm remove the Lua scripts of a context script-package add add a Lua package to the scripts allowlist script-package rm remove a Lua package from the scripts allowlist @@ -337,6 +338,7 @@ --skip-zero-entries log show only dumps entries that don't have zero value in one of the numeric field --infile= file to read in when setting data + --script-name= name of the Lua script --categories= comma separated list of categories, used in usage show --caps= list of caps (e.g., "usage=read, write; user=read") --op-mask= permission of user's operations (e.g., "read, write, delete, *") diff --git a/src/test/rgw/lua/test_lua.py b/src/test/rgw/lua/test_lua.py index 67e64340678..e9349b92727 100644 --- a/src/test/rgw/lua/test_lua.py +++ b/src/test/rgw/lua/test_lua.py @@ -31,9 +31,11 @@ test_path = os.path.normpath(os.path.dirname(os.path.realpath(__file__))) + '/.. def bash(cmd, **kwargs): log.debug('running command: %s', ' '.join(cmd)) kwargs['stdout'] = subprocess.PIPE + kwargs["stderr"] = subprocess.PIPE + kwargs["text"] = True process = subprocess.Popen(cmd, **kwargs) - s = process.communicate()[0].decode('utf-8') - return (s, process.returncode) + stdout, stderr = process.communicate() + return (stdout, stderr, process.returncode) def admin(args, **kwargs): @@ -96,9 +98,9 @@ def another_user(tenant=None): secret_key = str(time.time()) uid = 'superman' + str(time.time()) if tenant: - _, result = admin(['user', 'create', '--uid', uid, '--tenant', tenant, '--access-key', access_key, '--secret-key', secret_key, '--display-name', '"Super Man"']) + _, _, result = admin(['user', 'create', '--uid', uid, '--tenant', tenant, '--access-key', access_key, '--secret-key', secret_key, '--display-name', '"Super Man"']) else: - _, result = admin(['user', 'create', '--uid', uid, '--access-key', access_key, '--secret-key', secret_key, '--display-name', '"Super Man"']) + _, _, result = admin(['user', 'create', '--uid', uid, '--access-key', access_key, '--secret-key', secret_key, '--display-name', '"Super Man"']) assert result == 0 hostname = get_config_host() @@ -116,15 +118,20 @@ def another_user(tenant=None): return client -def put_script(script, context, tenant=None): +def put_script(script, context, tenant=None, script_name=None): fp = tempfile.NamedTemporaryFile(mode='w+') fp.write(script) fp.flush() + + args = ['script', 'put', '--infile', fp.name, '--context', context] if tenant: - result = admin(['script', 'put', '--infile', fp.name, '--context', context, '--tenant', tenant]) - else: - result = admin(['script', 'put', '--infile', fp.name, '--context', context]) + args.append('--tenant') + args.append(tenant) + if script_name: + args.append('--script-name') + args.append(script_name) + result = admin(args) fp.close() return result @@ -213,20 +220,71 @@ def test_script_management(): for context in contexts: script = 'print("hello from ' + context + '")' result = put_script(script, context) - assert result[1] == 0 + assert result[2] == 0 scripts[context] = script for context in contexts: result = admin(['script', 'get', '--context', context]) - assert result[1] == 0 + assert result[2] == 0 assert result[0].strip() == scripts[context] for context in contexts: result = admin(['script', 'rm', '--context', context]) - assert result[1] == 0 + assert result[2] == 0 for context in contexts: result = admin(['script', 'get', '--context', context]) - assert result[1] == 0 + assert result[2] == 0 assert result[0].strip() == 'no script exists for context: ' + context +@pytest.mark.basic_test +def test_multi_script_unsupported(): + contexts = ['getdata', 'putdata'] + for context in contexts: + result = put_script('print("hello")', context, None, 'test') + assert result[2] != 0 + assert f"ERROR: cannot create more than one Lua script in the {context} context; remove the --script-name flag" in result[1] + +@pytest.mark.basic_test +def test_multi_script_management(): + contexts = ['prerequest', 'postauth', 'postrequest', 'background'] # getdata and putdata contexts cannot use --script-name + script_names = ["c", "b", "a"] + scripts = {} + for context in contexts: + for script_name in script_names: + script = 'print("hello from ' + '{} ({})'.format(context, script_name) + ' script")' + result = put_script(script, context, None, script_name) + assert result[2] == 0 + scripts.setdefault(context, {})[script_name] = script + for context in contexts: + for script_name in script_names: + result = admin(['script', 'get', '--context', context, '--script-name', script_name]) + assert result[2] == 0 + assert result[0].strip() == scripts[context][script_name] + for context in contexts: + result = admin(['script', 'list', '--context', context]) + assert result[2] == 0 + assert result[0].strip() == "a\nb\nc" + # remove one script and verify the others remain in sorted order + for context in contexts: + result = admin(['script', 'rm', '--context', context, '--script-name', 'b']) + assert result[2] == 0 + for context in contexts: + result = admin(['script', 'get', '--context', context, '--script-name', 'a']) + assert result[2] == 0 + assert result[0].strip() == scripts[context]['a'] + result = admin(['script', 'get', '--context', context, '--script-name', 'c']) + assert result[2] == 0 + assert result[0].strip() == scripts[context]['c'] + result = admin(['script', 'list', '--context', context]) + assert result[2] == 0 + assert result[0].strip() == "a\nc" + for context in contexts: + for script_name in ['c', 'a']: + result = admin(['script', 'rm', '--context', context, '--script-name', script_name]) + assert result[2] == 0 + for context in contexts: + for script_name in script_names: + result = admin(['script', 'get', '--context', context, '--script-name', script_name]) + assert result[2] == 0 + assert result[0].strip() == "'{}' script does not exist in context: ".format(script_name) + context @pytest.mark.basic_test def test_script_management_with_tenant(): @@ -238,26 +296,58 @@ def test_script_management_with_tenant(): for t in ['', tenant]: script = 'print("hello from ' + context + ' and ' + tenant + '")' result = put_script(script, context, t) - assert result[1] == 0 + assert result[2] == 0 scripts[context+t] = script for context in contexts: result = admin(['script', 'get', '--context', context]) - assert result[1] == 0 + assert result[2] == 0 assert result[0].strip(), scripts[context] result = admin(['script', 'rm', '--context', context]) - assert result[1] == 0 + assert result[2] == 0 result = admin(['script', 'get', '--context', context]) - assert result[1] == 0 + assert result[2] == 0 assert result[0].strip(), 'no script exists for context: ' + context result = admin(['script', 'get', '--context', context, '--tenant', tenant]) - assert result[1] == 0 + assert result[2] == 0 assert result[0].strip(), scripts[context+tenant] result = admin(['script', 'rm', '--context', context, '--tenant', tenant]) - assert result[1] == 0 + assert result[2] == 0 result = admin(['script', 'get', '--context', context, '--tenant', tenant]) - assert result[1] == 0 + assert result[2] == 0 assert result[0].strip(), 'no script exists for context: ' + context + ' in tenant: ' + tenant +@pytest.mark.basic_test +def test_multi_script_management_with_tenant(): + tenant = 'mytenant' + conn2 = another_user(tenant) + contexts = ['prerequest', 'postauth', 'postrequest'] # background context cannot use a tenant, getdata and putdata contexts cannot use --script-name + script_names = ["c", "b", "a"] + scripts = {} + for context in contexts: + for t in ['', tenant]: + for script_name in script_names: + script = 'print("hello from ' + '{} ({})'.format(context, script_name) + ' and ' + tenant + '")' + result = put_script(script, context, t, script_name) + assert result[2] == 0 + scripts.setdefault(context+t, {})[script_name] = script + for context in contexts: + for script_name in script_names: + result = admin(['script', 'get', '--context', context, '--script-name', script_name]) + assert result[2] == 0 + assert result[0].strip(), scripts[context][script_name] + result = admin(['script', 'rm', '--context', context, '--script-name', script_name]) + assert result[2] == 0 + result = admin(['script', 'get', '--context', context, '--script-name', script_name]) + assert result[2] == 0 + assert result[0].strip(), "'{}' script does not exist in context: ".format(script_name) + context + result = admin(['script', 'get', '--context', context, '--tenant', tenant, '--script-name', script_name]) + assert result[2] == 0 + assert result[0].strip(), scripts[context+tenant][script_name] + result = admin(['script', 'rm', '--context', context, '--tenant', tenant, '--script-name', script_name]) + assert result[2] == 0 + result = admin(['script', 'get', '--context', context, '--tenant', tenant, '--script-name', script_name]) + assert result[2] == 0 + assert result[0].strip(), "'{}' script does not exist in context: ".format(script_name) + context + ' in tenant: ' + tenant @pytest.mark.request_test def test_put_obj(): @@ -273,7 +363,7 @@ end ''' context = "prerequest" result = put_script(script, context) - assert result[1] == 0 + assert result[2] == 0 conn = connection() bucket_name = gen_bucket_name() @@ -300,7 +390,7 @@ end contexts = ['prerequest', 'postauth', 'postrequest', 'getdata', 'putdata'] for context in contexts: result = admin(['script', 'rm', '--context', context]) - assert result[1] == 0 + assert result[2] == 0 @pytest.mark.example_test @@ -327,7 +417,7 @@ RGWDebugLog("op was: "..Request.RGWOp) for context in contexts: footer = '\nRGWDebugLog("context was: '+context+'\\n\\n")' result = put_script(script+footer, context) - assert result[1] == 0 + assert result[2] == 0 conn = connection() bucket_name = gen_bucket_name() @@ -352,7 +442,7 @@ RGWDebugLog("op was: "..Request.RGWOp) contexts = ['prerequest', 'postauth', 'postrequest', 'getdata', 'putdata'] for context in contexts: result = admin(['script', 'rm', '--context', context]) - assert result[1] == 0 + assert result[2] == 0 @pytest.mark.example_test @@ -390,7 +480,7 @@ RGWDebugLog("payload size of chunk of: " .. full_name .. " is: " .. #Data) ''' result = put_script(script, "putdata") - assert result[1] == 0 + assert result[2] == 0 conn = connection() bucket_name = gen_bucket_name() @@ -409,7 +499,7 @@ RGWDebugLog("payload size of chunk of: " .. full_name .. " is: " .. #Data) contexts = ['prerequest', 'postauth', 'postrequest', 'background', 'getdata', 'putdata'] for context in contexts: result = admin(['script', 'rm', '--context', context]) - assert result[1] == 0 + assert result[2] == 0 @pytest.mark.example_test @@ -440,13 +530,13 @@ end '''.format(socket_path) result = admin(['script-package', 'add', '--package=lua-cjson', '--allow-compilation']) - assert result[1] == 0 + assert result[2] == 0 result = admin(['script-package', 'add', '--package=luasocket', '--allow-compilation']) - assert result[1] == 0 + assert result[2] == 0 result = admin(['script-package', 'reload']) - assert result[1] == 0 + assert result[2] == 0 result = put_script(script, "postrequest") - assert result[1] == 0 + assert result[2] == 0 socket_server = UnixSocket(socket_path) try: @@ -480,8 +570,88 @@ end contexts = ['prerequest', 'postauth', 'postrequest', 'background', 'getdata', 'putdata'] for context in contexts: result = admin(['script', 'rm', '--context', context]) - assert result[1] == 0 + assert result[2] == 0 + +@pytest.mark.example_test +def test_multi_access_log(): + bucket_name = gen_bucket_name() + socket_path = '/tmp/'+bucket_name + + result = admin(['script-package', 'add', '--package=lua-cjson', '--allow-compilation']) + assert result[2] == 0 + result = admin(['script-package', 'add', '--package=luasocket', '--allow-compilation']) + assert result[2] == 0 + result = admin(['script-package', 'reload']) + assert result[2] == 0 + + script_names = ["b", "a"] + for name in script_names: + delay = 0.5*script_names.index(name) + script = ''' +if Request.RGWOp == "get_obj" then + local json = require("cjson") + local socket = require("socket") + local unix = require("socket.unix") + local s = unix() + E = {{}} + + -- sleep 0.5s for a and 0s for b + -- we should expect script a to execute first despite taking longer to connect to the socket + {} + msg = {{bucket = (Request.Bucket or (Request.CopyFrom or E).Bucket).Name, + object = Request.Object.Name, + time = Request.Time, + script_name = "{}", + operation = Request.RGWOp, + http_status = Request.Response.HTTPStatusCode, + error_code = Request.Response.HTTPStatus, + object_size = Request.Object.Size, + trans_id = Request.TransactionId}} + assert(s:connect("{}")) + s:send(json.encode(msg).."\\n") + s:close() +end + '''.format("" if delay == 0 else "socket.sleep({})".format(delay), name, socket_path) + result = put_script(script, "postrequest", None, name) + assert result[2] == 0 + + socket_server = UnixSocket(socket_path) + try: + conn = connection() + # create bucket + bucket = conn.create_bucket(Bucket=bucket_name) + # create objects in the bucket (async) + number_of_objects = 3 + keys = [] + for i in range(number_of_objects): + content = str(os.urandom(1024)).encode("ascii") + key = str(i) + conn.put_object(Body=content, Bucket=bucket_name, Key=key) + keys.append(key) + + for key in conn.list_objects(Bucket=bucket_name)['Contents']: + conn.get_object(Bucket=bucket_name, Key=key['Key']) + + time.sleep(3) + events = {} + for event in socket_server.get_and_reset_events(): + assert event['bucket'] == bucket_name + events[event['object']] = events.get(event['object'], []) + [event['script_name']] + + assert len(events) == number_of_objects + sorted_script_names = sorted(script_names) + for obj in events: + assert sorted_script_names == events[obj] + + finally: + socket_server.shutdown() + delete_all_objects(conn, bucket_name) + conn.delete_bucket(Bucket=bucket_name) + contexts = ['prerequest', 'postrequest', 'background', 'getdata', 'putdata'] + for context in contexts: + result = admin(['script', 'rm', '--context', context]) + assert result[2] == 0 @pytest.mark.example_test def test_interrupt_request(): @@ -494,7 +664,7 @@ def test_interrupt_request(): conn.create_bucket(Bucket=bucket_name) result = put_script(script, "prerequest") - assert result[1] == 0 + assert result[2] == 0 key = "hello" try: @@ -503,7 +673,7 @@ def test_interrupt_request(): except Exception as e: pass - out, err = admin(['script', 'rm', '--context', 'prerequest']) + out, _, err = admin(['script', 'rm', '--context', 'prerequest']) assert err == 0 try: diff --git a/src/test/rgw/test_rgw_lua.cc b/src/test/rgw/test_rgw_lua.cc index c6aaaa2ead3..88af725f750 100644 --- a/src/test/rgw/test_rgw_lua.cc +++ b/src/test/rgw/test_rgw_lua.cc @@ -165,11 +165,14 @@ class TestLuaManager : public rgw::sal::StoreLuaManager { TestLuaManager() { rgw_perf_start(g_ceph_context); } - int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override { + int get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) override { std::this_thread::sleep_for(std::chrono::seconds(read_time)); script = lua_script; return 0; } + int list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector& scripts) override { + return -ENOENT; + } std::tuple get_script_or_bytecode(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) override { std::this_thread::sleep_for(std::chrono::seconds(read_time)); return std::make_tuple(lua_script, 0); -- 2.47.3