Specify a file to read when setting data.
+.. option:: --script-name
+
+ Specify a Lua script name (default 'default').
+
.. option:: --categories=<list>
Comma separated list of categories, used in usage show.
An execution of a script in a context can use up to 128K byte of memory. This include all libraries used by Lua, but not the memory which is managed by the RGW itself, and may be accessed from Lua.
To change this default value, use the ``rgw_lua_max_memory_per_state`` configuration parameter. Note that the basic overhead of Lua with its standard libraries is ~32K bytes. To disable the limit, use zero.
By default, the execution of a Lua script is limited to a maximum runtime of 1000 milliseconds. This limit can be changed using the ``rgw_lua_max_runtime_per_state`` configuration parameter. If a Lua script exceeds this runtime, it will be terminated. To disable the runtime limit, use zero.
+Scripts maintain separate local scopes during execution. When a global variable is declared within a script, it remains confined to that script's scope and cannot be accessed by other scripts. The execution order follows lexicographical sorting of script names with the unnamed script running first. CPU and memory limits are enforced per script, except in the ``background`` context, where all scripts share the same resource limits. Multiple scripts can be specified for all contexts except ``getdata`` and ``putdata``. The name ``default`` is reserved for the unnamed script.
+
.. warning:: Be cautious when modifying the memory limit. If the current memory usage exceeds the newly set limit, all previously stored data in the background state will be lost.
::
- # radosgw-admin script put --infile={lua-file-path} --context={prerequest|postauth|postrequest|background|getdata|putdata} [--tenant={tenant-name}]
+ # radosgw-admin script put --infile={lua-file-path} --context={prerequest|postauth|postrequest|background|getdata|putdata} [--tenant={tenant-name}] [--script-name={script-name}]
* When uploading a script with the ``background`` context, a tenant name should not be specified.
+* When uploading a script with the ``getdata`` or ``putdata`` contexts, a script name should not be specified.
::
- # cephadm shell radosgw-admin script put --infile=/rootfs/{lua-file-path} --context={prerequest|postrequest|background|getdata|putdata} [--tenant={tenant-name}]
+ # cephadm shell radosgw-admin script put --infile=/rootfs/{lua-file-path} --context={prerequest|postauth|postrequest|background|getdata|putdata} [--tenant={tenant-name}] [--script-name={script-name}]
To print the content of the script to standard output:
::
- # radosgw-admin script get --context={preRequest|postAuth|postRequest|background|getdata|putdata} [--tenant={tenant-name}]
+ # radosgw-admin script get --context={preRequest|postAuth|postRequest|background|getdata|putdata} [--tenant={tenant-name}] [--script-name={script-name}]
+
+
+To list the scripts to standard output:
+
+::
+ # radosgw-admin script list --context={prerequest|postauth|postrequest|background|getdata|putdata} [--tenant={tenant-name}]
To remove the script:
::
- # radosgw-admin script rm --context={preRequest|postAuth|postRequest|background|getdata|putdata} [--tenant={tenant-name}]
+ # radosgw-admin script rm --context={preRequest|postAuth|postRequest|background|getdata|putdata} [--tenant={tenant-name}] [--script-name={script-name}]
Package Management via CLI
DaosLuaManager(DaosStore* _s) : store(_s) {}
virtual ~DaosLuaManager() = default;
- virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y,
+ virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv,
const std::string& key, std::string& script) override {
DAOS_NOT_IMPLEMENTED_LOG(dpp);
return -ENOENT;
};
+ virtual int list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key,
+ std::vector<std::string>& scripts) override {
+ DAOS_NOT_IMPLEMENTED_LOG(dpp);
+ return -ENOENT;
+ }
+
virtual int put_script(const DoutPrefixProvider* dpp, optional_yield y,
const std::string& key,
const std::string& script) override {
return 0;
}
- int MotrLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script)
+ int MotrLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script)
{
return -ENOENT;
}
+ int MotrLuaManager::list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector<std::string>& scripts) {
+ return -ENOENT;
+ }
+
std::tuple<rgw::lua::LuaCodeType, int> MotrLuaManager::get_script_or_bytecode(const DoutPrefixProvider* dpp, optional_yield y,
const std::string& key)
{
virtual ~MotrLuaManager() = default;
/** Get a script named with the given key from the backing store */
- virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override;
+ virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) override;
+ /** List all scripts named with the given key from the backing store */
+ virtual int list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector<std::string>& scripts) override;
/** Get the Lua bytecode if it exists, else script named with the given key from the backing store */
virtual std::tuple<rgw::lua::LuaCodeType, int> get_script_or_bytecode(const DoutPrefixProvider* dpp, optional_yield y,
const std::string& key) override;
return nullptr;
}
-int POSIXLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script)
+int POSIXLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script)
+{
+ return -ENOENT;
+}
+
+int POSIXLuaManager::list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector<std::string>& scripts)
{
return -ENOENT;
}
{ }
virtual ~POSIXLuaManager() = default;
- virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override;
+ virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) override;
+ virtual int list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector<std::string>& scripts) override;
virtual std::tuple<rgw::lua::LuaCodeType, int> get_script_or_bytecode(const DoutPrefixProvider* dpp, optional_yield y,
const std::string& key) override;
virtual int put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) override;
return 0;
}
-int RadosLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script)
-{
+int RadosLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) {
if (pool.empty()) {
ldpp_dout(dpp, 10) << "WARNING: missing pool when reading Lua script " << dendl;
return 0;
}
- bufferlist bl;
- int r = rgw_get_system_obj(store->svc()->sysobj, pool, key, bl, nullptr, nullptr, y, dpp);
+ bufferlist bl;
+ int r = rgw_get_system_obj(store->svc()->sysobj, pool, key, bl, objv, nullptr, y, dpp);
if (r < 0) {
return r;
}
try {
ceph::decode(script, iter);
} catch (buffer::error& err) {
+ ldpp_dout(dpp, 1) << "ERROR: failed to decode Lua script for "
+ << key << ", error: " << err.what() << dendl;
return -EIO;
}
return 0;
}
+int RadosLuaManager::list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector<std::string>& scripts) {
+ if (pool.empty()) {
+ ldpp_dout(dpp, 10) << "WARNING: missing pool when reading Lua script " << dendl;
+ return 0;
+ }
+
+ // get the script list metadata as a bufferlist
+ bufferlist bl;
+ int r = rgw_get_system_obj(store->svc()->sysobj, pool, list_meta_key, bl, objv, nullptr, y, dpp);
+ if (r < 0) {
+ ldpp_dout(dpp, 10) << "WARNING: failed to read Lua script metadata for " << list_meta_key << ", err:" << r << dendl;
+ return r;
+ }
+
+ // decode the bufferlist
+ auto iter = bl.cbegin();
+ try {
+ ceph::decode(scripts, iter);
+ } catch (buffer::error& err) {
+ ldpp_dout(dpp, 1) << "ERROR: failed to decode Lua script metadata for "
+ << list_meta_key << ", error: " << err.what() << dendl;
+ return -EIO;
+ }
+ return 0;
+}
+
std::tuple<rgw::lua::LuaCodeType, int> RadosLuaManager::get_script_or_bytecode(const DoutPrefixProvider* dpp, optional_yield y,
const std::string& key)
{
return std::make_tuple(script, 0);
}
+int RadosLuaManager::save_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, const std::vector<std::string>& scripts) {
+ // create a bufferlist of scripts
+ bufferlist bl;
+ ceph::encode(scripts, bl);
+
+ // write the list to the metadata file
+ int r;
+ int retries = 10;
+ do {
+ r = rgw_put_system_obj(dpp, store->svc()->sysobj, pool, list_meta_key, bl, false, objv, real_time(), y);
+ retries--;
+ } while(r == -EBUSY && retries > 0);
+
+ if (r < 0) {
+ ldpp_dout(dpp, 10) << "WARNING: failed to update Lua script list: " << list_meta_key << ", err:" << r << dendl;
+ return r;
+ }
+ return 0;
+}
+
int RadosLuaManager::put_script(const DoutPrefixProvider* dpp, optional_yield y,
const std::string& key, const std::string& script)
{
ldpp_dout(dpp, 10) << "WARNING: missing pool when writing Lua script " << dendl;
return 0;
}
+
+ // read the script list
+ RGWObjVersionTracker objv;
+ std::vector<std::string> scripts;
+ std::string script_tenant, script_name;
+ rgw::lua::context script_context;
+ rgw::lua::parse_script_oid(key, script_context, script_tenant, script_name);
+ std::string list_meta_key = rgw::lua::script_list_metadata_oid(script_context, script_tenant);
+ int r = list_scripts(dpp, y, list_meta_key, &objv, scripts);
+ if (r < 0 && r != -ENOENT) {
+ ldpp_dout(dpp, 10) << "WARNING: failed to list Lua scripts for " << list_meta_key << ", err:" << r << dendl;
+ return r;
+ }
+
+ // create the script
bufferlist bl;
ceph::encode(script, bl);
-
- int r = rgw_put_system_obj(dpp, store->svc()->sysobj, pool, key, bl, false, nullptr, real_time(), y);
+ r = rgw_put_system_obj(dpp, store->svc()->sysobj, pool, key, bl, false, nullptr, real_time(), y);
if (r < 0) {
+ ldpp_dout(dpp, 10) << "WARNING: failed to write Lua script " << key << ", err:" << r << dendl;
return r;
}
ldpp_dout(dpp, 10) << "WARNING: failed to send Lua script update notification :" << key << ", err:" << r << dendl;
return r;
}
+
+ // write the new script list
+ if (std::find(scripts.begin(), scripts.end(), script_name) == scripts.end()) {
+ // insert script in lexicographical order
+ auto i = std::lower_bound(scripts.begin(), scripts.end(), script_name);
+ scripts.insert(i, script_name);
+ return save_scripts(dpp, y, list_meta_key, &objv, scripts);
+ }
return 0;
}
ldpp_dout(dpp, 10) << "WARNING: missing pool when deleting Lua script " << dendl;
return 0;
}
- int r = rgw_delete_system_obj(dpp, store->svc()->sysobj, pool, key, nullptr, y);
+
+ // read the script list
+ RGWObjVersionTracker objv;
+ std::vector<std::string> scripts;
+ std::string script_tenant, script_name;
+ rgw::lua::context script_context;
+ rgw::lua::parse_script_oid(key, script_context, script_tenant, script_name);
+ std::string list_meta_key = rgw::lua::script_list_metadata_oid(script_context, script_tenant);
+ int r = list_scripts(dpp, y, list_meta_key, &objv, scripts);
if (r < 0 && r != -ENOENT) {
+ ldpp_dout(dpp, 10) << "WARNING: failed to list Lua scripts for " << list_meta_key << ", err:" << r << dendl;
return r;
}
+ // delete the script
+ r = rgw_delete_system_obj(dpp, store->svc()->sysobj, pool, key, nullptr, y);
+ if (r < 0 && r != -ENOENT) {
+ ldpp_dout(dpp, 10) << "WARNING: failed to delete Lua script " << key << ", err:" << r << dendl;
+ return r;
+ }
+
+ // write the new script list
+ auto it = std::find(scripts.begin(), scripts.end(), script_name);
+ if (it != scripts.end()) {
+ scripts.erase(it);
+ return save_scripts(dpp, y, rgw::lua::script_list_metadata_oid(script_context, script_tenant), &objv, scripts);
+ }
return 0;
}
#include "rgw_role.h"
#include "rgw_multi.h"
#include "rgw_putobj_processor.h"
+#include "rgw_lua.h"
#include "services/svc_tier_rados.h"
#include "cls/lock/cls_lock_client.h"
int notify_script_update(const DoutPrefixProvider *dpp, const std::string& script_oid, optional_yield y);
uint64_t get_watch_handle_for_script(const std::string& script_oid);
std::string get_script_for_watch_handle(uint64_t handle);
+ int save_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, const std::vector<std::string>& scripts);
uint64_t watch_handle = 0;
std::map<std::string, uint64_t> script_watches;
~RadosLuaManager() override = default;
// To be used by the radosgw-admin process
- int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override;
+ int get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) override;
+ int list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector<std::string>& scripts) override;
// To be used by the radosgw process
std::tuple<rgw::lua::LuaCodeType, int> get_script_or_bytecode(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) override;
int put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) override;
cout << " topic dump dump (in JSON format) all pending bucket notifications of a persistent topic\n";
cout << " script put upload a Lua script to a context\n";
cout << " script get get the Lua script of a context\n";
+ cout << " script list list the Lua scripts of a context\n";
cout << " script rm remove the Lua scripts of a context\n";
cout << " script-package add add a Lua package to the scripts allowlist\n";
cout << " script-package rm remove a Lua package from the scripts allowlist\n";
cout << " --skip-zero-entries log show only dumps entries that don't have zero value\n";
cout << " in one of the numeric field\n";
cout << " --infile=<file> file to read in when setting data\n";
+ cout << " --script-name=<script-name> name of the Lua script\n";
cout << " --categories=<list> comma separated list of categories, used in usage show\n";
cout << " --caps=<caps> list of caps (e.g., \"usage=read, write; user=read\")\n";
cout << " --op-mask=<op-mask> permission of user's operations (e.g., \"read, write, delete, *\")\n";
PUBSUB_TOPIC_STATS,
PUBSUB_TOPIC_DUMP,
SCRIPT_PUT,
+ SCRIPT_LIST,
SCRIPT_GET,
SCRIPT_RM,
SCRIPT_PACKAGE_ADD,
{ "topic stats", OPT::PUBSUB_TOPIC_STATS },
{ "topic dump", OPT::PUBSUB_TOPIC_DUMP },
{ "script put", OPT::SCRIPT_PUT },
+ { "script list", OPT::SCRIPT_LIST },
{ "script get", OPT::SCRIPT_GET },
{ "script rm", OPT::SCRIPT_RM },
{ "script-package add", OPT::SCRIPT_PACKAGE_ADD },
int check_objects = false;
RGWBucketAdminOpState bucket_op;
string infile;
+ string script_name;
string metadata_key;
RGWObjVersionTracker objv_tracker;
string marker;
caps = val;
} else if (ceph_argparse_witharg(args, i, &val, "--infile", (char*)NULL)) {
infile = val;
+ } else if (ceph_argparse_witharg(args, i, &val, "--script-name", (char*)NULL)) {
+ script_name = (val == "default") ? "" : val;
} else if (ceph_argparse_witharg(args, i, &val, "--metadata-key", (char*)NULL)) {
metadata_key = val;
} else if (ceph_argparse_witharg(args, i, &val, "--marker", (char*)NULL)) {
&& opt_cmd != OPT::PUBSUB_TOPIC_DUMP
&& opt_cmd != OPT::SCRIPT_PUT
&& opt_cmd != OPT::SCRIPT_GET
+ && opt_cmd != OPT::SCRIPT_LIST
&& opt_cmd != OPT::SCRIPT_RM
&& opt_cmd != OPT::ACCOUNT_CREATE
&& opt_cmd != OPT::ACCOUNT_MODIFY
cerr << "ERROR: cannot specify tenant in background context" << std::endl;
return EINVAL;
}
+ if ((script_ctx == rgw::lua::context::getData ||
+ script_ctx == rgw::lua::context::putData) &&
+ !script_name.empty()) {
+ cerr << "ERROR: cannot create more than one Lua script in the " << *str_script_ctx << " context; remove the --script-name flag" << std::endl;
+ return EINVAL;
+ }
auto lua_manager = driver->get_lua_manager("");
- rc = rgw::lua::write_script(dpp(), lua_manager.get(), tenant, null_yield, script_ctx, script);
+ rc = rgw::lua::write_script(dpp(), lua_manager.get(), null_yield, tenant, script_ctx, script, script_name);
if (rc < 0) {
cerr << "ERROR: failed to put script. error: " << rc << std::endl;
return -rc;
cerr << "ERROR: invalid script context: " << *str_script_ctx << ". must be one of: " << LUA_CONTEXT_LIST << std::endl;
return EINVAL;
}
+ if ((script_ctx == rgw::lua::context::getData ||
+ script_ctx == rgw::lua::context::putData) &&
+ !script_name.empty()) {
+ cerr << "ERROR: cannot get more than one Lua script in the " << *str_script_ctx << " context; remove the --script-name flag" << std::endl;
+ return EINVAL;
+ }
auto lua_manager = driver->get_lua_manager("");
std::string script;
- const auto rc = rgw::lua::read_script(dpp(), lua_manager.get(), tenant, null_yield, script_ctx, script);
+ const auto rc = rgw::lua::read_script(dpp(), lua_manager.get(), null_yield, tenant, script_ctx, script, script_name);
if (rc == -ENOENT) {
- std::cout << "no script exists for context: " << *str_script_ctx <<
+ if (script_name.empty() || script_name == "default") {
+ std::cout << "no script exists for context: " << *str_script_ctx << (tenant.empty() ? "" : (" in tenant: " + tenant)) << std::endl;
+ } else {
+ std::cout << "'" << script_name << "' script does not exist in context: " << *str_script_ctx <<
(tenant.empty() ? "" : (" in tenant: " + tenant)) << std::endl;
+ }
} else if (rc < 0) {
cerr << "ERROR: failed to read script. error: " << rc << std::endl;
return -rc;
std::cout << script << std::endl;
}
}
+
+ if (opt_cmd == OPT::SCRIPT_LIST) {
+ if (!str_script_ctx) {
+ cerr << "ERROR: context was not provided (via --context)" << std::endl;
+ return EINVAL;
+ }
+ const rgw::lua::context script_ctx = rgw::lua::to_context(*str_script_ctx);
+ if (script_ctx == rgw::lua::context::none) {
+ cerr << "ERROR: invalid script context: " << *str_script_ctx << ". must be one of: " << LUA_CONTEXT_LIST << std::endl;
+ return EINVAL;
+ }
+ auto lua_manager = driver->get_lua_manager("");
+ std::vector<std::string> scripts;
+ const auto rc = rgw::lua::list_scripts(dpp(), lua_manager.get(), null_yield, tenant, script_ctx, scripts);
+ if (rc < 0 && rc != -ENOENT) {
+ cerr << "ERROR: failed to list scripts. error: " << rc << std::endl;
+ return -rc;
+ } else if (scripts.size() == 0) {
+ std::cout << "no scripts to list for context: " << *str_script_ctx <<
+ (tenant.empty() ? "" : (" in tenant: " + tenant)) << std::endl;
+ } else {
+ for (const auto& script : scripts) {
+ if (script.empty()) {
+ std::cout << "default" << std::endl;
+ } else {
+ std::cout << script << std::endl;
+ }
+ }
+ }
+ }
if (opt_cmd == OPT::SCRIPT_RM) {
if (!str_script_ctx) {
return EINVAL;
}
auto lua_manager = driver->get_lua_manager("");
- const auto rc = rgw::lua::delete_script(dpp(), lua_manager.get(), tenant, null_yield, script_ctx);
+ const auto rc = rgw::lua::delete_script(dpp(), lua_manager.get(), null_yield, tenant, script_ctx, script_name);
if (rc < 0) {
cerr << "ERROR: failed to remove script. error: " << rc << std::endl;
return -rc;
return true;
}
-std::string script_oid(context ctx, const std::string& tenant) {
+// Parses a script oid key into its context, tenant, and name
+void parse_script_oid(const std::string& key, context& context, std::string& tenant, std::string& name) {
+ std::string prefix;
+ size_t pos1 = key.find('.');
+ if (pos1 == std::string::npos || key.substr(0, pos1) != "script") {
+ return;
+ }
+ size_t pos2 = key.find('.', pos1 + 1);
+ context = to_context(key.substr(pos1 + 1, pos2));
+ size_t pos3 = (pos2 != std::string::npos) ? key.find('.', pos2 + 1) : std::string::npos;
+ if (pos3 == std::string::npos) {
+ name = "";
+ tenant = key.substr(pos2+1);
+ } else {
+ name = key.substr(pos3+1);
+ tenant = key.substr(pos2+1, pos3);
+ }
+}
+
+// Returns the oid key of a script list object based on its context and tenant
+// For example, the script list for the postrequest context and testx tenant
+// will return "metadata.script.postrequest.testx"
+// Also, the script list for the background context and global tenant
+// will return "metadata.script.background."
+std::string script_list_metadata_oid(context ctx, const std::string& tenant) {
+ static const std::string SCRIPT_LIST_METADATA_PREFIX("metadata.");
+ return SCRIPT_LIST_METADATA_PREFIX + script_oid(ctx, tenant, "");
+}
+
+// Returns the oid key of a script object based on its context, tenant, and name.
+// For example, a default (unnamed) script within the background context and testx tenant
+// will return "script.background.testx"
+// Also, a script named "myscript" in the prerequest context and global tenant
+// will return "script.prerequest..myscript"
+std::string script_oid(context ctx, const std::string& tenant, const std::string& name) {
static const std::string SCRIPT_OID_PREFIX("script.");
+ if (!name.empty()) {
+ return SCRIPT_OID_PREFIX + to_string(ctx) + "." + tenant + "." + name;
+ }
return SCRIPT_OID_PREFIX + to_string(ctx) + "." + tenant;
}
+int read_script(const DoutPrefixProvider *dpp, sal::LuaManager* manager, optional_yield y, const std::string& tenant, context ctx, std::string& script, const std::string& name)
+{
+ return manager ? manager->get_script(dpp, y, nullptr, script_oid(ctx, tenant, name), script) : -ENOENT;
+}
-int read_script(const DoutPrefixProvider *dpp, sal::LuaManager* manager, const std::string& tenant, optional_yield y, context ctx, std::string& script)
+int list_scripts(const DoutPrefixProvider *dpp, sal::LuaManager* manager, optional_yield y, const std::string& tenant, context ctx, std::vector<std::string>& scripts)
{
- return manager ? manager->get_script(dpp, y, script_oid(ctx, tenant), script) : -ENOENT;
+ auto list_meta_key = script_list_metadata_oid (ctx, tenant);
+ return manager ? manager->list_scripts(dpp, y, list_meta_key, nullptr, scripts) : -ENOENT;
}
std::tuple<LuaCodeType, int> read_script_or_bytecode(const DoutPrefixProvider *dpp, sal::LuaManager* manager,
- const std::string& tenant, optional_yield y, context ctx)
+ optional_yield y, const std::string& tenant, context ctx, const std::string& name)
{
- return manager ? manager->get_script_or_bytecode(dpp, y, script_oid(ctx, tenant)) : std::make_tuple("", -ENOENT);
+ return manager ? manager->get_script_or_bytecode(dpp, y, script_oid(ctx, tenant, name)) : std::make_tuple("", -ENOENT);
}
-int write_script(const DoutPrefixProvider *dpp, sal::LuaManager* manager, const std::string& tenant, optional_yield y, context ctx, const std::string& script)
+int write_script(const DoutPrefixProvider *dpp, sal::LuaManager* manager, optional_yield y, const std::string& tenant, context ctx, const std::string& script, const std::string& name)
{
- return manager ? manager->put_script(dpp, y, script_oid(ctx, tenant), script) : -ENOENT;
+ return manager ? manager->put_script(dpp, y, script_oid(ctx, tenant, name), script) : -ENOENT;
}
-int delete_script(const DoutPrefixProvider *dpp, sal::LuaManager* manager, const std::string& tenant, optional_yield y, context ctx)
+int delete_script(const DoutPrefixProvider *dpp, sal::LuaManager* manager, optional_yield y, const std::string& tenant, context ctx, const std::string& name)
{
- return manager ? manager->del_script(dpp, y, script_oid(ctx, tenant)) : -ENOENT;
+ return manager ? manager->del_script(dpp, y, script_oid(ctx, tenant, name)) : -ENOENT;
}
#ifdef WITH_RADOSGW_LUA_PACKAGES
// return "none" if not matched
context to_context(const std::string& s);
+std::string to_string(context ctx);
+
// verify a lua script
bool verify(const std::string& script, std::string& err_msg);
+// lua script oid key helpers
+void parse_script_oid(const std::string& key, context& ctx, std::string& tenant, std::string& name);
+std::string script_list_metadata_oid(context ctx, const std::string& tenant);
+std::string script_oid(context ctx, const std::string& tenant, const std::string& name);
+
// driver a lua script in a context
-int write_script(const DoutPrefixProvider *dpp, rgw::sal::LuaManager* manager, const std::string& tenant, optional_yield y, context ctx, const std::string& script);
+int write_script(const DoutPrefixProvider *dpp, rgw::sal::LuaManager* manager, optional_yield y, const std::string& tenant, context ctx, const std::string& script, const std::string& name);
// read the stored lua script from a context
-int read_script(const DoutPrefixProvider *dpp, rgw::sal::LuaManager* manager, const std::string& tenant, optional_yield y, context ctx, std::string& script);
+int read_script(const DoutPrefixProvider *dpp, rgw::sal::LuaManager* manager, optional_yield y, const std::string& tenant, context ctx, std::string& script, const std::string& name);
+
+// list the stored lua scripts from a context
+int list_scripts(const DoutPrefixProvider *dpp, sal::LuaManager* manager, optional_yield y, const std::string&tenant, context ctx, std::vector<std::string>& scripts);
-std::tuple<LuaCodeType, int> read_script_or_bytecode(const DoutPrefixProvider *dpp, rgw::sal::LuaManager* manager, const std::string& tenant, optional_yield y, context ctx);
+std::tuple<LuaCodeType, int> read_script_or_bytecode(const DoutPrefixProvider *dpp, rgw::sal::LuaManager* manager, optional_yield y, const std::string& tenant, context ctx, const std::string& name);
// delete the stored lua script from a context
-int delete_script(const DoutPrefixProvider *dpp, rgw::sal::LuaManager* manager, const std::string& tenant, optional_yield y, context ctx);
+int delete_script(const DoutPrefixProvider *dpp, rgw::sal::LuaManager* manager, optional_yield y, const std::string& tenant, context ctx, const std::string& name);
using packages_t = std::set<std::string>;
cond.notify_all();
}
-int Background::read_script() {
+int Background::list_scripts(std::vector<std::string>& script_names) {
std::unique_lock cond_lock(pause_mutex);
if (paused) {
return -EAGAIN;
}
std::string tenant;
- return rgw::lua::read_script(&dp, lua_manager, tenant, null_yield, rgw::lua::context::background, rgw_script);
+ return rgw::lua::list_scripts(&dp, lua_manager, null_yield, tenant, rgw::lua::context::background, script_names);
+}
+
+int Background::read_script(const std::string& name) {
+ std::unique_lock cond_lock(pause_mutex);
+ if (paused) {
+ return -EAGAIN;
+ }
+ std::string tenant;
+ return rgw::lua::read_script(&dp, lua_manager, null_yield, tenant, rgw::lua::context::background, rgw_script, name);
}
std::unique_ptr<lua_state_guard> Background::initialize_lguard_state() {
if (!lguard) {
return;
}
- const auto rc = read_script();
- if (rc == -ENOENT || rc == -EAGAIN) {
- // either no script or paused, nothing to do
- } else if (rc < 0) {
- ldpp_dout(dpp, 1) << "WARNING: failed to read background script. error " << rc << dendl;
- } else {
+
+ std::vector<std::string> script_names;
+ const auto rc = list_scripts(script_names); // load all scripts stored in list metadata
+ // include the unnamed script for backward compatibility
+ if (std::find(script_names.begin(), script_names.end(), "") == script_names.end()) {
+ script_names.insert(script_names.begin(), "");
+ }
+ if (rc < 0 && rc != -ENOENT && rc != -EAGAIN) {
+ ldpp_dout(dpp, 1) << "WARNING: failed to list background scripts. error " << rc << dendl;
+ }
+
+ for (const auto& name : script_names) {
+ const auto rc = read_script(name);
+ if (rc == -ENOENT || rc == -EAGAIN) {
+ // either no script or paused, nothing to do
+ continue;
+ }
+ if (rc < 0) {
+ ldpp_dout(dpp, 1) << "WARNING: failed to read background script. error " << rc << dendl;
+ continue;
+ }
auto failed = false;
auto L = lguard->get();
try {
}
std::string script;
for (const auto& key: updated_scripts) {
- int r = lua_manager->get_script(&dp, null_yield, key, script);
+ int r = lua_manager->get_script(&dp, null_yield, nullptr, key, script);
if (r < 0 && r != -ENOENT) {
ldpp_dout(&dp, 10) << "ERROR: Failed to get script : " << key
<< ". r = " << r << dendl;
void run();
std::string rgw_script;
- int read_script();
+ int read_script(const std::string& script_name);
+ int list_scripts(std::vector<std::string>& script_names);
std::unique_ptr<lua_state_guard> initialize_lguard_state();
void process_scripts();
int RGWGetObj::get_lua_filter(std::unique_ptr<RGWGetObj_Filter>* filter, RGWGetObj_Filter* cb) {
const auto [script, rc] = rgw::lua::read_script_or_bytecode(s, s->penv.lua.manager.get(),
- s->bucket_tenant, s->yield, rgw::lua::context::getData);
+ s->yield, s->bucket_tenant, rgw::lua::context::getData, "");
if (rc == -ENOENT) {
// no script, nothing to do
return 0;
int RGWPutObj::get_lua_filter(std::unique_ptr<rgw::sal::DataProcessor>* filter, rgw::sal::DataProcessor* cb) {
const auto [script, rc] = rgw::lua::read_script_or_bytecode(s, s->penv.lua.manager.get(),
- s->bucket_tenant, s->yield, rgw::lua::context::putData);
+ s->yield, s->bucket_tenant, rgw::lua::context::putData, "");
if (rc == -ENOENT) {
// no script, nothing to do
return 0;
bool is_health_request = (op->get_type() == RGW_OP_GET_HEALTH_CHECK);
{
if (!is_health_request) {
- std::string script;
- auto rc = rgw::lua::read_script(s, s->penv.lua.manager.get(),
- s->bucket_tenant, s->yield,
- rgw::lua::context::postAuth, script);
- if (rc == -ENOENT) {
- // no script, nothing to do
- } else if (rc < 0) {
- ldpp_dout(op, 5) <<
- "WARNING: failed to execute post authorization script. "
- "error: " << rc << dendl;
- } else {
- int script_return_code = 0;
- rc = rgw::lua::request::execute(s->penv.rest, s->penv.olog.get(), s, op, script, script_return_code);
+ std::vector<std::string> script_names;
+ const auto rc = rgw::lua::list_scripts(s, s->penv.lua.manager.get(), s->yield,
+ s->bucket_tenant, rgw::lua::context::postAuth, script_names);
+ // include the unnamed script for backward compatibility
+ if (std::find(script_names.begin(), script_names.end(), "") == script_names.end()) {
+ script_names.insert(script_names.begin(), "");
+ }
+ if (rc < 0 && rc != -ENOENT) {
+ ldpp_dout(op, 5) << "WARNING: failed to list data scripts in tenant " << s->bucket_tenant
+ << " and context " << rgw::lua::to_string(rgw::lua::context::postAuth)
+ << ". error " << rc << dendl;
+ }
+
+ for (const auto& name : script_names) {
+ auto [lua_script, rc] = rgw::lua::read_script_or_bytecode(s, s->penv.lua.manager.get(),
+ s->yield, s->bucket_tenant,
+ rgw::lua::context::postAuth, name);
+ if (rc == -ENOENT) {
+ // no script, nothing to do
+ continue;
+ }
if (rc < 0) {
- ldpp_dout(op, 5) <<
- "WARNING: failed to execute post authorization script. "
- "error: " << rc << dendl;
+ ldpp_dout(op, 5) << "WARNING: failed to read post authorization script in tenant " << s->bucket_tenant
+ << ". error " << rc << dendl;
+ continue;
}
- if (script_return_code == -EPERM) {
- return script_return_code;
+
+ rc = rgw::lua::request::execute(s->penv.rest, s->penv.olog.get(), s, op, lua_script);
+ if (rc < 0) {
+ ldpp_dout(op, 5) << "WARNING: failed to execute post authorization script in tenant " << s->bucket_tenant
+ << ". error " << rc << dendl;
}
}
}
{
s->trace_enabled = tracing::rgw::tracer.is_enabled();
if (!is_health_request) {
- auto [lua_script, rc] = rgw::lua::read_script_or_bytecode(s, penv.lua.manager.get(),
- s->bucket_tenant, s->yield,
- rgw::lua::context::preRequest);
- if (rc == -ENOENT) {
- // no script, nothing to do
- } else if (rc < 0) {
- ldpp_dout(op, 5) <<
- "WARNING: failed to execute pre request script. "
- "error: " << rc << dendl;
- } else {
+ std::vector<std::string> script_names;
+ const auto rc = rgw::lua::list_scripts(s, penv.lua.manager.get(), s->yield,
+ s->bucket_tenant, rgw::lua::context::preRequest, script_names);
+ // include the unnamed script for backward compatibility
+ if (std::find(script_names.begin(), script_names.end(), "") == script_names.end()) {
+ script_names.insert(script_names.begin(), "");
+ }
+ if (rc < 0 && rc != -ENOENT) {
+ ldpp_dout(op, 5) << "WARNING: failed to list data scripts in tenant " << s->bucket_tenant
+ << " and context " << rgw::lua::to_string(rgw::lua::context::preRequest)
+ << ". error " << rc << dendl;
+ }
+
+ for (const auto& name : script_names) {
+ auto [lua_script, rc] = rgw::lua::read_script_or_bytecode(s, penv.lua.manager.get(),
+ s->yield, s->bucket_tenant,
+ rgw::lua::context::preRequest, name);
+ if (rc == -ENOENT) {
+ // no script, nothing to do
+ continue;
+ }
+ if (rc < 0) {
+ ldpp_dout(op, 5) << "WARNING: failed to read pre request script in tenant " << s->bucket_tenant
+ << ". error " << rc << dendl;
+ continue;
+ }
+
int script_return_code = 0;
rc = rgw::lua::request::execute(rest, penv.olog.get(), s, op, lua_script, script_return_code);
if (rc < 0) {
- ldpp_dout(op, 5) <<
- "WARNING: failed to execute pre request script. "
- "error: " << rc << dendl;
+ ldpp_dout(op, 5) << "WARNING: failed to execute pre request script in tenant " << s->bucket_tenant
+ << ". error " << rc << dendl;
}
if (script_return_code == -EPERM) {
abort_early(s, op, script_return_code, handler, yield);
}
}
if (!is_health_request) {
- auto [lua_script, rc] = rgw::lua::read_script_or_bytecode(s, penv.lua.manager.get(),
- s->bucket_tenant, s->yield,
- rgw::lua::context::postRequest);
- if (rc == -ENOENT) {
- // no script, nothing to do
- } else if (rc < 0) {
- ldpp_dout(op, 5) <<
- "WARNING: failed to read post request script. "
- "error: " << rc << dendl;
- } else {
+ std::vector<std::string> script_names;
+ const auto rc = rgw::lua::list_scripts(s, penv.lua.manager.get(), s->yield,
+ s->bucket_tenant, rgw::lua::context::postRequest, script_names);
+ // include the unnamed script for backward compatibility
+ if (std::find(script_names.begin(), script_names.end(), "") == script_names.end()) {
+ script_names.insert(script_names.begin(), "");
+ }
+ if (rc < 0 && rc != -ENOENT) {
+ ldpp_dout(op, 5) << "WARNING: failed to list data scripts in tenant " << s->bucket_tenant
+ << " and context " << rgw::lua::to_string(rgw::lua::context::postRequest)
+ << ". error " << rc << dendl;
+ }
+
+ for (const auto& name : script_names) {
+ auto [lua_script, rc] = rgw::lua::read_script_or_bytecode(s, penv.lua.manager.get(),
+ s->yield, s->bucket_tenant,
+ rgw::lua::context::postRequest, name);
+ if (rc == -ENOENT) {
+ // no script, nothing to do
+ continue;
+ }
+ if (rc < 0) {
+ ldpp_dout(op, 5) << "WARNING: failed to read post request script in tenant " << s->bucket_tenant
+ << ". error " << rc << dendl;
+ continue;
+ }
+
rc = rgw::lua::request::execute(rest, penv.olog.get(), s, op, lua_script);
if (rc < 0) {
- ldpp_dout(op, 5) <<
- "WARNING: failed to execute post request script. "
- "error: " << rc << dendl;
+ ldpp_dout(op, 5) << "WARNING: failed to execute post request script in tenant " << s->bucket_tenant
+ << ". error " << rc << dendl;
}
}
}
virtual ~LuaManager() = default;
/** Get a script named with the given key from the backing store */
- virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) = 0;
+ virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) = 0;
+ /** List all scripts named with the given key from the backing store */
+ virtual int list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector<std::string>& scripts) = 0;
/** Get a copy of the lua bytecode if it exists, else the script named with the given key from the backing store */
virtual std::tuple<rgw::lua::LuaCodeType, int> get_script_or_bytecode(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) = 0;
/** Put a script named with the given key to the backing store */
return ret;
}
- int DBLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script)
+ int DBLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script)
+ {
+ return -ENOENT;
+ }
+
+ int DBLuaManager::list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector<std::string>& scripts)
{
return -ENOENT;
}
virtual ~DBLuaManager() = default;
/** Get a script named with the given key from the backing store */
- virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override;
+ virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) override;
+ /** List all scripts named with the given key from the backing store */
+ virtual int list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector<std::string>& scripts) override;
/** Get a ref to the Lua bytecode if it exists, else the script named with the given key from the backing store */
virtual std::tuple<rgw::lua::LuaCodeType, int> get_script_or_bytecode(const DoutPrefixProvider* dpp, optional_yield y,
const std::string& key) override;
canceled, rctx, flags);
}
-int FilterLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y,
+int FilterLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv,
const std::string& key, std::string& script)
{
- return next->get_script(dpp, y, key, script);
+ return next->get_script(dpp, y, objv, key, script);
+}
+
+int FilterLuaManager::list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key,
+ RGWObjVersionTracker* objv, std::vector<std::string>& scripts)
+{
+ return next->list_scripts(dpp, y, list_meta_key, objv, scripts);
}
std::tuple<rgw::lua::LuaCodeType, int> FilterLuaManager::get_script_or_bytecode(const DoutPrefixProvider* dpp, optional_yield y,
FilterLuaManager(std::unique_ptr<LuaManager> _next) : next(std::move(_next)) {}
virtual ~FilterLuaManager() = default;
- virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override;
+ virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) override;
+ virtual int list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector<std::string>& scripts) override;
virtual std::tuple<rgw::lua::LuaCodeType, int> get_script_or_bytecode(const DoutPrefixProvider* dpp, optional_yield y,
const std::string& key) override;
virtual int put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) override;
topic dump dump (in JSON format) all pending bucket notifications of a persistent topic
script put upload a Lua script to a context
script get get the Lua script of a context
+ script list list the Lua scripts of a context
script rm remove the Lua scripts of a context
script-package add add a Lua package to the scripts allowlist
script-package rm remove a Lua package from the scripts allowlist
--skip-zero-entries log show only dumps entries that don't have zero value
in one of the numeric field
--infile=<file> file to read in when setting data
+ --script-name=<script-name> name of the Lua script
--categories=<list> comma separated list of categories, used in usage show
--caps=<caps> list of caps (e.g., "usage=read, write; user=read")
--op-mask=<op-mask> permission of user's operations (e.g., "read, write, delete, *")
def bash(cmd, **kwargs):
log.debug('running command: %s', ' '.join(cmd))
kwargs['stdout'] = subprocess.PIPE
+ kwargs["stderr"] = subprocess.PIPE
+ kwargs["text"] = True
process = subprocess.Popen(cmd, **kwargs)
- s = process.communicate()[0].decode('utf-8')
- return (s, process.returncode)
+ stdout, stderr = process.communicate()
+ return (stdout, stderr, process.returncode)
def admin(args, **kwargs):
secret_key = str(time.time())
uid = 'superman' + str(time.time())
if tenant:
- _, result = admin(['user', 'create', '--uid', uid, '--tenant', tenant, '--access-key', access_key, '--secret-key', secret_key, '--display-name', '"Super Man"'])
+ _, _, result = admin(['user', 'create', '--uid', uid, '--tenant', tenant, '--access-key', access_key, '--secret-key', secret_key, '--display-name', '"Super Man"'])
else:
- _, result = admin(['user', 'create', '--uid', uid, '--access-key', access_key, '--secret-key', secret_key, '--display-name', '"Super Man"'])
+ _, _, result = admin(['user', 'create', '--uid', uid, '--access-key', access_key, '--secret-key', secret_key, '--display-name', '"Super Man"'])
assert result == 0
hostname = get_config_host()
return client
-def put_script(script, context, tenant=None):
+def put_script(script, context, tenant=None, script_name=None):
fp = tempfile.NamedTemporaryFile(mode='w+')
fp.write(script)
fp.flush()
+
+ args = ['script', 'put', '--infile', fp.name, '--context', context]
if tenant:
- result = admin(['script', 'put', '--infile', fp.name, '--context', context, '--tenant', tenant])
- else:
- result = admin(['script', 'put', '--infile', fp.name, '--context', context])
+ args.append('--tenant')
+ args.append(tenant)
+ if script_name:
+ args.append('--script-name')
+ args.append(script_name)
+ result = admin(args)
fp.close()
return result
for context in contexts:
script = 'print("hello from ' + context + '")'
result = put_script(script, context)
- assert result[1] == 0
+ assert result[2] == 0
scripts[context] = script
for context in contexts:
result = admin(['script', 'get', '--context', context])
- assert result[1] == 0
+ assert result[2] == 0
assert result[0].strip() == scripts[context]
for context in contexts:
result = admin(['script', 'rm', '--context', context])
- assert result[1] == 0
+ assert result[2] == 0
for context in contexts:
result = admin(['script', 'get', '--context', context])
- assert result[1] == 0
+ assert result[2] == 0
assert result[0].strip() == 'no script exists for context: ' + context
+@pytest.mark.basic_test
+def test_multi_script_unsupported():
+ contexts = ['getdata', 'putdata']
+ for context in contexts:
+ result = put_script('print("hello")', context, None, 'test')
+ assert result[2] != 0
+ assert f"ERROR: cannot create more than one Lua script in the {context} context; remove the --script-name flag" in result[1]
+
+@pytest.mark.basic_test
+def test_multi_script_management():
+ contexts = ['prerequest', 'postauth', 'postrequest', 'background'] # getdata and putdata contexts cannot use --script-name
+ script_names = ["c", "b", "a"]
+ scripts = {}
+ for context in contexts:
+ for script_name in script_names:
+ script = 'print("hello from ' + '{} ({})'.format(context, script_name) + ' script")'
+ result = put_script(script, context, None, script_name)
+ assert result[2] == 0
+ scripts.setdefault(context, {})[script_name] = script
+ for context in contexts:
+ for script_name in script_names:
+ result = admin(['script', 'get', '--context', context, '--script-name', script_name])
+ assert result[2] == 0
+ assert result[0].strip() == scripts[context][script_name]
+ for context in contexts:
+ result = admin(['script', 'list', '--context', context])
+ assert result[2] == 0
+ assert result[0].strip() == "a\nb\nc"
+ # remove one script and verify the others remain in sorted order
+ for context in contexts:
+ result = admin(['script', 'rm', '--context', context, '--script-name', 'b'])
+ assert result[2] == 0
+ for context in contexts:
+ result = admin(['script', 'get', '--context', context, '--script-name', 'a'])
+ assert result[2] == 0
+ assert result[0].strip() == scripts[context]['a']
+ result = admin(['script', 'get', '--context', context, '--script-name', 'c'])
+ assert result[2] == 0
+ assert result[0].strip() == scripts[context]['c']
+ result = admin(['script', 'list', '--context', context])
+ assert result[2] == 0
+ assert result[0].strip() == "a\nc"
+ for context in contexts:
+ for script_name in ['c', 'a']:
+ result = admin(['script', 'rm', '--context', context, '--script-name', script_name])
+ assert result[2] == 0
+ for context in contexts:
+ for script_name in script_names:
+ result = admin(['script', 'get', '--context', context, '--script-name', script_name])
+ assert result[2] == 0
+ assert result[0].strip() == "'{}' script does not exist in context: ".format(script_name) + context
@pytest.mark.basic_test
def test_script_management_with_tenant():
for t in ['', tenant]:
script = 'print("hello from ' + context + ' and ' + tenant + '")'
result = put_script(script, context, t)
- assert result[1] == 0
+ assert result[2] == 0
scripts[context+t] = script
for context in contexts:
result = admin(['script', 'get', '--context', context])
- assert result[1] == 0
+ assert result[2] == 0
assert result[0].strip(), scripts[context]
result = admin(['script', 'rm', '--context', context])
- assert result[1] == 0
+ assert result[2] == 0
result = admin(['script', 'get', '--context', context])
- assert result[1] == 0
+ assert result[2] == 0
assert result[0].strip(), 'no script exists for context: ' + context
result = admin(['script', 'get', '--context', context, '--tenant', tenant])
- assert result[1] == 0
+ assert result[2] == 0
assert result[0].strip(), scripts[context+tenant]
result = admin(['script', 'rm', '--context', context, '--tenant', tenant])
- assert result[1] == 0
+ assert result[2] == 0
result = admin(['script', 'get', '--context', context, '--tenant', tenant])
- assert result[1] == 0
+ assert result[2] == 0
assert result[0].strip(), 'no script exists for context: ' + context + ' in tenant: ' + tenant
+@pytest.mark.basic_test
+def test_multi_script_management_with_tenant():
+ tenant = 'mytenant'
+ conn2 = another_user(tenant)
+ contexts = ['prerequest', 'postauth', 'postrequest'] # background context cannot use a tenant, getdata and putdata contexts cannot use --script-name
+ script_names = ["c", "b", "a"]
+ scripts = {}
+ for context in contexts:
+ for t in ['', tenant]:
+ for script_name in script_names:
+ script = 'print("hello from ' + '{} ({})'.format(context, script_name) + ' and ' + tenant + '")'
+ result = put_script(script, context, t, script_name)
+ assert result[2] == 0
+ scripts.setdefault(context+t, {})[script_name] = script
+ for context in contexts:
+ for script_name in script_names:
+ result = admin(['script', 'get', '--context', context, '--script-name', script_name])
+ assert result[2] == 0
+ assert result[0].strip(), scripts[context][script_name]
+ result = admin(['script', 'rm', '--context', context, '--script-name', script_name])
+ assert result[2] == 0
+ result = admin(['script', 'get', '--context', context, '--script-name', script_name])
+ assert result[2] == 0
+ assert result[0].strip(), "'{}' script does not exist in context: ".format(script_name) + context
+ result = admin(['script', 'get', '--context', context, '--tenant', tenant, '--script-name', script_name])
+ assert result[2] == 0
+ assert result[0].strip(), scripts[context+tenant][script_name]
+ result = admin(['script', 'rm', '--context', context, '--tenant', tenant, '--script-name', script_name])
+ assert result[2] == 0
+ result = admin(['script', 'get', '--context', context, '--tenant', tenant, '--script-name', script_name])
+ assert result[2] == 0
+ assert result[0].strip(), "'{}' script does not exist in context: ".format(script_name) + context + ' in tenant: ' + tenant
@pytest.mark.request_test
def test_put_obj():
'''
context = "prerequest"
result = put_script(script, context)
- assert result[1] == 0
+ assert result[2] == 0
conn = connection()
bucket_name = gen_bucket_name()
contexts = ['prerequest', 'postauth', 'postrequest', 'getdata', 'putdata']
for context in contexts:
result = admin(['script', 'rm', '--context', context])
- assert result[1] == 0
+ assert result[2] == 0
@pytest.mark.example_test
for context in contexts:
footer = '\nRGWDebugLog("context was: '+context+'\\n\\n")'
result = put_script(script+footer, context)
- assert result[1] == 0
+ assert result[2] == 0
conn = connection()
bucket_name = gen_bucket_name()
contexts = ['prerequest', 'postauth', 'postrequest', 'getdata', 'putdata']
for context in contexts:
result = admin(['script', 'rm', '--context', context])
- assert result[1] == 0
+ assert result[2] == 0
@pytest.mark.example_test
'''
result = put_script(script, "putdata")
- assert result[1] == 0
+ assert result[2] == 0
conn = connection()
bucket_name = gen_bucket_name()
contexts = ['prerequest', 'postauth', 'postrequest', 'background', 'getdata', 'putdata']
for context in contexts:
result = admin(['script', 'rm', '--context', context])
- assert result[1] == 0
+ assert result[2] == 0
@pytest.mark.example_test
'''.format(socket_path)
result = admin(['script-package', 'add', '--package=lua-cjson', '--allow-compilation'])
- assert result[1] == 0
+ assert result[2] == 0
result = admin(['script-package', 'add', '--package=luasocket', '--allow-compilation'])
- assert result[1] == 0
+ assert result[2] == 0
result = admin(['script-package', 'reload'])
- assert result[1] == 0
+ assert result[2] == 0
result = put_script(script, "postrequest")
- assert result[1] == 0
+ assert result[2] == 0
socket_server = UnixSocket(socket_path)
try:
contexts = ['prerequest', 'postauth', 'postrequest', 'background', 'getdata', 'putdata']
for context in contexts:
result = admin(['script', 'rm', '--context', context])
- assert result[1] == 0
+ assert result[2] == 0
+
+@pytest.mark.example_test
+def test_multi_access_log():
+ bucket_name = gen_bucket_name()
+ socket_path = '/tmp/'+bucket_name
+
+ result = admin(['script-package', 'add', '--package=lua-cjson', '--allow-compilation'])
+ assert result[2] == 0
+ result = admin(['script-package', 'add', '--package=luasocket', '--allow-compilation'])
+ assert result[2] == 0
+ result = admin(['script-package', 'reload'])
+ assert result[2] == 0
+
+ script_names = ["b", "a"]
+ for name in script_names:
+ delay = 0.5*script_names.index(name)
+ script = '''
+if Request.RGWOp == "get_obj" then
+ local json = require("cjson")
+ local socket = require("socket")
+ local unix = require("socket.unix")
+ local s = unix()
+ E = {{}}
+
+ -- sleep 0.5s for a and 0s for b
+ -- we should expect script a to execute first despite taking longer to connect to the socket
+ {}
+ msg = {{bucket = (Request.Bucket or (Request.CopyFrom or E).Bucket).Name,
+ object = Request.Object.Name,
+ time = Request.Time,
+ script_name = "{}",
+ operation = Request.RGWOp,
+ http_status = Request.Response.HTTPStatusCode,
+ error_code = Request.Response.HTTPStatus,
+ object_size = Request.Object.Size,
+ trans_id = Request.TransactionId}}
+ assert(s:connect("{}"))
+ s:send(json.encode(msg).."\\n")
+ s:close()
+end
+ '''.format("" if delay == 0 else "socket.sleep({})".format(delay), name, socket_path)
+ result = put_script(script, "postrequest", None, name)
+ assert result[2] == 0
+
+ socket_server = UnixSocket(socket_path)
+ try:
+ conn = connection()
+ # create bucket
+ bucket = conn.create_bucket(Bucket=bucket_name)
+ # create objects in the bucket (async)
+ number_of_objects = 3
+ keys = []
+ for i in range(number_of_objects):
+ content = str(os.urandom(1024)).encode("ascii")
+ key = str(i)
+ conn.put_object(Body=content, Bucket=bucket_name, Key=key)
+ keys.append(key)
+
+ for key in conn.list_objects(Bucket=bucket_name)['Contents']:
+ conn.get_object(Bucket=bucket_name, Key=key['Key'])
+
+ time.sleep(3)
+ events = {}
+ for event in socket_server.get_and_reset_events():
+ assert event['bucket'] == bucket_name
+ events[event['object']] = events.get(event['object'], []) + [event['script_name']]
+
+ assert len(events) == number_of_objects
+ sorted_script_names = sorted(script_names)
+ for obj in events:
+ assert sorted_script_names == events[obj]
+
+ finally:
+ socket_server.shutdown()
+ delete_all_objects(conn, bucket_name)
+ conn.delete_bucket(Bucket=bucket_name)
+ contexts = ['prerequest', 'postrequest', 'background', 'getdata', 'putdata']
+ for context in contexts:
+ result = admin(['script', 'rm', '--context', context])
+ assert result[2] == 0
@pytest.mark.example_test
def test_interrupt_request():
conn.create_bucket(Bucket=bucket_name)
result = put_script(script, "prerequest")
- assert result[1] == 0
+ assert result[2] == 0
key = "hello"
try:
except Exception as e:
pass
- out, err = admin(['script', 'rm', '--context', 'prerequest'])
+ out, _, err = admin(['script', 'rm', '--context', 'prerequest'])
assert err == 0
try:
TestLuaManager() {
rgw_perf_start(g_ceph_context);
}
- int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override {
+ int get_script(const DoutPrefixProvider* dpp, optional_yield y, RGWObjVersionTracker* objv, const std::string& key, std::string& script) override {
std::this_thread::sleep_for(std::chrono::seconds(read_time));
script = lua_script;
return 0;
}
+ int list_scripts(const DoutPrefixProvider* dpp, optional_yield y, const std::string& list_meta_key, RGWObjVersionTracker* objv, std::vector<std::string>& scripts) override {
+ return -ENOENT;
+ }
std::tuple<rgw::lua::LuaCodeType, int> get_script_or_bytecode(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) override {
std::this_thread::sleep_for(std::chrono::seconds(read_time));
return std::make_tuple(lua_script, 0);
projects / ceph-ci.git / commitdiff