The request context script can also access fields in the request and modify certain fields, as well as the `Global RGW Table`_.
The data context script can access the content of the object as well as the request fields and the `Global RGW Table`_.
All Lua language features can be used in all contexts.
+An execution of a script in a context can use up to 500K byte of memory. This include all libraries used by Lua, but not the memory which is managed by the RGW itself, and may be accessed from Lua.
+To change this default value, use the ``rgw_lua_max_memory_per_state`` configuration parameter. Note that the basic overhead of Lua with its standard libraries is ~32K bytes. To disable the limit, use zero or a negative number.
By default, all Lua standard libraries are available in the script, however, in order to allow for other Lua modules to be used in the script, we support adding packages to an allowlist:
flags:
- startup
with_legacy: true
-
+- name: rgw_lua_max_memory_per_state
+ type: uint
+ level: advanced
+ desc: Max size of memory used by a single lua state
+ long_desc: This is the maximum size in bytes that a lua state can allocate for its own use.
+ Note that this does not include any memory that can be accessed from lua, but managed by the RGW.
+ If not set, it would use a default of 128K. If set to zero, the amount of memory would
+ only be limited by the system.
+ default: 128000
+ services:
+ - rgw
+ with_legacy: true
bool verify(const std::string& script, std::string& err_msg)
{
- lua_State *L = luaL_newstate();
- lua_state_guard guard(L);
- open_standard_libs(L);
+ lua_state_guard lguard(0, nullptr); // no memory limit, sice we don't execute the script
+ auto L = lguard.get();
try {
+ open_standard_libs(L);
if (luaL_loadstring(L, script.c_str()) != LUA_OK) {
err_msg.assign(lua_tostring(L, -1));
return false;
//(2) Executes the script
//(3) Sleep (configurable)
void Background::run() {
- lua_State* const L = luaL_newstate();
- rgw::lua::lua_state_guard lguard(L);
- open_standard_libs(L);
- set_package_path(L, luarocks_path);
- create_debug_action(L, cct);
- create_background_metatable(L);
const DoutPrefixProvider* const dpp = &dp;
+ lua_state_guard lguard(cct->_conf->rgw_lua_max_memory_per_state, dpp);
+ auto L = lguard.get();
+ if (!L) {
+ ldpp_dout(dpp, 1) << "Failed to create state for Lua background thread" << dendl;
+ return;
+ }
+ try {
+ open_standard_libs(L);
+ set_package_path(L, luarocks_path);
+ create_debug_action(L, cct);
+ create_background_metatable(L);
+ } catch (const std::runtime_error& e) {
+ ldpp_dout(dpp, 1) << "Failed to create initial setup of Lua background thread. error "
+ << e.what() << dendl;
+ return;
+ }
while (!stopped) {
if (paused) {
ldpp_dout(dpp, 1) << "Lua ERROR: " << err << dendl;
failed = true;
}
- } catch (const std::exception& e) {
+ } catch (const std::runtime_error& e) {
ldpp_dout(dpp, 1) << "Lua ERROR: " << e.what() << dendl;
failed = true;
}
};
int RGWObjFilter::execute(bufferlist& bl, off_t offset, const char* op_name) const {
- auto L = luaL_newstate();
- lua_state_guard lguard(L);
+ lua_state_guard lguard(s->cct->_conf->rgw_lua_max_memory_per_state, s);
+ auto L = lguard.get();
+ if (!L) {
+ ldpp_dout(s, 1) << "Failed to create state for Lua data context" << dendl;
+ return -ENOMEM;
+ }
- open_standard_libs(L);
+ try {
+ open_standard_libs(L);
- create_debug_action(L, s->cct);
+ create_debug_action(L, s->cct);
- // create the "Data" table
- create_metatable<BufferlistMetaTable>(L, true, &bl);
- lua_getglobal(L, BufferlistMetaTable::TableName().c_str());
- ceph_assert(lua_istable(L, -1));
+ // create the "Data" table
+ create_metatable<BufferlistMetaTable>(L, true, &bl);
+ lua_getglobal(L, BufferlistMetaTable::TableName().c_str());
+ ceph_assert(lua_istable(L, -1));
- // create the "Request" table
- request::create_top_metatable(L, s, op_name);
+ // create the "Request" table
+ request::create_top_metatable(L, s, op_name);
- // create the "Offset" variable
- lua_pushinteger(L, offset);
- lua_setglobal(L, "Offset");
+ // create the "Offset" variable
+ lua_pushinteger(L, offset);
+ lua_setglobal(L, "Offset");
- if (s->penv.lua.background) {
- // create the "RGW" table
- s->penv.lua.background->create_background_metatable(L);
- lua_getglobal(L, rgw::lua::RGWTable::TableName().c_str());
- ceph_assert(lua_istable(L, -1));
- }
+ if (s->penv.lua.background) {
+ // create the "RGW" table
+ s->penv.lua.background->create_background_metatable(L);
+ lua_getglobal(L, rgw::lua::RGWTable::TableName().c_str());
+ ceph_assert(lua_istable(L, -1));
+ }
- try {
// execute the lua script
if (luaL_dostring(L, script.c_str()) != LUA_OK) {
const std::string err(lua_tostring(L, -1));
RGWOp* op,
const std::string& script)
{
- auto L = luaL_newstate();
+ lua_state_guard lguard(s->cct->_conf->rgw_lua_max_memory_per_state, s);
+ auto L = lguard.get();
+ if (!L) {
+ ldpp_dout(s, 1) << "Failed to create state for Lua request context" << dendl;
+ return -ENOMEM;
+ }
const char* op_name = op ? op->name() : "Unknown";
- lua_state_guard lguard(L);
-
- open_standard_libs(L);
- set_package_path(L, s->penv.lua.luarocks_path);
- create_debug_action(L, s->cct);
-
- create_metatable<RequestMetaTable>(L, true, s, const_cast<char*>(op_name));
+ int rc = 0;
+ try {
+ open_standard_libs(L);
+ set_package_path(L, s->penv.lua.luarocks_path);
- lua_getglobal(L, RequestMetaTable::TableName().c_str());
- ceph_assert(lua_istable(L, -1));
+ create_debug_action(L, s->cct);
+
+ create_metatable<RequestMetaTable>(L, true, s, const_cast<char*>(op_name));
- // add the ops log action
- pushstring(L, RequestLogAction);
- lua_pushlightuserdata(L, rest);
- lua_pushlightuserdata(L, olog);
- lua_pushlightuserdata(L, s);
- lua_pushlightuserdata(L, op);
- lua_pushcclosure(L, RequestLog, FOUR_UPVALS);
- lua_rawset(L, -3);
-
- if (s->penv.lua.background) {
- s->penv.lua.background->create_background_metatable(L);
- lua_getglobal(L, rgw::lua::RGWTable::TableName().c_str());
+ lua_getglobal(L, RequestMetaTable::TableName().c_str());
ceph_assert(lua_istable(L, -1));
- }
- int rc = 0;
- try {
+ // add the ops log action
+ pushstring(L, RequestLogAction);
+ lua_pushlightuserdata(L, rest);
+ lua_pushlightuserdata(L, olog);
+ lua_pushlightuserdata(L, s);
+ lua_pushlightuserdata(L, op);
+ lua_pushcclosure(L, RequestLog, FOUR_UPVALS);
+ lua_rawset(L, -3);
+
+ if (s->penv.lua.background) {
+ s->penv.lua.background->create_background_metatable(L);
+ lua_getglobal(L, rgw::lua::RGWTable::TableName().c_str());
+ ceph_assert(lua_istable(L, -1));
+ }
+
// execute the lua script
if (luaL_dostring(L, script.c_str()) != LUA_OK) {
const std::string err(lua_tostring(L, -1));
#include <string>
#include <lua.hpp>
#include "common/ceph_context.h"
-#include "common/dout.h"
+#include "common/debug.h"
#include "rgw_lua_utils.h"
#include "rgw_lua_version.h"
if (install_dir.empty()) {
return;
}
- lua_getglobal(L, "package");
- if (!lua_istable(L, -1)) {
+ if (lua_getglobal(L, "package") != LUA_TTABLE) {
return;
}
const auto path = install_dir+"/share/lua/"+CEPH_LUA_VERSION+"/?.lua";
unsetglobal(L, "dofile");
unsetglobal(L, "debug");
// remove os.exit()
- lua_getglobal(L, "os");
- lua_pushstring(L, "exit");
- lua_pushnil(L);
- lua_settable(L, -3);
+ if (lua_getglobal(L, "os") == LUA_TTABLE) {
+ lua_pushstring(L, "exit");
+ lua_pushnil(L);
+ lua_settable(L, -3);
+ }
+}
+
+// allocator function that verifies against maximum allowed memory value
+void* allocator(void* ud, void* ptr, std::size_t osize, std::size_t nsize) {
+ auto mem = reinterpret_cast<std::size_t*>(ud); // remaining memory
+ // free memory
+ if (nsize == 0) {
+ if (mem && ptr) {
+ *mem += osize;
+ }
+ free(ptr);
+ return nullptr;
+ }
+ // re/alloc memory
+ if (mem) {
+ const std::size_t realloc_size = ptr ? osize : 0;
+ if (nsize > realloc_size && (nsize - realloc_size) > *mem) {
+ return nullptr;
+ }
+ *mem += realloc_size;
+ *mem -= nsize;
+ }
+ return realloc(ptr, nsize);
+}
+
+// create new lua state together with its memory counter
+lua_State* newstate(int max_memory) {
+ std::size_t* remaining_memory = nullptr;
+ if (max_memory > 0) {
+ remaining_memory = new std::size_t(max_memory);
+ }
+ lua_State* L = lua_newstate(allocator, remaining_memory);
+ if (!L) {
+ delete remaining_memory;
+ remaining_memory = nullptr;
+ }
+ if (L) {
+ lua_atpanic(L, [](lua_State* L) -> int {
+ const char* msg = lua_tostring(L, -1);
+ if (msg == nullptr) msg = "error object is not a string";
+ throw std::runtime_error(msg);
+ });
+ }
+ return L;
+}
+
+// lua_state_guard ctor
+lua_state_guard::lua_state_guard(std::size_t _max_memory, const DoutPrefixProvider* _dpp) :
+ max_memory(_max_memory),
+ dpp(_dpp),
+ state(newstate(_max_memory)) {
+ if (state && perfcounter) {
+ perfcounter->inc(l_rgw_lua_current_vms, 1);
+ }
+}
+
+// lua_state_guard dtor
+lua_state_guard::~lua_state_guard() {
+ lua_State* L = state;
+ if (!L) {
+ return;
+ }
+ void* ud = nullptr;
+ lua_getallocf(L, &ud);
+ auto remaining_memory = static_cast<std::size_t*>(ud);
+
+ if (remaining_memory) {
+ const auto used_memory = max_memory - *remaining_memory;
+ ldpp_dout(dpp, 20) << "Lua is using: " << used_memory <<
+ " bytes (" << 100.0*used_memory/max_memory << "%)" << dendl;
+ // dont limit memory during cleanup
+ *remaining_memory = 0;
+ }
+ try {
+ lua_close(L);
+ } catch (const std::runtime_error& e) {
+ ldpp_dout(dpp, 20) << "Lua cleanup failed with: " << e.what() << dendl;
+ }
+
+ // TODO: use max_memory and remaining memory to check for leaks
+ // this could be done only if we don't zero the remianing memory during clanup
+
+ delete remaining_memory;
+ if (perfcounter) {
+ perfcounter->dec(l_rgw_lua_current_vms, 1);
+ }
}
} // namespace rgw::lua
struct is_variant<std::variant<Ts...>> :
std::true_type {};
+class DoutPrefixProvider;
+
namespace rgw::lua {
// push ceph time in string format: "%Y-%m-%d %H:%M:%S"
template <typename CephTime>
-void pushtime(lua_State* L, const CephTime& tp)
+const char* pushtime(lua_State* L, const CephTime& tp)
{
const auto tt = CephTime::clock::to_time_t(tp);
const auto tm = *std::localtime(&tt);
char buff[64];
- std::strftime(buff, sizeof(buff), "%Y-%m-%d %H:%M:%S", &tm);
- lua_pushstring(L, buff);
+ const auto len = std::strftime(buff, sizeof(buff), "%Y-%m-%d %H:%M:%S", &tm);
+ return lua_pushlstring(L, buff, len+1);
}
-inline void pushstring(lua_State* L, std::string_view str)
+static inline const char* pushstring(lua_State* L, std::string_view str)
{
- lua_pushlstring(L, str.data(), str.size());
+ return lua_pushlstring(L, str.data(), str.size());
}
inline void pushvalue(lua_State* L, const std::string& value) {
void stack_dump(lua_State* L);
class lua_state_guard {
- lua_State* l;
+ const std::size_t max_memory;
+ const DoutPrefixProvider* const dpp;
+ lua_State* const state;
public:
- lua_state_guard(lua_State* _l) : l(_l) {
- if (perfcounter) {
- perfcounter->inc(l_rgw_lua_current_vms, 1);
- }
- }
- ~lua_state_guard() {
- lua_close(l);
- if (perfcounter) {
- perfcounter->dec(l_rgw_lua_current_vms, 1);
- }
- }
- void reset(lua_State* _l=nullptr) {l = _l;}
+ lua_state_guard(std::size_t _max_memory, const DoutPrefixProvider* _dpp);
+ ~lua_state_guard();
+ lua_State* get() { return state; }
};
+int dostring(lua_State* L, const char* str);
+
constexpr const int MAX_LUA_VALUE_SIZE = 1000;
constexpr const int MAX_LUA_KEY_ENTRIES = 100000;
ASSERT_NE(rc, 0);
}
+TEST(TestRGWLua, MemoryLimit)
+{
+ std::string script = "print(\"hello world\")";
+
+ DEFINE_REQ_STATE;
+
+ // memory should be sufficient
+ s.cct->_conf->rgw_lua_max_memory_per_state = 1024*32;
+ auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
+ ASSERT_EQ(rc, 0);
+
+ // no memory limit
+ s.cct->_conf->rgw_lua_max_memory_per_state = 0;
+ rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
+ ASSERT_EQ(rc, 0);
+
+ // not enough memory to start lua
+ s.cct->_conf->rgw_lua_max_memory_per_state = 2048;
+ rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
+ ASSERT_NE(rc, 0);
+
+ // not enough memory for initial setup
+ s.cct->_conf->rgw_lua_max_memory_per_state = 1024*16;
+ rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
+ ASSERT_NE(rc, 0);
+
+ // not enough memory for the script
+ script = R"(
+ t = {}
+ for i = 1,1000 do
+ table.insert(t, i)
+ end
+ )";
+ s.cct->_conf->rgw_lua_max_memory_per_state = 1024*32;
+ rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
+ ASSERT_NE(rc, 0);
+}
+
projects / ceph.git / commitdiff