From: Yuval Lifshitz Date: Wed, 18 May 2022 14:18:14 +0000 (+0300) Subject: rgw/lua: support multiple types in lua background table X-Git-Tag: v18.0.0~711^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=402c35cf97256cbb29885cfe8b4106e9d2e2c962;p=ceph-ci.git rgw/lua: support multiple types in lua background table Signed-off-by: Yuval Lifshitz --- diff --git a/src/rgw/rgw_lua_background.cc b/src/rgw/rgw_lua_background.cc index ab4f9f5fc4d..21c5fcc1a3d 100644 --- a/src/rgw/rgw_lua_background.cc +++ b/src/rgw/rgw_lua_background.cc @@ -64,9 +64,9 @@ int Background::read_script() { return rgw::lua::read_script(&dp, store, tenant, null_yield, rgw::lua::context::background, rgw_script); } -const std::string Background::empty_table_value; +const BackgroundMapValue Background::empty_table_value; -const std::string& Background::get_table_value(const std::string& key) const { +const BackgroundMapValue& Background::get_table_value(const std::string& key) const { std::unique_lock cond_lock(table_mutex); const auto it = rgw_map.find(key); if (it == rgw_map.end()) { @@ -75,11 +75,6 @@ const std::string& Background::get_table_value(const std::string& key) const { return it->second; } -void Background::put_table_value(const std::string& key, const std::string& value) { - std::unique_lock cond_lock(table_mutex); - rgw_map[key] = value; -} - //(1) Loads the script from the object if not paused //(2) Executes the script //(3) Sleep (configurable) diff --git a/src/rgw/rgw_lua_background.h b/src/rgw/rgw_lua_background.h index 3baca6b321f..e2fa9a195e9 100644 --- a/src/rgw/rgw_lua_background.h +++ b/src/rgw/rgw_lua_background.h @@ -2,6 +2,8 @@ #include "common/dout.h" #include "rgw_common.h" #include +#include +#include #include "rgw_lua_utils.h" #include "rgw_realm_reloader.h" @@ -11,26 +13,142 @@ namespace rgw::lua { constexpr const int INIT_EXECUTE_INTERVAL = 5; //Writeable meta table named RGW with mutex protection -using BackgroundMap = std::unordered_map; -struct RGWTable : StringMapMetaTable> { - static std::string TableName() {return "RGW";} - static std::string Name() {return TableName() + "Meta";} - static int IndexClosure(lua_State* L) { - auto& mtx = *reinterpret_cast(lua_touserdata(L, lua_upvalueindex(2))); - std::lock_guard l(mtx); - return StringMapMetaTable::IndexClosure(L); +using BackgroundMapValue = std::variant; +using BackgroundMap = std::unordered_map; + +inline void pushvalue(lua_State* L, const std::string& value) { + pushstring(L, value); +} + +inline void pushvalue(lua_State* L, int64_t value) { + lua_pushinteger(L, value); +} + +inline void pushvalue(lua_State* L, double value) { + lua_pushnumber(L, value); +} + +inline void pushvalue(lua_State* L, bool value) { + lua_pushboolean(L, value); +} + +struct RGWTable : EmptyMetaTable { + static std::string TableName() {return "RGW";} + static std::string Name() {return TableName() + "Meta";} + + static int IndexClosure(lua_State* L) { + auto& mtx = *reinterpret_cast(lua_touserdata(L, lua_upvalueindex(2))); + const auto map = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(1))); + const char* index = luaL_checkstring(L, 2); + + std::lock_guard l(mtx); + + const auto it = map->find(std::string(index)); + if (it == map->end()) { + lua_pushnil(L); + } else { + std::visit([L](auto&& value) { pushvalue(L, value); }, it->second); + } + return ONE_RETURNVAL; + } + + static int LenClosure(lua_State* L) { + auto& mtx = *reinterpret_cast(lua_touserdata(L, lua_upvalueindex(2))); + const auto map = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(1))); + + std::lock_guard l(mtx); + + lua_pushinteger(L, map->size()); + + return ONE_RETURNVAL; + } + + static int NewIndexClosure(lua_State* L) { + auto& mtx = *reinterpret_cast(lua_touserdata(L, lua_upvalueindex(2))); + const auto map = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(1))); + const auto index = luaL_checkstring(L, 2); + + std::unique_lock l(mtx); + + size_t len; + BackgroundMapValue value; + const int value_type = lua_type(L, 3); + + switch (value_type) { + case LUA_TNIL: + map->erase(std::string(index)); + return NO_RETURNVAL; + case LUA_TBOOLEAN: + value = static_cast(lua_toboolean(L, 3)); + len = sizeof(bool); + break; + case LUA_TNUMBER: + if (lua_isinteger(L, 3)) { + value = lua_tointeger(L, 3); + len = sizeof(int64_t); + } else { + value = lua_tonumber(L, 3); + len = sizeof(double); + } + break; + case LUA_TSTRING: + value = lua_tolstring(L, 3, &len); + break; + default: + l.unlock(); + return luaL_error(L, "unsupported value type for RGW table"); + } + + if (len + strnlen(index, MAX_LUA_VALUE_SIZE) + > MAX_LUA_VALUE_SIZE) { + return luaL_error(L, "Lua maximum size of entry limit exceeded"); + } else if (map->size() > MAX_LUA_KEY_ENTRIES) { + l.unlock(); + return luaL_error(L, "Lua max number of entries limit exceeded"); + } else { + map->insert_or_assign(index, value); } - static int LenClosure(lua_State* L) { - auto& mtx = *reinterpret_cast(lua_touserdata(L, lua_upvalueindex(2))); - std::lock_guard l(mtx); - return StringMapMetaTable::LenClosure(L); + + return NO_RETURNVAL; + } + + static int PairsClosure(lua_State* L) { + auto map = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(1))); + ceph_assert(map); + lua_pushlightuserdata(L, map); + lua_pushcclosure(L, stateless_iter, ONE_UPVAL); // push the stateless iterator function + lua_pushnil(L); // indicate this is the first call + // return stateless_iter, nil + + return TWO_RETURNVALS; + } + + static int stateless_iter(lua_State* L) { + // based on: http://lua-users.org/wiki/GeneralizedPairsAndIpairs + auto map = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(1))); + typename BackgroundMap::const_iterator next_it; + if (lua_isnil(L, -1)) { + next_it = map->begin(); + } else { + const char* index = luaL_checkstring(L, 2); + const auto it = map->find(std::string(index)); + ceph_assert(it != map->end()); + next_it = std::next(it); } - static int NewIndexClosure(lua_State* L) { - auto& mtx = *reinterpret_cast(lua_touserdata(L, lua_upvalueindex(2))); - std::lock_guard l(mtx); - return StringMapMetaTable::NewIndexClosure(L); + + if (next_it == map->end()) { + // index of the last element was provided + lua_pushnil(L); + lua_pushnil(L); + // return nil, nil + } else { + pushstring(L, next_it->first); + std::visit([L](auto&& value) { pushvalue(L, value); }, next_it->second); + // return key, value } + + return TWO_RETURNVALS; + } }; class Background : public RGWRealmReloader::Pauser { @@ -50,7 +168,7 @@ private: std::mutex cond_mutex; std::mutex pause_mutex; std::condition_variable cond; - static const std::string empty_table_value; + static const BackgroundMapValue empty_table_value; void run(); @@ -68,8 +186,12 @@ public: void start(); void shutdown(); void create_background_metatable(lua_State* L); - const std::string& get_table_value(const std::string& key) const; - void put_table_value(const std::string& key, const std::string& value); + const BackgroundMapValue& get_table_value(const std::string& key) const; + template + void put_table_value(const std::string& key, T value) { + std::unique_lock cond_lock(table_mutex); + rgw_map[key] = value; + } void pause() override; void resume(rgw::sal::Store* _store) override; diff --git a/src/rgw/rgw_lua_utils.h b/src/rgw/rgw_lua_utils.h index 4d1e7510d10..ba582ee8756 100644 --- a/src/rgw/rgw_lua_utils.h +++ b/src/rgw/rgw_lua_utils.h @@ -1,6 +1,8 @@ #pragma once +#include #include +#include #include #include #include @@ -208,7 +210,7 @@ void open_standard_libs(lua_State* L); typedef int MetaTableClosure(lua_State* L); -template +template> int StringMapWriteableNewIndex(lua_State* L) { const auto map = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(1))); diff --git a/src/test/rgw/test_rgw_lua.cc b/src/test/rgw/test_rgw_lua.cc index 86bd79250a6..1d386197127 100644 --- a/src/test/rgw/test_rgw_lua.cc +++ b/src/test/rgw/test_rgw_lua.cc @@ -681,7 +681,17 @@ TEST(TestRGWLuaBackground, Start) } -constexpr auto wait_time = std::chrono::seconds(2); +constexpr auto wait_time = std::chrono::seconds(3); + +template +const T& get_table_value(const TestBackground& b, const std::string& index) { + try { + return std::get(b.get_table_value(index)); + } catch (std::bad_variant_access const& ex) { + std::cout << "expected RGW[" << index << "] to be: " << typeid(T).name() << std::endl; + throw(ex); + } +} TEST(TestRGWLuaBackground, Script) { @@ -694,7 +704,7 @@ TEST(TestRGWLuaBackground, Script) TestBackground lua_background(script); lua_background.start(); std::this_thread::sleep_for(wait_time); - EXPECT_EQ(lua_background.get_table_value("hello"), "world"); + EXPECT_EQ(get_table_value(lua_background, "hello"), "world"); } TEST(TestRGWLuaBackground, RequestScript) @@ -722,11 +732,11 @@ TEST(TestRGWLuaBackground, RequestScript) lua_background.pause(); const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", request_script, &lua_background); ASSERT_EQ(rc, 0); - EXPECT_EQ(lua_background.get_table_value("hello"), "from request"); + EXPECT_EQ(get_table_value(lua_background, "hello"), "from request"); // now we resume and let the background set the value lua_background.resume(nullptr); std::this_thread::sleep_for(wait_time); - EXPECT_EQ(lua_background.get_table_value("hello"), "from background"); + EXPECT_EQ(get_table_value(lua_background, "hello"), "from background"); } TEST(TestRGWLuaBackground, Pause) @@ -744,12 +754,12 @@ TEST(TestRGWLuaBackground, Pause) TestBackground lua_background(script); lua_background.start(); std::this_thread::sleep_for(wait_time); - const auto value_len = lua_background.get_table_value("hello").size(); + const auto value_len = get_table_value(lua_background, "hello").size(); EXPECT_GT(value_len, 0); lua_background.pause(); std::this_thread::sleep_for(wait_time); // no change in len - EXPECT_EQ(value_len, lua_background.get_table_value("hello").size()); + EXPECT_EQ(value_len, get_table_value(lua_background, "hello").size()); } TEST(TestRGWLuaBackground, PauseWhileReading) @@ -769,12 +779,12 @@ TEST(TestRGWLuaBackground, PauseWhileReading) TestBackground lua_background(script, 2); lua_background.start(); std::this_thread::sleep_for(long_wait_time); - const auto value_len = lua_background.get_table_value("hello").size(); + const auto value_len = get_table_value(lua_background, "hello").size(); EXPECT_GT(value_len, 0); lua_background.pause(); std::this_thread::sleep_for(long_wait_time); // one execution might occur after pause - EXPECT_TRUE(value_len + 1 >= lua_background.get_table_value("hello").size()); + EXPECT_TRUE(value_len + 1 >= get_table_value(lua_background, "hello").size()); } TEST(TestRGWLuaBackground, ReadWhilePaused) @@ -789,10 +799,10 @@ TEST(TestRGWLuaBackground, ReadWhilePaused) lua_background.pause(); lua_background.start(); std::this_thread::sleep_for(wait_time); - EXPECT_EQ(lua_background.get_table_value("hello"), ""); + EXPECT_EQ(get_table_value(lua_background, "hello"), ""); lua_background.resume(nullptr); std::this_thread::sleep_for(wait_time); - EXPECT_EQ(lua_background.get_table_value("hello"), "world"); + EXPECT_EQ(get_table_value(lua_background, "hello"), "world"); } TEST(TestRGWLuaBackground, PauseResume) @@ -810,16 +820,16 @@ TEST(TestRGWLuaBackground, PauseResume) TestBackground lua_background(script); lua_background.start(); std::this_thread::sleep_for(wait_time); - const auto value_len = lua_background.get_table_value("hello").size(); + const auto value_len = get_table_value(lua_background, "hello").size(); EXPECT_GT(value_len, 0); lua_background.pause(); std::this_thread::sleep_for(wait_time); // no change in len - EXPECT_EQ(value_len, lua_background.get_table_value("hello").size()); + EXPECT_EQ(value_len, get_table_value(lua_background, "hello").size()); lua_background.resume(nullptr); std::this_thread::sleep_for(wait_time); // should be a change in len - EXPECT_GT(lua_background.get_table_value("hello").size(), value_len); + EXPECT_GT(get_table_value(lua_background, "hello").size(), value_len); } TEST(TestRGWLuaBackground, MultipleStarts) @@ -837,7 +847,7 @@ TEST(TestRGWLuaBackground, MultipleStarts) TestBackground lua_background(script); lua_background.start(); std::this_thread::sleep_for(wait_time); - const auto value_len = lua_background.get_table_value("hello").size(); + const auto value_len = get_table_value(lua_background, "hello").size(); EXPECT_GT(value_len, 0); lua_background.start(); lua_background.shutdown(); @@ -846,6 +856,170 @@ TEST(TestRGWLuaBackground, MultipleStarts) lua_background.start(); std::this_thread::sleep_for(wait_time); // should be a change in len - EXPECT_GT(lua_background.get_table_value("hello").size(), value_len); + EXPECT_GT(get_table_value(lua_background, "hello").size(), value_len); +} + +TEST(TestRGWLuaBackground, TableValues) +{ + TestBackground lua_background(""); + + const std::string request_script = R"( + RGW["key1"] = "string value" + RGW["key2"] = 42 + RGW["key3"] = 42.2 + RGW["key4"] = true + )"; + + DEFINE_REQ_STATE; + + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", request_script, &lua_background); + ASSERT_EQ(rc, 0); + EXPECT_EQ(get_table_value(lua_background, "key1"), "string value"); + EXPECT_EQ(get_table_value(lua_background, "key2"), 42); + EXPECT_EQ(get_table_value(lua_background, "key3"), 42.2); + EXPECT_TRUE(get_table_value(lua_background, "key4")); +} + +TEST(TestRGWLuaBackground, TablePersist) +{ + TestBackground lua_background(""); + + std::string request_script = R"( + RGW["key1"] = "string value" + RGW["key2"] = 42 + )"; + + DEFINE_REQ_STATE; + + auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", request_script, &lua_background); + ASSERT_EQ(rc, 0); + EXPECT_EQ(get_table_value(lua_background, "key1"), "string value"); + EXPECT_EQ(get_table_value(lua_background, "key2"), 42); + + request_script = R"( + RGW["key3"] = RGW["key1"] + RGW["key4"] = RGW["key2"] + )"; + + rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", request_script, &lua_background); + ASSERT_EQ(rc, 0); + EXPECT_EQ(get_table_value(lua_background, "key1"), "string value"); + EXPECT_EQ(get_table_value(lua_background, "key2"), 42); + EXPECT_EQ(get_table_value(lua_background, "key3"), "string value"); + EXPECT_EQ(get_table_value(lua_background, "key4"), 42); +} + +TEST(TestRGWLuaBackground, TableValuesFromRequest) +{ + TestBackground lua_background(""); + lua_background.start(); + + const std::string request_script = R"( + RGW["key1"] = Request.Response.RGWCode + RGW["key2"] = Request.Response.Message + RGW["key3"] = Request.Response.RGWCode*0.1 + RGW["key4"] = Request.Tags["key1"] == Request.Tags["key2"] + )"; + + DEFINE_REQ_STATE; + s.tagset.add_tag("key1", "val1"); + s.tagset.add_tag("key2", "val1"); + s.err.ret = -99; + s.err.message = "hi"; + + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", request_script, &lua_background); + ASSERT_EQ(rc, 0); + EXPECT_EQ(get_table_value(lua_background, "key1"), -99); + EXPECT_EQ(get_table_value(lua_background, "key2"), "hi"); + EXPECT_EQ(get_table_value(lua_background, "key3"), -9.9); + EXPECT_EQ(get_table_value(lua_background, "key4"), true); +} + +TEST(TestRGWLuaBackground, TableInvalidValue) +{ + TestBackground lua_background(""); + lua_background.start(); + + const std::string request_script = R"( + RGW["key1"] = "val1" + RGW["key2"] = 42 + RGW["key3"] = 42.2 + RGW["key4"] = true + RGW["key5"] = Request.Tags + )"; + + DEFINE_REQ_STATE; + s.tagset.add_tag("key1", "val1"); + s.tagset.add_tag("key2", "val2"); + + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", request_script, &lua_background); + ASSERT_NE(rc, 0); + EXPECT_EQ(get_table_value(lua_background, "key1"), "val1"); + EXPECT_EQ(get_table_value(lua_background, "key2"), 42); + EXPECT_EQ(get_table_value(lua_background, "key3"), 42.2); + EXPECT_EQ(get_table_value(lua_background, "key4"), true); +} + +TEST(TestRGWLuaBackground, TableErase) +{ + TestBackground lua_background(""); + + std::string request_script = R"( + RGW["size"] = 0 + RGW["key1"] = "string value" + RGW["key2"] = 42 + RGW["key3"] = "another string value" + RGW["size"] = #RGW + )"; + + DEFINE_REQ_STATE; + + auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", request_script, &lua_background); + ASSERT_EQ(rc, 0); + EXPECT_EQ(get_table_value(lua_background, "key1"), "string value"); + EXPECT_EQ(get_table_value(lua_background, "key2"), 42); + EXPECT_EQ(get_table_value(lua_background, "key3"), "another string value"); + EXPECT_EQ(get_table_value(lua_background, "size"), 4); + + request_script = R"( + -- erase key1 + RGW["key1"] = nil + -- following should be a no op + RGW["key4"] = nil + RGW["size"] = #RGW + )"; + + rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", request_script, &lua_background); + ASSERT_EQ(rc, 0); + EXPECT_EQ(get_table_value(lua_background, "key1"), ""); + EXPECT_EQ(get_table_value(lua_background, "key2"), 42); + EXPECT_EQ(get_table_value(lua_background, "key3"), "another string value"); + EXPECT_EQ(get_table_value(lua_background, "size"), 3); +} + +TEST(TestRGWLuaBackground, TableIterate) +{ + TestBackground lua_background(""); + + const std::string request_script = R"( + RGW["key1"] = "string value" + RGW["key2"] = 42 + RGW["key3"] = 42.2 + RGW["key4"] = true + RGW["size"] = 0 + for k, v in pairs(RGW) do + RGW["size"] = RGW["size"] + 1 + end + )"; + + DEFINE_REQ_STATE; + + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", request_script, &lua_background); + ASSERT_EQ(rc, 0); + EXPECT_EQ(get_table_value(lua_background, "key1"), "string value"); + EXPECT_EQ(get_table_value(lua_background, "key2"), 42); + EXPECT_EQ(get_table_value(lua_background, "key3"), 42.2); + EXPECT_TRUE(get_table_value(lua_background, "key4")); + EXPECT_EQ(get_table_value(lua_background, "size"), 5); }