From 8d37f60c10fedbf6a5ecae52a8275c12d2b14e48 Mon Sep 17 00:00:00 2001 From: samarah Date: Fri, 11 Nov 2022 12:56:46 -0500 Subject: [PATCH] RGW: Add D4N classes and unit testing; update cpp_redis submodule Signed-off-by: samarah --- CMakeLists.txt | 3 +- ceph.spec.in | 1 - doc/radosgw/config-ref.rst | 13 + src/CMakeLists.txt | 4 +- src/common/options/rgw.yaml.in | 22 +- src/cpp_redis | 2 +- src/include/config-h.in.cmake | 3 + src/librados/CMakeLists.txt | 1 - src/rgw/CMakeLists.txt | 7 +- src/rgw/driver/d4n/d4n_datacache.cc | 490 +++++++ src/rgw/driver/d4n/d4n_datacache.h | 40 + src/rgw/driver/d4n/d4n_directory.cc | 179 +++ src/rgw/driver/d4n/d4n_directory.h | 53 + src/rgw/driver/d4n/rgw_sal_d4n.cc | 566 ++++++++ src/rgw/driver/d4n/rgw_sal_d4n.h | 199 +++ src/rgw/rgw_sal.cc | 24 +- src/rgw/rgw_sal.h | 10 + src/rgw/rgw_sal_filter.h | 5 + src/rgw/rgw_sal_store.h | 17 + src/test/rgw/CMakeLists.txt | 38 + src/test/rgw/run-d4n-unit-tests.sh | 20 + src/test/rgw/test_d4n_directory.cc | 206 +++ src/test/rgw/test_d4n_filter.cc | 1977 +++++++++++++++++++++++++++ 23 files changed, 3871 insertions(+), 9 deletions(-) create mode 100644 src/rgw/driver/d4n/d4n_datacache.cc create mode 100644 src/rgw/driver/d4n/d4n_datacache.h create mode 100644 src/rgw/driver/d4n/d4n_directory.cc create mode 100644 src/rgw/driver/d4n/d4n_directory.h create mode 100644 src/rgw/driver/d4n/rgw_sal_d4n.cc create mode 100644 src/rgw/driver/d4n/rgw_sal_d4n.h create mode 100755 src/test/rgw/run-d4n-unit-tests.sh create mode 100644 src/test/rgw/test_d4n_directory.cc create mode 100644 src/test/rgw/test_d4n_filter.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index f5046b93011..6039bd84e23 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -438,7 +438,8 @@ option(WITH_RADOSGW_KAFKA_ENDPOINT "Rados Gateway's pubsub support for Kafka pus option(WITH_RADOSGW_LUA_PACKAGES "Rados Gateway's support for dynamically adding lua packagess" ON) option(WITH_RADOSGW_DBSTORE "DBStore backend for Rados Gateway" ON) option(WITH_RADOSGW_MOTR "CORTX-Motr backend for Rados Gateway" OFF) -option(WITH_RADOSGW_DAOS "DAOS backend for RADOS Gateway" OFF) +option(WITH_RADOSGW_DAOS "DAOS backend for Rados Gateway" OFF) +option(WITH_RADOSGW_D4N "D4N wrapper for RADOS Gateway" ON) option(WITH_RADOSGW_SELECT_PARQUET "Support for s3 select on parquet objects" ON) option(WITH_RADOSGW_ARROW_FLIGHT "Build arrow flight when not using system-provided arrow" OFF) option(WITH_RADOSGW_BACKTRACE_LOGGING "Enable backtraces in rgw logs" OFF) diff --git a/ceph.spec.in b/ceph.spec.in index 6d21b3858fe..f0dd8e8a941 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -1349,7 +1349,6 @@ cmake .. \ -DWITH_MANPAGE:BOOL=ON \ -DWITH_PYTHON3:STRING=%{python3_version} \ -DWITH_MGR_DASHBOARD_FRONTEND:BOOL=OFF \ - -DWITH_CPP_REDIS=TRUE \ %if 0%{?suse_version} -DWITH_RADOSGW_SELECT_PARQUET:BOOL=OFF \ %endif diff --git a/doc/radosgw/config-ref.rst b/doc/radosgw/config-ref.rst index b6de649dfe9..c2d7633f297 100644 --- a/doc/radosgw/config-ref.rst +++ b/doc/radosgw/config-ref.rst @@ -296,3 +296,16 @@ implementation of *dmclock_client* op queue divides RGW Ops on admin, auth .. _Barbican: ../barbican .. _Encryption: ../encryption .. _HTTP Frontends: ../frontends + +D4N Settings +============ + +D4N is a caching architecture that utilizes Redis to speed up S3 object storage +operations by establishing shared databases between different RGW access points. + +Currently, the architecture can only function on one Redis instance at a time. +The address is configurable and can be changed by accessing the parameters +below. + +.. confval:: rgw_d4n_host +.. confval:: rgw_d4n_port diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 204142382ad..20219c7ac5e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -295,8 +295,8 @@ if(WITH_CEPHFS_JAVA) add_subdirectory(java) endif() -option(WITH_CPP_REDIS "Build radosgw with cpp_redis library" OFF) -if(WITH_CPP_REDIS) +if(WITH_RADOSGW_D4N) +>>>>>>> f409ee2d871 (RGW: Update D4N files and cpp_redis submodule) add_subdirectory(cpp_redis) endif() diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in index 71def2e9de7..84cef28084b 100644 --- a/src/common/options/rgw.yaml.in +++ b/src/common/options/rgw.yaml.in @@ -3597,7 +3597,7 @@ options: enum_values: - none - base - - trace + - d4n - name: dbstore_db_dir type: str level: advanced @@ -3765,3 +3765,23 @@ options: default: true services: - rgw +- name: rgw_d4n_host + type: str + level: advanced + desc: The rgw directory host + default: 127.0.0.1 + services: + - rgw + flags: + - startup + with_legacy: true +- name: rgw_d4n_port + type: int + level: advanced + desc: The rgw directory port + default: 6379 + services: + - rgw + flags: + - startup + with_legacy: true diff --git a/src/cpp_redis b/src/cpp_redis index f40a63d5bb3..c659475ea43 160000 --- a/src/cpp_redis +++ b/src/cpp_redis @@ -1 +1 @@ -Subproject commit f40a63d5bb33487346008f123b4a5cf86babd2e3 +Subproject commit c659475ea43bc77850018aa1433d55cad902ea85 diff --git a/src/include/config-h.in.cmake b/src/include/config-h.in.cmake index cc9ad0ec7f7..f0f1b5b529a 100644 --- a/src/include/config-h.in.cmake +++ b/src/include/config-h.in.cmake @@ -157,6 +157,9 @@ /* define if radosgw has openssl support */ #cmakedefine WITH_CURL_OPENSSL +/*define if D4N filter enabled */ +#cmakedefine WITH_RADOSGW_D4N + /* define if HAVE_THREAD_SAFE_RES_QUERY */ #cmakedefine HAVE_THREAD_SAFE_RES_QUERY diff --git a/src/librados/CMakeLists.txt b/src/librados/CMakeLists.txt index f2c426a37f9..9e469eb17ff 100644 --- a/src/librados/CMakeLists.txt +++ b/src/librados/CMakeLists.txt @@ -6,7 +6,6 @@ add_library(librados_impl STATIC librados_tp.cc) # C/C++ API - add_library(librados ${CEPH_SHARED} librados_c.cc librados_cxx.cc diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index b0f462566d6..a888c3313d7 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -148,6 +148,9 @@ set(librgw_common_srcs rgw_bucket_encryption.cc rgw_tracer.cc rgw_lua_background.cc + driver/d4n/d4n_directory.cc + driver/d4n/d4n_datacache.cc + driver/d4n/rgw_sal_d4n.cc driver/rados/cls_fifo_legacy.cc driver/rados/rgw_bucket.cc driver/rados/rgw_bucket_sync.cc @@ -196,6 +199,8 @@ set(librgw_common_srcs driver/rados/sync_fairness.cc) list(APPEND librgw_common_srcs + driver/d4n/d4n_directory.cc + driver/d4n/d4n_datacache.cc driver/immutable_config/store.cc driver/json_config/store.cc driver/rados/config/impl.cc @@ -277,7 +282,7 @@ target_include_directories(rgw_common PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw" PUBLIC "${LUA_INCLUDE_DIR}") -if(WITH_CPP_REDIS) +if(WITH_RADOSGW_D4N) add_dependencies(rgw_common cpp_redis) target_link_libraries(rgw_common PRIVATE cpp_redis) target_include_directories(rgw_common PUBLIC "${CMAKE_SOURCE_DIR}/src/cpp_redis/includes") diff --git a/src/rgw/driver/d4n/d4n_datacache.cc b/src/rgw/driver/d4n/d4n_datacache.cc new file mode 100644 index 00000000000..ec0338f5bd2 --- /dev/null +++ b/src/rgw/driver/d4n/d4n_datacache.cc @@ -0,0 +1,490 @@ +#include "d4n_datacache.h" + +#define dout_subsys ceph_subsys_rgw +#define dout_context g_ceph_context + +/* Base metadata and data fields should remain consistent */ +std::vector baseFields { + "mtime", + "object_size", + "accounted_size", + "epoch", + "version_id", + "source_zone_short_id", + "bucket_count", + "bucket_size", + "user_quota.max_size", + "user_quota.max_objects", + "max_buckets", + "data"}; + +std::vector< std::pair > RGWD4NCache::buildObject(rgw::sal::Attrs* binary) { + std::vector< std::pair > values; + rgw::sal::Attrs::iterator attrs; + + /* Convert to vector */ + if (binary != NULL) { + for (attrs = binary->begin(); attrs != binary->end(); ++attrs) { + values.push_back(std::make_pair(attrs->first, attrs->second.to_str())); + } + } + + return values; +} + +int RGWD4NCache::findClient(cpp_redis::client *client) { + if (client->is_connected()) + return 0; + + if (host == "" || port == 0) { + dout(10) << "RGW D4N Cache: D4N cache endpoint was not configured correctly" << dendl; + return EDESTADDRREQ; + } + + client->connect(host, port, nullptr); + + if (!client->is_connected()) + return ECONNREFUSED; + + return 0; +} + +int RGWD4NCache::existKey(std::string key) { + int result = -1; + std::vector keys; + keys.push_back(key); + + if (!client.is_connected()) { + return result; + } + + try { + client.exists(keys, [&result](cpp_redis::reply &reply) { + if (reply.is_integer()) { + result = reply.as_integer(); /* Returns 1 upon success */ + } + }); + + client.sync_commit(std::chrono::milliseconds(1000)); + } catch(std::exception &e) {} + + return result; +} + +int RGWD4NCache::setObject(std::string oid, rgw::sal::Attrs* attrs) { + /* Creating the index based on oid */ + std::string key = "rgw-object:" + oid + ":cache"; + std::string result; + + if (!client.is_connected()) { + findClient(&client); + } + + /* Every set will be treated as new */ + try { + std::vector< std::pair > redisObject = buildObject(attrs); + + if (redisObject.empty()) { + return -1; + } + + client.hmset(key, redisObject, [&result](cpp_redis::reply &reply) { + if (!reply.is_null()) { + result = reply.as_string(); + } + }); + + client.sync_commit(std::chrono::milliseconds(1000)); + + if (result != "OK") { + return -1; + } + } catch(std::exception &e) { + return -1; + } + + return 0; +} + +int RGWD4NCache::getObject(std::string oid, + rgw::sal::Attrs* newAttrs, + std::vector< std::pair >* newMetadata) +{ + std::string result; + std::string key = "rgw-object:" + oid + ":cache"; + + if (!client.is_connected()) { + findClient(&client); + } + + if (existKey(key)) { + int field_exist = -1; + + rgw::sal::Attrs::iterator it; + std::vector< std::pair > redisObject; + std::vector getFields; + + /* Retrieve existing fields from cache */ + try { + client.hgetall(key, [&getFields](cpp_redis::reply &reply) { + if (reply.is_array()) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + for (long unsigned int i = 0; i < arr.size() - 1; i += 2) { + getFields.push_back(arr[i].as_string()); + } + } + } + }); + + client.sync_commit(std::chrono::milliseconds(1000)); + } catch(std::exception &e) { + return -1; + } + + /* Only data exists */ + if (getFields.size() == 1 && getFields[0] == "data") + return 0; + + /* Ensure all metadata, attributes, and data has been set */ + for (const auto& field : baseFields) { + auto it = std::find_if(getFields.begin(), getFields.end(), + [&](const auto& comp) { return comp == field; }); + + if (it != getFields.end()) { + int index = std::distance(getFields.begin(), it); + getFields.erase(getFields.begin() + index); + } else { + return -1; + } + } + + /* Get attributes from cache */ + try { + client.hmget(key, getFields, [&field_exist, &newAttrs, &getFields](cpp_redis::reply &reply) { + if (reply.is_array()) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + field_exist = 0; + + for (long unsigned int i = 0; i < getFields.size(); ++i) { + std::string tmp = arr[i].as_string(); + buffer::list bl; + bl.append(tmp); + newAttrs->insert({getFields[i], bl}); + } + } + } + }); + + client.sync_commit(std::chrono::milliseconds(1000)); + } catch(std::exception &e) { + return -1; + } + + if (field_exist == 0) { + field_exist = -1; + + getFields.clear(); + getFields.insert(getFields.begin(), baseFields.begin(), baseFields.end()); + getFields.pop_back(); /* Do not query for data field */ + + /* Get metadata from cache */ + try { + client.hmget(key, getFields, [&field_exist, &newMetadata, &getFields](cpp_redis::reply &reply) { + if (reply.is_array()) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + field_exist = 0; + + for (long unsigned int i = 0; i < getFields.size(); ++i) { + newMetadata->push_back({getFields[i], arr[i].as_string()}); + } + } + } + }); + + client.sync_commit(std::chrono::milliseconds(1000)); + } catch(std::exception &e) { + return -1; + } + } else { + return -1; + } + } else { + dout(20) << "RGW D4N Cache: Object was not retrievable." << dendl; + return -2; + } + + return 0; +} + +int RGWD4NCache::copyObject(std::string original_oid, std::string copy_oid, rgw::sal::Attrs* attrs) { + std::string result; + std::vector< std::pair > redisObject; + std::string key = "rgw-object:" + original_oid + ":cache"; + + if (!client.is_connected()) { + findClient(&client); + } + + /* Read values from cache */ + if (existKey(key)) { + try { + client.hgetall(key, [&redisObject](cpp_redis::reply &reply) { + if (reply.is_array()) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + for (long unsigned int i = 0; i < arr.size() - 1; i += 2) { + redisObject.push_back({arr[i].as_string(), arr[i + 1].as_string()}); + } + } + } + }); + + client.sync_commit(std::chrono::milliseconds(1000)); + } catch(std::exception &e) { + return -1; + } + } else { + return -2; + } + + /* Build copy with updated values */ + if (!redisObject.empty()) { + rgw::sal::Attrs::iterator attr; + + for (attr = attrs->begin(); attr != attrs->end(); ++attr) { + auto it = std::find_if(redisObject.begin(), redisObject.end(), + [&](const auto& pair) { return pair.first == attr->first; }); + + if (it != redisObject.end()) { + int index = std::distance(redisObject.begin(), it); + redisObject[index] = {attr->first, attr->second.to_str()}; + } else { + redisObject.push_back(std::make_pair(attr->first, attr->second.to_str())); + } + } + } else { + return -1; + } + + /* Set copy with new values */ + key = "rgw-object:" + copy_oid + ":cache"; + + try { + client.hmset(key, redisObject, [&result](cpp_redis::reply &reply) { + if (!reply.is_null()) { + result = reply.as_string(); + } + }); + + client.sync_commit(std::chrono::milliseconds(1000)); + + if (result != "OK") { + return -1; + } + } catch(std::exception &e) { + return -1; + } + + return 0; +} + +int RGWD4NCache::delObject(std::string oid) { + int result = 0; + std::vector keys; + std::string key = "rgw-object:" + oid + ":cache"; + keys.push_back(key); + + if (!client.is_connected()) { + findClient(&client); + } + + if (existKey(key)) { + try { + client.del(keys, [&result](cpp_redis::reply &reply) { + if (reply.is_integer()) { + result = reply.as_integer(); + } + }); + + client.sync_commit(std::chrono::milliseconds(1000)); + + return result - 1; + } catch(std::exception &e) { + return -1; + } + } else { + dout(20) << "RGW D4N Cache: Object is not in cache." << dendl; + return -2; + } +} + +int RGWD4NCache::updateAttr(std::string oid, rgw::sal::Attrs* attr) { + std::string result; + std::string key = "rgw-object:" + oid + ":cache"; + + if (!client.is_connected()) { + findClient(&client); + } + + if (existKey(key)) { + try { + std::vector< std::pair > redisObject; + auto it = attr->begin(); + redisObject.push_back({it->first, it->second.to_str()}); + + client.hmset(key, redisObject, [&result](cpp_redis::reply &reply) { + if (!reply.is_null()) { + result = reply.as_string(); + } + }); + + client.sync_commit(std::chrono::milliseconds(1000)); + + if (result != "OK") { + return -1; + } + } catch(std::exception &e) { + return -1; + } + } else { + return -2; + } + + return 0; +} + +int RGWD4NCache::delAttrs(std::string oid, std::vector& baseFields, std::vector& deleteFields) { + int result = 0; + std::string key = "rgw-object:" + oid + ":cache"; + + if (!client.is_connected()) { + findClient(&client); + } + + if (existKey(key)) { + /* Find if attribute doesn't exist */ + for (const auto& delField : deleteFields) { + if (std::find(baseFields.begin(), baseFields.end(), delField) == baseFields.end()) { + deleteFields.erase(std::find(deleteFields.begin(), deleteFields.end(), delField)); + } + } + + try { + client.hdel(key, deleteFields, [&result](cpp_redis::reply &reply) { + if (reply.is_integer()) { + result = reply.as_integer(); + } + }); + + client.sync_commit(std::chrono::milliseconds(1000)); + + return result - 1; + } catch(std::exception &e) { + return -1; + } + } + + dout(20) << "RGW D4N Cache: Object is not in cache." << dendl; + return -2; +} + +int RGWD4NCache::appendData(std::string oid, buffer::list& data) { + std::string result; + std::string value = ""; + std::string key = "rgw-object:" + oid + ":cache"; + + if (!client.is_connected()) { + findClient(&client); + } + + if (existKey(key)) { + try { + client.hget(key, "data", [&value](cpp_redis::reply &reply) { + if (!reply.is_null()) { + value = reply.as_string(); + } + }); + + client.sync_commit(std::chrono::milliseconds(1000)); + } catch(std::exception &e) { + return -1; + } + } + + try { + /* Append to existing value or set as new value */ + std::string temp = value + data.to_str(); + std::vector< std::pair > field; + field.push_back({"data", temp}); + + client.hmset(key, field, [&result](cpp_redis::reply &reply) { + if (!reply.is_null()) { + result = reply.as_string(); + } + }); + + client.sync_commit(std::chrono::milliseconds(1000)); + + if (result != "OK") { + return -1; + } + } catch(std::exception &e) { + return -1; + } + + return 0; +} + +int RGWD4NCache::deleteData(std::string oid) { + int result = 0; + std::string key = "rgw-object:" + oid + ":cache"; + std::vector deleteField; + deleteField.push_back("data"); + + if (!client.is_connected()) { + findClient(&client); + } + + if (existKey(key)) { + int field_exist = -1; + + try { + client.hget(key, "data", [&field_exist](cpp_redis::reply &reply) { + if (!reply.is_null()) { + field_exist = 0; + } + }); + + client.sync_commit(std::chrono::milliseconds(1000)); + } catch(std::exception &e) { + return -1; + } + + if (field_exist == 0) { + try { + client.hdel(key, deleteField, [&result](cpp_redis::reply &reply) { + if (reply.is_integer()) { + result = reply.as_integer(); /* Returns 1 upon success */ + } + }); + + client.sync_commit(std::chrono::milliseconds(1000)); + + return result - 1; + } catch(std::exception &e) { + return -1; + } + } else { + return -1; + } + } else { + return 0; /* No delete was necessary */ + } +} diff --git a/src/rgw/driver/d4n/d4n_datacache.h b/src/rgw/driver/d4n/d4n_datacache.h new file mode 100644 index 00000000000..5faf7b6ce0e --- /dev/null +++ b/src/rgw/driver/d4n/d4n_datacache.h @@ -0,0 +1,40 @@ +#ifndef CEPH_RGWD4NCACHE_H +#define CEPH_RGWD4NCACHE_H + +#include "rgw_common.h" +#include +#include +#include + +class RGWD4NCache { + public: + CephContext *cct; + + RGWD4NCache() {} + RGWD4NCache(std::string cacheHost, int cachePort):host(cacheHost), port(cachePort) {} + + void init(CephContext *_cct) { + cct = _cct; + host = cct->_conf->rgw_d4n_host; + port = cct->_conf->rgw_d4n_port; + } + + int findClient(cpp_redis::client *client); + int existKey(std::string key); + int setObject(std::string oid, rgw::sal::Attrs* attrs); + int getObject(std::string oid, rgw::sal::Attrs* newAttrs, std::vector< std::pair >* newMetadata); + int copyObject(std::string original_oid, std::string copy_oid, rgw::sal::Attrs* attrs); + int delObject(std::string oid); + int updateAttr(std::string oid, rgw::sal::Attrs* attr); + int delAttrs(std::string oid, std::vector& baseFields, std::vector& deleteFields); + int appendData(std::string oid, buffer::list& data); + int deleteData(std::string oid); + + private: + cpp_redis::client client; + std::string host = ""; + int port = 0; + std::vector< std::pair > buildObject(rgw::sal::Attrs* binary); +}; + +#endif diff --git a/src/rgw/driver/d4n/d4n_directory.cc b/src/rgw/driver/d4n/d4n_directory.cc new file mode 100644 index 00000000000..96667295533 --- /dev/null +++ b/src/rgw/driver/d4n/d4n_directory.cc @@ -0,0 +1,179 @@ +#include "d4n_directory.h" + +#define dout_subsys ceph_subsys_rgw +#define dout_context g_ceph_context + +int RGWBlockDirectory::findClient(cpp_redis::client *client) { + if (client->is_connected()) + return 0; + + if (host == "" || port == 0) { + dout(10) << "RGW D4N Directory: D4N directory endpoint was not configured correctly" << dendl; + return EDESTADDRREQ; + } + + client->connect(host, port, nullptr); + + if (!client->is_connected()) + return ECONNREFUSED; + + return 0; +} + +std::string RGWBlockDirectory::buildIndex(cache_block *ptr) { + return "rgw-object:" + ptr->c_obj.obj_name + ":directory"; +} + +int RGWBlockDirectory::existKey(std::string key) { + int result = -1; + std::vector keys; + keys.push_back(key); + + if (!client.is_connected()) { + return result; + } + + try { + client.exists(keys, [&result](cpp_redis::reply &reply) { + if (reply.is_integer()) { + result = reply.as_integer(); /* Returns 1 upon success */ + } + }); + + client.sync_commit(std::chrono::milliseconds(1000)); + } catch(std::exception &e) {} + + return result; +} + +int RGWBlockDirectory::setValue(cache_block *ptr) { + /* Creating the index based on obj_name */ + std::string key = buildIndex(ptr); + if (!client.is_connected()) { + findClient(&client); + } + + std::string result; + std::vector keys; + keys.push_back(key); + + /* Every set will be new */ + if (host == "" || port == 0) { + dout(10) << "RGW D4N Directory: Directory endpoint not configured correctly" << dendl; + return -1; + } + + std::string endpoint = host + ":" + std::to_string(port); + std::vector> list; + + /* Creating a list of key's properties */ + list.push_back(make_pair("key", key)); + list.push_back(make_pair("size", std::to_string(ptr->size_in_bytes))); + list.push_back(make_pair("bucket_name", ptr->c_obj.bucket_name)); + list.push_back(make_pair("obj_name", ptr->c_obj.obj_name)); + list.push_back(make_pair("hosts", endpoint)); + + try { + client.hmset(key, list, [&result](cpp_redis::reply &reply) { + if (!reply.is_null()) { + result = reply.as_string(); + } + }); + + client.sync_commit(std::chrono::milliseconds(1000)); + + if (result != "OK") { + return -1; + } + } catch(std::exception &e) { + return -1; + } + + return 0; +} + +int RGWBlockDirectory::getValue(cache_block *ptr) { + std::string key = buildIndex(ptr); + + if (!client.is_connected()) { + findClient(&client); + } + + if (existKey(key)) { + int field_exist = -1; + + std::string hosts; + std::string size; + std::string bucket_name; + std::string obj_name; + std::vector fields; + + fields.push_back("key"); + fields.push_back("hosts"); + fields.push_back("size"); + fields.push_back("bucket_name"); + fields.push_back("obj_name"); + + try { + client.hmget(key, fields, [&key, &hosts, &size, &bucket_name, &obj_name, &field_exist](cpp_redis::reply &reply) { + if (reply.is_array()) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + field_exist = 0; + key = arr[0].as_string(); + hosts = arr[1].as_string(); + size = arr[2].as_string(); + bucket_name = arr[3].as_string(); + obj_name = arr[4].as_string(); + } + } + }); + + client.sync_commit(std::chrono::milliseconds(1000)); + + if (field_exist < 0) { + return field_exist; + } + + /* Currently, there can only be one host */ + ptr->size_in_bytes = std::stoi(size); + ptr->c_obj.bucket_name = bucket_name; + ptr->c_obj.obj_name = obj_name; + } catch(std::exception &e) { + return -1; + } + } + + return 0; +} + +int RGWBlockDirectory::delValue(cache_block *ptr) { + int result = 0; + std::vector keys; + std::string key = buildIndex(ptr); + keys.push_back(key); + + if (!client.is_connected()) { + findClient(&client); + } + + if (existKey(key)) { + try { + client.del(keys, [&result](cpp_redis::reply &reply) { + if (reply.is_integer()) { + result = reply.as_integer(); /* Returns 1 upon success */ + } + }); + + client.sync_commit(std::chrono::milliseconds(1000)); + + return result - 1; + } catch(std::exception &e) { + return -1; + } + } else { + dout(20) << "RGW D4N Directory: Block is not in directory." << dendl; + return -2; + } +} diff --git a/src/rgw/driver/d4n/d4n_directory.h b/src/rgw/driver/d4n/d4n_directory.h new file mode 100644 index 00000000000..95596db660b --- /dev/null +++ b/src/rgw/driver/d4n/d4n_directory.h @@ -0,0 +1,53 @@ +#ifndef CEPH_RGWD4NDIRECTORY_H +#define CEPH_RGWD4NDIRECTORY_H + +#include "rgw_common.h" +#include +#include +#include + +struct cache_obj { + std::string bucket_name; /* s3 bucket name */ + std::string obj_name; /* s3 obj name */ +}; + +struct cache_block { + cache_obj c_obj; + uint64_t size_in_bytes; /* block size_in_bytes */ + std::vector hosts_list; /* Currently not supported: list of hostnames of block locations */ +}; + +class RGWDirectory { + public: + RGWDirectory() {} + CephContext *cct; +}; + +class RGWBlockDirectory: RGWDirectory { + public: + RGWBlockDirectory() {} + RGWBlockDirectory(std::string blockHost, int blockPort):host(blockHost), port(blockPort) {} + + void init(CephContext *_cct) { + cct = _cct; + host = cct->_conf->rgw_d4n_host; + port = cct->_conf->rgw_d4n_port; + } + + int findClient(cpp_redis::client *client); + int existKey(std::string key); + int setValue(cache_block *ptr); + int getValue(cache_block *ptr); + int delValue(cache_block *ptr); + + std::string get_host() { return host; } + int get_port() { return port; } + + private: + cpp_redis::client client; + std::string buildIndex(cache_block *ptr); + std::string host = ""; + int port = 0; +}; + +#endif diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.cc b/src/rgw/driver/d4n/rgw_sal_d4n.cc new file mode 100644 index 00000000000..fe877cdb224 --- /dev/null +++ b/src/rgw/driver/d4n/rgw_sal_d4n.cc @@ -0,0 +1,566 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2022 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "rgw_sal_d4n.h" + +#define dout_subsys ceph_subsys_rgw +#define dout_context g_ceph_context + +namespace rgw { namespace sal { + +static inline Bucket* nextBucket(Bucket* t) +{ + if (!t) + return nullptr; + + return dynamic_cast(t)->get_next(); +} + +static inline Object* nextObject(Object* t) +{ + if (!t) + return nullptr; + + return dynamic_cast(t)->get_next(); +} + +int D4NFilterDriver::initialize(CephContext *cct, const DoutPrefixProvider *dpp) +{ + FilterDriver::initialize(cct, dpp); + blk_dir->init(cct); + d4n_cache->init(cct); + + return 0; +} + +std::unique_ptr D4NFilterDriver::get_user(const rgw_user &u) +{ + std::unique_ptr user = next->get_user(u); + + return std::make_unique(std::move(user), this); +} + +std::unique_ptr D4NFilterBucket::get_object(const rgw_obj_key& k) +{ + std::unique_ptr o = next->get_object(k); + + return std::make_unique(std::move(o), this, filter); +} + +int D4NFilterUser::create_bucket(const DoutPrefixProvider* dpp, + const rgw_bucket& b, + const std::string& zonegroup_id, + rgw_placement_rule& placement_rule, + std::string& swift_ver_location, + const RGWQuotaInfo * pquota_info, + const RGWAccessControlPolicy& policy, + Attrs& attrs, + RGWBucketInfo& info, + obj_version& ep_objv, + bool exclusive, + bool obj_lock_enabled, + bool* existed, + req_info& req_info, + std::unique_ptr* bucket_out, + optional_yield y) +{ + std::unique_ptr nb; + int ret; + + ret = next->create_bucket(dpp, b, zonegroup_id, placement_rule, swift_ver_location, pquota_info, policy, attrs, info, ep_objv, exclusive, obj_lock_enabled, existed, req_info, &nb, y); + if (ret < 0) + return ret; + + Bucket* fb = new D4NFilterBucket(std::move(nb), this, filter); + bucket_out->reset(fb); + return 0; +} + +int D4NFilterObject::copy_object(User* user, + req_info* info, + const rgw_zone_id& source_zone, + rgw::sal::Object* dest_object, + rgw::sal::Bucket* dest_bucket, + rgw::sal::Bucket* src_bucket, + const rgw_placement_rule& dest_placement, + ceph::real_time* src_mtime, + ceph::real_time* mtime, + const ceph::real_time* mod_ptr, + const ceph::real_time* unmod_ptr, + bool high_precision_time, + const char* if_match, + const char* if_nomatch, + AttrsMod attrs_mod, + bool copy_if_newer, + Attrs& attrs, + RGWObjCategory category, + uint64_t olh_epoch, + boost::optional delete_at, + std::string* version_id, + std::string* tag, + std::string* etag, + void (*progress_cb)(off_t, void *), + void* progress_data, + const DoutPrefixProvider* dpp, + optional_yield y) +{ + /* Append additional metadata to attributes */ + rgw::sal::Attrs baseAttrs = this->get_attrs(); + buffer::list bl; + + bl.append(to_iso_8601(*mtime)); + baseAttrs.insert({"mtime", bl}); + bl.clear(); + + if (version_id != NULL) { + bl.append(*version_id); + baseAttrs.insert({"version_id", bl}); + bl.clear(); + } + + if (!etag->empty()) { + bl.append(*etag); + baseAttrs.insert({"etag", bl}); + bl.clear(); + } + + if (attrs_mod == rgw::sal::ATTRSMOD_REPLACE) { /* Replace */ + rgw::sal::Attrs::iterator iter; + + for (const auto& pair : attrs) { + iter = baseAttrs.find(pair.first); + + if (iter != baseAttrs.end()) { + iter->second = pair.second; + } else { + baseAttrs.insert({pair.first, pair.second}); + } + } + } else if (attrs_mod == rgw::sal::ATTRSMOD_MERGE) { /* Merge */ + baseAttrs.insert(attrs.begin(), attrs.end()); + } + + int copyObjReturn = filter->get_d4n_cache()->copyObject(this->get_key().get_oid(), dest_object->get_key().get_oid(), &baseAttrs); + + if (copyObjReturn < 0) { + ldpp_dout(dpp, 20) << "D4N Filter: Cache copy object operation failed." << dendl; + } else { + ldpp_dout(dpp, 20) << "D4N Filter: Cache copy object operation succeeded." << dendl; + } + + return next->copy_object(user, info, source_zone, + nextObject(dest_object), + nextBucket(dest_bucket), + nextBucket(src_bucket), + dest_placement, src_mtime, mtime, + mod_ptr, unmod_ptr, high_precision_time, if_match, + if_nomatch, attrs_mod, copy_if_newer, attrs, + category, olh_epoch, delete_at, version_id, tag, + etag, progress_cb, progress_data, dpp, y); +} + +int D4NFilterObject::set_obj_attrs(const DoutPrefixProvider* dpp, Attrs* setattrs, + Attrs* delattrs, optional_yield y) +{ + if (setattrs != NULL) { + /* Ensure setattrs and delattrs do not overlap */ + if (delattrs != NULL) { + for (const auto& attr : *delattrs) { + if (std::find(setattrs->begin(), setattrs->end(), attr) != setattrs->end()) { + delattrs->erase(std::find(delattrs->begin(), delattrs->end(), attr)); + } + } + } + + int updateAttrsReturn = filter->get_d4n_cache()->setObject(this->get_key().get_oid(), setattrs); + + if (updateAttrsReturn < 0) { + ldpp_dout(dpp, 20) << "D4N Filter: Cache set object attributes operation failed." << dendl; + } else { + ldpp_dout(dpp, 20) << "D4N Filter: Cache set object attributes operation succeeded." << dendl; + } + } + + if (delattrs != NULL) { + std::vector delFields; + Attrs::iterator attrs; + + /* Extract fields from delattrs */ + for (attrs = delattrs->begin(); attrs != delattrs->end(); ++attrs) { + delFields.push_back(attrs->first); + } + + Attrs currentattrs = this->get_attrs(); + std::vector currentFields; + + /* Extract fields from current attrs */ + for (attrs = currentattrs.begin(); attrs != currentattrs.end(); ++attrs) { + currentFields.push_back(attrs->first); + } + + int delAttrsReturn = filter->get_d4n_cache()->delAttrs(this->get_key().get_oid(), currentFields, delFields); + + if (delAttrsReturn < 0) { + ldpp_dout(dpp, 20) << "D4N Filter: Cache delete object attributes operation failed." << dendl; + } else { + ldpp_dout(dpp, 20) << "D4N Filter: Cache delete object attributes operation succeeded." << dendl; + } + } + + return next->set_obj_attrs(dpp, setattrs, delattrs, y); +} + +int D4NFilterObject::get_obj_attrs(optional_yield y, const DoutPrefixProvider* dpp, + rgw_obj* target_obj) +{ + rgw::sal::Attrs newAttrs; + std::vector< std::pair > newMetadata; + int getAttrsReturn = filter->get_d4n_cache()->getObject(this->get_key().get_oid(), + &newAttrs, + &newMetadata); + + if (getAttrsReturn < 0) { + ldpp_dout(dpp, 20) << "D4N Filter: Cache get object attributes operation failed." << dendl; + + return next->get_obj_attrs(y, dpp, target_obj); + } else { + int setAttrsReturn = this->set_attrs(newAttrs); + + if (setAttrsReturn < 0) { + ldpp_dout(dpp, 20) << "D4N Filter: Cache get object attributes operation failed." << dendl; + + return next->get_obj_attrs(y, dpp, target_obj); + } else { + ldpp_dout(dpp, 20) << "D4N Filter: Cache get object attributes operation succeeded." << dendl; + + return 0; + } + } +} + +int D4NFilterObject::modify_obj_attrs(const char* attr_name, bufferlist& attr_val, + optional_yield y, const DoutPrefixProvider* dpp) +{ + Attrs update; + update[(std::string)attr_name] = attr_val; + int updateAttrsReturn = filter->get_d4n_cache()->updateAttr(this->get_key().get_oid(), &update); + + if (updateAttrsReturn < 0) { + ldpp_dout(dpp, 20) << "D4N Filter: Cache modify object attribute operation failed." << dendl; + } else { + ldpp_dout(dpp, 20) << "D4N Filter: Cache modify object attribute operation succeeded." << dendl; + } + + return next->modify_obj_attrs(attr_name, attr_val, y, dpp); +} + +int D4NFilterObject::delete_obj_attrs(const DoutPrefixProvider* dpp, const char* attr_name, + optional_yield y) +{ + std::vector delFields; + delFields.push_back((std::string)attr_name); + + Attrs::iterator attrs; + Attrs currentattrs = this->get_attrs(); + std::vector currentFields; + + /* Extract fields from current attrs */ + for (attrs = currentattrs.begin(); attrs != currentattrs.end(); ++attrs) { + currentFields.push_back(attrs->first); + } + + int delAttrReturn = filter->get_d4n_cache()->delAttrs(this->get_key().get_oid(), currentFields, delFields); + + if (delAttrReturn < 0) { + ldpp_dout(dpp, 20) << "D4N Filter: Cache delete object attribute operation failed." << dendl; + } else { + ldpp_dout(dpp, 20) << "D4N Filter: Cache delete object attribute operation succeeded." << dendl; + } + + return next->delete_obj_attrs(dpp, attr_name, y); +} + +std::unique_ptr D4NFilterDriver::get_object(const rgw_obj_key& k) +{ + std::unique_ptr o = next->get_object(k); + + return std::make_unique(std::move(o), this); +} + +std::unique_ptr D4NFilterDriver::get_atomic_writer(const DoutPrefixProvider *dpp, + optional_yield y, + rgw::sal::Object* obj, + const rgw_user& owner, + const rgw_placement_rule *ptail_placement_rule, + uint64_t olh_epoch, + const std::string& unique_tag) +{ + std::unique_ptr writer = next->get_atomic_writer(dpp, y, nextObject(obj), + owner, ptail_placement_rule, + olh_epoch, unique_tag); + + return std::make_unique(std::move(writer), this, obj, dpp, true); +} + +std::unique_ptr D4NFilterObject::get_read_op() +{ + std::unique_ptr r = next->get_read_op(); + return std::make_unique(std::move(r), this); +} + +std::unique_ptr D4NFilterObject::get_delete_op() +{ + std::unique_ptr d = next->get_delete_op(); + return std::make_unique(std::move(d), this); +} + +int D4NFilterObject::D4NFilterReadOp::prepare(optional_yield y, const DoutPrefixProvider* dpp) +{ + int getDirReturn = source->filter->get_block_dir()->getValue(source->filter->get_cache_block()); + + if (getDirReturn < 0) { + ldpp_dout(dpp, 20) << "D4N Filter: Directory get operation failed." << dendl; + } else { + ldpp_dout(dpp, 20) << "D4N Filter: Directory get operation succeeded." << dendl; + } + + rgw::sal::Attrs newAttrs; + std::vector< std::pair > newMetadata; + int getObjReturn = source->filter->get_d4n_cache()->getObject(source->get_key().get_oid(), + &newAttrs, + &newMetadata); + + int ret = next->prepare(y, dpp); + + if (getObjReturn < 0) { + ldpp_dout(dpp, 20) << "D4N Filter: Cache get object operation failed." << dendl; + } else { + /* Set metadata locally */ + RGWQuotaInfo quota_info; + RGWObjState* astate; + source->get_obj_state(dpp, &astate, y); + + for (auto it = newMetadata.begin(); it != newMetadata.end(); ++it) { + if (!std::strcmp(it->first.data(), "mtime")) { + parse_time(it->second.data(), &astate->mtime); + } else if (!std::strcmp(it->first.data(), "object_size")) { + source->set_obj_size(std::stoull(it->second)); + } else if (!std::strcmp(it->first.data(), "accounted_size")) { + astate->accounted_size = std::stoull(it->second); + } else if (!std::strcmp(it->first.data(), "epoch")) { + astate->epoch = std::stoull(it->second); + } else if (!std::strcmp(it->first.data(), "version_id")) { + source->set_instance(it->second); + } else if (!std::strcmp(it->first.data(), "source_zone_short_id")) { + astate->zone_short_id = static_cast(std::stoul(it->second)); + } else if (!std::strcmp(it->first.data(), "bucket_count")) { + source->get_bucket()->set_count(std::stoull(it->second)); + } else if (!std::strcmp(it->first.data(), "bucket_size")) { + source->get_bucket()->set_size(std::stoull(it->second)); + } else if (!std::strcmp(it->first.data(), "user_quota.max_size")) { + quota_info.max_size = std::stoull(it->second); + } else if (!std::strcmp(it->first.data(), "user_quota.max_objects")) { + quota_info.max_objects = std::stoull(it->second); + } else if (!std::strcmp(it->first.data(), "max_buckets")) { + source->get_bucket()->get_owner()->set_max_buckets(std::stoull(it->second)); + } + } + + source->get_bucket()->get_owner()->set_info(quota_info); + source->set_obj_state(*astate); + + /* Set attributes locally */ + int setAttrsReturn = source->set_attrs(newAttrs); + + if (setAttrsReturn < 0) { + ldpp_dout(dpp, 20) << "D4N Filter: Cache get object operation failed." << dendl; + } else { + ldpp_dout(dpp, 20) << "D4N Filter: Cache get object operation succeeded." << dendl; + } + } + + return ret; +} + +int D4NFilterObject::D4NFilterDeleteOp::delete_obj(const DoutPrefixProvider* dpp, + optional_yield y) +{ + int delDirReturn = source->filter->get_block_dir()->delValue(source->filter->get_cache_block()); + + if (delDirReturn < 0) { + ldpp_dout(dpp, 20) << "D4N Filter: Directory delete operation failed." << dendl; + } else { + ldpp_dout(dpp, 20) << "D4N Filter: Directory delete operation succeeded." << dendl; + } + + int delObjReturn = source->filter->get_d4n_cache()->delObject(source->get_key().get_oid()); + + if (delObjReturn < 0) { + ldpp_dout(dpp, 20) << "D4N Filter: Cache delete operation failed." << dendl; + } else { + ldpp_dout(dpp, 20) << "D4N Filter: Cache delete operation succeeded." << dendl; + } + + return next->delete_obj(dpp, y); +} + +int D4NFilterWriter::prepare(optional_yield y) +{ + int delDataReturn = filter->get_d4n_cache()->deleteData(obj->get_key().get_oid()); + + if (delDataReturn < 0) { + ldpp_dout(save_dpp, 20) << "D4N Filter: Cache delete data operation failed." << dendl; + } else { + ldpp_dout(save_dpp, 20) << "D4N Filter: Cache delete data operation succeeded." << dendl; + } + + return next->prepare(y); +} + +int D4NFilterWriter::process(bufferlist&& data, uint64_t offset) +{ + int appendDataReturn = filter->get_d4n_cache()->appendData(obj->get_key().get_oid(), data); + + if (appendDataReturn < 0) { + ldpp_dout(save_dpp, 20) << "D4N Filter: Cache append data operation failed." << dendl; + } else { + ldpp_dout(save_dpp, 20) << "D4N Filter: Cache append data operation succeeded." << dendl; + } + + return next->process(std::move(data), offset); +} + +int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag, + ceph::real_time *mtime, ceph::real_time set_mtime, + std::map& attrs, + ceph::real_time delete_at, + const char *if_match, const char *if_nomatch, + const std::string *user_data, + rgw_zone_set *zones_trace, bool *canceled, + optional_yield y) +{ + cache_block* temp_cache_block = filter->get_cache_block(); + RGWBlockDirectory* temp_block_dir = filter->get_block_dir(); + + temp_cache_block->hosts_list.push_back(temp_block_dir->get_host() + ":" + std::to_string(temp_block_dir->get_port())); + temp_cache_block->size_in_bytes = accounted_size; + temp_cache_block->c_obj.bucket_name = obj->get_bucket()->get_name(); + temp_cache_block->c_obj.obj_name = obj->get_key().get_oid(); + + int setDirReturn = temp_block_dir->setValue(temp_cache_block); + + if (setDirReturn < 0) { + ldpp_dout(save_dpp, 20) << "D4N Filter: Directory set operation failed." << dendl; + } else { + ldpp_dout(save_dpp, 20) << "D4N Filter: Directory set operation succeeded." << dendl; + } + + /* Retrieve complete set of attrs */ + RGWObjState* astate; + int ret = next->complete(accounted_size, etag, mtime, set_mtime, attrs, + delete_at, if_match, if_nomatch, user_data, zones_trace, + canceled, y); + obj->get_obj_attrs(y, save_dpp, NULL); + obj->get_obj_state(save_dpp, &astate, y); + + /* Append additional metadata to attributes */ + rgw::sal::Attrs baseAttrs = obj->get_attrs(); + rgw::sal::Attrs attrs_temp = baseAttrs; + buffer::list bl; + + bl.append(to_iso_8601(obj->get_mtime())); + baseAttrs.insert({"mtime", bl}); + bl.clear(); + + bl.append(std::to_string(obj->get_obj_size())); + baseAttrs.insert({"object_size", bl}); + bl.clear(); + + bl.append(std::to_string(accounted_size)); + baseAttrs.insert({"accounted_size", bl}); + bl.clear(); + + bl.append(std::to_string(astate->epoch)); + baseAttrs.insert({"epoch", bl}); + bl.clear(); + + if (obj->have_instance()) { + bl.append(obj->get_instance()); + baseAttrs.insert({"version_id", bl}); + bl.clear(); + } else { + bl.append(""); /* Empty value */ + baseAttrs.insert({"version_id", bl}); + bl.clear(); + } + + auto iter = attrs_temp.find(RGW_ATTR_SOURCE_ZONE); + if (iter != attrs_temp.end()) { + bl.append(std::to_string(astate->zone_short_id)); + baseAttrs.insert({"source_zone_short_id", bl}); + bl.clear(); + } else { + bl.append("0"); /* Initialized to zero */ + baseAttrs.insert({"source_zone_short_id", bl}); + bl.clear(); + } + + bl.append(std::to_string(obj->get_bucket()->get_count())); + baseAttrs.insert({"bucket_count", bl}); + bl.clear(); + + bl.append(std::to_string(obj->get_bucket()->get_size())); + baseAttrs.insert({"bucket_size", bl}); + bl.clear(); + + RGWUserInfo info = obj->get_bucket()->get_owner()->get_info(); + bl.append(std::to_string(info.quota.user_quota.max_size)); + baseAttrs.insert({"user_quota.max_size", bl}); + bl.clear(); + + bl.append(std::to_string(info.quota.user_quota.max_objects)); + baseAttrs.insert({"user_quota.max_objects", bl}); + bl.clear(); + + bl.append(std::to_string(obj->get_bucket()->get_owner()->get_max_buckets())); + baseAttrs.insert({"max_buckets", bl}); + bl.clear(); + + baseAttrs.insert(attrs.begin(), attrs.end()); + + int setObjReturn = filter->get_d4n_cache()->setObject(obj->get_key().get_oid(), &baseAttrs); + + if (setObjReturn < 0) { + ldpp_dout(save_dpp, 20) << "D4N Filter: Cache set operation failed." << dendl; + } else { + ldpp_dout(save_dpp, 20) << "D4N Filter: Cache set operation succeeded." << dendl; + } + + return ret; +} + +} } // namespace rgw::sal + +extern "C" { + +rgw::sal::Driver* newD4NFilter(rgw::sal::Driver* next) +{ + rgw::sal::D4NFilterDriver* driver = new rgw::sal::D4NFilterDriver(next); + + return driver; +} + +} + diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.h b/src/rgw/driver/d4n/rgw_sal_d4n.h new file mode 100644 index 00000000000..62c13f0abed --- /dev/null +++ b/src/rgw/driver/d4n/rgw_sal_d4n.h @@ -0,0 +1,199 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2022 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#pragma once + +#include "rgw_sal_filter.h" +#include "rgw_sal.h" +#include "rgw_oidc_provider.h" +#include "rgw_role.h" +#include "common/dout.h" + +#include "driver/d4n/d4n_directory.h" +#include "driver/d4n/d4n_datacache.h" + +namespace rgw { namespace sal { + +class D4NFilterDriver : public FilterDriver { + private: + RGWBlockDirectory* blk_dir; + cache_block* c_blk; + RGWD4NCache* d4n_cache; + + public: + D4NFilterDriver(Driver* _next) : FilterDriver(_next) + { + blk_dir = new RGWBlockDirectory(); /* Initialize directory address with cct */ + c_blk = new cache_block(); + d4n_cache = new RGWD4NCache(); + } + virtual ~D4NFilterDriver() { + delete blk_dir; + delete c_blk; + delete d4n_cache; + } + + virtual int initialize(CephContext *cct, const DoutPrefixProvider *dpp) override; + virtual std::unique_ptr get_user(const rgw_user& u) override; + + virtual std::unique_ptr get_object(const rgw_obj_key& k) override; + + virtual std::unique_ptr get_atomic_writer(const DoutPrefixProvider *dpp, + optional_yield y, + rgw::sal::Object* obj, + const rgw_user& owner, + const rgw_placement_rule *ptail_placement_rule, + uint64_t olh_epoch, + const std::string& unique_tag) override; + RGWBlockDirectory* get_block_dir() { return blk_dir; } + cache_block* get_cache_block() { return c_blk; } + RGWD4NCache* get_d4n_cache() { return d4n_cache; } +}; + +class D4NFilterUser : public FilterUser { + private: + D4NFilterDriver* filter; + + public: + D4NFilterUser(std::unique_ptr _next, D4NFilterDriver* _filter) : + FilterUser(std::move(_next)), + filter(_filter) {} + virtual ~D4NFilterUser() = default; + + virtual int create_bucket(const DoutPrefixProvider* dpp, + const rgw_bucket& b, + const std::string& zonegroup_id, + rgw_placement_rule& placement_rule, + std::string& swift_ver_location, + const RGWQuotaInfo* pquota_info, + const RGWAccessControlPolicy& policy, + Attrs& attrs, + RGWBucketInfo& info, + obj_version& ep_objv, + bool exclusive, + bool obj_lock_enabled, + bool* existed, + req_info& req_info, + std::unique_ptr* bucket, + optional_yield y) override; +}; + +class D4NFilterBucket : public FilterBucket { + private: + D4NFilterDriver* filter; + + public: + D4NFilterBucket(std::unique_ptr _next, User* _user, D4NFilterDriver* _filter) : + FilterBucket(std::move(_next), _user), + filter(_filter) {} + virtual ~D4NFilterBucket() = default; + + virtual std::unique_ptr get_object(const rgw_obj_key& key) override; +}; + +class D4NFilterObject : public FilterObject { + private: + D4NFilterDriver* filter; + + public: + struct D4NFilterReadOp : FilterReadOp { + D4NFilterObject* source; + + D4NFilterReadOp(std::unique_ptr _next, D4NFilterObject* _source) : FilterReadOp(std::move(_next)), + source(_source) {} + virtual ~D4NFilterReadOp() = default; + + virtual int prepare(optional_yield y, const DoutPrefixProvider* dpp) override; + }; + + struct D4NFilterDeleteOp : FilterDeleteOp { + D4NFilterObject* source; + + D4NFilterDeleteOp(std::unique_ptr _next, D4NFilterObject* _source) : FilterDeleteOp(std::move(_next)), + source(_source) {} + virtual ~D4NFilterDeleteOp() = default; + + virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y) override; + }; + + D4NFilterObject(std::unique_ptr _next, D4NFilterDriver* _filter) : FilterObject(std::move(_next)), + filter(_filter) {} + D4NFilterObject(std::unique_ptr _next, Bucket* _bucket, D4NFilterDriver* _filter) : FilterObject(std::move(_next), _bucket), + filter(_filter) {} + D4NFilterObject(D4NFilterObject& _o, D4NFilterDriver* _filter) : FilterObject(_o), + filter(_filter) {} + virtual ~D4NFilterObject() = default; + + virtual int copy_object(User* user, + req_info* info, const rgw_zone_id& source_zone, + rgw::sal::Object* dest_object, rgw::sal::Bucket* dest_bucket, + rgw::sal::Bucket* src_bucket, + const rgw_placement_rule& dest_placement, + ceph::real_time* src_mtime, ceph::real_time* mtime, + const ceph::real_time* mod_ptr, const ceph::real_time* unmod_ptr, + bool high_precision_time, + const char* if_match, const char* if_nomatch, + AttrsMod attrs_mod, bool copy_if_newer, Attrs& attrs, + RGWObjCategory category, uint64_t olh_epoch, + boost::optional delete_at, + std::string* version_id, std::string* tag, std::string* etag, + void (*progress_cb)(off_t, void *), void* progress_data, + const DoutPrefixProvider* dpp, optional_yield y) override; + virtual const std::string &get_name() const override { return next->get_name(); } + virtual int set_obj_attrs(const DoutPrefixProvider* dpp, Attrs* setattrs, + Attrs* delattrs, optional_yield y) override; + virtual int get_obj_attrs(optional_yield y, const DoutPrefixProvider* dpp, + rgw_obj* target_obj = NULL) override; + virtual int modify_obj_attrs(const char* attr_name, bufferlist& attr_val, + optional_yield y, const DoutPrefixProvider* dpp) override; + virtual int delete_obj_attrs(const DoutPrefixProvider* dpp, const char* attr_name, + optional_yield y) override; + + virtual std::unique_ptr get_read_op() override; + virtual std::unique_ptr get_delete_op() override; +}; + +class D4NFilterWriter : public FilterWriter { + private: + D4NFilterDriver* filter; + const DoutPrefixProvider* save_dpp; + bool atomic; + + public: + D4NFilterWriter(std::unique_ptr _next, D4NFilterDriver* _filter, Object* _obj, + const DoutPrefixProvider* _dpp) : FilterWriter(std::move(_next), _obj), + filter(_filter), + save_dpp(_dpp), atomic(false) {} + D4NFilterWriter(std::unique_ptr _next, D4NFilterDriver* _filter, Object* _obj, + const DoutPrefixProvider* _dpp, bool _atomic) : FilterWriter(std::move(_next), _obj), + filter(_filter), + save_dpp(_dpp), atomic(_atomic) {} + virtual ~D4NFilterWriter() = default; + + virtual int prepare(optional_yield y); + virtual int process(bufferlist&& data, uint64_t offset) override; + virtual int complete(size_t accounted_size, const std::string& etag, + ceph::real_time *mtime, ceph::real_time set_mtime, + std::map& attrs, + ceph::real_time delete_at, + const char *if_match, const char *if_nomatch, + const std::string *user_data, + rgw_zone_set *zones_trace, bool *canceled, + optional_yield y) override; + bool is_atomic() { return atomic; }; + const DoutPrefixProvider* dpp() { return save_dpp; } +}; + +} } // namespace rgw::sal diff --git a/src/rgw/rgw_sal.cc b/src/rgw/rgw_sal.cc index 714b10abf19..bda725d64a4 100644 --- a/src/rgw/rgw_sal.cc +++ b/src/rgw/rgw_sal.cc @@ -31,6 +31,9 @@ #include "rgw_sal_dbstore.h" #include "driver/dbstore/config/store.h" #endif +#ifdef WITH_RADOSGW_D4N +#include "driver/d4n/rgw_sal_d4n.h" +#endif #ifdef WITH_RADOSGW_MOTR #include "rgw_sal_motr.h" @@ -54,7 +57,9 @@ extern rgw::sal::Driver* newMotrStore(CephContext *cct); extern rgw::sal::Driver* newDaosStore(CephContext *cct); #endif extern rgw::sal::Driver* newBaseFilter(rgw::sal::Driver* next); - +#ifdef WITH_RADOSGW_D4N +extern rgw::sal::Driver* newD4NFilter(rgw::sal::Driver* next); +#endif } RGWObjState::RGWObjState() { @@ -217,12 +222,24 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider* rgw::sal::Driver* next = driver; driver = newBaseFilter(next); + if (driver->initialize(cct, dpp) < 0) { + delete driver; + delete next; + return nullptr; + } + } +#ifdef WITH_RADOSGW_D4N + else if (cfg.filter_name.compare("d4n") == 0) { + rgw::sal::Driver* next = driver; + driver = newD4NFilter(next); + if (driver->initialize(cct, dpp) < 0) { delete driver; delete next; return nullptr; } } +#endif return driver; } @@ -349,6 +366,11 @@ DriverManager::Config DriverManager::get_config(bool admin, CephContext* cct) if (config_filter == "base") { cfg.filter_name = "base"; } +#ifdef WITH_RADOSGW_D4N + else if (config_filter == "d4n") { + cfg.filter_name= "d4n"; + } +#endif return cfg; } diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index 2caf02f87ef..69436e13171 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -509,6 +509,10 @@ class User { virtual uint32_t get_type() const = 0; /** Get the maximum number of buckets allowed for this User */ virtual int32_t get_max_buckets() const = 0; + /** Set the maximum number of buckets allowed for this User */ + virtual void set_max_buckets(int32_t _max_buckets) = 0; + /** Set quota info */ + virtual void set_info(RGWQuotaInfo& _quota) = 0; /** Get the capabilities for this User */ virtual const RGWUserCaps& get_caps() const = 0; /** Get the version tracker for this User */ @@ -713,6 +717,10 @@ class Bucket { virtual int set_tag_timeout(const DoutPrefixProvider *dpp, uint64_t timeout) = 0; /** Remove this specific bucket instance from the backing store. May be removed from API */ virtual int purge_instance(const DoutPrefixProvider* dpp) = 0; + /** Set the cached object count of this bucket */ + virtual void set_count(uint64_t _count) = 0; + /** Set the cached size of this bucket */ + virtual void set_size(uint64_t _size) = 0; /** Check if this instantiation is empty */ virtual bool empty() const = 0; @@ -990,6 +998,8 @@ class Object { /** Get the object state for this object. Will be removed in the future */ virtual int get_obj_state(const DoutPrefixProvider* dpp, RGWObjState **state, optional_yield y, bool follow_olh = true) = 0; + /** Set the object state for this object */ + virtual void set_obj_state(RGWObjState& _state) = 0; /** Set attributes for this object from the backing store. Attrs can be set or * deleted. @note the attribute APIs may be revisited in the future. */ virtual int set_obj_attrs(const DoutPrefixProvider* dpp, Attrs* setattrs, Attrs* delattrs, optional_yield y) = 0; diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h index 6260f3acae3..b2ea781b122 100644 --- a/src/rgw/rgw_sal_filter.h +++ b/src/rgw/rgw_sal_filter.h @@ -355,6 +355,8 @@ public: virtual const rgw_user& get_id() const override { return next->get_id(); } virtual uint32_t get_type() const override { return next->get_type(); } virtual int32_t get_max_buckets() const override { return next->get_max_buckets(); } + virtual void set_max_buckets(int32_t _max_buckets) override { return next->set_max_buckets(_max_buckets); } + virtual void set_info(RGWQuotaInfo& _quota) override { return next->set_info(_quota); } virtual const RGWUserCaps& get_caps() const override { return next->get_caps(); } virtual RGWObjVersionTracker& get_version_tracker() override { return next->get_version_tracker(); @@ -469,6 +471,8 @@ public: virtual int rebuild_index(const DoutPrefixProvider *dpp) override; virtual int set_tag_timeout(const DoutPrefixProvider *dpp, uint64_t timeout) override; virtual int purge_instance(const DoutPrefixProvider* dpp) override; + virtual void set_count(uint64_t _count) override { return next->set_count(_count); } + virtual void set_size(uint64_t _size) override { return next->set_size(_size); } virtual bool empty() const override { return next->empty(); } virtual const std::string& get_name() const override { return next->get_name(); } virtual const std::string& get_tenant() const override { return next->get_tenant(); } @@ -605,6 +609,7 @@ public: virtual int get_obj_state(const DoutPrefixProvider* dpp, RGWObjState **state, optional_yield y, bool follow_olh = true) override; + virtual void set_obj_state(RGWObjState& _state) override { return next->set_obj_state(_state); } virtual int set_obj_attrs(const DoutPrefixProvider* dpp, Attrs* setattrs, Attrs* delattrs, optional_yield y) override; virtual int get_obj_attrs(optional_yield y, const DoutPrefixProvider* dpp, diff --git a/src/rgw/rgw_sal_store.h b/src/rgw/rgw_sal_store.h index e4822a6cb6e..b9d9be4c6e5 100644 --- a/src/rgw/rgw_sal_store.h +++ b/src/rgw/rgw_sal_store.h @@ -58,12 +58,20 @@ class StoreUser : public User { virtual const rgw_user& get_id() const override { return info.user_id; } virtual uint32_t get_type() const override { return info.type; } virtual int32_t get_max_buckets() const override { return info.max_buckets; } + virtual void set_max_buckets(int32_t _max_buckets) override { + info.max_buckets = _max_buckets; + } virtual const RGWUserCaps& get_caps() const override { return info.caps; } virtual RGWObjVersionTracker& get_version_tracker() override { return objv_tracker; } virtual Attrs& get_attrs() override { return attrs; } virtual void set_attrs(Attrs& _attrs) override { attrs = _attrs; } virtual bool empty() const override { return info.user_id.id.empty(); } virtual RGWUserInfo& get_info() override { return info; } + virtual void set_info(RGWQuotaInfo& _quota) override { + info.quota.user_quota.max_size = _quota.max_size; + info.quota.user_quota.max_objects = _quota.max_objects; + } + virtual void print(std::ostream& out) const override { out << info.user_id; } friend class StoreBucket; @@ -113,6 +121,12 @@ class StoreBucket : public Bucket { virtual void set_owner(rgw::sal::User* _owner) override { owner = _owner; } + virtual void set_count(uint64_t _count) override { + ent.count = _count; + } + virtual void set_size(uint64_t _size) override { + ent.size = _size; + } virtual User* get_owner(void) override { return owner; }; virtual ACLOwner get_acl_owner(void) override { return ACLOwner(info.owner); }; virtual bool empty() const override { return info.bucket.name.empty(); } @@ -204,6 +218,9 @@ class StoreObject : public Object { virtual bool empty() const override { return state.obj.empty(); } virtual const std::string &get_name() const override { return state.obj.key.name; } + virtual void set_obj_state(RGWObjState& _state) override { + state = _state; + } virtual Attrs& get_attrs(void) override { return state.attrset; } virtual const Attrs& get_attrs(void) const override { return state.attrset; } virtual int set_attrs(Attrs a) override { state.attrset = a; state.has_attrs = true; return 0; } diff --git a/src/test/rgw/CMakeLists.txt b/src/test/rgw/CMakeLists.txt index 96d157b7ced..0f99597c21e 100644 --- a/src/test/rgw/CMakeLists.txt +++ b/src/test/rgw/CMakeLists.txt @@ -20,6 +20,44 @@ if(WITH_JAEGER) list(APPEND rgw_libs ${jaeger_base}) endif() +if(WITH_RADOSGW_D4N) +add_executable(ceph_test_rgw_d4n_directory + test_d4n_directory.cc + ) +target_include_directories(ceph_test_rgw_d4n_directory + PUBLIC "${CMAKE_SOURCE_DIR}/src/dmclock/support/src" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/driver/d4n") +target_link_libraries(ceph_test_rgw_d4n_directory PRIVATE + rgw_common + librados + ceph-common + ${rgw_libs} + ${UNITTEST_LIBS} + ${EXTRALIBS} + ) + target_link_libraries(ceph_test_rgw_d4n_directory PRIVATE spawn) +install(TARGETS ceph_test_rgw_d4n_directory DESTINATION ${CMAKE_INSTALL_BINDIR}) +endif() + +if(WITH_RADOSGW_D4N) +add_executable(ceph_test_rgw_d4n_filter + test_d4n_filter.cc + ) +target_include_directories(ceph_test_rgw_d4n_filter + PUBLIC "${CMAKE_SOURCE_DIR}/src/dmclock/support/src" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/dbstore/common") +target_link_libraries(ceph_test_rgw_d4n_filter PRIVATE + rgw_common + librados + ceph-common + ${rgw_libs} + ${UNITTEST_LIBS} + ${EXTRALIBS} + ) + target_link_libraries(ceph_test_rgw_d4n_filter PRIVATE spawn) +install(TARGETS ceph_test_rgw_d4n_filter DESTINATION ${CMAKE_INSTALL_BINDIR}) +endif() + #unittest_rgw_bencode add_executable(unittest_rgw_bencode test_rgw_bencode.cc) add_ceph_unittest(unittest_rgw_bencode) diff --git a/src/test/rgw/run-d4n-unit-tests.sh b/src/test/rgw/run-d4n-unit-tests.sh new file mode 100755 index 00000000000..3490b8e2bcd --- /dev/null +++ b/src/test/rgw/run-d4n-unit-tests.sh @@ -0,0 +1,20 @@ +#!/bin/bash +ps cax | grep redis-server > /dev/null +if [ $? -eq 0 ]; +then + echo "Redis process found; flushing!" + redis-cli FLUSHALL +fi +redis-server --daemonize yes +echo "-----------Redis Server Started-----------" +../../../build/bin/ceph_test_rgw_d4n_directory +printf "\n-----------Directory Test Executed-----------\n" +redis-cli FLUSHALL +echo "-----------Redis Server Flushed-----------" +../../../build/bin/ceph_test_rgw_d4n_filter +printf "\n-----------Filter Test Executed-----------\n" +redis-cli FLUSHALL +echo "-----------Redis Server Flushed-----------" +REDIS_PID=$(lsof -i4TCP:6379 -sTCP:LISTEN -t) +kill $REDIS_PID +echo "-----------Redis Server Stopped-----------" diff --git a/src/test/rgw/test_d4n_directory.cc b/src/test/rgw/test_d4n_directory.cc new file mode 100644 index 00000000000..8aaf2acb0a6 --- /dev/null +++ b/src/test/rgw/test_d4n_directory.cc @@ -0,0 +1,206 @@ +#include "d4n_directory.h" +#include "rgw_process_env.h" +#include +#include +#include +#include "gtest/gtest.h" + +using namespace std; + +string portStr; +string hostStr; +string redisHost = ""; +string oid = "samoid"; +string bucketName = "testBucket"; +int blkSize = 123; + +class DirectoryFixture: public ::testing::Test { + protected: + virtual void SetUp() { + blk_dir = new RGWBlockDirectory(hostStr, stoi(portStr)); + c_blk = new cache_block(); + + c_blk->hosts_list.push_back(redisHost); + c_blk->size_in_bytes = blkSize; + c_blk->c_obj.bucket_name = bucketName; + c_blk->c_obj.obj_name = oid; + } + + virtual void TearDown() { + delete blk_dir; + blk_dir = nullptr; + + delete c_blk; + c_blk = nullptr; + } + + RGWBlockDirectory* blk_dir; + cache_block* c_blk; +}; + +/* Successful initialization */ +TEST_F(DirectoryFixture, DirectoryInit) { + ASSERT_NE(blk_dir, nullptr); + ASSERT_NE(c_blk, nullptr); + ASSERT_NE(redisHost.length(), (long unsigned int)0); +} + +/* Successful setValue Call and Redis Check */ +TEST_F(DirectoryFixture, SetValueTest) { + cpp_redis::client client; + int key_exist = -1; + string key; + string hosts; + string size; + string bucket_name; + string obj_name; + std::vector fields; + int setReturn = blk_dir->setValue(c_blk); + + ASSERT_EQ(setReturn, 0); + + fields.push_back("key"); + fields.push_back("hosts"); + fields.push_back("size"); + fields.push_back("bucket_name"); + fields.push_back("obj_name"); + + client.connect(hostStr, stoi(portStr), nullptr, 0, 5, 1000); + ASSERT_EQ((bool)client.is_connected(), (bool)1); + + client.hmget("rgw-object:" + oid + ":directory", fields, [&key, &hosts, &size, &bucket_name, &obj_name, &key_exist](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + key_exist = 0; + key = arr[0].as_string(); + hosts = arr[1].as_string(); + size = arr[2].as_string(); + bucket_name = arr[3].as_string(); + obj_name = arr[4].as_string(); + } + }); + + client.sync_commit(); + + EXPECT_EQ(key_exist, 0); + EXPECT_EQ(key, "rgw-object:" + oid + ":directory"); + EXPECT_EQ(hosts, redisHost); + EXPECT_EQ(size, to_string(blkSize)); + EXPECT_EQ(bucket_name, bucketName); + EXPECT_EQ(obj_name, oid); + + client.flushall(); +} + +/* Successful getValue Calls and Redis Check */ +TEST_F(DirectoryFixture, GetValueTest) { + cpp_redis::client client; + int key_exist = -1; + string key; + string hosts; + string size; + string bucket_name; + string obj_name; + std::vector fields; + int setReturn = blk_dir->setValue(c_blk); + + ASSERT_EQ(setReturn, 0); + + fields.push_back("key"); + fields.push_back("hosts"); + fields.push_back("size"); + fields.push_back("bucket_name"); + fields.push_back("obj_name"); + + client.connect(hostStr, stoi(portStr), nullptr, 0, 5, 1000); + ASSERT_EQ((bool)client.is_connected(), (bool)1); + + client.hmget("rgw-object:" + oid + ":directory", fields, [&key, &hosts, &size, &bucket_name, &obj_name, &key_exist](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + key_exist = 0; + key = arr[0].as_string(); + hosts = arr[1].as_string(); + size = arr[2].as_string(); + bucket_name = arr[3].as_string(); + obj_name = arr[4].as_string(); + } + }); + + client.sync_commit(); + + EXPECT_EQ(key_exist, 0); + EXPECT_EQ(key, "rgw-object:" + oid + ":directory"); + EXPECT_EQ(hosts, redisHost); + EXPECT_EQ(size, to_string(blkSize)); + EXPECT_EQ(bucket_name, bucketName); + EXPECT_EQ(obj_name, oid); + + /* Check if object name in directory instance matches redis update */ + client.hset("rgw-object:" + oid + ":directory", "obj_name", "newoid", [](cpp_redis::reply& reply) { + if (reply.is_integer()) { + ASSERT_EQ(reply.as_integer(), 0); /* Zero keys exist */ + } + }); + + client.sync_commit(); + + int getReturn = blk_dir->getValue(c_blk); + + ASSERT_EQ(getReturn, 0); + EXPECT_EQ(c_blk->c_obj.obj_name, "newoid"); + + client.flushall(); +} + +/* Successful delValue Call and Redis Check */ +TEST_F(DirectoryFixture, DelValueTest) { + cpp_redis::client client; + vector keys; + int setReturn = blk_dir->setValue(c_blk); + + ASSERT_EQ(setReturn, 0); + + /* Ensure cache entry exists in cache before deletion */ + keys.push_back("rgw-object:" + oid + ":directory"); + + client.exists(keys, [](cpp_redis::reply& reply) { + if (reply.is_integer()) { + ASSERT_EQ(reply.as_integer(), 1); + } + }); + + int delReturn = blk_dir->delValue(c_blk); + + ASSERT_EQ(delReturn, 0); + + client.exists(keys, [](cpp_redis::reply& reply) { + if (reply.is_integer()) { + ASSERT_EQ(reply.as_integer(), 0); /* Zero keys exist */ + } + }); + + client.flushall(); +} + +int main(int argc, char *argv[]) { + ::testing::InitGoogleTest(&argc, argv); + + /* Other ports can be passed to the program */ + if (argc == 1) { + portStr = "6379"; + hostStr = "127.0.0.1"; + } else if (argc == 3) { + hostStr = argv[1]; + portStr = argv[2]; + } else { + cout << "Incorrect number of arguments." << std::endl; + return -1; + } + + redisHost = hostStr + ":" + portStr; + + return RUN_ALL_TESTS(); +} diff --git a/src/test/rgw/test_d4n_filter.cc b/src/test/rgw/test_d4n_filter.cc new file mode 100644 index 00000000000..4d056a9d0f7 --- /dev/null +++ b/src/test/rgw/test_d4n_filter.cc @@ -0,0 +1,1977 @@ +#include "gtest/gtest.h" +#include "common/ceph_context.h" +#include +#include +#include "rgw_process_env.h" +#include +#include "driver/dbstore/common/dbstore.h" +#include "rgw_sal_store.h" +#include "driver/d4n/rgw_sal_d4n.h" + +#include "rgw_sal.h" +#include "rgw_auth.h" +#include "rgw_auth_registry.h" + +#define dout_subsys ceph_subsys_rgw + +#define METADATA_LENGTH 22 + +using namespace std; + +string portStr; +string hostStr; +string redisHost = ""; + +vector args; +class Environment* env; +const DoutPrefixProvider* dpp; + +class StoreObject : public rgw::sal::StoreObject { + friend class D4NFilterFixture; + FRIEND_TEST(D4NFilterFixture, StoreGetMetadata); +}; + +class Environment : public ::testing::Environment { + public: + Environment() {} + + virtual ~Environment() {} + + void SetUp() override { + /* Ensure redis instance is running */ + try { + env_client.connect(hostStr, stoi(portStr), nullptr, 0, 5, 1000); + } catch (std::exception &e) { + std::cerr << "[ ] ERROR: Redis instance not running." << std::endl; + } + + ASSERT_EQ((bool)env_client.is_connected(), (bool)1); + + /* Proceed with environment setup */ + code_environment_t code_env; + global_pre_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, code_env, 0); + cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, CINIT_FLAG_NO_MON_CONFIG, false); + + dpp = new DoutPrefix(cct->get(), dout_subsys, "d4n test: "); + DriverManager::Config cfg; + + cfg.store_name = "dbstore"; + cfg.filter_name = "d4n"; + + driver = DriverManager::get_storage(dpp, dpp->get_cct(), + cfg, + false, + false, + false, + false, + false, + false, + false); + + ASSERT_NE(driver, nullptr); + } + + void TearDown() override { + if (env_client.is_connected()) { + delete driver; + delete dpp; + + env_client.disconnect(); + } + } + + boost::intrusive_ptr cct; + rgw::sal::Driver* driver; + cpp_redis::client env_client; +}; + +class D4NFilterFixture : public ::testing::Test { + protected: + rgw::sal::Driver* driver; + unique_ptr testUser = nullptr; + unique_ptr testBucket = nullptr; + unique_ptr testWriter = nullptr; + + public: + D4NFilterFixture() {} + + void SetUp() { + driver = env->driver; + } + + void TearDown() {} + + int createUser() { + rgw_user u("test_tenant", "test_user", "ns"); + + testUser = driver->get_user(u); + testUser->get_info().user_id = u; + + int ret = testUser->store_user(dpp, null_yield, false); + + return ret; + } + + int createBucket() { + rgw_bucket b; + string zonegroup_id = "test_id"; + rgw_placement_rule placement_rule; + string swift_ver_location = "test_location"; + const RGWAccessControlPolicy policy; + rgw::sal::Attrs attrs; + RGWBucketInfo info; + obj_version ep_objv; + bool bucket_exists; + int ret; + + CephContext* cct = get_pointer(env->cct); + RGWProcessEnv penv; + RGWEnv rgw_env; + req_state s(cct->get(), penv, &rgw_env, 0); + req_info _req_info = s.info; + + b.name = "test_bucket"; + placement_rule.storage_class = "test_sc"; + + ret = testUser->create_bucket(dpp, b, + zonegroup_id, + placement_rule, + swift_ver_location, + nullptr, + policy, + attrs, + info, + ep_objv, + false, + false, + &bucket_exists, + _req_info, + &testBucket, + null_yield); + + return ret; + } + + int putObject(string name) { + string object_name = "test_object_" + name; + unique_ptr obj = testBucket->get_object(rgw_obj_key(object_name)); + rgw_user owner; + rgw_placement_rule ptail_placement_rule; + uint64_t olh_epoch = 123; + string unique_tag; + + obj->get_obj_attrs(null_yield, dpp); + + testWriter = driver->get_atomic_writer(dpp, + null_yield, + obj.get(), + owner, + &ptail_placement_rule, + olh_epoch, + unique_tag); + + size_t accounted_size = 4; + string etag("test_etag"); + ceph::real_time mtime; + ceph::real_time set_mtime; + + buffer::list bl; + string tmp = "test_attrs_value_" + name; + bl.append("test_attrs_value_" + name); + map attrs{{"test_attrs_key_" + name, bl}}; + + ceph::real_time delete_at; + char if_match; + char if_nomatch; + string user_data; + rgw_zone_set zones_trace; + bool canceled; + + int ret = testWriter->complete(accounted_size, etag, + &mtime, set_mtime, + attrs, + delete_at, + &if_match, &if_nomatch, + &user_data, + &zones_trace, &canceled, + null_yield); + + return ret; + } + + void clientSetUp(cpp_redis::client* client) { + client->connect(hostStr, stoi(portStr), nullptr, 0, 5, 1000); + ASSERT_EQ((bool)client->is_connected(), (bool)1); + + client->flushdb([](cpp_redis::reply& reply) {}); + client->sync_commit(); + } + + void clientReset(cpp_redis::client* client) { + client->flushdb([](cpp_redis::reply& reply) {}); + client->sync_commit(); + } +}; + +/* General operation-related tests */ +TEST_F(D4NFilterFixture, CreateUser) { + EXPECT_EQ(createUser(), 0); + EXPECT_NE(testUser, nullptr); +} + +TEST_F(D4NFilterFixture, CreateBucket) { + ASSERT_EQ(createUser(), 0); + ASSERT_NE(testUser, nullptr); + + EXPECT_EQ(createBucket(), 0); + EXPECT_NE(testBucket, nullptr); +} + +TEST_F(D4NFilterFixture, PutObject) { + cpp_redis::client client; + vector fields; + fields.push_back("test_attrs_key_0"); + clientSetUp(&client); + + ASSERT_EQ(createUser(), 0); + ASSERT_NE(testUser, nullptr); + + ASSERT_EQ(createBucket(), 0); + ASSERT_NE(testBucket, nullptr); + + EXPECT_EQ(putObject("PutObject"), 0); + EXPECT_NE(testWriter, nullptr); + + client.hgetall("rgw-object:test_object_PutObject:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 2 + METADATA_LENGTH); + } + }); + + client.sync_commit(); + + client.hmget("rgw-object:test_object_PutObject:cache", fields, [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ(arr[0].as_string(), "test_attrs_value_PutObject"); + } + }); + + client.sync_commit(); + + clientReset(&client); +} + +TEST_F(D4NFilterFixture, GetObject) { + cpp_redis::client client; + vector fields; + fields.push_back("test_attrs_key_GetObject"); + clientSetUp(&client); + + ASSERT_EQ(createUser(), 0); + ASSERT_NE(testUser, nullptr); + + ASSERT_EQ(createBucket(), 0); + ASSERT_NE(testBucket, nullptr); + + ASSERT_EQ(putObject("GetObject"), 0); + ASSERT_NE(testWriter, nullptr); + + unique_ptr testObject_GetObject = testBucket->get_object(rgw_obj_key("test_object_GetObject")); + + EXPECT_NE(testObject_GetObject, nullptr); + + static rgw::sal::Object* nextObject = dynamic_cast(testObject_GetObject.get())->get_next(); + + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + + unique_ptr testROp = testObject_GetObject->get_read_op(); + + EXPECT_NE(testROp, nullptr); + EXPECT_EQ(testROp->prepare(null_yield, dpp), 0); + + client.hgetall("rgw-object:test_object_GetObject:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 2 + METADATA_LENGTH); + } + }); + + client.sync_commit(); + + client.hmget("rgw-object:test_object_GetObject:cache", fields, [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ(arr[0].as_string(), "test_attrs_value_GetObject"); + } + }); + + client.sync_commit(); + + clientReset(&client); +} + +TEST_F(D4NFilterFixture, CopyObjectNone) { + cpp_redis::client client; + vector fields; + fields.push_back("test_attrs_key_CopyObjectNone"); + clientSetUp(&client); + + createUser(); + createBucket(); + putObject("CopyObjectNone"); + unique_ptr testObject_CopyObjectNone = testBucket->get_object(rgw_obj_key("test_object_CopyObjectNone")); + + ASSERT_NE(testObject_CopyObjectNone, nullptr); + + static rgw::sal::Object* nextObject = dynamic_cast(testObject_CopyObjectNone.get())->get_next(); + + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + ASSERT_NE(nextObject->get_attrs().empty(), true); + + /* Update object */ + RGWEnv rgw_env; + req_info info(get_pointer(env->cct), &rgw_env); + rgw_zone_id source_zone; + rgw_placement_rule dest_placement; + ceph::real_time src_mtime; + ceph::real_time mtime; + ceph::real_time mod_ptr; + ceph::real_time unmod_ptr; + char if_match; + char if_nomatch; + rgw::sal::AttrsMod attrs_mod = rgw::sal::ATTRSMOD_NONE; + rgw::sal::Attrs attrs; + RGWObjCategory category = RGWObjCategory::Main; + uint64_t olh_epoch = 0; + ceph::real_time delete_at; + string tag; + string etag; + + EXPECT_EQ(testObject_CopyObjectNone->copy_object(testUser.get(), + &info, source_zone, testObject_CopyObjectNone.get(), + testBucket.get(), testBucket.get(), + dest_placement, &src_mtime, &mtime, + &mod_ptr, &unmod_ptr, false, + &if_match, &if_nomatch, attrs_mod, + false, attrs, category, olh_epoch, + delete_at, NULL, &tag, &etag, + NULL, NULL, dpp, null_yield), 0); + + client.hgetall("rgw-object:test_object_CopyObjectNone:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 2 + METADATA_LENGTH); + } + }); + + client.sync_commit(); + + client.hmget("rgw-object:test_object_CopyObjectNone:cache", fields, [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ(arr[0].as_string(), "test_attrs_value_CopyObjectNone"); + } + }); + + client.sync_commit(); +} + +TEST_F(D4NFilterFixture, CopyObjectReplace) { + cpp_redis::client client; + vector fields; + clientSetUp(&client); + + createUser(); + createBucket(); + putObject("CopyObjectReplace"); + unique_ptr testObject_CopyObjectReplace = testBucket->get_object(rgw_obj_key("test_object_CopyObjectReplace")); + + ASSERT_NE(testObject_CopyObjectReplace, nullptr); + + static rgw::sal::Object* nextObject = dynamic_cast(testObject_CopyObjectReplace.get())->get_next(); + + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + ASSERT_NE(nextObject->get_attrs().empty(), true); + + /* Copy to new object */ + unique_ptr testWriterCopy = nullptr; + unique_ptr obj = testBucket->get_object(rgw_obj_key("test_object_copy")); + rgw_user owner; + rgw_placement_rule ptail_placement_rule; + uint64_t olh_epoch_copy = 123; + string unique_tag; + + obj->get_obj_attrs(null_yield, dpp); + + testWriterCopy = driver->get_atomic_writer(dpp, + null_yield, + obj.get(), + owner, + &ptail_placement_rule, + olh_epoch_copy, + unique_tag); + + RGWEnv rgw_env; + size_t accounted_size = 0; + req_info info(get_pointer(env->cct), &rgw_env); + rgw_zone_id source_zone; + rgw_placement_rule dest_placement; + ceph::real_time src_mtime; + ceph::real_time mtime; + ceph::real_time set_mtime; + ceph::real_time mod_ptr; + ceph::real_time unmod_ptr; + rgw::sal::AttrsMod attrs_mod = rgw::sal::ATTRSMOD_REPLACE; + char if_match; + char if_nomatch; + RGWObjCategory category = RGWObjCategory::Main; + uint64_t olh_epoch = 0; + ceph::real_time delete_at; + string tag; + string etag("test_etag_copy"); + + /* Attribute to replace */ + buffer::list bl; + bl.append("test_attrs_copy_value"); + rgw::sal::Attrs attrs{{"test_attrs_key_CopyObjectReplace", bl}}; + + string user_data; + rgw_zone_set zones_trace; + bool canceled; + + ASSERT_EQ(testWriterCopy->complete(accounted_size, etag, + &mtime, set_mtime, + attrs, + delete_at, + &if_match, &if_nomatch, + &user_data, + &zones_trace, &canceled, + null_yield), 0); + + unique_ptr testObject_copy = testBucket->get_object(rgw_obj_key("test_object_copy")); + + EXPECT_EQ(testObject_CopyObjectReplace->copy_object(testUser.get(), + &info, source_zone, testObject_copy.get(), + testBucket.get(), testBucket.get(), + dest_placement, &src_mtime, &mtime, + &mod_ptr, &unmod_ptr, false, + &if_match, &if_nomatch, attrs_mod, + false, attrs, category, olh_epoch, + delete_at, NULL, &tag, &etag, + NULL, NULL, dpp, null_yield), 0); + + /* Ensure the original object is still in the cache */ + vector keys; + keys.push_back("rgw-object:test_object_CopyObjectReplace:cache"); + + client.exists(keys, [](cpp_redis::reply& reply) { + if (reply.is_integer()) { + EXPECT_EQ(reply.as_integer(), 1); + } + }); + + client.sync_commit(); + + /* Check copy */ + client.hgetall("rgw-object:test_object_copy:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 4 + METADATA_LENGTH); /* With etag */ + } + }); + + client.sync_commit(); + + fields.push_back("test_attrs_key_CopyObjectReplace"); + + client.hmget("rgw-object:test_object_copy:cache", fields, [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ(arr[0].as_string(), "test_attrs_copy_value"); + } + }); + + client.sync_commit(); + + clientReset(&client); +} + +TEST_F(D4NFilterFixture, CopyObjectMerge) { + cpp_redis::client client; + vector fields; + clientSetUp(&client); + + createUser(); + createBucket(); + putObject("CopyObjectMerge"); + unique_ptr testObject_CopyObjectMerge = testBucket->get_object(rgw_obj_key("test_object_CopyObjectMerge")); + + ASSERT_NE(testObject_CopyObjectMerge, nullptr); + + static rgw::sal::Object* nextObject = dynamic_cast(testObject_CopyObjectMerge.get())->get_next(); + + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + ASSERT_NE(nextObject->get_attrs().empty(), true); + + /* Copy to new object */ + unique_ptr testWriterCopy = nullptr; + string object_name = "test_object_copy"; + unique_ptr obj = testBucket->get_object(rgw_obj_key(object_name)); + rgw_user owner; + rgw_placement_rule ptail_placement_rule; + uint64_t olh_epoch_copy = 123; + string unique_tag; + + obj->get_obj_attrs(null_yield, dpp); + + testWriterCopy = driver->get_atomic_writer(dpp, + null_yield, + obj.get(), + owner, + &ptail_placement_rule, + olh_epoch_copy, + unique_tag); + + RGWEnv rgw_env; + size_t accounted_size = 4; + req_info info(get_pointer(env->cct), &rgw_env); + rgw_zone_id source_zone; + rgw_placement_rule dest_placement; + ceph::real_time src_mtime; + ceph::real_time mtime; + ceph::real_time set_mtime; + ceph::real_time mod_ptr; + ceph::real_time unmod_ptr; + rgw::sal::AttrsMod attrs_mod = rgw::sal::ATTRSMOD_MERGE; + char if_match; + char if_nomatch; + RGWObjCategory category = RGWObjCategory::Main; + uint64_t olh_epoch = 0; + ceph::real_time delete_at; + string tag; + string etag("test_etag_copy"); + + buffer::list bl; + bl.append("bad_value"); + rgw::sal::Attrs attrs{{"test_attrs_key_CopyObjectMerge", bl}}; /* Existing attr */ + bl.clear(); + bl.append("test_attrs_copy_extra_value"); + attrs.insert({"test_attrs_copy_extra_key", bl}); /* New attr */ + + string user_data; + rgw_zone_set zones_trace; + bool canceled; + + ASSERT_EQ(testWriterCopy->complete(accounted_size, etag, + &mtime, set_mtime, + attrs, + delete_at, + &if_match, &if_nomatch, + &user_data, + &zones_trace, &canceled, + null_yield), 0); + + unique_ptr testObject_copy = testBucket->get_object(rgw_obj_key("test_object_copy")); + + EXPECT_EQ(testObject_CopyObjectMerge->copy_object(testUser.get(), + &info, source_zone, testObject_copy.get(), + testBucket.get(), testBucket.get(), + dest_placement, &src_mtime, &mtime, + &mod_ptr, &unmod_ptr, false, + &if_match, &if_nomatch, attrs_mod, + false, attrs, category, olh_epoch, + delete_at, NULL, &tag, &etag, + NULL, NULL, dpp, null_yield), 0); + + /* Ensure the original object is still in the cache */ + vector keys; + keys.push_back("rgw-object:test_object_CopyObjectMerge:cache"); + + client.exists(keys, [](cpp_redis::reply& reply) { + if (reply.is_integer()) { + EXPECT_EQ(reply.as_integer(), 1); + } + }); + + client.sync_commit(); + + /* Check copy */ + client.hgetall("rgw-object:test_object_copy:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 6 + METADATA_LENGTH); /* With etag */ + } + }); + + client.sync_commit(); + + fields.push_back("test_attrs_key_CopyObjectMerge"); + fields.push_back("test_attrs_copy_extra_key"); + + client.hmget("rgw-object:test_object_copy:cache", fields, [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ(arr[0].as_string(), "test_attrs_value_CopyObjectMerge"); + EXPECT_EQ(arr[1].as_string(), "test_attrs_copy_extra_value"); + } + }); + + client.sync_commit(); + + clientReset(&client); +} + +TEST_F(D4NFilterFixture, DelObject) { + cpp_redis::client client; + vector keys; + keys.push_back("rgw-object:test_object_DelObject:cache"); + clientSetUp(&client); + + ASSERT_EQ(createUser(), 0); + ASSERT_NE(testUser, nullptr); + + ASSERT_EQ(createBucket(), 0); + ASSERT_NE(testBucket, nullptr); + + ASSERT_EQ(putObject("DelObject"), 0); + ASSERT_NE(testWriter, nullptr); + + /* Check the object exists before delete op */ + client.exists(keys, [](cpp_redis::reply& reply) { + if (reply.is_integer()) { + EXPECT_EQ(reply.as_integer(), 1); + } + }); + + client.sync_commit(); + + unique_ptr testObject_DelObject = testBucket->get_object(rgw_obj_key("test_object_DelObject")); + + EXPECT_NE(testObject_DelObject, nullptr); + + unique_ptr testDOp = testObject_DelObject->get_delete_op(); + + EXPECT_NE(testDOp, nullptr); + EXPECT_EQ(testDOp->delete_obj(dpp, null_yield), 0); + + /* Check the object does not exist after delete op */ + client.exists(keys, [](cpp_redis::reply& reply) { + if (reply.is_integer()) { + EXPECT_EQ(reply.as_integer(), 0); /* Zero keys exist */ + } + }); + + client.sync_commit(); + + clientReset(&client); +} + +/* Attribute-related tests */ +TEST_F(D4NFilterFixture, SetObjectAttrs) { + cpp_redis::client client; + vector fields; + fields.push_back("test_attrs_key_SetObjectAttrs"); + clientSetUp(&client); + + createUser(); + createBucket(); + putObject("SetObjectAttrs"); + unique_ptr testObject_SetObjectAttrs = testBucket->get_object(rgw_obj_key("test_object_SetObjectAttrs")); + + ASSERT_NE(testObject_SetObjectAttrs, nullptr); + + buffer::list bl; + bl.append("test_attrs_value_extra"); + map test_attrs{{"test_attrs_key_extra", bl}}; + fields.push_back("test_attrs_key_extra"); + + EXPECT_EQ(testObject_SetObjectAttrs->set_obj_attrs(dpp, &test_attrs, NULL, null_yield), 0); + + client.hgetall("rgw-object:test_object_SetObjectAttrs:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 4 + METADATA_LENGTH); + } + }); + + client.sync_commit(); + + client.hmget("rgw-object:test_object_SetObjectAttrs:cache", fields, [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ(arr[0].as_string(), "test_attrs_value_SetObjectAttrs"); + EXPECT_EQ(arr[1].as_string(), "test_attrs_value_extra"); + } + }); + + client.sync_commit(); + + clientReset(&client); +} + +TEST_F(D4NFilterFixture, GetObjectAttrs) { + cpp_redis::client client; + vector fields; + fields.push_back("test_attrs_key_GetObjectAttrs"); + clientSetUp(&client); + + createUser(); + createBucket(); + putObject("GetObjectAttrs"); + unique_ptr testObject_GetObjectAttrs = testBucket->get_object(rgw_obj_key("test_object_GetObjectAttrs")); + + ASSERT_NE(testObject_GetObjectAttrs, nullptr); + + buffer::list bl; + bl.append("test_attrs_value_extra"); + map test_attrs{{"test_attrs_key_extra", bl}}; + fields.push_back("test_attrs_key_extra"); + + static rgw::sal::Object* nextObject = dynamic_cast(testObject_GetObjectAttrs.get())->get_next(); + + ASSERT_EQ(testObject_GetObjectAttrs->set_obj_attrs(dpp, &test_attrs, NULL, null_yield), 0); + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + ASSERT_NE(nextObject->get_attrs().empty(), true); + + EXPECT_EQ(testObject_GetObjectAttrs->get_obj_attrs(null_yield, dpp, NULL), 0); + + client.hgetall("rgw-object:test_object_GetObjectAttrs:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 4 + METADATA_LENGTH); + } + }); + + client.sync_commit(); + + client.hmget("rgw-object:test_object_GetObjectAttrs:cache", fields, [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ(arr[0].as_string(), "test_attrs_value_GetObjectAttrs"); + EXPECT_EQ(arr[1].as_string(), "test_attrs_value_extra"); + } + }); + + client.sync_commit(); + + clientReset(&client); +} + +TEST_F(D4NFilterFixture, DelObjectAttrs) { + cpp_redis::client client; + clientSetUp(&client); + + createUser(); + createBucket(); + putObject("DelObjectAttrs"); + unique_ptr testObject_DelObjectAttrs = testBucket->get_object(rgw_obj_key("test_object_DelObjectAttrs")); + + ASSERT_NE(testObject_DelObjectAttrs, nullptr); + + buffer::list bl; + bl.append("test_attrs_value_extra"); + map test_attrs{{"test_attrs_key_extra", bl}}; + + static rgw::sal::Object* nextObject = dynamic_cast(testObject_DelObjectAttrs.get())->get_next(); + + ASSERT_EQ(testObject_DelObjectAttrs->set_obj_attrs(dpp, &test_attrs, NULL, null_yield), 0); + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + ASSERT_NE(nextObject->get_attrs().empty(), true); + + /* Check that the attributes exist before deletion */ + client.hgetall("rgw-object:test_object_DelObjectAttrs:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 4 + METADATA_LENGTH); + } + }); + + client.sync_commit(); + + EXPECT_EQ(testObject_DelObjectAttrs->set_obj_attrs(dpp, NULL, &test_attrs, null_yield), 0); + + /* Check that the attribute does not exist after deletion */ + client.hgetall("rgw-object:test_object_DelObjectAttrs:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 2 + METADATA_LENGTH); + } + }); + + client.sync_commit(); + + client.hexists("rgw-object:test_object_DelObjectAttrs:cache", "test_attrs_key_extra", [](cpp_redis::reply& reply) { + if (reply.is_integer()) { + EXPECT_EQ(reply.as_integer(), 0); + } + }); + + client.sync_commit(); + + clientReset(&client); +} + +TEST_F(D4NFilterFixture, SetLongObjectAttrs) { + cpp_redis::client client; + map test_attrs_long; + vector fields; + fields.push_back("test_attrs_key_SetLongObjectAttrs"); + clientSetUp(&client); + + createUser(); + createBucket(); + putObject("SetLongObjectAttrs"); + unique_ptr testObject_SetLongObjectAttrs = testBucket->get_object(rgw_obj_key("test_object_SetLongObjectAttrs")); + + ASSERT_NE(testObject_SetLongObjectAttrs, nullptr); + + for (int i = 0; i < 10; ++i) { + buffer::list bl_tmp; + string tmp_value = "test_attrs_value_extra_" + to_string(i); + bl_tmp.append(tmp_value.data(), strlen(tmp_value.data())); + + string tmp_key = "test_attrs_key_extra_" + to_string(i); + test_attrs_long.insert({tmp_key, bl_tmp}); + fields.push_back(tmp_key); + } + + EXPECT_EQ(testObject_SetLongObjectAttrs->set_obj_attrs(dpp, &test_attrs_long, NULL, null_yield), 0); + + client.hgetall("rgw-object:test_object_SetLongObjectAttrs:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 22 + METADATA_LENGTH); + } + }); + + client.sync_commit(); + + client.hmget("rgw-object:test_object_SetLongObjectAttrs:cache", fields, [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ(arr[0].as_string(), "test_attrs_value_SetLongObjectAttrs"); + + for (int i = 1; i < 11; ++i) { + EXPECT_EQ(arr[i].as_string(), "test_attrs_value_extra_" + to_string(i - 1)); + } + } + }); + + client.sync_commit(); + + clientReset(&client); +} + +TEST_F(D4NFilterFixture, GetLongObjectAttrs) { + cpp_redis::client client; + map test_attrs_long; + vector fields; + fields.push_back("test_attrs_key_GetLongObjectAttrs"); + clientSetUp(&client); + + createUser(); + createBucket(); + putObject("GetLongObjectAttrs"); + unique_ptr testObject_GetLongObjectAttrs = testBucket->get_object(rgw_obj_key("test_object_GetLongObjectAttrs")); + + ASSERT_NE(testObject_GetLongObjectAttrs, nullptr); + + for (int i = 0; i < 10; ++i) { + buffer::list bl_tmp; + string tmp_value = "test_attrs_value_extra_" + to_string(i); + bl_tmp.append(tmp_value.data(), strlen(tmp_value.data())); + + string tmp_key = "test_attrs_key_extra_" + to_string(i); + test_attrs_long.insert({tmp_key, bl_tmp}); + fields.push_back(tmp_key); + } + + static rgw::sal::Object* nextObject = dynamic_cast(testObject_GetLongObjectAttrs.get())->get_next(); + + ASSERT_EQ(testObject_GetLongObjectAttrs->set_obj_attrs(dpp, &test_attrs_long, NULL, null_yield), 0); + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + ASSERT_NE(nextObject->get_attrs().empty(), true); + + EXPECT_EQ(testObject_GetLongObjectAttrs->get_obj_attrs(null_yield, dpp, NULL), 0); + + client.hgetall("rgw-object:test_object_GetLongObjectAttrs:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 22 + METADATA_LENGTH); + } + }); + + client.sync_commit(); + + client.hmget("rgw-object:test_object_GetLongObjectAttrs:cache", fields, [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ(arr[0].as_string(), "test_attrs_value_GetLongObjectAttrs"); + + for (int i = 1; i < 11; ++i) { + EXPECT_EQ(arr[i].as_string(), "test_attrs_value_extra_" + to_string(i - 1)); + } + } + }); + + client.sync_commit(); + + clientReset(&client); +} + +TEST_F(D4NFilterFixture, ModifyObjectAttr) { + cpp_redis::client client; + map test_attrs_long; + vector fields; + fields.push_back("test_attrs_key_ModifyObjectAttr"); + clientSetUp(&client); + + createUser(); + createBucket(); + putObject("ModifyObjectAttr"); + unique_ptr testObject_ModifyObjectAttr = testBucket->get_object(rgw_obj_key("test_object_ModifyObjectAttr")); + + ASSERT_NE(testObject_ModifyObjectAttr, nullptr); + + for (int i = 0; i < 10; ++i) { + buffer::list bl_tmp; + string tmp_value = "test_attrs_value_extra_" + to_string(i); + bl_tmp.append(tmp_value.data(), strlen(tmp_value.data())); + + string tmp_key = "test_attrs_key_extra_" + to_string(i); + test_attrs_long.insert({tmp_key, bl_tmp}); + fields.push_back(tmp_key); + } + + static rgw::sal::Object* nextObject = dynamic_cast(testObject_ModifyObjectAttr.get())->get_next(); + + ASSERT_EQ(testObject_ModifyObjectAttr->set_obj_attrs(dpp, &test_attrs_long, NULL, null_yield), 0); + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + ASSERT_NE(nextObject->get_attrs().empty(), true); + + buffer::list bl_tmp; + string tmp_value = "new_test_attrs_value_extra_5"; + bl_tmp.append(tmp_value.data(), strlen(tmp_value.data())); + + EXPECT_EQ(testObject_ModifyObjectAttr->modify_obj_attrs("test_attrs_key_extra_5", bl_tmp, null_yield, dpp), 0); + + client.hgetall("rgw-object:test_object_ModifyObjectAttr:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 22 + METADATA_LENGTH); + } + }); + + client.sync_commit(); + + client.hmget("rgw-object:test_object_ModifyObjectAttr:cache", fields, [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ(arr[0].as_string(), "test_attrs_value_ModifyObjectAttr"); + + for (int i = 1; i < 11; ++i) { + if (i == 6) { + EXPECT_EQ(arr[i].as_string(), "new_test_attrs_value_extra_" + to_string(i - 1)); + } else { + EXPECT_EQ(arr[i].as_string(), "test_attrs_value_extra_" + to_string(i - 1)); + } + } + } + }); + + client.sync_commit(); + + clientReset(&client); +} + +TEST_F(D4NFilterFixture, DelLongObjectAttrs) { + cpp_redis::client client; + map test_attrs_long; + vector fields; + fields.push_back("test_attrs_key_DelLongObjectAttrs"); + clientSetUp(&client); + + createUser(); + createBucket(); + putObject("DelLongObjectAttrs"); + unique_ptr testObject_DelLongObjectAttrs = testBucket->get_object(rgw_obj_key("test_object_DelLongObjectAttrs")); + + ASSERT_NE(testObject_DelLongObjectAttrs, nullptr); + + for (int i = 0; i < 10; ++i) { + buffer::list bl_tmp; + string tmp_value = "test_attrs_value_extra_" + to_string(i); + bl_tmp.append(tmp_value.data(), strlen(tmp_value.data())); + + string tmp_key = "test_attrs_key_extra_" + to_string(i); + test_attrs_long.insert({tmp_key, bl_tmp}); + fields.push_back(tmp_key); + } + + static rgw::sal::Object* nextObject = dynamic_cast(testObject_DelLongObjectAttrs.get())->get_next(); + + ASSERT_EQ(testObject_DelLongObjectAttrs->set_obj_attrs(dpp, &test_attrs_long, NULL, null_yield), 0); + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + ASSERT_NE(nextObject->get_attrs().empty(), true); + + /* Check that the attributes exist before deletion */ + client.hgetall("rgw-object:test_object_DelLongObjectAttrs:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 22 + METADATA_LENGTH); + } + }); + + client.sync_commit(); + + EXPECT_EQ(testObject_DelLongObjectAttrs->set_obj_attrs(dpp, NULL, &test_attrs_long, null_yield), 0); + + /* Check that the attributes do not exist after deletion */ + client.hgetall("rgw-object:test_object_DelLongObjectAttrs:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 2 + METADATA_LENGTH); + + for (int i = 0; i < (int)arr.size(); ++i) { + EXPECT_EQ((int)arr[i].as_string().find("extra"), -1); + } + } + }); + + client.sync_commit(); + + clientReset(&client); +} + +TEST_F(D4NFilterFixture, DelObjectAttr) { + cpp_redis::client client; + map test_attrs_long; + vector fields; + fields.push_back("test_attrs_key_DelObjectAttr"); + clientSetUp(&client); + + createUser(); + createBucket(); + putObject("DelObjectAttr"); + unique_ptr testObject_DelObjectAttr = testBucket->get_object(rgw_obj_key("test_object_DelObjectAttr")); + + ASSERT_NE(testObject_DelObjectAttr, nullptr); + + for (int i = 0; i < 10; ++i) { + buffer::list bl_tmp; + string tmp_value = "test_attrs_value_extra_" + to_string(i); + bl_tmp.append(tmp_value.data(), strlen(tmp_value.data())); + + string tmp_key = "test_attrs_key_extra_" + to_string(i); + test_attrs_long.insert({tmp_key, bl_tmp}); + fields.push_back(tmp_key); + } + + static rgw::sal::Object* nextObject = dynamic_cast(testObject_DelObjectAttr.get())->get_next(); + + ASSERT_EQ(testObject_DelObjectAttr->set_obj_attrs(dpp, &test_attrs_long, NULL, null_yield), 0); + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + ASSERT_NE(nextObject->get_attrs().empty(), true); + + /* Check that the attribute exists before deletion */ + client.hgetall("rgw-object:test_object_DelObjectAttr:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 22 + METADATA_LENGTH); + } + }); + + client.sync_commit(); + + EXPECT_EQ(testObject_DelObjectAttr->delete_obj_attrs(dpp, "test_attrs_key_extra_5", null_yield), 0); + + /* Check that the attribute does not exist after deletion */ + client.hgetall("rgw-object:test_object_DelObjectAttr:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 20 + METADATA_LENGTH); + } + }); + + client.sync_commit(); + + client.hexists("rgw-object:test_object_DelObjectAttr:cache", "test_attrs_key_extra_5", [](cpp_redis::reply& reply) { + if (reply.is_integer()) { + EXPECT_EQ(reply.as_integer(), 0); + } + }); + + client.sync_commit(); + + clientReset(&client); +} + +/* Edge cases */ +TEST_F(D4NFilterFixture, PrepareCopyObject) { + cpp_redis::client client; + vector fields; + fields.push_back("test_attrs_key_PrepareCopyObject"); + clientSetUp(&client); + + createUser(); + createBucket(); + putObject("PrepareCopyObject"); + unique_ptr testObject_PrepareCopyObject = testBucket->get_object(rgw_obj_key("test_object_PrepareCopyObject")); + + ASSERT_NE(testObject_PrepareCopyObject, nullptr); + + static rgw::sal::Object* nextObject = dynamic_cast(testObject_PrepareCopyObject.get())->get_next(); + + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + ASSERT_NE(nextObject->get_attrs().empty(), true); + + unique_ptr testROp = testObject_PrepareCopyObject->get_read_op(); + + ASSERT_NE(testROp, nullptr); + ASSERT_EQ(testROp->prepare(null_yield, dpp), 0); + + /* Update object */ + RGWEnv rgw_env; + req_info info(get_pointer(env->cct), &rgw_env); + rgw_zone_id source_zone; + rgw_placement_rule dest_placement; + ceph::real_time src_mtime; + ceph::real_time mtime; + ceph::real_time mod_ptr; + ceph::real_time unmod_ptr; + char if_match; + char if_nomatch; + rgw::sal::AttrsMod attrs_mod = rgw::sal::ATTRSMOD_NONE; + rgw::sal::Attrs attrs; + RGWObjCategory category = RGWObjCategory::Main; + uint64_t olh_epoch = 0; + ceph::real_time delete_at; + string tag; + string etag; + + EXPECT_EQ(testObject_PrepareCopyObject->copy_object(testUser.get(), + &info, source_zone, testObject_PrepareCopyObject.get(), + testBucket.get(), testBucket.get(), + dest_placement, &src_mtime, &mtime, + &mod_ptr, &unmod_ptr, false, + &if_match, &if_nomatch, attrs_mod, + false, attrs, category, olh_epoch, + delete_at, NULL, &tag, &etag, + NULL, NULL, dpp, null_yield), 0); + + client.hgetall("rgw-object:test_object_PrepareCopyObject:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 2 + METADATA_LENGTH); + } + }); + + client.sync_commit(); + + client.hmget("rgw-object:test_object_PrepareCopyObject:cache", fields, [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ(arr[0].as_string(), "test_attrs_value_PrepareCopyObject"); + } + }); + + client.sync_commit(); + + clientReset(&client); +} + +TEST_F(D4NFilterFixture, SetDelAttrs) { + cpp_redis::client client; + map test_attrs_base; + vector fields; + fields.push_back("test_attrs_key_SetDelAttrs"); + clientSetUp(&client); + + createUser(); + createBucket(); + putObject("SetDelAttrs"); + unique_ptr testObject_SetDelAttrs = testBucket->get_object(rgw_obj_key("test_object_SetDelAttrs")); + + ASSERT_NE(testObject_SetDelAttrs, nullptr); + + for (int i = 0; i < 10; ++i) { + buffer::list bl_tmp; + string tmp_value = "test_attrs_value_extra_" + to_string(i); + bl_tmp.append(tmp_value.data(), strlen(tmp_value.data())); + + string tmp_key = "test_attrs_key_extra_" + to_string(i); + test_attrs_base.insert({tmp_key, bl_tmp}); + } + + static rgw::sal::Object* nextObject = dynamic_cast(testObject_SetDelAttrs.get())->get_next(); + + ASSERT_EQ(testObject_SetDelAttrs->set_obj_attrs(dpp, &test_attrs_base, NULL, null_yield), 0); + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + ASSERT_NE(nextObject->get_attrs().empty(), true); + + /* Attempt to set and delete attrs with the same API call */ + buffer::list bl; + bl.append("test_attrs_value_extra"); + map test_attrs_new{{"test_attrs_key_extra", bl}}; + fields.push_back("test_attrs_key_extra"); + + EXPECT_EQ(testObject_SetDelAttrs->set_obj_attrs(dpp, &test_attrs_new, &test_attrs_base, null_yield), 0); + + client.hgetall("rgw-object:test_object_SetDelAttrs:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 4 + METADATA_LENGTH); + } + }); + + client.sync_commit(); + + client.hmget("rgw-object:test_object_SetDelAttrs:cache", fields, [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ(arr[0].as_string(), "test_attrs_value_SetDelAttrs"); + EXPECT_EQ(arr[1].as_string(), "test_attrs_value_extra"); + } + }); + + client.sync_commit(); + + clientReset(&client); +} + +TEST_F(D4NFilterFixture, ModifyNonexistentAttr) { + cpp_redis::client client; + map test_attrs_base; + vector fields; + fields.push_back("test_attrs_key_ModifyNonexistentAttr"); + clientSetUp(&client); + + createUser(); + createBucket(); + putObject("ModifyNonexistentAttr"); + unique_ptr testObject_ModifyNonexistentAttr = testBucket->get_object(rgw_obj_key("test_object_ModifyNonexistentAttr")); + + ASSERT_NE(testObject_ModifyNonexistentAttr, nullptr); + + for (int i = 0; i < 10; ++i) { + buffer::list bl_tmp; + string tmp_value = "test_attrs_value_extra_" + to_string(i); + bl_tmp.append(tmp_value.data(), strlen(tmp_value.data())); + + string tmp_key = "test_attrs_key_extra_" + to_string(i); + test_attrs_base.insert({tmp_key, bl_tmp}); + fields.push_back(tmp_key); + } + + static rgw::sal::Object* nextObject = dynamic_cast(testObject_ModifyNonexistentAttr.get())->get_next(); + + ASSERT_EQ(testObject_ModifyNonexistentAttr->set_obj_attrs(dpp, &test_attrs_base, NULL, null_yield), 0); + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + ASSERT_NE(nextObject->get_attrs().empty(), true); + + buffer::list bl_tmp; + bl_tmp.append("new_test_attrs_value_extra_ModifyNonexistentAttr"); + + EXPECT_EQ(testObject_ModifyNonexistentAttr->modify_obj_attrs("test_attrs_key_extra_ModifyNonexistentAttr", bl_tmp, null_yield, dpp), 0); + + fields.push_back("test_attrs_key_extra_ModifyNonexistentAttr"); + + client.hgetall("rgw-object:test_object_ModifyNonexistentAttr:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 24 + METADATA_LENGTH); + } + }); + + client.sync_commit(); + + client.hmget("rgw-object:test_object_ModifyNonexistentAttr:cache", fields, [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ(arr[0].as_string(), "test_attrs_value_ModifyNonexistentAttr"); + + for (int i = 1; i < 11; ++i) { + EXPECT_EQ(arr[i].as_string(), "test_attrs_value_extra_" + to_string(i - 1)); + } + + /* New attribute will be created and stored since it was not found in the existing attributes */ + EXPECT_EQ(arr[11].as_string(), "new_test_attrs_value_extra_ModifyNonexistentAttr"); + } + }); + + client.sync_commit(); + + clientReset(&client); +} + +TEST_F(D4NFilterFixture, ModifyGetAttrs) { + cpp_redis::client client; + map test_attrs_base; + vector fields; + fields.push_back("test_attrs_key_ModifyGetAttrs"); + clientSetUp(&client); + + createUser(); + createBucket(); + putObject("ModifyGetAttrs"); + unique_ptr testObject_ModifyGetAttrs = testBucket->get_object(rgw_obj_key("test_object_ModifyGetAttrs")); + + ASSERT_NE(testObject_ModifyGetAttrs, nullptr); + + for (int i = 0; i < 10; ++i) { + buffer::list bl_tmp; + string tmp_value = "test_attrs_value_extra_" + to_string(i); + bl_tmp.append(tmp_value.data(), strlen(tmp_value.data())); + + string tmp_key = "test_attrs_key_extra_" + to_string(i); + test_attrs_base.insert({tmp_key, bl_tmp}); + fields.push_back(tmp_key); + } + + static rgw::sal::Object* nextObject = dynamic_cast(testObject_ModifyGetAttrs.get())->get_next(); + + ASSERT_EQ(testObject_ModifyGetAttrs->set_obj_attrs(dpp, &test_attrs_base, NULL, null_yield), 0); + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + ASSERT_NE(nextObject->get_attrs().empty(), true); + + /* Attempt to get immediately after a modification */ + buffer::list bl_tmp; + bl_tmp.append("new_test_attrs_value_extra_5"); + + ASSERT_EQ(testObject_ModifyGetAttrs->modify_obj_attrs("test_attrs_key_extra_5", bl_tmp, null_yield, dpp), 0); + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + EXPECT_EQ(testObject_ModifyGetAttrs->get_obj_attrs(null_yield, dpp, NULL), 0); + + client.hgetall("rgw-object:test_object_ModifyGetAttrs:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 22 + METADATA_LENGTH); + } + }); + + client.sync_commit(); + + client.hmget("rgw-object:test_object_ModifyGetAttrs:cache", fields, [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ(arr[0].as_string(), "test_attrs_value_ModifyGetAttrs"); + + for (int i = 1; i < 11; ++i) { + if (i == 6) { + EXPECT_EQ(arr[i].as_string(), "new_test_attrs_value_extra_5"); + } else { + EXPECT_EQ(arr[i].as_string(), "test_attrs_value_extra_" + to_string(i - 1)); + } + } + } + }); + + client.sync_commit(); + + clientReset(&client); +} + +TEST_F(D4NFilterFixture, DelNonexistentAttr) { + cpp_redis::client client; + map test_attrs_base; + vector fields; + fields.push_back("test_attrs_key_DelNonexistentAttr"); + clientSetUp(&client); + + createUser(); + createBucket(); + putObject("DelNonexistentAttr"); + unique_ptr testObject_DelNonexistentAttr = testBucket->get_object(rgw_obj_key("test_object_DelNonexistentAttr")); + + ASSERT_NE(testObject_DelNonexistentAttr, nullptr); + + for (int i = 0; i < 10; ++i) { + buffer::list bl_tmp; + string tmp_value = "test_attrs_value_extra_" + to_string(i); + bl_tmp.append(tmp_value.data(), strlen(tmp_value.data())); + + string tmp_key = "test_attrs_key_extra_" + to_string(i); + test_attrs_base.insert({tmp_key, bl_tmp}); + fields.push_back(tmp_key); + } + + static rgw::sal::Object* nextObject = dynamic_cast(testObject_DelNonexistentAttr.get())->get_next(); + + ASSERT_EQ(testObject_DelNonexistentAttr->set_obj_attrs(dpp, &test_attrs_base, NULL, null_yield), 0); + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + ASSERT_NE(nextObject->get_attrs().empty(), true); + + /* Attempt to delete an attribute that does not exist */ + ASSERT_EQ(testObject_DelNonexistentAttr->delete_obj_attrs(dpp, "test_attrs_key_extra_12", null_yield), 0); + + client.hgetall("rgw-object:test_object_DelNonexistentAttr:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 22 + METADATA_LENGTH); + } + }); + + client.sync_commit(); + + client.hmget("rgw-object:test_object_DelNonexistentAttr:cache", fields, [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ(arr[0].as_string(), "test_attrs_value_DelNonexistentAttr"); + + for (int i = 1; i < 11; ++i) { + EXPECT_EQ(arr[i].as_string(), "test_attrs_value_extra_" + to_string(i - 1)); + } + } + }); + + client.sync_commit(); + + clientReset(&client); +} + +TEST_F(D4NFilterFixture, DelSetWithNonexisentAttr) { + cpp_redis::client client; + map test_attrs_base; + vector fields; + fields.push_back("test_attrs_key_DelSetWithNonexistentAttr"); + clientSetUp(&client); + + createUser(); + createBucket(); + putObject("DelSetWithNonexistentAttr"); + unique_ptr testObject_DelSetWithNonexistentAttr = testBucket->get_object(rgw_obj_key("test_object_DelSetWithNonexistentAttr")); + + ASSERT_NE(testObject_DelSetWithNonexistentAttr, nullptr); + + for (int i = 0; i < 10; ++i) { + buffer::list bl_tmp; + string tmp_value = "test_attrs_value_extra_" + to_string(i); + bl_tmp.append(tmp_value.data(), strlen(tmp_value.data())); + + string tmp_key = "test_attrs_key_extra_" + to_string(i); + test_attrs_base.insert({tmp_key, bl_tmp}); + } + + ASSERT_EQ(testObject_DelSetWithNonexistentAttr->set_obj_attrs(dpp, &test_attrs_base, NULL, null_yield), 0); + + static rgw::sal::Object* nextObject = dynamic_cast(testObject_DelSetWithNonexistentAttr.get())->get_next(); + + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + ASSERT_NE(nextObject->get_attrs().empty(), true); + + EXPECT_EQ(testObject_DelSetWithNonexistentAttr->delete_obj_attrs(dpp, "test_attrs_key_extra_5", null_yield), 0); + + /* Attempt to delete a set of attrs, including one that does not exist */ + EXPECT_EQ(testObject_DelSetWithNonexistentAttr->set_obj_attrs(dpp, NULL, &test_attrs_base, null_yield), 0); + + client.hgetall("rgw-object:test_object_DelSetWithNonexistentAttr:cache", [](cpp_redis::reply& reply) { + auto arr = reply.as_array(); + + if (!arr[0].is_null()) { + EXPECT_EQ((int)arr.size(), 2 + METADATA_LENGTH); + } + }); + + client.sync_commit(); + + clientReset(&client); +} + +/* Underlying store attribute check */ +TEST_F(D4NFilterFixture, StoreSetAttr) { + createUser(); + createBucket(); + putObject("StoreSetAttr"); + unique_ptr testObject_StoreSetAttr = testBucket->get_object(rgw_obj_key("test_object_StoreSetAttr")); + + /* Get the underlying store */ + static rgw::sal::Object* nextObject = dynamic_cast(testObject_StoreSetAttr.get())->get_next(); + + EXPECT_NE(nextObject, nullptr); + + /* Set one attribute */ + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + ASSERT_NE(nextObject->get_attrs().empty(), true); + + /* Check the attribute */ + rgw::sal::Attrs driverAttrs = nextObject->get_attrs(); + pair value(driverAttrs.begin()->first, driverAttrs.begin()->second.to_str()); + + EXPECT_EQ(value, make_pair(string("test_attrs_key_StoreSetAttr"), string("test_attrs_value_StoreSetAttr"))); +} + +TEST_F(D4NFilterFixture, StoreSetAttrs) { + createUser(); + createBucket(); + putObject("StoreSetAttrs"); + unique_ptr testObject_StoreSetAttrs = testBucket->get_object(rgw_obj_key("test_object_StoreSetAttrs")); + + /* Get the underlying store */ + static rgw::sal::Object* nextObject = dynamic_cast(testObject_StoreSetAttrs.get())->get_next(); + + EXPECT_NE(nextObject, nullptr); + + /* Delete base attribute for easier comparison */ + testObject_StoreSetAttrs->delete_obj_attrs(dpp, "test_attrs_key_StoreSetAttrs", null_yield); + + /* Set more attributes */ + map test_attrs_base; + + for (int i = 0; i < 10; ++i) { + buffer::list bl_tmp; + string tmp_value = "test_attrs_value_extra_" + to_string(i); + bl_tmp.append(tmp_value.data(), strlen(tmp_value.data())); + + string tmp_key = "test_attrs_key_extra_" + to_string(i); + test_attrs_base.insert({tmp_key, bl_tmp}); + } + + testObject_StoreSetAttrs->set_obj_attrs(dpp, &test_attrs_base, NULL, null_yield); + + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + + /* Check the attributes */ + rgw::sal::Attrs driverAttrs = nextObject->get_attrs(); + rgw::sal::Attrs::iterator attrs; + vector< pair > values; + + for (attrs = driverAttrs.begin(); attrs != driverAttrs.end(); ++attrs) { + values.push_back(make_pair(attrs->first, attrs->second.to_str())); + } + + int i = 0; + + for (const auto& pair : values) { + string tmp_key = "test_attrs_key_extra_" + to_string(i); + string tmp_value = "test_attrs_value_extra_" + to_string(i); + + EXPECT_EQ(pair, make_pair(tmp_key, tmp_value)); + ++i; + } +} + +TEST_F(D4NFilterFixture, StoreGetAttrs) { + cpp_redis::client client; + map test_attrs_base; + clientSetUp(&client); + + createUser(); + createBucket(); + putObject("StoreGetAttrs"); + unique_ptr testObject_StoreGetAttrs = testBucket->get_object(rgw_obj_key("test_object_StoreGetAttrs")); + + /* Get the underlying store */ + static rgw::sal::Object* nextObject = dynamic_cast(testObject_StoreGetAttrs.get())->get_next(); + + EXPECT_NE(nextObject, nullptr); + + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + + /* Delete base attribute for easier comparison */ + testObject_StoreGetAttrs->delete_obj_attrs(dpp, "test_attrs_key_StoreGetAttrs", null_yield); + + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + + /* Set more attributes */ + for (int i = 0; i < 10; ++i) { + buffer::list bl_tmp; + string tmp_value = "test_attrs_value_extra_" + to_string(i); + bl_tmp.append(tmp_value.data(), strlen(tmp_value.data())); + + string tmp_key = "test_attrs_key_extra_" + to_string(i); + test_attrs_base.insert({tmp_key, bl_tmp}); + } + + testObject_StoreGetAttrs->set_obj_attrs(dpp, &test_attrs_base, NULL, null_yield); + nextObject->get_obj_attrs(null_yield, dpp, NULL); + + /* Change an attribute through redis */ + vector< pair > value; + value.push_back(make_pair("test_attrs_key_extra_5", "new_test_attrs_value_extra_5")); + + client.hmset("rgw-object:test_object_StoreGetAttrs:cache", value, [&](cpp_redis::reply& reply) { + if (!reply.is_null()) { + EXPECT_EQ(reply.as_string(), "OK"); + } + }); + + client.sync_commit(); + + /* Artificially adding the data field so getObject will succeed + for the purposes of this test */ + value.clear(); + value.push_back(make_pair("data", "")); + + client.hmset("rgw-object:test_object_StoreGetAttrs:cache", value, [&](cpp_redis::reply& reply) { + if (!reply.is_null()) { + ASSERT_EQ(reply.as_string(), "OK"); + } + }); + + client.sync_commit(); + + ASSERT_EQ(testObject_StoreGetAttrs->get_obj_attrs(null_yield, dpp, NULL), 0); /* Cache attributes */ + + /* Check the attributes on the store layer */ + rgw::sal::Attrs driverAttrs = nextObject->get_attrs(); + rgw::sal::Attrs::iterator driverattrs; + vector< pair > driverValues; + + for (driverattrs = driverAttrs.begin(); driverattrs != driverAttrs.end(); ++driverattrs) { + driverValues.push_back(make_pair(driverattrs->first, driverattrs->second.to_str())); + } + + EXPECT_EQ((int)driverValues.size(), 10); + + int i = 0; + + for (const auto& pair : driverValues) { + string tmp_key = "test_attrs_key_extra_" + to_string(i); + string tmp_value = "test_attrs_value_extra_" + to_string(i); + + if (i == 5) { + tmp_value = "new_" + tmp_value; + } + + EXPECT_EQ(pair, make_pair(tmp_key, tmp_value)); + ++i; + } + + /* Restore and check original attributes */ + nextObject->get_obj_attrs(null_yield, dpp, NULL); + driverAttrs = nextObject->get_attrs(); + driverValues.clear(); + + for (driverattrs = driverAttrs.begin(); driverattrs != driverAttrs.end(); ++driverattrs) { + driverValues.push_back(make_pair(driverattrs->first, driverattrs->second.to_str())); + } + + EXPECT_EQ((int)driverValues.size(), 10); + + i = 0; + + for (const auto& pair : driverValues) { + string tmp_key = "test_attrs_key_extra_" + to_string(i); + string tmp_value = "test_attrs_value_extra_" + to_string(i); + + EXPECT_EQ(pair, make_pair(tmp_key, tmp_value)); + ++i; + } + + clientReset(&client); +} + +TEST_F(D4NFilterFixture, StoreGetMetadata) { + cpp_redis::client client; + clientSetUp(&client); + + createUser(); + createBucket(); + putObject("StoreGetMetadata"); + unique_ptr testObject_StoreGetMetadata = testBucket->get_object(rgw_obj_key("test_object_StoreGetMetadata")); + + /* Get the underlying store */ + static rgw::sal::Object* nextObject = dynamic_cast(testObject_StoreGetMetadata.get())->get_next(); + + EXPECT_NE(nextObject, nullptr); + + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + + /* Change metadata values through redis */ + vector< pair > value; + value.push_back(make_pair("mtime", "2021-11-08T21:13:38.334696731Z")); + value.push_back(make_pair("object_size", "100")); + value.push_back(make_pair("accounted_size", "200")); + value.push_back(make_pair("epoch", "3")); /* version_id is not tested because the object does not have an instance */ + value.push_back(make_pair("source_zone_short_id", "300")); + value.push_back(make_pair("bucket_count", "10")); + value.push_back(make_pair("bucket_size", "20")); + value.push_back(make_pair("user_quota.max_size", "0")); + value.push_back(make_pair("user_quota.max_objects", "0")); + value.push_back(make_pair("max_buckets", "2000")); + + client.hmset("rgw-object:test_object_StoreGetMetadata:cache", value, [](cpp_redis::reply& reply) { + if (!reply.is_null()) { + EXPECT_EQ(reply.as_string(), "OK"); + } + }); + + client.sync_commit(); + + /* Artificially adding the data field so getObject will succeed + for the purposes of this test */ + value.clear(); + value.push_back(make_pair("data", "")); + + client.hmset("rgw-object:test_object_StoreGetMetadata:cache", value, [](cpp_redis::reply& reply) { + if (!reply.is_null()) { + ASSERT_EQ(reply.as_string(), "OK"); + } + }); + + client.sync_commit(); + + unique_ptr testROp = testObject_StoreGetMetadata->get_read_op(); + + ASSERT_NE(testROp, nullptr); + ASSERT_EQ(testROp->prepare(null_yield, dpp), 0); + + /* Check updated metadata values */ + RGWUserInfo info = testObject_StoreGetMetadata->get_bucket()->get_owner()->get_info(); + static StoreObject* storeObject = static_cast(dynamic_cast(testObject_StoreGetMetadata.get())->get_next()); + + EXPECT_EQ(to_iso_8601(storeObject->state.mtime), "2021-11-08T21:13:38.334696731Z"); + EXPECT_EQ(testObject_StoreGetMetadata->get_obj_size(), (uint64_t)100); + EXPECT_EQ(storeObject->state.accounted_size, (uint64_t)200); + EXPECT_EQ(storeObject->state.epoch, (uint64_t)3); + EXPECT_EQ(storeObject->state.zone_short_id, (uint32_t)300); + EXPECT_EQ(testObject_StoreGetMetadata->get_bucket()->get_count(), (uint64_t)10); + EXPECT_EQ(testObject_StoreGetMetadata->get_bucket()->get_size(), (uint64_t)20); + EXPECT_EQ(info.quota.user_quota.max_size, (int64_t)0); + EXPECT_EQ(info.quota.user_quota.max_objects, (int64_t)0); + EXPECT_EQ(testObject_StoreGetMetadata->get_bucket()->get_owner()->get_max_buckets(), (int32_t)2000); +} + +TEST_F(D4NFilterFixture, StoreModifyAttr) { + createUser(); + createBucket(); + putObject("StoreModifyAttr"); + unique_ptr testObject_StoreModifyAttr = testBucket->get_object(rgw_obj_key("test_object_StoreModifyAttr")); + + /* Get the underlying store */ + static rgw::sal::Object* nextObject = dynamic_cast(testObject_StoreModifyAttr.get())->get_next(); + + ASSERT_NE(nextObject, nullptr); + + /* Modify existing attribute */ + buffer::list bl_tmp; + string tmp_value = "new_test_attrs_value_StoreModifyAttr"; + bl_tmp.append(tmp_value.data(), strlen(tmp_value.data())); + + testObject_StoreModifyAttr->modify_obj_attrs("test_attrs_key_StoreModifyAttr", bl_tmp, null_yield, dpp); + + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + + /* Check the attribute */ + rgw::sal::Attrs driverAttrs = nextObject->get_attrs(); + pair value(driverAttrs.begin()->first, driverAttrs.begin()->second.to_str()); + + EXPECT_EQ(value, make_pair(string("test_attrs_key_StoreModifyAttr"), string("new_test_attrs_value_StoreModifyAttr"))); +} + +TEST_F(D4NFilterFixture, StoreDelAttrs) { + createUser(); + createBucket(); + putObject("StoreDelAttrs"); + unique_ptr testObject_StoreDelAttrs = testBucket->get_object(rgw_obj_key("test_object_StoreDelAttrs")); + + /* Get the underlying store */ + static rgw::sal::Object* nextObject = dynamic_cast(testObject_StoreDelAttrs.get())->get_next(); + + ASSERT_NE(nextObject, nullptr); + + /* Set more attributes */ + map test_attrs_base; + + for (int i = 0; i < 10; ++i) { + buffer::list bl_tmp; + string tmp_value = "test_attrs_value_extra_" + to_string(i); + bl_tmp.append(tmp_value.data(), strlen(tmp_value.data())); + + string tmp_key = "test_attrs_key_extra_" + to_string(i); + test_attrs_base.insert({tmp_key, bl_tmp}); + } + + testObject_StoreDelAttrs->set_obj_attrs(dpp, &test_attrs_base, NULL, null_yield); + + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + + /* Check that the attributes exist before deletion */ + rgw::sal::Attrs driverAttrs = nextObject->get_attrs(); + + EXPECT_EQ(driverAttrs.size(), (long unsigned int)11); + + rgw::sal::Attrs::iterator driverattrs; + vector< pair > driverValues; + + for (driverattrs = ++driverAttrs.begin(); driverattrs != driverAttrs.end(); ++driverattrs) { + driverValues.push_back(make_pair(driverattrs->first, driverattrs->second.to_str())); + } + + int i = 0; + + for (const auto& pair : driverValues) { + string tmp_key = "test_attrs_key_extra_" + to_string(i); + string tmp_value = "test_attrs_value_extra_" + to_string(i); + + EXPECT_EQ(pair, make_pair(tmp_key, tmp_value)); + ++i; + } + + testObject_StoreDelAttrs->set_obj_attrs(dpp, NULL, &test_attrs_base, null_yield); + + ASSERT_EQ(nextObject->get_obj_attrs(null_yield, dpp, NULL), 0); + + /* Check that the attributes do not exist after deletion */ + driverAttrs = nextObject->get_attrs(); + + EXPECT_EQ(driverAttrs.size(), (long unsigned int)1); + + pair value(driverAttrs.begin()->first, driverAttrs.begin()->second.to_str()); + + EXPECT_EQ(value, make_pair(string("test_attrs_key_StoreDelAttrs"), string("test_attrs_value_StoreDelAttrs"))); +} + +/* SAL object data storage check */ +TEST_F(D4NFilterFixture, DataCheck) { + cpp_redis::client client; + clientSetUp(&client); + + createUser(); + createBucket(); + + /* Prepare, process, and complete object write */ + unique_ptr obj = testBucket->get_object(rgw_obj_key("test_object_DataCheck")); + rgw_user owner; + rgw_placement_rule ptail_placement_rule; + uint64_t olh_epoch = 123; + string unique_tag; + + obj->get_obj_attrs(null_yield, dpp); + + testWriter = driver->get_atomic_writer(dpp, + null_yield, + obj.get(), + owner, + &ptail_placement_rule, + olh_epoch, + unique_tag); + + size_t accounted_size = 4; + string etag("test_etag"); + ceph::real_time mtime; + ceph::real_time set_mtime; + + buffer::list bl; + string tmp = "test_attrs_value_DataCheck"; + bl.append("test_attrs_value_DataCheck"); + map attrs{{"test_attrs_key_DataCheck", bl}}; + buffer::list data; + data.append("test data"); + + ceph::real_time delete_at; + char if_match; + char if_nomatch; + string user_data; + rgw_zone_set zones_trace; + bool canceled; + + ASSERT_EQ(testWriter->prepare(null_yield), 0); + + ASSERT_EQ(testWriter->process(move(data), 0), 0); + + ASSERT_EQ(testWriter->complete(accounted_size, etag, + &mtime, set_mtime, + attrs, + delete_at, + &if_match, &if_nomatch, + &user_data, + &zones_trace, &canceled, + null_yield), 0); + + client.hget("rgw-object:test_object_DataCheck:cache", "data", [&data](cpp_redis::reply& reply) { + if (reply.is_string()) { + EXPECT_EQ(reply.as_string(), data.to_str()); + } + }); + + client.sync_commit(); + + /* Change data and ensure redis stores the new value */ + buffer::list dataNew; + dataNew.append("new test data"); + + ASSERT_EQ(testWriter->prepare(null_yield), 0); + + ASSERT_EQ(testWriter->process(move(dataNew), 0), 0); + + ASSERT_EQ(testWriter->complete(accounted_size, etag, + &mtime, set_mtime, + attrs, + delete_at, + &if_match, &if_nomatch, + &user_data, + &zones_trace, &canceled, + null_yield), 0); + + client.hget("rgw-object:test_object_DataCheck:cache", "data", [&dataNew](cpp_redis::reply& reply) { + if (reply.is_string()) { + EXPECT_EQ(reply.as_string(), dataNew.to_str()); + } + }); + + client.sync_commit(); + + clientReset(&client); +} + +int main(int argc, char *argv[]) { + ::testing::InitGoogleTest(&argc, argv); + + /* Other host and port can be passed to the program */ + if (argc == 1) { + portStr = "6379"; + hostStr = "127.0.0.1"; + } else if (argc == 3) { + hostStr = argv[1]; + portStr = argv[2]; + } else { + std::cout << "Incorrect number of arguments." << std::endl; + return -1; + } + + redisHost = hostStr + ":" + portStr; + + env = new Environment(); + ::testing::AddGlobalTestEnvironment(env); + + return RUN_ALL_TESTS(); +} -- 2.39.5