d4n/test: squashing the following commits related to

D4N test failures.

1 d4n: Add `rename` method and unit test and update D4N unit tests
2. d4n/test: Fix linking issue

Signed-off-by: Samarah <samarah.uriarte@ibm.com>

diff --git a/src/rgw/rgw_redis_driver.cc b/src/rgw/rgw_redis_driver.cc
index 0d8e462365eeed862edc693413ef1da3901d2ac4..c30dfc3b41e2de3b25a1e5ae4fb908117264c0d3 100644 (file)
--- a/src/rgw/rgw_redis_driver.cc
+++ b/src/rgw/rgw_redis_driver.cc
@@ -343,6 +343,29 @@ int RedisDriver::delete_data(const DoutPrefixProvider* dpp, const::std::string&
   return 0;
 }
 
+int RedisDriver::rename(const DoutPrefixProvider* dpp, const::std::string& oldKey, const::std::string& newKey, optional_yield y) {
+  std::string entry = partition_info.location + oldKey;
+  std::string newEntry = partition_info.location + newKey;
+
+  try {
+    boost::system::error_code ec;
+    request req;
+    response<boost::redis::ignore_t> resp;
+    req.push("RENAME", entry, newEntry);
+
+    redis_exec(conn, ec, req, resp, y);
+
+    if (ec) {
+      return -ec.value();
+    }
+  } catch (std::exception &e) {
+    ldpp_dout(dpp, 0) << "RedisDriver::" << __func__ << "(): ERROR: " << e.what() << dendl;
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
 int RedisDriver::get_attrs(const DoutPrefixProvider* dpp, const std::string& key, rgw::sal::Attrs& attrs, optional_yield y) 
 {
   std::string entry = partition_info.location + key;

diff --git a/src/rgw/rgw_redis_driver.h b/src/rgw/rgw_redis_driver.h
index fe401c07e5a578b37664850aa1ff867db751878a..da5203b3cb3793f364c93b6567b4cf26d5f69e7b 100644 (file)
--- a/src/rgw/rgw_redis_driver.h
+++ b/src/rgw/rgw_redis_driver.h
@@ -38,6 +38,7 @@ class RedisDriver : public CacheDriver {
     virtual int del(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y) override;
     virtual int append_data(const DoutPrefixProvider* dpp, const::std::string& key, const bufferlist& bl_data, optional_yield y) override;
     virtual int delete_data(const DoutPrefixProvider* dpp, const::std::string& key, optional_yield y) override;
+    virtual int rename(const DoutPrefixProvider* dpp, const::std::string& oldKey, const::std::string& newKey, optional_yield y) override;
     virtual int set_attrs(const DoutPrefixProvider* dpp, const std::string& key, const rgw::sal::Attrs& attrs, optional_yield y) override;
     virtual int get_attrs(const DoutPrefixProvider* dpp, const std::string& key, rgw::sal::Attrs& attrs, optional_yield y) override;
     virtual int update_attrs(const DoutPrefixProvider* dpp, const std::string& key, const rgw::sal::Attrs& attrs, optional_yield y) override;

diff --git a/src/test/rgw/test_d4n_directory.cc b/src/test/rgw/test_d4n_directory.cc
index fbebcc8e4ab3c8603176e89b17bbd6c822d2eda2..d06286891ea81168b84f3f9cdecf517a20c3bd0f 100644 (file)
--- a/src/test/rgw/test_d4n_directory.cc
+++ b/src/test/rgw/test_d4n_directory.cc
@@ -301,7 +301,7 @@ TEST_F(ObjectDirectoryFixture, UpdateFieldYield)
 TEST_F(BlockDirectoryFixture, SetYield)
 {
   boost::asio::spawn(io, [this] (boost::asio::yield_context yield) {
-    ASSERT_EQ(0, dir->set(block, yield));
+    ASSERT_EQ(0, dir->set(env->dpp, block, optional_yield{yield}));
 
     boost::system::error_code ec;
     request req;
@@ -324,7 +324,7 @@ TEST_F(BlockDirectoryFixture, SetYield)
 TEST_F(BlockDirectoryFixture, GetYield)
 {
   boost::asio::spawn(io, [this] (boost::asio::yield_context yield) {
-    ASSERT_EQ(0, dir->set(block, yield));
+    ASSERT_EQ(0, dir->set(env->dpp, block, optional_yield{yield}));
 
     {
       boost::system::error_code ec;
@@ -338,7 +338,7 @@ TEST_F(BlockDirectoryFixture, GetYield)
       EXPECT_EQ(std::get<0>(resp).value(), 0);
     }
 
-    ASSERT_EQ(0, dir->get(block, yield));
+    ASSERT_EQ(0, dir->get(env->dpp, block, optional_yield{yield}));
     EXPECT_EQ(block->cacheObj.objName, "newoid");
 
     {
@@ -360,8 +360,8 @@ TEST_F(BlockDirectoryFixture, GetYield)
 TEST_F(BlockDirectoryFixture, CopyYield)
 {
   boost::asio::spawn(io, [this] (boost::asio::yield_context yield) {
-    ASSERT_EQ(0, dir->set(block, yield));
-    ASSERT_EQ(0, dir->copy(block, "copyTestName", "copyBucketName", yield));
+    ASSERT_EQ(0, dir->set(env->dpp, block, optional_yield{yield}));
+    ASSERT_EQ(0, dir->copy(env->dpp, block, "copyTestName", "copyBucketName", optional_yield{yield}));
 
     boost::system::error_code ec;
     request req;
@@ -392,7 +392,7 @@ TEST_F(BlockDirectoryFixture, CopyYield)
 TEST_F(BlockDirectoryFixture, DelYield)
 {
   boost::asio::spawn(io, [this] (boost::asio::yield_context yield) {
-    ASSERT_EQ(0, dir->set(block, yield));
+    ASSERT_EQ(0, dir->set(env->dpp, block, optional_yield{yield}));
 
     {
       boost::system::error_code ec;
@@ -406,7 +406,7 @@ TEST_F(BlockDirectoryFixture, DelYield)
       EXPECT_EQ(std::get<0>(resp).value(), 1);
     }
 
-    ASSERT_EQ(0, dir->del(block, yield));
+    ASSERT_EQ(0, dir->del(env->dpp, block, optional_yield{yield}));
 
     {
       boost::system::error_code ec;
@@ -430,9 +430,9 @@ TEST_F(BlockDirectoryFixture, DelYield)
 TEST_F(BlockDirectoryFixture, UpdateFieldYield)
 {
   boost::asio::spawn(io, [this] (boost::asio::yield_context yield) {
-    ASSERT_EQ(0, dir->set(block, yield));
-    ASSERT_EQ(0, dir->update_field(block, "objName", "newTestName", yield));
-    ASSERT_EQ(0, dir->update_field(block, "blockHosts", "127.0.0.1:5000", yield));
+    ASSERT_EQ(0, dir->set(env->dpp, block, optional_yield{yield}));
+    ASSERT_EQ(0, dir->update_field(env->dpp, block, "objName", "newTestName", optional_yield{yield}));
+    ASSERT_EQ(0, dir->update_field(env->dpp, block, "blockHosts", "127.0.0.1:5000", optional_yield{yield}));
 
     boost::system::error_code ec;
     request req;
@@ -457,8 +457,8 @@ TEST_F(BlockDirectoryFixture, RemoveHostYield)
 {
   boost::asio::spawn(io, [this] (boost::asio::yield_context yield) {
     block->hostsList.push_back("127.0.0.1:6000");
-    ASSERT_EQ(0, dir->set(block, yield));
-    ASSERT_EQ(0, dir->remove_host(block, "127.0.0.1:6379", yield));
+    ASSERT_EQ(0, dir->set(env->dpp, block, optional_yield{yield}));
+    ASSERT_EQ(0, dir->remove_host(env->dpp, block, "127.0.0.1:6379", optional_yield{yield}));
 
     {
       boost::system::error_code ec;
@@ -474,7 +474,7 @@ TEST_F(BlockDirectoryFixture, RemoveHostYield)
       EXPECT_EQ(std::get<1>(resp).value(), "127.0.0.1:6000");
     }
 
-    ASSERT_EQ(0, dir->remove_host(block, "127.0.0.1:6000", yield));
+    ASSERT_EQ(0, dir->remove_host(env->dpp, block, "127.0.0.1:6000", optional_yield{yield}));
 
     {
       boost::system::error_code ec;

diff --git a/src/test/rgw/test_d4n_policy.cc b/src/test/rgw/test_d4n_policy.cc
index 12bfd7913b79bf3cf20f57971b3ad38cc3663295..4c57fb0d7fd08e4c4d7d16a11114bb05cd6bf7d0 100644 (file)
--- a/src/test/rgw/test_d4n_policy.cc
+++ b/src/test/rgw/test_d4n_policy.cc
@@ -72,7 +72,7 @@ class LFUDAPolicyFixture : public ::testing::Test {
       ASSERT_NE(policyDriver, nullptr);
       ASSERT_NE(conn, nullptr);
 
-      dir->init(env->cct);
+      env->cct->_conf->rgw_d4n_l1_datacache_address = "127.0.0.1:6379";
       cacheDriver->initialize(env->dpp);
 
       bl.append("test data");
@@ -106,21 +106,21 @@ class LFUDAPolicyFixture : public ::testing::Test {
       std::string oid = build_index(block->cacheObj.bucketName, block->cacheObj.objName, block->blockID, block->size);
 
       if (this->policyDriver->get_cache_policy()->exist_key(build_index(block->cacheObj.bucketName, block->cacheObj.objName, block->blockID, block->size))) { /* Local copy */
-       policyDriver->get_cache_policy()->update(env->dpp, oid, 0, bl.length(), "", y);
+       policyDriver->get_cache_policy()->update(env->dpp, oid, 0, bl.length(), "", false, y);
         return 0;
       } else {
        if (this->policyDriver->get_cache_policy()->eviction(dpp, block->size, y) < 0)
          return -1;
 
-       int exists = dir->exist_key(block, y);
+       int exists = dir->exist_key(env->dpp, block, y);
        if (exists > 0) { /* Remote copy */
-         if (dir->get(block, y) < 0) {
+         if (dir->get(env->dpp, block, y) < 0) {
            return -1;
          } else {
            if (!block->hostsList.empty()) { 
              block->globalWeight += age;
              auto globalWeight = std::to_string(block->globalWeight);
-             if (dir->update_field(block, "globalWeight", globalWeight, y) < 0) {
+             if (dir->update_field(env->dpp, block, "globalWeight", globalWeight, y) < 0) {
                return -1;
              } else {
                return 0;
@@ -131,10 +131,10 @@ class LFUDAPolicyFixture : public ::testing::Test {
          }
        } else if (!exists) { /* No remote copy */
          block->hostsList.push_back(dir->cct->_conf->rgw_d4n_l1_datacache_address);
-         if (dir->set(block, y) < 0)
+         if (dir->set(env->dpp, block, y) < 0)
            return -1;
 
-         this->policyDriver->get_cache_policy()->update(dpp, oid, 0, bl.length(), "", y);
+         this->policyDriver->get_cache_policy()->update(dpp, oid, 0, bl.length(), "", false, y);
          if (cacheDriver->put(dpp, oid, bl, bl.length(), attrs, y) < 0)
             return -1;
          return cacheDriver->set_attr(dpp, oid, "localWeight", std::to_string(age), y);
@@ -148,6 +148,7 @@ class LFUDAPolicyFixture : public ::testing::Test {
     rgw::d4n::BlockDirectory* dir;
     rgw::d4n::PolicyDriver* policyDriver;
     rgw::cache::RedisDriver* cacheDriver;
+    rgw::sal::D4NFilterDriver* driver = nullptr;
 
     net::io_context io;
     std::shared_ptr<connection> conn;
@@ -163,9 +164,13 @@ void rethrow(std::exception_ptr eptr) {
 TEST_F(LFUDAPolicyFixture, LocalGetBlockYield)
 {
   boost::asio::spawn(io, [this] (boost::asio::yield_context yield) {
+    env->cct->_conf->rgw_lfuda_sync_frequency = 1;
+    dynamic_cast<rgw::d4n::LFUDAPolicy*>(policyDriver->get_cache_policy())->save_y(optional_yield{yield});
+    policyDriver->get_cache_policy()->init(env->cct, env->dpp, io, driver);
+
     std::string key = block->cacheObj.bucketName + "_" + block->cacheObj.objName + "_" + std::to_string(block->blockID) + "_" + std::to_string(block->size);
-    ASSERT_EQ(0, cacheDriver->put(env->dpp, key, bl, bl.length(), attrs, yield));
-    policyDriver->get_cache_policy()->update(env->dpp, key, 0, bl.length(), "", yield);
+    ASSERT_EQ(0, cacheDriver->put(env->dpp, key, bl, bl.length(), attrs, optional_yield{yield}));
+    policyDriver->get_cache_policy()->update(env->dpp, key, 0, bl.length(), "", false, optional_yield{yield});
 
     ASSERT_EQ(lfuda(env->dpp, block, cacheDriver, yield), 0);
 
@@ -183,6 +188,9 @@ TEST_F(LFUDAPolicyFixture, LocalGetBlockYield)
     ASSERT_EQ((bool)ec, false);
     EXPECT_EQ(std::get<0>(resp).value(), "2");
     conn->cancel();
+    
+    delete policyDriver; 
+    policyDriver = nullptr;
   }, rethrow);
 
   io.run();
@@ -214,10 +222,14 @@ TEST_F(LFUDAPolicyFixture, RemoteGetBlockYield)
     attrVal.append("testBucket");
     attrs.insert({"bucket_name", attrVal});
 
-    ASSERT_EQ(0, dir->set(&victim, yield));
+    env->cct->_conf->rgw_lfuda_sync_frequency = 1;
+    dynamic_cast<rgw::d4n::LFUDAPolicy*>(policyDriver->get_cache_policy())->save_y(optional_yield{yield});
+    policyDriver->get_cache_policy()->init(env->cct, env->dpp, io, driver);
+
+    ASSERT_EQ(0, dir->set(env->dpp, &victim, optional_yield{yield}));
     std::string victimKey = victim.cacheObj.bucketName + "_" + victim.cacheObj.objName + "_" + std::to_string(victim.blockID) + "_" + std::to_string(victim.size);
-    ASSERT_EQ(0, cacheDriver->put(env->dpp, victimKey, bl, bl.length(), attrs, yield));
-    policyDriver->get_cache_policy()->update(env->dpp, victimKey, 0, bl.length(), "", yield);
+    ASSERT_EQ(0, cacheDriver->put(env->dpp, victimKey, bl, bl.length(), attrs, optional_yield{yield}));
+    policyDriver->get_cache_policy()->update(env->dpp, victimKey, 0, bl.length(), "", false, optional_yield{yield});
 
     /* Remote block */
     block->size = cacheDriver->get_free_space(env->dpp) + 1; /* To trigger eviction */
@@ -226,7 +238,17 @@ TEST_F(LFUDAPolicyFixture, RemoteGetBlockYield)
     block->hostsList.push_back("127.0.0.1:6000");
     block->cacheObj.hostsList.push_back("127.0.0.1:6000");
 
-    ASSERT_EQ(0, dir->set(block, yield));
+    ASSERT_EQ(0, dir->set(env->dpp, block, optional_yield{yield}));
+
+    { /* Avoid sending victim block to remote cache since no network is available */
+      boost::system::error_code ec;
+      request req;
+      req.push("HSET", "lfuda", "minLocalWeights_sum", "10", "minLocalWeights_size", "1");
+
+      response<boost::redis::ignore_t> resp;
+
+      conn->async_exec(req, resp, yield[ec]);
+    }
 
     ASSERT_GE(lfuda(env->dpp, block, cacheDriver, yield), 0);
 
@@ -250,15 +272,22 @@ TEST_F(LFUDAPolicyFixture, RemoteGetBlockYield)
     EXPECT_EQ(std::get<1>(resp).value(), 0);
     EXPECT_EQ(std::get<2>(resp).value(), "1");
     conn->cancel();
+    
+    delete policyDriver; 
+    policyDriver = nullptr;
   }, rethrow);
 
-  io.run();
+  io.run(); 
 }
 
 TEST_F(LFUDAPolicyFixture, BackendGetBlockYield)
 {
   boost::asio::spawn(io, [this] (boost::asio::yield_context yield) {
-    ASSERT_GE(lfuda(env->dpp, block, cacheDriver, yield), 0);
+    env->cct->_conf->rgw_lfuda_sync_frequency = 1;
+    dynamic_cast<rgw::d4n::LFUDAPolicy*>(policyDriver->get_cache_policy())->save_y(optional_yield{yield});
+    policyDriver->get_cache_policy()->init(env->cct, env->dpp, io, driver);
+
+    ASSERT_GE(lfuda(env->dpp, block, cacheDriver, optional_yield{yield}), 0);
 
     cacheDriver->shutdown();
 
@@ -271,6 +300,9 @@ TEST_F(LFUDAPolicyFixture, BackendGetBlockYield)
     conn->async_exec(req, resp, yield[ec]);
 
     conn->cancel();
+
+    delete policyDriver; 
+    policyDriver = nullptr;
   }, rethrow);
 
   io.run();
@@ -280,8 +312,8 @@ TEST_F(LFUDAPolicyFixture, RedisSyncTest)
 {
   boost::asio::spawn(io, [this] (boost::asio::yield_context yield) {
     env->cct->_conf->rgw_lfuda_sync_frequency = 1;
-    dynamic_cast<rgw::d4n::LFUDAPolicy*>(policyDriver->get_cache_policy())->save_y(yield);
-    policyDriver->get_cache_policy()->init(env->cct, env->dpp, io);
+    dynamic_cast<rgw::d4n::LFUDAPolicy*>(policyDriver->get_cache_policy())->save_y(optional_yield{yield});
+    policyDriver->get_cache_policy()->init(env->cct, env->dpp, io, driver);
   
     cacheDriver->shutdown();
 

diff --git a/src/test/rgw/test_redis_driver.cc b/src/test/rgw/test_redis_driver.cc
index be5f1496a52c2afb65cf9a15dee88bc1bc7c3004..8aefe7fbc398f175321fae6af88c436f01c76644 100644 (file)
--- a/src/test/rgw/test_redis_driver.cc
+++ b/src/test/rgw/test_redis_driver.cc
@@ -93,6 +93,7 @@ class RedisDriverFixture: public ::testing::Test {
       ASSERT_NE(cacheDriver, nullptr);
       ASSERT_NE(conn, nullptr);
 
+      env->cct->_conf->rgw_d4n_l1_datacache_address = "127.0.0.1:6379";
       cacheDriver->initialize(env->dpp);
 
       bl.append("test data");
@@ -365,6 +366,34 @@ TEST_F(RedisDriverFixture, DeleteDataYield)
   io.run();
 }
 
+TEST_F(RedisDriverFixture, RenameYield)
+{
+  boost::asio::spawn(io, [this] (boost::asio::yield_context yield) {
+    ASSERT_EQ(0, cacheDriver->put(env->dpp, "testName", bl, bl.length(), attrs, optional_yield{yield}));
+    ASSERT_EQ(0, cacheDriver->rename(env->dpp, "testName", "newTestName", optional_yield{yield}));
+    cacheDriver->shutdown();
+
+    {
+      boost::system::error_code ec;
+      request req;
+      req.push("EXISTS", "RedisCache/testName");
+      req.push("EXISTS", "RedisCache/newTestName");
+      req.push("FLUSHALL");
+      response<int, int, boost::redis::ignore_t> resp;
+
+      conn->async_exec(req, resp, yield[ec]);
+
+      ASSERT_EQ((bool)ec, false);
+      EXPECT_EQ(std::get<0>(resp).value(), 0);
+      EXPECT_EQ(std::get<1>(resp).value(), 1);
+    }
+
+    conn->cancel();
+  }, rethrow);
+
+  io.run();
+}
+
 TEST_F(RedisDriverFixture, SetAttrsYield)
 {
   boost::asio::spawn(io, [this] (boost::asio::yield_context yield) {