rgw: add max-entries, marker for log operations

Signed-off-by: Babu Shanmugam <anbu@enovance.com>
Conflicts:
	src/rgw/rgw_admin.cc
	src/rgw/rgw_rest_log.cc

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 0d06e1352bb969edad80eb4e77891f16fbad9816..ce18e2da43f2383ab2b71ae30f9eb5e3b17fb0ee 100644 (file)
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -1798,7 +1798,6 @@ next:
     if (ret < 0)
       return -ret;
 
-
     int i = (specified_shard_id ? shard_id : 0);
 
     formatter->open_array_section("entries");
@@ -1808,7 +1807,7 @@ next:
       list<cls_log_entry> entries;
 
 
-      meta_log->init_list_entries(i, start_time, end_time, &handle);
+      meta_log->init_list_entries(i, start_time, end_time, marker, &handle);
 
       bool truncated;
       do {

diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index 7a6423f660bc0fb01624cf87bbf5b2b864f56015..7b758b08fd19ee61ab73bf80841516e8fbceb963 100644 (file)
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -1159,7 +1159,7 @@ int RGWDataChangesLog::list_entries(int shard, utime_t& start_time, utime_t& end
   list<cls_log_entry> log_entries;
 
   int ret = store->time_log_list(oids[shard], start_time, end_time,
-                                 max_entries, log_entries, marker, truncated);
+                                 max_entries, log_entries, marker, truncated); 
   if (ret < 0)
     return ret;
 
@@ -1182,7 +1182,6 @@ int RGWDataChangesLog::list_entries(int shard, utime_t& start_time, utime_t& end
 int RGWDataChangesLog::list_entries(utime_t& start_time, utime_t& end_time, int max_entries,
              list<rgw_data_change>& entries, LogMarker& marker, bool *ptruncated) {
   bool truncated;
-
   entries.clear();
 
   for (; marker.shard < num_shards && (int)entries.size() < max_entries;

diff --git a/src/rgw/rgw_metadata.cc b/src/rgw/rgw_metadata.cc
index 2a90f0d0e0c8dbd90798fa56facdda2615d92046..6e0d4b3b6322026193f8b3f370208170836607be 100644 (file)
--- a/src/rgw/rgw_metadata.cc
+++ b/src/rgw/rgw_metadata.cc
@@ -79,13 +79,15 @@ int RGWMetadataLog::add_entry(RGWRados *store, string& section, string& key, buf
   return store->time_log_add(oid, now, section, key, bl);
 }
 
-void RGWMetadataLog::init_list_entries(int shard_id, utime_t& from_time, utime_t& end_time, void **handle)
+void RGWMetadataLog::init_list_entries(int shard_id, utime_t& from_time, utime_t& end_time, 
+                                       string& marker, void **handle)
 {
   LogListCtx *ctx = new LogListCtx();
 
   ctx->cur_shard = shard_id;
   ctx->from_time = from_time;
-  ctx->end_time = end_time;
+  ctx->end_time  = end_time;
+  ctx->marker    = marker;
 
   get_shard_oid(ctx->cur_shard, ctx->cur_oid);
 
@@ -99,7 +101,7 @@ void RGWMetadataLog::complete_list_entries(void *handle) {
 
 int RGWMetadataLog::list_entries(void *handle,
                  int max_entries,
-                 list<cls_log_entry>& entries,
+                 list<cls_log_entry>& entries, 
                  bool *truncated) {
   LogListCtx *ctx = (LogListCtx *)handle;
 
@@ -108,8 +110,6 @@ int RGWMetadataLog::list_entries(void *handle,
     return 0;
   }
 
-  entries.clear();
-
   int ret = store->time_log_list(ctx->cur_oid, ctx->from_time, ctx->end_time,
                                  max_entries, entries, ctx->marker, truncated);
   if ((ret < 0) && (ret != -ENOENT))

diff --git a/src/rgw/rgw_metadata.h b/src/rgw/rgw_metadata.h
index eaac1fac987388381b317cff7d94427c0e28bb0a..5f0fafefc5f427786936fcd743f06c3a79b8d048 100644 (file)
--- a/src/rgw/rgw_metadata.h
+++ b/src/rgw/rgw_metadata.h
@@ -91,12 +91,11 @@ public:
     LogListCtx() : done(false) {}
   };
 
-  void init_list_entries(int shard_id, utime_t& from_time, utime_t& end_time, void **handle);
+  void init_list_entries(int shard_id, utime_t& from_time, utime_t& end_time, string& marker, void **handle);
   void complete_list_entries(void *handle);
   int list_entries(void *handle,
                    int max_entries,
-                   list<cls_log_entry>& entries,
-                   bool *truncated);
+                   list<cls_log_entry>& entries, bool *truncated);
 
   int trim(int shard_id, utime_t& from_time, utime_t& end_time);
   int lock_exclusive(int shard_id, utime_t& duration, string& owner_id);

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 7ad84c44617aad02e0b5f7458ca53754a8adb612..9a485600b6d8ec3a9c0f556ddda86e66e1499a96 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -60,6 +60,7 @@ static string region_info_oid_prefix = "region_info.";
 
 static string default_region_info_oid = "default.region";
 static string region_map_oid = "region_map";
+static string log_lock_name = "rgw_log_lock";
 
 static RGWObjCategory main_category = RGW_OBJ_CATEGORY_MAIN;
 
@@ -1454,8 +1455,8 @@ int RGWRados::time_log_list(const string& oid, utime_t& start_time, utime_t& end
   int r = rados->ioctx_create(log_pool, io_ctx);
   if (r < 0)
     return r;
-
   librados::ObjectReadOperation op;
+
   cls_log_list(op, start_time, end_time, marker, max_entries, entries, &marker, truncated);
 
   bufferlist obl;
@@ -1489,8 +1490,7 @@ int RGWRados::lock_exclusive(rgw_bucket& pool, const string& oid, utime_t& durat
   if (r < 0)
     return r;
   
-  string lock_name = RGW_INDEX_LOCK_NAME;
-  rados::cls::lock::Lock l(lock_name);
+  rados::cls::lock::Lock l(log_lock_name);
   l.set_duration(duration);
   l.set_cookie(owner_id);
   
@@ -1506,8 +1506,7 @@ int RGWRados::unlock(rgw_bucket& pool, const string& oid, string& owner_id) {
   if (r < 0)
     return r;
   
-  string lock_name = RGW_INDEX_LOCK_NAME;
-  rados::cls::lock::Lock l(lock_name);
+  rados::cls::lock::Lock l(log_lock_name);
   l.set_cookie(owner_id);
   
   return l.unlock(&io_ctx, oid);

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index f292f47cb0e0ec2a9613c02b27c8010316f0762c..7e08d8016c0fc48fcc5f43fe74826aadc6becad6 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -25,7 +25,6 @@ class RGWGC;
 
 #define RGW_OBJ_NS_MULTIPART "multipart"
 #define RGW_OBJ_NS_SHADOW    "shadow"
-#define RGW_INDEX_LOCK_NAME  "rgw_process"
 
 static inline void prepend_bucket_marker(rgw_bucket& bucket, string& orig_oid, string& oid)
 {

diff --git a/src/rgw/rgw_rest_log.cc b/src/rgw/rgw_rest_log.cc
index ba821c9b9a2efc6f565c9eb1e52fadab18ac70ea..f066b6b4b0491effd152b45ca13bebae64aa99ab 100644 (file)
--- a/src/rgw/rgw_rest_log.cc
+++ b/src/rgw/rgw_rest_log.cc
@@ -20,6 +20,7 @@
 #include "rgw_client_io.h"
 #include "common/errno.h"
 
+#define LOG_CLASS_LIST_MAX_ENTRIES (1000)
 #define dout_subsys ceph_subsys_rgw
 
 static int parse_date_str(string& in, utime_t& out) {
@@ -37,16 +38,17 @@ static int parse_date_str(string& in, utime_t& out) {
 
 void RGWOp_MDLog_List::execute() {
   string   shard = s->info.args.get("id");
-
+  string   max_entries_str = s->info.args.get("max-entries");
   string   st = s->info.args.get("start-time"),
            et = s->info.args.get("end-time"),
+           marker = s->info.args.get("marker"),
            err;
   utime_t  ut_st, 
            ut_et;
   void    *handle;
-  int      shard_id;
+  unsigned shard_id, max_entries = 0;
 
-  shard_id = strict_strtol(shard.c_str(), 10, &err);
+  shard_id = (unsigned)strict_strtol(shard.c_str(), 10, &err);
   if (!err.empty()) {
     dout(5) << "Error parsing shard_id " << shard << dendl;
     http_ret = -EINVAL;
@@ -63,13 +65,29 @@ void RGWOp_MDLog_List::execute() {
     return;
   }
 
+  if (!max_entries_str.empty()) {
+    max_entries = (unsigned)strict_strtol(max_entries_str.c_str(), 10, &err);
+    if (!err.empty()) {
+      dout(5) << "Error parsing max-entries " << max_entries_str << dendl;
+      http_ret = -EINVAL;
+      return;
+    }
+  } else 
+    max_entries = LOG_CLASS_LIST_MAX_ENTRIES;
+
   RGWMetadataLog *meta_log = store->meta_mgr->get_log();
 
-  meta_log->init_list_entries(shard_id, ut_st, ut_et, &handle);
+  meta_log->init_list_entries(shard_id, ut_st, ut_et, marker, &handle);
 
   bool truncated;
+  do {
+    http_ret = meta_log->list_entries(handle, max_entries, entries, &truncated);
+    if (http_ret < 0) 
+      break;
 
-  http_ret = meta_log->list_entries(handle, 1000, entries, &truncated);
+    if (!max_entries_str.empty()) 
+      max_entries -= entries.size();
+  } while (truncated && (max_entries > 0));
 }
 
 void RGWOp_MDLog_List::send_response() {
@@ -114,11 +132,11 @@ void RGWOp_MDLog_Delete::execute() {
            err;
   utime_t  ut_st, 
            ut_et;
-  int     shard_id;
+  unsigned shard_id;
 
   http_ret = 0;
 
-  shard_id = strict_strtol(shard.c_str(), 10, &err);
+  shard_id = (unsigned)strict_strtol(shard.c_str(), 10, &err);
   if (!err.empty()) {
     dout(5) << "Error parsing shard_id " << shard << dendl;
     http_ret = -EINVAL;
@@ -145,7 +163,7 @@ void RGWOp_MDLog_Delete::execute() {
 
 void RGWOp_MDLog_Lock::execute() {
   string shard_id_str, duration_str, lock_id;
-  int shard_id;
+  unsigned shard_id;
 
   http_ret = 0;
 
@@ -162,7 +180,7 @@ void RGWOp_MDLog_Lock::execute() {
   }
 
   string err;
-  shard_id = strict_strtol(shard_id_str.c_str(), 10, &err);
+  shard_id = (unsigned)strict_strtol(shard_id_str.c_str(), 10, &err);
   if (!err.empty()) {
     dout(5) << "Error parsing shard_id param " << shard_id_str << dendl;
     http_ret = -EINVAL;
@@ -170,8 +188,8 @@ void RGWOp_MDLog_Lock::execute() {
   }
 
   RGWMetadataLog *meta_log = store->meta_mgr->get_log();
-  int dur;
-  dur = strict_strtol(duration_str.c_str(), 10, &err);
+  unsigned dur;
+  dur = (unsigned)strict_strtol(duration_str.c_str(), 10, &err);
   if (!err.empty() || dur <= 0) {
     dout(5) << "invalid length param " << duration_str << dendl;
     http_ret = -EINVAL;
@@ -183,7 +201,7 @@ void RGWOp_MDLog_Lock::execute() {
 
 void RGWOp_MDLog_Unlock::execute() {
   string shard_id_str, lock_id;
-  int shard_id;
+  unsigned shard_id;
 
   http_ret = 0;
 
@@ -198,7 +216,7 @@ void RGWOp_MDLog_Unlock::execute() {
   }
 
   string err;
-  shard_id = strict_strtol(shard_id_str.c_str(), 10, &err);
+  shard_id = (unsigned)strict_strtol(shard_id_str.c_str(), 10, &err);
   if (!err.empty()) {
     dout(5) << "Error parsing shard_id param " << shard_id_str << dendl;
     http_ret = -EINVAL;
@@ -214,7 +232,7 @@ void RGWOp_BILog_List::execute() {
          marker = s->info.args.get("marker"),
          max_entries_str = s->info.args.get("max-entries");
   RGWBucketInfo bucket_info;
-  int max_entries;
+  unsigned max_entries;
 
   if (bucket_name.empty()) {
     dout(5) << "ERROR: bucket not specified" << dendl;
@@ -229,12 +247,12 @@ void RGWOp_BILog_List::execute() {
   }
 
   bool truncated;
-  int count = 0;
+  unsigned count = 0;
   string err;
 
-  max_entries = strict_strtol(max_entries_str.c_str(), 10, &err);
+  max_entries = (unsigned)strict_strtol(max_entries_str.c_str(), 10, &err);
   if (!err.empty())
-    max_entries = 1000;
+    max_entries = LOG_CLASS_LIST_MAX_ENTRIES;
 
   send_response();
   do {
@@ -318,12 +336,14 @@ void RGWOp_DATALog_List::execute() {
 
   string   st = s->info.args.get("start-time"),
            et = s->info.args.get("end-time"),
+           max_entries_str = s->info.args.get("max-entries"),
+           marker = s->info.args.get("marker"),
            err;
   utime_t  ut_st, 
            ut_et;
-  int      shard_id;
+  unsigned shard_id, max_entries = 0;
 
-  shard_id = strict_strtol(shard.c_str(), 10, &err);
+  shard_id = (unsigned)strict_strtol(shard.c_str(), 10, &err);
   if (!err.empty()) {
     dout(5) << "Error parsing shard_id " << shard << dendl;
     http_ret = -EINVAL;
@@ -340,12 +360,26 @@ void RGWOp_DATALog_List::execute() {
     return;
   }
 
-  string marker;
-  bool truncated;
-#define DATALOG_LIST_MAX_ENTRIES 1000
+  if (!max_entries_str.empty()) {
+    max_entries = (unsigned)strict_strtol(max_entries_str.c_str(), 10, &err);
+    if (!err.empty()) {
+      dout(5) << "Error parsing max-entries " << max_entries_str << dendl;
+      http_ret = -EINVAL;
+      return;
+    }
+  } else 
+    max_entries = LOG_CLASS_LIST_MAX_ENTRIES;
 
-  http_ret = store->data_log->list_entries(shard_id, ut_st, ut_et, 
-                               DATALOG_LIST_MAX_ENTRIES, entries, marker, &truncated);
+  bool truncated;
+  do {
+    http_ret = store->data_log->list_entries(shard_id, ut_st, ut_et, 
+                               max_entries, entries, marker, &truncated);
+    if (http_ret < 0) 
+      break;
+
+    if (!max_entries_str.empty()) 
+      max_entries -= entries.size();
+  } while (truncated && (max_entries > 0));
 }
 
 void RGWOp_DATALog_List::send_response() {
@@ -386,7 +420,7 @@ void RGWOp_DATALog_GetShardsInfo::send_response() {
 
 void RGWOp_DATALog_Lock::execute() {
   string shard_id_str, duration_str, lock_id;
-  int shard_id;
+  unsigned shard_id;
 
   http_ret = 0;
 
@@ -403,15 +437,15 @@ void RGWOp_DATALog_Lock::execute() {
   }
 
   string err;
-  shard_id = strict_strtol(shard_id_str.c_str(), 10, &err);
+  shard_id = (unsigned)strict_strtol(shard_id_str.c_str(), 10, &err);
   if (!err.empty()) {
     dout(5) << "Error parsing shard_id param " << shard_id_str << dendl;
     http_ret = -EINVAL;
     return;
   }
 
-  int dur;
-  dur = strict_strtol(duration_str.c_str(), 10, &err);
+  unsigned dur;
+  dur = (unsigned)strict_strtol(duration_str.c_str(), 10, &err);
   if (!err.empty() || dur <= 0) {
     dout(5) << "invalid length param " << duration_str << dendl;
     http_ret = -EINVAL;
@@ -423,7 +457,7 @@ void RGWOp_DATALog_Lock::execute() {
 
 void RGWOp_DATALog_Unlock::execute() {
   string shard_id_str, lock_id;
-  int shard_id;
+  unsigned shard_id;
 
   http_ret = 0;
 
@@ -438,7 +472,7 @@ void RGWOp_DATALog_Unlock::execute() {
   }
 
   string err;
-  shard_id = strict_strtol(shard_id_str.c_str(), 10, &err);
+  shard_id = (unsigned)strict_strtol(shard_id_str.c_str(), 10, &err);
   if (!err.empty()) {
     dout(5) << "Error parsing shard_id param " << shard_id_str << dendl;
     http_ret = -EINVAL;
@@ -455,11 +489,11 @@ void RGWOp_DATALog_Delete::execute() {
            err;
   utime_t  ut_st, 
            ut_et;
-  int     shard_id;
+  unsigned shard_id;
 
   http_ret = 0;
 
-  shard_id = strict_strtol(shard.c_str(), 10, &err);
+  shard_id = (unsigned)strict_strtol(shard.c_str(), 10, &err);
   if (!err.empty()) {
     dout(5) << "Error parsing shard_id " << shard << dendl;
     http_ret = -EINVAL;

diff --git a/src/test/test_rgw_admin_log.cc b/src/test/test_rgw_admin_log.cc
index 40d0f70c3bc8ed0143d9606f784ff6ce4e442e32..3ce853ab67e19703a29fb1c760e65b2f00f843b7 100644 (file)
--- a/src/test/test_rgw_admin_log.cc
+++ b/src/test/test_rgw_admin_log.cc
@@ -798,6 +798,16 @@ TEST(TestRGWAdmin, datalog_list) {
     EXPECT_EQ((*it).key.compare(TEST_BUCKET_NAME), 0);
   }
 
+  ss.str("");
+  ss << "/admin/log?type=data&id=" << shard_id << "&start-time=" << start_time
+    << "&max-entries=1";
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  entries.clear();
+  get_datalog_list(entries);
+  EXPECT_EQ(1U, entries.size());
+  
   ss.str("");
   ss << "/admin/log?type=data&id=" << shard_id << "&start-time=" << start_time 
     << "&end-time=" << end_time;
@@ -1157,6 +1167,25 @@ TEST(TestRGWAdmin, mdlog_list) {
   EXPECT_EQ(get_log_list(entries), 0);
   EXPECT_EQ(entries.size(), 14U);
 
+  ss.str("");
+  ss << "/admin/log?type=metadata&id=" << shard_id << "&start-time=" << start_time 
+    << "&max-entries=" << 1;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  entries.clear();
+  EXPECT_EQ(get_log_list(entries), 0);
+  EXPECT_EQ(entries.size(), 1U);
+
+  ss.str("");
+  ss << "/admin/log?type=metadata&id=" << shard_id << "&start-time=" << start_time 
+    << "&max-entries=" << 6;
+  rest_req = ss.str();
+  g_test->send_request(string("GET"), rest_req);
+  EXPECT_EQ(200U, g_test->get_resp_code());
+  entries.clear();
+  EXPECT_EQ(get_log_list(entries), 0);
+  EXPECT_EQ(entries.size(), 6U);
 
   ASSERT_EQ(0, caps_rm(cname, perm));
   ss.str("");