cls/*: adjust use of cls_cxx_map_get_vals()

Now that objclass call gets a new 'more' param.

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>

diff --git a/src/cls/log/cls_log.cc b/src/cls/log/cls_log.cc
index bfc0e499f0228b6510b94cd8e23b917623a31b5f..411cbc91954d4f57cb09ae6ae47f6dd6c73fd008 100644 (file)
--- a/src/cls/log/cls_log.cc
+++ b/src/cls/log/cls_log.cc
@@ -170,24 +170,22 @@ static int cls_log_list(cls_method_context_t hctx, bufferlist *in, bufferlist *o
   if (!max_entries || max_entries > MAX_ENTRIES)
     max_entries = MAX_ENTRIES;
 
-  int rc = cls_cxx_map_get_vals(hctx, from_index, log_index_prefix, max_entries + 1, &keys);
+  cls_log_list_ret ret;
+
+  int rc = cls_cxx_map_get_vals(hctx, from_index, log_index_prefix, max_entries, &keys, &ret.truncated);
   if (rc < 0)
     return rc;
 
-  cls_log_list_ret ret;
-
   list<cls_log_entry>& entries = ret.entries;
   map<string, bufferlist>::iterator iter = keys.begin();
 
-  bool done = false;
   string marker;
 
-  size_t i;
-  for (i = 0; i < max_entries && iter != keys.end(); ++i, ++iter) {
+  for (; iter != keys.end(); ++iter) {
     const string& index = iter->first;
     marker = index;
     if (use_time_boundary && index.compare(0, to_index.size(), to_index) >= 0) {
-      done = true;
+      ret.truncated = false;
       break;
     }
 
@@ -202,11 +200,9 @@ static int cls_log_list(cls_method_context_t hctx, bufferlist *in, bufferlist *o
     }
   }
 
-  if (iter == keys.end())
-    done = true;
-
-  ret.marker = marker;
-  ret.truncated = !done;
+  if (ret.truncated) {
+    ret.marker = marker;
+  }
 
   ::encode(ret, *out);
 
@@ -244,16 +240,16 @@ static int cls_log_trim(cls_method_context_t hctx, bufferlist *in, bufferlist *o
 
 #define MAX_TRIM_ENTRIES 1000
   size_t max_entries = MAX_TRIM_ENTRIES;
+  bool more;
 
-  int rc = cls_cxx_map_get_vals(hctx, from_index, log_index_prefix, max_entries, &keys);
+  int rc = cls_cxx_map_get_vals(hctx, from_index, log_index_prefix, max_entries, &keys, &more);
   if (rc < 0)
     return rc;
 
   map<string, bufferlist>::iterator iter = keys.begin();
 
-  size_t i;
   bool removed = false;
-  for (i = 0; i < max_entries && iter != keys.end(); ++i, ++iter) {
+  for (; iter != keys.end(); ++iter) {
     const string& index = iter->first;
 
     CLS_LOG(20, "index=%s to_index=%s", index.c_str(), to_index.c_str());

diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc
index 9af96fe6ab4852fb4f9a706261f0fb3e3980a795..f7e2f6b6568472390bd91603e6f3520ff57f290e 100644 (file)
--- a/src/cls/rgw/cls_rgw.cc
+++ b/src/cls/rgw/cls_rgw.cc
@@ -145,9 +145,9 @@ static int log_index_operation(cls_method_context_t hctx, cls_rgw_obj_key& obj_k
  * read list of objects, skips objects in the ugly namespace
  */
 static int get_obj_vals(cls_method_context_t hctx, const string& start, const string& filter_prefix,
-                        int num_entries, map<string, bufferlist> *pkeys)
+                        int num_entries, map<string, bufferlist> *pkeys, bool *pmore)
 {
-  int ret = cls_cxx_map_get_vals(hctx, start, filter_prefix, num_entries, pkeys);
+  int ret = cls_cxx_map_get_vals(hctx, start, filter_prefix, num_entries, pkeys, pmore);
   if (ret < 0)
     return ret;
 
@@ -183,7 +183,7 @@ static int get_obj_vals(cls_method_context_t hctx, const string& start, const st
   string new_start = c;
 
   /* now get some more keys */
-  ret = cls_cxx_map_get_vals(hctx, new_start, filter_prefix, num_entries - pkeys->size(), &new_keys);
+  ret = cls_cxx_map_get_vals(hctx, new_start, filter_prefix, num_entries - pkeys->size(), &new_keys, pmore);
   if (ret < 0)
     return ret;
 
@@ -405,10 +405,11 @@ int rgw_bucket_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
   string start_key;
   encode_list_index_key(hctx, op.start_obj, &start_key);
   bool done = false;
-  uint32_t left_to_read = op.num_entries + 1;
+  uint32_t left_to_read = op.num_entries;
+  bool more;
 
   do {
-    rc = get_obj_vals(hctx, start_key, op.filter_prefix, left_to_read, &keys);
+    rc = get_obj_vals(hctx, start_key, op.filter_prefix, left_to_read, &keys, &more);
     if (rc < 0)
       return rc;
 
@@ -458,8 +459,7 @@ int rgw_bucket_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
     }
   } while (left_to_read > 0 && !done);
 
-  ret.is_truncated = (left_to_read == 0) && /* we found more entries than we were requested, meaning response is truncated */
-                     !done;
+  ret.is_truncated = more && !done;
 
   ::encode(ret, *out);
   return 0;
@@ -482,9 +482,10 @@ static int check_index(cls_method_context_t hctx, struct rgw_bucket_dir_header *
 
 #define CHECK_CHUNK_SIZE 1000
   bool done = false;
+  bool more;
 
   do {
-    rc = get_obj_vals(hctx, start_obj, filter_prefix, CHECK_CHUNK_SIZE, &keys);
+    rc = get_obj_vals(hctx, start_obj, filter_prefix, CHECK_CHUNK_SIZE, &keys, &more);
     if (rc < 0)
       return rc;
 
@@ -1157,9 +1158,10 @@ public:
     get_list_index_key(instance_entry, &list_idx);
     /* this is the current head, need to update! */
     map<string, bufferlist> keys;
+    bool more;
     string filter = key.name; /* list key starts with key name, filter it to avoid a case where we cross to
                                  different namespace */
-    int ret = cls_cxx_map_get_vals(hctx, list_idx, filter, 1, &keys);
+    int ret = cls_cxx_map_get_vals(hctx, list_idx, filter, 1, &keys, &more);
     if (ret < 0) {
       return ret;
     }
@@ -2260,7 +2262,7 @@ static int rgw_bi_put_op(cls_method_context_t hctx, bufferlist *in, bufferlist *
 }
 
 static int list_plain_entries(cls_method_context_t hctx, const string& name, const string& marker, uint32_t max,
-                              list<rgw_cls_bi_entry> *entries)
+                              list<rgw_cls_bi_entry> *entries, bool *pmore)
 {
   string filter = name;
   string start_key = marker;
@@ -2270,59 +2272,52 @@ static int list_plain_entries(cls_method_context_t hctx, const string& name, con
 
   int count = 0;
   map<string, bufferlist> keys;
-  do {
-    if (count >= (int)max) {
+  int ret = cls_cxx_map_get_vals(hctx, start_key, filter, max, &keys, pmore);
+  if (ret < 0) {
+    return ret;
+  }
+
+  map<string, bufferlist>::iterator iter;
+  for (iter = keys.begin(); iter != keys.end(); ++iter) {
+    if (iter->first >= end_key) {
+      /* past the end of plain namespace */
       return count;
     }
-    keys.clear();
-#define BI_GET_NUM_KEYS 128
-    int ret = cls_cxx_map_get_vals(hctx, start_key, filter, BI_GET_NUM_KEYS, &keys);
-    if (ret < 0) {
-      return ret;
-    }
-
-    map<string, bufferlist>::iterator iter;
-    for (iter = keys.begin(); iter != keys.end(); ++iter) {
-      if (iter->first >= end_key) {
-        /* past the end of plain namespace */
-        return count;
-      }
 
-      rgw_cls_bi_entry entry;
-      entry.type = PlainIdx;
-      entry.idx = iter->first;
-      entry.data = iter->second;
+    rgw_cls_bi_entry entry;
+    entry.type = PlainIdx;
+    entry.idx = iter->first;
+    entry.data = iter->second;
 
-      bufferlist::iterator biter = entry.data.begin();
+    bufferlist::iterator biter = entry.data.begin();
 
-      rgw_bucket_dir_entry e;
-      try {
-        ::decode(e, biter);
-      } catch (buffer::error& err) {
-        CLS_LOG(0, "ERROR: %s(): failed to decode buffer", __func__);
-        return -EIO;
-      }
+    rgw_bucket_dir_entry e;
+    try {
+      ::decode(e, biter);
+    } catch (buffer::error& err) {
+      CLS_LOG(0, "ERROR: %s(): failed to decode buffer", __func__);
+      return -EIO;
+    }
 
-      CLS_LOG(20, "%s(): entry.idx=%s e.key.name=%s", __func__, escape_str(entry.idx).c_str(), escape_str(e.key.name).c_str());
+    CLS_LOG(20, "%s(): entry.idx=%s e.key.name=%s", __func__, escape_str(entry.idx).c_str(), escape_str(e.key.name).c_str());
 
-      if (!name.empty() && e.key.name != name) {
-        return count;
-      }
+    if (!name.empty() && e.key.name != name) {
+      return count;
+    }
 
-      entries->push_back(entry);
-      count++;
-      if (count >= (int)max) {
-        return count;
-      }
-      start_key = entry.idx;
+    entries->push_back(entry);
+    count++;
+    if (count >= (int)max) {
+      return count;
     }
-  } while (!keys.empty());
+    start_key = entry.idx;
+  }
 
   return count;
 }
 
 static int list_instance_entries(cls_method_context_t hctx, const string& name, const string& marker, uint32_t max,
-                                 list<rgw_cls_bi_entry> *entries)
+                                 list<rgw_cls_bi_entry> *entries, bool *pmore)
 {
   cls_rgw_obj_key key(name);
   string first_instance_idx;
@@ -2341,66 +2336,63 @@ static int list_instance_entries(cls_method_context_t hctx, const string& name,
   }
   int count = 0;
   map<string, bufferlist> keys;
-  bool started = true;
-  do {
-    if (count >= (int)max) {
-      return count;
-    }
-    keys.clear();
-#define BI_GET_NUM_KEYS 128
-    int ret;
-    if (started) {
-      ret = cls_cxx_map_get_val(hctx, start_key, &keys[start_key]);
-      if (ret == -ENOENT) {
-        ret = cls_cxx_map_get_vals(hctx, start_key, string(), BI_GET_NUM_KEYS, &keys);
-      }
-      started = false;
-    } else {
-      ret = cls_cxx_map_get_vals(hctx, start_key, string(), BI_GET_NUM_KEYS, &keys);
-    }
+  bufferlist k;
+  int ret = cls_cxx_map_get_val(hctx, start_key, &k);
+  if (ret < 0 && ret != -ENOENT) {
+    return ret;
+  }
+  bool found_first = (ret == 0);
+  if (found_first) {
+    --max;
+  }
+  if (max > 0) {
+    ret = cls_cxx_map_get_vals(hctx, start_key, string(), max, &keys, pmore);
     CLS_LOG(20, "%s(): start_key=%s first_instance_idx=%s keys.size()=%d", __func__, escape_str(start_key).c_str(), escape_str(first_instance_idx).c_str(), (int)keys.size());
     if (ret < 0) {
       return ret;
     }
+  }
+  if (found_first) {
+    keys[start_key].claim(k);
+  }
 
-    map<string, bufferlist>::iterator iter;
-    for (iter = keys.begin(); iter != keys.end(); ++iter) {
-      rgw_cls_bi_entry entry;
-      entry.type = InstanceIdx;
-      entry.idx = iter->first;
-      entry.data = iter->second;
-
-      if (!filter.empty() && entry.idx.compare(0, filter.size(), filter) != 0) {
-        return count;
-      }
+  map<string, bufferlist>::iterator iter;
+  for (iter = keys.begin(); iter != keys.end(); ++iter) {
+    rgw_cls_bi_entry entry;
+    entry.type = InstanceIdx;
+    entry.idx = iter->first;
+    entry.data = iter->second;
 
-      CLS_LOG(20, "%s(): entry.idx=%s", __func__, escape_str(entry.idx).c_str());
+    if (!filter.empty() && entry.idx.compare(0, filter.size(), filter) != 0) {
+      return count;
+    }
 
-      bufferlist::iterator biter = entry.data.begin();
+    CLS_LOG(20, "%s(): entry.idx=%s", __func__, escape_str(entry.idx).c_str());
 
-      rgw_bucket_dir_entry e;
-      try {
-        ::decode(e, biter);
-      } catch (buffer::error& err) {
-        CLS_LOG(0, "ERROR: %s(): failed to decode buffer (size=%d)", __func__, entry.data.length());
-        return -EIO;
-      }
+    bufferlist::iterator biter = entry.data.begin();
 
-      if (!name.empty() && e.key.name != name) {
-        return count;
-      }
+    rgw_bucket_dir_entry e;
+    try {
+      ::decode(e, biter);
+    } catch (buffer::error& err) {
+      CLS_LOG(0, "ERROR: %s(): failed to decode buffer (size=%d)", __func__, entry.data.length());
+      return -EIO;
+    }
 
-      entries->push_back(entry);
-      count++;
-      start_key = entry.idx;
+    if (!name.empty() && e.key.name != name) {
+      return count;
     }
-  } while (!keys.empty());
+
+    entries->push_back(entry);
+    count++;
+    start_key = entry.idx;
+  }
 
   return count;
 }
 
 static int list_olh_entries(cls_method_context_t hctx, const string& name, const string& marker, uint32_t max,
-                            list<rgw_cls_bi_entry> *entries)
+                            list<rgw_cls_bi_entry> *entries, bool *pmore)
 {
   cls_rgw_obj_key key(name);
   string first_instance_idx;
@@ -2419,60 +2411,59 @@ static int list_olh_entries(cls_method_context_t hctx, const string& name, const
   }
   int count = 0;
   map<string, bufferlist> keys;
-  bool started = true;
-  do {
-    if (count >= (int)max) {
-      return count;
-    }
-    keys.clear();
-#define BI_GET_NUM_KEYS 128
-    int ret;
-    if (started) {
-      ret = cls_cxx_map_get_val(hctx, start_key, &keys[start_key]);
-      if (ret == -ENOENT) {
-        ret = cls_cxx_map_get_vals(hctx, start_key, string(), BI_GET_NUM_KEYS, &keys);
-      }
-      started = false;
-    } else {
-      ret = cls_cxx_map_get_vals(hctx, start_key, string(), BI_GET_NUM_KEYS, &keys);
-    }
+  int ret;
+  bufferlist k;
+  ret = cls_cxx_map_get_val(hctx, start_key, &k);
+  if (ret < 0 && ret != -ENOENT) {
+    return ret;
+  }
+  bool found_first = (ret == 0);
+  if (found_first) {
+    --max;
+  }
+  if (max > 0) {
+    ret = cls_cxx_map_get_vals(hctx, start_key, string(), max, &keys, pmore);
     CLS_LOG(20, "%s(): start_key=%s first_instance_idx=%s keys.size()=%d", __func__, escape_str(start_key).c_str(), escape_str(first_instance_idx).c_str(), (int)keys.size());
     if (ret < 0) {
       return ret;
     }
+  }
 
-    map<string, bufferlist>::iterator iter;
-    for (iter = keys.begin(); iter != keys.end(); ++iter) {
-      rgw_cls_bi_entry entry;
-      entry.type = OLHIdx;
-      entry.idx = iter->first;
-      entry.data = iter->second;
+  if (found_first) {
+    keys[start_key].claim(k);
+  }
 
-      if (!filter.empty() && entry.idx.compare(0, filter.size(), filter) != 0) {
-        return count;
-      }
+  map<string, bufferlist>::iterator iter;
+  for (iter = keys.begin(); iter != keys.end(); ++iter) {
+    rgw_cls_bi_entry entry;
+    entry.type = OLHIdx;
+    entry.idx = iter->first;
+    entry.data = iter->second;
 
-      CLS_LOG(20, "%s(): entry.idx=%s", __func__, escape_str(entry.idx).c_str());
+    if (!filter.empty() && entry.idx.compare(0, filter.size(), filter) != 0) {
+      return count;
+    }
 
-      bufferlist::iterator biter = entry.data.begin();
+    CLS_LOG(20, "%s(): entry.idx=%s", __func__, escape_str(entry.idx).c_str());
 
-      rgw_bucket_olh_entry e;
-      try {
-        ::decode(e, biter);
-      } catch (buffer::error& err) {
-        CLS_LOG(0, "ERROR: %s(): failed to decode buffer (size=%d)", __func__, entry.data.length());
-        return -EIO;
-      }
+    bufferlist::iterator biter = entry.data.begin();
 
-      if (!name.empty() && e.key.name != name) {
-        return count;
-      }
+    rgw_bucket_olh_entry e;
+    try {
+      ::decode(e, biter);
+    } catch (buffer::error& err) {
+      CLS_LOG(0, "ERROR: %s(): failed to decode buffer (size=%d)", __func__, entry.data.length());
+      return -EIO;
+    }
 
-      entries->push_back(entry);
-      count++;
-      start_key = entry.idx;
+    if (!name.empty() && e.key.name != name) {
+      return count;
     }
-  } while (!keys.empty());
+
+    entries->push_back(entry);
+    count++;
+    start_key = entry.idx;
+  }
 
   return count;
 }
@@ -2493,9 +2484,10 @@ static int rgw_bi_list_op(cls_method_context_t hctx, bufferlist *in, bufferlist
 
   string filter = op.name;
 #define MAX_BI_LIST_ENTRIES 1000
-  int32_t max = (op.max < MAX_BI_LIST_ENTRIES ? op.max : MAX_BI_LIST_ENTRIES) + 1; /* one extra entry for identifying truncation */
+  int32_t max = (op.max < MAX_BI_LIST_ENTRIES ? op.max : MAX_BI_LIST_ENTRIES);
   string start_key = op.marker;
-  int ret = list_plain_entries(hctx, op.name, op.marker, max, &op_ret.entries); 
+  bool more;
+  int ret = list_plain_entries(hctx, op.name, op.marker, max, &op_ret.entries, &more); 
   if (ret < 0) {
     CLS_LOG(0, "ERROR: %s(): list_plain_entries retured ret=%d", __func__, ret);
     return ret;
@@ -2504,23 +2496,27 @@ static int rgw_bi_list_op(cls_method_context_t hctx, bufferlist *in, bufferlist
 
   CLS_LOG(20, "found %d plain entries", count);
 
-  ret = list_instance_entries(hctx, op.name, op.marker, max - count, &op_ret.entries);
-  if (ret < 0) {
-    CLS_LOG(0, "ERROR: %s(): list_instance_entries retured ret=%d", __func__, ret);
-    return ret;
+  if (!more) {
+    ret = list_instance_entries(hctx, op.name, op.marker, max - count, &op_ret.entries, &more);
+    if (ret < 0) {
+      CLS_LOG(0, "ERROR: %s(): list_instance_entries retured ret=%d", __func__, ret);
+      return ret;
+    }
+
+    count += ret;
   }
 
-  count += ret;
+  if (!more) {
+    ret = list_olh_entries(hctx, op.name, op.marker, max - count, &op_ret.entries, &more);
+    if (ret < 0) {
+      CLS_LOG(0, "ERROR: %s(): list_olh_entries retured ret=%d", __func__, ret);
+      return ret;
+    }
 
-  ret = list_olh_entries(hctx, op.name, op.marker, max - count, &op_ret.entries);
-  if (ret < 0) {
-    CLS_LOG(0, "ERROR: %s(): list_olh_entries retured ret=%d", __func__, ret);
-    return ret;
+    count += ret;
   }
 
-  count += ret;
-
-  op_ret.is_truncated = (count >= max);
+  op_ret.is_truncated = (count >= max) || more;
   while (count >= max) {
     op_ret.entries.pop_back();
     count--;
@@ -2582,45 +2578,40 @@ static int bi_log_iterate_entries(cls_method_context_t hctx, const string& marke
 
   string filter;
 
-  do {
-#define BI_NUM_KEYS 128
-    int ret = cls_cxx_map_get_vals(hctx, start_key, filter, BI_NUM_KEYS, &keys);
-    if (ret < 0)
-      return ret;
+  int ret = cls_cxx_map_get_vals(hctx, start_key, filter, max_entries, &keys, truncated);
+  if (ret < 0)
+    return ret;
 
-    map<string, bufferlist>::iterator iter = keys.begin();
-    if (iter == keys.end())
-      break;
+  map<string, bufferlist>::iterator iter = keys.begin();
+  if (iter == keys.end())
+    return 0;
 
-    for (; iter != keys.end(); ++iter) {
-      const string& key = iter->first;
-      rgw_bi_log_entry e;
+  uint32_t num_keys = keys.size();
 
-      CLS_LOG(0, "bi_log_iterate_entries key=%s bl.length=%d\n", key.c_str(), (int)iter->second.length());
+  for (; iter != keys.end(); ++iter,++i) {
+    const string& key = iter->first;
+    rgw_bi_log_entry e;
 
-      if (key.compare(end_key) > 0)
-        return 0;
+    CLS_LOG(0, "bi_log_iterate_entries key=%s bl.length=%d\n", key.c_str(), (int)iter->second.length());
 
-      ret = bi_log_record_decode(iter->second, e);
-      if (ret < 0)
-        return ret;
+    if (key.compare(end_key) > 0) {
+      key_iter = key;
+      return 0;
+    }
 
-      if (max_entries && (i >= max_entries)) {
-        if (truncated)
-          *truncated = true;
-        key_iter = key;
-        return 0;
-      }
+    ret = bi_log_record_decode(iter->second, e);
+    if (ret < 0)
+      return ret;
 
-      ret = cb(hctx, key, e, param);
-      if (ret < 0)
-        return ret;
-      i++;
+    ret = cb(hctx, key, e, param);
+    if (ret < 0)
+      return ret;
 
+    if (i == num_keys - 1) {
+      key_iter = key;
     }
-    --iter;
-    start_key = iter->first;
-  } while (true);
+  }
+
   return 0;
 }
 
@@ -2862,58 +2853,53 @@ static int usage_iterate_range(cls_method_context_t hctx, uint64_t start, uint64
     start_key = key_iter;
   }
 
-  do {
-    CLS_LOG(20, "usage_iterate_range start_key=%s", start_key.c_str());
-    int ret = cls_cxx_map_get_vals(hctx, start_key, filter_prefix, NUM_KEYS, &keys);
-    if (ret < 0)
-      return ret;
+  CLS_LOG(20, "usage_iterate_range start_key=%s", start_key.c_str());
+  int ret = cls_cxx_map_get_vals(hctx, start_key, filter_prefix, max_entries, &keys, truncated);
+  if (ret < 0)
+    return ret;
 
 
-    map<string, bufferlist>::iterator iter = keys.begin();
-    if (iter == keys.end())
-      break;
+  map<string, bufferlist>::iterator iter = keys.begin();
+  if (iter == keys.end())
+    return 0;
 
-    for (; iter != keys.end(); ++iter) {
-      const string& key = iter->first;
-      rgw_usage_log_entry e;
+  uint32_t num_keys = keys.size();
 
-      if (!by_user && key.compare(end_key) >= 0) {
-        CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str());
-        return 0;
-      }
+  for (; iter != keys.end(); ++iter,++i) {
+    const string& key = iter->first;
+    rgw_usage_log_entry e;
 
-      if (by_user && key.compare(0, user_key.size(), user_key) != 0) {
-        CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str());
-        return 0;
-      }
+    if (!by_user && key.compare(end_key) >= 0) {
+      CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str());
+      return 0;
+    }
 
-      ret = usage_record_decode(iter->second, e);
-      if (ret < 0)
-        return ret;
+    if (by_user && key.compare(0, user_key.size(), user_key) != 0) {
+      CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str());
+      return 0;
+    }
 
-      if (e.epoch < start)
-       continue;
+    ret = usage_record_decode(iter->second, e);
+    if (ret < 0)
+      return ret;
 
-      /* keys are sorted by epoch, so once we're past end we're done */
-      if (e.epoch >= end)
-        return 0;
+    if (e.epoch < start)
+      continue;
 
-      ret = cb(hctx, key, e, param);
-      if (ret < 0)
-        return ret;
+    /* keys are sorted by epoch, so once we're past end we're done */
+    if (e.epoch >= end)
+      return 0;
 
+    ret = cb(hctx, key, e, param);
+    if (ret < 0)
+      return ret;
 
-      i++;
-      if (max_entries && (i > max_entries)) {
-        CLS_LOG(20, "usage_iterate_range reached max_entries (%d), done", max_entries);
-        *truncated = true;
-        key_iter = key;
-        return 0;
-      }
+
+    if (i == num_keys - 1) {
+      key_iter = key;
+      return 0;
     }
-    --iter;
-    start_key = iter->first;
-  } while (true);
+  }
   return 0;
 }
 
@@ -2999,7 +2985,9 @@ int rgw_user_usage_log_trim(cls_method_context_t hctx, bufferlist *in, bufferlis
   }
 
   string iter;
-  ret = usage_iterate_range(hctx, op.start_epoch, op.end_epoch, op.user, iter, 0, NULL, usage_log_trim_cb, NULL);
+  bool more;
+#define MAX_USAGE_TRIM_ENTRIES 128
+  ret = usage_iterate_range(hctx, op.start_epoch, op.end_epoch, op.user, iter, MAX_USAGE_TRIM_ENTRIES, &more, usage_log_trim_cb, NULL);
   if (ret < 0)
     return ret;
 
@@ -3197,50 +3185,42 @@ static int gc_iterate_entries(cls_method_context_t hctx, const string& marker, b
 
   string filter;
 
-  do {
-#define GC_NUM_KEYS 32
-    int ret = cls_cxx_map_get_vals(hctx, start_key, filter, GC_NUM_KEYS, &keys);
-    if (ret < 0)
-      return ret;
+  int ret = cls_cxx_map_get_vals(hctx, start_key, filter, max_entries, &keys, truncated);
+  if (ret < 0)
+    return ret;
 
 
-    map<string, bufferlist>::iterator iter = keys.begin();
-    if (iter == keys.end())
-      break;
+  map<string, bufferlist>::iterator iter = keys.begin();
+  if (iter == keys.end())
+    return 0;
 
-    for (; iter != keys.end(); ++iter) {
-      const string& key = iter->first;
-      cls_rgw_gc_obj_info e;
+  uint32_t num_keys = keys.size();
 
-      CLS_LOG(10, "gc_iterate_entries key=%s\n", key.c_str());
+  for (; iter != keys.end(); ++iter, ++i) {
+    const string& key = iter->first;
+    cls_rgw_gc_obj_info e;
 
-      if (!end_key.empty() && key.compare(end_key) >= 0)
-        return 0;
+    CLS_LOG(10, "gc_iterate_entries key=%s\n", key.c_str());
 
-      if (!key_in_index(key, GC_OBJ_TIME_INDEX))
-       return 0;
+    if (!end_key.empty() && key.compare(end_key) >= 0)
+      return 0;
 
-      ret = gc_record_decode(iter->second, e);
-      if (ret < 0)
-        return ret;
+    if (!key_in_index(key, GC_OBJ_TIME_INDEX))
+      return 0;
 
-      if (max_entries && (i >= max_entries)) {
-        if (truncated)
-          *truncated = true;
-        --iter;
-        key_iter = iter->first;
-        return 0;
-      }
+    ret = gc_record_decode(iter->second, e);
+    if (ret < 0)
+      return ret;
 
-      ret = cb(hctx, key, e, param);
-      if (ret < 0)
-        return ret;
-      i++;
+    ret = cb(hctx, key, e, param);
+    if (ret < 0)
+      return ret;
 
+    if (i == num_keys - 1) {
+      key_iter = key;
     }
-    --iter;
-    start_key = iter->first;
-  } while (true);
+  }
+
   return 0;
 }
 
@@ -3274,7 +3254,8 @@ static int rgw_cls_gc_list(cls_method_context_t hctx, bufferlist *in, bufferlist
   }
 
   cls_rgw_gc_list_ret op_ret;
-  int ret = gc_list_entries(hctx, op.marker, op.max, op.expired_only, 
+#define GC_LIST_ENTRIES_DEFAULT 128
+  int ret = gc_list_entries(hctx, op.marker, (op.max ? op.max : GC_LIST_ENTRIES_DEFAULT), op.expired_only, 
    op_ret.entries, &op_ret.truncated, op_ret.next_marker);
   if (ret < 0)
     return ret;
@@ -3384,7 +3365,8 @@ static int rgw_cls_lc_get_next_entry(cls_method_context_t hctx, bufferlist *in,
 
   map<string, bufferlist> vals;
   string filter_prefix;
-  int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, 1, &vals);
+  bool more;
+  int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, 1, &vals, &more);
   if (ret < 0)
     return ret;
   map<string, bufferlist>::iterator it;
@@ -3419,7 +3401,7 @@ static int rgw_cls_lc_list_entries(cls_method_context_t hctx, bufferlist *in, bu
   bufferlist::iterator iter;
   map<string, bufferlist> vals;
   string filter_prefix;
-  int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, op.max_entries, &vals);
+  int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, op.max_entries, &vals, &op_ret.is_truncated);
   if (ret < 0)
     return ret;
   map<string, bufferlist>::iterator it;
@@ -3524,8 +3506,8 @@ static int rgw_reshard_list(cls_method_context_t hctx, bufferlist *in, bufferlis
   string filter_prefix;
 #define MAX_RESHARD_LIST_ENTRIES 1000
   /* one extra entry for identifying truncation */
-  int32_t max = (op.max < MAX_RESHARD_LIST_ENTRIES ? op.max : MAX_RESHARD_LIST_ENTRIES) + 1;
-  int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, max, &vals);
+  int32_t max = (op.max && (op.max < MAX_RESHARD_LIST_ENTRIES) ? op.max : MAX_RESHARD_LIST_ENTRIES);
+  int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, max, &vals, &op_ret.is_truncated);
   if (ret < 0)
     return ret;
   map<string, bufferlist>::iterator it;
@@ -3541,7 +3523,6 @@ static int rgw_reshard_list(cls_method_context_t hctx, bufferlist *in, bufferlis
    }
     op_ret.entries.push_back(entry);
   }
-  op_ret.is_truncated = op.max && (vals.size() > op.max);
   ::encode(op_ret, *out);
   return 0;
 }

diff --git a/src/cls/rgw/cls_rgw_ops.h b/src/cls/rgw/cls_rgw_ops.h
index 538443adf40b40179904a4d1f439d7efd4b573a4..07ebeea12fde758122a897ffc2cdd766bfdea6ae 100644 (file)
--- a/src/cls/rgw/cls_rgw_ops.h
+++ b/src/cls/rgw/cls_rgw_ops.h
@@ -1123,18 +1123,23 @@ WRITE_CLASS_ENCODER(cls_rgw_lc_list_entries_op)
 
 struct cls_rgw_lc_list_entries_ret {
   map<string, int> entries;
+  bool is_truncated{false};
 
   cls_rgw_lc_list_entries_ret() {}
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(1, 1, bl);
+    ENCODE_START(2, 1, bl);
     ::encode(entries, bl);
+    ::encode(is_truncated, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::iterator& bl) {
-    DECODE_START(1, bl);
+    DECODE_START(2, bl);
     ::decode(entries, bl);
+    if (struct_v >= 2) {
+      ::decode(is_truncated, bl);
+    }
     DECODE_FINISH(bl);
   }
 

diff --git a/src/cls/statelog/cls_statelog.cc b/src/cls/statelog/cls_statelog.cc
index f404fa77b60b38b1af6a914552352cdbd75d9ece..b2a13764d1e8f2cd6d0eb36e50b60427343d1f79 100644 (file)
--- a/src/cls/statelog/cls_statelog.cc
+++ b/src/cls/statelog/cls_statelog.cc
@@ -168,21 +168,20 @@ static int cls_statelog_list(cls_method_context_t hctx, bufferlist *in, bufferli
   if (!max_entries || max_entries > MAX_ENTRIES)
     max_entries = MAX_ENTRIES;
 
-  int rc = cls_cxx_map_get_vals(hctx, from_index, match_prefix, max_entries + 1, &keys);
+  cls_statelog_list_ret ret;
+
+  int rc = cls_cxx_map_get_vals(hctx, from_index, match_prefix, max_entries, &keys, &ret.truncated);
   if (rc < 0)
     return rc;
 
   CLS_LOG(20, "from_index=%s match_prefix=%s", from_index.c_str(), match_prefix.c_str());
-  cls_statelog_list_ret ret;
 
   list<cls_statelog_entry>& entries = ret.entries;
   map<string, bufferlist>::iterator iter = keys.begin();
 
-  bool done = false;
   string marker;
 
-  size_t i;
-  for (i = 0; i < max_entries && iter != keys.end(); ++i, ++iter) {
+  for (; iter != keys.end(); ++iter) {
     const string& index = iter->first;
     marker = index;
 
@@ -197,12 +196,9 @@ static int cls_statelog_list(cls_method_context_t hctx, bufferlist *in, bufferli
     }
   }
 
-  if (iter == keys.end())
-    done = true;
-  else
+  if (ret.truncated) {
     ret.marker = marker;
-
-  ret.truncated = !done;
+  }
 
   ::encode(ret, *out);
 

diff --git a/src/cls/timeindex/cls_timeindex.cc b/src/cls/timeindex/cls_timeindex.cc
index 202b653567ea7c2d119757e9cd15d08a52286e85..bed96e5cd74d6e405ed65276d1309b5ac1dc5985 100644 (file)
--- a/src/cls/timeindex/cls_timeindex.cc
+++ b/src/cls/timeindex/cls_timeindex.cc
@@ -120,21 +120,20 @@ static int cls_timeindex_list(cls_method_context_t hctx,
     max_entries = MAX_LIST_ENTRIES;
   }
 
+  cls_timeindex_list_ret ret;
+
   int rc = cls_cxx_map_get_vals(hctx, from_index, TIMEINDEX_PREFIX,
-          max_entries + 1, &keys);
+          max_entries, &keys, &ret.truncated);
   if (rc < 0) {
     return rc;
   }
 
-  cls_timeindex_list_ret ret;
-
   list<cls_timeindex_entry>& entries = ret.entries;
   map<string, bufferlist>::iterator iter = keys.begin();
 
-  bool done = false;
   string marker;
 
-  for (size_t i = 0; i < max_entries && iter != keys.end(); ++i, ++iter) {
+  for (; iter != keys.end(); ++iter) {
     const string& index = iter->first;
     bufferlist& bl = iter->second;
 
@@ -142,7 +141,7 @@ static int cls_timeindex_list(cls_method_context_t hctx,
     if (use_time_boundary && index.compare(0, to_index.size(), to_index) >= 0) {
       CLS_LOG(20, "DEBUG: cls_timeindex_list: finishing on to_index=%s",
               to_index.c_str());
-      done = true;
+      ret.truncated = false;
       break;
     }
 
@@ -159,12 +158,7 @@ static int cls_timeindex_list(cls_method_context_t hctx,
     }
   }
 
-  if (iter == keys.end()) {
-    done = true;
-  }
-
   ret.marker = marker;
-  ret.truncated = !done;
 
   ::encode(ret, *out);
 
@@ -203,8 +197,10 @@ static int cls_timeindex_trim(cls_method_context_t hctx,
     to_index = op.to_marker;
   }
 
+  bool more;
+
   int rc = cls_cxx_map_get_vals(hctx, from_index, TIMEINDEX_PREFIX,
-          MAX_TRIM_ENTRIES, &keys);
+          MAX_TRIM_ENTRIES, &keys, &more);
   if (rc < 0) {
     return rc;
   }
@@ -212,7 +208,7 @@ static int cls_timeindex_trim(cls_method_context_t hctx,
   map<string, bufferlist>::iterator iter = keys.begin();
 
   bool removed = false;
-  for (size_t i = 0; i < MAX_TRIM_ENTRIES && iter != keys.end(); ++i, ++iter) {
+  for (; iter != keys.end(); ++iter) {
     const string& index = iter->first;
 
     CLS_LOG(20, "index=%s to_index=%s", index.c_str(), to_index.c_str());

diff --git a/src/cls/user/cls_user.cc b/src/cls/user/cls_user.cc
index c616d1ae6563988f89121bc6b922747a46476f02..7912578aa36c8aed8fdea07c8cb1849e760f0aea 100644 (file)
--- a/src/cls/user/cls_user.cc
+++ b/src/cls/user/cls_user.cc
@@ -292,8 +292,9 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff
     max_entries = MAX_ENTRIES;
 
   string match_prefix;
+  cls_user_list_buckets_ret ret;
 
-  int rc = cls_cxx_map_get_vals(hctx, from_index, match_prefix, max_entries + 1, &keys);
+  int rc = cls_cxx_map_get_vals(hctx, from_index, match_prefix, max_entries, &keys, &ret.truncated);
   if (rc < 0)
     return rc;
 
@@ -301,21 +302,20 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff
           from_index.c_str(),
           to_index.c_str(),
           match_prefix.c_str());
-  cls_user_list_buckets_ret ret;
 
   list<cls_user_bucket_entry>& entries = ret.entries;
   map<string, bufferlist>::iterator iter = keys.begin();
 
-  bool done = false;
   string marker;
 
-  size_t i;
-  for (i = 0; i < max_entries && iter != keys.end(); ++i, ++iter) {
+  for (; iter != keys.end(); ++iter) {
     const string& index = iter->first;
     marker = index;
 
-    if (to_index_valid && to_index.compare(index) <= 0)
+    if (to_index_valid && to_index.compare(index) <= 0) {
+      ret.truncated = false;
       break;
+    }
 
     bufferlist& bl = iter->second;
     bufferlist::iterator biter = bl.begin();
@@ -328,12 +328,9 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff
     }
   }
 
-  if (iter == keys.end())
-    done = true;
-  else
+  if (ret.truncated) {
     ret.marker = marker;
-
-  ret.truncated = !done;
+  }
 
   ::encode(ret, *out);