From: Yehuda Sadeh Date: Fri, 14 Jul 2017 22:28:09 +0000 (-0700) Subject: cls/*: adjust use of cls_cxx_map_get_vals() X-Git-Tag: v12.1.2~36^2~4 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=5334622a8365520fa4247241f97422c044cbf5b2;p=ceph.git cls/*: adjust use of cls_cxx_map_get_vals() Now that objclass call gets a new 'more' param. Signed-off-by: Yehuda Sadeh --- diff --git a/src/cls/log/cls_log.cc b/src/cls/log/cls_log.cc index bfc0e499f02..411cbc91954 100644 --- a/src/cls/log/cls_log.cc +++ b/src/cls/log/cls_log.cc @@ -170,24 +170,22 @@ static int cls_log_list(cls_method_context_t hctx, bufferlist *in, bufferlist *o if (!max_entries || max_entries > MAX_ENTRIES) max_entries = MAX_ENTRIES; - int rc = cls_cxx_map_get_vals(hctx, from_index, log_index_prefix, max_entries + 1, &keys); + cls_log_list_ret ret; + + int rc = cls_cxx_map_get_vals(hctx, from_index, log_index_prefix, max_entries, &keys, &ret.truncated); if (rc < 0) return rc; - cls_log_list_ret ret; - list& entries = ret.entries; map::iterator iter = keys.begin(); - bool done = false; string marker; - size_t i; - for (i = 0; i < max_entries && iter != keys.end(); ++i, ++iter) { + for (; iter != keys.end(); ++iter) { const string& index = iter->first; marker = index; if (use_time_boundary && index.compare(0, to_index.size(), to_index) >= 0) { - done = true; + ret.truncated = false; break; } @@ -202,11 +200,9 @@ static int cls_log_list(cls_method_context_t hctx, bufferlist *in, bufferlist *o } } - if (iter == keys.end()) - done = true; - - ret.marker = marker; - ret.truncated = !done; + if (ret.truncated) { + ret.marker = marker; + } ::encode(ret, *out); @@ -244,16 +240,16 @@ static int cls_log_trim(cls_method_context_t hctx, bufferlist *in, bufferlist *o #define MAX_TRIM_ENTRIES 1000 size_t max_entries = MAX_TRIM_ENTRIES; + bool more; - int rc = cls_cxx_map_get_vals(hctx, from_index, log_index_prefix, max_entries, &keys); + int rc = cls_cxx_map_get_vals(hctx, from_index, log_index_prefix, max_entries, &keys, &more); if (rc < 0) return rc; map::iterator iter = keys.begin(); - size_t i; bool removed = false; - for (i = 0; i < max_entries && iter != keys.end(); ++i, ++iter) { + for (; iter != keys.end(); ++iter) { const string& index = iter->first; CLS_LOG(20, "index=%s to_index=%s", index.c_str(), to_index.c_str()); diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc index 9af96fe6ab4..f7e2f6b6568 100644 --- a/src/cls/rgw/cls_rgw.cc +++ b/src/cls/rgw/cls_rgw.cc @@ -145,9 +145,9 @@ static int log_index_operation(cls_method_context_t hctx, cls_rgw_obj_key& obj_k * read list of objects, skips objects in the ugly namespace */ static int get_obj_vals(cls_method_context_t hctx, const string& start, const string& filter_prefix, - int num_entries, map *pkeys) + int num_entries, map *pkeys, bool *pmore) { - int ret = cls_cxx_map_get_vals(hctx, start, filter_prefix, num_entries, pkeys); + int ret = cls_cxx_map_get_vals(hctx, start, filter_prefix, num_entries, pkeys, pmore); if (ret < 0) return ret; @@ -183,7 +183,7 @@ static int get_obj_vals(cls_method_context_t hctx, const string& start, const st string new_start = c; /* now get some more keys */ - ret = cls_cxx_map_get_vals(hctx, new_start, filter_prefix, num_entries - pkeys->size(), &new_keys); + ret = cls_cxx_map_get_vals(hctx, new_start, filter_prefix, num_entries - pkeys->size(), &new_keys, pmore); if (ret < 0) return ret; @@ -405,10 +405,11 @@ int rgw_bucket_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out) string start_key; encode_list_index_key(hctx, op.start_obj, &start_key); bool done = false; - uint32_t left_to_read = op.num_entries + 1; + uint32_t left_to_read = op.num_entries; + bool more; do { - rc = get_obj_vals(hctx, start_key, op.filter_prefix, left_to_read, &keys); + rc = get_obj_vals(hctx, start_key, op.filter_prefix, left_to_read, &keys, &more); if (rc < 0) return rc; @@ -458,8 +459,7 @@ int rgw_bucket_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out) } } while (left_to_read > 0 && !done); - ret.is_truncated = (left_to_read == 0) && /* we found more entries than we were requested, meaning response is truncated */ - !done; + ret.is_truncated = more && !done; ::encode(ret, *out); return 0; @@ -482,9 +482,10 @@ static int check_index(cls_method_context_t hctx, struct rgw_bucket_dir_header * #define CHECK_CHUNK_SIZE 1000 bool done = false; + bool more; do { - rc = get_obj_vals(hctx, start_obj, filter_prefix, CHECK_CHUNK_SIZE, &keys); + rc = get_obj_vals(hctx, start_obj, filter_prefix, CHECK_CHUNK_SIZE, &keys, &more); if (rc < 0) return rc; @@ -1157,9 +1158,10 @@ public: get_list_index_key(instance_entry, &list_idx); /* this is the current head, need to update! */ map keys; + bool more; string filter = key.name; /* list key starts with key name, filter it to avoid a case where we cross to different namespace */ - int ret = cls_cxx_map_get_vals(hctx, list_idx, filter, 1, &keys); + int ret = cls_cxx_map_get_vals(hctx, list_idx, filter, 1, &keys, &more); if (ret < 0) { return ret; } @@ -2260,7 +2262,7 @@ static int rgw_bi_put_op(cls_method_context_t hctx, bufferlist *in, bufferlist * } static int list_plain_entries(cls_method_context_t hctx, const string& name, const string& marker, uint32_t max, - list *entries) + list *entries, bool *pmore) { string filter = name; string start_key = marker; @@ -2270,59 +2272,52 @@ static int list_plain_entries(cls_method_context_t hctx, const string& name, con int count = 0; map keys; - do { - if (count >= (int)max) { + int ret = cls_cxx_map_get_vals(hctx, start_key, filter, max, &keys, pmore); + if (ret < 0) { + return ret; + } + + map::iterator iter; + for (iter = keys.begin(); iter != keys.end(); ++iter) { + if (iter->first >= end_key) { + /* past the end of plain namespace */ return count; } - keys.clear(); -#define BI_GET_NUM_KEYS 128 - int ret = cls_cxx_map_get_vals(hctx, start_key, filter, BI_GET_NUM_KEYS, &keys); - if (ret < 0) { - return ret; - } - - map::iterator iter; - for (iter = keys.begin(); iter != keys.end(); ++iter) { - if (iter->first >= end_key) { - /* past the end of plain namespace */ - return count; - } - rgw_cls_bi_entry entry; - entry.type = PlainIdx; - entry.idx = iter->first; - entry.data = iter->second; + rgw_cls_bi_entry entry; + entry.type = PlainIdx; + entry.idx = iter->first; + entry.data = iter->second; - bufferlist::iterator biter = entry.data.begin(); + bufferlist::iterator biter = entry.data.begin(); - rgw_bucket_dir_entry e; - try { - ::decode(e, biter); - } catch (buffer::error& err) { - CLS_LOG(0, "ERROR: %s(): failed to decode buffer", __func__); - return -EIO; - } + rgw_bucket_dir_entry e; + try { + ::decode(e, biter); + } catch (buffer::error& err) { + CLS_LOG(0, "ERROR: %s(): failed to decode buffer", __func__); + return -EIO; + } - CLS_LOG(20, "%s(): entry.idx=%s e.key.name=%s", __func__, escape_str(entry.idx).c_str(), escape_str(e.key.name).c_str()); + CLS_LOG(20, "%s(): entry.idx=%s e.key.name=%s", __func__, escape_str(entry.idx).c_str(), escape_str(e.key.name).c_str()); - if (!name.empty() && e.key.name != name) { - return count; - } + if (!name.empty() && e.key.name != name) { + return count; + } - entries->push_back(entry); - count++; - if (count >= (int)max) { - return count; - } - start_key = entry.idx; + entries->push_back(entry); + count++; + if (count >= (int)max) { + return count; } - } while (!keys.empty()); + start_key = entry.idx; + } return count; } static int list_instance_entries(cls_method_context_t hctx, const string& name, const string& marker, uint32_t max, - list *entries) + list *entries, bool *pmore) { cls_rgw_obj_key key(name); string first_instance_idx; @@ -2341,66 +2336,63 @@ static int list_instance_entries(cls_method_context_t hctx, const string& name, } int count = 0; map keys; - bool started = true; - do { - if (count >= (int)max) { - return count; - } - keys.clear(); -#define BI_GET_NUM_KEYS 128 - int ret; - if (started) { - ret = cls_cxx_map_get_val(hctx, start_key, &keys[start_key]); - if (ret == -ENOENT) { - ret = cls_cxx_map_get_vals(hctx, start_key, string(), BI_GET_NUM_KEYS, &keys); - } - started = false; - } else { - ret = cls_cxx_map_get_vals(hctx, start_key, string(), BI_GET_NUM_KEYS, &keys); - } + bufferlist k; + int ret = cls_cxx_map_get_val(hctx, start_key, &k); + if (ret < 0 && ret != -ENOENT) { + return ret; + } + bool found_first = (ret == 0); + if (found_first) { + --max; + } + if (max > 0) { + ret = cls_cxx_map_get_vals(hctx, start_key, string(), max, &keys, pmore); CLS_LOG(20, "%s(): start_key=%s first_instance_idx=%s keys.size()=%d", __func__, escape_str(start_key).c_str(), escape_str(first_instance_idx).c_str(), (int)keys.size()); if (ret < 0) { return ret; } + } + if (found_first) { + keys[start_key].claim(k); + } - map::iterator iter; - for (iter = keys.begin(); iter != keys.end(); ++iter) { - rgw_cls_bi_entry entry; - entry.type = InstanceIdx; - entry.idx = iter->first; - entry.data = iter->second; - - if (!filter.empty() && entry.idx.compare(0, filter.size(), filter) != 0) { - return count; - } + map::iterator iter; + for (iter = keys.begin(); iter != keys.end(); ++iter) { + rgw_cls_bi_entry entry; + entry.type = InstanceIdx; + entry.idx = iter->first; + entry.data = iter->second; - CLS_LOG(20, "%s(): entry.idx=%s", __func__, escape_str(entry.idx).c_str()); + if (!filter.empty() && entry.idx.compare(0, filter.size(), filter) != 0) { + return count; + } - bufferlist::iterator biter = entry.data.begin(); + CLS_LOG(20, "%s(): entry.idx=%s", __func__, escape_str(entry.idx).c_str()); - rgw_bucket_dir_entry e; - try { - ::decode(e, biter); - } catch (buffer::error& err) { - CLS_LOG(0, "ERROR: %s(): failed to decode buffer (size=%d)", __func__, entry.data.length()); - return -EIO; - } + bufferlist::iterator biter = entry.data.begin(); - if (!name.empty() && e.key.name != name) { - return count; - } + rgw_bucket_dir_entry e; + try { + ::decode(e, biter); + } catch (buffer::error& err) { + CLS_LOG(0, "ERROR: %s(): failed to decode buffer (size=%d)", __func__, entry.data.length()); + return -EIO; + } - entries->push_back(entry); - count++; - start_key = entry.idx; + if (!name.empty() && e.key.name != name) { + return count; } - } while (!keys.empty()); + + entries->push_back(entry); + count++; + start_key = entry.idx; + } return count; } static int list_olh_entries(cls_method_context_t hctx, const string& name, const string& marker, uint32_t max, - list *entries) + list *entries, bool *pmore) { cls_rgw_obj_key key(name); string first_instance_idx; @@ -2419,60 +2411,59 @@ static int list_olh_entries(cls_method_context_t hctx, const string& name, const } int count = 0; map keys; - bool started = true; - do { - if (count >= (int)max) { - return count; - } - keys.clear(); -#define BI_GET_NUM_KEYS 128 - int ret; - if (started) { - ret = cls_cxx_map_get_val(hctx, start_key, &keys[start_key]); - if (ret == -ENOENT) { - ret = cls_cxx_map_get_vals(hctx, start_key, string(), BI_GET_NUM_KEYS, &keys); - } - started = false; - } else { - ret = cls_cxx_map_get_vals(hctx, start_key, string(), BI_GET_NUM_KEYS, &keys); - } + int ret; + bufferlist k; + ret = cls_cxx_map_get_val(hctx, start_key, &k); + if (ret < 0 && ret != -ENOENT) { + return ret; + } + bool found_first = (ret == 0); + if (found_first) { + --max; + } + if (max > 0) { + ret = cls_cxx_map_get_vals(hctx, start_key, string(), max, &keys, pmore); CLS_LOG(20, "%s(): start_key=%s first_instance_idx=%s keys.size()=%d", __func__, escape_str(start_key).c_str(), escape_str(first_instance_idx).c_str(), (int)keys.size()); if (ret < 0) { return ret; } + } - map::iterator iter; - for (iter = keys.begin(); iter != keys.end(); ++iter) { - rgw_cls_bi_entry entry; - entry.type = OLHIdx; - entry.idx = iter->first; - entry.data = iter->second; + if (found_first) { + keys[start_key].claim(k); + } - if (!filter.empty() && entry.idx.compare(0, filter.size(), filter) != 0) { - return count; - } + map::iterator iter; + for (iter = keys.begin(); iter != keys.end(); ++iter) { + rgw_cls_bi_entry entry; + entry.type = OLHIdx; + entry.idx = iter->first; + entry.data = iter->second; - CLS_LOG(20, "%s(): entry.idx=%s", __func__, escape_str(entry.idx).c_str()); + if (!filter.empty() && entry.idx.compare(0, filter.size(), filter) != 0) { + return count; + } - bufferlist::iterator biter = entry.data.begin(); + CLS_LOG(20, "%s(): entry.idx=%s", __func__, escape_str(entry.idx).c_str()); - rgw_bucket_olh_entry e; - try { - ::decode(e, biter); - } catch (buffer::error& err) { - CLS_LOG(0, "ERROR: %s(): failed to decode buffer (size=%d)", __func__, entry.data.length()); - return -EIO; - } + bufferlist::iterator biter = entry.data.begin(); - if (!name.empty() && e.key.name != name) { - return count; - } + rgw_bucket_olh_entry e; + try { + ::decode(e, biter); + } catch (buffer::error& err) { + CLS_LOG(0, "ERROR: %s(): failed to decode buffer (size=%d)", __func__, entry.data.length()); + return -EIO; + } - entries->push_back(entry); - count++; - start_key = entry.idx; + if (!name.empty() && e.key.name != name) { + return count; } - } while (!keys.empty()); + + entries->push_back(entry); + count++; + start_key = entry.idx; + } return count; } @@ -2493,9 +2484,10 @@ static int rgw_bi_list_op(cls_method_context_t hctx, bufferlist *in, bufferlist string filter = op.name; #define MAX_BI_LIST_ENTRIES 1000 - int32_t max = (op.max < MAX_BI_LIST_ENTRIES ? op.max : MAX_BI_LIST_ENTRIES) + 1; /* one extra entry for identifying truncation */ + int32_t max = (op.max < MAX_BI_LIST_ENTRIES ? op.max : MAX_BI_LIST_ENTRIES); string start_key = op.marker; - int ret = list_plain_entries(hctx, op.name, op.marker, max, &op_ret.entries); + bool more; + int ret = list_plain_entries(hctx, op.name, op.marker, max, &op_ret.entries, &more); if (ret < 0) { CLS_LOG(0, "ERROR: %s(): list_plain_entries retured ret=%d", __func__, ret); return ret; @@ -2504,23 +2496,27 @@ static int rgw_bi_list_op(cls_method_context_t hctx, bufferlist *in, bufferlist CLS_LOG(20, "found %d plain entries", count); - ret = list_instance_entries(hctx, op.name, op.marker, max - count, &op_ret.entries); - if (ret < 0) { - CLS_LOG(0, "ERROR: %s(): list_instance_entries retured ret=%d", __func__, ret); - return ret; + if (!more) { + ret = list_instance_entries(hctx, op.name, op.marker, max - count, &op_ret.entries, &more); + if (ret < 0) { + CLS_LOG(0, "ERROR: %s(): list_instance_entries retured ret=%d", __func__, ret); + return ret; + } + + count += ret; } - count += ret; + if (!more) { + ret = list_olh_entries(hctx, op.name, op.marker, max - count, &op_ret.entries, &more); + if (ret < 0) { + CLS_LOG(0, "ERROR: %s(): list_olh_entries retured ret=%d", __func__, ret); + return ret; + } - ret = list_olh_entries(hctx, op.name, op.marker, max - count, &op_ret.entries); - if (ret < 0) { - CLS_LOG(0, "ERROR: %s(): list_olh_entries retured ret=%d", __func__, ret); - return ret; + count += ret; } - count += ret; - - op_ret.is_truncated = (count >= max); + op_ret.is_truncated = (count >= max) || more; while (count >= max) { op_ret.entries.pop_back(); count--; @@ -2582,45 +2578,40 @@ static int bi_log_iterate_entries(cls_method_context_t hctx, const string& marke string filter; - do { -#define BI_NUM_KEYS 128 - int ret = cls_cxx_map_get_vals(hctx, start_key, filter, BI_NUM_KEYS, &keys); - if (ret < 0) - return ret; + int ret = cls_cxx_map_get_vals(hctx, start_key, filter, max_entries, &keys, truncated); + if (ret < 0) + return ret; - map::iterator iter = keys.begin(); - if (iter == keys.end()) - break; + map::iterator iter = keys.begin(); + if (iter == keys.end()) + return 0; - for (; iter != keys.end(); ++iter) { - const string& key = iter->first; - rgw_bi_log_entry e; + uint32_t num_keys = keys.size(); - CLS_LOG(0, "bi_log_iterate_entries key=%s bl.length=%d\n", key.c_str(), (int)iter->second.length()); + for (; iter != keys.end(); ++iter,++i) { + const string& key = iter->first; + rgw_bi_log_entry e; - if (key.compare(end_key) > 0) - return 0; + CLS_LOG(0, "bi_log_iterate_entries key=%s bl.length=%d\n", key.c_str(), (int)iter->second.length()); - ret = bi_log_record_decode(iter->second, e); - if (ret < 0) - return ret; + if (key.compare(end_key) > 0) { + key_iter = key; + return 0; + } - if (max_entries && (i >= max_entries)) { - if (truncated) - *truncated = true; - key_iter = key; - return 0; - } + ret = bi_log_record_decode(iter->second, e); + if (ret < 0) + return ret; - ret = cb(hctx, key, e, param); - if (ret < 0) - return ret; - i++; + ret = cb(hctx, key, e, param); + if (ret < 0) + return ret; + if (i == num_keys - 1) { + key_iter = key; } - --iter; - start_key = iter->first; - } while (true); + } + return 0; } @@ -2862,58 +2853,53 @@ static int usage_iterate_range(cls_method_context_t hctx, uint64_t start, uint64 start_key = key_iter; } - do { - CLS_LOG(20, "usage_iterate_range start_key=%s", start_key.c_str()); - int ret = cls_cxx_map_get_vals(hctx, start_key, filter_prefix, NUM_KEYS, &keys); - if (ret < 0) - return ret; + CLS_LOG(20, "usage_iterate_range start_key=%s", start_key.c_str()); + int ret = cls_cxx_map_get_vals(hctx, start_key, filter_prefix, max_entries, &keys, truncated); + if (ret < 0) + return ret; - map::iterator iter = keys.begin(); - if (iter == keys.end()) - break; + map::iterator iter = keys.begin(); + if (iter == keys.end()) + return 0; - for (; iter != keys.end(); ++iter) { - const string& key = iter->first; - rgw_usage_log_entry e; + uint32_t num_keys = keys.size(); - if (!by_user && key.compare(end_key) >= 0) { - CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str()); - return 0; - } + for (; iter != keys.end(); ++iter,++i) { + const string& key = iter->first; + rgw_usage_log_entry e; - if (by_user && key.compare(0, user_key.size(), user_key) != 0) { - CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str()); - return 0; - } + if (!by_user && key.compare(end_key) >= 0) { + CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str()); + return 0; + } - ret = usage_record_decode(iter->second, e); - if (ret < 0) - return ret; + if (by_user && key.compare(0, user_key.size(), user_key) != 0) { + CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str()); + return 0; + } - if (e.epoch < start) - continue; + ret = usage_record_decode(iter->second, e); + if (ret < 0) + return ret; - /* keys are sorted by epoch, so once we're past end we're done */ - if (e.epoch >= end) - return 0; + if (e.epoch < start) + continue; - ret = cb(hctx, key, e, param); - if (ret < 0) - return ret; + /* keys are sorted by epoch, so once we're past end we're done */ + if (e.epoch >= end) + return 0; + ret = cb(hctx, key, e, param); + if (ret < 0) + return ret; - i++; - if (max_entries && (i > max_entries)) { - CLS_LOG(20, "usage_iterate_range reached max_entries (%d), done", max_entries); - *truncated = true; - key_iter = key; - return 0; - } + + if (i == num_keys - 1) { + key_iter = key; + return 0; } - --iter; - start_key = iter->first; - } while (true); + } return 0; } @@ -2999,7 +2985,9 @@ int rgw_user_usage_log_trim(cls_method_context_t hctx, bufferlist *in, bufferlis } string iter; - ret = usage_iterate_range(hctx, op.start_epoch, op.end_epoch, op.user, iter, 0, NULL, usage_log_trim_cb, NULL); + bool more; +#define MAX_USAGE_TRIM_ENTRIES 128 + ret = usage_iterate_range(hctx, op.start_epoch, op.end_epoch, op.user, iter, MAX_USAGE_TRIM_ENTRIES, &more, usage_log_trim_cb, NULL); if (ret < 0) return ret; @@ -3197,50 +3185,42 @@ static int gc_iterate_entries(cls_method_context_t hctx, const string& marker, b string filter; - do { -#define GC_NUM_KEYS 32 - int ret = cls_cxx_map_get_vals(hctx, start_key, filter, GC_NUM_KEYS, &keys); - if (ret < 0) - return ret; + int ret = cls_cxx_map_get_vals(hctx, start_key, filter, max_entries, &keys, truncated); + if (ret < 0) + return ret; - map::iterator iter = keys.begin(); - if (iter == keys.end()) - break; + map::iterator iter = keys.begin(); + if (iter == keys.end()) + return 0; - for (; iter != keys.end(); ++iter) { - const string& key = iter->first; - cls_rgw_gc_obj_info e; + uint32_t num_keys = keys.size(); - CLS_LOG(10, "gc_iterate_entries key=%s\n", key.c_str()); + for (; iter != keys.end(); ++iter, ++i) { + const string& key = iter->first; + cls_rgw_gc_obj_info e; - if (!end_key.empty() && key.compare(end_key) >= 0) - return 0; + CLS_LOG(10, "gc_iterate_entries key=%s\n", key.c_str()); - if (!key_in_index(key, GC_OBJ_TIME_INDEX)) - return 0; + if (!end_key.empty() && key.compare(end_key) >= 0) + return 0; - ret = gc_record_decode(iter->second, e); - if (ret < 0) - return ret; + if (!key_in_index(key, GC_OBJ_TIME_INDEX)) + return 0; - if (max_entries && (i >= max_entries)) { - if (truncated) - *truncated = true; - --iter; - key_iter = iter->first; - return 0; - } + ret = gc_record_decode(iter->second, e); + if (ret < 0) + return ret; - ret = cb(hctx, key, e, param); - if (ret < 0) - return ret; - i++; + ret = cb(hctx, key, e, param); + if (ret < 0) + return ret; + if (i == num_keys - 1) { + key_iter = key; } - --iter; - start_key = iter->first; - } while (true); + } + return 0; } @@ -3274,7 +3254,8 @@ static int rgw_cls_gc_list(cls_method_context_t hctx, bufferlist *in, bufferlist } cls_rgw_gc_list_ret op_ret; - int ret = gc_list_entries(hctx, op.marker, op.max, op.expired_only, +#define GC_LIST_ENTRIES_DEFAULT 128 + int ret = gc_list_entries(hctx, op.marker, (op.max ? op.max : GC_LIST_ENTRIES_DEFAULT), op.expired_only, op_ret.entries, &op_ret.truncated, op_ret.next_marker); if (ret < 0) return ret; @@ -3384,7 +3365,8 @@ static int rgw_cls_lc_get_next_entry(cls_method_context_t hctx, bufferlist *in, map vals; string filter_prefix; - int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, 1, &vals); + bool more; + int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, 1, &vals, &more); if (ret < 0) return ret; map::iterator it; @@ -3419,7 +3401,7 @@ static int rgw_cls_lc_list_entries(cls_method_context_t hctx, bufferlist *in, bu bufferlist::iterator iter; map vals; string filter_prefix; - int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, op.max_entries, &vals); + int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, op.max_entries, &vals, &op_ret.is_truncated); if (ret < 0) return ret; map::iterator it; @@ -3524,8 +3506,8 @@ static int rgw_reshard_list(cls_method_context_t hctx, bufferlist *in, bufferlis string filter_prefix; #define MAX_RESHARD_LIST_ENTRIES 1000 /* one extra entry for identifying truncation */ - int32_t max = (op.max < MAX_RESHARD_LIST_ENTRIES ? op.max : MAX_RESHARD_LIST_ENTRIES) + 1; - int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, max, &vals); + int32_t max = (op.max && (op.max < MAX_RESHARD_LIST_ENTRIES) ? op.max : MAX_RESHARD_LIST_ENTRIES); + int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, max, &vals, &op_ret.is_truncated); if (ret < 0) return ret; map::iterator it; @@ -3541,7 +3523,6 @@ static int rgw_reshard_list(cls_method_context_t hctx, bufferlist *in, bufferlis } op_ret.entries.push_back(entry); } - op_ret.is_truncated = op.max && (vals.size() > op.max); ::encode(op_ret, *out); return 0; } diff --git a/src/cls/rgw/cls_rgw_ops.h b/src/cls/rgw/cls_rgw_ops.h index 538443adf40..07ebeea12fd 100644 --- a/src/cls/rgw/cls_rgw_ops.h +++ b/src/cls/rgw/cls_rgw_ops.h @@ -1123,18 +1123,23 @@ WRITE_CLASS_ENCODER(cls_rgw_lc_list_entries_op) struct cls_rgw_lc_list_entries_ret { map entries; + bool is_truncated{false}; cls_rgw_lc_list_entries_ret() {} void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); + ENCODE_START(2, 1, bl); ::encode(entries, bl); + ::encode(is_truncated, bl); ENCODE_FINISH(bl); } void decode(bufferlist::iterator& bl) { - DECODE_START(1, bl); + DECODE_START(2, bl); ::decode(entries, bl); + if (struct_v >= 2) { + ::decode(is_truncated, bl); + } DECODE_FINISH(bl); } diff --git a/src/cls/statelog/cls_statelog.cc b/src/cls/statelog/cls_statelog.cc index f404fa77b60..b2a13764d1e 100644 --- a/src/cls/statelog/cls_statelog.cc +++ b/src/cls/statelog/cls_statelog.cc @@ -168,21 +168,20 @@ static int cls_statelog_list(cls_method_context_t hctx, bufferlist *in, bufferli if (!max_entries || max_entries > MAX_ENTRIES) max_entries = MAX_ENTRIES; - int rc = cls_cxx_map_get_vals(hctx, from_index, match_prefix, max_entries + 1, &keys); + cls_statelog_list_ret ret; + + int rc = cls_cxx_map_get_vals(hctx, from_index, match_prefix, max_entries, &keys, &ret.truncated); if (rc < 0) return rc; CLS_LOG(20, "from_index=%s match_prefix=%s", from_index.c_str(), match_prefix.c_str()); - cls_statelog_list_ret ret; list& entries = ret.entries; map::iterator iter = keys.begin(); - bool done = false; string marker; - size_t i; - for (i = 0; i < max_entries && iter != keys.end(); ++i, ++iter) { + for (; iter != keys.end(); ++iter) { const string& index = iter->first; marker = index; @@ -197,12 +196,9 @@ static int cls_statelog_list(cls_method_context_t hctx, bufferlist *in, bufferli } } - if (iter == keys.end()) - done = true; - else + if (ret.truncated) { ret.marker = marker; - - ret.truncated = !done; + } ::encode(ret, *out); diff --git a/src/cls/timeindex/cls_timeindex.cc b/src/cls/timeindex/cls_timeindex.cc index 202b653567e..bed96e5cd74 100644 --- a/src/cls/timeindex/cls_timeindex.cc +++ b/src/cls/timeindex/cls_timeindex.cc @@ -120,21 +120,20 @@ static int cls_timeindex_list(cls_method_context_t hctx, max_entries = MAX_LIST_ENTRIES; } + cls_timeindex_list_ret ret; + int rc = cls_cxx_map_get_vals(hctx, from_index, TIMEINDEX_PREFIX, - max_entries + 1, &keys); + max_entries, &keys, &ret.truncated); if (rc < 0) { return rc; } - cls_timeindex_list_ret ret; - list& entries = ret.entries; map::iterator iter = keys.begin(); - bool done = false; string marker; - for (size_t i = 0; i < max_entries && iter != keys.end(); ++i, ++iter) { + for (; iter != keys.end(); ++iter) { const string& index = iter->first; bufferlist& bl = iter->second; @@ -142,7 +141,7 @@ static int cls_timeindex_list(cls_method_context_t hctx, if (use_time_boundary && index.compare(0, to_index.size(), to_index) >= 0) { CLS_LOG(20, "DEBUG: cls_timeindex_list: finishing on to_index=%s", to_index.c_str()); - done = true; + ret.truncated = false; break; } @@ -159,12 +158,7 @@ static int cls_timeindex_list(cls_method_context_t hctx, } } - if (iter == keys.end()) { - done = true; - } - ret.marker = marker; - ret.truncated = !done; ::encode(ret, *out); @@ -203,8 +197,10 @@ static int cls_timeindex_trim(cls_method_context_t hctx, to_index = op.to_marker; } + bool more; + int rc = cls_cxx_map_get_vals(hctx, from_index, TIMEINDEX_PREFIX, - MAX_TRIM_ENTRIES, &keys); + MAX_TRIM_ENTRIES, &keys, &more); if (rc < 0) { return rc; } @@ -212,7 +208,7 @@ static int cls_timeindex_trim(cls_method_context_t hctx, map::iterator iter = keys.begin(); bool removed = false; - for (size_t i = 0; i < MAX_TRIM_ENTRIES && iter != keys.end(); ++i, ++iter) { + for (; iter != keys.end(); ++iter) { const string& index = iter->first; CLS_LOG(20, "index=%s to_index=%s", index.c_str(), to_index.c_str()); diff --git a/src/cls/user/cls_user.cc b/src/cls/user/cls_user.cc index c616d1ae656..7912578aa36 100644 --- a/src/cls/user/cls_user.cc +++ b/src/cls/user/cls_user.cc @@ -292,8 +292,9 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff max_entries = MAX_ENTRIES; string match_prefix; + cls_user_list_buckets_ret ret; - int rc = cls_cxx_map_get_vals(hctx, from_index, match_prefix, max_entries + 1, &keys); + int rc = cls_cxx_map_get_vals(hctx, from_index, match_prefix, max_entries, &keys, &ret.truncated); if (rc < 0) return rc; @@ -301,21 +302,20 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff from_index.c_str(), to_index.c_str(), match_prefix.c_str()); - cls_user_list_buckets_ret ret; list& entries = ret.entries; map::iterator iter = keys.begin(); - bool done = false; string marker; - size_t i; - for (i = 0; i < max_entries && iter != keys.end(); ++i, ++iter) { + for (; iter != keys.end(); ++iter) { const string& index = iter->first; marker = index; - if (to_index_valid && to_index.compare(index) <= 0) + if (to_index_valid && to_index.compare(index) <= 0) { + ret.truncated = false; break; + } bufferlist& bl = iter->second; bufferlist::iterator biter = bl.begin(); @@ -328,12 +328,9 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff } } - if (iter == keys.end()) - done = true; - else + if (ret.truncated) { ret.marker = marker; - - ret.truncated = !done; + } ::encode(ret, *out);