rgw: clean up address 0-length listing results...

Some minor clean-ups to the previous commit, including adjust logging
messages, rename variable, convert a #define to a constexpr (and
adjust its scope).

Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>
(cherry picked from commit 4470ca8a3ff22aeb140e0c3be396e5f1bb25aa26)

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 20ede3f2e69ff2192f2d7177eeb513578b76430f..fc66d4b503ce49523c4b15832a60d48b53ed7a95 100644 (file)
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -6255,12 +6255,12 @@ next:
     formatter->open_array_section("objects");
 
     constexpr uint32_t NUM_ENTRIES = 1000;
-    uint16_t attempt = 1;
+    uint16_t expansion_factor = 1;
     while (is_truncated) {
       map<string, rgw_bucket_dir_entry> result;
       int r =
        store->cls_bucket_list_ordered(bucket_info, RGW_NO_SHARD, marker,
-                                      prefix, NUM_ENTRIES, true, attempt,
+                                      prefix, NUM_ENTRIES, true, expansion_factor,
                                       result, &is_truncated, &marker,
                                       bucket_object_check_filter);
       if (r < 0 && r != -ENOENT) {
@@ -6270,9 +6270,10 @@ next:
       }
 
       if (result.size() < NUM_ENTRIES / 8) {
-       ++attempt;
-      } else if (result.size() > NUM_ENTRIES * 7 / 8 && attempt > 1) {
-       --attempt;
+       ++expansion_factor;
+      } else if (result.size() > NUM_ENTRIES * 7 / 8 &&
+                expansion_factor > 1) {
+       --expansion_factor;
       }
 
       map<string, rgw_bucket_dir_entry>::iterator iter;

diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index 5092278c4f0dd4c69f3e4c46d59f3ef94a939271..8145c1f8d78cd0c263d19110d61e7ee11daeefe8 100644 (file)
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -39,6 +39,10 @@
 
 #define BUCKET_TAG_TIMEOUT 30
 
+// default number of entries to list with each bucket listing call
+// (use marker to bridge between calls)
+static constexpr size_t listing_max_entries = 1000;
+
 
 static RGWMetadataHandler *bucket_meta_handler = NULL;
 static RGWMetadataHandler *bucket_instance_meta_handler = NULL;
@@ -1164,13 +1168,14 @@ int RGWBucket::check_object_index(RGWBucketAdminOpState& op_state,
 
   Formatter *formatter = flusher.get_formatter();
   formatter->open_object_section("objects");
-  constexpr uint32_t NUM_ENTRIES = 1000;
-  uint16_t attempt = 1;
+  uint16_t expansion_factor = 1;
   while (is_truncated) {
     map<string, rgw_bucket_dir_entry> result;
 
     int r = store->cls_bucket_list_ordered(bucket_info, RGW_NO_SHARD,
-                                          marker, prefix, NUM_ENTRIES, true, attempt,
+                                          marker, prefix,
+                                          listing_max_entries, true,
+                                          expansion_factor,
                                           result, &is_truncated, &marker,
                                           bucket_object_check_filter);
     if (r == -ENOENT) {
@@ -1179,10 +1184,11 @@ int RGWBucket::check_object_index(RGWBucketAdminOpState& op_state,
       set_err_msg(err_msg, "ERROR: failed operation r=" + cpp_strerror(-r));
     }
 
-    if (result.size() < NUM_ENTRIES / 8) {
-      ++attempt;
-    } else if (result.size() > NUM_ENTRIES * 7 / 8 && attempt > 1) {
-      --attempt;
+    if (result.size() < listing_max_entries / 8) {
+      ++expansion_factor;
+    } else if (result.size() > listing_max_entries * 7 / 8 &&
+              expansion_factor > 1) {
+      --expansion_factor;
     }
 
     dump_bucket_index(result, formatter);

diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h
index b32049639d446804013eff0a54b6d94ac3ef3d90..302dc84f91ea58766cf720ba5b346aa27a661dc2 100644 (file)
--- a/src/rgw/rgw_bucket.h
+++ b/src/rgw/rgw_bucket.h
@@ -20,6 +20,7 @@
 #include "common/ceph_time.h"
 #include "rgw_formats.h"
 
+
 // define as static when RGWBucket implementation completes
 extern void rgw_get_buckets_obj(const rgw_user& user_id, string& buckets_obj_id);
 

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 79ab2b3186b7bdbcc0333a14cd13246ba599c4b1..0cc0bc55254712f29a32b5cc48413256d7bed617 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -2489,9 +2489,11 @@ int RGWRados::Bucket::List::list_objects_ordered(
     for (auto eiter = ent_map.begin(); eiter != ent_map.end(); ++eiter) {
       rgw_bucket_dir_entry& entry = eiter->second;
       rgw_obj_index_key index_key = entry.key;
-
       rgw_obj_key obj(index_key);
 
+      ldout(cct, 20) << "RGWRados::Bucket::List::" << __func__ <<
+       " considering entry " << entry.key << dendl;
+
       /* note that parse_raw_oid() here will not set the correct
        * object's instance, as rgw_obj_index_key encodes that
        * separately. We don't need to set the instance because it's
@@ -2504,12 +2506,12 @@ int RGWRados::Bucket::List::list_objects_ordered(
         continue;
       }
 
-      bool check_ns = (obj.ns == params.ns);
+      bool matched_ns = (obj.ns == params.ns);
       if (!params.list_versions && !entry.is_visible()) {
         continue;
       }
 
-      if (params.enforce_ns && !check_ns) {
+      if (params.enforce_ns && !matched_ns) {
         if (!params.ns.empty()) {
           /* we've iterated past the namespace we're searching -- done now */
           truncated = false;
@@ -9200,19 +9202,24 @@ int RGWRados::cls_bucket_list_ordered(RGWBucketInfo& bucket_info,
                                      const string& prefix,
                                      const uint32_t num_entries,
                                      const bool list_versions,
-                                     const uint16_t attempt,
+                                     const uint16_t expansion_factor,
                                      map<string, rgw_bucket_dir_entry>& m,
                                      bool *is_truncated,
                                      rgw_obj_index_key *last_entry,
                                      bool (*force_check_filter)(const string& name))
 {
+  /* expansion_factor allows the number of entries to read to grow
+   * exponentially; this is used when earlier reads are producing too
+   * few results, perhaps due to filtering or to a series of
+   * namespaced entries */
+
   ldout(cct, 10) << "RGWRados::" << __func__ << ": " << bucket_info.bucket <<
     " start_after=\"" << start_after.name <<
     "[" << start_after.instance <<
     "]\", prefix=\"" << prefix <<
     "\" num_entries=" << num_entries <<
     ", list_versions=" << list_versions <<
-    ", attempt=" << attempt << dendl;
+    ", expansion_factor=" << expansion_factor << dendl;
 
   librados::IoCtx index_ctx;
   // key   - oid (for different shards if there is any)
@@ -9226,14 +9233,14 @@ int RGWRados::cls_bucket_list_ordered(RGWBucketInfo& bucket_info,
 
   const uint32_t shard_count = oids.size();
   uint32_t num_entries_per_shard;
-  if (attempt == 0) {
+  if (expansion_factor == 0) {
     num_entries_per_shard =
       calc_ordered_bucket_list_per_shard(num_entries, shard_count);
-  } else if (attempt <= 11) {
+  } else if (expansion_factor <= 11) {
     // we'll max out the exponential multiplication factor at 1024 (2<<10)
     num_entries_per_shard =
       std::min(num_entries,
-              (uint32_t(1 << (attempt - 1)) *
+              (uint32_t(1 << (expansion_factor - 1)) *
                calc_ordered_bucket_list_per_shard(num_entries, shard_count)));
   } else {
     num_entries_per_shard = num_entries;
@@ -9253,7 +9260,7 @@ int RGWRados::cls_bucket_list_ordered(RGWBucketInfo& bucket_info,
     return r;
   }
 
-  // Create a list of iterators that are used to iterate each shard
+  // create a list of iterators that are used to iterate each shard
   vector<map<string, struct rgw_bucket_dir_entry>::iterator> vcurrents;
   vector<map<string, struct rgw_bucket_dir_entry>::iterator> vends;
   vector<string> vnames;

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 162a9b15adb1743076d7d1b178fe972235b9b5f5..0c7bf5718fba63bbb8e776305c2382e8e6c6b857 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -2211,7 +2211,7 @@ public:
                              const string& prefix,
                              const uint32_t num_entries,
                              const bool list_versions,
-                             const uint16_t attempt, // 0 means ignore
+                             const uint16_t exp_factor, // 0 means ignore
                              map<string, rgw_bucket_dir_entry>& m,
                              bool *is_truncated,
                              rgw_obj_index_key *last_entry,