librgw: is_truncated experiments

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>

diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index f9e15e1f108ecb3487a8b8182174ccab5fdcee8d..3357b7a083bba2a582a8128226f33ff88d299337 100644 (file)
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -102,7 +102,8 @@ int rgw_read_user_buckets(RGWRados * store,
                           const string& end_marker,
                           uint64_t max,
                           bool need_stats,
-                          uint64_t default_amount)
+                         bool *is_truncated,
+                         uint64_t default_amount)
 {
   int ret;
   buckets.clear();
@@ -368,10 +369,12 @@ static void dump_mulipart_index_results(list<rgw_obj_key>& objs_to_unlink,
   f->close_section();
 }
 
-void check_bad_user_bucket_mapping(RGWRados *store, const rgw_user& user_id, bool fix)
+void check_bad_user_bucket_mapping(RGWRados *store, const rgw_user& user_id,
+                                  bool fix)
 {
   RGWUserBuckets user_buckets;
   bool done;
+  bool is_truncated;
   string marker;
 
   CephContext *cct = store->ctx();
@@ -379,10 +382,12 @@ void check_bad_user_bucket_mapping(RGWRados *store, const rgw_user& user_id, boo
   size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk;
 
   do {
-    int ret = rgw_read_user_buckets(store, user_id, user_buckets,
-                                    marker, string(), max_entries, false);
+    int ret = rgw_read_user_buckets(store, user_id, user_buckets, marker,
+                                   string(), max_entries, false,
+                                   &is_truncated);
     if (ret < 0) {
-      ldout(store->ctx(), 0) << "failed to read user buckets: " << cpp_strerror(-ret) << dendl;
+      ldout(store->ctx(), 0) << "failed to read user buckets: "
+                            << cpp_strerror(-ret) << dendl;
       return;
     }
 
@@ -1142,10 +1147,12 @@ int RGWBucketAdminOp::info(RGWRados *store, RGWBucketAdminOpState& op_state,
     RGWUserBuckets buckets;
     string marker;
     bool done;
+    bool is_truncated;
 
     do {
-      ret = rgw_read_user_buckets(store, user_id, buckets,
-                                  marker, string(), max_entries, false);
+      ret = rgw_read_user_buckets(store, op_state.get_user_id(), buckets,
+                                 marker, string(), max_entries, false,
+                                 &is_truncated);
       if (ret < 0)
         return ret;
 

diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h
index 61f5c6218b05bdd4ef0e1126a03e9b35fdf5253a..7a0a5c3db0d301fa97cbbf3eef0327acbfb3a244 100644 (file)
--- a/src/rgw/rgw_bucket.h
+++ b/src/rgw/rgw_bucket.h
@@ -123,6 +123,7 @@ extern int rgw_read_user_buckets(RGWRados *store,
                                  const string& end_marker,
                                  uint64_t max,
                                  bool need_stats,
+                                bool* is_truncated,
                                  uint64_t default_amount = 1000);
 
 extern int rgw_link_bucket(RGWRados *store, const rgw_user& user_id, rgw_bucket& bucket, time_t creation_time, bool update_entrypoint = true);

diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc
index 49cb70fc1821562225b958984ad5fd66a464dd9f..1ba39d92c8d07fe66230ac2766636d8a450dc5e9 100644 (file)
--- a/src/rgw/rgw_file.cc
+++ b/src/rgw/rgw_file.cc
@@ -745,6 +745,7 @@ void dump_buckets(void) {
 
   RGWUserBuckets buckets;
   uint64_t max_buckets = g_ceph_context->_conf->rgw_list_buckets_max_chunk;
+  bool is_truncated;
 
   RGWRados* store = rgwlib.get_store();
 
@@ -752,7 +753,7 @@ void dump_buckets(void) {
   uint64_t ix = 3;
   rgw_user uid("testuser");
   int rc = rgw_read_user_buckets(store, uid, buckets, marker, end_marker,
-                                max_buckets, true);
+                                max_buckets, true, &is_truncated);
   if (rc == 0) {
     bucket_count = 0;
     bucket_objcount = 0;

diff --git a/src/rgw/rgw_file.h b/src/rgw/rgw_file.h
index 4a3b7edc14f2709e9ac0a926f0280d2c3d8302f1..0f5fed0ebc5e582dcc9eb776038cf58aba49a029 100644 (file)
--- a/src/rgw/rgw_file.h
+++ b/src/rgw/rgw_file.h
@@ -869,7 +869,12 @@ public:
     return 0;
   }
 
-  bool eof() { return ix == 0; }
+  bool eof() {
+    std::cout << "READDIR offset: " << *offset
+             << " is_truncated: " << is_truncated
+             << std::endl;
+    return !is_truncated;
+  }
 
 }; /* RGWListBucketsRequest */
 
@@ -1018,7 +1023,13 @@ public:
     send_response();
   }
 
-  bool eof() { return ix == 0; }
+  bool eof() {
+    std::cout << "READDIR offset: " << *offset
+             << " next marker: " << next_marker
+             << " is_truncated: " << is_truncated
+             << std::endl;
+    return !is_truncated;
+  }
 
 }; /* RGWReaddirRequest */
 

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 4b6004b58f79d353aba568eea20c555cfabcb2f9..df884086ec02b72dcd8045a75c96a2b424c5813b 100644 (file)
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -1276,7 +1276,7 @@ void RGWListBuckets::execute()
 
     op_ret = rgw_read_user_buckets(store, s->user->user_id, buckets,
                                   marker, end_marker, read_count,
-                                  should_get_stats(), 0);
+                                  should_get_stats(), &is_truncated);
     if (op_ret < 0) {
       /* hmm.. something wrong here.. the user was authenticated, so it
          should exist */
@@ -1328,13 +1328,14 @@ void RGWStatAccount::execute()
 {
   string marker;
   bool done;
+  bool is_truncated;
   uint64_t max_buckets = s->cct->_conf->rgw_list_buckets_max_chunk;
 
   do {
     RGWUserBuckets buckets;
 
-    op_ret = rgw_read_user_buckets(store, s->user->user_id, buckets,
-                                  marker, string(), max_buckets, false);
+    op_ret = rgw_read_user_buckets(store, s->user->user_id, buckets, marker,
+                                  string(), max_buckets, true, &is_truncated);
     if (op_ret < 0) {
       /* hmm.. something wrong here.. the user was authenticated, so it
          should exist */
@@ -1616,9 +1617,10 @@ int RGWCreateBucket::verify_permission()
   if (s->user->max_buckets) {
     RGWUserBuckets buckets;
     string marker;
+    bool is_truncated;
     op_ret = rgw_read_user_buckets(store, s->user->user_id, buckets,
                                   marker, string(), s->user->max_buckets,
-                                  false);
+                                  false, &is_truncated);
     if (op_ret < 0)
       return op_ret;
 

diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index f2819e320c3fd620edecba3a84c0526af1a7b499..517fa476ec3950b4cfeb50400b91c80a1c989833 100644 (file)
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -319,6 +319,7 @@ protected:
   uint64_t buckets_size;
   uint64_t buckets_size_rounded;
   map<string, bufferlist> attrs;
+  bool is_truncated;
 
 public:
   RGWListBuckets() : sent_data(false) {

diff --git a/src/rgw/rgw_user.cc b/src/rgw/rgw_user.cc
index cd27721f03c56539a3b7652d97889a9a16733bdb..b043cf1d5c79b3d63e270182b1c453f966fca7c8 100644 (file)
--- a/src/rgw/rgw_user.cc
+++ b/src/rgw/rgw_user.cc
@@ -50,13 +50,14 @@ int rgw_user_sync_all_stats(RGWRados *store, const rgw_user& user_id)
   CephContext *cct = store->ctx();
   size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk;
   bool done;
+  bool is_truncated;
   string marker;
   int ret;
 
   do {
     RGWUserBuckets user_buckets;
-    ret = rgw_read_user_buckets(store, user_id, user_buckets,
-                                marker, string(), max_entries, false);
+    ret = rgw_read_user_buckets(store, user_id, user_buckets, marker,
+                               string(), max_entries, false, &is_truncated);
     if (ret < 0) {
       ldout(cct, 0) << "failed to read user buckets: ret=" << ret << dendl;
       return ret;
@@ -406,14 +407,15 @@ int rgw_delete_user(RGWRados *store, RGWUserInfo& info, RGWObjVersionTracker& ob
   vector<rgw_bucket> buckets_vec;
 
   bool done;
+  bool is_truncated;
   int ret;
   CephContext *cct = store->ctx();
   size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk;
 
   do {
     RGWUserBuckets user_buckets;
-    ret = rgw_read_user_buckets(store, info.user_id, user_buckets,
-                                marker, string(), max_buckets, false);
+    ret = rgw_read_user_buckets(store, info.user_id, user_buckets, marker,
+                               string(), max_buckets, false, &is_truncated);
     if (ret < 0)
       return ret;
 
@@ -2023,13 +2025,14 @@ int RGWUser::execute_remove(RGWUserAdminOpState& op_state, std::string *err_msg)
   }
 
   bool done;
+  bool is_truncated;
   string marker;
   CephContext *cct = store->ctx();
   size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk;
   do {
     RGWUserBuckets buckets;
-    ret = rgw_read_user_buckets(store, uid, buckets,
-                                marker, string(), max_buckets, false);
+    ret = rgw_read_user_buckets(store, uid, buckets, marker, string(),
+                               max_buckets, false, &is_truncated);
     if (ret < 0) {
       set_err_msg(err_msg, "unable to read user bucket info");
       return ret;
@@ -2174,12 +2177,13 @@ int RGWUser::execute_modify(RGWUserAdminOpState& op_state, std::string *err_msg)
     }
 
     bool done;
+    bool is_truncated;
     string marker;
     CephContext *cct = store->ctx();
     size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk;
     do {
-      ret = rgw_read_user_buckets(store, user_id, buckets,
-                                  marker, string(), max_buckets, false);
+      ret = rgw_read_user_buckets(store, user_id, buckets, marker, string(),
+                                 max_buckets, false, &is_truncated);
       if (ret < 0) {
         set_err_msg(err_msg, "could not get buckets for uid:  " + user_id.to_str());
         return ret;