radosgw-admin: gc list --include-all

A new option to the radosgw-admin gc list command that dumps *all* gc
entries, and not just the expired ones. This is useful for debugging.

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>

diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc
index 2f5711ecb32d574c94d7875a73b1f46aa1356f1e..b02668f35558a8022fa905034ea615f0311f2d01 100644 (file)
--- a/src/cls/rgw/cls_rgw.cc
+++ b/src/cls/rgw/cls_rgw.cc
@@ -1427,7 +1427,7 @@ static int rgw_cls_gc_defer_entry(cls_method_context_t hctx, bufferlist *in, buf
   return gc_defer_entry(hctx, op.tag, op.expiration_secs);
 }
 
-static int gc_iterate_entries(cls_method_context_t hctx, const string& marker,
+static int gc_iterate_entries(cls_method_context_t hctx, const string& marker, bool expired_only,
                               string& key_iter, uint32_t max_entries, bool *truncated,
                               int (*cb)(cls_method_context_t, const string&, cls_rgw_gc_obj_info&, void *),
                               void *param)
@@ -1449,12 +1449,14 @@ static int gc_iterate_entries(cls_method_context_t hctx, const string& marker,
     start_key = key_iter;
   }
 
-  utime_t now = ceph_clock_now(g_ceph_context);
-  string now_str;
-  get_time_key(now, &now_str);
-  prepend_index_prefix(now_str, GC_OBJ_TIME_INDEX, &end_key);
+  if (expired_only) {
+    utime_t now = ceph_clock_now(g_ceph_context);
+    string now_str;
+    get_time_key(now, &now_str);
+    prepend_index_prefix(now_str, GC_OBJ_TIME_INDEX, &end_key);
 
-  CLS_LOG(0, "gc_iterate_entries end_key=%s\n", end_key.c_str());
+    CLS_LOG(0, "gc_iterate_entries end_key=%s\n", end_key.c_str());
+  }
 
   string filter;
 
@@ -1475,7 +1477,7 @@ static int gc_iterate_entries(cls_method_context_t hctx, const string& marker,
 
       CLS_LOG(10, "gc_iterate_entries key=%s\n", key.c_str());
 
-      if (key.compare(end_key) >= 0)
+      if (!end_key.empty() && key.compare(end_key) >= 0)
         return 0;
 
       if (!key_in_index(key, GC_OBJ_TIME_INDEX))
@@ -1512,10 +1514,11 @@ static int gc_list_cb(cls_method_context_t hctx, const string& key, cls_rgw_gc_o
 }
 
 static int gc_list_entries(cls_method_context_t hctx, const string& marker,
-                          uint32_t max, list<cls_rgw_gc_obj_info>& entries, bool *truncated)
+                          uint32_t max, bool expired_only,
+                           list<cls_rgw_gc_obj_info>& entries, bool *truncated)
 {
   string key_iter;
-  int ret = gc_iterate_entries(hctx, marker,
+  int ret = gc_iterate_entries(hctx, marker, expired_only,
                               key_iter, max, truncated,
                               gc_list_cb, &entries);
   return ret;
@@ -1534,7 +1537,7 @@ static int rgw_cls_gc_list(cls_method_context_t hctx, bufferlist *in, bufferlist
   }
 
   cls_rgw_gc_list_ret op_ret;
-  int ret = gc_list_entries(hctx, op.marker, op.max, op_ret.entries, &op_ret.truncated);
+  int ret = gc_list_entries(hctx, op.marker, op.max, op.expired_only, op_ret.entries, &op_ret.truncated);
   if (ret < 0)
     return ret;
 

diff --git a/src/cls/rgw/cls_rgw_client.cc b/src/cls/rgw/cls_rgw_client.cc
index 2851f2bd70261a49a1a3f68c862b80c25f16d513..a26d0afdf4c6ad316e76c7d92f94b417eb1e22e2 100644 (file)
--- a/src/cls/rgw/cls_rgw_client.cc
+++ b/src/cls/rgw/cls_rgw_client.cc
@@ -324,13 +324,14 @@ void cls_rgw_gc_defer_entry(ObjectWriteOperation& op, uint32_t expiration_secs,
   op.exec("rgw", "gc_defer_entry", in);
 }
 
-int cls_rgw_gc_list(IoCtx& io_ctx, string& oid, string& marker, uint32_t max,
+int cls_rgw_gc_list(IoCtx& io_ctx, string& oid, string& marker, uint32_t max, bool expired_only,
                     list<cls_rgw_gc_obj_info>& entries, bool *truncated)
 {
   bufferlist in, out;
   cls_rgw_gc_list_op call;
   call.marker = marker;
   call.max = max;
+  call.expired_only = expired_only;
   ::encode(call, in);
   int r = io_ctx.exec(oid, "rgw", "gc_list", in, out);
   if (r < 0)

diff --git a/src/cls/rgw/cls_rgw_client.h b/src/cls/rgw/cls_rgw_client.h
index 39bb3c9fc4a9eb58e1d5eeb8e0e2de0e912d38a5..c6b5b757fa843d0e587cb2a43ecb94951ceb8f6e 100644 (file)
--- a/src/cls/rgw/cls_rgw_client.h
+++ b/src/cls/rgw/cls_rgw_client.h
@@ -61,7 +61,7 @@ void cls_rgw_usage_log_add(librados::ObjectWriteOperation& op, rgw_usage_log_inf
 void cls_rgw_gc_set_entry(librados::ObjectWriteOperation& op, uint32_t expiration_secs, cls_rgw_gc_obj_info& info);
 void cls_rgw_gc_defer_entry(librados::ObjectWriteOperation& op, uint32_t expiration_secs, const string& tag);
 
-int cls_rgw_gc_list(librados::IoCtx& io_ctx, string& oid, string& marker, uint32_t max,
+int cls_rgw_gc_list(librados::IoCtx& io_ctx, string& oid, string& marker, uint32_t max, bool expired_only,
                     list<cls_rgw_gc_obj_info>& entries, bool *truncated);
 
 void cls_rgw_gc_remove(librados::ObjectWriteOperation& op, const list<string>& tags);

diff --git a/src/cls/rgw/cls_rgw_ops.cc b/src/cls/rgw/cls_rgw_ops.cc
index 2ffc53c92bec93718c46c971f095234d2ed46eca..01c0666d9ebe9bf1069732b963341494b9c42e79 100644 (file)
--- a/src/cls/rgw/cls_rgw_ops.cc
+++ b/src/cls/rgw/cls_rgw_ops.cc
@@ -49,6 +49,7 @@ void cls_rgw_gc_list_op::dump(Formatter *f) const
 {
   f->dump_string("marker", marker);
   f->dump_unsigned("max", max);
+  f->dump_bool("expired_only", expired_only);
 }
 
 void cls_rgw_gc_list_op::generate_test_instances(list<cls_rgw_gc_list_op*>& ls)

diff --git a/src/cls/rgw/cls_rgw_ops.h b/src/cls/rgw/cls_rgw_ops.h
index 8f0ecfbdd26cc56750511634d09f8765bf1071f5..e5db06010dc240c0929f46f22097e9f54e1eae27 100644 (file)
--- a/src/cls/rgw/cls_rgw_ops.h
+++ b/src/cls/rgw/cls_rgw_ops.h
@@ -345,20 +345,25 @@ WRITE_CLASS_ENCODER(cls_rgw_gc_defer_entry_op)
 struct cls_rgw_gc_list_op {
   string marker;
   uint32_t max;
+  bool expired_only;
 
-  cls_rgw_gc_list_op() : max(0) {}
+  cls_rgw_gc_list_op() : max(0), expired_only(true) {}
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(1, 1, bl);
+    ENCODE_START(2, 1, bl);
     ::encode(marker, bl);
     ::encode(max, bl);
+    ::encode(expired_only, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::iterator& bl) {
-    DECODE_START(1, bl);
+    DECODE_START(2, bl);
     ::decode(marker, bl);
     ::decode(max, bl);
+    if (struct_v >= 2) {
+      ::decode(expired_only, bl);
+    }
     DECODE_FINISH(bl);
   }
 

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 26da02e6779cba54f43634b153b9cc73432a6e0a..ac192aaa6c73ec1166f8be35c0ad56ec65cd9098 100644 (file)
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -87,7 +87,8 @@ void _usage()
   cerr << "  usage trim                 trim usage (by user, date range)\n";
   cerr << "  temp remove                remove temporary objects that were created up to\n";
   cerr << "                             specified date (and optional time)\n";
-  cerr << "  gc list                    dump expired garbage collection objects\n";
+  cerr << "  gc list                    dump expired garbage collection objects (specify\n";
+  cerr << "                             --include-all to list all entries, including unexpired)\n";
   cerr << "  gc process                 manually process garbage\n";
   cerr << "  metadata get               get metadata info\n";
   cerr << "  metadata put               put metadata info\n";
@@ -812,6 +813,7 @@ int main(int argc, char **argv)
   int64_t max_size = -1;
   bool have_max_objects = false;
   bool have_max_size = false;
+  int include_all = false;
 
   int sync_stats = false;
 
@@ -952,6 +954,8 @@ int main(int argc, char **argv)
      // do nothing
     } else if (ceph_argparse_binary_flag(args, i, &sync_stats, NULL, "--sync-stats", (char*)NULL)) {
      // do nothing
+    } else if (ceph_argparse_binary_flag(args, i, &include_all, NULL, "--include-all", (char*)NULL)) {
+     // do nothing
     } else if (ceph_argparse_witharg(args, i, &val, "--caps", (char*)NULL)) {
       caps = val;
     } else if (ceph_argparse_witharg(args, i, &val, "-i", "--infile", (char*)NULL)) {
@@ -1891,7 +1895,7 @@ next:
 
     do {
       list<cls_rgw_gc_obj_info> result;
-      int ret = store->list_gc_objs(&index, marker, 1000, result, &truncated);
+      int ret = store->list_gc_objs(&index, marker, 1000, !include_all, result, &truncated);
       if (ret < 0) {
        cerr << "ERROR: failed to list objs: " << cpp_strerror(-ret) << std::endl;
        return 1;

diff --git a/src/rgw/rgw_gc.cc b/src/rgw/rgw_gc.cc
index 1b2b8ca9afb3f75d1102533030e9562bf5202776..dab07f341abc9d2055f3a95ea2c2b7d4071d643f 100644 (file)
--- a/src/rgw/rgw_gc.cc
+++ b/src/rgw/rgw_gc.cc
@@ -90,13 +90,13 @@ int RGWGC::remove(int index, const std::list<string>& tags)
   return store->gc_operate(obj_names[index], &op);
 }
 
-int RGWGC::list(int *index, string& marker, uint32_t max, std::list<cls_rgw_gc_obj_info>& result, bool *truncated)
+int RGWGC::list(int *index, string& marker, uint32_t max, bool expired_only, std::list<cls_rgw_gc_obj_info>& result, bool *truncated)
 {
   result.clear();
 
   for (; *index < cct->_conf->rgw_gc_max_objs && result.size() < max; (*index)++, marker.clear()) {
     std::list<cls_rgw_gc_obj_info> entries;
-    int ret = cls_rgw_gc_list(store->gc_pool_ctx, obj_names[*index], marker, max - result.size(), entries, truncated);
+    int ret = cls_rgw_gc_list(store->gc_pool_ctx, obj_names[*index], marker, max - result.size(), expired_only, entries, truncated);
     if (ret == -ENOENT)
       continue;
     if (ret < 0)
@@ -158,7 +158,7 @@ int RGWGC::process(int index, int max_secs)
   do {
     int max = 100;
     std::list<cls_rgw_gc_obj_info> entries;
-    ret = cls_rgw_gc_list(store->gc_pool_ctx, obj_names[index], marker, max, entries, &truncated);
+    ret = cls_rgw_gc_list(store->gc_pool_ctx, obj_names[index], marker, max, true, entries, &truncated);
     if (ret == -ENOENT) {
       ret = 0;
       goto done;

diff --git a/src/rgw/rgw_gc.h b/src/rgw/rgw_gc.h
index b19afc55dd58c2face7deedcaf1351895773411d..afaead49258f8f75994eb001438fe06f042e9fc2 100644 (file)
--- a/src/rgw/rgw_gc.h
+++ b/src/rgw/rgw_gc.h
@@ -49,7 +49,7 @@ public:
   void initialize(CephContext *_cct, RGWRados *_store);
   void finalize();
 
-  int list(int *index, string& marker, uint32_t max, std::list<cls_rgw_gc_obj_info>& result, bool *truncated);
+  int list(int *index, string& marker, uint32_t max, bool expired_only, std::list<cls_rgw_gc_obj_info>& result, bool *truncated);
   void list_init(int *index) { *index = 0; }
   int process(int index, int process_max_secs);
   int process();

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index cce463b324b25aea3dd5d765e81807ce1a489349..f6c8a4fa8fe83528fb4a14a9172c3ca0ed1a56b8 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -5339,9 +5339,9 @@ int RGWRados::gc_operate(string& oid, librados::ObjectReadOperation *op, bufferl
   return gc_pool_ctx.operate(oid, op, pbl);
 }
 
-int RGWRados::list_gc_objs(int *index, string& marker, uint32_t max, std::list<cls_rgw_gc_obj_info>& result, bool *truncated)
+int RGWRados::list_gc_objs(int *index, string& marker, uint32_t max, bool expired_only, std::list<cls_rgw_gc_obj_info>& result, bool *truncated)
 {
-  return gc->list(index, marker, max, result, truncated);
+  return gc->list(index, marker, max, expired_only, result, truncated);
 }
 
 int RGWRados::process_gc()

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 64b9ef062ab63cb2b4a03ccd5e5c7ad008977452..d3f9998e7ccb0623a0d54d76363d2011e352a073 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -1431,7 +1431,7 @@ public:
   int gc_aio_operate(string& oid, librados::ObjectWriteOperation *op);
   int gc_operate(string& oid, librados::ObjectReadOperation *op, bufferlist *pbl);
 
-  int list_gc_objs(int *index, string& marker, uint32_t max, std::list<cls_rgw_gc_obj_info>& result, bool *truncated);
+  int list_gc_objs(int *index, string& marker, uint32_t max, bool expired_only, std::list<cls_rgw_gc_obj_info>& result, bool *truncated);
   int process_gc();
   int defer_gc(void *ctx, rgw_obj& obj);
 

diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t
index a7921e2b93483e8a1518f6a1fac9612e0df7ccbe..ca3e7471b10abc1e930170d7c0fe7847cd561447 100644 (file)
--- a/src/test/cli/radosgw-admin/help.t
+++ b/src/test/cli/radosgw-admin/help.t
@@ -48,7 +48,8 @@
     usage trim                 trim usage (by user, date range)
     temp remove                remove temporary objects that were created up to
                                specified date (and optional time)
-    gc list                    dump expired garbage collection objects
+    gc list                    dump expired garbage collection objects (specify
+                               --include-all to list all entries, including unexpired)
     gc process                 manually process garbage
     metadata get               get metadata info
     metadata put               put metadata info

diff --git a/src/test/cls_rgw/test_cls_rgw.cc b/src/test/cls_rgw/test_cls_rgw.cc
index 7a37deec10512f66293a940a27aa0cdc407a3934..44cb30307245d2117e4adb7cb628d1d08719aa1c 100644 (file)
--- a/src/test/cls_rgw/test_cls_rgw.cc
+++ b/src/test/cls_rgw/test_cls_rgw.cc
@@ -409,14 +409,14 @@ TEST(cls_rgw, gc_set)
   string marker;
 
   /* list chains, verify truncated */
-  ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 8, entries, &truncated));
+  ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 8, true, entries, &truncated));
   ASSERT_EQ(8, (int)entries.size());
   ASSERT_EQ(1, truncated);
 
   entries.clear();
 
   /* list all chains, verify not truncated */
-  ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 10, entries, &truncated));
+  ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 10, true, entries, &truncated));
   ASSERT_EQ(10, (int)entries.size());
   ASSERT_EQ(0, truncated);
  
@@ -483,7 +483,7 @@ TEST(cls_rgw, gc_defer)
   string marker;
 
   /* list chains, verify num entries as expected */
-  ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, entries, &truncated));
+  ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated));
   ASSERT_EQ(1, (int)entries.size());
   ASSERT_EQ(0, truncated);
 
@@ -496,7 +496,7 @@ TEST(cls_rgw, gc_defer)
   entries.clear();
 
   /* verify list doesn't show deferred entry (this may fail if cluster is thrashing) */
-  ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, entries, &truncated));
+  ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated));
   ASSERT_EQ(0, (int)entries.size());
   ASSERT_EQ(0, truncated);
 
@@ -504,7 +504,7 @@ TEST(cls_rgw, gc_defer)
   sleep(5);
 
   /* verify list shows deferred entry */
-  ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, entries, &truncated));
+  ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated));
   ASSERT_EQ(1, (int)entries.size());
   ASSERT_EQ(0, truncated);
 
@@ -519,7 +519,7 @@ TEST(cls_rgw, gc_defer)
   entries.clear();
 
   /* verify entry was removed */
-  ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, entries, &truncated));
+  ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated));
   ASSERT_EQ(0, (int)entries.size());
   ASSERT_EQ(0, truncated);