rgw: "bucket check --fix" should delete damaged multipart uploads from bi

As one of the steps in `radosgw-admin bucket check --fix ...` it looks
for bucket index entries for incomplete multipart uploads that do not
have a corresponding ".meta" entry in the same bucket index. It then
intends to delete those entries, however the function that it calls
to perform the bucket index deletions was flawed and did not direct
the removals to the appropriate shard(s), but instead a non-existant
oid.

This commit determines the appropriate shard for each of the entries
to be removed and asynchronously issues a librados call to
omap_rm_keys.

Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>
(cherry picked from commit 0521c2ae830d54c4fee37ef5f1ea77ed298c27a0)

    Conflicts:
        src/rgw/driver/rados/rgw_rados.cc
            - remove bitx debugging

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 5461b6aee0b725fc06171936ac1e6b6493ed5fc0..fed2c28fb170dc712ba6c8d4f22925dd1502ce57 100644 (file)
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -7794,6 +7794,8 @@ next:
     rgw_obj_index_key index_key;
     key.get_index_key(&index_key);
     oid_list.push_back(index_key);
+
+    // note: under rados this removes directly from rados index objects
     ret = bucket->remove_objs_from_index(dpp(), oid_list);
     if (ret < 0) {
       cerr << "ERROR: remove_obj_from_index() returned error: " << cpp_strerror(-ret) << std::endl;

diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index be9eddd05d50ac6a50db7e11f5503665ce8ee666..004ec3e170eedf25f1f4b0787033e790a7135a42 100644 (file)
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -659,9 +659,6 @@ static int check_bad_index_multipart(rgw::sal::RadosStore* const rados_store,
     if (fix_index) {
       ret = store->remove_objs_from_index(dpp, bucket->get_info(),
                                          entries_to_unlink);
-for (const auto& entry : entries_to_unlink) {
-  ldpp_dout(dpp, 0) << "    ERIC: " << entry << dendl;
-}
       if (ret < 0) {
        ldpp_dout(dpp, 0) <<
          "ERROR: remove_objs_from_index failed returning " <<

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 003a889e231769d6bf65e4285a4f8009211f41e3..ac7081403353be2e72b213128418b6466b46d178 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -9454,31 +9454,75 @@ int RGWRados::cls_obj_usage_log_clear(const DoutPrefixProvider *dpp, string& oid
 }
 
 
-int RGWRados::remove_objs_from_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, list<rgw_obj_index_key>& oid_list)
-{
+// note: this removes entries from the rados bucket index objects
+// without going through CLS; this is known to be called from
+// "radosgw-admin unlink" and "radosgw-admin bucket check --fix"
+int RGWRados::remove_objs_from_index(const DoutPrefixProvider *dpp,
+                                    RGWBucketInfo& bucket_info,
+                                    const std::list<rgw_obj_index_key>& entry_key_list)
+{
+  const rgw::bucket_index_layout_generation& latest_layout = bucket_info.layout.current_index;
+  if (latest_layout.layout.type != rgw::BucketIndexType::Normal ||
+      latest_layout.layout.normal.hash_type != rgw::BucketHashType::Mod) {
+    ldpp_dout(dpp, 0) << "ERROR: " << __func__ << " index for bucket=" <<
+      bucket_info.bucket << " is not a normal modulo index" << dendl;
+    return -EINVAL;
+  }
+  const uint32_t num_shards = latest_layout.layout.normal.num_shards;
+
   RGWSI_RADOS::Pool index_pool;
-  string dir_oid;
+  std::map<int, std::string> index_oids;
+  int r = svc.bi_rados->open_bucket_index(dpp, bucket_info, std::nullopt, &index_pool,
+                                         &index_oids, nullptr);
+  if (r < 0) {
+    ldpp_dout(dpp, 0) << "ERROR: " << __func__ <<
+      " open_bucket_index returned " << r << dendl;
+    return r;
+  }
 
-  uint8_t suggest_flag = (svc.zone->get_zone().log_data ? CEPH_RGW_DIR_SUGGEST_LOG_OP : 0);
+  // split up removals by shard
+  std::map<int, std::set<std::string>> sharded_removals;
+  for (const auto& entry_key : entry_key_list) {
+    const rgw_obj_key obj_key(entry_key);
+    const uint32_t shard =
+      RGWSI_BucketIndex_RADOS::bucket_shard_index(obj_key, num_shards);
 
-  int r = svc.bi_rados->open_bucket_index(dpp, bucket_info, &index_pool, &dir_oid);
-  if (r < 0)
-    return r;
+    // entry_key already combines namespace and name, so we first have
+    // to break that apart before we can then combine with instance
+    std::string name;
+    std::string ns; // namespace
+    rgw_obj_key::parse_index_key(entry_key.name, &name, &ns);
+    rgw_obj_key full_key(name, entry_key.instance, ns);
+    std::string combined_key = full_key.get_oid();
 
-  bufferlist updates;
+    sharded_removals[shard].insert(combined_key);
 
-  for (auto iter = oid_list.begin(); iter != oid_list.end(); ++iter) {
-    rgw_bucket_dir_entry entry;
-    entry.key = *iter;
-    ldpp_dout(dpp, 2) << "RGWRados::remove_objs_from_index bucket=" << bucket_info.bucket << " obj=" << entry.key.name << ":" << entry.key.instance << dendl;
-    entry.ver.epoch = (uint64_t)-1; // ULLONG_MAX, needed to that objclass doesn't skip out request
-    updates.append(CEPH_RGW_REMOVE | suggest_flag);
-    encode(entry, updates);
+    ldpp_dout(dpp, 20) << "INFO: " << __func__ <<
+      ": removal from bucket index, bucket=" << bucket_info.bucket <<
+      " key=" << combined_key << " designated for shard " << shard <<
+      dendl;
   }
 
-  bufferlist out;
+  for (const auto& removals : sharded_removals) {
+    const int shard = removals.first;
+    const std::string& oid = index_oids[shard];
+
+    ldpp_dout(dpp, 10) << "INFO: " << __func__ <<
+      ": removal from bucket index, bucket=" << bucket_info.bucket <<
+      ", shard=" << shard << ", oid=" << oid << ", num_keys=" <<
+      removals.second.size() << dendl;
+
+    r = index_pool.ioctx().omap_rm_keys(oid, removals.second);
+    if (r < 0) {
+      ldpp_dout(dpp, 0) << "ERROR: " << __func__ <<
+       ": omap_rm_keys returned ret=" << r <<
+       dendl;
+      return r;
+    }
+  }
 
-  r = index_pool.ioctx().exec(dir_oid, RGW_CLASS, RGW_DIR_SUGGEST_CHANGES, updates, out);
+  ldpp_dout(dpp, 5) <<
+    "EXITING " << __func__ << " and returning " << r << dendl;
 
   return r;
 }

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 082a6a23248bb9980d235c95293d3e77eff3c29d..d13350a12e3252e64ab00896cc08ae1f30384d51 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -64,9 +64,6 @@ struct get_obj_data;
 #define PUT_OBJ_EXCL        0x02
 #define PUT_OBJ_CREATE_EXCL (PUT_OBJ_CREATE | PUT_OBJ_EXCL)
 
-#define RGW_OBJ_NS_MULTIPART "multipart"
-#define RGW_OBJ_NS_SHADOW    "shadow"
-
 static inline void prepend_bucket_marker(const rgw_bucket& bucket, const std::string& orig_oid, std::string& oid)
 {
   if (bucket.marker.empty() || orig_oid.empty()) {
@@ -1493,7 +1490,9 @@ public:
                                         RGWFormatterFlusher& flusher);
 
   int bucket_set_reshard(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const cls_rgw_bucket_instance_entry& entry);
-  int remove_objs_from_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, std::list<rgw_obj_index_key>& oid_list);
+  int remove_objs_from_index(const DoutPrefixProvider *dpp,
+                            RGWBucketInfo& bucket_info,
+                            const std::list<rgw_obj_index_key>& oid_list);
   int move_rados_obj(const DoutPrefixProvider *dpp,
                      librados::IoCtx& src_ioctx,
                     const std::string& src_oid, const std::string& src_locator,

diff --git a/src/rgw/services/svc_bi_rados.h b/src/rgw/services/svc_bi_rados.h
index 85eb3e5b7fedefb49f35d6ddadc60c20a54a7052..337836203bcc016e5c5ffd7b9c43037b9100d625 100644 (file)
--- a/src/rgw/services/svc_bi_rados.h
+++ b/src/rgw/services/svc_bi_rados.h
@@ -22,6 +22,7 @@
 
 #include "svc_bi.h"
 #include "svc_rados.h"
+#include "svc_tier_rados.h"
 
 struct rgw_bucket_dir_header;
 
@@ -32,6 +33,12 @@ class RGWSI_BILog_RADOS;
 #define RGW_SHARDS_PRIME_0 7877
 #define RGW_SHARDS_PRIME_1 65521
 
+/*
+ * Defined Bucket Index Namespaces
+ */
+#define RGW_OBJ_NS_MULTIPART "multipart"
+#define RGW_OBJ_NS_SHADOW    "shadow"
+
 class RGWSI_BucketIndex_RADOS : public RGWSI_BucketIndex
 {
   friend class RGWSI_BILog_RADOS;
@@ -96,6 +103,21 @@ public:
     return rgw_shards_mod(sid2, num_shards);
   }
 
+  static uint32_t bucket_shard_index(const rgw_obj_key& obj_key,
+                                    int num_shards)
+  {
+    std::string sharding_key;
+    if (obj_key.ns == RGW_OBJ_NS_MULTIPART) {
+      RGWMPObj mp;
+      mp.from_meta(obj_key.name);
+      sharding_key = mp.get_key();
+    } else {
+      sharding_key = obj_key.name;
+    }
+
+    return bucket_shard_index(sharding_key, num_shards);
+  }
+
   int init_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info);
   int clean_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info);