From: J. Eric Ivancich Date: Tue, 26 Feb 2019 16:50:14 +0000 (-0500) Subject: rgw: fix unordered bucket listing when object names are adorned X-Git-Tag: v14.2.0~52^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=5309dd96519668d98dca0f8d0bcc5f56bd0fa334;p=ceph.git rgw: fix unordered bucket listing when object names are adorned Namespaces, instances, and multipart metadata adorn object names and are not used in the computation of an object's bucket index shard. The logic for unordered bucket listing ignored these complexities, which yielded incorrect results when dealing with such adorned object names. Markers, for example, failed to continue an unordered bucket listing where it left off and could sometimes cause listing loops. The lifecycle rule dealing with multipart expiration exposed these issues. This commit makes unordered bucket listing handle these complexities. Fixes: http://tracker.ceph.com/issues/38486 Signed-off-by: J. Eric Ivancich --- diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index c424e5bca8b6..f2e308543df0 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -2517,6 +2517,7 @@ int RGWRados::Bucket::List::list_objects_ordered(int64_t max, ldout(cct, 0) << "ERROR: could not parse object name: " << obj.name << dendl; continue; } + bool check_ns = (obj.ns == params.ns); if (!params.list_versions && !entry.is_visible()) { continue; @@ -2636,11 +2637,14 @@ int RGWRados::Bucket::List::list_objects_unordered(int64_t max, result->clear(); - rgw_obj_key marker_obj(params.marker.name, params.marker.instance, params.ns); + rgw_obj_key marker_obj(params.marker.name, + params.marker.instance, + params.ns); rgw_obj_index_key cur_marker; marker_obj.get_index_key(&cur_marker); - rgw_obj_key end_marker_obj(params.end_marker.name, params.end_marker.instance, + rgw_obj_key end_marker_obj(params.end_marker.name, + params.end_marker.instance, params.ns); rgw_obj_index_key cur_end_marker; end_marker_obj.get_index_key(&cur_end_marker); @@ -2700,8 +2704,8 @@ int RGWRados::Bucket::List::list_objects_unordered(int64_t max, } if (count < max) { - params.marker = index_key; - next_marker = index_key; + params.marker.set(index_key); + next_marker.set(index_key); } if (params.filter && !params.filter->filter(obj.name, index_key.name)) @@ -9159,30 +9163,60 @@ int RGWRados::cls_bucket_list_unordered(RGWBucketInfo& bucket_info, " start " << start.name << "[" << start.instance << "] num_entries " << num_entries << dendl; + static MultipartMetaFilter multipart_meta_filter; + *is_truncated = false; librados::IoCtx index_ctx; - rgw_obj_index_key my_start = start; - map oids; int r = open_bucket_index(bucket_info, index_ctx, oids, shard_id); if (r < 0) return r; const uint32_t num_shards = oids.size(); + rgw_obj_index_key marker = start; uint32_t current_shard; if (shard_id >= 0) { current_shard = shard_id; - } else if (my_start.empty()) { + } else if (start.empty()) { current_shard = 0u; } else { - current_shard = - rgw_bucket_shard_index(my_start.name, num_shards); + // at this point we have a marker (start) that has something in + // it, so we need to get to the bucket shard index, so we can + // start reading from there + + std::string key; + // test whether object name is a multipart meta name + if(! multipart_meta_filter.filter(start.name, key)) { + // if multipart_meta_filter fails, must be "regular" (i.e., + // unadorned) and the name is the key + key = start.name; + } + + // now convert the key (oid) to an rgw_obj_key since that will + // separate out the namespace, name, and instance + rgw_obj_key obj_key; + bool parsed = rgw_obj_key::parse_raw_oid(key, &obj_key); + if (!parsed) { + ldout(cct, 0) << + "ERROR: RGWRados::cls_bucket_list_unordered received an invalid " + "start marker: '" << start << "'" << dendl; + return -EINVAL; + } else if (obj_key.name.empty()) { + // if the name is empty that means the object name came in with + // a namespace only, and therefore we need to start our scan at + // the first bucket index shard + current_shard = 0u; + } else { + // so now we have the key used to compute the bucket index shard + // and can extract the specific shard from it + current_shard = rgw_bucket_shard_index(obj_key.name, num_shards); + } } uint32_t count = 0u; map updates; - std::string last_added_entry; + rgw_obj_index_key last_added_entry; while (count <= num_entries && ((shard_id >= 0 && current_shard == uint32_t(shard_id)) || current_shard < num_shards)) { @@ -9190,7 +9224,7 @@ int RGWRados::cls_bucket_list_unordered(RGWBucketInfo& bucket_info, // value - list result for the corresponding oid (shard), it is filled by // the AIO callback map list_results; - r = CLSRGWIssueBucketList(index_ctx, my_start, prefix, num_entries, + r = CLSRGWIssueBucketList(index_ctx, marker, prefix, num_entries, list_versions, oids, list_results, cct->_conf->rgw_bucket_index_max_aio)(); if (r < 0) @@ -9223,8 +9257,7 @@ int RGWRados::cls_bucket_list_unordered(RGWBucketInfo& bucket_info, dirent.key.name << "[" << dirent.key.instance << "]" << dendl; if (count < num_entries) { - last_added_entry = entry.first; - my_start = dirent.key; + marker = last_added_entry = dirent.key; // double assign ent_list.emplace_back(std::move(dirent)); ++count; } else { @@ -9234,18 +9267,19 @@ int RGWRados::cls_bucket_list_unordered(RGWBucketInfo& bucket_info, } else { // r == -ENOENT // in the case of -ENOENT, make sure we're advancing marker // for possible next call to CLSRGWIssueBucketList - my_start = dirent.key; + marker = dirent.key; } } // entry for loop if (!result.is_truncated) { // if we reached the end of the shard read next shard ++current_shard; - my_start = rgw_obj_index_key(); + marker = rgw_obj_index_key(); } } // shard loop check_updates: + // suggest updates if there is any map::iterator miter = updates.begin(); for (; miter != updates.end(); ++miter) { @@ -9264,7 +9298,7 @@ check_updates: } return 0; -} +} // RGWRados::cls_bucket_list_unordered int RGWRados::cls_obj_usage_log_add(const string& oid, @@ -10382,4 +10416,3 @@ int RGWRados::list_mfa(const string& oid, list *res return 0; } -