if (retcode < 0 && retcode != -ENOENT) {
return set_cr_error(retcode);
- } else if (retcode == -ENOENT && bucket_info->layout.logs.front().layout.type == rgw::BucketLogType::Deleted) {
+ } else if (retcode == -ENOENT) {
+ // the peer has no bucket-index sync status for this bucket which does
+ // not by itself mean it is deleted there (an uninitialized sync status,
+ // e.g. no data written yet, looks the same). UINT64_MAX just excludes
+ // this peer from the minimum trim generation; we will confirm real
+ // deletion via the bucket entrypoint before removing anything.
p->generation = UINT64_MAX;
- ldpp_dout(dpp, 10) << "INFO: could not read shard status for bucket:" << bucket_instance
- << " from zone: " << zid.id << dendl;
+ ldpp_dout(dpp, 10) << "bucket=" << bucket_instance
+ << ": peer zone=" << zid.id << " has no sync status (-ENOENT); "
+ << "excluding it from the trim generation" << dendl;
}
return set_cr_done();
};
+/// result of a metadata GET for a bucket ENTRYPOINT, unwrapping the
+/// {"key":..., "ver":..., "mtime":..., "data":{...}} envelope returned by
+/// /admin/metadata/bucket/<key>
+struct RGWBucketEntrypointMetadataResult {
+ RGWBucketEntryPoint data;
+
+ void decode_json(JSONObj *obj) {
+ JSONDecoder::decode_json("data", data, obj);
+ }
+};
+
+/// Resolve a bucket's entrypoint to the instance id it currently points at.
+///
+/// The entrypoint is a reliable deletion signal: its removal is a real metadata
+/// op (unlike the no-op instance removal), so its absence means a deletion has synced.
+/// But a zone's *local* entrypoint copy is not authoritative during creation: the
+/// instance syncs to peers before the entrypoint does (put_linked_bucket_info() in
+/// rgw_rados.cc writes the instance, then the entrypoint), so a peer can hold the
+/// instance without yet holding the entrypoint: i.e., indistinguishable locally from
+/// a real deletion.
+/// The metadata master is authoritative and has the entrypoint immediately on
+/// creation, so consult it, unless this zone is the master, whose local copy
+/// is itself authoritative.
+///
+/// On success sets *bucket_id to the resolved instance id, or empty if the
+/// entrypoint no longer exists. Returns an error only if the authoritative copy
+/// could not be read (the caller should then skip and retry on a later cycle).
+class RGWReadMasterBucketEntrypointCR : public RGWCoroutine {
+ rgw::sal::RadosStore* const store;
+ RGWHTTPManager *http;
+ const rgw_bucket bucket;
+ std::string *bucket_id;
+ std::shared_ptr<rgw_get_bucket_info_result> local_result;
+ RGWBucketEntrypointMetadataResult master_result;
+
+public:
+ RGWReadMasterBucketEntrypointCR(rgw::sal::RadosStore* store,
+ RGWHTTPManager *http,
+ const rgw_bucket& bucket,
+ std::string *bucket_id)
+ : RGWCoroutine(store->ctx()), store(store), http(http),
+ bucket(bucket), bucket_id(bucket_id),
+ local_result(make_shared<rgw_get_bucket_info_result>()) {}
+
+ int operate(const DoutPrefixProvider *dpp) override {
+ reenter(this) {
+ if (store->svc()->zone->is_meta_master()) {
+ yield call(new RGWGetBucketInfoCR(store->svc()->async_processor, store,
+ {bucket.tenant, bucket.name},
+ local_result, dpp));
+ if (retcode == -ENOENT) {
+ bucket_id->clear();
+ return set_cr_done();
+ }
+ if (retcode < 0) {
+ return set_cr_error(retcode);
+ }
+ *bucket_id = local_result->bucket ? local_result->bucket->get_bucket_id()
+ : std::string();
+ return set_cr_done();
+ }
+
+ yield {
+ auto *master_conn = store->svc()->zone->get_master_conn();
+ if (!master_conn) {
+ ldpp_dout(dpp, 0) << "WARNING: no connection to metadata master zone, "
+ "can't resolve entrypoint for bucket=" << bucket << dendl;
+ return set_cr_error(-ECANCELED);
+ }
+ std::string key = rgw_bucket(bucket.tenant, bucket.name).get_key();
+ rgw_http_param_pair params[] = { { "key", key.c_str() }, { nullptr, nullptr } };
+ call(new RGWReadRESTResourceCR<RGWBucketEntrypointMetadataResult>(
+ cct, master_conn, http, "/admin/metadata/bucket", params, &master_result));
+ }
+ if (retcode == -ENOENT) {
+ bucket_id->clear();
+ return set_cr_done();
+ }
+ if (retcode < 0) {
+ return set_cr_error(retcode);
+ }
+ *bucket_id = master_result.data.bucket.bucket_id;
+ return set_cr_done();
+ }
+ return 0;
+ }
+};
+
+
/// trim the bilog of all of the given bucket instance's shards
class BucketTrimInstanceCR : public RGWCoroutine {
static constexpr auto MAX_RETRIES = 25u;
std::string bucket_instance;
rgw_bucket_get_sync_policy_params get_policy_params;
std::shared_ptr<rgw_bucket_get_sync_policy_result> source_policy;
+ std::string cur_entrypoint_bucket_id;
rgw_bucket bucket;
const std::string& zone_id; //< my zone id
RGWBucketInfo _bucket_info;
}
if (min_generation == UINT64_MAX) {
- // if all peers have deleted this bucket, purge the rest of our log generations
+ // no peer is syncing this bucket (all returned -ENOENT); this alone does
+ // not confirm deletion: we will check the entrypoint before cleanup.
totrim.gen = UINT64_MAX;
return 0;
}
return 0;
}
+ // every peer returned -ENOENT for this bucket's sync status (no peer is
+ // syncing it). NOTE: this alone does not confirm deletion. A live bucket
+ // whose sync status is not yet initialized looks the same; the entrypoint
+ // check in operate() disambiguates.
+ bool all_peers_report_bucket_gone() const {
+ return totrim.gen == UINT64_MAX;
+ }
+
+ // the local layout already carries the Deleted flag, i.e. metadata sync has
+ // delivered the deletion to this zone. That is authoritative on its own, so
+ // the normal Deleted-driven cleanup applies and we can skip the entrypoint
+ // round-trip to the metadata master.
+ bool locally_confirmed_deleted() const {
+ return pbucket_info->layout.logs.back().layout.type == rgw::BucketLogType::Deleted;
+ }
+
/// If there is a generation below the minimum, prepare to clean it up.
int maybe_remove_generation() {
if (clean_info)
return 0;
- bool deleted_type = (pbucket_info->layout.logs.back().layout.type == rgw::BucketLogType::Deleted);
+ bool deleted_type = locally_confirmed_deleted();
if (pbucket_info->layout.logs.front().gen < totrim.gen ||
(pbucket_info->layout.logs.front().gen <= totrim.gen && deleted_type)) {
clean_info = {*pbucket_info, {}};
auto log = clean_info->first.layout.logs.cbegin();
clean_info->second = *log;
+ // the local zone still sees this bucket as a normal live bucket
+ // with one generation and no Deleted flag.
if (clean_info->first.layout.logs.size() == 1 && !deleted_type) {
- ldpp_dout(dpp, -1)
- << "Critical error! Attempt to remove only log generation! "
- << "log.gen=" << log->gen << ", totrim.gen=" << totrim.gen
- << dendl;
- return -EIO;
+ if (!all_peers_report_bucket_gone()) { // unexpected: a peer reported a real generation but local has only one
+ ldpp_dout(dpp, -1)
+ << "Critical error! Attempt to remove only log generation! "
+ << "log.gen=" << log->gen << ", totrim.gen=" << totrim.gen
+ << dendl;
+ return -EIO;
+ }
+ ldpp_dout(dpp, 10) << "bucket=" << bucket_instance
+ << " all peers gone, deferring instance removal" << dendl;
+ clean_info = std::nullopt;
+ return 0;
}
clean_info->first.layout.logs.erase(log);
}
ldpp_dout(dpp, 4) << "failed to find minimum generation" << dendl;
return set_cr_error(retcode);
}
+
+ // When all peers report -ENOENT the bucket may be deleted, or it may be a
+ // live bucket whose sync status is not yet initialized (both look the same).
+ // But, if the local layout already carries the Deleted flag, metadata sync has
+ // delivered the deletion and that is authoritative on its own; i.e., go through
+ // to the normal Deleted-driven cleanup below. Otherwise, confirm real deletion
+ // by resolving the authoritative entrypoint before touching the instance
+ // metadata; only an absent (or repointed) entrypoint means this instance is
+ // truly gone.
+ if (all_peers_report_bucket_gone() && !locally_confirmed_deleted()) {
+ yield call(new RGWReadMasterBucketEntrypointCR(store, http, bucket,
+ &cur_entrypoint_bucket_id));
+ if (retcode < 0) {
+ ldpp_dout(dpp, 4) << "bucket=" << bucket_instance
+ << " could not confirm deletion via entrypoint: " << cpp_strerror(retcode)
+ << ", skipping to avoid removing a live bucket" << dendl;
+ return set_cr_done();
+ }
+ if (cur_entrypoint_bucket_id == pbucket_info->bucket.bucket_id) {
+ // entrypoint still resolves to this instance: the bucket is live and its
+ // sync status must be uninitialized, not deleted. Leave it alone.
+ ldpp_dout(dpp, 10) << "bucket=" << bucket_instance
+ << " still has a live entrypoint, skipping (not deleted)" << dendl;
+ return set_cr_done();
+ }
+ ldpp_dout(dpp, 10) << "bucket=" << bucket_instance
+ << " entrypoint absent or repointed, treating as deleted" << dendl;
+ }
+
retcode = maybe_remove_generation();
if (retcode < 0) {
ldpp_dout(dpp, 4) << "error removing old generation from log: "
return set_cr_error(retcode);
}
+ // The no-Deleted-flag orphan path (#70858): all peers are gone, the
+ // entrypoint check above confirmed the bucket is deleted, and there's no
+ // generation cleanup pending. This is only reachable when the Deleted flag
+ // never arrived. What remains is a single InIndex generation, so remove the
+ // orphaned instance metadata directly.
+ if (all_peers_report_bucket_gone() && !clean_info) {
+ ldpp_dout(dpp, 1) << "all peers report bucket gone, removing "
+ << "orphaned bucket instance metadata" << dendl;
+ _bucket_info = *pbucket_info;
+ yield call(new RGWRemoveBucketInstanceInfoCR(
+ store->svc()->async_processor,
+ store, _bucket_info.bucket,
+ _bucket_info, nullptr, dpp));
+ if (retcode < 0 && retcode != -ENOENT) {
+ ldpp_dout(dpp, 0) << "failed to remove instance bucket info: "
+ << cpp_strerror(retcode) << dendl;
+ return set_cr_error(retcode);
+ }
+ observer->on_bucket_trimmed(std::move(bucket_instance));
+ return set_cr_done();
+ }
+
if (clean_info) {
if (clean_info->second.layout.type != rgw::BucketLogType::InIndex) {
ldpp_dout(dpp, 0) << "Unable to convert log of unknown type "
projects / ceph.git / commitdiff