From: Oguzhan Ozmen Date: Mon, 13 Jul 2026 19:49:41 +0000 (+0000) Subject: rgw/test/multisite: tests for bilog autotrim -ENOENT interpretation X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=6cc8ee204339aae86a968310d23c57e397d4c9de;p=ceph.git rgw/test/multisite: tests for bilog autotrim -ENOENT interpretation An all-peer -ENOENT does not by itself mean a bucket is deleted: a live bucket whose sync status is not yet initialized returns -ENOENT too. Add two testcases that verifies we do not accidentally delete instance of a live bucket: - test_bucket_log_trim_live_empty_bucket_not_removed: a data-less bucket never gets sync status, so every peer reports -ENOENT; autotrim must not remove its instance. - test_bucket_log_trim_new_bucket_entrypoint_not_synced: a new bucket whose instance has synced to the secondary before its entrypoint must survive autotrim. A delay_meta_sync_bucket_entrypoint_store injection forces that window. Tests: https://tracker.ceph.com/issues/70858 Signed-off-by: Oguzhan Ozmen --- diff --git a/src/rgw/driver/rados/rgw_sync.cc b/src/rgw/driver/rados/rgw_sync.cc index 973fb276f39..ec5b8798c35 100644 --- a/src/rgw/driver/rados/rgw_sync.cc +++ b/src/rgw/driver/rados/rgw_sync.cc @@ -1371,6 +1371,17 @@ int RGWMetaSyncSingleEntryCR::operate(const DoutPrefixProvider *dpp) { wait(dur); } } + if (section == "bucket" && + cct->_conf->rgw_inject_delay_sec > 0 && + std::string_view(cct->_conf->rgw_inject_delay_pattern) == + "delay_meta_sync_bucket_entrypoint_store") { + yield { + utime_t dur; + dur.set_from_double(cct->_conf->rgw_inject_delay_sec); + tn->log(0, SSTR("injecting a delay of " << dur << "s for bucket entrypoint metadata store")); + wait(dur); + } + } tn->log(10, SSTR("storing local metadata entry: " << section << ":" << key)); yield call(new RGWMetaStoreEntryCR(sync_env, raw_key, md_bl)); } else { diff --git a/src/test/rgw/rgw_multi/tests.py b/src/test/rgw/rgw_multi/tests.py index 465a3c3fba8..a9caf0f5229 100644 --- a/src/test/rgw/rgw_multi/tests.py +++ b/src/test/rgw/rgw_multi/tests.py @@ -2296,6 +2296,118 @@ def test_bucket_log_trim_enoent_race_after_reshard(): assert check_bucket_instance_metadata(zone.zone, test_bucket.name) +@attr('bucket_trim') +def test_bucket_log_trim_live_empty_bucket_not_removed(): + # A live bucket with no data written yet also returns -ENOENT for its bucket sync + # status on every peer as sync status is not initialized until data flows. + # autotrim must NOT mistake that for a deletion and remove the bucket + # instance metadata. This is a guard testcase for the -ENOENT disambiguation + # (tracker #70858): i.e., if the entrypoint still exists, so the bucket is live. + zonegroup = realm.master_zonegroup() + zonegroup_conns = ZonegroupConns(zonegroup) + primary = zonegroup_conns.rw_zones[0] + + # create a bucket but write no objects, so no bucket sync status is created + name = gen_bucket_name() + log.info('create empty bucket zone=%s name=%s', primary.zone.name, name) + bucket = primary.create_bucket(name) + + # make sure the entrypoint and instance metadata reach every zone + zonegroup_meta_checkpoint(zonegroup) + + # run autotrim everywhere; peers return -ENOENT for sync status, but the + # bucket is live (entrypoint present) and must be left intact. + for zg in realm.current_period.zonegroups: + zg_conns = ZonegroupConns(zg) + for zone in zg_conns.zones: + log.info('trimming on zone=%s', zone.name) + bilog_autotrim(zone.zone, ['--rgw-sync-log-trim-max-buckets', '50']) + time.sleep(config.checkpoint_delay) + + # the bucket instance metadata must still be present on every zone. + # check_bucket_instance_metadata() returns False when the instance is still + # present (i.e. it was NOT trimmed away). + for zg in realm.current_period.zonegroups: + zg_conns = ZonegroupConns(zg) + for zone in zg_conns.zones: + assert not check_bucket_instance_metadata(zone.zone, bucket.name), \ + 'live bucket %s instance wrongly removed on zone %s' % (bucket.name, zone.name) + + +@attr('bucket_trim') +def test_bucket_log_trim_new_bucket_entrypoint_not_synced(): + # A bucket freshly created on the primary syncs its INSTANCE metadata to the + # secondary before its ENTRYPOINT (creation writes the instance first, see + # put_linked_bucket_info() in rgw_rados.cc). If autotrim runs on the secondary + # in that window, every peer returns -ENOENT (new, data-less bucket => sync + # status uninitialized). autotrim must confirm the deletion against the + # metadata master (which already has the entrypoint) rather than the secondary's + # not-yet-synced local copy, and so must leave the live bucket's instance intact. + # + # Force the window described above by delaying entrypoint ("bucket" section) metadata + # sync on the secondary while letting the instance ("bucket.instance") go through. + zonegroup = realm.master_zonegroup() + zonegroup_conns = ZonegroupConns(zonegroup) + primary = zonegroup_conns.rw_zones[0] + secondary = zonegroup_conns.rw_zones[1] + + def set_entrypoint_sync_delay(delay_sec): + cluster = secondary.zone.cluster + if delay_sec > 0: + cluster.ceph_admin(['config', 'set', 'client', 'rgw_inject_delay_sec', str(delay_sec)]) + cluster.ceph_admin(['config', 'set', 'client', 'rgw_inject_delay_pattern', 'delay_meta_sync_bucket_entrypoint_store']) + else: + cluster.ceph_admin(['config', 'rm', 'client', 'rgw_inject_delay_sec']) + cluster.ceph_admin(['config', 'rm', 'client', 'rgw_inject_delay_pattern']) + + # metadata list is per-zone, so this reads the secondary's LOCAL entrypoints + def secondary_has_entrypoint(name): + cmd = ['metadata', 'list', 'bucket'] + secondary.zone.zone_args() + out, _ = secondary.zone.cluster.admin(cmd, check_retcode=False, read_only=True) + try: + return name in json.loads(out) + except ValueError: + return False + + with override_config(checkpoint_retries=30, checkpoint_delay=2): + set_entrypoint_sync_delay(config.checkpoint_retries * config.checkpoint_delay) + try: + time.sleep(10) # let the delay config reach the radosgws before creating + + name = gen_bucket_name() + log.info('create bucket zone=%s name=%s (no objects)', primary.zone.name, name) + primary.create_bucket(name) + + # wait until the INSTANCE has synced to the secondary + # while its ENTRYPOINT is still delayed (absent) + window_hit = False + for _ in range(config.checkpoint_retries): + time.sleep(config.checkpoint_delay) + inst_present = not check_bucket_instance_metadata(secondary.zone, name) + ep_present = secondary_has_entrypoint(name) + log.info('secondary sync state: instance=%s entrypoint=%s', inst_present, ep_present) + if inst_present and not ep_present: + window_hit = True + break + if inst_present and ep_present: + break # entrypoint arrived too; window missed + assert window_hit, \ + 'could not reproduce instance-synced-but-entrypoint-not window on secondary' + + # autotrim on the secondary while its local entrypoint is still absent + log.info('running autotrim on secondary while entrypoint is not yet synced') + bilog_autotrim(secondary.zone, ['--rgw-sync-log-trim-max-buckets', '50']) + time.sleep(config.checkpoint_delay) + + # the instance metadata must NOT have been removed since the bucket is live + assert not check_bucket_instance_metadata(secondary.zone, name), \ + 'live (mid-sync) bucket %s instance wrongly removed on secondary zone' % name + finally: + set_entrypoint_sync_delay(0) + # let metadata sync catch up so the entrypoint lands and state is consistent + zonegroup_meta_checkpoint(zonegroup) + + @attr('bucket_reshard') def test_bucket_reshard_incremental(): zonegroup = realm.master_zonegroup()