From: Oguzhan Ozmen Date: Sun, 5 Jul 2026 00:45:40 +0000 (+0000) Subject: rgw/test/multisite: test for bilog autotrim -ENOENT race after secondary reshard X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=43518e2f3e093a608a47d4af91bddc0b94615ac4;p=ceph.git rgw/test/multisite: test for bilog autotrim -ENOENT race after secondary reshard Adds test_bucket_log_trim_enoent_race_after_reshard to reproduce the exact failure from tracker #70858: when autotrim runs on the primary zone first and fully cleans up (including removing its own bucket.instance metadata), then an autotrim on the secondary zone asks the primary and receives -ENOENT. The secondary hasn't received the BucketLogType::Deleted flag yet (metadata sync is stalled via delay injection), so the current code left StatusShards{generation=0, shards=[]} which caused take_min_status() to fail with -EINVAL, leaving the bucket.instance metadata permanently orphaned. Tests: https://tracker.ceph.com/issues/70858 Signed-off-by: Oguzhan Ozmen --- diff --git a/src/rgw/driver/rados/rgw_sync.cc b/src/rgw/driver/rados/rgw_sync.cc index 810f35b77e3..973fb276f39 100644 --- a/src/rgw/driver/rados/rgw_sync.cc +++ b/src/rgw/driver/rados/rgw_sync.cc @@ -1360,6 +1360,17 @@ int RGWMetaSyncSingleEntryCR::operate(const DoutPrefixProvider *dpp) { retcode = 0; for (tries = 0; tries < NUM_TRANSIENT_ERROR_RETRIES; tries++) { if (sync_status != -ENOENT) { + if (section == "bucket.instance" && + cct->_conf->rgw_inject_delay_sec > 0 && + std::string_view(cct->_conf->rgw_inject_delay_pattern) == + "delay_meta_sync_bucket_instance_store") { + yield { + utime_t dur; + dur.set_from_double(cct->_conf->rgw_inject_delay_sec); + tn->log(0, SSTR("injecting a delay of " << dur << "s for bucket.instance metadata store")); + wait(dur); + } + } tn->log(10, SSTR("storing local metadata entry: " << section << ":" << key)); yield call(new RGWMetaStoreEntryCR(sync_env, raw_key, md_bl)); } else { diff --git a/src/test/rgw/rgw_multi/tests.py b/src/test/rgw/rgw_multi/tests.py index cca37d294ec..465a3c3fba8 100644 --- a/src/test/rgw/rgw_multi/tests.py +++ b/src/test/rgw/rgw_multi/tests.py @@ -2200,6 +2200,102 @@ def test_bucket_log_trim_after_delete_bucket_secondary_reshard(): assert check_bucket_instance_metadata(zone.zone, test_bucket.name) +@attr('bucket_trim') +def test_bucket_log_trim_enoent_race_after_reshard(): + zonegroup = realm.master_zonegroup() + zonegroup_conns = ZonegroupConns(zonegroup) + + primary = zonegroup_conns.rw_zones[0] + secondary = zonegroup_conns.rw_zones[1] + + all_clusters = set() + for zg in realm.current_period.zonegroups: + for zone in zg.zones: + all_clusters.add(zone.cluster) + + def set_meta_sync_delay(delay_sec): + for cluster in all_clusters: + if delay_sec > 0: + cluster.ceph_admin( + ['config', 'set', 'client', 'rgw_inject_delay_sec', str(delay_sec)]) + cluster.ceph_admin( + ['config', 'set', 'client', 'rgw_inject_delay_pattern', 'delay_meta_sync_bucket_instance_store']) + else: + cluster.ceph_admin( + ['config', 'rm', 'client', 'rgw_inject_delay_sec']) + cluster.ceph_admin( + ['config', 'rm', 'client', 'rgw_inject_delay_pattern']) + + def make_test_bucket(): + name = gen_bucket_name() + log.info('create bucket zone=%s name=%s', primary.zone.name, name) + bucket = primary.create_bucket(name) + for objname in ('a', 'b', 'c', 'd'): + primary.s3_client.put_object(Bucket=bucket.name, Key=objname, Body='foo') + zonegroup_meta_checkpoint(zonegroup) + zonegroup_bucket_checkpoint(zonegroup_conns, name) + return bucket + + test_bucket = make_test_bucket() + + # reshard on secondary to create a generation mismatch + secondary.zone.cluster.admin(['bucket', 'reshard', + '--bucket', test_bucket.name, + '--num-shards', '13', + '--yes-i-really-mean-it'] + secondary.zone.zone_args()) + + for obj in ('a', 'b', 'c', 'd'): + cmd = ['object', 'rm'] + primary.zone.zone_args() + cmd += ['--bucket', test_bucket.name] + cmd += ['--object', obj] + primary.zone.cluster.admin(cmd + primary.zone.zone_args()) + + log.info('setting metadata sync delay to reproduce ENOENT trim race') + set_meta_sync_delay(30) + time.sleep(10) # let the delay config reach the radosgws before deleting + + try: + primary.s3_client.delete_bucket(Bucket=test_bucket.name) + zonegroup_data_checkpoint(zonegroup_conns) + + # run autotrim on primary first — primary has the Deleted flag + # so it will fully clean up including removing its own + # bucket.instance metadata + bilog_autotrim(primary.zone, ['--rgw-sync-log-trim-max-buckets', '50'],) + time.sleep(config.checkpoint_delay) + bilog_autotrim(primary.zone, ['--rgw-sync-log-trim-max-buckets', '50'],) + time.sleep(config.checkpoint_delay) + + # now autotrim on secondary — secondary queries primary, but + # primary's instance metadata is gone so primary responds -ENOENT. + # secondary doesn't have the Deleted flag yet (metadata sync is + # stalled). In #70858, this leaves StatusShards{gen=0, shards=[]} + # causing take_min_status() to fail with -EINVAL. + bilog_autotrim(secondary.zone, ['--rgw-sync-log-trim-max-buckets', '50'],) + time.sleep(config.checkpoint_delay) + bilog_autotrim(secondary.zone, ['--rgw-sync-log-trim-max-buckets', '50'],) + finally: + log.info('removing metadata sync delay') + set_meta_sync_delay(0) + + for zonegroup in realm.current_period.zonegroups: + zonegroup_conns = ZonegroupConns(zonegroup) + zonegroup_meta_checkpoint(zonegroup) + + for zone in zonegroup_conns.zones: + log.info('trimming on zone=%s', zone.name) + bilog_autotrim(zone.zone, ['--rgw-sync-log-trim-max-buckets', '50'],) + time.sleep(config.checkpoint_delay) + + bilog_autotrim(secondary.zone, ['--rgw-sync-log-trim-max-buckets', '50'],) + time.sleep(config.checkpoint_delay) + + for zonegroup in realm.current_period.zonegroups: + zonegroup_conns = ZonegroupConns(zonegroup) + for zone in zonegroup_conns.zones: + assert check_bucket_instance_metadata(zone.zone, test_bucket.name) + + @attr('bucket_reshard') def test_bucket_reshard_incremental(): zonegroup = realm.master_zonegroup()