From: Yehuda Sadeh Date: Tue, 5 Jan 2016 21:51:34 +0000 (-0800) Subject: rgw: handle delete marker when doing incremental data sync X-Git-Tag: v10.1.0~354^2~92 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=850dd42cef2386df11634c8965e9be3abb26addd;p=ceph.git rgw: handle delete marker when doing incremental data sync Still need to handle the full sync, and to make sure we apply it only if-newer (as with the other data sync operations) to avoid circular sync. Signed-off-by: Yehuda Sadeh --- diff --git a/src/rgw/rgw_cr_rados.cc b/src/rgw/rgw_cr_rados.cc index 181e8e4b64d..ea5687fdc84 100644 --- a/src/rgw/rgw_cr_rados.cc +++ b/src/rgw/rgw_cr_rados.cc @@ -523,6 +523,10 @@ int RGWAsyncRemoveObj::_send_request() if (del_if_older) { del_op.params.unmod_since = timestamp; } + del_op.params.olh_epoch = versioned_epoch; + del_op.params.marker_version_id = marker_version_id; + del_op.params.obj_owner.set_id(owner); + del_op.params.obj_owner.set_name(owner_display_name); ret = del_op.delete_obj(); if (ret < 0) { diff --git a/src/rgw/rgw_cr_rados.h b/src/rgw/rgw_cr_rados.h index 20422eb4259..30761c9b69e 100644 --- a/src/rgw/rgw_cr_rados.h +++ b/src/rgw/rgw_cr_rados.h @@ -675,7 +675,10 @@ class RGWAsyncRemoveObj : public RGWAsyncRadosRequest { RGWBucketInfo bucket_info; rgw_obj_key key; + string owner; + string owner_display_name; uint64_t versioned_epoch; + string marker_version_id; bool del_if_older; utime_t timestamp; @@ -687,15 +690,24 @@ public: const string& _source_zone, RGWBucketInfo& _bucket_info, const rgw_obj_key& _key, + const string& _owner, + const string& _owner_display_name, uint64_t _versioned_epoch, + bool _delete_marker, bool _if_older, utime_t& _timestamp) : RGWAsyncRadosRequest(cn), store(_store), source_zone(_source_zone), bucket_info(_bucket_info), key(_key), + owner(_owner), + owner_display_name(_owner_display_name), versioned_epoch(_versioned_epoch), del_if_older(_if_older), - timestamp(_timestamp) {} + timestamp(_timestamp) { + if (_delete_marker) { + marker_version_id = key.instance; + } + } }; class RGWRemoveObjCR : public RGWSimpleCoroutine { @@ -708,6 +720,9 @@ class RGWRemoveObjCR : public RGWSimpleCoroutine { rgw_obj_key key; uint64_t versioned_epoch; + bool delete_marker; + string owner; + string owner_display_name; bool del_if_older; utime_t timestamp; @@ -720,16 +735,28 @@ public: RGWBucketInfo& _bucket_info, const rgw_obj_key& _key, uint64_t _versioned_epoch, + string *_owner, + string *_owner_display_name, + bool _delete_marker, utime_t *_timestamp) : RGWSimpleCoroutine(_store->ctx()), cct(_store->ctx()), async_rados(_async_rados), store(_store), source_zone(_source_zone), bucket_info(_bucket_info), key(_key), - versioned_epoch(_versioned_epoch) { + versioned_epoch(_versioned_epoch), + delete_marker(_delete_marker) { del_if_older = (_timestamp != NULL); if (_timestamp) { timestamp = *_timestamp; } + + if (_owner) { + owner = *_owner; + } + + if (_owner_display_name) { + owner_display_name = *_owner_display_name; + } } ~RGWRemoveObjCR() { @@ -738,7 +765,8 @@ public: int send_request() { req = new RGWAsyncRemoveObj(stack->create_completion_notifier(), store, source_zone, bucket_info, - key, versioned_epoch, del_if_older, timestamp); + key, owner, owner_display_name, versioned_epoch, + delete_marker, del_if_older, timestamp); async_rados->queue(req); return 0; } diff --git a/src/rgw/rgw_data_sync.cc b/src/rgw/rgw_data_sync.cc index b16dbe11a6a..df9fc472aec 100644 --- a/src/rgw/rgw_data_sync.cc +++ b/src/rgw/rgw_data_sync.cc @@ -1588,6 +1588,9 @@ struct bucket_entry_owner { string id; string display_name; + bucket_entry_owner() {} + bucket_entry_owner(const string& _id, const string& _display_name) : id(_id), display_name(_display_name) {} + void decode_json(JSONObj *obj) { JSONDecoder::decode_json("ID", id, obj); JSONDecoder::decode_json("DisplayName", display_name, obj); @@ -1879,6 +1882,7 @@ class RGWBucketSyncSingleEntryCR : public RGWCoroutine { rgw_obj_key key; uint64_t versioned_epoch; + bucket_entry_owner owner; utime_t timestamp; RGWModifyOp op; RGWPendingState op_state; @@ -1894,18 +1898,21 @@ public: const string& _source_zone, RGWBucketInfo *_bucket_info, int _shard_id, const rgw_obj_key& _key, uint64_t _versioned_epoch, utime_t& _timestamp, + const bucket_entry_owner& _owner, RGWModifyOp _op, RGWPendingState _op_state, const T& _entry_marker, RGWSyncShardMarkerTrack *_marker_tracker) : RGWCoroutine(_store->ctx()), store(_store), async_rados(_async_rados), source_zone(_source_zone), bucket_info(_bucket_info), shard_id(_shard_id), key(_key), versioned_epoch(_versioned_epoch), + owner(_owner), timestamp(_timestamp), op(_op), op_state(_op_state), entry_marker(_entry_marker), marker_tracker(_marker_tracker), sync_status(0) { - set_description() << "bucket sync single entry (source_zone=" << source_zone << ") b=" << bucket_info->bucket << ":" << shard_id <<"/" << key << "[" << versioned_epoch << "] log_entry=" << entry_marker; + set_description() << "bucket sync single entry (source_zone=" << source_zone << ") b=" << bucket_info->bucket << ":" << shard_id <<"/" << key << "[" << versioned_epoch << "] log_entry=" << entry_marker << " op=" << (int)op << " op_state=" << (int)op_state; + ldout(store->ctx(), 20) << "bucket sync single entry (source_zone=" << source_zone << ") b=" << bucket_info->bucket << ":" << shard_id <<"/" << key << "[" << versioned_epoch << "] log_entry=" << entry_marker << " op=" << (int)op << " op_state=" << (int)op_state << dendl; set_status("init"); } @@ -1932,7 +1939,12 @@ public: key, versioned_epoch, true)); } else if (op == CLS_RGW_OP_DEL) { - call(new RGWRemoveObjCR(async_rados, store, source_zone, *bucket_info, key, versioned_epoch, ×tamp)); + set_status("removing obj"); + call(new RGWRemoveObjCR(async_rados, store, source_zone, *bucket_info, key, versioned_epoch, NULL, NULL, false, ×tamp)); + } else if (op == CLS_RGW_OP_LINK_OLH_DM) { + set_status("creating delete marker"); + ldout(store->ctx(), 10) << "creating delete marker: obj: " << source_zone << "/" << bucket_info->bucket << "/" << key << "[" << versioned_epoch << "]" << dendl; + call(new RGWRemoveObjCR(async_rados, store, source_zone, *bucket_info, key, versioned_epoch, &owner.id, &owner.display_name, true, ×tamp)); } } } while (marker_tracker->need_retry(key)); @@ -2066,7 +2078,8 @@ int RGWBucketShardFullSyncCR::operate() yield { spawn(new RGWBucketSyncSingleEntryCR(store, async_rados, source_zone, bucket_info, shard_id, - entry->key, entry->versioned_epoch, entry->mtime, op, CLS_RGW_STATE_COMPLETE, entry->key, marker_tracker), false); + entry->key, entry->versioned_epoch, entry->mtime, + entry->owner, op, CLS_RGW_STATE_COMPLETE, entry->key, marker_tracker), false); } } while ((int)num_spawned() > spawn_window) { @@ -2253,11 +2266,13 @@ int RGWBucketShardIncrementalSyncCR::operate() ldout(store->ctx(), 0) << "ERROR: cannot start syncing " << entry->id << ". Duplicate entry?" << dendl; } else { uint64_t versioned_epoch = 0; + bucket_entry_owner owner(entry->owner, entry->owner_display_name); if (entry->ver.pool < 0) { versioned_epoch = entry->ver.epoch; } spawn(new RGWBucketSyncSingleEntryCR(store, async_rados, source_zone, bucket_info, shard_id, - key, versioned_epoch, entry->timestamp, entry->op, entry->state, entry->id, marker_tracker), false); + key, versioned_epoch, entry->timestamp, owner, entry->op, + entry->state, entry->id, marker_tracker), false); } // } while ((int)num_spawned() > spawn_window) {