From: kchheda3 Date: Tue, 27 Feb 2024 19:06:47 +0000 (-0500) Subject: rgw/notification: Make the Lifecycle events AWS compatible X-Git-Tag: v19.1.0~104^2~11 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=1f686cb4dbe52512d9f2c90137e8b4f620dfb97b;p=ceph.git rgw/notification: Make the Lifecycle events AWS compatible Signed-off-by: kchheda3 (cherry picked from commit ea6611327c9aa678c524e0d71e954bd735d60789) --- diff --git a/doc/radosgw/s3-notification-compatibility.rst b/doc/radosgw/s3-notification-compatibility.rst index cced60924d09e..b29989977b231 100644 --- a/doc/radosgw/s3-notification-compatibility.rst +++ b/doc/radosgw/s3-notification-compatibility.rst @@ -97,6 +97,14 @@ Event Types +--------------------------------------------------------+-----------------------------------------+ | ``s3:ObjectLifecycle:Transition:NonCurrent`` | Ceph extension | +--------------------------------------------------------+-----------------------------------------+ +| ``s3:LifecycleExpiration:*`` | Supported. Equivalent to: s3:LifecycleExpiration:Delete, s3:LifecycleExpiration:DeleteMarkerCreated| ++--------------------------------------------------------+-----------------------------------------+ +| ``s3:LifecycleExpiration:Delete`` | Supported. Equivalent to: s3:ObjectLifecycle:Expiration:Current.| ++--------------------------------------------------------+-----------------------------------------+ +| ``s3:LifecycleExpiration:DeleteMarkerCreated`` | Supported. Equivalent to: s3:ObjectLifecycle:Expiration:DeleteMarker.| ++--------------------------------------------------------+-----------------------------------------+ +| ``s3:LifecycleTransition`` | Supported. Equivalent to: s3:ObjectLifecycle:Transition:Current| ++--------------------------------------------------------+-----------------------------------------+ | ``s3:ObjectSynced:*`` | Ceph extension | +--------------------------------------------------------+-----------------------------------------+ | ``s3:ObjectSynced:Create`` | Ceph Extension | diff --git a/src/rgw/driver/daos/rgw_sal_daos.cc b/src/rgw/driver/daos/rgw_sal_daos.cc index 43ae35be42658..a6136d9059988 100644 --- a/src/rgw/driver/daos/rgw_sal_daos.cc +++ b/src/rgw/driver/daos/rgw_sal_daos.cc @@ -2271,16 +2271,22 @@ std::unique_ptr DaosStore::get_lifecycle(void) { std::unique_ptr DaosStore::get_notification( rgw::sal::Object* obj, rgw::sal::Object* src_obj, struct req_state* s, rgw::notify::EventType event_type, const std::string* object_name) { - return std::make_unique(obj, src_obj, event_type); + rgw::notify::EventTypeList event_types = {event_type}; + return std::make_unique(obj, src_obj, event_types); } std::unique_ptr DaosStore::get_notification( - const DoutPrefixProvider* dpp, Object* obj, Object* src_obj, - rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket, - std::string& _user_id, std::string& _user_tenant, std::string& _req_id, + const DoutPrefixProvider* dpp, + Object* obj, + Object* src_obj, + const rgw::notify::EventTypeList& event_types, + rgw::sal::Bucket* _bucket, + std::string& _user_id, + std::string& _user_tenant, + std::string& _req_id, optional_yield y) { ldpp_dout(dpp, 20) << "get_notification" << dendl; - return std::make_unique(obj, src_obj, event_type); + return std::make_unique(obj, src_obj, event_types); } int DaosStore::log_usage(const DoutPrefixProvider* dpp, diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h index c5cfefc222d10..2d74f9c17cc71 100644 --- a/src/rgw/driver/daos/rgw_sal_daos.h +++ b/src/rgw/driver/daos/rgw_sal_daos.h @@ -135,8 +135,10 @@ WRITE_CLASS_ENCODER(DaosUserInfo); class DaosNotification : public StoreNotification { public: - DaosNotification(Object* _obj, Object* _src_obj, rgw::notify::EventType _type) - : StoreNotification(_obj, _src_obj, _type) {} + DaosNotification(Object* _obj, + Object* _src_obj, + const rgw::notify::EventTypeList& _types) + : StoreNotification(_obj, _src_obj, _types) {} ~DaosNotification() = default; virtual int publish_reserve(const DoutPrefixProvider* dpp, @@ -911,10 +913,14 @@ class DaosStore : public StoreDriver { rgw::notify::EventType event_type, optional_yield y, const std::string* object_name = nullptr) override; virtual std::unique_ptr get_notification( - const DoutPrefixProvider* dpp, rgw::sal::Object* obj, - rgw::sal::Object* src_obj, rgw::notify::EventType event_type, - rgw::sal::Bucket* _bucket, std::string& _user_id, - std::string& _user_tenant, std::string& _req_id, + const DoutPrefixProvider* dpp, + rgw::sal::Object* obj, + rgw::sal::Object* src_obj, + const rgw::notify::EventTypeList& event_types, + rgw::sal::Bucket* _bucket, + std::string& _user_id, + std::string& _user_tenant, + std::string& _req_id, optional_yield y) override; virtual RGWLC* get_rgwlc(void) override { return NULL; } virtual RGWCoroutinesManagerRegistry* get_cr_registry() override { diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc index 55a396436167c..3ee60c9c4d5db 100644 --- a/src/rgw/driver/motr/rgw_sal_motr.cc +++ b/src/rgw/driver/motr/rgw_sal_motr.cc @@ -3289,14 +3289,21 @@ std::unique_ptr MotrStore::get_lifecycle(void) std::unique_ptr MotrStore::get_notification(Object* obj, Object* src_obj, req_state* s, rgw::notify::EventType event_type, optional_yield y, const string* object_name) { - return std::make_unique(obj, src_obj, event_type); + const rgw::notify::EventTypeList event_types = {event_type}; + return std::make_unique(obj, src_obj, event_types); } -std::unique_ptr MotrStore::get_notification(const DoutPrefixProvider* dpp, Object* obj, - Object* src_obj, rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket, - std::string& _user_id, std::string& _user_tenant, std::string& _req_id, optional_yield y) -{ - return std::make_unique(obj, src_obj, event_type); +std::unique_ptr MotrStore::get_notification( + const DoutPrefixProvider* dpp, + Object* obj, + Object* src_obj, + const rgw::notify::EventTypeList& event_types, + rgw::sal::Bucket* _bucket, + std::string& _user_id, + std::string& _user_tenant, + std::string& _req_id, + optional_yield y) { + return std::make_unique(obj, src_obj, event_types); } int MotrStore::log_usage(const DoutPrefixProvider *dpp, map& usage_info) diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h index 3cc3b37fa9ad0..63c2b9d9dc801 100644 --- a/src/rgw/driver/motr/rgw_sal_motr.h +++ b/src/rgw/driver/motr/rgw_sal_motr.h @@ -191,11 +191,13 @@ WRITE_CLASS_ENCODER(MotrAccessKey); class MotrNotification : public StoreNotification { public: - MotrNotification(Object* _obj, Object* _src_obj, rgw::notify::EventType _type) : - StoreNotification(_obj, _src_obj, _type) {} - ~MotrNotification() = default; + MotrNotification(Object* _obj, + Object* _src_obj, + const rgw::notify::EventTypeList& _types) + : StoreNotification(_obj, _src_obj, _types) {} + ~MotrNotification() = default; - virtual int publish_reserve(const DoutPrefixProvider *dpp, RGWObjTags* obj_tags = nullptr) override { return 0;} + virtual int publish_reserve(const DoutPrefixProvider *dpp, RGWObjTags* obj_tags = nullptr) override { return 0;} virtual int publish_commit(const DoutPrefixProvider* dpp, uint64_t size, const ceph::real_time& mtime, const std::string& etag, const std::string& version) override { return 0; } }; @@ -1006,9 +1008,16 @@ class MotrStore : public StoreDriver { virtual std::unique_ptr get_lifecycle(void) override; virtual std::unique_ptr get_notification(rgw::sal::Object* obj, rgw::sal::Object* src_obj, req_state* s, rgw::notify::EventType event_type, optional_yield y, const std::string* object_name=nullptr) override; - virtual std::unique_ptr get_notification(const DoutPrefixProvider* dpp, rgw::sal::Object* obj, - rgw::sal::Object* src_obj, rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket, - std::string& _user_id, std::string& _user_tenant, std::string& _req_id, optional_yield y) override; + virtual std::unique_ptr get_notification( + const DoutPrefixProvider* dpp, + rgw::sal::Object* obj, + rgw::sal::Object* src_obj, + const rgw::notify::EventTypeList& event_types, + rgw::sal::Bucket* _bucket, + std::string& _user_id, + std::string& _user_tenant, + std::string& _req_id, + optional_yield y) override; virtual RGWLC* get_rgwlc(void) override { return NULL; } virtual RGWCoroutinesManagerRegistry* get_cr_registry() override { return NULL; } diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc index 8ca6a09275bd5..a8b3fe2f3e342 100644 --- a/src/rgw/driver/posix/rgw_sal_posix.cc +++ b/src/rgw/driver/posix/rgw_sal_posix.cc @@ -417,14 +417,18 @@ std::unique_ptr POSIXDriver::get_notification(rgw::sal::Object* ob return next->get_notification(obj, src_obj, s, event_type, y, object_name); } -std::unique_ptr POSIXDriver::get_notification(const DoutPrefixProvider* dpp, - rgw::sal::Object* obj, rgw::sal::Object* src_obj, - rgw::notify::EventType event_type, - rgw::sal::Bucket* _bucket, - std::string& _user_id, std::string& _user_tenant, - std::string& _req_id, optional_yield y) -{ - return next->get_notification(dpp, obj, src_obj, event_type, _bucket, _user_id, _user_tenant, _req_id, y); +std::unique_ptr POSIXDriver::get_notification( + const DoutPrefixProvider* dpp, + rgw::sal::Object* obj, + rgw::sal::Object* src_obj, + const rgw::notify::EventTypeList& event_types, + rgw::sal::Bucket* _bucket, + std::string& _user_id, + std::string& _user_tenant, + std::string& _req_id, + optional_yield y) { + return next->get_notification(dpp, obj, src_obj, event_types, _bucket, + _user_id, _user_tenant, _req_id, y); } int POSIXDriver::close() diff --git a/src/rgw/driver/posix/rgw_sal_posix.h b/src/rgw/driver/posix/rgw_sal_posix.h index 61ca13dc76f59..ed7630a7d65a2 100644 --- a/src/rgw/driver/posix/rgw_sal_posix.h +++ b/src/rgw/driver/posix/rgw_sal_posix.h @@ -82,15 +82,15 @@ public: const std::string* object_name=nullptr) override; virtual std::unique_ptr get_notification( - const DoutPrefixProvider* dpp, - rgw::sal::Object* obj, - rgw::sal::Object* src_obj, - rgw::notify::EventType event_type, - rgw::sal::Bucket* _bucket, - std::string& _user_id, - std::string& _user_tenant, - std::string& _req_id, - optional_yield y) override; + const DoutPrefixProvider* dpp, + rgw::sal::Object* obj, + rgw::sal::Object* src_obj, + const rgw::notify::EventTypeList& event_type, + rgw::sal::Bucket* _bucket, + std::string& _user_id, + std::string& _user_tenant, + std::string& _req_id, + optional_yield y) override; /* Internal APIs */ int get_root_fd() { return root_fd; } diff --git a/src/rgw/driver/rados/rgw_cr_rados.cc b/src/rgw/driver/rados/rgw_cr_rados.cc index fb5722528c398..a6d4719e71864 100644 --- a/src/rgw/driver/rados/rgw_cr_rados.cc +++ b/src/rgw/driver/rados/rgw_cr_rados.cc @@ -787,40 +787,52 @@ int RGWAsyncFetchRemoteObj::_send_request(const DoutPrefixProvider *dpp) // send notification that object was successfully synced std::string user_id = "rgw sync"; std::string req_id = "0"; - + RGWObjTags obj_tags; auto iter = attrs.find(RGW_ATTR_TAGS); if (iter != attrs.end()) { try { auto it = iter->second.cbegin(); obj_tags.decode(it); - } catch (buffer::error &err) { - ldpp_dout(dpp, 1) << "ERROR: " << __func__ << ": caught buffer::error couldn't decode TagSet " << dendl; + } catch (buffer::error& err) { + ldpp_dout(dpp, 1) + << "ERROR: " << __func__ + << ": caught buffer::error couldn't decode TagSet " << dendl; } } - // NOTE: we create a mutable copy of bucket.get_tenant as the get_notification function expects a std::string&, not const + // NOTE: we create a mutable copy of bucket.get_tenant as the + // get_notification function expects a std::string&, not const std::string tenant(dest_bucket.get_tenant()); - std::unique_ptr notify - = store->get_notification(dpp, &dest_obj, nullptr, rgw::notify::ObjectSyncedCreate, - &dest_bucket, user_id, - tenant, - req_id, null_yield); - - auto notify_res = static_cast(notify.get())->get_reservation(); - int ret = rgw::notify::publish_reserve(dpp, *store->svc()->site, rgw::notify::ObjectSyncedCreate, notify_res, &obj_tags); + std::unique_ptr notify = + store->get_notification( + dpp, &dest_obj, nullptr, {rgw::notify::ObjectSyncedCreate}, + &dest_bucket, user_id, tenant, req_id, null_yield); + + auto notify_res = + static_cast(notify.get()) + ->get_reservation(); + int ret = rgw::notify::publish_reserve( + dpp, *store->svc()->site, {rgw::notify::ObjectSyncedCreate}, + notify_res, &obj_tags); if (ret < 0) { - ldpp_dout(dpp, 1) << "ERROR: reserving notification failed, with error: " << ret << dendl; + ldpp_dout(dpp, 1) + << "ERROR: reserving notification failed, with error: " << ret + << dendl; // no need to return, the sync already happened } else { - ret = rgw::notify::publish_commit(&dest_obj, *bytes_transferred, ceph::real_clock::now(), etag, dest_obj.get_instance(), rgw::notify::ObjectSyncedCreate, notify_res, dpp); + ret = rgw::notify::publish_commit( + &dest_obj, *bytes_transferred, ceph::real_clock::now(), etag, + dest_obj.get_instance(), notify_res, dpp); if (ret < 0) { - ldpp_dout(dpp, 1) << "ERROR: publishing notification failed, with error: " << ret << dendl; + ldpp_dout(dpp, 1) + << "ERROR: publishing notification failed, with error: " << ret + << dendl; } } } - + if (counters) { if (bytes_transferred) { counters->inc(sync_counters::l_fetch, *bytes_transferred); diff --git a/src/rgw/driver/rados/rgw_notify.cc b/src/rgw/driver/rados/rgw_notify.cc index 3fe441eec14ee..b2e91e3a77afe 100644 --- a/src/rgw/driver/rados/rgw_notify.cc +++ b/src/rgw/driver/rados/rgw_notify.cc @@ -981,12 +981,11 @@ static inline bool notification_match(reservation_t& res, return true; } - int publish_reserve(const DoutPrefixProvider* dpp, - const SiteConfig& site, - EventType event_type, - reservation_t& res, - const RGWObjTags* req_tags) -{ +int publish_reserve(const DoutPrefixProvider* dpp, + const SiteConfig& site, + const EventTypeList& event_types, + reservation_t& res, + const RGWObjTags* req_tags) { rgw_pubsub_bucket_topics bucket_topics; if (all_zonegroups_support(site, zone_features::notification_v2) && res.store->stat_topics_v1(res.user_tenant, res.yield, res.dpp) == -ENOENT) { @@ -1020,64 +1019,71 @@ static inline bool notification_match(reservation_t& res, for (const auto& bucket_topic : bucket_topics.topics) { const rgw_pubsub_topic_filter& topic_filter = bucket_topic.second; const rgw_pubsub_topic& topic_cfg = topic_filter.topic; - if (!notification_match(res, topic_filter, event_type, req_tags)) { - // notification does not apply to req_state - continue; - } - ldpp_dout(res.dpp, 20) << "INFO: notification: '" << topic_filter.s3_id << - "' on topic: '" << topic_cfg.dest.arn_topic << - "' and bucket: '" << res.bucket->get_name() << - "' (unique topic: '" << topic_cfg.name << - "') apply to event of type: '" << to_string(event_type) << "'" << dendl; - - cls_2pc_reservation::id_t res_id = cls_2pc_reservation::NO_ID; - if (topic_cfg.dest.persistent) { - // TODO: take default reservation size from conf - constexpr auto DEFAULT_RESERVATION = 4*1024U; // 4K - res.size = DEFAULT_RESERVATION; - librados::ObjectWriteOperation op; - bufferlist obl; - int rval; - const auto& queue_name = topic_cfg.dest.arn_topic; - cls_2pc_queue_reserve(op, res.size, 1, &obl, &rval); - auto ret = rgw_rados_operate( - res.dpp, res.store->getRados()->get_notif_pool_ctx(), - queue_name, &op, res.yield, librados::OPERATION_RETURNVEC); - if (ret < 0) { - ldpp_dout(res.dpp, 1) << - "ERROR: failed to reserve notification on queue: " - << queue_name << ". error: " << ret << dendl; - // if no space is left in queue we ask client to slow down - return (ret == -ENOSPC) ? -ERR_RATE_LIMITED : ret; + for (auto& event_type : event_types) { + if (!notification_match(res, topic_filter, event_type, req_tags)) { + // notification does not apply to req_state + continue; } - ret = cls_2pc_queue_reserve_result(obl, res_id); - if (ret < 0) { - ldpp_dout(res.dpp, 1) << "ERROR: failed to parse reservation id. error: " << ret << dendl; - return ret; + ldpp_dout(res.dpp, 20) + << "INFO: notification: '" << topic_filter.s3_id << "' on topic: '" + << topic_cfg.dest.arn_topic << "' and bucket: '" + << res.bucket->get_name() << "' (unique topic: '" << topic_cfg.name + << "') apply to event of type: '" << to_string(event_type) << "'" + << dendl; + + cls_2pc_reservation::id_t res_id = cls_2pc_reservation::NO_ID; + if (topic_cfg.dest.persistent) { + // TODO: take default reservation size from conf + constexpr auto DEFAULT_RESERVATION = 4 * 1024U; // 4K + res.size = DEFAULT_RESERVATION; + librados::ObjectWriteOperation op; + bufferlist obl; + int rval; + const auto& queue_name = topic_cfg.dest.arn_topic; + cls_2pc_queue_reserve(op, res.size, 1, &obl, &rval); + auto ret = rgw_rados_operate( + res.dpp, res.store->getRados()->get_notif_pool_ctx(), queue_name, + &op, res.yield, librados::OPERATION_RETURNVEC); + if (ret < 0) { + ldpp_dout(res.dpp, 1) + << "ERROR: failed to reserve notification on queue: " + << queue_name << ". error: " << ret << dendl; + // if no space is left in queue we ask client to slow down + return (ret == -ENOSPC) ? -ERR_RATE_LIMITED : ret; + } + ret = cls_2pc_queue_reserve_result(obl, res_id); + if (ret < 0) { + ldpp_dout(res.dpp, 1) + << "ERROR: failed to parse reservation id. error: " << ret + << dendl; + return ret; + } } - } - // load the topic,if there is change in topic config while it's stored in - // notification. - rgw_pubsub_topic result; - const RGWPubSub ps(res.store, res.user_tenant, site); - auto ret = ps.get_topic(res.dpp, topic_cfg.name, result, res.yield, nullptr); - if (ret < 0) { - ldpp_dout(res.dpp, 1) - << "INFO: failed to load topic: " << topic_cfg.name - << ". error: " << ret - << " while reserving persistent notification event" << dendl; - if (ret == -ENOENT) { - // either the topic is deleted but the corresponding notification still - // exist or in v2 mode the notification could have synced first but - // topic is not synced yet. - return 0; + // load the topic,if there is change in topic config while it's stored in + // notification. + rgw_pubsub_topic result; + const RGWPubSub ps(res.store, res.user_tenant, site); + auto ret = + ps.get_topic(res.dpp, topic_cfg.name, result, res.yield, nullptr); + if (ret < 0) { + ldpp_dout(res.dpp, 1) + << "INFO: failed to load topic: " << topic_cfg.name + << ". error: " << ret + << " while reserving persistent notification event" << dendl; + if (ret == -ENOENT) { + // either the topic is deleted but the corresponding notification + // still exist or in v2 mode the notification could have synced first + // but topic is not synced yet. + return 0; + } + ldpp_dout(res.dpp, 1) + << "WARN: Using the stored topic from bucket notification struct." + << dendl; + res.topics.emplace_back(topic_filter.s3_id, topic_cfg, res_id, + event_type); + } else { + res.topics.emplace_back(topic_filter.s3_id, result, res_id, event_type); } - ldpp_dout(res.dpp, 1) - << "WARN: Using the stored topic from bucket notification struct." - << dendl; - res.topics.emplace_back(topic_filter.s3_id, topic_cfg, res_id); - } else { - res.topics.emplace_back(topic_filter.s3_id, result, res_id); } } return 0; @@ -1088,7 +1094,6 @@ int publish_commit(rgw::sal::Object* obj, const ceph::real_time& mtime, const std::string& etag, const std::string& version, - EventType event_type, reservation_t& res, const DoutPrefixProvider* dpp) { @@ -1099,7 +1104,8 @@ int publish_commit(rgw::sal::Object* obj, continue; } event_entry_t event_entry; - populate_event(res, obj, size, mtime, etag, version, event_type, event_entry.event); + populate_event(res, obj, size, mtime, etag, version, topic.event_type, + event_entry.event); event_entry.event.configurationId = topic.configurationId; event_entry.event.opaque_data = topic.cfg.opaque_data; if (topic.cfg.dest.persistent) { diff --git a/src/rgw/driver/rados/rgw_notify.h b/src/rgw/driver/rados/rgw_notify.h index 4f2f57341ecb2..ec8117c2f573b 100644 --- a/src/rgw/driver/rados/rgw_notify.h +++ b/src/rgw/driver/rados/rgw_notify.h @@ -49,13 +49,18 @@ int remove_persistent_topic(const DoutPrefixProvider* dpp, librados::IoCtx& rado struct reservation_t { struct topic_t { topic_t(const std::string& _configurationId, const rgw_pubsub_topic& _cfg, - cls_2pc_reservation::id_t _res_id) : - configurationId(_configurationId), cfg(_cfg), res_id(_res_id) {} + cls_2pc_reservation::id_t _res_id, + rgw::notify::EventType _event_type) + : configurationId(_configurationId), + cfg(_cfg), + res_id(_res_id), + event_type(_event_type) {} const std::string configurationId; const rgw_pubsub_topic cfg; // res_id is reset after topic is committed/aborted cls_2pc_reservation::id_t res_id; + rgw::notify::EventType event_type; }; const DoutPrefixProvider* const dpp; @@ -112,10 +117,10 @@ struct rgw_topic_stats { // create a reservation on the 2-phase-commit queue int publish_reserve(const DoutPrefixProvider *dpp, - const SiteConfig& site, - EventType event_type, - reservation_t& reservation, - const RGWObjTags* req_tags); + const SiteConfig& site, + const EventTypeList& event_types, + reservation_t& reservation, + const RGWObjTags* req_tags); // commit the reservation to the queue int publish_commit(rgw::sal::Object* obj, @@ -123,7 +128,6 @@ int publish_commit(rgw::sal::Object* obj, const ceph::real_time& mtime, const std::string& etag, const std::string& version, - EventType event_type, reservation_t& reservation, const DoutPrefixProvider *dpp); diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc index 193dfb007e805..597f4f1ccf29f 100644 --- a/src/rgw/driver/rados/rgw_sal_rados.cc +++ b/src/rgw/driver/rados/rgw_sal_rados.cc @@ -1084,9 +1084,19 @@ std::unique_ptr RadosStore::get_notification( return std::make_unique(s, this, obj, src_obj, s, event_type, y, object_name); } -std::unique_ptr RadosStore::get_notification(const DoutPrefixProvider* dpp, rgw::sal::Object* obj, rgw::sal::Object* src_obj, rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket, std::string& _user_id, std::string& _user_tenant, std::string& _req_id, optional_yield y) -{ - return std::make_unique(dpp, this, obj, src_obj, event_type, _bucket, _user_id, _user_tenant, _req_id, y); +std::unique_ptr RadosStore::get_notification( + const DoutPrefixProvider* dpp, + rgw::sal::Object* obj, + rgw::sal::Object* src_obj, + const rgw::notify::EventTypeList& event_types, + rgw::sal::Bucket* _bucket, + std::string& _user_id, + std::string& _user_tenant, + std::string& _req_id, + optional_yield y) { + return std::make_unique(dpp, this, obj, src_obj, + event_types, _bucket, _user_id, + _user_tenant, _req_id, y); } std::string RadosStore::topics_oid(const std::string& tenant) const { @@ -3007,13 +3017,13 @@ std::unique_ptr RadosLifecycle::get_serializer(const std::string& int RadosNotification::publish_reserve(const DoutPrefixProvider *dpp, RGWObjTags* obj_tags) { - return rgw::notify::publish_reserve(dpp, *store->svc()->site, event_type, res, obj_tags); + return rgw::notify::publish_reserve(dpp, *store->svc()->site, event_types, res, obj_tags); } int RadosNotification::publish_commit(const DoutPrefixProvider* dpp, uint64_t size, const ceph::real_time& mtime, const std::string& etag, const std::string& version) { - return rgw::notify::publish_commit(obj, size, mtime, etag, version, event_type, res, dpp); + return rgw::notify::publish_commit(obj, size, mtime, etag, version, res, dpp); } int RadosAtomicWriter::prepare(optional_yield y) diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h index c97d5e1832d47..4e71045cda1e5 100644 --- a/src/rgw/driver/rados/rgw_sal_rados.h +++ b/src/rgw/driver/rados/rgw_sal_rados.h @@ -155,9 +155,15 @@ class RadosStore : public StoreDriver { virtual std::unique_ptr get_lifecycle(void) override; virtual std::unique_ptr get_notification(rgw::sal::Object* obj, rgw::sal::Object* src_obj, req_state* s, rgw::notify::EventType event_type, optional_yield y, const std::string* object_name=nullptr) override; virtual std::unique_ptr get_notification( - const DoutPrefixProvider* dpp, rgw::sal::Object* obj, rgw::sal::Object* src_obj, - rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket, std::string& _user_id, std::string& _user_tenant, - std::string& _req_id, optional_yield y) override; + const DoutPrefixProvider* dpp, + rgw::sal::Object* obj, + rgw::sal::Object* src_obj, + const rgw::notify::EventTypeList& event_types, + rgw::sal::Bucket* _bucket, + std::string& _user_id, + std::string& _user_tenant, + std::string& _req_id, + optional_yield y) override; int read_topics(const std::string& tenant, rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker, optional_yield y, const DoutPrefixProvider *dpp) override; int stat_topics_v1(const std::string& tenant, optional_yield y, const DoutPrefixProvider *dpp) override; @@ -733,13 +739,41 @@ class RadosNotification : public StoreNotification { rgw::notify::reservation_t res; public: - RadosNotification(const DoutPrefixProvider* _dpp, RadosStore* _store, Object* _obj, Object* _src_obj, req_state* _s, rgw::notify::EventType _type, optional_yield y, const std::string* object_name) : - StoreNotification(_obj, _src_obj, _type), store(_store), res(_dpp, _store, _s, _obj, _src_obj, object_name, y) { } - - RadosNotification(const DoutPrefixProvider* _dpp, RadosStore* _store, Object* _obj, Object* _src_obj, rgw::notify::EventType _type, rgw::sal::Bucket* _bucket, std::string& _user_id, std::string& _user_tenant, std::string& _req_id, optional_yield y) : - StoreNotification(_obj, _src_obj, _type), store(_store), res(_dpp, _store, _obj, _src_obj, _bucket, _user_id, _user_tenant, _req_id, y) {} - - ~RadosNotification() = default; + RadosNotification(const DoutPrefixProvider* _dpp, + RadosStore* _store, + Object* _obj, + Object* _src_obj, + req_state* _s, + rgw::notify::EventType _type, + optional_yield y, + const std::string* object_name) + : StoreNotification(_obj, _src_obj, {_type}), + store(_store), + res(_dpp, _store, _s, _obj, _src_obj, object_name, y) {} + + RadosNotification(const DoutPrefixProvider* _dpp, + RadosStore* _store, + Object* _obj, + Object* _src_obj, + const rgw::notify::EventTypeList& _types, + rgw::sal::Bucket* _bucket, + std::string& _user_id, + std::string& _user_tenant, + std::string& _req_id, + optional_yield y) + : StoreNotification(_obj, _src_obj, _types), + store(_store), + res(_dpp, + _store, + _obj, + _src_obj, + _bucket, + _user_id, + _user_tenant, + _req_id, + y) {} + + ~RadosNotification() = default; rgw::notify::reservation_t& get_reservation(void) { return res; diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index 7169e09edbc39..deb74465600b2 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -561,10 +561,10 @@ static bool zonegroup_lc_check(const DoutPrefixProvider *dpp, rgw::sal::Zone* zo }); } -static int remove_expired_obj( - const DoutPrefixProvider *dpp, lc_op_ctx& oc, bool remove_indeed, - rgw::notify::EventType event_type) -{ +static int remove_expired_obj(const DoutPrefixProvider* dpp, + lc_op_ctx& oc, + bool remove_indeed, + const rgw::notify::EventTypeList& event_types) { auto& driver = oc.driver; auto& bucket_info = oc.bucket->get_info(); auto& o = oc.o; @@ -601,10 +601,9 @@ static int remove_expired_obj( del_op->params.unmod_since = meta.mtime; // notification supported only for RADOS driver for now - notify = driver->get_notification(dpp, obj.get(), nullptr, event_type, - oc.bucket, lc_id, - const_cast(oc.bucket->get_tenant()), - lc_req_id, null_yield); + notify = driver->get_notification( + dpp, obj.get(), nullptr, event_types, oc.bucket, lc_id, + const_cast(oc.bucket->get_tenant()), lc_req_id, null_yield); ret = notify->publish_reserve(dpp, nullptr); if ( ret < 0) { @@ -898,10 +897,9 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target, std::unique_ptr notify = driver->get_notification( - this, sal_obj.get(), nullptr, event_type, - target, lc_id, - const_cast(target->get_tenant()), - lc_req_id, null_yield); + this, sal_obj.get(), nullptr, {event_type}, target, lc_id, + const_cast(target->get_tenant()), lc_req_id, + null_yield); auto version_id = obj.key.instance; ret = notify->publish_reserve(this, nullptr); @@ -1175,8 +1173,10 @@ public: auto& o = oc.o; int r; if (o.is_delete_marker()) { - r = remove_expired_obj(oc.dpp, oc, true, - rgw::notify::ObjectExpirationDeleteMarker); + r = remove_expired_obj( + oc.dpp, oc, true, + {rgw::notify::ObjectExpirationDeleteMarker, + rgw::notify::LifecycleExpirationDeleteMarkerCreated}); if (r < 0) { ldpp_dout(oc.dpp, 0) << "ERROR: current is-dm remove_expired_obj " << oc.bucket << ":" << o.key @@ -1190,7 +1190,8 @@ public: } else { /* ! o.is_delete_marker() */ r = remove_expired_obj(oc.dpp, oc, !oc.bucket->versioned(), - rgw::notify::ObjectExpirationCurrent); + {rgw::notify::ObjectExpirationCurrent, + rgw::notify::LifecycleExpirationDelete}); if (r < 0) { ldpp_dout(oc.dpp, 0) << "ERROR: remove_expired_obj " << oc.bucket << ":" << o.key @@ -1244,7 +1245,7 @@ public: int process(lc_op_ctx& oc) override { auto& o = oc.o; int r = remove_expired_obj(oc.dpp, oc, true, - rgw::notify::ObjectExpirationNoncurrent); + {rgw::notify::ObjectExpirationNoncurrent}); if (r < 0) { ldpp_dout(oc.dpp, 0) << "ERROR: remove_expired_obj (non-current expiration) " << oc.bucket << ":" << o.key @@ -1289,7 +1290,8 @@ public: int process(lc_op_ctx& oc) override { auto& o = oc.o; int r = remove_expired_obj(oc.dpp, oc, true, - rgw::notify::ObjectExpirationDeleteMarker); + {rgw::notify::ObjectExpirationDeleteMarker, + rgw::notify::LifecycleExpirationDeleteMarkerCreated}); if (r < 0) { ldpp_dout(oc.dpp, 0) << "ERROR: remove_expired_obj (delete marker expiration) " << oc.bucket << ":" << o.key @@ -1370,21 +1372,23 @@ public: /* If bucket is versioned, create delete_marker for current version */ if (! oc.bucket->versioned()) { - ret = remove_expired_obj(oc.dpp, oc, true, rgw::notify::ObjectTransition); + ret = + remove_expired_obj(oc.dpp, oc, true, {rgw::notify::ObjectTransition}); ldpp_dout(oc.dpp, 20) << "delete_tier_obj Object(key:" << oc.o.key << ") not versioned flags: " << oc.o.flags << dendl; } else { /* versioned */ if (oc.o.is_current() && !oc.o.is_delete_marker()) { ret = remove_expired_obj(oc.dpp, oc, false, - rgw::notify::ObjectTransitionCurrent); + {rgw::notify::ObjectTransitionCurrent, + rgw::notify::LifecycleTransition}); ldpp_dout(oc.dpp, 20) << "delete_tier_obj Object(key:" << oc.o.key << ") current & not delete_marker" << " versioned_epoch: " << oc.o.versioned_epoch << "flags: " << oc.o.flags << dendl; } else { ret = remove_expired_obj(oc.dpp, oc, true, - rgw::notify::ObjectTransitionNoncurrent); + {rgw::notify::ObjectTransitionNoncurrent}); ldpp_dout(oc.dpp, 20) << "delete_tier_obj Object(key:" << oc.o.key << ") not current " << "versioned_epoch: " << oc.o.versioned_epoch @@ -1411,17 +1415,20 @@ public: return ret; } - const auto event_type = (bucket->versioned() && - oc.o.is_current() && !oc.o.is_delete_marker()) ? - rgw::notify::ObjectTransitionCurrent : - rgw::notify::ObjectTransitionNoncurrent; + rgw::notify::EventTypeList event_types; + if (bucket->versioned() && oc.o.is_current() && !oc.o.is_delete_marker()) { + event_types.insert(event_types.end(), + {rgw::notify::ObjectTransitionCurrent, + rgw::notify::LifecycleTransition}); + } else { + event_types.push_back(rgw::notify::ObjectTransitionNoncurrent); + } - std::unique_ptr notify - = oc.driver->get_notification( - oc.dpp, obj.get(), nullptr, event_type, - bucket, lc_id, - const_cast(oc.bucket->get_tenant()), - lc_req_id, null_yield); + std::unique_ptr notify = + oc.driver->get_notification( + oc.dpp, obj.get(), nullptr, event_types, bucket, lc_id, + const_cast(oc.bucket->get_tenant()), lc_req_id, + null_yield); auto version_id = oc.o.key.instance; ret = notify->publish_reserve(oc.dpp, nullptr); diff --git a/src/rgw/rgw_notify_event_type.cc b/src/rgw/rgw_notify_event_type.cc index 7a0ef9568e15a..d36e10c9c2d3b 100644 --- a/src/rgw/rgw_notify_event_type.cc +++ b/src/rgw/rgw_notify_event_type.cc @@ -50,6 +50,14 @@ namespace rgw::notify { return "s3:ObjectSynced:Delete"; case ObjectSyncedDeletionMarkerCreated: return "s3:ObjectSynced:DeletionMarkerCreated"; + case LifecycleExpiration: + return "s3:LifecycleExpiration:*"; + case LifecycleExpirationDelete: + return "s3:LifecycleExpiration:Delete"; + case LifecycleExpirationDeleteMarkerCreated: + return "s3:LifecycleExpiration:DeleteMarkerCreated"; + case LifecycleTransition: + return "s3:LifecycleTransition"; case UnknownEvent: return "s3:UnknownEvent"; } @@ -103,6 +111,14 @@ namespace rgw::notify { return ObjectSyncedDelete; if (s == "s3:ObjectSynced:DeletionMarkerCreated") return ObjectSyncedDeletionMarkerCreated; + if (s == "s3:LifecycleExpiration:*") + return LifecycleExpiration; + if (s == "s3:LifecycleExpiration:Delete") + return LifecycleExpirationDelete; + if (s == "s3:LifecycleExpiration:DeleteMarkerCreated") + return LifecycleExpirationDeleteMarkerCreated; + if (s == "s3:LifecycleTransition") + return LifecycleTransition; return UnknownEvent; } diff --git a/src/rgw/rgw_notify_event_type.h b/src/rgw/rgw_notify_event_type.h index 4fe1b5c90c624..d7f70682d1627 100644 --- a/src/rgw/rgw_notify_event_type.h +++ b/src/rgw/rgw_notify_event_type.h @@ -29,8 +29,12 @@ namespace rgw::notify { ObjectSyncedCreate = 0x10000, ObjectSyncedDelete = 0x20000, ObjectSyncedDeletionMarkerCreated = 0x40000, - UnknownEvent = 0x100000 - }; + LifecycleExpiration = 0xF00000, + LifecycleExpirationDelete = 0x100000, + LifecycleExpirationDeleteMarkerCreated = 0x200000, + LifecycleTransition = 0xF000000, + UnknownEvent = 0x10000000 +}; using EventTypeList = std::vector; diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index 7202d9c90dca9..e21a6180f8b4f 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -302,9 +302,15 @@ class Driver { rgw::notify::EventType event_type, optional_yield y, const std::string* object_name=nullptr) = 0; /** No-req_state variant (e.g., rgwlc) */ virtual std::unique_ptr get_notification( - const DoutPrefixProvider* dpp, rgw::sal::Object* obj, rgw::sal::Object* src_obj, - rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket, std::string& _user_id, std::string& _user_tenant, - std::string& _req_id, optional_yield y) = 0; + const DoutPrefixProvider* dpp, + rgw::sal::Object* obj, + rgw::sal::Object* src_obj, + const rgw::notify::EventTypeList& event_types, + rgw::sal::Bucket* _bucket, + std::string& _user_id, + std::string& _user_tenant, + std::string& _req_id, + optional_yield y) = 0; /** Read the topic config entry into @a data and (optionally) @a objv_tracker */ virtual int read_topics(const std::string& tenant, rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker, optional_yield y, const DoutPrefixProvider *dpp) = 0; diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc index 2dc9fc95d105d..0019178f640f8 100644 --- a/src/rgw/rgw_sal_dbstore.cc +++ b/src/rgw/rgw_sal_dbstore.cc @@ -1662,17 +1662,21 @@ namespace rgw::sal { rgw::notify::EventType event_type, optional_yield y, const std::string* object_name) { - return std::make_unique(obj, src_obj, event_type); + rgw::notify::EventTypeList event_types = {event_type}; + return std::make_unique(obj, src_obj, event_types); } std::unique_ptr DBStore::get_notification( - const DoutPrefixProvider* dpp, rgw::sal::Object* obj, - rgw::sal::Object* src_obj, - rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket, - std::string& _user_id, std::string& _user_tenant, std::string& _req_id, - optional_yield y) - { - return std::make_unique(obj, src_obj, event_type); + const DoutPrefixProvider* dpp, + rgw::sal::Object* obj, + rgw::sal::Object* src_obj, + const rgw::notify::EventTypeList& event_types, + rgw::sal::Bucket* _bucket, + std::string& _user_id, + std::string& _user_tenant, + std::string& _req_id, + optional_yield y) { + return std::make_unique(obj, src_obj, event_types); } RGWLC* DBStore::get_rgwlc(void) { diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h index 3c0c7c765198b..6ce6398d0626e 100644 --- a/src/rgw/rgw_sal_dbstore.h +++ b/src/rgw/rgw_sal_dbstore.h @@ -63,11 +63,13 @@ public: class DBNotification : public StoreNotification { protected: public: - DBNotification(Object* _obj, Object* _src_obj, rgw::notify::EventType _type) - : StoreNotification(_obj, _src_obj, _type) {} - ~DBNotification() = default; + DBNotification(Object* _obj, + Object* _src_obj, + const rgw::notify::EventTypeList& _types) + : StoreNotification(_obj, _src_obj, _types) {} + ~DBNotification() = default; - virtual int publish_reserve(const DoutPrefixProvider *dpp, RGWObjTags* obj_tags = nullptr) override { return 0;} + virtual int publish_reserve(const DoutPrefixProvider *dpp, RGWObjTags* obj_tags = nullptr) override { return 0;} virtual int publish_commit(const DoutPrefixProvider* dpp, uint64_t size, const ceph::real_time& mtime, const std::string& etag, const std::string& version) override { return 0; } }; @@ -765,13 +767,17 @@ public: rgw::notify::EventType event_type, optional_yield y, const std::string* object_name) override; virtual std::unique_ptr get_notification( - const DoutPrefixProvider* dpp, rgw::sal::Object* obj, - rgw::sal::Object* src_obj, - rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket, - std::string& _user_id, std::string& _user_tenant, std::string& _req_id, - optional_yield y) override; - - virtual RGWLC* get_rgwlc(void) override; + const DoutPrefixProvider* dpp, + rgw::sal::Object* obj, + rgw::sal::Object* src_obj, + const rgw::notify::EventTypeList& event_types, + rgw::sal::Bucket* _bucket, + std::string& _user_id, + std::string& _user_tenant, + std::string& _req_id, + optional_yield y) override; + + virtual RGWLC* get_rgwlc(void) override; virtual RGWCoroutinesManagerRegistry* get_cr_registry() override { return NULL; } virtual int log_usage(const DoutPrefixProvider *dpp, std::map& usage_info, optional_yield y) override; virtual int log_op(const DoutPrefixProvider *dpp, std::string& oid, bufferlist& bl) override; diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc index 6e0ca8ccd8210..94b922acff48c 100644 --- a/src/rgw/rgw_sal_filter.cc +++ b/src/rgw/rgw_sal_filter.cc @@ -228,20 +228,19 @@ std::unique_ptr FilterDriver::get_notification(rgw::sal::Object* o return std::make_unique(std::move(n)); } -std::unique_ptr FilterDriver::get_notification(const DoutPrefixProvider* dpp, - rgw::sal::Object* obj, rgw::sal::Object* src_obj, - rgw::notify::EventType event_type, - rgw::sal::Bucket* _bucket, std::string& _user_id, - std::string& _user_tenant, std::string& _req_id, - optional_yield y) -{ - std::unique_ptr n = next->get_notification(dpp, nextObject(obj), - nextObject(src_obj), - event_type, - nextBucket(_bucket), - _user_id, - _user_tenant, - _req_id, y); +std::unique_ptr FilterDriver::get_notification( + const DoutPrefixProvider* dpp, + rgw::sal::Object* obj, + rgw::sal::Object* src_obj, + const rgw::notify::EventTypeList& event_types, + rgw::sal::Bucket* _bucket, + std::string& _user_id, + std::string& _user_tenant, + std::string& _req_id, + optional_yield y) { + std::unique_ptr n = next->get_notification( + dpp, nextObject(obj), nextObject(src_obj), event_types, + nextBucket(_bucket), _user_id, _user_tenant, _req_id, y); return std::make_unique(std::move(n)); } diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h index 5095f675f1625..b07179aa44e5a 100644 --- a/src/rgw/rgw_sal_filter.h +++ b/src/rgw/rgw_sal_filter.h @@ -176,11 +176,15 @@ public: rgw::notify::EventType event_type, optional_yield y, const std::string* object_name=nullptr) override; virtual std::unique_ptr get_notification( - const DoutPrefixProvider* dpp, rgw::sal::Object* obj, rgw::sal::Object* src_obj, - - rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket, - std::string& _user_id, std::string& _user_tenant, - std::string& _req_id, optional_yield y) override; + const DoutPrefixProvider* dpp, + rgw::sal::Object* obj, + rgw::sal::Object* src_obj, + const rgw::notify::EventTypeList& event_types, + rgw::sal::Bucket* _bucket, + std::string& _user_id, + std::string& _user_tenant, + std::string& _req_id, + optional_yield y) override; int read_topics(const std::string& tenant, rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker, optional_yield y, const DoutPrefixProvider *dpp) override { diff --git a/src/rgw/rgw_sal_store.h b/src/rgw/rgw_sal_store.h index 23fc3eb768565..1ba44bc02ec4a 100644 --- a/src/rgw/rgw_sal_store.h +++ b/src/rgw/rgw_sal_store.h @@ -408,14 +408,15 @@ class StoreNotification : public Notification { protected: Object* obj; Object* src_obj; - rgw::notify::EventType event_type; + rgw::notify::EventTypeList event_types; - public: - StoreNotification(Object* _obj, Object* _src_obj, rgw::notify::EventType _type) - : obj(_obj), src_obj(_src_obj), event_type(_type) - {} + public: + StoreNotification(Object* _obj, + Object* _src_obj, + rgw::notify::EventTypeList _types) + : obj(_obj), src_obj(_src_obj), event_types(std::move(_types)) {} - virtual ~StoreNotification() = default; + virtual ~StoreNotification() = default; }; class StoreWriter : public Writer { diff --git a/src/test/rgw/bucket_notification/test_bn.py b/src/test/rgw/bucket_notification/test_bn.py index 8e9ebd126c633..c9049dfd1f8fa 100644 --- a/src/test/rgw/bucket_notification/test_bn.py +++ b/src/test/rgw/bucket_notification/test_bn.py @@ -1907,7 +1907,8 @@ def test_ps_s3_lifecycle_on_master(): notification_name = bucket_name + NOTIFICATION_SUFFIX topic_conf_list = [{'Id': notification_name, 'TopicArn': topic_arn, - 'Events': ['s3:ObjectLifecycle:Expiration:*'] + 'Events': ['s3:ObjectLifecycle:Expiration:*', + 's3:LifecycleExpiration:*'] }] s3_notification_conf = PSNotificationS3(conn, bucket_name, topic_conf_list) response, status = s3_notification_conf.set_config() @@ -1927,14 +1928,14 @@ def test_ps_s3_lifecycle_on_master(): time_diff = time.time() - start_time print('average time for creation + http notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds') - + # create lifecycle policy client = boto3.client('s3', endpoint_url='http://'+conn.host+':'+str(conn.port), aws_access_key_id=conn.aws_access_key_id, aws_secret_access_key=conn.aws_secret_access_key) yesterday = datetime.date.today() - datetime.timedelta(days=1) - response = client.put_bucket_lifecycle_configuration(Bucket=bucket_name, + response = client.put_bucket_lifecycle_configuration(Bucket=bucket_name, LifecycleConfiguration={'Rules': [ { 'ID': 'rule1', @@ -1956,8 +1957,11 @@ def test_ps_s3_lifecycle_on_master(): print('total number of objects: ' + str(len(keys))) event_keys = [] events = http_server.get_and_reset_events() + assert_equal(number_of_objects * 2, len(events)) for event in events: - assert_equal(event['Records'][0]['eventName'], 'ObjectLifecycle:Expiration:Current') + assert_in(event['Records'][0]['eventName'], + ['LifecycleExpiration:Delete', + 'ObjectLifecycle:Expiration:Current']) event_keys.append(event['Records'][0]['s3']['object']['key']) for key in keys: key_found = False