bucket notification configuration is currently stored in system objects.
adding API to be able to store it in any sotre.
note that other aspects of sending bucket notifications via the
cls_2pc_queu are still only supported in RADOS.
also making sure that all APIs use yielding correctly.
Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
rgw_notify_event_type.cc
rgw_period_history.cc
rgw_period_puller.cc
+ rgw_pubsub.cc
rgw_coroutine.cc
rgw_cr_rest.cc
rgw_op.cc
rgw_rest_ratelimit.cc
rgw_rest_role.cc
rgw_rest_s3.cc
+ rgw_rest_pubsub.cc
rgw_s3select.cc
rgw_role.cc
rgw_sal.cc
driver/rados/rgw_object_expirer_core.cc
driver/rados/rgw_otp.cc
driver/rados/rgw_period.cc
- driver/rados/rgw_pubsub.cc
driver/rados/rgw_pubsub_push.cc
driver/rados/rgw_putobj_processor.cc
driver/rados/rgw_rados.cc
driver/rados/rgw_reshard.cc
driver/rados/rgw_rest_bucket.cc
driver/rados/rgw_rest_log.cc
- driver/rados/rgw_rest_pubsub.cc
driver/rados/rgw_rest_realm.cc
driver/rados/rgw_rest_user.cc
driver/rados/rgw_sal_rados.cc
const RGWObjTags* req_tags)
{
const RGWPubSub ps(res.store, res.user_tenant);
- const RGWPubSub::Bucket ps_bucket(ps, res.bucket->get_key());
+ const RGWPubSub::Bucket ps_bucket(ps, res.bucket);
rgw_pubsub_bucket_topics bucket_topics;
- auto rc = ps_bucket.get_topics(&bucket_topics);
+ auto rc = ps_bucket.get_topics(res.dpp, &bucket_topics, res.yield);
if (rc < 0) {
// failed to fetch bucket topics
return rc;
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab ft=cpp
-
-#include "services/svc_zone.h"
-#include "rgw_b64.h"
-#include "rgw_sal.h"
-#include "rgw_sal_rados.h"
-#include "rgw_pubsub.h"
-#include "rgw_tools.h"
-#include "rgw_xml.h"
-#include "rgw_arn.h"
-#include "rgw_pubsub_push.h"
-#include <regex>
-#include <algorithm>
-
-#define dout_subsys ceph_subsys_rgw
-
-void set_event_id(std::string& id, const std::string& hash, const utime_t& ts) {
- char buf[64];
- const auto len = snprintf(buf, sizeof(buf), "%010ld.%06ld.%s", (long)ts.sec(), (long)ts.usec(), hash.c_str());
- if (len > 0) {
- id.assign(buf, len);
- }
-}
-
-bool rgw_s3_key_filter::decode_xml(XMLObj* obj) {
- XMLObjIter iter = obj->find("FilterRule");
- XMLObj *o;
-
- const auto throw_if_missing = true;
- auto prefix_not_set = true;
- auto suffix_not_set = true;
- auto regex_not_set = true;
- std::string name;
-
- while ((o = iter.get_next())) {
- RGWXMLDecoder::decode_xml("Name", name, o, throw_if_missing);
- if (name == "prefix" && prefix_not_set) {
- prefix_not_set = false;
- RGWXMLDecoder::decode_xml("Value", prefix_rule, o, throw_if_missing);
- } else if (name == "suffix" && suffix_not_set) {
- suffix_not_set = false;
- RGWXMLDecoder::decode_xml("Value", suffix_rule, o, throw_if_missing);
- } else if (name == "regex" && regex_not_set) {
- regex_not_set = false;
- RGWXMLDecoder::decode_xml("Value", regex_rule, o, throw_if_missing);
- } else {
- throw RGWXMLDecoder::err("invalid/duplicate S3Key filter rule name: '" + name + "'");
- }
- }
- return true;
-}
-
-void rgw_s3_key_filter::dump_xml(Formatter *f) const {
- if (!prefix_rule.empty()) {
- f->open_object_section("FilterRule");
- ::encode_xml("Name", "prefix", f);
- ::encode_xml("Value", prefix_rule, f);
- f->close_section();
- }
- if (!suffix_rule.empty()) {
- f->open_object_section("FilterRule");
- ::encode_xml("Name", "suffix", f);
- ::encode_xml("Value", suffix_rule, f);
- f->close_section();
- }
- if (!regex_rule.empty()) {
- f->open_object_section("FilterRule");
- ::encode_xml("Name", "regex", f);
- ::encode_xml("Value", regex_rule, f);
- f->close_section();
- }
-}
-
-bool rgw_s3_key_filter::has_content() const {
- return !(prefix_rule.empty() && suffix_rule.empty() && regex_rule.empty());
-}
-
-bool rgw_s3_key_value_filter::decode_xml(XMLObj* obj) {
- kv.clear();
- XMLObjIter iter = obj->find("FilterRule");
- XMLObj *o;
-
- const auto throw_if_missing = true;
-
- std::string key;
- std::string value;
-
- while ((o = iter.get_next())) {
- RGWXMLDecoder::decode_xml("Name", key, o, throw_if_missing);
- RGWXMLDecoder::decode_xml("Value", value, o, throw_if_missing);
- kv.emplace(key, value);
- }
- return true;
-}
-
-void rgw_s3_key_value_filter::dump_xml(Formatter *f) const {
- for (const auto& key_value : kv) {
- f->open_object_section("FilterRule");
- ::encode_xml("Name", key_value.first, f);
- ::encode_xml("Value", key_value.second, f);
- f->close_section();
- }
-}
-
-bool rgw_s3_key_value_filter::has_content() const {
- return !kv.empty();
-}
-
-bool rgw_s3_filter::decode_xml(XMLObj* obj) {
- RGWXMLDecoder::decode_xml("S3Key", key_filter, obj);
- RGWXMLDecoder::decode_xml("S3Metadata", metadata_filter, obj);
- RGWXMLDecoder::decode_xml("S3Tags", tag_filter, obj);
- return true;
-}
-
-void rgw_s3_filter::dump_xml(Formatter *f) const {
- if (key_filter.has_content()) {
- ::encode_xml("S3Key", key_filter, f);
- }
- if (metadata_filter.has_content()) {
- ::encode_xml("S3Metadata", metadata_filter, f);
- }
- if (tag_filter.has_content()) {
- ::encode_xml("S3Tags", tag_filter, f);
- }
-}
-
-bool rgw_s3_filter::has_content() const {
- return key_filter.has_content() ||
- metadata_filter.has_content() ||
- tag_filter.has_content();
-}
-
-bool match(const rgw_s3_key_filter& filter, const std::string& key) {
- const auto key_size = key.size();
- const auto prefix_size = filter.prefix_rule.size();
- if (prefix_size != 0) {
- // prefix rule exists
- if (prefix_size > key_size) {
- // if prefix is longer than key, we fail
- return false;
- }
- if (!std::equal(filter.prefix_rule.begin(), filter.prefix_rule.end(), key.begin())) {
- return false;
- }
- }
- const auto suffix_size = filter.suffix_rule.size();
- if (suffix_size != 0) {
- // suffix rule exists
- if (suffix_size > key_size) {
- // if suffix is longer than key, we fail
- return false;
- }
- if (!std::equal(filter.suffix_rule.begin(), filter.suffix_rule.end(), (key.end() - suffix_size))) {
- return false;
- }
- }
- if (!filter.regex_rule.empty()) {
- // TODO add regex chaching in the filter
- const std::regex base_regex(filter.regex_rule);
- if (!std::regex_match(key, base_regex)) {
- return false;
- }
- }
- return true;
-}
-
-bool match(const rgw_s3_key_value_filter& filter, const KeyValueMap& kv) {
- // all filter pairs must exist with the same value in the object's metadata/tags
- // object metadata/tags may include items not in the filter
- return std::includes(kv.begin(), kv.end(), filter.kv.begin(), filter.kv.end());
-}
-
-bool match(const rgw_s3_key_value_filter& filter, const KeyMultiValueMap& kv) {
- // all filter pairs must exist with the same value in the object's metadata/tags
- // object metadata/tags may include items not in the filter
- for (auto& filter : filter.kv) {
- auto result = kv.equal_range(filter.first);
- if (std::any_of(result.first, result.second, [&filter](const std::pair<std::string, std::string>& p) { return p.second == filter.second;}))
- continue;
- else
- return false;
- }
- return true;
-}
-
-bool match(const rgw::notify::EventTypeList& events, rgw::notify::EventType event) {
- // if event list exists, and none of the events in the list matches the event type, filter the message
- if (!events.empty() && std::find(events.begin(), events.end(), event) == events.end()) {
- return false;
- }
- return true;
-}
-
-void do_decode_xml_obj(rgw::notify::EventTypeList& l, const std::string& name, XMLObj *obj) {
- l.clear();
-
- XMLObjIter iter = obj->find(name);
- XMLObj *o;
-
- while ((o = iter.get_next())) {
- std::string val;
- decode_xml_obj(val, o);
- l.push_back(rgw::notify::from_string(val));
- }
-}
-
-bool rgw_pubsub_s3_notification::decode_xml(XMLObj *obj) {
- const auto throw_if_missing = true;
- RGWXMLDecoder::decode_xml("Id", id, obj, throw_if_missing);
-
- RGWXMLDecoder::decode_xml("Topic", topic_arn, obj, throw_if_missing);
-
- RGWXMLDecoder::decode_xml("Filter", filter, obj);
-
- do_decode_xml_obj(events, "Event", obj);
- if (events.empty()) {
- // if no events are provided, we assume all events
- events.push_back(rgw::notify::ObjectCreated);
- events.push_back(rgw::notify::ObjectRemoved);
- }
- return true;
-}
-
-void rgw_pubsub_s3_notification::dump_xml(Formatter *f) const {
- ::encode_xml("Id", id, f);
- ::encode_xml("Topic", topic_arn.c_str(), f);
- if (filter.has_content()) {
- ::encode_xml("Filter", filter, f);
- }
- for (const auto& event : events) {
- ::encode_xml("Event", rgw::notify::to_string(event), f);
- }
-}
-
-bool rgw_pubsub_s3_notifications::decode_xml(XMLObj *obj) {
- do_decode_xml_obj(list, "TopicConfiguration", obj);
- return true;
-}
-
-rgw_pubsub_s3_notification::rgw_pubsub_s3_notification(const rgw_pubsub_topic_filter& topic_filter) :
- id(topic_filter.s3_id), events(topic_filter.events), topic_arn(topic_filter.topic.arn), filter(topic_filter.s3_filter) {}
-
-void rgw_pubsub_s3_notifications::dump_xml(Formatter *f) const {
- do_encode_xml("NotificationConfiguration", list, "TopicConfiguration", f);
-}
-
-void rgw_pubsub_s3_event::dump(Formatter *f) const {
- encode_json("eventVersion", eventVersion, f);
- encode_json("eventSource", eventSource, f);
- encode_json("awsRegion", awsRegion, f);
- utime_t ut(eventTime);
- encode_json("eventTime", ut, f);
- encode_json("eventName", eventName, f);
- {
- Formatter::ObjectSection s(*f, "userIdentity");
- encode_json("principalId", userIdentity, f);
- }
- {
- Formatter::ObjectSection s(*f, "requestParameters");
- encode_json("sourceIPAddress", sourceIPAddress, f);
- }
- {
- Formatter::ObjectSection s(*f, "responseElements");
- encode_json("x-amz-request-id", x_amz_request_id, f);
- encode_json("x-amz-id-2", x_amz_id_2, f);
- }
- {
- Formatter::ObjectSection s(*f, "s3");
- encode_json("s3SchemaVersion", s3SchemaVersion, f);
- encode_json("configurationId", configurationId, f);
- {
- Formatter::ObjectSection sub_s(*f, "bucket");
- encode_json("name", bucket_name, f);
- {
- Formatter::ObjectSection sub_sub_s(*f, "ownerIdentity");
- encode_json("principalId", bucket_ownerIdentity, f);
- }
- encode_json("arn", bucket_arn, f);
- encode_json("id", bucket_id, f);
- }
- {
- Formatter::ObjectSection sub_s(*f, "object");
- encode_json("key", object_key, f);
- encode_json("size", object_size, f);
- encode_json("eTag", object_etag, f);
- encode_json("versionId", object_versionId, f);
- encode_json("sequencer", object_sequencer, f);
- encode_json("metadata", x_meta_map, f);
- encode_json("tags", tags, f);
- }
- }
- encode_json("eventId", id, f);
- encode_json("opaqueData", opaque_data, f);
-}
-
-void rgw_pubsub_topic::dump(Formatter *f) const
-{
- encode_json("user", user, f);
- encode_json("name", name, f);
- encode_json("dest", dest, f);
- encode_json("arn", arn, f);
- encode_json("opaqueData", opaque_data, f);
-}
-
-void rgw_pubsub_topic::dump_xml(Formatter *f) const
-{
- encode_xml("User", user, f);
- encode_xml("Name", name, f);
- encode_xml("EndPoint", dest, f);
- encode_xml("TopicArn", arn, f);
- encode_xml("OpaqueData", opaque_data, f);
-}
-
-void encode_xml_key_value_entry(const std::string& key, const std::string& value, Formatter *f) {
- f->open_object_section("entry");
- encode_xml("key", key, f);
- encode_xml("value", value, f);
- f->close_section(); // entry
-}
-
-void rgw_pubsub_topic::dump_xml_as_attributes(Formatter *f) const
-{
- f->open_array_section("Attributes");
- std::string str_user;
- user.to_str(str_user);
- encode_xml_key_value_entry("User", str_user, f);
- encode_xml_key_value_entry("Name", name, f);
- encode_xml_key_value_entry("EndPoint", dest.to_json_str(), f);
- encode_xml_key_value_entry("TopicArn", arn, f);
- encode_xml_key_value_entry("OpaqueData", opaque_data, f);
- f->close_section(); // Attributes
-}
-
-void encode_json(const char *name, const rgw::notify::EventTypeList& l, Formatter *f)
-{
- f->open_array_section(name);
- for (auto iter = l.cbegin(); iter != l.cend(); ++iter) {
- f->dump_string("obj", rgw::notify::to_string(*iter));
- }
- f->close_section();
-}
-
-void rgw_pubsub_topic_filter::dump(Formatter *f) const
-{
- encode_json("topic", topic, f);
- encode_json("events", events, f);
-}
-
-void rgw_pubsub_bucket_topics::dump(Formatter *f) const
-{
- Formatter::ArraySection s(*f, "topics");
- for (auto& t : topics) {
- encode_json(t.first.c_str(), t.second, f);
- }
-}
-
-void rgw_pubsub_topics::dump(Formatter *f) const
-{
- Formatter::ArraySection s(*f, "topics");
- for (auto& t : topics) {
- encode_json(t.first.c_str(), t.second, f);
- }
-}
-
-void rgw_pubsub_topics::dump_xml(Formatter *f) const
-{
- for (auto& t : topics) {
- encode_xml("member", t.second, f);
- }
-}
-
-void rgw_pubsub_dest::dump(Formatter *f) const
-{
- encode_json("push_endpoint", push_endpoint, f);
- encode_json("push_endpoint_args", push_endpoint_args, f);
- encode_json("push_endpoint_topic", arn_topic, f);
- encode_json("stored_secret", stored_secret, f);
- encode_json("persistent", persistent, f);
-}
-
-void rgw_pubsub_dest::dump_xml(Formatter *f) const
-{
- encode_xml("EndpointAddress", push_endpoint, f);
- encode_xml("EndpointArgs", push_endpoint_args, f);
- encode_xml("EndpointTopic", arn_topic, f);
- encode_xml("HasStoredSecret", stored_secret, f);
- encode_xml("Persistent", persistent, f);
-}
-
-std::string rgw_pubsub_dest::to_json_str() const
-{
- JSONFormatter f;
- f.open_object_section("");
- encode_json("EndpointAddress", push_endpoint, &f);
- encode_json("EndpointArgs", push_endpoint_args, &f);
- encode_json("EndpointTopic", arn_topic, &f);
- encode_json("HasStoredSecret", stored_secret, &f);
- encode_json("Persistent", persistent, &f);
- f.close_section();
- std::stringstream ss;
- f.flush(ss);
- return ss.str();
-}
-
-RGWPubSub::RGWPubSub(rgw::sal::RadosStore* _store, const std::string& _tenant)
- : store(_store), tenant(_tenant), svc_sysobj(store->svc()->sysobj)
-{
- get_meta_obj(&meta_obj);
-}
-
-int RGWPubSub::remove(const DoutPrefixProvider *dpp,
- const rgw_raw_obj& obj,
- RGWObjVersionTracker *objv_tracker,
- optional_yield y) const
-{
- int ret = rgw_delete_system_obj(dpp, store->svc()->sysobj, obj.pool, obj.oid, objv_tracker, y);
- if (ret < 0) {
- return ret;
- }
-
- return 0;
-}
-
-int RGWPubSub::read_topics(rgw_pubsub_topics *result, RGWObjVersionTracker *objv_tracker) const
-{
- int ret = read(meta_obj, result, objv_tracker);
- if (ret < 0) {
- ldout(store->ctx(), 10) << "WARNING: failed to read topics info: ret=" << ret << dendl;
- return ret;
- }
- return 0;
-}
-
-int RGWPubSub::write_topics(const DoutPrefixProvider *dpp, const rgw_pubsub_topics& topics,
- RGWObjVersionTracker *objv_tracker, optional_yield y) const
-{
- int ret = write(dpp, meta_obj, topics, objv_tracker, y);
- if (ret < 0 && ret != -ENOENT) {
- ldpp_dout(dpp, 1) << "ERROR: failed to write topics info: ret=" << ret << dendl;
- return ret;
- }
- return 0;
-}
-
-int RGWPubSub::get_topics(rgw_pubsub_topics *result) const
-{
- return read_topics(result, nullptr);
-}
-
-int RGWPubSub::Bucket::read_topics(rgw_pubsub_bucket_topics *result, RGWObjVersionTracker *objv_tracker) const
-{
- int ret = ps.read(bucket_meta_obj, result, objv_tracker);
- if (ret < 0 && ret != -ENOENT) {
- ldout(ps.store->ctx(), 1) << "ERROR: failed to read bucket topics info: ret=" << ret << dendl;
- return ret;
- }
- return 0;
-}
-
-int RGWPubSub::Bucket::write_topics(const DoutPrefixProvider *dpp, const rgw_pubsub_bucket_topics& topics,
- RGWObjVersionTracker *objv_tracker,
- optional_yield y) const
-{
- int ret = ps.write(dpp, bucket_meta_obj, topics, objv_tracker, y);
- if (ret < 0) {
- ldout(ps.store->ctx(), 1) << "ERROR: failed to write bucket topics info: ret=" << ret << dendl;
- return ret;
- }
-
- return 0;
-}
-
-int RGWPubSub::Bucket::get_topics(rgw_pubsub_bucket_topics *result) const
-{
- return read_topics(result, nullptr);
-}
-
-int RGWPubSub::get_topic(const std::string& name, rgw_pubsub_topic *result) const
-{
- rgw_pubsub_topics topics;
- int ret = get_topics(&topics);
- if (ret < 0) {
- ldout(store->ctx(), 1) << "ERROR: failed to read topics info: ret=" << ret << dendl;
- return ret;
- }
-
- auto iter = topics.topics.find(name);
- if (iter == topics.topics.end()) {
- ldout(store->ctx(), 1) << "ERROR: topic not found" << dendl;
- return -ENOENT;
- }
-
- *result = iter->second;
- return 0;
-}
-
-int RGWPubSub::Bucket::create_notification(const DoutPrefixProvider *dpp, const std::string& topic_name,
- const rgw::notify::EventTypeList& events, optional_yield y) const {
- return create_notification(dpp, topic_name, events, std::nullopt, "", y);
-}
-
-int RGWPubSub::Bucket::create_notification(const DoutPrefixProvider *dpp, const std::string& topic_name,
- const rgw::notify::EventTypeList& events, OptionalFilter s3_filter, const std::string& notif_name, optional_yield y) const {
- rgw_pubsub_topic topic_info;
-
- int ret = ps.get_topic(topic_name, &topic_info);
- if (ret < 0) {
- ldpp_dout(dpp, 1) << "ERROR: failed to read topic '" << topic_name << "' info: ret=" << ret << dendl;
- return ret;
- }
- ldpp_dout(dpp, 20) << "successfully read topic '" << topic_name << "' info" << dendl;
-
- RGWObjVersionTracker objv_tracker;
- rgw_pubsub_bucket_topics bucket_topics;
-
- ret = read_topics(&bucket_topics, &objv_tracker);
- if (ret < 0) {
- ldpp_dout(dpp, 1) << "ERROR: failed to read topics from bucket '" <<
- bucket.name << "': ret=" << ret << dendl;
- return ret;
- }
- ldpp_dout(dpp, 20) << "successfully read " << bucket_topics.topics.size() << " topics from bucket '" <<
- bucket.name << "'" << dendl;
-
- auto& topic_filter = bucket_topics.topics[topic_name];
- topic_filter.topic = topic_info;
- topic_filter.events = events;
- topic_filter.s3_id = notif_name;
- if (s3_filter) {
- topic_filter.s3_filter = *s3_filter;
- }
-
- ret = write_topics(dpp, bucket_topics, &objv_tracker, y);
- if (ret < 0) {
- ldpp_dout(dpp, 1) << "ERROR: failed to write topics to bucket '" << bucket.name << "': ret=" << ret << dendl;
- return ret;
- }
-
- ldpp_dout(dpp, 20) << "successfully wrote " << bucket_topics.topics.size() << " topics to bucket '" << bucket.name << "'" << dendl;
-
- return 0;
-}
-
-int RGWPubSub::Bucket::remove_notification(const DoutPrefixProvider *dpp, const std::string& topic_name, optional_yield y) const
-{
- rgw_pubsub_topic topic_info;
-
- int ret = ps.get_topic(topic_name, &topic_info);
- if (ret < 0) {
- ldpp_dout(dpp, 1) << "ERROR: failed to read topic info: ret=" << ret << dendl;
- return ret;
- }
-
- RGWObjVersionTracker objv_tracker;
- rgw_pubsub_bucket_topics bucket_topics;
-
- ret = read_topics(&bucket_topics, &objv_tracker);
- if (ret < 0) {
- ldpp_dout(dpp, 1) << "ERROR: failed to read bucket topics info: ret=" << ret << dendl;
- return ret;
- }
-
- bucket_topics.topics.erase(topic_name);
-
- if (bucket_topics.topics.empty()) {
- // no more topics - delete the notification object of the bucket
- ret = ps.remove(dpp, bucket_meta_obj, &objv_tracker, y);
- if (ret < 0 && ret != -ENOENT) {
- ldpp_dout(dpp, 1) << "ERROR: failed to remove bucket topics: ret=" << ret << dendl;
- return ret;
- }
- return 0;
- }
-
- // write back the notifications without the deleted one
- ret = write_topics(dpp, bucket_topics, &objv_tracker, y);
- if (ret < 0) {
- ldpp_dout(dpp, 1) << "ERROR: failed to write topics info: ret=" << ret << dendl;
- return ret;
- }
-
- return 0;
-}
-
-int RGWPubSub::Bucket::remove_notifications(const DoutPrefixProvider *dpp, optional_yield y) const
-{
- // get all topics on a bucket
- rgw_pubsub_bucket_topics bucket_topics;
- auto ret = get_topics(&bucket_topics);
- if (ret < 0 && ret != -ENOENT) {
- ldpp_dout(dpp, 1) << "ERROR: failed to get list of topics from bucket '" << bucket.name << "', ret=" << ret << dendl;
- return ret ;
- }
-
- // remove all auto-genrated topics
- for (const auto& topic : bucket_topics.topics) {
- const auto& topic_name = topic.first;
- ret = ps.remove_topic(dpp, topic_name, y);
- if (ret < 0 && ret != -ENOENT) {
- ldpp_dout(dpp, 5) << "WARNING: failed to remove auto-generated topic '" << topic_name << "', ret=" << ret << dendl;
- }
- }
-
- // delete the notification object of the bucket
- ret = ps.remove(dpp, bucket_meta_obj, nullptr, y);
- if (ret < 0 && ret != -ENOENT) {
- ldpp_dout(dpp, 1) << "ERROR: failed to remove bucket topics: ret=" << ret << dendl;
- return ret;
- }
-
- return 0;
-}
-
-int RGWPubSub::create_topic(const DoutPrefixProvider *dpp, const std::string& name, optional_yield y) const {
- return create_topic(dpp, name, rgw_pubsub_dest{}, "", "", y);
-}
-
-int RGWPubSub::create_topic(const DoutPrefixProvider *dpp, const std::string& name, const rgw_pubsub_dest& dest,
- const std::string& arn, const std::string& opaque_data, optional_yield y) const {
- RGWObjVersionTracker objv_tracker;
- rgw_pubsub_topics topics;
-
- int ret = read_topics(&topics, &objv_tracker);
- if (ret < 0 && ret != -ENOENT) {
- // its not an error if not topics exist, we create one
- ldpp_dout(dpp, 1) << "ERROR: failed to read topics info: ret=" << ret << dendl;
- return ret;
- }
-
- rgw_pubsub_topic& new_topic = topics.topics[name];
- new_topic.user = rgw_user("", tenant);
- new_topic.name = name;
- new_topic.dest = dest;
- new_topic.arn = arn;
- new_topic.opaque_data = opaque_data;
-
- ret = write_topics(dpp, topics, &objv_tracker, y);
- if (ret < 0) {
- ldpp_dout(dpp, 1) << "ERROR: failed to write topics info: ret=" << ret << dendl;
- return ret;
- }
-
- return 0;
-}
-
-int RGWPubSub::remove_topic(const DoutPrefixProvider *dpp, const std::string& name, optional_yield y) const
-{
- RGWObjVersionTracker objv_tracker;
- rgw_pubsub_topics topics;
-
- int ret = read_topics(&topics, &objv_tracker);
- if (ret < 0 && ret != -ENOENT) {
- ldpp_dout(dpp, 1) << "ERROR: failed to read topics info: ret=" << ret << dendl;
- return ret;
- } else if (ret == -ENOENT) {
- // its not an error if no topics exist, just a no-op
- ldpp_dout(dpp, 10) << "WARNING: failed to read topics info, deletion is a no-op: ret=" << ret << dendl;
- return 0;
- }
-
- topics.topics.erase(name);
-
- ret = write_topics(dpp, topics, &objv_tracker, y);
- if (ret < 0) {
- ldpp_dout(dpp, 1) << "ERROR: failed to remove topics info: ret=" << ret << dendl;
- return ret;
- }
-
- return 0;
-}
-
-void RGWPubSub::get_meta_obj(rgw_raw_obj *obj) const {
- *obj = rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, meta_oid());
-}
-
-void RGWPubSub::get_bucket_meta_obj(const rgw_bucket& bucket, rgw_raw_obj *obj) const {
- *obj = rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, bucket_meta_oid(bucket));
-}
-
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab ft=cpp
-
-#pragma once
-
-#include "services/svc_sys_obj.h"
-#include "rgw_tools.h"
-#include "rgw_zone.h"
-#include "rgw_notify_event_type.h"
-#include <boost/container/flat_map.hpp>
-
-namespace rgw::sal { class RadosStore; }
-
-class XMLObj;
-
-struct rgw_s3_key_filter {
- std::string prefix_rule;
- std::string suffix_rule;
- std::string regex_rule;
-
- bool has_content() const;
-
- bool decode_xml(XMLObj *obj);
- void dump_xml(Formatter *f) const;
-
- void encode(bufferlist& bl) const {
- ENCODE_START(1, 1, bl);
- encode(prefix_rule, bl);
- encode(suffix_rule, bl);
- encode(regex_rule, bl);
- ENCODE_FINISH(bl);
- }
-
- void decode(bufferlist::const_iterator& bl) {
- DECODE_START(1, bl);
- decode(prefix_rule, bl);
- decode(suffix_rule, bl);
- decode(regex_rule, bl);
- DECODE_FINISH(bl);
- }
-};
-WRITE_CLASS_ENCODER(rgw_s3_key_filter)
-
-using KeyValueMap = boost::container::flat_map<std::string, std::string>;
-using KeyMultiValueMap = std::multimap<std::string, std::string>;
-
-struct rgw_s3_key_value_filter {
- KeyValueMap kv;
-
- bool has_content() const;
-
- bool decode_xml(XMLObj *obj);
- void dump_xml(Formatter *f) const;
-
- void encode(bufferlist& bl) const {
- ENCODE_START(1, 1, bl);
- encode(kv, bl);
- ENCODE_FINISH(bl);
- }
- void decode(bufferlist::const_iterator& bl) {
- DECODE_START(1, bl);
- decode(kv, bl);
- DECODE_FINISH(bl);
- }
-};
-WRITE_CLASS_ENCODER(rgw_s3_key_value_filter)
-
-struct rgw_s3_filter {
- rgw_s3_key_filter key_filter;
- rgw_s3_key_value_filter metadata_filter;
- rgw_s3_key_value_filter tag_filter;
-
- bool has_content() const;
-
- bool decode_xml(XMLObj *obj);
- void dump_xml(Formatter *f) const;
-
- void encode(bufferlist& bl) const {
- ENCODE_START(2, 1, bl);
- encode(key_filter, bl);
- encode(metadata_filter, bl);
- encode(tag_filter, bl);
- ENCODE_FINISH(bl);
- }
-
- void decode(bufferlist::const_iterator& bl) {
- DECODE_START(2, bl);
- decode(key_filter, bl);
- decode(metadata_filter, bl);
- if (struct_v >= 2) {
- decode(tag_filter, bl);
- }
- DECODE_FINISH(bl);
- }
-};
-WRITE_CLASS_ENCODER(rgw_s3_filter)
-
-using OptionalFilter = std::optional<rgw_s3_filter>;
-
-struct rgw_pubsub_topic_filter;
-/* S3 notification configuration
- * based on: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTnotification.html
-<NotificationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
- <TopicConfiguration>
- <Filter>
- <S3Key>
- <FilterRule>
- <Name>suffix</Name>
- <Value>jpg</Value>
- </FilterRule>
- </S3Key>
- <S3Metadata>
- <FilterRule>
- <Name></Name>
- <Value></Value>
- </FilterRule>
- </S3Metadata>
- <S3Tags>
- <FilterRule>
- <Name></Name>
- <Value></Value>
- </FilterRule>
- </S3Tags>
- </Filter>
- <Id>notification1</Id>
- <Topic>arn:aws:sns:<region>:<account>:<topic></Topic>
- <Event>s3:ObjectCreated:*</Event>
- <Event>s3:ObjectRemoved:*</Event>
- </TopicConfiguration>
-</NotificationConfiguration>
-*/
-struct rgw_pubsub_s3_notification {
- // notification id
- std::string id;
- // types of events
- rgw::notify::EventTypeList events;
- // topic ARN
- std::string topic_arn;
- // filter rules
- rgw_s3_filter filter;
-
- bool decode_xml(XMLObj *obj);
- void dump_xml(Formatter *f) const;
-
- rgw_pubsub_s3_notification() = default;
- // construct from rgw_pubsub_topic_filter (used by get/list notifications)
- explicit rgw_pubsub_s3_notification(const rgw_pubsub_topic_filter& topic_filter);
-};
-
-// return true if the key matches the prefix/suffix/regex rules of the key filter
-bool match(const rgw_s3_key_filter& filter, const std::string& key);
-
-// return true if the key matches the metadata rules of the metadata filter
-bool match(const rgw_s3_key_value_filter& filter, const KeyValueMap& kv);
-
-// return true if the key matches the tag rules of the tag filter
-bool match(const rgw_s3_key_value_filter& filter, const KeyMultiValueMap& kv);
-
-// return true if the event type matches (equal or contained in) one of the events in the list
-bool match(const rgw::notify::EventTypeList& events, rgw::notify::EventType event);
-
-struct rgw_pubsub_s3_notifications {
- std::list<rgw_pubsub_s3_notification> list;
- bool decode_xml(XMLObj *obj);
- void dump_xml(Formatter *f) const;
-};
-
-/* S3 event records structure
- * based on: https://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
-{
-"Records":[
- {
- "eventVersion":""
- "eventSource":"",
- "awsRegion":"",
- "eventTime":"",
- "eventName":"",
- "userIdentity":{
- "principalId":""
- },
- "requestParameters":{
- "sourceIPAddress":""
- },
- "responseElements":{
- "x-amz-request-id":"",
- "x-amz-id-2":""
- },
- "s3":{
- "s3SchemaVersion":"1.0",
- "configurationId":"",
- "bucket":{
- "name":"",
- "ownerIdentity":{
- "principalId":""
- },
- "arn":""
- "id": ""
- },
- "object":{
- "key":"",
- "size": ,
- "eTag":"",
- "versionId":"",
- "sequencer": "",
- "metadata": ""
- "tags": ""
- }
- },
- "eventId":"",
- }
-]
-}*/
-
-struct rgw_pubsub_s3_event {
- constexpr static const char* const json_type_plural = "Records";
- std::string eventVersion = "2.2";
- // aws:s3
- std::string eventSource = "ceph:s3";
- // zonegroup
- std::string awsRegion;
- // time of the request
- ceph::real_time eventTime;
- // type of the event
- std::string eventName;
- // user that sent the request
- std::string userIdentity;
- // IP address of source of the request (not implemented)
- std::string sourceIPAddress;
- // request ID (not implemented)
- std::string x_amz_request_id;
- // radosgw that received the request
- std::string x_amz_id_2;
- std::string s3SchemaVersion = "1.0";
- // ID received in the notification request
- std::string configurationId;
- // bucket name
- std::string bucket_name;
- // bucket owner
- std::string bucket_ownerIdentity;
- // bucket ARN
- std::string bucket_arn;
- // object key
- std::string object_key;
- // object size
- uint64_t object_size = 0;
- // object etag
- std::string object_etag;
- // object version id bucket is versioned
- std::string object_versionId;
- // hexadecimal value used to determine event order for specific key
- std::string object_sequencer;
- // this is an rgw extension (not S3 standard)
- // used to store a globally unique identifier of the event
- // that could be used for acking or any other identification of the event
- std::string id;
- // this is an rgw extension holding the internal bucket id
- std::string bucket_id;
- // meta data
- KeyValueMap x_meta_map;
- // tags
- KeyMultiValueMap tags;
- // opaque data received from the topic
- // could be used to identify the gateway
- std::string opaque_data;
-
- void encode(bufferlist& bl) const {
- ENCODE_START(4, 1, bl);
- encode(eventVersion, bl);
- encode(eventSource, bl);
- encode(awsRegion, bl);
- encode(eventTime, bl);
- encode(eventName, bl);
- encode(userIdentity, bl);
- encode(sourceIPAddress, bl);
- encode(x_amz_request_id, bl);
- encode(x_amz_id_2, bl);
- encode(s3SchemaVersion, bl);
- encode(configurationId, bl);
- encode(bucket_name, bl);
- encode(bucket_ownerIdentity, bl);
- encode(bucket_arn, bl);
- encode(object_key, bl);
- encode(object_size, bl);
- encode(object_etag, bl);
- encode(object_versionId, bl);
- encode(object_sequencer, bl);
- encode(id, bl);
- encode(bucket_id, bl);
- encode(x_meta_map, bl);
- encode(tags, bl);
- encode(opaque_data, bl);
- ENCODE_FINISH(bl);
- }
-
- void decode(bufferlist::const_iterator& bl) {
- DECODE_START(4, bl);
- decode(eventVersion, bl);
- decode(eventSource, bl);
- decode(awsRegion, bl);
- decode(eventTime, bl);
- decode(eventName, bl);
- decode(userIdentity, bl);
- decode(sourceIPAddress, bl);
- decode(x_amz_request_id, bl);
- decode(x_amz_id_2, bl);
- decode(s3SchemaVersion, bl);
- decode(configurationId, bl);
- decode(bucket_name, bl);
- decode(bucket_ownerIdentity, bl);
- decode(bucket_arn, bl);
- decode(object_key, bl);
- decode(object_size, bl);
- decode(object_etag, bl);
- decode(object_versionId, bl);
- decode(object_sequencer, bl);
- decode(id, bl);
- if (struct_v >= 2) {
- decode(bucket_id, bl);
- decode(x_meta_map, bl);
- }
- if (struct_v >= 3) {
- decode(tags, bl);
- }
- if (struct_v >= 4) {
- decode(opaque_data, bl);
- }
- DECODE_FINISH(bl);
- }
-
- void dump(Formatter *f) const;
-};
-WRITE_CLASS_ENCODER(rgw_pubsub_s3_event)
-
-// setting a unique ID for an event based on object hash and timestamp
-void set_event_id(std::string& id, const std::string& hash, const utime_t& ts);
-
-struct rgw_pubsub_dest {
- std::string push_endpoint;
- std::string push_endpoint_args;
- std::string arn_topic;
- bool stored_secret = false;
- bool persistent = false;
-
- void encode(bufferlist& bl) const {
- ENCODE_START(5, 1, bl);
- encode("", bl);
- encode("", bl);
- encode(push_endpoint, bl);
- encode(push_endpoint_args, bl);
- encode(arn_topic, bl);
- encode(stored_secret, bl);
- encode(persistent, bl);
- ENCODE_FINISH(bl);
- }
-
- void decode(bufferlist::const_iterator& bl) {
- DECODE_START(5, bl);
- std::string dummy;
- decode(dummy, bl);
- decode(dummy, bl);
- decode(push_endpoint, bl);
- if (struct_v >= 2) {
- decode(push_endpoint_args, bl);
- }
- if (struct_v >= 3) {
- decode(arn_topic, bl);
- }
- if (struct_v >= 4) {
- decode(stored_secret, bl);
- }
- if (struct_v >= 5) {
- decode(persistent, bl);
- }
- DECODE_FINISH(bl);
- }
-
- void dump(Formatter *f) const;
- void dump_xml(Formatter *f) const;
- std::string to_json_str() const;
-};
-WRITE_CLASS_ENCODER(rgw_pubsub_dest)
-
-struct rgw_pubsub_topic {
- rgw_user user;
- std::string name;
- rgw_pubsub_dest dest;
- std::string arn;
- std::string opaque_data;
-
- void encode(bufferlist& bl) const {
- ENCODE_START(3, 1, bl);
- encode(user, bl);
- encode(name, bl);
- encode(dest, bl);
- encode(arn, bl);
- encode(opaque_data, bl);
- ENCODE_FINISH(bl);
- }
-
- void decode(bufferlist::const_iterator& bl) {
- DECODE_START(3, bl);
- decode(user, bl);
- decode(name, bl);
- if (struct_v >= 2) {
- decode(dest, bl);
- decode(arn, bl);
- }
- if (struct_v >= 3) {
- decode(opaque_data, bl);
- }
- DECODE_FINISH(bl);
- }
-
- std::string to_str() const {
- return user.tenant + "/" + name;
- }
-
- void dump(Formatter *f) const;
- void dump_xml(Formatter *f) const;
- void dump_xml_as_attributes(Formatter *f) const;
-
- bool operator<(const rgw_pubsub_topic& t) const {
- return to_str().compare(t.to_str());
- }
-};
-WRITE_CLASS_ENCODER(rgw_pubsub_topic)
-
-// this struct deprecated and remain only for backward compatibility
-struct rgw_pubsub_topic_subs {
- rgw_pubsub_topic topic;
- std::set<std::string> subs;
-
- void encode(bufferlist& bl) const {
- ENCODE_START(1, 1, bl);
- encode(topic, bl);
- encode(subs, bl);
- ENCODE_FINISH(bl);
- }
-
- void decode(bufferlist::const_iterator& bl) {
- DECODE_START(1, bl);
- decode(topic, bl);
- decode(subs, bl);
- DECODE_FINISH(bl);
- }
-
- void dump(Formatter *f) const;
-};
-WRITE_CLASS_ENCODER(rgw_pubsub_topic_subs)
-
-struct rgw_pubsub_topic_filter {
- rgw_pubsub_topic topic;
- rgw::notify::EventTypeList events;
- std::string s3_id;
- rgw_s3_filter s3_filter;
-
- void encode(bufferlist& bl) const {
- ENCODE_START(3, 1, bl);
- encode(topic, bl);
- // events are stored as a vector of std::strings
- std::vector<std::string> tmp_events;
- std::transform(events.begin(), events.end(), std::back_inserter(tmp_events), rgw::notify::to_string);
- encode(tmp_events, bl);
- encode(s3_id, bl);
- encode(s3_filter, bl);
- ENCODE_FINISH(bl);
- }
-
- void decode(bufferlist::const_iterator& bl) {
- DECODE_START(3, bl);
- decode(topic, bl);
- // events are stored as a vector of std::strings
- events.clear();
- std::vector<std::string> tmp_events;
- decode(tmp_events, bl);
- std::transform(tmp_events.begin(), tmp_events.end(), std::back_inserter(events), rgw::notify::from_string);
- if (struct_v >= 2) {
- decode(s3_id, bl);
- }
- if (struct_v >= 3) {
- decode(s3_filter, bl);
- }
- DECODE_FINISH(bl);
- }
-
- void dump(Formatter *f) const;
-};
-WRITE_CLASS_ENCODER(rgw_pubsub_topic_filter)
-
-struct rgw_pubsub_bucket_topics {
- std::map<std::string, rgw_pubsub_topic_filter> topics;
-
- void encode(bufferlist& bl) const {
- ENCODE_START(1, 1, bl);
- encode(topics, bl);
- ENCODE_FINISH(bl);
- }
-
- void decode(bufferlist::const_iterator& bl) {
- DECODE_START(1, bl);
- decode(topics, bl);
- DECODE_FINISH(bl);
- }
-
- void dump(Formatter *f) const;
-};
-WRITE_CLASS_ENCODER(rgw_pubsub_bucket_topics)
-
-struct rgw_pubsub_topics {
- std::map<std::string, rgw_pubsub_topic> topics;
-
- void encode(bufferlist& bl) const {
- ENCODE_START(2, 2, bl);
- encode(topics, bl);
- ENCODE_FINISH(bl);
- }
-
- void decode(bufferlist::const_iterator& bl) {
- DECODE_START(2, bl);
- if (struct_v >= 2) {
- decode(topics, bl);
- } else {
- std::map<std::string, rgw_pubsub_topic_subs> v1topics;
- decode(v1topics, bl);
- std::transform(v1topics.begin(), v1topics.end(), std::inserter(topics, topics.end()),
- [](const auto& entry) {
- return std::pair<std::string, rgw_pubsub_topic>(entry.first, entry.second.topic);
- });
- }
- DECODE_FINISH(bl);
- }
-
- void dump(Formatter *f) const;
- void dump_xml(Formatter *f) const;
-};
-WRITE_CLASS_ENCODER(rgw_pubsub_topics)
-
-static std::string pubsub_oid_prefix = "pubsub.";
-
-class RGWPubSub
-{
- friend class Bucket;
-
- rgw::sal::RadosStore* const store;
- const std::string tenant;
- RGWSI_SysObj* const svc_sysobj;
-
- rgw_raw_obj meta_obj;
-
- std::string meta_oid() const {
- return pubsub_oid_prefix + tenant;
- }
-
- std::string bucket_meta_oid(const rgw_bucket& bucket) const {
- return pubsub_oid_prefix + tenant + ".bucket." + bucket.name + "/" + bucket.marker;
- }
-
- template <class T>
- int read(const rgw_raw_obj& obj, T* data, RGWObjVersionTracker* objv_tracker) const;
-
- template <class T>
- int write(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, const T& info,
- RGWObjVersionTracker* obj_tracker, optional_yield y) const;
-
- int remove(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, RGWObjVersionTracker* objv_tracker,
- optional_yield y) const;
-
- int read_topics(rgw_pubsub_topics *result, RGWObjVersionTracker* objv_tracker) const;
- int write_topics(const DoutPrefixProvider *dpp, const rgw_pubsub_topics& topics,
- RGWObjVersionTracker* objv_tracker, optional_yield y) const;
-
-public:
- RGWPubSub(rgw::sal::RadosStore* _store, const std::string& tenant);
-
- class Bucket {
- friend class RGWPubSub;
- const RGWPubSub& ps;
- const rgw_bucket& bucket;
- rgw_raw_obj bucket_meta_obj;
-
- // read the list of topics associated with a bucket and populate into result
- // use version tacker to enforce atomicity between read/write
- // return 0 on success or if no topic was associated with the bucket, error code otherwise
- int read_topics(rgw_pubsub_bucket_topics *result, RGWObjVersionTracker* objv_tracker) const;
- // set the list of topics associated with a bucket
- // use version tacker to enforce atomicity between read/write
- // return 0 on success, error code otherwise
- int write_topics(const DoutPrefixProvider *dpp, const rgw_pubsub_bucket_topics& topics,
- RGWObjVersionTracker* objv_tracker, optional_yield y) const;
- public:
- Bucket(const RGWPubSub& _ps, const rgw_bucket& _bucket) : ps(_ps), bucket(_bucket) {
- ps.get_bucket_meta_obj(bucket, &bucket_meta_obj);
- }
-
- // read the list of topics associated with a bucket and populate into result
- // return 0 on success or if no topic was associated with the bucket, error code otherwise
- int get_topics(rgw_pubsub_bucket_topics *result) const;
- // adds a topic + filter (event list, and possibly name metadata or tags filters) to a bucket
- // assigning a notification name is optional (needed for S3 compatible notifications)
- // if the topic already exist on the bucket, the filter event list may be updated
- // for S3 compliant notifications the version with: s3_filter and notif_name should be used
- // return -ENOENT if the topic does not exists
- // return 0 on success, error code otherwise
- int create_notification(const DoutPrefixProvider *dpp, const std::string& topic_name,
- const rgw::notify::EventTypeList& events, optional_yield y) const;
- int create_notification(const DoutPrefixProvider *dpp, const std::string& topic_name,
- const rgw::notify::EventTypeList& events, OptionalFilter s3_filter, const std::string& notif_name, optional_yield y) const;
- // remove a topic and filter from bucket
- // if the topic does not exists on the bucket it is a no-op (considered success)
- // return -ENOENT if the topic does not exists
- // return 0 on success, error code otherwise
- int remove_notification(const DoutPrefixProvider *dpp, const std::string& topic_name, optional_yield y) const;
- // remove all notifications (and autogenerated topics) associated with the bucket
- // return 0 on success or if no topic was associated with the bucket, error code otherwise
- int remove_notifications(const DoutPrefixProvider *dpp, optional_yield y) const;
- };
-
- void get_meta_obj(rgw_raw_obj *obj) const;
- void get_bucket_meta_obj(const rgw_bucket& bucket, rgw_raw_obj *obj) const;
-
- // get all topics (per tenant, if used)) and populate them into "result"
- // return 0 on success or if no topics exist, error code otherwise
- int get_topics(rgw_pubsub_topics *result) const;
- // get a topic with by its name and populate it into "result"
- // return -ENOENT if the topic does not exists
- // return 0 on success, error code otherwise
- int get_topic(const std::string& name, rgw_pubsub_topic *result) const;
- // create a topic with a name only
- // if the topic already exists it is a no-op (considered success)
- // return 0 on success, error code otherwise
- int create_topic(const DoutPrefixProvider *dpp, const std::string& name, optional_yield y) const;
- // create a topic with push destination information and ARN
- // if the topic already exists the destination and ARN values may be updated (considered succsess)
- // return 0 on success, error code otherwise
- int create_topic(const DoutPrefixProvider *dpp, const std::string& name, const rgw_pubsub_dest& dest,
- const std::string& arn, const std::string& opaque_data, optional_yield y) const;
- // remove a topic according to its name
- // if the topic does not exists it is a no-op (considered success)
- // return 0 on success, error code otherwise
- int remove_topic(const DoutPrefixProvider *dpp, const std::string& name, optional_yield y) const;
-};
-
-template <class T>
-int RGWPubSub::read(const rgw_raw_obj& obj, T* result, RGWObjVersionTracker* objv_tracker) const
-{
- bufferlist bl;
- int ret = rgw_get_system_obj(svc_sysobj,
- obj.pool, obj.oid,
- bl,
- objv_tracker,
- nullptr, null_yield, nullptr, nullptr);
- if (ret < 0) {
- return ret;
- }
-
- auto iter = bl.cbegin();
- try {
- decode(*result, iter);
- } catch (buffer::error& err) {
- return -EIO;
- }
-
- return 0;
-}
-
-template <class T>
-int RGWPubSub::write(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, const T& info,
- RGWObjVersionTracker* objv_tracker, optional_yield y) const
-{
- bufferlist bl;
- encode(info, bl);
-
- return rgw_put_system_obj(dpp, svc_sysobj, obj.pool, obj.oid,
- bl, false, objv_tracker, real_time(), y);
-}
-
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-
-#include <algorithm>
-#include <boost/tokenizer.hpp>
-#include <optional>
-#include "rgw_rest_pubsub.h"
-#include "rgw_pubsub_push.h"
-#include "rgw_pubsub.h"
-#include "rgw_op.h"
-#include "rgw_rest.h"
-#include "rgw_rest_s3.h"
-#include "rgw_arn.h"
-#include "rgw_auth_s3.h"
-#include "rgw_notify.h"
-#include "rgw_sal_rados.h"
-#include "services/svc_zone.h"
-#include "common/dout.h"
-#include "rgw_url.h"
-
-#define dout_context g_ceph_context
-#define dout_subsys ceph_subsys_rgw
-
-static const char* AWS_SNS_NS("https://sns.amazonaws.com/doc/2010-03-31/");
-
-bool verify_transport_security(CephContext *cct, const RGWEnv& env) {
- const auto is_secure = rgw_transport_is_secure(cct, env);
- if (!is_secure && g_conf().get_val<bool>("rgw_allow_notification_secrets_in_cleartext")) {
- ldout(cct, 0) << "WARNING: bypassing endpoint validation, allows sending secrets over insecure transport" << dendl;
- return true;
- }
- return is_secure;
-}
-
-// make sure that endpoint is a valid URL
-// make sure that if user/password are passed inside URL, it is over secure connection
-// update rgw_pubsub_dest to indicate that a password is stored in the URL
-bool validate_and_update_endpoint_secret(rgw_pubsub_dest& dest, CephContext *cct, const RGWEnv& env) {
- if (dest.push_endpoint.empty()) {
- return true;
- }
- std::string user;
- std::string password;
- if (!rgw::parse_url_userinfo(dest.push_endpoint, user, password)) {
- ldout(cct, 1) << "endpoint validation error: malformed endpoint URL:" << dest.push_endpoint << dendl;
- return false;
- }
- // this should be verified inside parse_url()
- ceph_assert(user.empty() == password.empty());
- if (!user.empty()) {
- dest.stored_secret = true;
- if (!verify_transport_security(cct, env)) {
- ldout(cct, 1) << "endpoint validation error: sending secrets over insecure transport" << dendl;
- return false;
- }
- }
- return true;
-}
-
-bool topic_has_endpoint_secret(const rgw_pubsub_topic& topic) {
- return topic.dest.stored_secret;
-}
-
-bool topics_has_endpoint_secret(const rgw_pubsub_topics& topics) {
- for (const auto& topic : topics.topics) {
- if (topic_has_endpoint_secret(topic.second)) return true;
- }
- return false;
-}
-
-// command (AWS compliant):
-// POST
-// Action=CreateTopic&Name=<topic-name>[&OpaqueData=data][&push-endpoint=<endpoint>[&persistent][&<arg1>=<value1>]]
-class RGWPSCreateTopicOp : public RGWOp {
- private:
- std::string topic_name;
- rgw_pubsub_dest dest;
- std::string topic_arn;
- std::string opaque_data;
-
- int get_params() {
- topic_name = s->info.args.get("Name");
- if (topic_name.empty()) {
- ldpp_dout(this, 1) << "CreateTopic Action 'Name' argument is missing" << dendl;
- return -EINVAL;
- }
-
- opaque_data = s->info.args.get("OpaqueData");
-
- dest.push_endpoint = s->info.args.get("push-endpoint");
- s->info.args.get_bool("persistent", &dest.persistent, false);
-
- if (!validate_and_update_endpoint_secret(dest, s->cct, *(s->info.env))) {
- return -EINVAL;
- }
- for (const auto& param : s->info.args.get_params()) {
- if (param.first == "Action" || param.first == "Name" || param.first == "PayloadHash") {
- continue;
- }
- dest.push_endpoint_args.append(param.first+"="+param.second+"&");
- }
-
- if (!dest.push_endpoint_args.empty()) {
- // remove last separator
- dest.push_endpoint_args.pop_back();
- }
- if (!dest.push_endpoint.empty() && dest.persistent) {
- const auto ret = rgw::notify::add_persistent_topic(topic_name, s->yield);
- if (ret < 0) {
- ldpp_dout(this, 1) << "CreateTopic Action failed to create queue for persistent topics. error:" << ret << dendl;
- return ret;
- }
- }
-
- // dest object only stores endpoint info
- dest.arn_topic = topic_name;
- // the topic ARN will be sent in the reply
- const rgw::ARN arn(rgw::Partition::aws, rgw::Service::sns,
- driver->get_zone()->get_zonegroup().get_name(),
- s->user->get_tenant(), topic_name);
- topic_arn = arn.to_string();
- return 0;
- }
-
- public:
- int verify_permission(optional_yield) override {
- return 0;
- }
-
- void pre_exec() override {
- rgw_bucket_object_pre_exec(s);
- }
- void execute(optional_yield) override;
-
- const char* name() const override { return "pubsub_topic_create"; }
- RGWOpType get_type() override { return RGW_OP_PUBSUB_TOPIC_CREATE; }
- uint32_t op_mask() override { return RGW_OP_TYPE_WRITE; }
-
- void send_response() override {
- if (op_ret) {
- set_req_state_err(s, op_ret);
- }
- dump_errno(s);
- end_header(s, this, "application/xml");
-
- if (op_ret < 0) {
- return;
- }
-
- const auto f = s->formatter;
- f->open_object_section_in_ns("CreateTopicResponse", AWS_SNS_NS);
- f->open_object_section("CreateTopicResult");
- encode_xml("TopicArn", topic_arn, f);
- f->close_section(); // CreateTopicResult
- f->open_object_section("ResponseMetadata");
- encode_xml("RequestId", s->req_id, f);
- f->close_section(); // ResponseMetadata
- f->close_section(); // CreateTopicResponse
- rgw_flush_formatter_and_reset(s, f);
- }
-};
-
-void RGWPSCreateTopicOp::execute(optional_yield y) {
- op_ret = get_params();
- if (op_ret < 0) {
- return;
- }
-
- RGWPubSub ps(static_cast<rgw::sal::RadosStore*>(driver), s->owner.get_id().tenant);
- op_ret = ps.create_topic(this, topic_name, dest, topic_arn, opaque_data, y);
- if (op_ret < 0) {
- ldpp_dout(this, 1) << "failed to create topic '" << topic_name << "', ret=" << op_ret << dendl;
- return;
- }
- ldpp_dout(this, 20) << "successfully created topic '" << topic_name << "'" << dendl;
-}
-
-// command (AWS compliant):
-// POST
-// Action=ListTopics
-class RGWPSListTopicsOp : public RGWOp {
-private:
- rgw_pubsub_topics result;
-
-public:
- int verify_permission(optional_yield) override {
- return 0;
- }
- void pre_exec() override {
- rgw_bucket_object_pre_exec(s);
- }
- void execute(optional_yield) override;
-
- const char* name() const override { return "pubsub_topics_list"; }
- RGWOpType get_type() override { return RGW_OP_PUBSUB_TOPICS_LIST; }
- uint32_t op_mask() override { return RGW_OP_TYPE_READ; }
-
- void send_response() override {
- if (op_ret) {
- set_req_state_err(s, op_ret);
- }
- dump_errno(s);
- end_header(s, this, "application/xml");
-
- if (op_ret < 0) {
- return;
- }
-
- const auto f = s->formatter;
- f->open_object_section_in_ns("ListTopicsResponse", AWS_SNS_NS);
- f->open_object_section("ListTopicsResult");
- encode_xml("Topics", result, f);
- f->close_section(); // ListTopicsResult
- f->open_object_section("ResponseMetadata");
- encode_xml("RequestId", s->req_id, f);
- f->close_section(); // ResponseMetadat
- f->close_section(); // ListTopicsResponse
- rgw_flush_formatter_and_reset(s, f);
- }
-};
-
-void RGWPSListTopicsOp::execute(optional_yield y) {
- RGWPubSub ps(static_cast<rgw::sal::RadosStore*>(driver), s->owner.get_id().tenant);
- op_ret = ps.get_topics(&result);
- // if there are no topics it is not considered an error
- op_ret = op_ret == -ENOENT ? 0 : op_ret;
- if (op_ret < 0) {
- ldpp_dout(this, 1) << "failed to get topics, ret=" << op_ret << dendl;
- return;
- }
- if (topics_has_endpoint_secret(result) && !verify_transport_security(s->cct, *(s->info.env))) {
- ldpp_dout(this, 1) << "topics contain secrets and cannot be sent over insecure transport" << dendl;
- op_ret = -EPERM;
- return;
- }
- ldpp_dout(this, 20) << "successfully got topics" << dendl;
-}
-
-// command (extension to AWS):
-// POST
-// Action=GetTopic&TopicArn=<topic-arn>
-class RGWPSGetTopicOp : public RGWOp {
- private:
- std::string topic_name;
- rgw_pubsub_topic result;
-
- int get_params() {
- const auto topic_arn = rgw::ARN::parse((s->info.args.get("TopicArn")));
-
- if (!topic_arn || topic_arn->resource.empty()) {
- ldpp_dout(this, 1) << "GetTopic Action 'TopicArn' argument is missing or invalid" << dendl;
- return -EINVAL;
- }
-
- topic_name = topic_arn->resource;
- return 0;
- }
-
- public:
- int verify_permission(optional_yield y) override {
- return 0;
- }
- void pre_exec() override {
- rgw_bucket_object_pre_exec(s);
- }
- void execute(optional_yield y) override;
-
- const char* name() const override { return "pubsub_topic_get"; }
- RGWOpType get_type() override { return RGW_OP_PUBSUB_TOPIC_GET; }
- uint32_t op_mask() override { return RGW_OP_TYPE_READ; }
-
- void send_response() override {
- if (op_ret) {
- set_req_state_err(s, op_ret);
- }
- dump_errno(s);
- end_header(s, this, "application/xml");
-
- if (op_ret < 0) {
- return;
- }
-
- const auto f = s->formatter;
- f->open_object_section("GetTopicResponse");
- f->open_object_section("GetTopicResult");
- encode_xml("Topic", result, f);
- f->close_section();
- f->open_object_section("ResponseMetadata");
- encode_xml("RequestId", s->req_id, f);
- f->close_section();
- f->close_section();
- rgw_flush_formatter_and_reset(s, f);
- }
-};
-
-void RGWPSGetTopicOp::execute(optional_yield y) {
- op_ret = get_params();
- if (op_ret < 0) {
- return;
- }
- RGWPubSub ps(static_cast<rgw::sal::RadosStore*>(driver), s->owner.get_id().tenant);
- op_ret = ps.get_topic(topic_name, &result);
- if (op_ret < 0) {
- ldpp_dout(this, 1) << "failed to get topic '" << topic_name << "', ret=" << op_ret << dendl;
- return;
- }
- if (topic_has_endpoint_secret(result) && !verify_transport_security(s->cct, *(s->info.env))) {
- ldpp_dout(this, 1) << "topic '" << topic_name << "' contain secret and cannot be sent over insecure transport" << dendl;
- op_ret = -EPERM;
- return;
- }
- ldpp_dout(this, 1) << "successfully got topic '" << topic_name << "'" << dendl;
-}
-
-// command (AWS compliant):
-// POST
-// Action=GetTopicAttributes&TopicArn=<topic-arn>
-class RGWPSGetTopicAttributesOp : public RGWOp {
- private:
- std::string topic_name;
- rgw_pubsub_topic result;
-
- int get_params() {
- const auto topic_arn = rgw::ARN::parse((s->info.args.get("TopicArn")));
-
- if (!topic_arn || topic_arn->resource.empty()) {
- ldpp_dout(this, 1) << "GetTopicAttribute Action 'TopicArn' argument is missing or invalid" << dendl;
- return -EINVAL;
- }
-
- topic_name = topic_arn->resource;
- return 0;
- }
-
- public:
- int verify_permission(optional_yield y) override {
- return 0;
- }
- void pre_exec() override {
- rgw_bucket_object_pre_exec(s);
- }
- void execute(optional_yield y) override;
-
- const char* name() const override { return "pubsub_topic_get"; }
- RGWOpType get_type() override { return RGW_OP_PUBSUB_TOPIC_GET; }
- uint32_t op_mask() override { return RGW_OP_TYPE_READ; }
-
- void send_response() override {
- if (op_ret) {
- set_req_state_err(s, op_ret);
- }
- dump_errno(s);
- end_header(s, this, "application/xml");
-
- if (op_ret < 0) {
- return;
- }
-
- const auto f = s->formatter;
- f->open_object_section_in_ns("GetTopicAttributesResponse", AWS_SNS_NS);
- f->open_object_section("GetTopicAttributesResult");
- result.dump_xml_as_attributes(f);
- f->close_section(); // GetTopicAttributesResult
- f->open_object_section("ResponseMetadata");
- encode_xml("RequestId", s->req_id, f);
- f->close_section(); // ResponseMetadata
- f->close_section(); // GetTopicAttributesResponse
- rgw_flush_formatter_and_reset(s, f);
- }
-};
-
-void RGWPSGetTopicAttributesOp::execute(optional_yield y) {
- op_ret = get_params();
- if (op_ret < 0) {
- return;
- }
- RGWPubSub ps(static_cast<rgw::sal::RadosStore*>(driver), s->owner.get_id().tenant);
- op_ret = ps.get_topic(topic_name, &result);
- if (op_ret < 0) {
- ldpp_dout(this, 1) << "failed to get topic '" << topic_name << "', ret=" << op_ret << dendl;
- return;
- }
- if (topic_has_endpoint_secret(result) && !verify_transport_security(s->cct, *(s->info.env))) {
- ldpp_dout(this, 1) << "topic '" << topic_name << "' contain secret and cannot be sent over insecure transport" << dendl;
- op_ret = -EPERM;
- return;
- }
- ldpp_dout(this, 1) << "successfully got topic '" << topic_name << "'" << dendl;
-}
-
-// command (AWS compliant):
-// POST
-// Action=DeleteTopic&TopicArn=<topic-arn>
-class RGWPSDeleteTopicOp : public RGWOp {
- private:
- std::string topic_name;
-
- int get_params() {
- const auto topic_arn = rgw::ARN::parse((s->info.args.get("TopicArn")));
-
- if (!topic_arn || topic_arn->resource.empty()) {
- ldpp_dout(this, 1) << "DeleteTopic Action 'TopicArn' argument is missing or invalid" << dendl;
- return -EINVAL;
- }
-
- topic_name = topic_arn->resource;
-
- // upon deletion it is not known if topic is persistent or not
- // will try to delete the persistent topic anyway
- const auto ret = rgw::notify::remove_persistent_topic(topic_name, s->yield);
- if (ret == -ENOENT) {
- // topic was not persistent, or already deleted
- return 0;
- }
- if (ret < 0) {
- ldpp_dout(this, 1) << "DeleteTopic Action failed to remove queue for persistent topics. error:" << ret << dendl;
- return ret;
- }
-
- return 0;
- }
-
- public:
- int verify_permission(optional_yield) override {
- return 0;
- }
- void pre_exec() override {
- rgw_bucket_object_pre_exec(s);
- }
- void execute(optional_yield y) override;
-
- const char* name() const override { return "pubsub_topic_delete"; }
- RGWOpType get_type() override { return RGW_OP_PUBSUB_TOPIC_DELETE; }
- uint32_t op_mask() override { return RGW_OP_TYPE_DELETE; }
-
- void send_response() override {
- if (op_ret) {
- set_req_state_err(s, op_ret);
- }
- dump_errno(s);
- end_header(s, this, "application/xml");
-
- if (op_ret < 0) {
- return;
- }
-
- const auto f = s->formatter;
- f->open_object_section_in_ns("DeleteTopicResponse", AWS_SNS_NS);
- f->open_object_section("ResponseMetadata");
- encode_xml("RequestId", s->req_id, f);
- f->close_section(); // ResponseMetadata
- f->close_section(); // DeleteTopicResponse
- rgw_flush_formatter_and_reset(s, f);
- }
-};
-
-void RGWPSDeleteTopicOp::execute(optional_yield y) {
- op_ret = get_params();
- if (op_ret < 0) {
- return;
- }
- RGWPubSub ps(static_cast<rgw::sal::RadosStore*>(driver), s->owner.get_id().tenant);
- op_ret = ps.remove_topic(this, topic_name, y);
- if (op_ret < 0) {
- ldpp_dout(this, 1) << "failed to remove topic '" << topic_name << ", ret=" << op_ret << dendl;
- return;
- }
- ldpp_dout(this, 1) << "successfully removed topic '" << topic_name << "'" << dendl;
-}
-
-using op_generator = RGWOp*(*)();
-static const std::unordered_map<std::string, op_generator> op_generators = {
- {"CreateTopic", []() -> RGWOp* {return new RGWPSCreateTopicOp;}},
- {"DeleteTopic", []() -> RGWOp* {return new RGWPSDeleteTopicOp;}},
- {"ListTopics", []() -> RGWOp* {return new RGWPSListTopicsOp;}},
- {"GetTopic", []() -> RGWOp* {return new RGWPSGetTopicOp;}},
- {"GetTopicAttributes", []() -> RGWOp* {return new RGWPSGetTopicAttributesOp;}}
-};
-
-bool RGWHandler_REST_PSTopic_AWS::action_exists(const req_state* s)
-{
- if (s->info.args.exists("Action")) {
- const std::string action_name = s->info.args.get("Action");
- return op_generators.contains(action_name);
- }
- return false;
-}
-
-RGWOp *RGWHandler_REST_PSTopic_AWS::op_post()
-{
- s->dialect = "sns";
- s->prot_flags = RGW_REST_STS;
-
- if (s->info.args.exists("Action")) {
- const std::string action_name = s->info.args.get("Action");
- const auto action_it = op_generators.find(action_name);
- if (action_it != op_generators.end()) {
- return action_it->second();
- }
- ldpp_dout(s, 10) << "unknown action '" << action_name << "' for Topic handler" << dendl;
- } else {
- ldpp_dout(s, 10) << "missing action argument in Topic handler" << dendl;
- }
- return nullptr;
-}
-
-int RGWHandler_REST_PSTopic_AWS::authorize(const DoutPrefixProvider* dpp, optional_yield y) {
- const auto rc = RGW_Auth_S3::authorize(dpp, driver, auth_registry, s, y);
- if (rc < 0) {
- return rc;
- }
- if (s->auth.identity->is_anonymous()) {
- ldpp_dout(dpp, 1) << "anonymous user not allowed in topic operations" << dendl;
- return -ERR_INVALID_REQUEST;
- }
- return 0;
-}
-
-namespace {
-// return a unique topic by prefexing with the notification name: <notification>_<topic>
-std::string topic_to_unique(const std::string& topic, const std::string& notification) {
- return notification + "_" + topic;
-}
-
-// extract the topic from a unique topic of the form: <notification>_<topic>
-[[maybe_unused]] std::string unique_to_topic(const std::string& unique_topic, const std::string& notification) {
- if (unique_topic.find(notification + "_") == std::string::npos) {
- return "";
- }
- return unique_topic.substr(notification.length() + 1);
-}
-
-// from list of bucket topics, find the one that was auto-generated by a notification
-auto find_unique_topic(const rgw_pubsub_bucket_topics& bucket_topics, const std::string& notif_name) {
- auto it = std::find_if(bucket_topics.topics.begin(), bucket_topics.topics.end(), [&](const auto& val) { return notif_name == val.second.s3_id; });
- return it != bucket_topics.topics.end() ?
- std::optional<std::reference_wrapper<const rgw_pubsub_topic_filter>>(it->second):
- std::nullopt;
-}
-}
-
-int remove_notification_by_topic(const DoutPrefixProvider *dpp, const std::string& topic_name, const RGWPubSub::Bucket& b, optional_yield y, const RGWPubSub& ps) {
- int op_ret = b.remove_notification(dpp, topic_name, y);
- if (op_ret < 0) {
- ldpp_dout(dpp, 1) << "failed to remove notification of topic '" << topic_name << "', ret=" << op_ret << dendl;
- }
- op_ret = ps.remove_topic(dpp, topic_name, y);
- if (op_ret < 0) {
- ldpp_dout(dpp, 1) << "failed to remove auto-generated topic '" << topic_name << "', ret=" << op_ret << dendl;
- }
- return op_ret;
-}
-
-int delete_all_notifications(const DoutPrefixProvider *dpp, const rgw_pubsub_bucket_topics& bucket_topics, const RGWPubSub::Bucket& b, optional_yield y, const RGWPubSub& ps) {
- // delete all notifications of on a bucket
- for (const auto& topic : bucket_topics.topics) {
- const auto op_ret = remove_notification_by_topic(dpp, topic.first, b, y, ps);
- if (op_ret < 0) {
- return op_ret;
- }
- }
- return 0;
-}
-
-// command (S3 compliant): PUT /<bucket name>?notification
-// a "notification" and a subscription will be auto-generated
-// actual configuration is XML encoded in the body of the message
-class RGWPSCreateNotifOp : public RGWDefaultResponseOp {
- private:
- std::string bucket_name;
- RGWBucketInfo bucket_info;
- rgw_pubsub_s3_notifications configurations;
-
- int get_params() {
- bool exists;
- const auto no_value = s->info.args.get("notification", &exists);
- if (!exists) {
- ldpp_dout(this, 1) << "missing required param 'notification'" << dendl;
- return -EINVAL;
- }
- if (no_value.length() > 0) {
- ldpp_dout(this, 1) << "param 'notification' should not have any value" << dendl;
- return -EINVAL;
- }
- if (s->bucket_name.empty()) {
- ldpp_dout(this, 1) << "request must be on a bucket" << dendl;
- return -EINVAL;
- }
- bucket_name = s->bucket_name;
- return 0;
- }
-
- public:
- int verify_permission(optional_yield y) override;
-
- void pre_exec() override {
- rgw_bucket_object_pre_exec(s);
- }
-
- const char* name() const override { return "pubsub_notification_create_s3"; }
- RGWOpType get_type() override { return RGW_OP_PUBSUB_NOTIF_CREATE; }
- uint32_t op_mask() override { return RGW_OP_TYPE_WRITE; }
-
- int get_params_from_body() {
- const auto max_size = s->cct->_conf->rgw_max_put_param_size;
- int r;
- bufferlist data;
- std::tie(r, data) = read_all_input(s, max_size, false);
-
- if (r < 0) {
- ldpp_dout(this, 1) << "failed to read XML payload" << dendl;
- return r;
- }
- if (data.length() == 0) {
- ldpp_dout(this, 1) << "XML payload missing" << dendl;
- return -EINVAL;
- }
-
- RGWXMLDecoder::XMLParser parser;
-
- if (!parser.init()){
- ldpp_dout(this, 1) << "failed to initialize XML parser" << dendl;
- return -EINVAL;
- }
- if (!parser.parse(data.c_str(), data.length(), 1)) {
- ldpp_dout(this, 1) << "failed to parse XML payload" << dendl;
- return -ERR_MALFORMED_XML;
- }
- try {
- // NotificationConfigurations is mandatory
- // It can be empty which means we delete all the notifications
- RGWXMLDecoder::decode_xml("NotificationConfiguration", configurations, &parser, true);
- } catch (RGWXMLDecoder::err& err) {
- ldpp_dout(this, 1) << "failed to parse XML payload. error: " << err << dendl;
- return -ERR_MALFORMED_XML;
- }
- return 0;
- }
-
- void execute(optional_yield) override;
-};
-
-void RGWPSCreateNotifOp::execute(optional_yield y) {
- op_ret = get_params_from_body();
- if (op_ret < 0) {
- return;
- }
-
- const RGWPubSub ps(static_cast<rgw::sal::RadosStore*>(driver), s->owner.get_id().tenant);
- const RGWPubSub::Bucket b(ps, bucket_info.bucket);
-
- if(configurations.list.empty()) {
- // get all topics on a bucket
- rgw_pubsub_bucket_topics bucket_topics;
- op_ret = b.get_topics(&bucket_topics);
- if (op_ret < 0) {
- ldpp_dout(this, 1) << "failed to get list of topics from bucket '" << bucket_info.bucket.name << "', ret=" << op_ret << dendl;
- return;
- }
-
- op_ret = delete_all_notifications(this, bucket_topics, b, y, ps);
- return;
- }
-
- for (const auto& c : configurations.list) {
- const auto& notif_name = c.id;
- if (notif_name.empty()) {
- ldpp_dout(this, 1) << "missing notification id" << dendl;
- op_ret = -EINVAL;
- return;
- }
- if (c.topic_arn.empty()) {
- ldpp_dout(this, 1) << "missing topic ARN in notification: '" << notif_name << "'" << dendl;
- op_ret = -EINVAL;
- return;
- }
-
- const auto arn = rgw::ARN::parse(c.topic_arn);
- if (!arn || arn->resource.empty()) {
- ldpp_dout(this, 1) << "topic ARN has invalid format: '" << c.topic_arn << "' in notification: '" << notif_name << "'" << dendl;
- op_ret = -EINVAL;
- return;
- }
-
- if (std::find(c.events.begin(), c.events.end(), rgw::notify::UnknownEvent) != c.events.end()) {
- ldpp_dout(this, 1) << "unknown event type in notification: '" << notif_name << "'" << dendl;
- op_ret = -EINVAL;
- return;
- }
-
- const auto topic_name = arn->resource;
-
- // get topic information. destination information is stored in the topic
- rgw_pubsub_topic topic_info;
- op_ret = ps.get_topic(topic_name, &topic_info);
- if (op_ret < 0) {
- ldpp_dout(this, 1) << "failed to get topic '" << topic_name << "', ret=" << op_ret << dendl;
- return;
- }
- // make sure that full topic configuration match
- // TODO: use ARN match function
-
- // create unique topic name. this has 2 reasons:
- // (1) topics cannot be shared between different S3 notifications because they hold the filter information
- // (2) make topic clneaup easier, when notification is removed
- const auto unique_topic_name = topic_to_unique(topic_name, notif_name);
- // generate the internal topic. destination is stored here for the "push-only" case
- // when no subscription exists
- // ARN is cached to make the "GET" method faster
- op_ret = ps.create_topic(this, unique_topic_name, topic_info.dest, topic_info.arn, topic_info.opaque_data, y);
- if (op_ret < 0) {
- ldpp_dout(this, 1) << "failed to auto-generate unique topic '" << unique_topic_name <<
- "', ret=" << op_ret << dendl;
- return;
- }
- ldpp_dout(this, 20) << "successfully auto-generated unique topic '" << unique_topic_name << "'" << dendl;
- // generate the notification
- rgw::notify::EventTypeList events;
- op_ret = b.create_notification(this, unique_topic_name, c.events, std::make_optional(c.filter), notif_name, y);
- if (op_ret < 0) {
- ldpp_dout(this, 1) << "failed to auto-generate notification for unique topic '" << unique_topic_name <<
- "', ret=" << op_ret << dendl;
- // rollback generated topic (ignore return value)
- ps.remove_topic(this, unique_topic_name, y);
- return;
- }
- ldpp_dout(this, 20) << "successfully auto-generated notification for unique topic '" << unique_topic_name << "'" << dendl;
- }
-}
-
-int RGWPSCreateNotifOp::verify_permission(optional_yield y) {
- int ret = get_params();
- if (ret < 0) {
- return ret;
- }
-
- std::unique_ptr<rgw::sal::User> user = driver->get_user(s->owner.get_id());
- std::unique_ptr<rgw::sal::Bucket> bucket;
- ret = driver->get_bucket(this, user.get(), s->owner.get_id().tenant, bucket_name, &bucket, y);
- if (ret < 0) {
- ldpp_dout(this, 1) << "failed to get bucket info, cannot verify ownership" << dendl;
- return ret;
- }
- bucket_info = bucket->get_info();
-
- if (bucket_info.owner != s->owner.get_id()) {
- ldpp_dout(this, 1) << "user doesn't own bucket, not allowed to create notification" << dendl;
- return -EPERM;
- }
- return 0;
-}
-
-// command (extension to S3): DELETE /bucket?notification[=<notification-id>]
-class RGWPSDeleteNotifOp : public RGWDefaultResponseOp {
- private:
- std::string bucket_name;
- RGWBucketInfo bucket_info;
- std::string notif_name;
-
- public:
- int verify_permission(optional_yield y) override;
-
- void pre_exec() override {
- rgw_bucket_object_pre_exec(s);
- }
-
- const char* name() const override { return "pubsub_notification_delete_s3"; }
- RGWOpType get_type() override { return RGW_OP_PUBSUB_NOTIF_DELETE; }
- uint32_t op_mask() override { return RGW_OP_TYPE_DELETE; }
-
- int get_params() {
- bool exists;
- notif_name = s->info.args.get("notification", &exists);
- if (!exists) {
- ldpp_dout(this, 1) << "missing required param 'notification'" << dendl;
- return -EINVAL;
- }
- if (s->bucket_name.empty()) {
- ldpp_dout(this, 1) << "request must be on a bucket" << dendl;
- return -EINVAL;
- }
- bucket_name = s->bucket_name;
- return 0;
- }
-
- void execute(optional_yield y) override;
-};
-
-void RGWPSDeleteNotifOp::execute(optional_yield y) {
- op_ret = get_params();
- if (op_ret < 0) {
- return;
- }
-
- const RGWPubSub ps(static_cast<rgw::sal::RadosStore*>(driver), s->owner.get_id().tenant);
- const RGWPubSub::Bucket b(ps, bucket_info.bucket);
-
- // get all topics on a bucket
- rgw_pubsub_bucket_topics bucket_topics;
- op_ret = b.get_topics(&bucket_topics);
- if (op_ret < 0) {
- ldpp_dout(this, 1) << "failed to get list of topics from bucket '" << bucket_info.bucket.name << "', ret=" << op_ret << dendl;
- return;
- }
-
- if (!notif_name.empty()) {
- // delete a specific notification
- const auto unique_topic = find_unique_topic(bucket_topics, notif_name);
- if (unique_topic) {
- const auto unique_topic_name = unique_topic->get().topic.name;
- op_ret = remove_notification_by_topic(this, unique_topic_name, b, y, ps);
- return;
- }
- // notification to be removed is not found - considered success
- ldpp_dout(this, 20) << "notification '" << notif_name << "' already removed" << dendl;
- return;
- }
-
- op_ret = delete_all_notifications(this, bucket_topics, b, y, ps);
-}
-
-int RGWPSDeleteNotifOp::verify_permission(optional_yield y) {
- int ret = get_params();
- if (ret < 0) {
- return ret;
- }
-
- std::unique_ptr<rgw::sal::User> user = driver->get_user(s->owner.get_id());
- std::unique_ptr<rgw::sal::Bucket> bucket;
- ret = driver->get_bucket(this, user.get(), s->owner.get_id().tenant, bucket_name, &bucket, y);
- if (ret < 0) {
- return ret;
- }
- bucket_info = bucket->get_info();
-
- if (bucket_info.owner != s->owner.get_id()) {
- ldpp_dout(this, 1) << "user doesn't own bucket, cannot remove notification" << dendl;
- return -EPERM;
- }
- return 0;
-}
-
-// command (S3 compliant): GET /bucket?notification[=<notification-id>]
-class RGWPSListNotifsOp : public RGWOp {
-private:
- std::string bucket_name;
- RGWBucketInfo bucket_info;
- std::string notif_name;
- rgw_pubsub_s3_notifications notifications;
-
- int get_params() {
- bool exists;
- notif_name = s->info.args.get("notification", &exists);
- if (!exists) {
- ldpp_dout(this, 1) << "missing required param 'notification'" << dendl;
- return -EINVAL;
- }
- if (s->bucket_name.empty()) {
- ldpp_dout(this, 1) << "request must be on a bucket" << dendl;
- return -EINVAL;
- }
- bucket_name = s->bucket_name;
- return 0;
- }
-
- public:
- int verify_permission(optional_yield y) override;
-
- void pre_exec() override {
- rgw_bucket_object_pre_exec(s);
- }
-
- const char* name() const override { return "pubsub_notifications_get_s3"; }
- RGWOpType get_type() override { return RGW_OP_PUBSUB_NOTIF_LIST; }
- uint32_t op_mask() override { return RGW_OP_TYPE_READ; }
-
- void execute(optional_yield y) override;
- void send_response() override {
- if (op_ret) {
- set_req_state_err(s, op_ret);
- }
- dump_errno(s);
- end_header(s, this, "application/xml");
-
- if (op_ret < 0) {
- return;
- }
- notifications.dump_xml(s->formatter);
- rgw_flush_formatter_and_reset(s, s->formatter);
- }
-};
-
-void RGWPSListNotifsOp::execute(optional_yield y) {
- const RGWPubSub ps(static_cast<rgw::sal::RadosStore*>(driver), s->owner.get_id().tenant);
- const RGWPubSub::Bucket b(ps, bucket_info.bucket);
-
- // get all topics on a bucket
- rgw_pubsub_bucket_topics bucket_topics;
- op_ret = b.get_topics(&bucket_topics);
- if (op_ret < 0) {
- ldpp_dout(this, 1) << "failed to get list of topics from bucket '" << bucket_info.bucket.name << "', ret=" << op_ret << dendl;
- return;
- }
- if (!notif_name.empty()) {
- // get info of a specific notification
- const auto unique_topic = find_unique_topic(bucket_topics, notif_name);
- if (unique_topic) {
- notifications.list.emplace_back(unique_topic->get());
- return;
- }
- op_ret = -ENOENT;
- ldpp_dout(this, 1) << "failed to get notification info for '" << notif_name << "', ret=" << op_ret << dendl;
- return;
- }
- // loop through all topics of the bucket
- for (const auto& topic : bucket_topics.topics) {
- if (topic.second.s3_id.empty()) {
- // not an s3 notification
- continue;
- }
- notifications.list.emplace_back(topic.second);
- }
-}
-
-int RGWPSListNotifsOp::verify_permission(optional_yield y) {
- int ret = get_params();
- if (ret < 0) {
- return ret;
- }
-
- std::unique_ptr<rgw::sal::User> user = driver->get_user(s->owner.get_id());
- std::unique_ptr<rgw::sal::Bucket> bucket;
- ret = driver->get_bucket(this, user.get(), s->owner.get_id().tenant, bucket_name, &bucket, y);
- if (ret < 0) {
- return ret;
- }
- bucket_info = bucket->get_info();
-
- if (bucket_info.owner != s->owner.get_id()) {
- ldpp_dout(this, 1) << "user doesn't own bucket, cannot get notification list" << dendl;
- return -EPERM;
- }
-
- return 0;
-}
-
-RGWOp* RGWHandler_REST_PSNotifs_S3::op_get() {
- return new RGWPSListNotifsOp();
-}
-
-RGWOp* RGWHandler_REST_PSNotifs_S3::op_put() {
- return new RGWPSCreateNotifOp();
-}
-
-RGWOp* RGWHandler_REST_PSNotifs_S3::op_delete() {
- return new RGWPSDeleteNotifOp();
-}
-
-RGWOp* RGWHandler_REST_PSNotifs_S3::create_get_op() {
- return new RGWPSListNotifsOp();
-}
-
-RGWOp* RGWHandler_REST_PSNotifs_S3::create_put_op() {
- return new RGWPSCreateNotifOp();
-}
-
-RGWOp* RGWHandler_REST_PSNotifs_S3::create_delete_op() {
- return new RGWPSDeleteNotifOp();
-}
-
// default number of entries to list with each bucket listing call
// (use marker to bridge between calls)
static constexpr size_t listing_max_entries = 1000;
+static std::string pubsub_oid_prefix = "pubsub.";
static int decode_policy(CephContext* cct,
bufferlist& bl,
// if bucket has notification definitions associated with it
// they should be removed (note that any pending notifications on the bucket are still going to be sent)
const RGWPubSub ps(store, info.owner.tenant);
- const RGWPubSub::Bucket ps_bucket(ps, info.bucket);
+ const RGWPubSub::Bucket ps_bucket(ps, this);
const auto ps_ret = ps_bucket.remove_notifications(dpp, y);
if (ps_ret < 0 && ps_ret != -ENOENT) {
ldpp_dout(dpp, -1) << "ERROR: unable to remove notifications from bucket. ret=" << ps_ret << dendl;
return 0;
}
+std::string RadosBucket::topics_oid() const {
+ return pubsub_oid_prefix + get_tenant() + ".bucket." + get_name() + "/" + get_marker();
+}
+
+int RadosBucket::read_topics(rgw_pubsub_bucket_topics& notifications,
+ RGWObjVersionTracker* objv_tracker, optional_yield y, const DoutPrefixProvider *dpp)
+{
+ bufferlist bl;
+ const int ret = rgw_get_system_obj(store->svc()->sysobj,
+ store->svc()->zone->get_zone_params().log_pool,
+ topics_oid(),
+ bl,
+ objv_tracker,
+ nullptr, y, dpp, nullptr);
+ if (ret < 0) {
+ return ret;
+ }
+
+ auto iter = bl.cbegin();
+ try {
+ decode(notifications, iter);
+ } catch (buffer::error& err) {
+ ldpp_dout(dpp, 20) << " failed to decode bucket notifications from oid: " << topics_oid() << ". for bucket: "
+ << get_name() << ". error: " << err.what() << dendl;
+ return -EIO;
+ }
+
+ return 0;
+}
+
+int RadosBucket::write_topics(const rgw_pubsub_bucket_topics& notifications,
+ RGWObjVersionTracker* objv_tracker, optional_yield y, const DoutPrefixProvider *dpp) {
+ bufferlist bl;
+ encode(notifications, bl);
+
+ return rgw_put_system_obj(dpp, store->svc()->sysobj,
+ store->svc()->zone->get_zone_params().log_pool,
+ topics_oid(),
+ bl, false, objv_tracker, real_time(), y);
+}
+
+int RadosBucket::remove_topics(RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) {
+ return rgw_delete_system_obj(dpp, store->svc()->sysobj,
+ store->svc()->zone->get_zone_params().log_pool,
+ topics_oid(),
+ objv_tracker, y);
+}
+
std::unique_ptr<User> RadosStore::get_user(const rgw_user &u)
{
return std::make_unique<RadosUser>(this, u);
return std::make_unique<RadosNotification>(dpp, this, obj, src_obj, event_type, _bucket, _user_id, _user_tenant, _req_id, y);
}
+std::string RadosStore::topics_oid(const std::string& tenant) const {
+ return pubsub_oid_prefix + tenant;
+}
+
+int RadosStore::read_topics(const std::string& tenant, rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) {
+ bufferlist bl;
+ const int ret = rgw_get_system_obj(svc()->sysobj,
+ svc()->zone->get_zone_params().log_pool,
+ topics_oid(tenant),
+ bl,
+ objv_tracker,
+ nullptr, y, dpp, nullptr);
+ if (ret < 0) {
+ return ret;
+ }
+
+ auto iter = bl.cbegin();
+ try {
+ decode(topics, iter);
+ } catch (buffer::error& err) {
+ ldpp_dout(dpp, 20) << " failed to decode topics from oid: " << topics_oid(tenant) <<
+ ". error: " << err.what() << dendl;
+ return -EIO;
+ }
+
+ return 0;
+}
+
+int RadosStore::write_topics(const std::string& tenant, const rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) {
+ bufferlist bl;
+ encode(topics, bl);
+
+ return rgw_put_system_obj(dpp, svc()->sysobj,
+ svc()->zone->get_zone_params().log_pool,
+ topics_oid(tenant),
+ bl, false, objv_tracker, real_time(), y);
+}
+
+int RadosStore::remove_topics(const std::string& tenant, RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) {
+ return rgw_delete_system_obj(dpp, svc()->sysobj,
+ svc()->zone->get_zone_params().log_pool,
+ topics_oid(tenant),
+ objv_tracker, y);
+}
+
int RadosStore::delete_raw_obj(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj)
{
return rados->delete_raw_obj(dpp, obj);
RGWRados* rados;
RGWUserCtl* user_ctl;
std::unique_ptr<RadosZone> zone;
+ std::string topics_oid(const std::string& tenant) const;
public:
RadosStore()
const DoutPrefixProvider* dpp, rgw::sal::Object* obj, rgw::sal::Object* src_obj,
rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket, std::string& _user_id, std::string& _user_tenant,
std::string& _req_id, optional_yield y) override;
+ int read_topics(const std::string& tenant, rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) override;
+ int write_topics(const std::string& tenant, const rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) override;
+ int remove_topics(const std::string& tenant, RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) override;
virtual RGWLC* get_rgwlc(void) override { return rados->get_lc(); }
virtual RGWCoroutinesManagerRegistry* get_cr_registry() override { return rados->get_cr_registry(); }
private:
RadosStore* store;
RGWAccessControlPolicy acls;
+ std::string topics_oid() const;
public:
RadosBucket(RadosStore *_st)
bool *is_truncated) override;
virtual int abort_multiparts(const DoutPrefixProvider* dpp,
CephContext* cct) override;
+ int read_topics(rgw_pubsub_bucket_topics& notifications, RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) override;
+ int write_topics(const rgw_pubsub_bucket_topics& notifications, RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) override;
+ int remove_topics(RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) override;
private:
int link(const DoutPrefixProvider* dpp, User* new_user, optional_yield y, bool update_entrypoint = true, RGWObjVersionTracker* objv = nullptr);
if (opt_cmd == OPT::PUBSUB_TOPICS_LIST) {
- RGWPubSub ps(static_cast<rgw::sal::RadosStore*>(driver), tenant);
+ RGWPubSub ps(driver, tenant);
if (!bucket_name.empty()) {
rgw_pubsub_bucket_topics result;
return -ret;
}
- const RGWPubSub::Bucket b(ps, bucket->get_key());
- ret = b.get_topics(&result);
+ const RGWPubSub::Bucket b(ps, bucket.get());
+ ret = b.get_topics(dpp(), &result, null_yield);
if (ret < 0 && ret != -ENOENT) {
cerr << "ERROR: could not get topics: " << cpp_strerror(-ret) << std::endl;
return -ret;
encode_json("result", result, formatter.get());
} else {
rgw_pubsub_topics result;
- int ret = ps.get_topics(&result);
+ int ret = ps.get_topics(dpp(), &result, null_yield);
if (ret < 0 && ret != -ENOENT) {
cerr << "ERROR: could not get topics: " << cpp_strerror(-ret) << std::endl;
return -ret;
return EINVAL;
}
- RGWPubSub ps(static_cast<rgw::sal::RadosStore*>(driver), tenant);
+ RGWPubSub ps(driver, tenant);
rgw_pubsub_topic topic;
- ret = ps.get_topic(topic_name, &topic);
+ ret = ps.get_topic(dpp(), topic_name, &topic, null_yield);
if (ret < 0) {
cerr << "ERROR: could not get topic: " << cpp_strerror(-ret) << std::endl;
return -ret;
return EINVAL;
}
- RGWPubSub ps(static_cast<rgw::sal::RadosStore*>(driver), tenant);
+ RGWPubSub ps(driver, tenant);
ret = ps.remove_topic(dpp(), topic_name, null_yield);
if (ret < 0) {
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+#include "services/svc_zone.h"
+#include "rgw_b64.h"
+#include "rgw_sal.h"
+#include "rgw_pubsub.h"
+#include "rgw_tools.h"
+#include "rgw_xml.h"
+#include "rgw_arn.h"
+#include "rgw_pubsub_push.h"
+#include <regex>
+#include <algorithm>
+
+#define dout_subsys ceph_subsys_rgw
+
+void set_event_id(std::string& id, const std::string& hash, const utime_t& ts) {
+ char buf[64];
+ const auto len = snprintf(buf, sizeof(buf), "%010ld.%06ld.%s", (long)ts.sec(), (long)ts.usec(), hash.c_str());
+ if (len > 0) {
+ id.assign(buf, len);
+ }
+}
+
+bool rgw_s3_key_filter::decode_xml(XMLObj* obj) {
+ XMLObjIter iter = obj->find("FilterRule");
+ XMLObj *o;
+
+ const auto throw_if_missing = true;
+ auto prefix_not_set = true;
+ auto suffix_not_set = true;
+ auto regex_not_set = true;
+ std::string name;
+
+ while ((o = iter.get_next())) {
+ RGWXMLDecoder::decode_xml("Name", name, o, throw_if_missing);
+ if (name == "prefix" && prefix_not_set) {
+ prefix_not_set = false;
+ RGWXMLDecoder::decode_xml("Value", prefix_rule, o, throw_if_missing);
+ } else if (name == "suffix" && suffix_not_set) {
+ suffix_not_set = false;
+ RGWXMLDecoder::decode_xml("Value", suffix_rule, o, throw_if_missing);
+ } else if (name == "regex" && regex_not_set) {
+ regex_not_set = false;
+ RGWXMLDecoder::decode_xml("Value", regex_rule, o, throw_if_missing);
+ } else {
+ throw RGWXMLDecoder::err("invalid/duplicate S3Key filter rule name: '" + name + "'");
+ }
+ }
+ return true;
+}
+
+void rgw_s3_key_filter::dump_xml(Formatter *f) const {
+ if (!prefix_rule.empty()) {
+ f->open_object_section("FilterRule");
+ ::encode_xml("Name", "prefix", f);
+ ::encode_xml("Value", prefix_rule, f);
+ f->close_section();
+ }
+ if (!suffix_rule.empty()) {
+ f->open_object_section("FilterRule");
+ ::encode_xml("Name", "suffix", f);
+ ::encode_xml("Value", suffix_rule, f);
+ f->close_section();
+ }
+ if (!regex_rule.empty()) {
+ f->open_object_section("FilterRule");
+ ::encode_xml("Name", "regex", f);
+ ::encode_xml("Value", regex_rule, f);
+ f->close_section();
+ }
+}
+
+bool rgw_s3_key_filter::has_content() const {
+ return !(prefix_rule.empty() && suffix_rule.empty() && regex_rule.empty());
+}
+
+bool rgw_s3_key_value_filter::decode_xml(XMLObj* obj) {
+ kv.clear();
+ XMLObjIter iter = obj->find("FilterRule");
+ XMLObj *o;
+
+ const auto throw_if_missing = true;
+
+ std::string key;
+ std::string value;
+
+ while ((o = iter.get_next())) {
+ RGWXMLDecoder::decode_xml("Name", key, o, throw_if_missing);
+ RGWXMLDecoder::decode_xml("Value", value, o, throw_if_missing);
+ kv.emplace(key, value);
+ }
+ return true;
+}
+
+void rgw_s3_key_value_filter::dump_xml(Formatter *f) const {
+ for (const auto& key_value : kv) {
+ f->open_object_section("FilterRule");
+ ::encode_xml("Name", key_value.first, f);
+ ::encode_xml("Value", key_value.second, f);
+ f->close_section();
+ }
+}
+
+bool rgw_s3_key_value_filter::has_content() const {
+ return !kv.empty();
+}
+
+bool rgw_s3_filter::decode_xml(XMLObj* obj) {
+ RGWXMLDecoder::decode_xml("S3Key", key_filter, obj);
+ RGWXMLDecoder::decode_xml("S3Metadata", metadata_filter, obj);
+ RGWXMLDecoder::decode_xml("S3Tags", tag_filter, obj);
+ return true;
+}
+
+void rgw_s3_filter::dump_xml(Formatter *f) const {
+ if (key_filter.has_content()) {
+ ::encode_xml("S3Key", key_filter, f);
+ }
+ if (metadata_filter.has_content()) {
+ ::encode_xml("S3Metadata", metadata_filter, f);
+ }
+ if (tag_filter.has_content()) {
+ ::encode_xml("S3Tags", tag_filter, f);
+ }
+}
+
+bool rgw_s3_filter::has_content() const {
+ return key_filter.has_content() ||
+ metadata_filter.has_content() ||
+ tag_filter.has_content();
+}
+
+bool match(const rgw_s3_key_filter& filter, const std::string& key) {
+ const auto key_size = key.size();
+ const auto prefix_size = filter.prefix_rule.size();
+ if (prefix_size != 0) {
+ // prefix rule exists
+ if (prefix_size > key_size) {
+ // if prefix is longer than key, we fail
+ return false;
+ }
+ if (!std::equal(filter.prefix_rule.begin(), filter.prefix_rule.end(), key.begin())) {
+ return false;
+ }
+ }
+ const auto suffix_size = filter.suffix_rule.size();
+ if (suffix_size != 0) {
+ // suffix rule exists
+ if (suffix_size > key_size) {
+ // if suffix is longer than key, we fail
+ return false;
+ }
+ if (!std::equal(filter.suffix_rule.begin(), filter.suffix_rule.end(), (key.end() - suffix_size))) {
+ return false;
+ }
+ }
+ if (!filter.regex_rule.empty()) {
+ // TODO add regex chaching in the filter
+ const std::regex base_regex(filter.regex_rule);
+ if (!std::regex_match(key, base_regex)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+bool match(const rgw_s3_key_value_filter& filter, const KeyValueMap& kv) {
+ // all filter pairs must exist with the same value in the object's metadata/tags
+ // object metadata/tags may include items not in the filter
+ return std::includes(kv.begin(), kv.end(), filter.kv.begin(), filter.kv.end());
+}
+
+bool match(const rgw_s3_key_value_filter& filter, const KeyMultiValueMap& kv) {
+ // all filter pairs must exist with the same value in the object's metadata/tags
+ // object metadata/tags may include items not in the filter
+ for (auto& filter : filter.kv) {
+ auto result = kv.equal_range(filter.first);
+ if (std::any_of(result.first, result.second, [&filter](const std::pair<std::string, std::string>& p) { return p.second == filter.second;}))
+ continue;
+ else
+ return false;
+ }
+ return true;
+}
+
+bool match(const rgw::notify::EventTypeList& events, rgw::notify::EventType event) {
+ // if event list exists, and none of the events in the list matches the event type, filter the message
+ if (!events.empty() && std::find(events.begin(), events.end(), event) == events.end()) {
+ return false;
+ }
+ return true;
+}
+
+void do_decode_xml_obj(rgw::notify::EventTypeList& l, const std::string& name, XMLObj *obj) {
+ l.clear();
+
+ XMLObjIter iter = obj->find(name);
+ XMLObj *o;
+
+ while ((o = iter.get_next())) {
+ std::string val;
+ decode_xml_obj(val, o);
+ l.push_back(rgw::notify::from_string(val));
+ }
+}
+
+bool rgw_pubsub_s3_notification::decode_xml(XMLObj *obj) {
+ const auto throw_if_missing = true;
+ RGWXMLDecoder::decode_xml("Id", id, obj, throw_if_missing);
+
+ RGWXMLDecoder::decode_xml("Topic", topic_arn, obj, throw_if_missing);
+
+ RGWXMLDecoder::decode_xml("Filter", filter, obj);
+
+ do_decode_xml_obj(events, "Event", obj);
+ if (events.empty()) {
+ // if no events are provided, we assume all events
+ events.push_back(rgw::notify::ObjectCreated);
+ events.push_back(rgw::notify::ObjectRemoved);
+ }
+ return true;
+}
+
+void rgw_pubsub_s3_notification::dump_xml(Formatter *f) const {
+ ::encode_xml("Id", id, f);
+ ::encode_xml("Topic", topic_arn.c_str(), f);
+ if (filter.has_content()) {
+ ::encode_xml("Filter", filter, f);
+ }
+ for (const auto& event : events) {
+ ::encode_xml("Event", rgw::notify::to_string(event), f);
+ }
+}
+
+bool rgw_pubsub_s3_notifications::decode_xml(XMLObj *obj) {
+ do_decode_xml_obj(list, "TopicConfiguration", obj);
+ return true;
+}
+
+rgw_pubsub_s3_notification::rgw_pubsub_s3_notification(const rgw_pubsub_topic_filter& topic_filter) :
+ id(topic_filter.s3_id), events(topic_filter.events), topic_arn(topic_filter.topic.arn), filter(topic_filter.s3_filter) {}
+
+void rgw_pubsub_s3_notifications::dump_xml(Formatter *f) const {
+ do_encode_xml("NotificationConfiguration", list, "TopicConfiguration", f);
+}
+
+void rgw_pubsub_s3_event::dump(Formatter *f) const {
+ encode_json("eventVersion", eventVersion, f);
+ encode_json("eventSource", eventSource, f);
+ encode_json("awsRegion", awsRegion, f);
+ utime_t ut(eventTime);
+ encode_json("eventTime", ut, f);
+ encode_json("eventName", eventName, f);
+ {
+ Formatter::ObjectSection s(*f, "userIdentity");
+ encode_json("principalId", userIdentity, f);
+ }
+ {
+ Formatter::ObjectSection s(*f, "requestParameters");
+ encode_json("sourceIPAddress", sourceIPAddress, f);
+ }
+ {
+ Formatter::ObjectSection s(*f, "responseElements");
+ encode_json("x-amz-request-id", x_amz_request_id, f);
+ encode_json("x-amz-id-2", x_amz_id_2, f);
+ }
+ {
+ Formatter::ObjectSection s(*f, "s3");
+ encode_json("s3SchemaVersion", s3SchemaVersion, f);
+ encode_json("configurationId", configurationId, f);
+ {
+ Formatter::ObjectSection sub_s(*f, "bucket");
+ encode_json("name", bucket_name, f);
+ {
+ Formatter::ObjectSection sub_sub_s(*f, "ownerIdentity");
+ encode_json("principalId", bucket_ownerIdentity, f);
+ }
+ encode_json("arn", bucket_arn, f);
+ encode_json("id", bucket_id, f);
+ }
+ {
+ Formatter::ObjectSection sub_s(*f, "object");
+ encode_json("key", object_key, f);
+ encode_json("size", object_size, f);
+ encode_json("eTag", object_etag, f);
+ encode_json("versionId", object_versionId, f);
+ encode_json("sequencer", object_sequencer, f);
+ encode_json("metadata", x_meta_map, f);
+ encode_json("tags", tags, f);
+ }
+ }
+ encode_json("eventId", id, f);
+ encode_json("opaqueData", opaque_data, f);
+}
+
+void rgw_pubsub_topic::dump(Formatter *f) const
+{
+ encode_json("user", user, f);
+ encode_json("name", name, f);
+ encode_json("dest", dest, f);
+ encode_json("arn", arn, f);
+ encode_json("opaqueData", opaque_data, f);
+}
+
+void rgw_pubsub_topic::dump_xml(Formatter *f) const
+{
+ encode_xml("User", user, f);
+ encode_xml("Name", name, f);
+ encode_xml("EndPoint", dest, f);
+ encode_xml("TopicArn", arn, f);
+ encode_xml("OpaqueData", opaque_data, f);
+}
+
+void encode_xml_key_value_entry(const std::string& key, const std::string& value, Formatter *f) {
+ f->open_object_section("entry");
+ encode_xml("key", key, f);
+ encode_xml("value", value, f);
+ f->close_section(); // entry
+}
+
+void rgw_pubsub_topic::dump_xml_as_attributes(Formatter *f) const
+{
+ f->open_array_section("Attributes");
+ std::string str_user;
+ user.to_str(str_user);
+ encode_xml_key_value_entry("User", str_user, f);
+ encode_xml_key_value_entry("Name", name, f);
+ encode_xml_key_value_entry("EndPoint", dest.to_json_str(), f);
+ encode_xml_key_value_entry("TopicArn", arn, f);
+ encode_xml_key_value_entry("OpaqueData", opaque_data, f);
+ f->close_section(); // Attributes
+}
+
+void encode_json(const char *name, const rgw::notify::EventTypeList& l, Formatter *f)
+{
+ f->open_array_section(name);
+ for (auto iter = l.cbegin(); iter != l.cend(); ++iter) {
+ f->dump_string("obj", rgw::notify::to_string(*iter));
+ }
+ f->close_section();
+}
+
+void rgw_pubsub_topic_filter::dump(Formatter *f) const
+{
+ encode_json("topic", topic, f);
+ encode_json("events", events, f);
+}
+
+void rgw_pubsub_bucket_topics::dump(Formatter *f) const
+{
+ Formatter::ArraySection s(*f, "topics");
+ for (auto& t : topics) {
+ encode_json(t.first.c_str(), t.second, f);
+ }
+}
+
+void rgw_pubsub_topics::dump(Formatter *f) const
+{
+ Formatter::ArraySection s(*f, "topics");
+ for (auto& t : topics) {
+ encode_json(t.first.c_str(), t.second, f);
+ }
+}
+
+void rgw_pubsub_topics::dump_xml(Formatter *f) const
+{
+ for (auto& t : topics) {
+ encode_xml("member", t.second, f);
+ }
+}
+
+void rgw_pubsub_dest::dump(Formatter *f) const
+{
+ encode_json("push_endpoint", push_endpoint, f);
+ encode_json("push_endpoint_args", push_endpoint_args, f);
+ encode_json("push_endpoint_topic", arn_topic, f);
+ encode_json("stored_secret", stored_secret, f);
+ encode_json("persistent", persistent, f);
+}
+
+void rgw_pubsub_dest::dump_xml(Formatter *f) const
+{
+ encode_xml("EndpointAddress", push_endpoint, f);
+ encode_xml("EndpointArgs", push_endpoint_args, f);
+ encode_xml("EndpointTopic", arn_topic, f);
+ encode_xml("HasStoredSecret", stored_secret, f);
+ encode_xml("Persistent", persistent, f);
+}
+
+std::string rgw_pubsub_dest::to_json_str() const
+{
+ JSONFormatter f;
+ f.open_object_section("");
+ encode_json("EndpointAddress", push_endpoint, &f);
+ encode_json("EndpointArgs", push_endpoint_args, &f);
+ encode_json("EndpointTopic", arn_topic, &f);
+ encode_json("HasStoredSecret", stored_secret, &f);
+ encode_json("Persistent", persistent, &f);
+ f.close_section();
+ std::stringstream ss;
+ f.flush(ss);
+ return ss.str();
+}
+
+RGWPubSub::RGWPubSub(rgw::sal::Driver* _driver, const std::string& _tenant)
+ : driver(_driver), tenant(_tenant)
+{}
+
+int RGWPubSub::read_topics(const DoutPrefixProvider *dpp, rgw_pubsub_topics *result,
+ RGWObjVersionTracker *objv_tracker, optional_yield y) const
+{
+ const int ret = driver->read_topics(tenant, *result, objv_tracker, y, dpp);
+ if (ret < 0) {
+ ldpp_dout(dpp, 10) << "WARNING: failed to read topics info: ret=" << ret << dendl;
+ return ret;
+ }
+ return 0;
+}
+
+int RGWPubSub::write_topics(const DoutPrefixProvider *dpp, const rgw_pubsub_topics& topics,
+ RGWObjVersionTracker *objv_tracker, optional_yield y) const
+{
+ const int ret = driver->write_topics(tenant, topics, objv_tracker, y, dpp);
+ if (ret < 0 && ret != -ENOENT) {
+ ldpp_dout(dpp, 1) << "ERROR: failed to write topics info: ret=" << ret << dendl;
+ return ret;
+ }
+ return 0;
+}
+
+int RGWPubSub::Bucket::read_topics(const DoutPrefixProvider *dpp, rgw_pubsub_bucket_topics *result,
+ RGWObjVersionTracker *objv_tracker, optional_yield y) const
+{
+ const int ret = bucket->read_topics(*result, objv_tracker, y, dpp);
+ if (ret < 0 && ret != -ENOENT) {
+ ldpp_dout(dpp, 1) << "ERROR: failed to read bucket topics info: ret=" << ret << dendl;
+ return ret;
+ }
+ return 0;
+}
+
+int RGWPubSub::Bucket::write_topics(const DoutPrefixProvider *dpp, const rgw_pubsub_bucket_topics& topics,
+ RGWObjVersionTracker *objv_tracker,
+ optional_yield y) const
+{
+ const int ret = bucket->write_topics(topics, objv_tracker, y, dpp);
+ if (ret < 0) {
+ ldpp_dout(dpp, 1) << "ERROR: failed to write bucket topics info: ret=" << ret << dendl;
+ return ret;
+ }
+
+ return 0;
+}
+
+int RGWPubSub::get_topic(const DoutPrefixProvider *dpp, const std::string& name, rgw_pubsub_topic *result, optional_yield y) const
+{
+ rgw_pubsub_topics topics;
+ const int ret = read_topics(dpp, &topics, nullptr, y);
+ if (ret < 0) {
+ ldpp_dout(dpp, 1) << "ERROR: failed to read topics info: ret=" << ret << dendl;
+ return ret;
+ }
+
+ auto iter = topics.topics.find(name);
+ if (iter == topics.topics.end()) {
+ ldpp_dout(dpp, 1) << "ERROR: topic not found" << dendl;
+ return -ENOENT;
+ }
+
+ *result = iter->second;
+ return 0;
+}
+
+int RGWPubSub::Bucket::create_notification(const DoutPrefixProvider *dpp, const std::string& topic_name,
+ const rgw::notify::EventTypeList& events, optional_yield y) const {
+ return create_notification(dpp, topic_name, events, std::nullopt, "", y);
+}
+
+int RGWPubSub::Bucket::create_notification(const DoutPrefixProvider *dpp, const std::string& topic_name,
+ const rgw::notify::EventTypeList& events, OptionalFilter s3_filter, const std::string& notif_name, optional_yield y) const {
+ rgw_pubsub_topic topic_info;
+
+ int ret = ps.get_topic(dpp, topic_name, &topic_info, y);
+ if (ret < 0) {
+ ldpp_dout(dpp, 1) << "ERROR: failed to read topic '" << topic_name << "' info: ret=" << ret << dendl;
+ return ret;
+ }
+ ldpp_dout(dpp, 20) << "successfully read topic '" << topic_name << "' info" << dendl;
+
+ RGWObjVersionTracker objv_tracker;
+ rgw_pubsub_bucket_topics bucket_topics;
+
+ ret = read_topics(dpp, &bucket_topics, &objv_tracker, y);
+ if (ret < 0) {
+ ldpp_dout(dpp, 1) << "ERROR: failed to read topics from bucket '" <<
+ bucket->get_name() << "': ret=" << ret << dendl;
+ return ret;
+ }
+ ldpp_dout(dpp, 20) << "successfully read " << bucket_topics.topics.size() << " topics from bucket '" <<
+ bucket->get_name() << "'" << dendl;
+
+ auto& topic_filter = bucket_topics.topics[topic_name];
+ topic_filter.topic = topic_info;
+ topic_filter.events = events;
+ topic_filter.s3_id = notif_name;
+ if (s3_filter) {
+ topic_filter.s3_filter = *s3_filter;
+ }
+
+ ret = write_topics(dpp, bucket_topics, &objv_tracker, y);
+ if (ret < 0) {
+ ldpp_dout(dpp, 1) << "ERROR: failed to write topics to bucket '" << bucket->get_name() << "': ret=" << ret << dendl;
+ return ret;
+ }
+
+ ldpp_dout(dpp, 20) << "successfully wrote " << bucket_topics.topics.size() << " topics to bucket '" << bucket->get_name() << "'" << dendl;
+
+ return 0;
+}
+
+int RGWPubSub::Bucket::remove_notification(const DoutPrefixProvider *dpp, const std::string& topic_name, optional_yield y) const
+{
+ RGWObjVersionTracker objv_tracker;
+ rgw_pubsub_bucket_topics bucket_topics;
+
+ auto ret = read_topics(dpp, &bucket_topics, &objv_tracker, y);
+ if (ret < 0) {
+ ldpp_dout(dpp, 1) << "ERROR: failed to read bucket topics info: ret=" << ret << dendl;
+ return ret;
+ }
+
+ if (bucket_topics.topics.erase(topic_name) == 0) {
+ ldpp_dout(dpp, 1) << "INFO: no need to remove, topic does not exist" << dendl;
+ return 0;
+ }
+
+ if (bucket_topics.topics.empty()) {
+ // no more topics - delete the notification object of the bucket
+ ret = bucket->remove_topics(&objv_tracker, y, dpp);
+ if (ret < 0 && ret != -ENOENT) {
+ ldpp_dout(dpp, 1) << "ERROR: failed to remove bucket topics: ret=" << ret << dendl;
+ return ret;
+ }
+ return 0;
+ }
+
+ // write back the notifications without the deleted one
+ ret = write_topics(dpp, bucket_topics, &objv_tracker, y);
+ if (ret < 0) {
+ ldpp_dout(dpp, 1) << "ERROR: failed to write topics info: ret=" << ret << dendl;
+ return ret;
+ }
+
+ return 0;
+}
+
+int RGWPubSub::Bucket::remove_notifications(const DoutPrefixProvider *dpp, optional_yield y) const
+{
+ // get all topics on a bucket
+ rgw_pubsub_bucket_topics bucket_topics;
+ auto ret = get_topics(dpp, &bucket_topics, y);
+ if (ret < 0 && ret != -ENOENT) {
+ ldpp_dout(dpp, 1) << "ERROR: failed to get list of topics from bucket '" << bucket->get_name() << "', ret=" << ret << dendl;
+ return ret ;
+ }
+
+ // remove all auto-genrated topics
+ for (const auto& topic : bucket_topics.topics) {
+ const auto& topic_name = topic.first;
+ ret = ps.remove_topic(dpp, topic_name, y);
+ if (ret < 0 && ret != -ENOENT) {
+ ldpp_dout(dpp, 5) << "WARNING: failed to remove auto-generated topic '" << topic_name << "', ret=" << ret << dendl;
+ }
+ }
+
+ // delete the notification object of the bucket
+ ret = bucket->remove_topics(nullptr, y, dpp);
+ if (ret < 0 && ret != -ENOENT) {
+ ldpp_dout(dpp, 1) << "ERROR: failed to remove bucket topics: ret=" << ret << dendl;
+ return ret;
+ }
+
+ return 0;
+}
+
+int RGWPubSub::create_topic(const DoutPrefixProvider *dpp, const std::string& name, optional_yield y) const {
+ return create_topic(dpp, name, rgw_pubsub_dest{}, "", "", y);
+}
+
+int RGWPubSub::create_topic(const DoutPrefixProvider *dpp, const std::string& name, const rgw_pubsub_dest& dest,
+ const std::string& arn, const std::string& opaque_data, optional_yield y) const {
+ RGWObjVersionTracker objv_tracker;
+ rgw_pubsub_topics topics;
+
+ int ret = read_topics(dpp, &topics, &objv_tracker, y);
+ if (ret < 0 && ret != -ENOENT) {
+ // its not an error if not topics exist, we create one
+ ldpp_dout(dpp, 1) << "ERROR: failed to read topics info: ret=" << ret << dendl;
+ return ret;
+ }
+
+ rgw_pubsub_topic& new_topic = topics.topics[name];
+ new_topic.user = rgw_user("", tenant);
+ new_topic.name = name;
+ new_topic.dest = dest;
+ new_topic.arn = arn;
+ new_topic.opaque_data = opaque_data;
+
+ ret = write_topics(dpp, topics, &objv_tracker, y);
+ if (ret < 0) {
+ ldpp_dout(dpp, 1) << "ERROR: failed to write topics info: ret=" << ret << dendl;
+ return ret;
+ }
+
+ return 0;
+}
+
+int RGWPubSub::remove_topic(const DoutPrefixProvider *dpp, const std::string& name, optional_yield y) const
+{
+ RGWObjVersionTracker objv_tracker;
+ rgw_pubsub_topics topics;
+
+ int ret = read_topics(dpp, &topics, &objv_tracker, y);
+ if (ret < 0 && ret != -ENOENT) {
+ ldpp_dout(dpp, 1) << "ERROR: failed to read topics info: ret=" << ret << dendl;
+ return ret;
+ } else if (ret == -ENOENT) {
+ // its not an error if no topics exist, just a no-op
+ ldpp_dout(dpp, 10) << "WARNING: failed to read topics info, deletion is a no-op: ret=" << ret << dendl;
+ return 0;
+ }
+
+ topics.topics.erase(name);
+
+ ret = write_topics(dpp, topics, &objv_tracker, y);
+ if (ret < 0) {
+ ldpp_dout(dpp, 1) << "ERROR: failed to remove topics info: ret=" << ret << dendl;
+ return ret;
+ }
+
+ return 0;
+}
+
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+#pragma once
+
+#include "rgw_sal.h"
+#include "rgw_tools.h"
+#include "rgw_zone.h"
+#include "rgw_notify_event_type.h"
+#include <boost/container/flat_map.hpp>
+
+class XMLObj;
+
+struct rgw_s3_key_filter {
+ std::string prefix_rule;
+ std::string suffix_rule;
+ std::string regex_rule;
+
+ bool has_content() const;
+
+ bool decode_xml(XMLObj *obj);
+ void dump_xml(Formatter *f) const;
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ encode(prefix_rule, bl);
+ encode(suffix_rule, bl);
+ encode(regex_rule, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::const_iterator& bl) {
+ DECODE_START(1, bl);
+ decode(prefix_rule, bl);
+ decode(suffix_rule, bl);
+ decode(regex_rule, bl);
+ DECODE_FINISH(bl);
+ }
+};
+WRITE_CLASS_ENCODER(rgw_s3_key_filter)
+
+using KeyValueMap = boost::container::flat_map<std::string, std::string>;
+using KeyMultiValueMap = std::multimap<std::string, std::string>;
+
+struct rgw_s3_key_value_filter {
+ KeyValueMap kv;
+
+ bool has_content() const;
+
+ bool decode_xml(XMLObj *obj);
+ void dump_xml(Formatter *f) const;
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ encode(kv, bl);
+ ENCODE_FINISH(bl);
+ }
+ void decode(bufferlist::const_iterator& bl) {
+ DECODE_START(1, bl);
+ decode(kv, bl);
+ DECODE_FINISH(bl);
+ }
+};
+WRITE_CLASS_ENCODER(rgw_s3_key_value_filter)
+
+struct rgw_s3_filter {
+ rgw_s3_key_filter key_filter;
+ rgw_s3_key_value_filter metadata_filter;
+ rgw_s3_key_value_filter tag_filter;
+
+ bool has_content() const;
+
+ bool decode_xml(XMLObj *obj);
+ void dump_xml(Formatter *f) const;
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(2, 1, bl);
+ encode(key_filter, bl);
+ encode(metadata_filter, bl);
+ encode(tag_filter, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::const_iterator& bl) {
+ DECODE_START(2, bl);
+ decode(key_filter, bl);
+ decode(metadata_filter, bl);
+ if (struct_v >= 2) {
+ decode(tag_filter, bl);
+ }
+ DECODE_FINISH(bl);
+ }
+};
+WRITE_CLASS_ENCODER(rgw_s3_filter)
+
+using OptionalFilter = std::optional<rgw_s3_filter>;
+
+struct rgw_pubsub_topic_filter;
+/* S3 notification configuration
+ * based on: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTnotification.html
+<NotificationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+ <TopicConfiguration>
+ <Filter>
+ <S3Key>
+ <FilterRule>
+ <Name>suffix</Name>
+ <Value>jpg</Value>
+ </FilterRule>
+ </S3Key>
+ <S3Metadata>
+ <FilterRule>
+ <Name></Name>
+ <Value></Value>
+ </FilterRule>
+ </S3Metadata>
+ <S3Tags>
+ <FilterRule>
+ <Name></Name>
+ <Value></Value>
+ </FilterRule>
+ </S3Tags>
+ </Filter>
+ <Id>notification1</Id>
+ <Topic>arn:aws:sns:<region>:<account>:<topic></Topic>
+ <Event>s3:ObjectCreated:*</Event>
+ <Event>s3:ObjectRemoved:*</Event>
+ </TopicConfiguration>
+</NotificationConfiguration>
+*/
+struct rgw_pubsub_s3_notification {
+ // notification id
+ std::string id;
+ // types of events
+ rgw::notify::EventTypeList events;
+ // topic ARN
+ std::string topic_arn;
+ // filter rules
+ rgw_s3_filter filter;
+
+ bool decode_xml(XMLObj *obj);
+ void dump_xml(Formatter *f) const;
+
+ rgw_pubsub_s3_notification() = default;
+ // construct from rgw_pubsub_topic_filter (used by get/list notifications)
+ explicit rgw_pubsub_s3_notification(const rgw_pubsub_topic_filter& topic_filter);
+};
+
+// return true if the key matches the prefix/suffix/regex rules of the key filter
+bool match(const rgw_s3_key_filter& filter, const std::string& key);
+
+// return true if the key matches the metadata rules of the metadata filter
+bool match(const rgw_s3_key_value_filter& filter, const KeyValueMap& kv);
+
+// return true if the key matches the tag rules of the tag filter
+bool match(const rgw_s3_key_value_filter& filter, const KeyMultiValueMap& kv);
+
+// return true if the event type matches (equal or contained in) one of the events in the list
+bool match(const rgw::notify::EventTypeList& events, rgw::notify::EventType event);
+
+struct rgw_pubsub_s3_notifications {
+ std::list<rgw_pubsub_s3_notification> list;
+ bool decode_xml(XMLObj *obj);
+ void dump_xml(Formatter *f) const;
+};
+
+/* S3 event records structure
+ * based on: https://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
+{
+"Records":[
+ {
+ "eventVersion":""
+ "eventSource":"",
+ "awsRegion":"",
+ "eventTime":"",
+ "eventName":"",
+ "userIdentity":{
+ "principalId":""
+ },
+ "requestParameters":{
+ "sourceIPAddress":""
+ },
+ "responseElements":{
+ "x-amz-request-id":"",
+ "x-amz-id-2":""
+ },
+ "s3":{
+ "s3SchemaVersion":"1.0",
+ "configurationId":"",
+ "bucket":{
+ "name":"",
+ "ownerIdentity":{
+ "principalId":""
+ },
+ "arn":""
+ "id": ""
+ },
+ "object":{
+ "key":"",
+ "size": ,
+ "eTag":"",
+ "versionId":"",
+ "sequencer": "",
+ "metadata": ""
+ "tags": ""
+ }
+ },
+ "eventId":"",
+ }
+]
+}*/
+
+struct rgw_pubsub_s3_event {
+ constexpr static const char* const json_type_plural = "Records";
+ std::string eventVersion = "2.2";
+ // aws:s3
+ std::string eventSource = "ceph:s3";
+ // zonegroup
+ std::string awsRegion;
+ // time of the request
+ ceph::real_time eventTime;
+ // type of the event
+ std::string eventName;
+ // user that sent the request
+ std::string userIdentity;
+ // IP address of source of the request (not implemented)
+ std::string sourceIPAddress;
+ // request ID (not implemented)
+ std::string x_amz_request_id;
+ // radosgw that received the request
+ std::string x_amz_id_2;
+ std::string s3SchemaVersion = "1.0";
+ // ID received in the notification request
+ std::string configurationId;
+ // bucket name
+ std::string bucket_name;
+ // bucket owner
+ std::string bucket_ownerIdentity;
+ // bucket ARN
+ std::string bucket_arn;
+ // object key
+ std::string object_key;
+ // object size
+ uint64_t object_size = 0;
+ // object etag
+ std::string object_etag;
+ // object version id bucket is versioned
+ std::string object_versionId;
+ // hexadecimal value used to determine event order for specific key
+ std::string object_sequencer;
+ // this is an rgw extension (not S3 standard)
+ // used to store a globally unique identifier of the event
+ // that could be used for acking or any other identification of the event
+ std::string id;
+ // this is an rgw extension holding the internal bucket id
+ std::string bucket_id;
+ // meta data
+ KeyValueMap x_meta_map;
+ // tags
+ KeyMultiValueMap tags;
+ // opaque data received from the topic
+ // could be used to identify the gateway
+ std::string opaque_data;
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(4, 1, bl);
+ encode(eventVersion, bl);
+ encode(eventSource, bl);
+ encode(awsRegion, bl);
+ encode(eventTime, bl);
+ encode(eventName, bl);
+ encode(userIdentity, bl);
+ encode(sourceIPAddress, bl);
+ encode(x_amz_request_id, bl);
+ encode(x_amz_id_2, bl);
+ encode(s3SchemaVersion, bl);
+ encode(configurationId, bl);
+ encode(bucket_name, bl);
+ encode(bucket_ownerIdentity, bl);
+ encode(bucket_arn, bl);
+ encode(object_key, bl);
+ encode(object_size, bl);
+ encode(object_etag, bl);
+ encode(object_versionId, bl);
+ encode(object_sequencer, bl);
+ encode(id, bl);
+ encode(bucket_id, bl);
+ encode(x_meta_map, bl);
+ encode(tags, bl);
+ encode(opaque_data, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::const_iterator& bl) {
+ DECODE_START(4, bl);
+ decode(eventVersion, bl);
+ decode(eventSource, bl);
+ decode(awsRegion, bl);
+ decode(eventTime, bl);
+ decode(eventName, bl);
+ decode(userIdentity, bl);
+ decode(sourceIPAddress, bl);
+ decode(x_amz_request_id, bl);
+ decode(x_amz_id_2, bl);
+ decode(s3SchemaVersion, bl);
+ decode(configurationId, bl);
+ decode(bucket_name, bl);
+ decode(bucket_ownerIdentity, bl);
+ decode(bucket_arn, bl);
+ decode(object_key, bl);
+ decode(object_size, bl);
+ decode(object_etag, bl);
+ decode(object_versionId, bl);
+ decode(object_sequencer, bl);
+ decode(id, bl);
+ if (struct_v >= 2) {
+ decode(bucket_id, bl);
+ decode(x_meta_map, bl);
+ }
+ if (struct_v >= 3) {
+ decode(tags, bl);
+ }
+ if (struct_v >= 4) {
+ decode(opaque_data, bl);
+ }
+ DECODE_FINISH(bl);
+ }
+
+ void dump(Formatter *f) const;
+};
+WRITE_CLASS_ENCODER(rgw_pubsub_s3_event)
+
+// setting a unique ID for an event based on object hash and timestamp
+void set_event_id(std::string& id, const std::string& hash, const utime_t& ts);
+
+struct rgw_pubsub_dest {
+ std::string push_endpoint;
+ std::string push_endpoint_args;
+ std::string arn_topic;
+ bool stored_secret = false;
+ bool persistent = false;
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(5, 1, bl);
+ encode("", bl);
+ encode("", bl);
+ encode(push_endpoint, bl);
+ encode(push_endpoint_args, bl);
+ encode(arn_topic, bl);
+ encode(stored_secret, bl);
+ encode(persistent, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::const_iterator& bl) {
+ DECODE_START(5, bl);
+ std::string dummy;
+ decode(dummy, bl);
+ decode(dummy, bl);
+ decode(push_endpoint, bl);
+ if (struct_v >= 2) {
+ decode(push_endpoint_args, bl);
+ }
+ if (struct_v >= 3) {
+ decode(arn_topic, bl);
+ }
+ if (struct_v >= 4) {
+ decode(stored_secret, bl);
+ }
+ if (struct_v >= 5) {
+ decode(persistent, bl);
+ }
+ DECODE_FINISH(bl);
+ }
+
+ void dump(Formatter *f) const;
+ void dump_xml(Formatter *f) const;
+ std::string to_json_str() const;
+};
+WRITE_CLASS_ENCODER(rgw_pubsub_dest)
+
+struct rgw_pubsub_topic {
+ rgw_user user;
+ std::string name;
+ rgw_pubsub_dest dest;
+ std::string arn;
+ std::string opaque_data;
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(3, 1, bl);
+ encode(user, bl);
+ encode(name, bl);
+ encode(dest, bl);
+ encode(arn, bl);
+ encode(opaque_data, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::const_iterator& bl) {
+ DECODE_START(3, bl);
+ decode(user, bl);
+ decode(name, bl);
+ if (struct_v >= 2) {
+ decode(dest, bl);
+ decode(arn, bl);
+ }
+ if (struct_v >= 3) {
+ decode(opaque_data, bl);
+ }
+ DECODE_FINISH(bl);
+ }
+
+ std::string to_str() const {
+ return user.tenant + "/" + name;
+ }
+
+ void dump(Formatter *f) const;
+ void dump_xml(Formatter *f) const;
+ void dump_xml_as_attributes(Formatter *f) const;
+
+ bool operator<(const rgw_pubsub_topic& t) const {
+ return to_str().compare(t.to_str());
+ }
+};
+WRITE_CLASS_ENCODER(rgw_pubsub_topic)
+
+// this struct deprecated and remain only for backward compatibility
+struct rgw_pubsub_topic_subs {
+ rgw_pubsub_topic topic;
+ std::set<std::string> subs;
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ encode(topic, bl);
+ encode(subs, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::const_iterator& bl) {
+ DECODE_START(1, bl);
+ decode(topic, bl);
+ decode(subs, bl);
+ DECODE_FINISH(bl);
+ }
+
+ void dump(Formatter *f) const;
+};
+WRITE_CLASS_ENCODER(rgw_pubsub_topic_subs)
+
+struct rgw_pubsub_topic_filter {
+ rgw_pubsub_topic topic;
+ rgw::notify::EventTypeList events;
+ std::string s3_id;
+ rgw_s3_filter s3_filter;
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(3, 1, bl);
+ encode(topic, bl);
+ // events are stored as a vector of std::strings
+ std::vector<std::string> tmp_events;
+ std::transform(events.begin(), events.end(), std::back_inserter(tmp_events), rgw::notify::to_string);
+ encode(tmp_events, bl);
+ encode(s3_id, bl);
+ encode(s3_filter, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::const_iterator& bl) {
+ DECODE_START(3, bl);
+ decode(topic, bl);
+ // events are stored as a vector of std::strings
+ events.clear();
+ std::vector<std::string> tmp_events;
+ decode(tmp_events, bl);
+ std::transform(tmp_events.begin(), tmp_events.end(), std::back_inserter(events), rgw::notify::from_string);
+ if (struct_v >= 2) {
+ decode(s3_id, bl);
+ }
+ if (struct_v >= 3) {
+ decode(s3_filter, bl);
+ }
+ DECODE_FINISH(bl);
+ }
+
+ void dump(Formatter *f) const;
+};
+WRITE_CLASS_ENCODER(rgw_pubsub_topic_filter)
+
+struct rgw_pubsub_bucket_topics {
+ std::map<std::string, rgw_pubsub_topic_filter> topics;
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(1, 1, bl);
+ encode(topics, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::const_iterator& bl) {
+ DECODE_START(1, bl);
+ decode(topics, bl);
+ DECODE_FINISH(bl);
+ }
+
+ void dump(Formatter *f) const;
+};
+WRITE_CLASS_ENCODER(rgw_pubsub_bucket_topics)
+
+struct rgw_pubsub_topics {
+ std::map<std::string, rgw_pubsub_topic> topics;
+
+ void encode(bufferlist& bl) const {
+ ENCODE_START(2, 2, bl);
+ encode(topics, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::const_iterator& bl) {
+ DECODE_START(2, bl);
+ if (struct_v >= 2) {
+ decode(topics, bl);
+ } else {
+ std::map<std::string, rgw_pubsub_topic_subs> v1topics;
+ decode(v1topics, bl);
+ std::transform(v1topics.begin(), v1topics.end(), std::inserter(topics, topics.end()),
+ [](const auto& entry) {
+ return std::pair<std::string, rgw_pubsub_topic>(entry.first, entry.second.topic);
+ });
+ }
+ DECODE_FINISH(bl);
+ }
+
+ void dump(Formatter *f) const;
+ void dump_xml(Formatter *f) const;
+};
+WRITE_CLASS_ENCODER(rgw_pubsub_topics)
+
+class RGWPubSub
+{
+ friend class Bucket;
+
+ rgw::sal::Driver* const driver;
+ const std::string tenant;
+
+ int read_topics(const DoutPrefixProvider *dpp, rgw_pubsub_topics *result,
+ RGWObjVersionTracker* objv_tracker, optional_yield y) const;
+ int write_topics(const DoutPrefixProvider *dpp, const rgw_pubsub_topics& topics,
+ RGWObjVersionTracker* objv_tracker, optional_yield y) const;
+
+public:
+ RGWPubSub(rgw::sal::Driver* _driver, const std::string& tenant);
+
+ class Bucket {
+ friend class RGWPubSub;
+ const RGWPubSub& ps;
+ rgw::sal::Bucket* const bucket;
+
+ // read the list of topics associated with a bucket and populate into result
+ // use version tacker to enforce atomicity between read/write
+ // return 0 on success or if no topic was associated with the bucket, error code otherwise
+ int read_topics(const DoutPrefixProvider *dpp, rgw_pubsub_bucket_topics *result,
+ RGWObjVersionTracker* objv_tracker, optional_yield y) const;
+ // set the list of topics associated with a bucket
+ // use version tacker to enforce atomicity between read/write
+ // return 0 on success, error code otherwise
+ int write_topics(const DoutPrefixProvider *dpp, const rgw_pubsub_bucket_topics& topics,
+ RGWObjVersionTracker* objv_tracker, optional_yield y) const;
+ public:
+ Bucket(const RGWPubSub& _ps, rgw::sal::Bucket* _bucket) :
+ ps(_ps), bucket(_bucket)
+ {}
+
+ // get the list of topics associated with a bucket and populate into result
+ // return 0 on success or if no topic was associated with the bucket, error code otherwise
+ int get_topics(const DoutPrefixProvider *dpp, rgw_pubsub_bucket_topics *result, optional_yield y) const {
+ return read_topics(dpp, result, nullptr, y);
+ }
+ // adds a topic + filter (event list, and possibly name metadata or tags filters) to a bucket
+ // assigning a notification name is optional (needed for S3 compatible notifications)
+ // if the topic already exist on the bucket, the filter event list may be updated
+ // for S3 compliant notifications the version with: s3_filter and notif_name should be used
+ // return -ENOENT if the topic does not exists
+ // return 0 on success, error code otherwise
+ int create_notification(const DoutPrefixProvider *dpp, const std::string& topic_name,
+ const rgw::notify::EventTypeList& events, optional_yield y) const;
+ int create_notification(const DoutPrefixProvider *dpp, const std::string& topic_name,
+ const rgw::notify::EventTypeList& events, OptionalFilter s3_filter, const std::string& notif_name, optional_yield y) const;
+ // remove a topic and filter from bucket
+ // if the topic does not exists on the bucket it is a no-op (considered success)
+ // return -ENOENT if the topic does not exists
+ // return 0 on success, error code otherwise
+ int remove_notification(const DoutPrefixProvider *dpp, const std::string& topic_name, optional_yield y) const;
+ // remove all notifications (and autogenerated topics) associated with the bucket
+ // return 0 on success or if no topic was associated with the bucket, error code otherwise
+ int remove_notifications(const DoutPrefixProvider *dpp, optional_yield y) const;
+ };
+
+ // get the list of topics
+ // return 0 on success or if no topic was associated with the bucket, error code otherwise
+ int get_topics(const DoutPrefixProvider *dpp, rgw_pubsub_topics *result, optional_yield y) const {
+ return read_topics(dpp, result, nullptr, y);
+ }
+ // get a topic with by its name and populate it into "result"
+ // return -ENOENT if the topic does not exists
+ // return 0 on success, error code otherwise
+ int get_topic(const DoutPrefixProvider *dpp, const std::string& name, rgw_pubsub_topic *result, optional_yield y) const;
+ // create a topic with a name only
+ // if the topic already exists it is a no-op (considered success)
+ // return 0 on success, error code otherwise
+ int create_topic(const DoutPrefixProvider *dpp, const std::string& name, optional_yield y) const;
+ // create a topic with push destination information and ARN
+ // if the topic already exists the destination and ARN values may be updated (considered succsess)
+ // return 0 on success, error code otherwise
+ int create_topic(const DoutPrefixProvider *dpp, const std::string& name, const rgw_pubsub_dest& dest,
+ const std::string& arn, const std::string& opaque_data, optional_yield y) const;
+ // remove a topic according to its name
+ // if the topic does not exists it is a no-op (considered success)
+ // return 0 on success, error code otherwise
+ int remove_topic(const DoutPrefixProvider *dpp, const std::string& name, optional_yield y) const;
+};
+
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <algorithm>
+#include <boost/tokenizer.hpp>
+#include <optional>
+#include "rgw_rest_pubsub.h"
+#include "rgw_pubsub_push.h"
+#include "rgw_pubsub.h"
+#include "rgw_op.h"
+#include "rgw_rest.h"
+#include "rgw_rest_s3.h"
+#include "rgw_arn.h"
+#include "rgw_auth_s3.h"
+#include "rgw_notify.h"
+#include "services/svc_zone.h"
+#include "common/dout.h"
+#include "rgw_url.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rgw
+
+static const char* AWS_SNS_NS("https://sns.amazonaws.com/doc/2010-03-31/");
+
+bool verify_transport_security(CephContext *cct, const RGWEnv& env) {
+ const auto is_secure = rgw_transport_is_secure(cct, env);
+ if (!is_secure && g_conf().get_val<bool>("rgw_allow_notification_secrets_in_cleartext")) {
+ ldout(cct, 0) << "WARNING: bypassing endpoint validation, allows sending secrets over insecure transport" << dendl;
+ return true;
+ }
+ return is_secure;
+}
+
+// make sure that endpoint is a valid URL
+// make sure that if user/password are passed inside URL, it is over secure connection
+// update rgw_pubsub_dest to indicate that a password is stored in the URL
+bool validate_and_update_endpoint_secret(rgw_pubsub_dest& dest, CephContext *cct, const RGWEnv& env) {
+ if (dest.push_endpoint.empty()) {
+ return true;
+ }
+ std::string user;
+ std::string password;
+ if (!rgw::parse_url_userinfo(dest.push_endpoint, user, password)) {
+ ldout(cct, 1) << "endpoint validation error: malformed endpoint URL:" << dest.push_endpoint << dendl;
+ return false;
+ }
+ // this should be verified inside parse_url()
+ ceph_assert(user.empty() == password.empty());
+ if (!user.empty()) {
+ dest.stored_secret = true;
+ if (!verify_transport_security(cct, env)) {
+ ldout(cct, 1) << "endpoint validation error: sending secrets over insecure transport" << dendl;
+ return false;
+ }
+ }
+ return true;
+}
+
+bool topic_has_endpoint_secret(const rgw_pubsub_topic& topic) {
+ return topic.dest.stored_secret;
+}
+
+bool topics_has_endpoint_secret(const rgw_pubsub_topics& topics) {
+ for (const auto& topic : topics.topics) {
+ if (topic_has_endpoint_secret(topic.second)) return true;
+ }
+ return false;
+}
+
+// command (AWS compliant):
+// POST
+// Action=CreateTopic&Name=<topic-name>[&OpaqueData=data][&push-endpoint=<endpoint>[&persistent][&<arg1>=<value1>]]
+class RGWPSCreateTopicOp : public RGWOp {
+ private:
+ std::string topic_name;
+ rgw_pubsub_dest dest;
+ std::string topic_arn;
+ std::string opaque_data;
+
+ int get_params() {
+ topic_name = s->info.args.get("Name");
+ if (topic_name.empty()) {
+ ldpp_dout(this, 1) << "CreateTopic Action 'Name' argument is missing" << dendl;
+ return -EINVAL;
+ }
+
+ opaque_data = s->info.args.get("OpaqueData");
+
+ dest.push_endpoint = s->info.args.get("push-endpoint");
+ s->info.args.get_bool("persistent", &dest.persistent, false);
+
+ if (!validate_and_update_endpoint_secret(dest, s->cct, *(s->info.env))) {
+ return -EINVAL;
+ }
+ for (const auto& param : s->info.args.get_params()) {
+ if (param.first == "Action" || param.first == "Name" || param.first == "PayloadHash") {
+ continue;
+ }
+ dest.push_endpoint_args.append(param.first+"="+param.second+"&");
+ }
+
+ if (!dest.push_endpoint_args.empty()) {
+ // remove last separator
+ dest.push_endpoint_args.pop_back();
+ }
+ if (!dest.push_endpoint.empty() && dest.persistent) {
+ const auto ret = rgw::notify::add_persistent_topic(topic_name, s->yield);
+ if (ret < 0) {
+ ldpp_dout(this, 1) << "CreateTopic Action failed to create queue for persistent topics. error:" << ret << dendl;
+ return ret;
+ }
+ }
+
+ // dest object only stores endpoint info
+ dest.arn_topic = topic_name;
+ // the topic ARN will be sent in the reply
+ const rgw::ARN arn(rgw::Partition::aws, rgw::Service::sns,
+ driver->get_zone()->get_zonegroup().get_name(),
+ s->user->get_tenant(), topic_name);
+ topic_arn = arn.to_string();
+ return 0;
+ }
+
+ public:
+ int verify_permission(optional_yield) override {
+ return 0;
+ }
+
+ void pre_exec() override {
+ rgw_bucket_object_pre_exec(s);
+ }
+ void execute(optional_yield) override;
+
+ const char* name() const override { return "pubsub_topic_create"; }
+ RGWOpType get_type() override { return RGW_OP_PUBSUB_TOPIC_CREATE; }
+ uint32_t op_mask() override { return RGW_OP_TYPE_WRITE; }
+
+ void send_response() override {
+ if (op_ret) {
+ set_req_state_err(s, op_ret);
+ }
+ dump_errno(s);
+ end_header(s, this, "application/xml");
+
+ if (op_ret < 0) {
+ return;
+ }
+
+ const auto f = s->formatter;
+ f->open_object_section_in_ns("CreateTopicResponse", AWS_SNS_NS);
+ f->open_object_section("CreateTopicResult");
+ encode_xml("TopicArn", topic_arn, f);
+ f->close_section(); // CreateTopicResult
+ f->open_object_section("ResponseMetadata");
+ encode_xml("RequestId", s->req_id, f);
+ f->close_section(); // ResponseMetadata
+ f->close_section(); // CreateTopicResponse
+ rgw_flush_formatter_and_reset(s, f);
+ }
+};
+
+void RGWPSCreateTopicOp::execute(optional_yield y) {
+ op_ret = get_params();
+ if (op_ret < 0) {
+ return;
+ }
+
+ const RGWPubSub ps(driver, s->owner.get_id().tenant);
+ op_ret = ps.create_topic(this, topic_name, dest, topic_arn, opaque_data, y);
+ if (op_ret < 0) {
+ ldpp_dout(this, 1) << "failed to create topic '" << topic_name << "', ret=" << op_ret << dendl;
+ return;
+ }
+ ldpp_dout(this, 20) << "successfully created topic '" << topic_name << "'" << dendl;
+}
+
+// command (AWS compliant):
+// POST
+// Action=ListTopics
+class RGWPSListTopicsOp : public RGWOp {
+private:
+ rgw_pubsub_topics result;
+
+public:
+ int verify_permission(optional_yield) override {
+ return 0;
+ }
+ void pre_exec() override {
+ rgw_bucket_object_pre_exec(s);
+ }
+ void execute(optional_yield) override;
+
+ const char* name() const override { return "pubsub_topics_list"; }
+ RGWOpType get_type() override { return RGW_OP_PUBSUB_TOPICS_LIST; }
+ uint32_t op_mask() override { return RGW_OP_TYPE_READ; }
+
+ void send_response() override {
+ if (op_ret) {
+ set_req_state_err(s, op_ret);
+ }
+ dump_errno(s);
+ end_header(s, this, "application/xml");
+
+ if (op_ret < 0) {
+ return;
+ }
+
+ const auto f = s->formatter;
+ f->open_object_section_in_ns("ListTopicsResponse", AWS_SNS_NS);
+ f->open_object_section("ListTopicsResult");
+ encode_xml("Topics", result, f);
+ f->close_section(); // ListTopicsResult
+ f->open_object_section("ResponseMetadata");
+ encode_xml("RequestId", s->req_id, f);
+ f->close_section(); // ResponseMetadat
+ f->close_section(); // ListTopicsResponse
+ rgw_flush_formatter_and_reset(s, f);
+ }
+};
+
+void RGWPSListTopicsOp::execute(optional_yield y) {
+ const RGWPubSub ps(driver, s->owner.get_id().tenant);
+ op_ret = ps.get_topics(this, &result, y);
+ // if there are no topics it is not considered an error
+ op_ret = op_ret == -ENOENT ? 0 : op_ret;
+ if (op_ret < 0) {
+ ldpp_dout(this, 1) << "failed to get topics, ret=" << op_ret << dendl;
+ return;
+ }
+ if (topics_has_endpoint_secret(result) && !verify_transport_security(s->cct, *(s->info.env))) {
+ ldpp_dout(this, 1) << "topics contain secrets and cannot be sent over insecure transport" << dendl;
+ op_ret = -EPERM;
+ return;
+ }
+ ldpp_dout(this, 20) << "successfully got topics" << dendl;
+}
+
+// command (extension to AWS):
+// POST
+// Action=GetTopic&TopicArn=<topic-arn>
+class RGWPSGetTopicOp : public RGWOp {
+ private:
+ std::string topic_name;
+ rgw_pubsub_topic result;
+
+ int get_params() {
+ const auto topic_arn = rgw::ARN::parse((s->info.args.get("TopicArn")));
+
+ if (!topic_arn || topic_arn->resource.empty()) {
+ ldpp_dout(this, 1) << "GetTopic Action 'TopicArn' argument is missing or invalid" << dendl;
+ return -EINVAL;
+ }
+
+ topic_name = topic_arn->resource;
+ return 0;
+ }
+
+ public:
+ int verify_permission(optional_yield y) override {
+ return 0;
+ }
+ void pre_exec() override {
+ rgw_bucket_object_pre_exec(s);
+ }
+ void execute(optional_yield y) override;
+
+ const char* name() const override { return "pubsub_topic_get"; }
+ RGWOpType get_type() override { return RGW_OP_PUBSUB_TOPIC_GET; }
+ uint32_t op_mask() override { return RGW_OP_TYPE_READ; }
+
+ void send_response() override {
+ if (op_ret) {
+ set_req_state_err(s, op_ret);
+ }
+ dump_errno(s);
+ end_header(s, this, "application/xml");
+
+ if (op_ret < 0) {
+ return;
+ }
+
+ const auto f = s->formatter;
+ f->open_object_section("GetTopicResponse");
+ f->open_object_section("GetTopicResult");
+ encode_xml("Topic", result, f);
+ f->close_section();
+ f->open_object_section("ResponseMetadata");
+ encode_xml("RequestId", s->req_id, f);
+ f->close_section();
+ f->close_section();
+ rgw_flush_formatter_and_reset(s, f);
+ }
+};
+
+void RGWPSGetTopicOp::execute(optional_yield y) {
+ op_ret = get_params();
+ if (op_ret < 0) {
+ return;
+ }
+ const RGWPubSub ps(driver, s->owner.get_id().tenant);
+ op_ret = ps.get_topic(this, topic_name, &result, y);
+ if (op_ret < 0) {
+ ldpp_dout(this, 1) << "failed to get topic '" << topic_name << "', ret=" << op_ret << dendl;
+ return;
+ }
+ if (topic_has_endpoint_secret(result) && !verify_transport_security(s->cct, *(s->info.env))) {
+ ldpp_dout(this, 1) << "topic '" << topic_name << "' contain secret and cannot be sent over insecure transport" << dendl;
+ op_ret = -EPERM;
+ return;
+ }
+ ldpp_dout(this, 1) << "successfully got topic '" << topic_name << "'" << dendl;
+}
+
+// command (AWS compliant):
+// POST
+// Action=GetTopicAttributes&TopicArn=<topic-arn>
+class RGWPSGetTopicAttributesOp : public RGWOp {
+ private:
+ std::string topic_name;
+ rgw_pubsub_topic result;
+
+ int get_params() {
+ const auto topic_arn = rgw::ARN::parse((s->info.args.get("TopicArn")));
+
+ if (!topic_arn || topic_arn->resource.empty()) {
+ ldpp_dout(this, 1) << "GetTopicAttribute Action 'TopicArn' argument is missing or invalid" << dendl;
+ return -EINVAL;
+ }
+
+ topic_name = topic_arn->resource;
+ return 0;
+ }
+
+ public:
+ int verify_permission(optional_yield y) override {
+ return 0;
+ }
+ void pre_exec() override {
+ rgw_bucket_object_pre_exec(s);
+ }
+ void execute(optional_yield y) override;
+
+ const char* name() const override { return "pubsub_topic_get"; }
+ RGWOpType get_type() override { return RGW_OP_PUBSUB_TOPIC_GET; }
+ uint32_t op_mask() override { return RGW_OP_TYPE_READ; }
+
+ void send_response() override {
+ if (op_ret) {
+ set_req_state_err(s, op_ret);
+ }
+ dump_errno(s);
+ end_header(s, this, "application/xml");
+
+ if (op_ret < 0) {
+ return;
+ }
+
+ const auto f = s->formatter;
+ f->open_object_section_in_ns("GetTopicAttributesResponse", AWS_SNS_NS);
+ f->open_object_section("GetTopicAttributesResult");
+ result.dump_xml_as_attributes(f);
+ f->close_section(); // GetTopicAttributesResult
+ f->open_object_section("ResponseMetadata");
+ encode_xml("RequestId", s->req_id, f);
+ f->close_section(); // ResponseMetadata
+ f->close_section(); // GetTopicAttributesResponse
+ rgw_flush_formatter_and_reset(s, f);
+ }
+};
+
+void RGWPSGetTopicAttributesOp::execute(optional_yield y) {
+ op_ret = get_params();
+ if (op_ret < 0) {
+ return;
+ }
+ const RGWPubSub ps(driver, s->owner.get_id().tenant);
+ op_ret = ps.get_topic(this, topic_name, &result, y);
+ if (op_ret < 0) {
+ ldpp_dout(this, 1) << "failed to get topic '" << topic_name << "', ret=" << op_ret << dendl;
+ return;
+ }
+ if (topic_has_endpoint_secret(result) && !verify_transport_security(s->cct, *(s->info.env))) {
+ ldpp_dout(this, 1) << "topic '" << topic_name << "' contain secret and cannot be sent over insecure transport" << dendl;
+ op_ret = -EPERM;
+ return;
+ }
+ ldpp_dout(this, 1) << "successfully got topic '" << topic_name << "'" << dendl;
+}
+
+// command (AWS compliant):
+// POST
+// Action=DeleteTopic&TopicArn=<topic-arn>
+class RGWPSDeleteTopicOp : public RGWOp {
+ private:
+ std::string topic_name;
+
+ int get_params() {
+ const auto topic_arn = rgw::ARN::parse((s->info.args.get("TopicArn")));
+
+ if (!topic_arn || topic_arn->resource.empty()) {
+ ldpp_dout(this, 1) << "DeleteTopic Action 'TopicArn' argument is missing or invalid" << dendl;
+ return -EINVAL;
+ }
+
+ topic_name = topic_arn->resource;
+
+ // upon deletion it is not known if topic is persistent or not
+ // will try to delete the persistent topic anyway
+ const auto ret = rgw::notify::remove_persistent_topic(topic_name, s->yield);
+ if (ret == -ENOENT) {
+ // topic was not persistent, or already deleted
+ return 0;
+ }
+ if (ret < 0) {
+ ldpp_dout(this, 1) << "DeleteTopic Action failed to remove queue for persistent topics. error:" << ret << dendl;
+ return ret;
+ }
+
+ return 0;
+ }
+
+ public:
+ int verify_permission(optional_yield) override {
+ return 0;
+ }
+ void pre_exec() override {
+ rgw_bucket_object_pre_exec(s);
+ }
+ void execute(optional_yield y) override;
+
+ const char* name() const override { return "pubsub_topic_delete"; }
+ RGWOpType get_type() override { return RGW_OP_PUBSUB_TOPIC_DELETE; }
+ uint32_t op_mask() override { return RGW_OP_TYPE_DELETE; }
+
+ void send_response() override {
+ if (op_ret) {
+ set_req_state_err(s, op_ret);
+ }
+ dump_errno(s);
+ end_header(s, this, "application/xml");
+
+ if (op_ret < 0) {
+ return;
+ }
+
+ const auto f = s->formatter;
+ f->open_object_section_in_ns("DeleteTopicResponse", AWS_SNS_NS);
+ f->open_object_section("ResponseMetadata");
+ encode_xml("RequestId", s->req_id, f);
+ f->close_section(); // ResponseMetadata
+ f->close_section(); // DeleteTopicResponse
+ rgw_flush_formatter_and_reset(s, f);
+ }
+};
+
+void RGWPSDeleteTopicOp::execute(optional_yield y) {
+ op_ret = get_params();
+ if (op_ret < 0) {
+ return;
+ }
+ const RGWPubSub ps(driver, s->owner.get_id().tenant);
+ op_ret = ps.remove_topic(this, topic_name, y);
+ if (op_ret < 0) {
+ ldpp_dout(this, 1) << "failed to remove topic '" << topic_name << ", ret=" << op_ret << dendl;
+ return;
+ }
+ ldpp_dout(this, 1) << "successfully removed topic '" << topic_name << "'" << dendl;
+}
+
+using op_generator = RGWOp*(*)();
+static const std::unordered_map<std::string, op_generator> op_generators = {
+ {"CreateTopic", []() -> RGWOp* {return new RGWPSCreateTopicOp;}},
+ {"DeleteTopic", []() -> RGWOp* {return new RGWPSDeleteTopicOp;}},
+ {"ListTopics", []() -> RGWOp* {return new RGWPSListTopicsOp;}},
+ {"GetTopic", []() -> RGWOp* {return new RGWPSGetTopicOp;}},
+ {"GetTopicAttributes", []() -> RGWOp* {return new RGWPSGetTopicAttributesOp;}}
+};
+
+bool RGWHandler_REST_PSTopic_AWS::action_exists(const req_state* s)
+{
+ if (s->info.args.exists("Action")) {
+ const std::string action_name = s->info.args.get("Action");
+ return op_generators.contains(action_name);
+ }
+ return false;
+}
+
+RGWOp *RGWHandler_REST_PSTopic_AWS::op_post()
+{
+ s->dialect = "sns";
+ s->prot_flags = RGW_REST_STS;
+
+ if (s->info.args.exists("Action")) {
+ const std::string action_name = s->info.args.get("Action");
+ const auto action_it = op_generators.find(action_name);
+ if (action_it != op_generators.end()) {
+ return action_it->second();
+ }
+ ldpp_dout(s, 10) << "unknown action '" << action_name << "' for Topic handler" << dendl;
+ } else {
+ ldpp_dout(s, 10) << "missing action argument in Topic handler" << dendl;
+ }
+ return nullptr;
+}
+
+int RGWHandler_REST_PSTopic_AWS::authorize(const DoutPrefixProvider* dpp, optional_yield y) {
+ const auto rc = RGW_Auth_S3::authorize(dpp, driver, auth_registry, s, y);
+ if (rc < 0) {
+ return rc;
+ }
+ if (s->auth.identity->is_anonymous()) {
+ ldpp_dout(dpp, 1) << "anonymous user not allowed in topic operations" << dendl;
+ return -ERR_INVALID_REQUEST;
+ }
+ return 0;
+}
+
+namespace {
+// return a unique topic by prefexing with the notification name: <notification>_<topic>
+std::string topic_to_unique(const std::string& topic, const std::string& notification) {
+ return notification + "_" + topic;
+}
+
+// extract the topic from a unique topic of the form: <notification>_<topic>
+[[maybe_unused]] std::string unique_to_topic(const std::string& unique_topic, const std::string& notification) {
+ if (unique_topic.find(notification + "_") == std::string::npos) {
+ return "";
+ }
+ return unique_topic.substr(notification.length() + 1);
+}
+
+// from list of bucket topics, find the one that was auto-generated by a notification
+auto find_unique_topic(const rgw_pubsub_bucket_topics& bucket_topics, const std::string& notif_name) {
+ auto it = std::find_if(bucket_topics.topics.begin(), bucket_topics.topics.end(), [&](const auto& val) { return notif_name == val.second.s3_id; });
+ return it != bucket_topics.topics.end() ?
+ std::optional<std::reference_wrapper<const rgw_pubsub_topic_filter>>(it->second):
+ std::nullopt;
+}
+}
+
+int remove_notification_by_topic(const DoutPrefixProvider *dpp, const std::string& topic_name, const RGWPubSub::Bucket& b, optional_yield y, const RGWPubSub& ps) {
+ int op_ret = b.remove_notification(dpp, topic_name, y);
+ if (op_ret < 0) {
+ ldpp_dout(dpp, 1) << "failed to remove notification of topic '" << topic_name << "', ret=" << op_ret << dendl;
+ }
+ op_ret = ps.remove_topic(dpp, topic_name, y);
+ if (op_ret < 0) {
+ ldpp_dout(dpp, 1) << "failed to remove auto-generated topic '" << topic_name << "', ret=" << op_ret << dendl;
+ }
+ return op_ret;
+}
+
+int delete_all_notifications(const DoutPrefixProvider *dpp, const rgw_pubsub_bucket_topics& bucket_topics, const RGWPubSub::Bucket& b, optional_yield y, const RGWPubSub& ps) {
+ // delete all notifications of on a bucket
+ for (const auto& topic : bucket_topics.topics) {
+ const auto op_ret = remove_notification_by_topic(dpp, topic.first, b, y, ps);
+ if (op_ret < 0) {
+ return op_ret;
+ }
+ }
+ return 0;
+}
+
+// command (S3 compliant): PUT /<bucket name>?notification
+// a "notification" and a subscription will be auto-generated
+// actual configuration is XML encoded in the body of the message
+class RGWPSCreateNotifOp : public RGWDefaultResponseOp {
+ private:
+ std::string bucket_name;
+ std::unique_ptr<rgw::sal::Bucket> bucket;
+ rgw_pubsub_s3_notifications configurations;
+
+ int get_params() {
+ bool exists;
+ const auto no_value = s->info.args.get("notification", &exists);
+ if (!exists) {
+ ldpp_dout(this, 1) << "missing required param 'notification'" << dendl;
+ return -EINVAL;
+ }
+ if (no_value.length() > 0) {
+ ldpp_dout(this, 1) << "param 'notification' should not have any value" << dendl;
+ return -EINVAL;
+ }
+ if (s->bucket_name.empty()) {
+ ldpp_dout(this, 1) << "request must be on a bucket" << dendl;
+ return -EINVAL;
+ }
+ bucket_name = s->bucket_name;
+ return 0;
+ }
+
+ public:
+ int verify_permission(optional_yield y) override;
+
+ void pre_exec() override {
+ rgw_bucket_object_pre_exec(s);
+ }
+
+ const char* name() const override { return "pubsub_notification_create_s3"; }
+ RGWOpType get_type() override { return RGW_OP_PUBSUB_NOTIF_CREATE; }
+ uint32_t op_mask() override { return RGW_OP_TYPE_WRITE; }
+
+ int get_params_from_body() {
+ const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+ int r;
+ bufferlist data;
+ std::tie(r, data) = read_all_input(s, max_size, false);
+
+ if (r < 0) {
+ ldpp_dout(this, 1) << "failed to read XML payload" << dendl;
+ return r;
+ }
+ if (data.length() == 0) {
+ ldpp_dout(this, 1) << "XML payload missing" << dendl;
+ return -EINVAL;
+ }
+
+ RGWXMLDecoder::XMLParser parser;
+
+ if (!parser.init()){
+ ldpp_dout(this, 1) << "failed to initialize XML parser" << dendl;
+ return -EINVAL;
+ }
+ if (!parser.parse(data.c_str(), data.length(), 1)) {
+ ldpp_dout(this, 1) << "failed to parse XML payload" << dendl;
+ return -ERR_MALFORMED_XML;
+ }
+ try {
+ // NotificationConfigurations is mandatory
+ // It can be empty which means we delete all the notifications
+ RGWXMLDecoder::decode_xml("NotificationConfiguration", configurations, &parser, true);
+ } catch (RGWXMLDecoder::err& err) {
+ ldpp_dout(this, 1) << "failed to parse XML payload. error: " << err << dendl;
+ return -ERR_MALFORMED_XML;
+ }
+ return 0;
+ }
+
+ void execute(optional_yield) override;
+};
+
+void RGWPSCreateNotifOp::execute(optional_yield y) {
+ op_ret = get_params_from_body();
+ if (op_ret < 0) {
+ return;
+ }
+
+ const RGWPubSub ps(driver, s->owner.get_id().tenant);
+ const RGWPubSub::Bucket b(ps, bucket.get());
+
+ if(configurations.list.empty()) {
+ // get all topics on a bucket
+ rgw_pubsub_bucket_topics bucket_topics;
+ op_ret = b.get_topics(this, &bucket_topics, y);
+ if (op_ret < 0) {
+ ldpp_dout(this, 1) << "failed to get list of topics from bucket '" << bucket_name << "', ret=" << op_ret << dendl;
+ return;
+ }
+
+ op_ret = delete_all_notifications(this, bucket_topics, b, y, ps);
+ return;
+ }
+
+ for (const auto& c : configurations.list) {
+ const auto& notif_name = c.id;
+ if (notif_name.empty()) {
+ ldpp_dout(this, 1) << "missing notification id" << dendl;
+ op_ret = -EINVAL;
+ return;
+ }
+ if (c.topic_arn.empty()) {
+ ldpp_dout(this, 1) << "missing topic ARN in notification: '" << notif_name << "'" << dendl;
+ op_ret = -EINVAL;
+ return;
+ }
+
+ const auto arn = rgw::ARN::parse(c.topic_arn);
+ if (!arn || arn->resource.empty()) {
+ ldpp_dout(this, 1) << "topic ARN has invalid format: '" << c.topic_arn << "' in notification: '" << notif_name << "'" << dendl;
+ op_ret = -EINVAL;
+ return;
+ }
+
+ if (std::find(c.events.begin(), c.events.end(), rgw::notify::UnknownEvent) != c.events.end()) {
+ ldpp_dout(this, 1) << "unknown event type in notification: '" << notif_name << "'" << dendl;
+ op_ret = -EINVAL;
+ return;
+ }
+
+ const auto topic_name = arn->resource;
+
+ // get topic information. destination information is stored in the topic
+ rgw_pubsub_topic topic_info;
+ op_ret = ps.get_topic(this, topic_name, &topic_info, y);
+ if (op_ret < 0) {
+ ldpp_dout(this, 1) << "failed to get topic '" << topic_name << "', ret=" << op_ret << dendl;
+ return;
+ }
+ // make sure that full topic configuration match
+ // TODO: use ARN match function
+
+ // create unique topic name. this has 2 reasons:
+ // (1) topics cannot be shared between different S3 notifications because they hold the filter information
+ // (2) make topic clneaup easier, when notification is removed
+ const auto unique_topic_name = topic_to_unique(topic_name, notif_name);
+ // generate the internal topic. destination is stored here for the "push-only" case
+ // when no subscription exists
+ // ARN is cached to make the "GET" method faster
+ op_ret = ps.create_topic(this, unique_topic_name, topic_info.dest, topic_info.arn, topic_info.opaque_data, y);
+ if (op_ret < 0) {
+ ldpp_dout(this, 1) << "failed to auto-generate unique topic '" << unique_topic_name <<
+ "', ret=" << op_ret << dendl;
+ return;
+ }
+ ldpp_dout(this, 20) << "successfully auto-generated unique topic '" << unique_topic_name << "'" << dendl;
+ // generate the notification
+ rgw::notify::EventTypeList events;
+ op_ret = b.create_notification(this, unique_topic_name, c.events, std::make_optional(c.filter), notif_name, y);
+ if (op_ret < 0) {
+ ldpp_dout(this, 1) << "failed to auto-generate notification for unique topic '" << unique_topic_name <<
+ "', ret=" << op_ret << dendl;
+ // rollback generated topic (ignore return value)
+ ps.remove_topic(this, unique_topic_name, y);
+ return;
+ }
+ ldpp_dout(this, 20) << "successfully auto-generated notification for unique topic '" << unique_topic_name << "'" << dendl;
+ }
+}
+
+int RGWPSCreateNotifOp::verify_permission(optional_yield y) {
+ int ret = get_params();
+ if (ret < 0) {
+ return ret;
+ }
+
+ std::unique_ptr<rgw::sal::User> user = driver->get_user(s->owner.get_id());
+ ret = driver->get_bucket(this, user.get(), s->owner.get_id().tenant, bucket_name, &bucket, y);
+ if (ret < 0) {
+ ldpp_dout(this, 1) << "failed to get bucket info, cannot verify ownership" << dendl;
+ return ret;
+ }
+
+ if (bucket->get_info().owner != s->owner.get_id()) {
+ ldpp_dout(this, 1) << "user doesn't own bucket, not allowed to create notification" << dendl;
+ return -EPERM;
+ }
+ return 0;
+}
+
+// command (extension to S3): DELETE /bucket?notification[=<notification-id>]
+class RGWPSDeleteNotifOp : public RGWDefaultResponseOp {
+ private:
+ std::string bucket_name;
+ std::unique_ptr<rgw::sal::Bucket> bucket;
+ std::string notif_name;
+
+ public:
+ int verify_permission(optional_yield y) override;
+
+ void pre_exec() override {
+ rgw_bucket_object_pre_exec(s);
+ }
+
+ const char* name() const override { return "pubsub_notification_delete_s3"; }
+ RGWOpType get_type() override { return RGW_OP_PUBSUB_NOTIF_DELETE; }
+ uint32_t op_mask() override { return RGW_OP_TYPE_DELETE; }
+
+ int get_params() {
+ bool exists;
+ notif_name = s->info.args.get("notification", &exists);
+ if (!exists) {
+ ldpp_dout(this, 1) << "missing required param 'notification'" << dendl;
+ return -EINVAL;
+ }
+ if (s->bucket_name.empty()) {
+ ldpp_dout(this, 1) << "request must be on a bucket" << dendl;
+ return -EINVAL;
+ }
+ bucket_name = s->bucket_name;
+ return 0;
+ }
+
+ void execute(optional_yield y) override;
+};
+
+void RGWPSDeleteNotifOp::execute(optional_yield y) {
+ op_ret = get_params();
+ if (op_ret < 0) {
+ return;
+ }
+
+ const RGWPubSub ps(driver, s->owner.get_id().tenant);
+ const RGWPubSub::Bucket b(ps, bucket.get());
+
+ // get all topics on a bucket
+ rgw_pubsub_bucket_topics bucket_topics;
+ op_ret = b.get_topics(this, &bucket_topics, y);
+ if (op_ret < 0) {
+ ldpp_dout(this, 1) << "failed to get list of topics from bucket '" << bucket_name << "', ret=" << op_ret << dendl;
+ return;
+ }
+
+ if (!notif_name.empty()) {
+ // delete a specific notification
+ const auto unique_topic = find_unique_topic(bucket_topics, notif_name);
+ if (unique_topic) {
+ const auto unique_topic_name = unique_topic->get().topic.name;
+ op_ret = remove_notification_by_topic(this, unique_topic_name, b, y, ps);
+ return;
+ }
+ // notification to be removed is not found - considered success
+ ldpp_dout(this, 20) << "notification '" << notif_name << "' already removed" << dendl;
+ return;
+ }
+
+ op_ret = delete_all_notifications(this, bucket_topics, b, y, ps);
+}
+
+int RGWPSDeleteNotifOp::verify_permission(optional_yield y) {
+ int ret = get_params();
+ if (ret < 0) {
+ return ret;
+ }
+
+ std::unique_ptr<rgw::sal::User> user = driver->get_user(s->owner.get_id());
+ ret = driver->get_bucket(this, user.get(), s->owner.get_id().tenant, bucket_name, &bucket, y);
+ if (ret < 0) {
+ return ret;
+ }
+
+ if (bucket->get_info().owner != s->owner.get_id()) {
+ ldpp_dout(this, 1) << "user doesn't own bucket, cannot remove notification" << dendl;
+ return -EPERM;
+ }
+ return 0;
+}
+
+// command (S3 compliant): GET /bucket?notification[=<notification-id>]
+class RGWPSListNotifsOp : public RGWOp {
+private:
+ std::string bucket_name;
+ std::unique_ptr<rgw::sal::Bucket> bucket;
+ std::string notif_name;
+ rgw_pubsub_s3_notifications notifications;
+
+ int get_params() {
+ bool exists;
+ notif_name = s->info.args.get("notification", &exists);
+ if (!exists) {
+ ldpp_dout(this, 1) << "missing required param 'notification'" << dendl;
+ return -EINVAL;
+ }
+ if (s->bucket_name.empty()) {
+ ldpp_dout(this, 1) << "request must be on a bucket" << dendl;
+ return -EINVAL;
+ }
+ bucket_name = s->bucket_name;
+ return 0;
+ }
+
+ public:
+ int verify_permission(optional_yield y) override;
+
+ void pre_exec() override {
+ rgw_bucket_object_pre_exec(s);
+ }
+
+ const char* name() const override { return "pubsub_notifications_get_s3"; }
+ RGWOpType get_type() override { return RGW_OP_PUBSUB_NOTIF_LIST; }
+ uint32_t op_mask() override { return RGW_OP_TYPE_READ; }
+
+ void execute(optional_yield y) override;
+ void send_response() override {
+ if (op_ret) {
+ set_req_state_err(s, op_ret);
+ }
+ dump_errno(s);
+ end_header(s, this, "application/xml");
+
+ if (op_ret < 0) {
+ return;
+ }
+ notifications.dump_xml(s->formatter);
+ rgw_flush_formatter_and_reset(s, s->formatter);
+ }
+};
+
+void RGWPSListNotifsOp::execute(optional_yield y) {
+ const RGWPubSub ps(driver, s->owner.get_id().tenant);
+ const RGWPubSub::Bucket b(ps, bucket.get());
+
+ // get all topics on a bucket
+ rgw_pubsub_bucket_topics bucket_topics;
+ op_ret = b.get_topics(this, &bucket_topics, y);
+ if (op_ret < 0) {
+ ldpp_dout(this, 1) << "failed to get list of topics from bucket '" << bucket_name << "', ret=" << op_ret << dendl;
+ return;
+ }
+ if (!notif_name.empty()) {
+ // get info of a specific notification
+ const auto unique_topic = find_unique_topic(bucket_topics, notif_name);
+ if (unique_topic) {
+ notifications.list.emplace_back(unique_topic->get());
+ return;
+ }
+ op_ret = -ENOENT;
+ ldpp_dout(this, 1) << "failed to get notification info for '" << notif_name << "', ret=" << op_ret << dendl;
+ return;
+ }
+ // loop through all topics of the bucket
+ for (const auto& topic : bucket_topics.topics) {
+ if (topic.second.s3_id.empty()) {
+ // not an s3 notification
+ continue;
+ }
+ notifications.list.emplace_back(topic.second);
+ }
+}
+
+int RGWPSListNotifsOp::verify_permission(optional_yield y) {
+ int ret = get_params();
+ if (ret < 0) {
+ return ret;
+ }
+
+ std::unique_ptr<rgw::sal::User> user = driver->get_user(s->owner.get_id());
+ ret = driver->get_bucket(this, user.get(), s->owner.get_id().tenant, bucket_name, &bucket, y);
+ if (ret < 0) {
+ return ret;
+ }
+
+ if (bucket->get_info().owner != s->owner.get_id()) {
+ ldpp_dout(this, 1) << "user doesn't own bucket, cannot get notification list" << dendl;
+ return -EPERM;
+ }
+
+ return 0;
+}
+
+RGWOp* RGWHandler_REST_PSNotifs_S3::op_get() {
+ return new RGWPSListNotifsOp();
+}
+
+RGWOp* RGWHandler_REST_PSNotifs_S3::op_put() {
+ return new RGWPSCreateNotifOp();
+}
+
+RGWOp* RGWHandler_REST_PSNotifs_S3::op_delete() {
+ return new RGWPSDeleteNotifOp();
+}
+
+RGWOp* RGWHandler_REST_PSNotifs_S3::create_get_op() {
+ return new RGWPSListNotifsOp();
+}
+
+RGWOp* RGWHandler_REST_PSNotifs_S3::create_put_op() {
+ return new RGWPSCreateNotifOp();
+}
+
+RGWOp* RGWHandler_REST_PSNotifs_S3::create_delete_op() {
+ return new RGWPSDeleteNotifOp();
+}
+
class RGWSyncModuleInstance;
typedef std::shared_ptr<RGWSyncModuleInstance> RGWSyncModuleInstanceRef;
class RGWCompressionInfo;
+struct rgw_pubsub_topics;
+struct rgw_pubsub_bucket_topics;
using RGWBucketListNameFilter = std::function<bool (const std::string&)>;
const DoutPrefixProvider* dpp, rgw::sal::Object* obj, rgw::sal::Object* src_obj,
rgw::notify::EventType event_type, rgw::sal::Bucket* _bucket, std::string& _user_id, std::string& _user_tenant,
std::string& _req_id, optional_yield y) = 0;
-
+ /** Read the topic config entry into @a data and (optionally) @a objv_tracker */
+ virtual int read_topics(const std::string& tenant, rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) = 0;
+ /** Write @a info and (optionally) @a objv_tracker into the config */
+ virtual int write_topics(const std::string& tenant, const rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) = 0;
+ /** Remove the topic config, optionally a specific version */
+ virtual int remove_topics(const std::string& tenant, RGWObjVersionTracker* objv_tracker,
+ optional_yield y,const DoutPrefixProvider *dpp) = 0;
/** Get access to the lifecycle management thread */
virtual RGWLC* get_rgwlc(void) = 0;
/** Get access to the coroutine registry. Used to create new coroutine managers */
virtual int abort_multiparts(const DoutPrefixProvider* dpp,
CephContext* cct) = 0;
+ /** Read the bucket notification config into @a notifications with and (optionally) @a objv_tracker */
+ virtual int read_topics(rgw_pubsub_bucket_topics& notifications,
+ RGWObjVersionTracker* objv_tracker, optional_yield y, const DoutPrefixProvider *dpp) = 0;
+ /** Write @a notifications with (optionally) @a objv_tracker into the bucket notification config */
+ virtual int write_topics(const rgw_pubsub_bucket_topics& notifications, RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) = 0;
+ /** Remove the bucket notification config with (optionally) @a objv_tracker */
+ virtual int remove_topics(RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) = 0;
+
/* dang - This is temporary, until the API is completed */
virtual rgw_bucket& get_key() = 0;
virtual RGWBucketInfo& get_info() = 0;
std::string& _user_id, std::string& _user_tenant,
std::string& _req_id, optional_yield y) override;
+ int read_topics(const std::string& tenant, rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) override {
+ return next->read_topics(tenant, topics, objv_tracker, y, dpp);
+ }
+ int write_topics(const std::string& tenant, const rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) override {
+ return next->write_topics(tenant, topics, objv_tracker, y, dpp);
+ }
+ int remove_topics(const std::string& tenant, RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) override {
+ return next->remove_topics(tenant, objv_tracker, y, dpp);
+ }
+
virtual RGWLC* get_rgwlc(void) override;
virtual RGWCoroutinesManagerRegistry* get_cr_registry() override;
virtual int abort_multiparts(const DoutPrefixProvider* dpp,
CephContext* cct) override;
+ int read_topics(rgw_pubsub_bucket_topics& notifications, RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) override {
+ return next->read_topics(notifications, objv_tracker, y, dpp);
+ }
+ int write_topics(const rgw_pubsub_bucket_topics& notifications, RGWObjVersionTracker* obj_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) override {
+ return next->write_topics(notifications, obj_tracker, y, dpp);
+ }
+ int remove_topics(RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) override {
+ return next->remove_topics(objv_tracker, y, dpp);
+ }
+
virtual rgw_bucket& get_key() override { return next->get_key(); }
virtual RGWBucketInfo& get_info() override { return next->get_info(); }
virtual uint64_t get_new_req_id() override {
return ceph::util::generate_random_number<uint64_t>();
}
+
+ int read_topics(const std::string& tenant, rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) override {return -EOPNOTSUPP;}
+ int write_topics(const std::string& tenant, const rgw_pubsub_topics& topics, RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) override {return -ENOENT;}
+ int remove_topics(const std::string& tenant, RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) override {return -ENOENT;}
};
class StoreUser : public User {
(info.bucket.bucket_id != sb.info.bucket.bucket_id);
}
+ int read_topics(rgw_pubsub_bucket_topics& notifications, RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) override {return 0;}
+ int write_topics(const rgw_pubsub_bucket_topics& notifications, RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) override {return 0;}
+ int remove_topics(RGWObjVersionTracker* objv_tracker,
+ optional_yield y, const DoutPrefixProvider *dpp) override {return 0;}
+
friend class BucketList;
protected:
virtual void set_ent(RGWBucketEnt& _ent) { ent = _ent; info.bucket = ent.bucket; info.placement_rule = ent.placement_rule; }
projects / ceph.git / commitdiff