return -EIO;
}
- mutex.Lock();
+ mutex.lock();
init_timer.cancel_all_events();
init_timer.shutdown();
- mutex.Unlock();
+ mutex.unlock();
- rgw_user_init(store);
-
init_async_signal_handler();
register_async_signal_handler(SIGUSR1, handle_sigterm);
* the wiser.
* If that fails, we look up in the requested (possibly empty) tenant.
* If that fails too, we create the account within the global or separated
- * namespace depending on rgw_keystone_implicit_tenants. */
- if (acct_user.tenant.empty()) {
+ * namespace depending on rgw_keystone_implicit_tenants.
+ * For compatibility with previous versions of ceph, it is possible
+ * to enable implicit_tenants for only s3 or only swift.
+ * in this mode ("split_mode"), we must constrain the id lookups to
+ * only use the identifier space that would be used if the id were
+ * to be created. */
+
+ if (split_mode && !implicit_tenant)
+ ; /* suppress lookup for id used by "other" protocol */
+ else if (acct_user.tenant.empty()) {
const rgw_user tenanted_uid(acct_user.id, acct_user.id);
- if (rgw_get_user_info_by_uid(store, tenanted_uid, user_info) >= 0) {
+ if (ctl->user->get_info_by_uid(tenanted_uid, &user_info, null_yield) >= 0) {
/* Succeeded. */
return;
}
}
- if (ctl->user->get_info_by_uid(acct_user, &user_info, null_yield) < 0) {
- ldpp_dout(dpp, 0) << "NOTICE: couldn't map swift user " << acct_user << dendl;
- create_account(dpp, acct_user, user_info);
+ if (split_mode && implicit_tenant)
+ ; /* suppress lookup for id used by "other" protocol */
- else if (rgw_get_user_info_by_uid(store, acct_user, user_info) >= 0) {
++ else if (ctl->user->get_info_by_uid(acct_user, &user_info, null_yield) >= 0) {
+ /* Succeeded. */
+ return;
}
- ldout(cct, 0) << "NOTICE: couldn't map swift user " << acct_user << dendl;
++ ldpp_dout(dpp, 0) << "NOTICE: couldn't map swift user " << acct_user << dendl;
+ create_account(dpp, acct_user, implicit_tenant, user_info);
+
/* Succeeded if we are here (create_account() hasn't throwed). */
}
public:
RemoteApplier(CephContext* const cct,
- RGWRados* const store,
+ RGWCtl* const ctl,
acl_strategy_t&& extra_acl_strategy,
const AuthInfo& info,
- const bool implicit_tenants)
+ rgw::auth::ImplicitTenants& implicit_tenant_context,
+ rgw::auth::ImplicitTenants::implicit_tenant_flag_bits implicit_tenant_bit)
: cct(cct),
- store(store),
+ ctl(ctl),
extra_acl_strategy(std::move(extra_acl_strategy)),
info(info),
- implicit_tenants(implicit_tenants) {
+ implicit_tenant_context(implicit_tenant_context),
+ implicit_tenant_bit(implicit_tenant_bit) {
}
uint32_t get_perms_from_aclspec(const DoutPrefixProvider* dpp, const aclspec_t& aclspec) const override;
s3_main_strategy_plain_t s3_main_strategy_plain;
s3_main_strategy_boto2_t s3_main_strategy_boto2;
- s3_main_strategy_t(CephContext* const cct, RGWCtl* const ctl)
- : s3_main_strategy_plain(cct, ctl),
- s3_main_strategy_boto2(cct, ctl) {
+ s3_main_strategy_t(CephContext* const cct,
+ ImplicitTenants& implicit_tenant_context,
- RGWRados* const store)
- : s3_main_strategy_plain(cct, implicit_tenant_context, store),
- s3_main_strategy_boto2(cct, implicit_tenant_context, store) {
++ RGWCtl* const ctl)
++ : s3_main_strategy_plain(cct, implicit_tenant_context, ctl),
++ s3_main_strategy_boto2(cct, implicit_tenant_context, ctl) {
add_engine(Strategy::Control::SUFFICIENT, s3_main_strategy_plain);
add_engine(Strategy::Control::FALLBACK, s3_main_strategy_boto2);
}
public:
StrategyRegistry(CephContext* const cct,
- RGWRados* const store)
- : s3_main_strategy(cct, implicit_tenant_context, store),
- s3_post_strategy(cct, implicit_tenant_context, store),
- swift_strategy(cct, implicit_tenant_context, store),
- sts_strategy(cct, implicit_tenant_context, store) {
+ ImplicitTenants& implicit_tenant_context,
- : s3_main_strategy(cct, ctl),
- s3_post_strategy(cct, ctl),
- swift_strategy(cct, ctl),
- sts_strategy(cct, ctl) {
+ RGWCtl* const ctl)
++ : s3_main_strategy(cct, implicit_tenant_context, ctl),
++ s3_post_strategy(cct, implicit_tenant_context, ctl),
++ swift_strategy(cct, implicit_tenant_context, ctl),
++ sts_strategy(cct, implicit_tenant_context, ctl) {
}
const s3_main_strategy_t& get_s3_main() const {
static std::shared_ptr<StrategyRegistry>
create(CephContext* const cct,
- RGWRados* const store) {
- return std::make_shared<StrategyRegistry>(cct, implicit_tenant_context, store);
+ ImplicitTenants& implicit_tenant_context,
- return std::make_shared<StrategyRegistry>(cct, ctl);
+ RGWCtl* const ctl) {
++ return std::make_shared<StrategyRegistry>(cct, implicit_tenant_context, ctl);
}
};
public rgw::auth::LocalApplier::Factory,
public rgw::auth::RoleApplier::Factory {
typedef rgw::auth::IdentityApplier::aplptr_t aplptr_t;
- RGWRados* const store;
+ RGWCtl* const ctl;
+ rgw::auth::ImplicitTenants& implicit_tenant_context;
STSEngine sts_engine;
rgw::auth::RemoteApplier::acl_strategy_t&& acl_alg,
const rgw::auth::RemoteApplier::AuthInfo &info
) const override {
- auto apl = rgw::auth::add_sysreq(cct, store, s,
- rgw::auth::RemoteApplier(cct, store, std::move(acl_alg), info,
+ auto apl = rgw::auth::add_sysreq(cct, ctl, s,
+ rgw::auth::RemoteApplier(cct, ctl, std::move(acl_alg), info,
- cct->_conf->rgw_keystone_implicit_tenants));
+ implicit_tenant_context,
+ rgw::auth::ImplicitTenants::IMPLICIT_TENANTS_S3));
return aplptr_t(new decltype(apl)(std::move(apl)));
}
public:
STSAuthStrategy(CephContext* const cct,
- RGWRados* const store,
+ RGWCtl* const ctl,
+ rgw::auth::ImplicitTenants& implicit_tenant_context,
AWSEngine::VersionAbstractor* const ver_abstractor)
- : store(store),
+ : ctl(ctl),
+ implicit_tenant_context(implicit_tenant_context),
- sts_engine(cct, store, *ver_abstractor,
+ sts_engine(cct, ctl, *ver_abstractor,
static_cast<rgw::auth::LocalApplier::Factory*>(this),
static_cast<rgw::auth::RemoteApplier::Factory*>(this),
static_cast<rgw::auth::RoleApplier::Factory*>(this)) {
class ExternalAuthStrategy : public rgw::auth::Strategy,
public rgw::auth::RemoteApplier::Factory {
typedef rgw::auth::IdentityApplier::aplptr_t aplptr_t;
- RGWRados* const store;
+ RGWCtl* const ctl;
+ rgw::auth::ImplicitTenants& implicit_tenant_context;
using keystone_config_t = rgw::keystone::CephCtxConfig;
using keystone_cache_t = rgw::keystone::TokenCache;
rgw::auth::RemoteApplier::acl_strategy_t&& acl_alg,
const rgw::auth::RemoteApplier::AuthInfo &info
) const override {
- auto apl = rgw::auth::add_sysreq(cct, store, s,
- rgw::auth::RemoteApplier(cct, store, std::move(acl_alg), info,
+ auto apl = rgw::auth::add_sysreq(cct, ctl, s,
+ rgw::auth::RemoteApplier(cct, ctl, std::move(acl_alg), info,
- cct->_conf->rgw_keystone_implicit_tenants));
+ implicit_tenant_context,
+ rgw::auth::ImplicitTenants::IMPLICIT_TENANTS_S3));
/* TODO(rzarzynski): replace with static_ptr. */
return aplptr_t(new decltype(apl)(std::move(apl)));
}
public:
ExternalAuthStrategy(CephContext* const cct,
- RGWRados* const store,
+ RGWCtl* const ctl,
+ rgw::auth::ImplicitTenants& implicit_tenant_context,
AWSEngine::VersionAbstractor* const ver_abstractor)
- : store(store),
+ : ctl(ctl),
+ implicit_tenant_context(implicit_tenant_context),
- ldap_engine(cct, store, *ver_abstractor,
+ ldap_engine(cct, ctl, *ver_abstractor,
static_cast<rgw::auth::RemoteApplier::Factory*>(this)) {
if (cct->_conf->rgw_s3_auth_use_keystone &&
}
AWSAuthStrategy(CephContext* const cct,
- RGWRados* const store)
- : store(store),
+ rgw::auth::ImplicitTenants& implicit_tenant_context,
+ RGWCtl* const ctl)
+ : ctl(ctl),
ver_abstractor(cct),
anonymous_engine(cct,
static_cast<rgw::auth::LocalApplier::Factory*>(this)),
- external_engines(cct, store, implicit_tenant_context, &ver_abstractor),
- sts_engine(cct, store, implicit_tenant_context, &ver_abstractor),
- local_engine(cct, store, ver_abstractor,
- external_engines(cct, ctl, &ver_abstractor),
- sts_engine(cct, ctl, &ver_abstractor),
++ external_engines(cct, ctl, implicit_tenant_context, &ver_abstractor),
++ sts_engine(cct, ctl, implicit_tenant_context, &ver_abstractor),
+ local_engine(cct, ctl, ver_abstractor,
static_cast<rgw::auth::LocalApplier::Factory*>(this)) {
/* The anonymous auth. */
if (AllowAnonAccessT) {
// split instance:shard
pos = instance.find(':');
- if (pos == boost::string_ref::npos) {
+ if (pos == string::npos) {
bucket->bucket_id.assign(instance.begin(), instance.end());
- *shard_id = -1;
+ if (shard_id) {
+ *shard_id = -1;
+ }
return 0;
}
*sink = msg;
}
-int RGWBucket::init(RGWRados *storage, RGWBucketAdminOpState& op_state)
+int RGWBucket::init(RGWRados *storage, RGWBucketAdminOpState& op_state,
- std::string *err_msg, map<string, bufferlist> *pattrs)
++ optional_yield y, std::string *err_msg,
++ map<string, bufferlist> *pattrs)
{
- std::string bucket_tenant;
- if (!storage)
+ if (!storage) {
+ set_err_msg(err_msg, "no storage!");
return -EINVAL;
+ }
store = storage;
rgw_user user_id = op_state.get_user_id();
-- tenant = user_id.tenant;
- bucket_tenant = tenant;
-- bucket_name = op_state.get_bucket_name();
++ bucket.tenant = user_id.tenant;
++ bucket.name = op_state.get_bucket_name();
RGWUserBuckets user_buckets;
-- auto obj_ctx = store->svc.sysobj->init_obj_ctx();
-- if (bucket_name.empty() && user_id.empty())
++ if (bucket.name.empty() && user_id.empty())
return -EINVAL;
- if (!bucket_name.empty()) {
- int r = store->get_bucket_info(obj_ctx, tenant, bucket_name, bucket_info, NULL, null_yield);
+ // split possible tenant/name
- auto pos = bucket_name.find('/');
++ auto pos = bucket.name.find('/');
+ if (pos != string::npos) {
- bucket_tenant = bucket_name.substr(0, pos);
- bucket_name = bucket_name.substr(pos + 1);
++ bucket.tenant = bucket.name.substr(0, pos);
++ bucket.name = bucket.name.substr(pos + 1);
+ }
+
- if (!bucket_name.empty()) {
++ if (!bucket.name.empty()) {
+ ceph::real_time mtime;
- int r = store->get_bucket_info(obj_ctx, bucket_tenant, bucket_name,
- bucket_info, &mtime, null_yield, pattrs);
++ int r = store->ctl.bucket->read_bucket_info(
++ bucket, &bucket_info, y,
++ RGWBucketCtl::BucketInstance::GetParams().set_attrs(pattrs),
++ &ep_objv);
if (r < 0) {
- set_err_msg(err_msg, "failed to fetch bucket info for bucket=" + bucket_name);
-- ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket_name << dendl;
++ set_err_msg(err_msg, "failed to fetch bucket info for bucket=" + bucket.name);
return r;
}
}
if (!user_id.empty()) {
- int r = rgw_get_user_info_by_uid(store, user_id, user_info);
- int r = store->ctl.user->get_info_by_uid(user_id, &user_info, null_yield);
- if (r < 0)
++ int r = store->ctl.user->get_info_by_uid(user_id, &user_info, y);
+ if (r < 0) {
+ set_err_msg(err_msg, "failed to fetch user info");
return r;
+ }
op_state.display_name = user_info.display_name;
}
return 0;
}
- int RGWBucket::link(RGWBucketAdminOpState& op_state,
- map<string, bufferlist>& attrs, std::string *err_msg)
-int RGWBucket::link(RGWBucketAdminOpState& op_state, optional_yield y, std::string *err_msg)
++int RGWBucket::link(RGWBucketAdminOpState& op_state, optional_yield y,
++ map<string, bufferlist>& attrs, std::string *err_msg)
{
if (!op_state.is_user_op()) {
set_err_msg(err_msg, "empty user id");
}
string bucket_id = op_state.get_bucket_id();
- if (bucket_id.empty()) {
- set_err_msg(err_msg, "empty bucket instance id");
- return -EINVAL;
- }
std::string display_name = op_state.get_user_display_name();
- rgw_bucket bucket = op_state.get_bucket();
+ rgw_bucket& bucket = op_state.get_bucket();
+ if (!bucket_id.empty() && bucket_id != bucket.bucket_id) {
+ set_err_msg(err_msg,
+ "specified bucket id does not match " + bucket.bucket_id);
+ return -EINVAL;
+ }
+ rgw_bucket old_bucket = bucket;
- bucket.tenant = tenant;
+ if (!op_state.new_bucket_name.empty()) {
+ auto pos = op_state.new_bucket_name.find('/');
+ if (pos != string::npos) {
+ bucket.tenant = op_state.new_bucket_name.substr(0, pos);
+ bucket.name = op_state.new_bucket_name.substr(pos + 1);
+ } else {
+ bucket.name = op_state.new_bucket_name;
+ }
+ }
- const rgw_pool& root_pool = store->svc.zone->get_zone_params().domain_root;
- std::string bucket_entry;
- rgw_make_bucket_entry_name(tenant, bucket_name, bucket_entry);
- rgw_raw_obj obj(root_pool, bucket_entry);
RGWObjVersionTracker objv_tracker;
+ RGWObjVersionTracker old_version = bucket_info.objv_tracker;
- map<string, bufferlist> attrs;
- RGWBucketInfo bucket_info;
-
- auto obj_ctx = store->svc.sysobj->init_obj_ctx();
-
- RGWSI_MetaBackend_CtxParams ctx_params = RGWSI_MetaBackend_CtxParams_SObj(&obj_ctx);
+ map<string, bufferlist>::iterator aiter = attrs.find(RGW_ATTR_ACL);
+ if (aiter == attrs.end()) {
+ // should never happen; only pre-argonaut buckets lacked this.
+ ldout(store->ctx(), 0) << "WARNING: can't bucket link because no acl on bucket=" << old_bucket.name << dendl;
+ set_err_msg(err_msg,
+ "While crossing the Anavros you have displeased the goddess Hera."
+ " You must sacrifice your ancient bucket " + bucket.bucket_id);
+ return -EINVAL;
+ }
- bufferlist aclbl = aiter->second;
++ bufferlist& aclbl = aiter->second;
+ RGWAccessControlPolicy policy;
+ ACLOwner owner;
+ try {
+ auto iter = aclbl.cbegin();
+ decode(policy, iter);
+ owner = policy.get_owner();
+ } catch (buffer::error& err) {
+ set_err_msg(err_msg, "couldn't decode policy");
+ return -EIO;
+ }
- int r = rgw_unlink_bucket(store, owner.get_id(),
- old_bucket.tenant, old_bucket.name, false);
+ auto bucket_ctl = store->ctl.bucket;
-
- int r = bucket_ctl->read_bucket_instance_info(bucket, &bucket_info, y, RGWBucketCtl::BucketInstance::GetParams()
- .set_attrs(&attrs)
- .set_objv_tracker(&objv_tracker)
- .set_bectx_params(&obj_ctx));
++ int r = bucket_ctl->unlink_bucket(owner.get_id(), old_bucket, y, false);
if (r < 0) {
+ set_err_msg(err_msg, "could not unlink policy from user " + owner.get_id().to_str());
return r;
}
- map<string, bufferlist>::iterator aiter = attrs.find(RGW_ATTR_ACL);
- if (aiter != attrs.end()) {
- bufferlist aclbl = aiter->second;
- RGWAccessControlPolicy policy;
- ACLOwner owner;
- try {
- auto iter = aclbl.cbegin();
- decode(policy, iter);
- owner = policy.get_owner();
- } catch (buffer::error& err) {
- set_err_msg(err_msg, "couldn't decode policy");
- return -EIO;
- }
-
- r = store->ctl.bucket->unlink_bucket(owner.get_id(), bucket, y, false);
- if (r < 0) {
- set_err_msg(err_msg, "could not unlink policy from user " + owner.get_id().to_str());
- return r;
- }
-
- // now update the user for the bucket...
- if (display_name.empty()) {
- ldout(store->ctx(), 0) << "WARNING: user " << user_info.user_id << " has no display name set" << dendl;
- }
- policy.create_default(user_info.user_id, display_name);
+ // now update the user for the bucket...
+ if (display_name.empty()) {
+ ldout(store->ctx(), 0) << "WARNING: user " << user_info.user_id << " has no display name set" << dendl;
+ }
- bucket_info.owner = user_info.user_id;
+ RGWAccessControlPolicy policy_instance;
+ policy_instance.create_default(user_info.user_id, display_name);
+ owner = policy_instance.get_owner();
- // ...and encode the acl
- attrs[RGW_ATTR_ACL].clear();
- policy.encode(attrs[RGW_ATTR_ACL]);
+ aclbl.clear();
+ policy_instance.encode(aclbl);
- if (bucket == old_bucket) {
- r = rgw_set_bucket_acl(store, owner, bucket, bucket_info, aclbl);
- r = bucket_ctl->store_bucket_instance_info(bucket, bucket_info, y,
- RGWBucketCtl::BucketInstance::PutParams()
- .set_attrs(&attrs)
- .set_objv_tracker(&objv_tracker));
-- if (r < 0) {
- set_err_msg(err_msg, "failed to set new acl");
-- return r;
-- }
- } else {
- attrs[RGW_ATTR_ACL] = aclbl;
++ auto instance_params = RGWBucketCtl::BucketInstance::PutParams().set_attrs(&attrs);
+
- /* update entrypoint */
++ bucket_info.owner = user_info.user_id;
++ if (bucket != old_bucket) {
+ bucket_info.bucket = bucket;
- bucket_info.owner = user_info.user_id;
+ bucket_info.objv_tracker.version_for_read()->ver = 0;
- r = store->put_bucket_instance_info(bucket_info, true, real_time(), &attrs);
- if (r < 0) {
- set_err_msg(err_msg, "ERROR: failed writing bucket instance info: " + cpp_strerror(-r));
- return r;
- }
++ instance_params.set_exclusive(true);
++ }
+
- RGWBucketEntryPoint be;
- map<string, bufferlist> be_attrs;
- RGWObjVersionTracker be_ot;
- r = bucket_ctl->read_bucket_entrypoint_info(bucket, &be, y, RGWBucketCtl::Bucket::GetParams()
- .set_objv_tracker(&be_ot)
- .set_attrs(&be_attrs));
- if (r < 0) {
- return r;
- }
++ r = bucket_ctl->store_bucket_instance_info(bucket, bucket_info, y, instance_params);
++ if (r < 0) {
++ set_err_msg(err_msg, "ERROR: failed writing bucket instance info: " + cpp_strerror(-r));
++ return r;
+ }
- be.owner = bucket_info.owner;
- be_attrs[RGW_ATTR_ACL] = attrs[RGW_ATTR_ACL];
+ RGWBucketEntryPoint ep;
+ ep.bucket = bucket_info.bucket;
+ ep.owner = user_info.user_id;
+ ep.creation_time = bucket_info.creation_time;
+ ep.linked = true;
+ map<string, bufferlist> ep_attrs;
+ rgw_ep_info ep_data{ep, ep_attrs};
+
- r = rgw_link_bucket(store, user_info.user_id, bucket_info.bucket,
- ceph::real_time(), true, &ep_data);
++ /* link to user */
++ r = store->ctl.bucket->link_bucket(user_info.user_id,
++ bucket_info.bucket,
++ ceph::real_time(),
++ y, true, &ep_data);
+ if (r < 0) {
+ set_err_msg(err_msg, "failed to relink bucket");
+ return r;
+ }
+
- r = bucket_ctl->store_bucket_entrypoint_info(bucket, be, y, RGWBucketCtl::Bucket::PutParams()
- .set_objv_tracker(&be_ot)
- .set_attrs(&be_attrs));
+ if (bucket != old_bucket) {
- RGWObjVersionTracker ep_version;
- *ep_version.version_for_read() = bucket_info.ep_objv;
+ // like RGWRados::delete_bucket -- excepting no bucket_index work.
- r = rgw_bucket_delete_bucket_obj(store,
- old_bucket.tenant, old_bucket.name, ep_version);
++ r = bucket_ctl->remove_bucket_entrypoint_info(old_bucket, y,
++ RGWBucketCtl::Bucket::RemoveParams()
++ .set_objv_tracker(&ep_objv));
if (r < 0) {
+ set_err_msg(err_msg, "failed to unlink old bucket endpoint " + old_bucket.tenant + "/" + old_bucket.name);
return r;
}
- string entry = old_bucket.get_key();
- r = rgw_bucket_instance_remove_entry(store, entry, &old_version);
+
- /* link to user */
- r = store->ctl.bucket->link_bucket(user_info.user_id,
- bucket_info.bucket,
- ceph::real_time(),
- y);
++ r = bucket_ctl->remove_bucket_instance_info(old_bucket, bucket_info, y,
++ RGWBucketCtl::BucketInstance::RemoveParams()
++ .set_objv_tracker(&old_version));
if (r < 0) {
- set_err_msg(err_msg, "failed to unlink old bucket info " + entry);
++ set_err_msg(err_msg, "failed to unlink old bucket info");
return r;
}
-
}
return 0;
}
- int RGWBucket::chown(RGWBucketAdminOpState& op_state, const string& marker, std::string *err_msg)
++int RGWBucket::chown(RGWBucketAdminOpState& op_state, const string& marker,
++ optional_yield y, std::string *err_msg)
+{
- RGWUserInfo user_info;
- int ret = rgw_get_user_info_by_uid(store, bucket_info.owner, user_info);
++ int ret = store->ctl.bucket->chown(store, bucket_info, user_info.user_id,
++ user_info.display_name, marker, y);
+ if (ret < 0) {
- set_err_msg(err_msg, "user info failed: " + cpp_strerror(-ret));
- return ret;
- }
-
- ret = rgw_bucket_chown(store, bucket_info, user_info.user_id,
- user_info.display_name, marker);
- if (ret < 0) {
- set_err_msg(err_msg, "Failed to change object ownership" + cpp_strerror(-ret));
++ set_err_msg(err_msg, "Failed to change object ownership: " + cpp_strerror(-ret));
+ }
+
+ return ret;
+}
+
- int RGWBucket::unlink(RGWBucketAdminOpState& op_state, std::string *err_msg)
+ int RGWBucket::unlink(RGWBucketAdminOpState& op_state, optional_yield y, std::string *err_msg)
{
rgw_bucket bucket = op_state.get_bucket();
return 0;
}
-static void dump_bucket_index(map<string, rgw_bucket_dir_entry> result, Formatter *f)
+static void dump_bucket_index(const RGWRados::ent_map_t& result, Formatter *f)
{
- map<string, rgw_bucket_dir_entry>::iterator iter;
- for (iter = result.begin(); iter != result.end(); ++iter) {
+ for (auto iter = result.begin(); iter != result.end(); ++iter) {
f->dump_string("object", iter->first);
-- }
++ }
}
static void dump_bucket_usage(map<RGWObjCategory, RGWStorageStats>& stats, Formatter *formatter)
Formatter *formatter = flusher.get_formatter();
formatter->open_object_section("objects");
while (is_truncated) {
- map<string, rgw_bucket_dir_entry> result;
+ RGWRados::ent_map_t result;
- result.reserve(1000);
int r = store->cls_bucket_list_ordered(bucket_info, RGW_NO_SHARD,
marker, prefix, 1000, true,
{
RGWBucket bucket;
-- int ret = bucket.init(store, op_state);
++ int ret = bucket.init(store, op_state, null_yield);
if (ret < 0)
return ret;
{
RGWBucket bucket;
-- int ret = bucket.init(store, op_state);
++ int ret = bucket.init(store, op_state, null_yield);
if (ret < 0)
return ret;
int RGWBucketAdminOp::link(RGWRados *store, RGWBucketAdminOpState& op_state, string *err)
{
RGWBucket bucket;
- int ret = bucket.init(store, op_state, err, &attrs);
+ map<string, bufferlist> attrs;
+
- return bucket.link(op_state, attrs, err);
++ int ret = bucket.init(store, op_state, null_yield, err, &attrs);
+ if (ret < 0)
+ return ret;
+
- int ret = bucket.init(store, op_state, err, &attrs);
++ return bucket.link(op_state, null_yield, attrs, err);
+
+}
+
+int RGWBucketAdminOp::chown(RGWRados *store, RGWBucketAdminOpState& op_state, const string& marker, string *err)
+{
+ RGWBucket bucket;
+ map<string, bufferlist> attrs;
+
++ int ret = bucket.init(store, op_state, null_yield, err, &attrs);
+ if (ret < 0)
+ return ret;
- ret = bucket.link(op_state, attrs, err);
- int ret = bucket.init(store, op_state);
++ ret = bucket.link(op_state, null_yield, attrs, err);
if (ret < 0)
return ret;
- return bucket.chown(op_state, marker, err);
- return bucket.link(op_state, null_yield, err);
++ return bucket.chown(op_state, marker, null_yield, err);
}
RGWBucket bucket;
-- ret = bucket.init(store, op_state);
++ ret = bucket.init(store, op_state, null_yield);
if (ret < 0)
return ret;
{
RGWBucket bucket;
-- int ret = bucket.init(store, op_state);
++ int ret = bucket.init(store, op_state, y);
if (ret < 0)
return ret;
{
RGWBucket bucket;
-- int ret = bucket.init(store, op_state);
++ int ret = bucket.init(store, op_state, null_yield);
if (ret < 0)
return ret;
{
RGWBucket bucket;
-- int ret = bucket.init(store, op_state);
++ int ret = bucket.init(store, op_state, null_yield);
if (ret < 0)
return ret;
return bucket.set_quota(op_state);
RGWFormatterFlusher& flusher, bool dry_run)
{
RGWBucket admin_bucket;
-- int ret = admin_bucket.init(store, op_state);
++ int ret = admin_bucket.init(store, op_state, null_yield);
if (ret < 0) {
lderr(store->ctx()) << "failed to initialize bucket" << dendl;
return ret;
JSONDecoder::decode_json("entry", entry, obj);
}
-RGWDataChangesLog::RGWDataChangesLog(RGWSI_Zone *zone_svc,
- RGWSI_Cls *cls_svc) : cct(zone_svc->ctx()),
- lock("RGWDataChangesLog::lock"), modified_lock("RGWDataChangesLog::modified_lock"),
- changes(cct->_conf->rgw_data_log_changes_size)
+
++RGWDataChangesLog::RGWDataChangesLog(RGWSI_Zone *zone_svc, RGWSI_Cls *cls_svc)
++ : cct(zone_svc->ctx()), changes(cct->_conf->rgw_data_log_changes_size)
+ {
+ svc.zone = zone_svc;
+ svc.cls = cls_svc;
+
+ num_shards = cct->_conf->rgw_data_log_num_shards;
+
+ oids = new string[num_shards];
+
+ string prefix = cct->_conf->rgw_data_log_obj_prefix;
+
+ if (prefix.empty()) {
+ prefix = "data_log";
+ }
+
+ for (int i = 0; i < num_shards; i++) {
+ char buf[16];
+ snprintf(buf, sizeof(buf), "%s.%d", prefix.c_str(), i);
+ oids[i] = buf;
+ }
+
+ renew_thread = new ChangesRenewThread(cct, this);
+ renew_thread->create("rgw_dt_lg_renew");
+ }
+
int RGWDataChangesLog::choose_oid(const rgw_bucket_shard& bs) {
const string& name = bs.bucket.name;
int shard_shift = (bs.shard_id > 0 ? bs.shard_id : 0);
return 0;
}
- int list_keys_next(void *handle, int max, list<string>& keys, bool *truncated) override {
- list_keys_info *info = static_cast<list_keys_info *>(handle);
+ info = entry_point.old_bucket_info;
- string no_filter;
+ ot.generate_new_write_ver(cct);
- keys.clear();
+ ret = do_store_linked_bucket_info(ctx, info, nullptr, false, ep_mtime, &ot.write_version, &attrs, true, y);
+ if (ret < 0) {
+ ldout(cct, 0) << "ERROR: failed to put_linked_bucket_info(): " << ret << dendl;
+ return ret;
+ }
- RGWRados *store = info->store;
+ return 0;
+ }
- list<string> unfiltered_keys;
+ int RGWBucketCtl::set_bucket_instance_attrs(RGWBucketInfo& bucket_info,
+ map<string, bufferlist>& attrs,
+ RGWObjVersionTracker *objv_tracker,
+ optional_yield y)
+ {
+ return call([&](RGWSI_Bucket_X_Ctx& ctx) {
+ rgw_bucket& bucket = bucket_info.bucket;
- int ret = store->list_raw_objects_next(no_filter, max, info->ctx,
- unfiltered_keys, truncated);
- if (ret < 0 && ret != -ENOENT)
- return ret;
- if (ret == -ENOENT) {
- if (truncated)
- *truncated = false;
- return 0;
+ if (!bucket_info.has_instance_obj) {
+ /* an old bucket object, need to convert it */
+ int ret = convert_old_bucket_info(ctx, bucket, y);
+ if (ret < 0) {
+ ldout(cct, 0) << "ERROR: failed converting old bucket info: " << ret << dendl;
+ return ret;
+ }
}
- constexpr int prefix_size = sizeof(RGW_BUCKET_INSTANCE_MD_PREFIX) - 1;
- // now filter in the relevant entries
- list<string>::iterator iter;
- for (iter = unfiltered_keys.begin(); iter != unfiltered_keys.end(); ++iter) {
- string& k = *iter;
+ return do_store_bucket_instance_info(ctx.bi,
+ bucket,
+ bucket_info,
+ y,
+ BucketInstance::PutParams().set_attrs(&attrs)
+ .set_objv_tracker(objv_tracker)
+ .set_orig_info(&bucket_info));
+ });
+ }
+
- if (k.compare(0, prefix_size, RGW_BUCKET_INSTANCE_MD_PREFIX) == 0) {
- auto oid = k.substr(prefix_size);
- rgw_bucket_instance_oid_to_key(oid);
- keys.emplace_back(std::move(oid));
+ int RGWBucketCtl::link_bucket(const rgw_user& user_id,
+ const rgw_bucket& bucket,
+ ceph::real_time creation_time,
+ optional_yield y,
- bool update_entrypoint)
++ bool update_entrypoint,
++ rgw_ep_info *pinfo)
+ {
+ return bm_handler->call([&](RGWSI_Bucket_EP_Ctx& ctx) {
- return do_link_bucket(ctx, user_id, bucket, creation_time, y, update_entrypoint);
++ return do_link_bucket(ctx, user_id, bucket, creation_time, y,
++ update_entrypoint, pinfo);
+ });
+ }
+
+ int RGWBucketCtl::do_link_bucket(RGWSI_Bucket_EP_Ctx& ctx,
+ const rgw_user& user_id,
+ const rgw_bucket& bucket,
+ ceph::real_time creation_time,
+ optional_yield y,
- bool update_entrypoint)
++ bool update_entrypoint,
++ rgw_ep_info *pinfo)
+ {
+ int ret;
+
+ RGWBucketEntryPoint ep;
+ RGWObjVersionTracker ot;
+
- map<string, bufferlist> attrs;
++ map<string, bufferlist> attrs, *pattrs = nullptr;
+
+ string meta_key;
+
+ if (update_entrypoint) {
+ meta_key = RGWSI_Bucket::get_entrypoint_meta_key(bucket);
- ret = svc.bucket->read_bucket_entrypoint_info(ctx,
- meta_key,
- &ep, &ot,
- nullptr, &attrs,
- y);
- if (ret < 0 && ret != -ENOENT) {
- ldout(cct, 0) << "ERROR: store->get_bucket_entrypoint_info() returned: "
- << cpp_strerror(-ret) << dendl;
++ if (pinfo) {
++ ep = pinfo->ep;
++ pattrs = &pinfo->attrs;
++ } else {
++ ret = svc.bucket->read_bucket_entrypoint_info(ctx,
++ meta_key,
++ &ep, &ot,
++ nullptr, &attrs,
++ y);
++ if (ret < 0 && ret != -ENOENT) {
++ ldout(cct, 0) << "ERROR: store->get_bucket_entrypoint_info() returned: "
++ << cpp_strerror(-ret) << dendl;
+ }
++ pattrs = &attrs;
}
-
- return 0;
}
- void list_keys_complete(void *handle) override {
- list_keys_info *info = static_cast<list_keys_info *>(handle);
- delete info;
+ ret = ctl.user->add_bucket(user_id, bucket, creation_time);
+ if (ret < 0) {
+ ldout(cct, 0) << "ERROR: error adding bucket to user directory: user=" << user_id
+ << " bucket=" << bucket << " err=" << cpp_strerror(-ret) << dendl;
+ goto done_err;
}
- string get_marker(void *handle) override {
- list_keys_info *info = static_cast<list_keys_info *>(handle);
- return info->store->list_raw_objs_get_cursor(info->ctx);
+ if (!update_entrypoint)
+ return 0;
+
+ ep.linked = true;
+ ep.owner = user_id;
+ ep.bucket = bucket;
- ret = svc.bucket->store_bucket_entrypoint_info(ctx, meta_key, ep, false, real_time(), &attrs, &ot, y);
++ ret = svc.bucket->store_bucket_entrypoint_info(ctx, meta_key, ep, false,
++ real_time(), pattrs, &ot, y);
+ if (ret < 0)
+ goto done_err;
+
+ return 0;
+ done_err:
+ int r = do_unlink_bucket(ctx, user_id, bucket, y, true);
+ if (r < 0) {
+ ldout(cct, 0) << "ERROR: failed unlinking bucket on error cleanup: "
+ << cpp_strerror(-r) << dendl;
}
+ return ret;
+ }
- /*
- * hash entry for mdlog placement. Use the same hash key we'd have for the bucket entry
- * point, so that the log entries end up at the same log shard, so that we process them
- * in order
- */
- void get_hash_key(const string& section, const string& key, string& hash_key) override {
- string k;
- int pos = key.find(':');
- if (pos < 0)
- k = key;
- else
- k = key.substr(0, pos);
- hash_key = "bucket:" + k;
+ int RGWBucketCtl::unlink_bucket(const rgw_user& user_id, const rgw_bucket& bucket, optional_yield y, bool update_entrypoint)
+ {
+ return bm_handler->call([&](RGWSI_Bucket_EP_Ctx& ctx) {
+ return do_unlink_bucket(ctx, user_id, bucket, y, update_entrypoint);
+ });
+ }
+
+ int RGWBucketCtl::do_unlink_bucket(RGWSI_Bucket_EP_Ctx& ctx,
+ const rgw_user& user_id,
+ const rgw_bucket& bucket,
+ optional_yield y,
+ bool update_entrypoint)
+ {
+ int ret = ctl.user->remove_bucket(user_id, bucket);
+ if (ret < 0) {
+ ldout(cct, 0) << "ERROR: error removing bucket from directory: "
+ << cpp_strerror(-ret)<< dendl;
}
- };
- class RGWArchiveBucketInstanceMetadataHandler : public RGWBucketInstanceMetadataHandler {
- public:
+ if (!update_entrypoint)
+ return 0;
+
+ RGWBucketEntryPoint ep;
+ RGWObjVersionTracker ot;
+ map<string, bufferlist> attrs;
+ string meta_key = RGWSI_Bucket::get_entrypoint_meta_key(bucket);
+ ret = svc.bucket->read_bucket_entrypoint_info(ctx, meta_key, &ep, &ot, nullptr, &attrs, y);
+ if (ret == -ENOENT)
+ return 0;
+ if (ret < 0)
+ return ret;
- int remove(RGWRados *store, string& entry, RGWObjVersionTracker& objv_tracker) override {
- ldout(store->ctx(), 0) << "SKIP: bucket instance removal is not allowed on archive zone: bucket.instance:" << entry << dendl;
+ if (!ep.linked)
return 0;
+
+ if (ep.owner != user_id) {
+ ldout(cct, 0) << "bucket entry point user mismatch, can't unlink bucket: " << ep.owner << " != " << user_id << dendl;
+ return -EINVAL;
}
- };
- RGWMetadataHandler *RGWBucketMetaHandlerAllocator::alloc() {
- return new RGWBucketMetadataHandler;
+ ep.linked = false;
+ return svc.bucket->store_bucket_entrypoint_info(ctx, meta_key, ep, false, real_time(), &attrs, &ot, y);
}
- RGWMetadataHandler *RGWBucketInstanceMetaHandlerAllocator::alloc() {
- return new RGWBucketInstanceMetadataHandler;
++int RGWBucketCtl::set_acl(ACLOwner& owner, rgw_bucket& bucket,
++ RGWBucketInfo& bucket_info, bufferlist& bl,
++ optional_yield y)
++{
++ // set owner and acl
++ bucket_info.owner = owner.get_id();
++ std::map<std::string, bufferlist> attrs{{RGW_ATTR_ACL, bl}};
++
++ int r = store_bucket_instance_info(bucket, bucket_info, y,
++ BucketInstance::PutParams().set_attrs(&attrs));
++ if (r < 0) {
++ cerr << "ERROR: failed to set bucket owner: " << cpp_strerror(-r) << std::endl;
++ return r;
++ }
++
++ return 0;
+}
+
- RGWMetadataHandler *RGWArchiveBucketMetaHandlerAllocator::alloc() {
- return new RGWArchiveBucketMetadataHandler;
++// TODO: remove RGWRados dependency for bucket listing
++int RGWBucketCtl::chown(RGWRados *store, RGWBucketInfo& bucket_info,
++ const rgw_user& user_id, const std::string& display_name,
++ const std::string& marker, optional_yield y)
++{
++ RGWObjectCtx obj_ctx(store);
++ std::vector<rgw_bucket_dir_entry> objs;
++ map<string, bool> common_prefixes;
++
++ RGWRados::Bucket target(store, bucket_info);
++ RGWRados::Bucket::List list_op(&target);
++
++ list_op.params.list_versions = true;
++ list_op.params.allow_unordered = true;
++ list_op.params.marker = marker;
++
++ bool is_truncated = false;
++ int count = 0;
++ int max_entries = 1000;
++
++ //Loop through objects and update object acls to point to bucket owner
++
++ do {
++ objs.clear();
++ int ret = list_op.list_objects(max_entries, &objs, &common_prefixes, &is_truncated, y);
++ if (ret < 0) {
++ ldout(store->ctx(), 0) << "ERROR: list objects failed: " << cpp_strerror(-ret) << dendl;
++ return ret;
++ }
++
++ list_op.params.marker = list_op.get_next_marker();
++ count += objs.size();
++
++ for (const auto& obj : objs) {
++
++ rgw_obj r_obj(bucket_info.bucket, obj.key);
++ RGWRados::Object op_target(store, bucket_info, obj_ctx, r_obj);
++ RGWRados::Object::Read read_op(&op_target);
++
++ map<string, bufferlist> attrs;
++ read_op.params.attrs = &attrs;
++ ret = read_op.prepare(y);
++ if (ret < 0){
++ ldout(store->ctx(), 0) << "ERROR: failed to read object " << obj.key.name << cpp_strerror(-ret) << dendl;
++ continue;
++ }
++ const auto& aiter = attrs.find(RGW_ATTR_ACL);
++ if (aiter == attrs.end()) {
++ ldout(store->ctx(), 0) << "ERROR: no acls found for object " << obj.key.name << " .Continuing with next object." << dendl;
++ continue;
++ } else {
++ bufferlist& bl = aiter->second;
++ RGWAccessControlPolicy policy(store->ctx());
++ ACLOwner owner;
++ try {
++ decode(policy, bl);
++ owner = policy.get_owner();
++ } catch (buffer::error& err) {
++ ldout(store->ctx(), 0) << "ERROR: decode policy failed" << err << dendl;
++ return -EIO;
++ }
++
++ //Get the ACL from the policy
++ RGWAccessControlList& acl = policy.get_acl();
++
++ //Remove grant that is set to old owner
++ acl.remove_canon_user_grant(owner.get_id());
++
++ //Create a grant and add grant
++ ACLGrant grant;
++ grant.set_canon(user_id, display_name, RGW_PERM_FULL_CONTROL);
++ acl.add_grant(&grant);
++
++ //Update the ACL owner to the new user
++ owner.set_id(user_id);
++ owner.set_name(display_name);
++ policy.set_owner(owner);
++
++ bl.clear();
++ encode(policy, bl);
++
++ obj_ctx.set_atomic(r_obj);
++ ret = store->set_attr(&obj_ctx, bucket_info, r_obj, RGW_ATTR_ACL, bl);
++ if (ret < 0) {
++ ldout(store->ctx(), 0) << "ERROR: modify attr failed " << cpp_strerror(-ret) << dendl;
++ return ret;
++ }
++ }
++ }
++ cerr << count << " objects processed in " << bucket_info.bucket.name
++ << ". Next marker " << list_op.params.marker.name << std::endl;
++ } while(is_truncated);
++ return 0;
+}
+
- RGWMetadataHandler *RGWArchiveBucketInstanceMetaHandlerAllocator::alloc() {
- return new RGWArchiveBucketInstanceMetadataHandler;
+ int RGWBucketCtl::read_bucket_stats(const rgw_bucket& bucket,
+ RGWBucketEnt *result,
+ optional_yield y)
+ {
+ return call([&](RGWSI_Bucket_X_Ctx& ctx) {
+ return svc.bucket->read_bucket_stats(ctx, bucket, result, y);
+ });
}
- void rgw_bucket_init(RGWMetadataManager *mm)
+ int RGWBucketCtl::read_buckets_stats(map<string, RGWBucketEnt>& m,
+ optional_yield y)
{
- auto sync_module = mm->get_store()->get_sync_module();
- if (sync_module) {
- bucket_meta_handler = sync_module->alloc_bucket_meta_handler();
- bucket_instance_meta_handler = sync_module->alloc_bucket_instance_meta_handler();
- } else {
- bucket_meta_handler = RGWBucketMetaHandlerAllocator::alloc();
- bucket_instance_meta_handler = RGWBucketInstanceMetaHandlerAllocator::alloc();
+ return call([&](RGWSI_Bucket_X_Ctx& ctx) {
+ return svc.bucket->read_buckets_stats(ctx, m, y);
+ });
+ }
+
+ int RGWBucketCtl::sync_user_stats(const rgw_user& user_id, const RGWBucketInfo& bucket_info)
+ {
+ RGWBucketEnt ent;
+ int r = svc.bi->read_stats(bucket_info, &ent, null_yield);
+ if (r < 0) {
+ ldout(cct, 20) << __func__ << "(): failed to read bucket stats (r=" << r << ")" << dendl;
+ return r;
}
- mm->register_handler(bucket_meta_handler);
- mm->register_handler(bucket_instance_meta_handler);
+
+ return ctl.user->flush_bucket_stats(user_id, ent);
}
+
+ RGWBucketMetadataHandlerBase *RGWBucketMetaHandlerAllocator::alloc()
+ {
+ return new RGWBucketMetadataHandler();
+ }
+
+ RGWBucketInstanceMetadataHandlerBase *RGWBucketInstanceMetaHandlerAllocator::alloc()
+ {
+ return new RGWBucketInstanceMetadataHandler();
+ }
+
+ RGWBucketMetadataHandlerBase *RGWArchiveBucketMetaHandlerAllocator::alloc()
+ {
+ return new RGWArchiveBucketMetadataHandler();
+ }
+
+ RGWBucketInstanceMetadataHandlerBase *RGWArchiveBucketInstanceMetaHandlerAllocator::alloc()
+ {
+ return new RGWArchiveBucketInstanceMetadataHandler();
+ }
+
RGWAccessHandle handle;
RGWUserInfo user_info;
-- std::string tenant;
-- std::string bucket_name;
++ rgw_bucket bucket;
bool failure;
RGWBucketInfo bucket_info;
++ RGWObjVersionTracker ep_objv; // entrypoint object version
public:
RGWBucket() : store(NULL), handle(NULL), failure(false) {}
- int init(RGWRados *storage, RGWBucketAdminOpState& op_state,
- int init(RGWRados *storage, RGWBucketAdminOpState& op_state);
++ int init(RGWRados *storage, RGWBucketAdminOpState& op_state, optional_yield y,
+ std::string *err_msg = NULL, map<string, bufferlist> *pattrs = NULL);
int check_bad_index_multipart(RGWBucketAdminOpState& op_state,
RGWFormatterFlusher& flusher, std::string *err_msg = NULL);
std::string *err_msg = NULL);
int remove(RGWBucketAdminOpState& op_state, optional_yield y, bool bypass_gc = false, bool keep_index_consistent = true, std::string *err_msg = NULL);
- int link(RGWBucketAdminOpState& op_state, map<string, bufferlist>& attrs,
- std::string *err_msg = NULL);
- int chown(RGWBucketAdminOpState& op_state, const string& marker, std::string *err_msg = NULL);
- int unlink(RGWBucketAdminOpState& op_state, std::string *err_msg = NULL);
- int link(RGWBucketAdminOpState& op_state, optional_yield y, std::string *err_msg = NULL);
++ int link(RGWBucketAdminOpState& op_state, optional_yield y,
++ map<string, bufferlist>& attrs, std::string *err_msg = NULL);
++ int chown(RGWBucketAdminOpState& op_state, const string& marker,
++ optional_yield y, std::string *err_msg = NULL);
+ int unlink(RGWBucketAdminOpState& op_state, optional_yield y, std::string *err_msg = NULL);
int set_quota(RGWBucketAdminOpState& op_state, std::string *err_msg = NULL);
int remove_object(RGWBucketAdminOpState& op_state, std::string *err_msg = NULL);
struct ChangeStatus {
real_time cur_expiration;
real_time cur_sent;
- bool pending;
- RefCountedCond *cond;
- Mutex *lock;
-
- ChangeStatus() : pending(false), cond(NULL) {
- lock = new Mutex("RGWDataChangesLog::ChangeStatus");
- }
-
- ~ChangeStatus() {
- delete lock;
- }
+ bool pending = false;
+ RefCountedCond *cond = nullptr;
+ ceph::mutex lock =
+ ceph::make_mutex("RGWDataChangesLog::ChangeStatus");
-
- ChangeStatus() = default;
- ~ChangeStatus() = default;
};
typedef std::shared_ptr<ChangeStatus> ChangeStatusPtr;
bool going_down();
};
- bool update_entrypoint = true);
++struct rgw_ep_info {
++ RGWBucketEntryPoint &ep;
++ map<string, bufferlist>& attrs;
++ rgw_ep_info(RGWBucketEntryPoint &ep, map<string, bufferlist>& attrs)
++ : ep(ep), attrs(attrs) { }
++};
++
+ class RGWBucketCtl
+ {
+ CephContext *cct;
+
+ struct Svc {
+ RGWSI_Zone *zone{nullptr};
+ RGWSI_Bucket *bucket{nullptr};
+ RGWSI_BucketIndex *bi{nullptr};
+ } svc;
+
+ struct Ctl {
+ RGWUserCtl *user{nullptr};
+ } ctl;
+
+ RGWBucketMetadataHandler *bm_handler;
+ RGWBucketInstanceMetadataHandler *bmi_handler;
+
+ RGWSI_Bucket_BE_Handler bucket_be_handler; /* bucket backend handler */
+ RGWSI_BucketInstance_BE_Handler bi_be_handler; /* bucket instance backend handler */
+
+ int call(std::function<int(RGWSI_Bucket_X_Ctx& ctx)> f);
+
+ public:
+ RGWBucketCtl(RGWSI_Zone *zone_svc,
+ RGWSI_Bucket *bucket_svc,
+ RGWSI_BucketIndex *bi_svc);
+
+ void init(RGWUserCtl *user_ctl,
+ RGWBucketMetadataHandler *_bm_handler,
+ RGWBucketInstanceMetadataHandler *_bmi_handler);
+
+ struct Bucket {
+ struct GetParams {
+ RGWObjVersionTracker *objv_tracker{nullptr};
+ real_time *mtime{nullptr};
+ map<string, bufferlist> *attrs{nullptr};
+ rgw_cache_entry_info *cache_info{nullptr};
+ boost::optional<obj_version> refresh_version;
+ std::optional<RGWSI_MetaBackend_CtxParams> bectx_params;
+
+ GetParams() {}
+
+ GetParams& set_objv_tracker(RGWObjVersionTracker *_objv_tracker) {
+ objv_tracker = _objv_tracker;
+ return *this;
+ }
+
+ GetParams& set_mtime(ceph::real_time *_mtime) {
+ mtime = _mtime;
+ return *this;
+ }
+
+ GetParams& set_attrs(map<string, bufferlist> *_attrs) {
+ attrs = _attrs;
+ return *this;
+ }
+
+ GetParams& set_cache_info(rgw_cache_entry_info *_cache_info) {
+ cache_info = _cache_info;
+ return *this;
+ }
+
+ GetParams& set_refresh_version(const obj_version& _refresh_version) {
+ refresh_version = _refresh_version;
+ return *this;
+ }
+
+ GetParams& set_bectx_params(std::optional<RGWSI_MetaBackend_CtxParams> _bectx_params) {
+ bectx_params = _bectx_params;
+ return *this;
+ }
+ };
+
+ struct PutParams {
+ RGWObjVersionTracker *objv_tracker{nullptr};
+ ceph::real_time mtime;
+ bool exclusive{false};
+ map<string, bufferlist> *attrs{nullptr};
+
+ PutParams() {}
+
+ PutParams& set_objv_tracker(RGWObjVersionTracker *_objv_tracker) {
+ objv_tracker = _objv_tracker;
+ return *this;
+ }
+
+ PutParams& set_mtime(const ceph::real_time& _mtime) {
+ mtime = _mtime;
+ return *this;
+ }
+
+ PutParams& set_exclusive(bool _exclusive) {
+ exclusive = _exclusive;
+ return *this;
+ }
+
+ PutParams& set_attrs(map<string, bufferlist> *_attrs) {
+ attrs = _attrs;
+ return *this;
+ }
+ };
+
+ struct RemoveParams {
+ RGWObjVersionTracker *objv_tracker{nullptr};
+
+ RemoveParams() {}
+
+ RemoveParams& set_objv_tracker(RGWObjVersionTracker *_objv_tracker) {
+ objv_tracker = _objv_tracker;
+ return *this;
+ }
+ };
+ };
+
+ struct BucketInstance {
+ struct GetParams {
+ real_time *mtime{nullptr};
+ map<string, bufferlist> *attrs{nullptr};
+ rgw_cache_entry_info *cache_info{nullptr};
+ boost::optional<obj_version> refresh_version;
+ RGWObjVersionTracker *objv_tracker{nullptr};
+ std::optional<RGWSI_MetaBackend_CtxParams> bectx_params;
+
+ GetParams() {}
+
+ GetParams& set_mtime(ceph::real_time *_mtime) {
+ mtime = _mtime;
+ return *this;
+ }
+
+ GetParams& set_attrs(map<string, bufferlist> *_attrs) {
+ attrs = _attrs;
+ return *this;
+ }
+
+ GetParams& set_cache_info(rgw_cache_entry_info *_cache_info) {
+ cache_info = _cache_info;
+ return *this;
+ }
+
+ GetParams& set_refresh_version(const obj_version& _refresh_version) {
+ refresh_version = _refresh_version;
+ return *this;
+ }
+
+ GetParams& set_objv_tracker(RGWObjVersionTracker *_objv_tracker) {
+ objv_tracker = _objv_tracker;
+ return *this;
+ }
+
+ GetParams& set_bectx_params(std::optional<RGWSI_MetaBackend_CtxParams> _bectx_params) {
+ bectx_params = _bectx_params;
+ return *this;
+ }
+ };
+
+ struct PutParams {
+ std::optional<RGWBucketInfo *> orig_info; /* nullopt: orig_info was not fetched,
+ nullptr: orig_info was not found (new bucket instance */
+ ceph::real_time mtime;
+ bool exclusive{false};
+ map<string, bufferlist> *attrs{nullptr};
+ RGWObjVersionTracker *objv_tracker{nullptr};
+
+ PutParams() {}
+
+ PutParams& set_orig_info(RGWBucketInfo *pinfo) {
+ orig_info = pinfo;
+ return *this;
+ }
+
+ PutParams& set_mtime(const ceph::real_time& _mtime) {
+ mtime = _mtime;
+ return *this;
+ }
+
+ PutParams& set_exclusive(bool _exclusive) {
+ exclusive = _exclusive;
+ return *this;
+ }
+
+ PutParams& set_attrs(map<string, bufferlist> *_attrs) {
+ attrs = _attrs;
+ return *this;
+ }
+
+ PutParams& set_objv_tracker(RGWObjVersionTracker *_objv_tracker) {
+ objv_tracker = _objv_tracker;
+ return *this;
+ }
+ };
+
+ struct RemoveParams {
+ RGWObjVersionTracker *objv_tracker{nullptr};
+
+ RemoveParams() {}
+
+ RemoveParams& set_objv_tracker(RGWObjVersionTracker *_objv_tracker) {
+ objv_tracker = _objv_tracker;
+ return *this;
+ }
+ };
+ };
+
+ /* bucket entrypoint */
+ int read_bucket_entrypoint_info(const rgw_bucket& bucket,
+ RGWBucketEntryPoint *info,
+ optional_yield y,
+ const Bucket::GetParams& params = {});
+ int store_bucket_entrypoint_info(const rgw_bucket& bucket,
+ RGWBucketEntryPoint& info,
+ optional_yield y,
+ const Bucket::PutParams& params = {});
+ int remove_bucket_entrypoint_info(const rgw_bucket& bucket,
+ optional_yield y,
+ const Bucket::RemoveParams& params = {});
+
+ /* bucket instance */
+ int read_bucket_instance_info(const rgw_bucket& bucket,
+ RGWBucketInfo *info,
+ optional_yield y,
+ const BucketInstance::GetParams& params = {});
+ int store_bucket_instance_info(const rgw_bucket& bucket,
+ RGWBucketInfo& info,
+ optional_yield y,
+ const BucketInstance::PutParams& params = {});
+ int remove_bucket_instance_info(const rgw_bucket& bucket,
+ RGWBucketInfo& info,
+ optional_yield y,
+ const BucketInstance::RemoveParams& params = {});
+
+ /*
+ * bucket_id may or may not be provided
+ *
+ * ep_objv_tracker might not be populated even if provided. Will only be set if entrypoint is read
+ * (that is: if bucket_id is empty).
+ */
+ int read_bucket_info(const rgw_bucket& bucket,
+ RGWBucketInfo *info,
+ optional_yield y,
+ const BucketInstance::GetParams& params = {},
+ RGWObjVersionTracker *ep_objv_tracker = nullptr);
+
+
+ int set_bucket_instance_attrs(RGWBucketInfo& bucket_info,
+ map<string, bufferlist>& attrs,
+ RGWObjVersionTracker *objv_tracker,
+ optional_yield y);
+
+ /* user/bucket */
+ int link_bucket(const rgw_user& user_id,
+ const rgw_bucket& bucket,
+ ceph::real_time creation_time,
+ optional_yield y,
- bool update_entrypoint);
++ bool update_entrypoint = true,
++ rgw_ep_info *pinfo = nullptr);
+
+ int unlink_bucket(const rgw_user& user_id,
+ const rgw_bucket& bucket,
+ optional_yield y,
+ bool update_entrypoint = true);
+
++ int chown(RGWRados *store, RGWBucketInfo& bucket_info,
++ const rgw_user& user_id, const std::string& display_name,
++ const std::string& marker, optional_yield y);
++
++ int set_acl(ACLOwner& owner, rgw_bucket& bucket,
++ RGWBucketInfo& bucket_info, bufferlist& bl, optional_yield y);
++
+ int read_buckets_stats(map<string, RGWBucketEnt>& m,
+ optional_yield y);
+
+ int read_bucket_stats(const rgw_bucket& bucket,
+ RGWBucketEnt *result,
+ optional_yield y);
+
+ /* quota related */
+ int sync_user_stats(const rgw_user& user_id, const RGWBucketInfo& bucket_info);
+
+ private:
+ int convert_old_bucket_info(RGWSI_Bucket_X_Ctx& ctx,
+ const rgw_bucket& bucket,
+ optional_yield y);
+
+ int do_store_bucket_instance_info(RGWSI_Bucket_BI_Ctx& ctx,
+ const rgw_bucket& bucket,
+ RGWBucketInfo& info,
+ optional_yield y,
+ const BucketInstance::PutParams& params);
+
+ int do_store_linked_bucket_info(RGWSI_Bucket_X_Ctx& ctx,
+ RGWBucketInfo& info,
+ RGWBucketInfo *orig_info,
+ bool exclusive, real_time mtime,
+ obj_version *pep_objv,
+ map<string, bufferlist> *pattrs,
+ bool create_entry_point,
+ optional_yield);
+
+ int do_link_bucket(RGWSI_Bucket_EP_Ctx& ctx,
+ const rgw_user& user,
+ const rgw_bucket& bucket,
+ ceph::real_time creation_time,
+ optional_yield y,
++ bool update_entrypoint,
++ rgw_ep_info *pinfo);
+
+ int do_unlink_bucket(RGWSI_Bucket_EP_Ctx& ctx,
+ const rgw_user& user_id,
+ const rgw_bucket& bucket,
+ optional_yield y,
+ bool update_entrypoint);
+
+ };
+
#endif
#include "include/types.h"
#include "include/utime.h"
#include "include/ceph_assert.h"
-#include "common/RWLock.h"
+#include "common/ceph_mutex.h"
+ #include "cls/version/cls_version_types.h"
+ #include "rgw_common.h"
+
enum {
UPDATE_OBJ,
REMOVE_OBJ,
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
++#include "include/Context.h"
#include "common/ceph_json.h"
#include "rgw_coroutine.h"
}
};
- lock("RGWRemoteDataLog::lock"), data_sync_cr(NULL),
+ RGWRemoteDataLog::RGWRemoteDataLog(const DoutPrefixProvider *dpp, RGWRados *_store,
+ RGWAsyncRadosProcessor *async_rados)
+ : RGWCoroutinesManager(_store->ctx(), _store->get_cr_registry()),
+ dpp(dpp), store(_store), async_rados(async_rados),
+ http_manager(store->ctx(), completion_mgr),
++ data_sync_cr(NULL),
+ initialized(false)
+ {
+ }
+
int RGWRemoteDataLog::read_log_info(rgw_datalog_info *log_info)
{
rgw_http_param_pair pairs[] = { { "type", "data" },
/* Initialize the registry of auth strategies which will coordinate
* the dynamic reconfiguration. */
auto auth_registry = \
- rgw::auth::StrategyRegistry::create(g_ceph_context, implicit_tenants, store);
- rgw::auth::StrategyRegistry::create(g_ceph_context, store->pctl);
++ rgw::auth::StrategyRegistry::create(g_ceph_context, implicit_tenants, store->pctl);
for (auto frontend : frontends)
frontend->unpause_with_new_config(store, auth_registry);
#ifndef CEPH_RGW_LOG_H
#define CEPH_RGW_LOG_H
+
#include <boost/container/flat_map.hpp>
#include "rgw_common.h"
- #include "common/Formatter.h"
#include "common/OutputDataSocket.h"
-#include "common/Mutex.h"
class RGWRados;
WRITE_CLASS_ENCODER(rgw_log_entry)
class OpsLogSocket : public OutputDataSocket {
- Formatter *formatter;
+ ceph::Formatter *formatter;
- Mutex lock;
+ ceph::mutex lock = ceph::make_mutex("OpsLogSocket");
void formatter_to_bl(bufferlist& bl);
rgw_rest_init(g_ceph_context, store, store->svc.zone->get_zonegroup());
- mutex.Lock();
+ mutex.lock();
init_timer.cancel_all_events();
init_timer.shutdown();
- mutex.Unlock();
+ mutex.unlock();
- rgw_user_init(store);
- rgw_bucket_init(store->meta_mgr);
- rgw_otp_init(store);
rgw_log_usage_init(g_ceph_context, store);
RGWREST rest;
/* Initialize the registry of auth strategies which will coordinate
* the dynamic reconfiguration. */
+ rgw::auth::ImplicitTenants implicit_tenant_context{g_conf()};
+ g_conf().add_observer(&implicit_tenant_context);
auto auth_registry = \
- rgw::auth::StrategyRegistry::create(g_ceph_context, implicit_tenant_context, store);
- rgw::auth::StrategyRegistry::create(g_ceph_context, store->pctl);
++ rgw::auth::StrategyRegistry::create(g_ceph_context, implicit_tenant_context, store->pctl);
/* Header custom behavior */
rest.register_x_headers(g_conf()->rgw_log_http_headers);
}
}
- void RGWObjManifest::obj_iterator::operator++()
- {
- if (manifest->explicit_objs) {
- ++explicit_iter;
-
- update_explicit_pos();
-
- update_location();
- return;
- }
-
- uint64_t obj_size = manifest->get_obj_size();
- uint64_t head_size = manifest->get_head_size();
-
- if (ofs == obj_size) {
- return;
- }
-
- if (manifest->rules.empty()) {
- return;
- }
-
- /* are we still pointing at the head? */
- if (ofs < head_size) {
- rule_iter = manifest->rules.begin();
- RGWObjManifestRule *rule = &rule_iter->second;
- ofs = std::min(head_size, obj_size);
- stripe_ofs = ofs;
- cur_stripe = 1;
- stripe_size = std::min(obj_size - ofs, rule->stripe_max_size);
- if (rule->part_size > 0) {
- stripe_size = std::min(stripe_size, rule->part_size);
- }
- update_location();
- return;
- }
-
- RGWObjManifestRule *rule = &rule_iter->second;
-
- stripe_ofs += rule->stripe_max_size;
- cur_stripe++;
- dout(20) << "RGWObjManifest::operator++(): rule->part_size=" << rule->part_size << " rules.size()=" << manifest->rules.size() << dendl;
-
- if (rule->part_size > 0) {
- /* multi part, multi stripes object */
-
- dout(20) << "RGWObjManifest::operator++(): stripe_ofs=" << stripe_ofs << " part_ofs=" << part_ofs << " rule->part_size=" << rule->part_size << dendl;
-
- if (stripe_ofs >= part_ofs + rule->part_size) {
- /* moved to the next part */
- cur_stripe = 0;
- part_ofs += rule->part_size;
- stripe_ofs = part_ofs;
-
- bool last_rule = (next_rule_iter == manifest->rules.end());
- /* move to the next rule? */
- if (!last_rule && stripe_ofs >= next_rule_iter->second.start_ofs) {
- rule_iter = next_rule_iter;
- last_rule = (next_rule_iter == manifest->rules.end());
- if (!last_rule) {
- ++next_rule_iter;
- }
- cur_part_id = rule_iter->second.start_part_num;
- } else {
- cur_part_id++;
- }
-
- rule = &rule_iter->second;
- }
-
- stripe_size = std::min(rule->part_size - (stripe_ofs - part_ofs), rule->stripe_max_size);
- }
-
- cur_override_prefix = rule->override_prefix;
-
- ofs = stripe_ofs;
- if (ofs > obj_size) {
- ofs = obj_size;
- stripe_ofs = ofs;
- stripe_size = 0;
- }
-
- dout(20) << "RGWObjManifest::operator++(): result: ofs=" << ofs << " stripe_ofs=" << stripe_ofs << " part_ofs=" << part_ofs << " rule->part_size=" << rule->part_size << dendl;
- update_location();
+ RGWObjState::RGWObjState() {
}
- int RGWObjManifest::generator::create_begin(CephContext *cct, RGWObjManifest *_m,
- const rgw_placement_rule& head_placement_rule,
- const rgw_placement_rule *tail_placement_rule,
- const rgw_bucket& _b, const rgw_obj& _obj)
- {
- manifest = _m;
-
- if (!tail_placement_rule) {
- manifest->set_tail_placement(head_placement_rule, _b);
- } else {
- rgw_placement_rule new_tail_rule = *tail_placement_rule;
- new_tail_rule.inherit_from(head_placement_rule);
- manifest->set_tail_placement(new_tail_rule, _b);
- }
-
- manifest->set_head(head_placement_rule, _obj, 0);
- last_ofs = 0;
-
- if (manifest->get_prefix().empty()) {
- char buf[33];
- gen_rand_alphanumeric(cct, buf, sizeof(buf) - 1);
-
- string oid_prefix = ".";
- oid_prefix.append(buf);
- oid_prefix.append("_");
-
- manifest->set_prefix(oid_prefix);
- }
-
- bool found = manifest->get_rule(0, &rule);
- if (!found) {
- derr << "ERROR: manifest->get_rule() could not find rule" << dendl;
- return -EIO;
- }
-
- uint64_t head_size = manifest->get_head_size();
-
- if (head_size > 0) {
- cur_stripe_size = head_size;
- } else {
- cur_stripe_size = rule.stripe_max_size;
- }
-
- cur_part_id = rule.start_part_num;
-
- manifest->get_implicit_location(cur_part_id, cur_stripe, 0, NULL, &cur_obj);
-
- // Normal object which not generated through copy operation
- manifest->set_tail_instance(_obj.key.instance);
-
- manifest->update_iterators();
-
- return 0;
+ RGWObjState::~RGWObjState() {
}
- int RGWObjManifest::generator::create_next(uint64_t ofs)
- {
- if (ofs < last_ofs) /* only going forward */
- return -EINVAL;
-
- uint64_t max_head_size = manifest->get_max_head_size();
-
- if (ofs < max_head_size) {
- manifest->set_head_size(ofs);
+ RGWObjState::RGWObjState(const RGWObjState& rhs) : obj (rhs.obj) {
+ is_atomic = rhs.is_atomic;
+ has_attrs = rhs.has_attrs;
+ exists = rhs.exists;
+ size = rhs.size;
+ accounted_size = rhs.accounted_size;
+ mtime = rhs.mtime;
+ epoch = rhs.epoch;
+ if (rhs.obj_tag.length()) {
+ obj_tag = rhs.obj_tag;
}
-
- if (ofs >= max_head_size) {
- manifest->set_head_size(max_head_size);
- cur_stripe = (ofs - max_head_size) / rule.stripe_max_size;
- cur_stripe_size = rule.stripe_max_size;
-
- if (cur_part_id == 0 && max_head_size > 0) {
- cur_stripe++;
- }
+ if (rhs.tail_tag.length()) {
+ tail_tag = rhs.tail_tag;
}
-
- last_ofs = ofs;
- manifest->set_obj_size(ofs);
-
- manifest->get_implicit_location(cur_part_id, cur_stripe, ofs, NULL, &cur_obj);
-
- manifest->update_iterators();
-
- return 0;
- }
-
- const RGWObjManifest::obj_iterator& RGWObjManifest::obj_begin()
- {
- return begin_iter;
- }
-
- const RGWObjManifest::obj_iterator& RGWObjManifest::obj_end()
- {
- return end_iter;
- }
-
- RGWObjManifest::obj_iterator RGWObjManifest::obj_find(uint64_t ofs)
- {
- if (ofs > obj_size) {
- ofs = obj_size;
+ write_tag = rhs.write_tag;
+ fake_tag = rhs.fake_tag;
+ manifest = rhs.manifest;
+ shadow_obj = rhs.shadow_obj;
+ has_data = rhs.has_data;
+ if (rhs.data.length()) {
+ data = rhs.data;
}
- RGWObjManifest::obj_iterator iter(this);
- iter.seek(ofs);
- return iter;
+ prefetch_data = rhs.prefetch_data;
+ keep_tail = rhs.keep_tail;
+ is_olh = rhs.is_olh;
+ objv_tracker = rhs.objv_tracker;
+ pg_ver = rhs.pg_ver;
}
- int RGWObjManifest::append(RGWObjManifest& m, const RGWZoneGroup& zonegroup,
- const RGWZoneParams& zone_params)
- {
- if (explicit_objs || m.explicit_objs) {
- return append_explicit(m, zonegroup, zone_params);
- }
-
- if (rules.empty()) {
- *this = m;
- return 0;
- }
-
- string override_prefix;
-
- if (prefix.empty()) {
- prefix = m.prefix;
- }
-
- if (prefix != m.prefix) {
- override_prefix = m.prefix;
- }
-
- map<uint64_t, RGWObjManifestRule>::iterator miter = m.rules.begin();
- if (miter == m.rules.end()) {
- return append_explicit(m, zonegroup, zone_params);
- }
-
- for (; miter != m.rules.end(); ++miter) {
- map<uint64_t, RGWObjManifestRule>::reverse_iterator last_rule = rules.rbegin();
-
- RGWObjManifestRule& rule = last_rule->second;
-
- if (rule.part_size == 0) {
- rule.part_size = obj_size - rule.start_ofs;
- }
-
- RGWObjManifestRule& next_rule = miter->second;
- if (!next_rule.part_size) {
- next_rule.part_size = m.obj_size - next_rule.start_ofs;
- }
-
- string rule_prefix = prefix;
- if (!rule.override_prefix.empty()) {
- rule_prefix = rule.override_prefix;
- }
-
- string next_rule_prefix = m.prefix;
- if (!next_rule.override_prefix.empty()) {
- next_rule_prefix = next_rule.override_prefix;
- }
-
- if (rule.part_size != next_rule.part_size ||
- rule.stripe_max_size != next_rule.stripe_max_size ||
- rule_prefix != next_rule_prefix) {
- if (next_rule_prefix != prefix) {
- append_rules(m, miter, &next_rule_prefix);
- } else {
- append_rules(m, miter, NULL);
- }
- break;
- }
-
- uint64_t expected_part_num = rule.start_part_num + 1;
- if (rule.part_size > 0) {
- expected_part_num = rule.start_part_num + (obj_size + next_rule.start_ofs - rule.start_ofs) / rule.part_size;
- }
-
- if (expected_part_num != next_rule.start_part_num) {
- append_rules(m, miter, NULL);
- break;
- }
+ RGWObjState *RGWObjectCtx::get_state(const rgw_obj& obj) {
+ RGWObjState *result;
+ typename std::map<rgw_obj, RGWObjState>::iterator iter;
- lock.get_read();
++ lock.lock_shared();
+ assert (!obj.empty());
+ iter = objs_state.find(obj);
+ if (iter != objs_state.end()) {
+ result = &iter->second;
- lock.unlock();
++ lock.unlock_shared();
+ } else {
- lock.unlock();
- lock.get_write();
++ lock.unlock_shared();
++ lock.lock();
+ result = &objs_state[obj];
+ lock.unlock();
}
-
- set_obj_size(obj_size + m.obj_size);
-
- return 0;
+ return result;
}
- int RGWObjManifest::append(RGWObjManifest& m, RGWSI_Zone *zone_svc)
- {
- return append(m, zone_svc->get_zonegroup(), zone_svc->get_zone_params());
+ void RGWObjectCtx::set_atomic(rgw_obj& obj) {
- RWLock::WLocker wl(lock);
++ std::unique_lock wl{lock};
+ assert (!obj.empty());
+ objs_state[obj].is_atomic = true;
}
--
- void RGWObjManifest::append_rules(RGWObjManifest& m, map<uint64_t, RGWObjManifestRule>::iterator& miter,
- string *override_prefix)
- {
- for (; miter != m.rules.end(); ++miter) {
- RGWObjManifestRule rule = miter->second;
- rule.start_ofs += obj_size;
- if (override_prefix)
- rule.override_prefix = *override_prefix;
- rules[rule.start_ofs] = rule;
- }
+ void RGWObjectCtx::set_prefetch_data(const rgw_obj& obj) {
- RWLock::WLocker wl(lock);
++ std::unique_lock wl{lock};
+ assert (!obj.empty());
+ objs_state[obj].prefetch_data = true;
}
- void RGWObjManifest::convert_to_explicit(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params)
- {
- if (explicit_objs) {
+ void RGWObjectCtx::invalidate(const rgw_obj& obj) {
- RWLock::WLocker wl(lock);
++ std::unique_lock wl{lock};
+ auto iter = objs_state.find(obj);
+ if (iter == objs_state.end()) {
return;
}
- obj_iterator iter = obj_begin();
-
- while (iter != obj_end()) {
- RGWObjManifestPart& part = objs[iter.get_stripe_ofs()];
- const rgw_obj_select& os = iter.get_location();
- const rgw_raw_obj& raw_loc = os.get_raw_obj(zonegroup, zone_params);
- part.loc_ofs = 0;
-
- uint64_t ofs = iter.get_stripe_ofs();
-
- if (ofs == 0) {
- part.loc = obj;
- } else {
- rgw_raw_obj_to_obj(tail_placement.bucket, raw_loc, &part.loc);
- }
- ++iter;
- uint64_t next_ofs = iter.get_stripe_ofs();
-
- part.size = next_ofs - ofs;
- }
+ bool is_atomic = iter->second.is_atomic;
+ bool prefetch_data = iter->second.prefetch_data;
- explicit_objs = true;
- rules.clear();
- prefix.clear();
- }
-
- int RGWObjManifest::append_explicit(RGWObjManifest& m, const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params)
- {
- if (!explicit_objs) {
- convert_to_explicit(zonegroup, zone_params);
- }
- if (!m.explicit_objs) {
- m.convert_to_explicit(zonegroup, zone_params);
- }
- map<uint64_t, RGWObjManifestPart>::iterator iter;
- uint64_t base = obj_size;
- for (iter = m.objs.begin(); iter != m.objs.end(); ++iter) {
- RGWObjManifestPart& part = iter->second;
- objs[base + iter->first] = part;
- }
- obj_size += m.obj_size;
-
- return 0;
- }
-
- bool RGWObjManifest::get_rule(uint64_t ofs, RGWObjManifestRule *rule)
- {
- if (rules.empty()) {
- return false;
- }
+ objs_state.erase(iter);
- map<uint64_t, RGWObjManifestRule>::iterator iter = rules.upper_bound(ofs);
- if (iter != rules.begin()) {
- --iter;
+ if (is_atomic || prefetch_data) {
+ auto& state = objs_state[obj];
+ state.is_atomic = is_atomic;
+ state.prefetch_data = prefetch_data;
}
-
- *rule = iter->second;
-
- return true;
}
void RGWObjVersionTracker::generate_new_write_ver(CephContext *cct)
void RGWRados::finalize()
{
- cct->get_admin_socket()->unregister_commands(this);
if (run_sync_thread) {
- Mutex::Locker l(meta_sync_thread_lock);
+ std::lock_guard l{meta_sync_thread_lock};
meta_sync_processor_thread->stop();
- Mutex::Locker dl(data_sync_thread_lock);
+ std::lock_guard dl{data_sync_thread_lock};
for (auto iter : data_sync_processor_threads) {
RGWDataSyncProcessorThread *thread = iter.second;
thread->stop();
<< pt.second.name << " present in zonegroup" << dendl;
}
}
- Mutex::Locker l(meta_sync_thread_lock);
+ auto async_processor = svc.rados->get_async_processor();
- meta_sync_processor_thread = new RGWMetaSyncProcessorThread(this, async_rados);
+ std::lock_guard l{meta_sync_thread_lock};
+ meta_sync_processor_thread = new RGWMetaSyncProcessorThread(this, async_processor);
ret = meta_sync_processor_thread->init();
if (ret < 0) {
ldout(cct, 0) << "ERROR: failed to initialize meta sync thread" << dendl;
ldout(cct, 0) << "ERROR: failed to start bucket trim manager" << dendl;
return ret;
}
- data_log->set_observer(&*bucket_trim);
+ svc.datalog_rados->set_observer(&*bucket_trim);
- Mutex::Locker dl(data_sync_thread_lock);
+ std::lock_guard dl{data_sync_thread_lock};
for (auto source_zone : svc.zone->get_data_sync_source_zones()) {
ldout(cct, 5) << "starting data sync thread for zone " << source_zone->name << dendl;
- auto *thread = new RGWDataSyncProcessorThread(this, async_rados, source_zone);
+ auto *thread = new RGWDataSyncProcessorThread(this, svc.rados->get_async_processor(), source_zone);
ret = thread->init();
if (ret < 0) {
ldout(cct, 0) << "ERROR: failed to initialize data sync thread" << dendl;
return ret;
}
- int RGWRados::key_to_shard_id(const string& key, int max_shards)
- {
- return rgw_shard_id(key, max_shards);
- }
-
- void RGWRados::shard_name(const string& prefix, unsigned max_shards, const string& key, string& name, int *shard_id)
+ int RGWRados::decode_policy(bufferlist& bl, ACLOwner *owner)
{
- uint32_t val = ceph_str_hash_linux(key.c_str(), key.size());
- char buf[16];
- if (shard_id) {
- *shard_id = val % max_shards;
+ auto i = bl.cbegin();
+ RGWAccessControlPolicy policy(cct);
+ try {
+ policy.decode_owner(i);
+ } catch (buffer::error& err) {
+ ldout(cct, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl;
+ return -EIO;
}
- snprintf(buf, sizeof(buf), "%u", (unsigned)(val % max_shards));
- name = prefix + buf;
- }
-
- void RGWRados::shard_name(const string& prefix, unsigned max_shards, const string& section, const string& key, string& name)
- {
- uint32_t val = ceph_str_hash_linux(key.c_str(), key.size());
- val ^= ceph_str_hash_linux(section.c_str(), section.size());
- char buf[16];
- snprintf(buf, sizeof(buf), "%u", (unsigned)(val % max_shards));
- name = prefix + buf;
+ *owner = policy.get_owner();
+ return 0;
}
- void RGWRados::shard_name(const string& prefix, unsigned shard_id, string& name)
+ int rgw_policy_from_attrset(CephContext *cct, map<string, bufferlist>& attrset, RGWAccessControlPolicy *policy)
{
- char buf[16];
- snprintf(buf, sizeof(buf), "%u", shard_id);
- name = prefix + buf;
+ map<string, bufferlist>::iterator aiter = attrset.find(RGW_ATTR_ACL);
+ if (aiter == attrset.end())
+ return -EIO;
+ bufferlist& bl = aiter->second;
+ auto iter = bl.cbegin();
+ try {
+ policy->decode(iter);
+ } catch (buffer::error& err) {
+ ldout(cct, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl;
+ return -EIO;
+ }
+ if (cct->_conf->subsys.should_gather<ceph_subsys_rgw, 15>()) {
+ RGWAccessControlPolicy_S3 *s3policy = static_cast<RGWAccessControlPolicy_S3 *>(policy);
+ ldout(cct, 15) << __func__ << " Read AccessControlPolicy";
+ s3policy->to_xml(*_dout);
+ *_dout << dendl;
+ }
+ return 0;
}
- void RGWRados::time_log_prepare_entry(cls_log_entry& entry, const real_time& ut, const string& section, const string& key, bufferlist& bl)
- {
- cls_log_add_prepare_entry(entry, utime_t(ut), section, key, bl);
- }
- int RGWRados::time_log_add_init(librados::IoCtx& io_ctx)
+ int RGWRados::Bucket::update_bucket_id(const string& new_bucket_id)
{
- return rgw_init_ioctx(get_rados_handle(), svc.zone->get_zone_params().log_pool, io_ctx, true);
-
- }
+ rgw_bucket bucket = bucket_info.bucket;
+ bucket.update_bucket_id(new_bucket_id);
- int RGWRados::time_log_add(const string& oid, const real_time& ut, const string& section, const string& key, bufferlist& bl)
- {
- librados::IoCtx io_ctx;
+ auto obj_ctx = store->svc.sysobj->init_obj_ctx();
- int r = time_log_add_init(io_ctx);
- if (r < 0) {
- return r;
+ bucket_info.objv_tracker.clear();
+ int ret = store->get_bucket_instance_info(obj_ctx, bucket, bucket_info, nullptr, nullptr, null_yield);
+ if (ret < 0) {
+ return ret;
}
- ObjectWriteOperation op;
- utime_t t(ut);
- cls_log_add(op, t, section, key, bl);
-
- return io_ctx.operate(oid, &op);
+ return 0;
}
- int RGWRados::time_log_add(const string& oid, list<cls_log_entry>& entries,
- librados::AioCompletion *completion, bool monotonic_inc)
+
++static inline std::string after_delim(std::string_view delim)
+{
- librados::IoCtx io_ctx;
-
- int r = time_log_add_init(io_ctx);
- if (r < 0) {
- return r;
- }
-
- ObjectWriteOperation op;
- cls_log_add(op, entries, monotonic_inc);
-
- if (!completion) {
- r = io_ctx.operate(oid, &op);
- } else {
- r = io_ctx.aio_operate(oid, completion, &op);
- }
- return r;
- }
-
- int RGWRados::time_log_list(const string& oid, const real_time& start_time, const real_time& end_time,
- int max_entries, list<cls_log_entry>& entries,
- const string& marker,
- string *out_marker,
- bool *truncated)
- {
- librados::IoCtx io_ctx;
-
- int r = rgw_init_ioctx(get_rados_handle(), svc.zone->get_zone_params().log_pool, io_ctx);
- if (r < 0)
- return r;
- librados::ObjectReadOperation op;
-
- utime_t st(start_time);
- utime_t et(end_time);
-
- cls_log_list(op, st, et, marker, max_entries, entries,
- out_marker, truncated);
-
- bufferlist obl;
-
- int ret = io_ctx.operate(oid, &op, &obl);
- if (ret < 0)
- return ret;
-
- return 0;
- }
-
- int RGWRados::time_log_info(const string& oid, cls_log_header *header)
- {
- librados::IoCtx io_ctx;
-
- int r = rgw_init_ioctx(get_rados_handle(), svc.zone->get_zone_params().log_pool, io_ctx);
- if (r < 0)
- return r;
- librados::ObjectReadOperation op;
-
- cls_log_info(op, header);
-
- bufferlist obl;
-
- int ret = io_ctx.operate(oid, &op, &obl);
- if (ret < 0)
- return ret;
-
- return 0;
- }
-
- int RGWRados::time_log_info_async(librados::IoCtx& io_ctx, const string& oid, cls_log_header *header, librados::AioCompletion *completion)
- {
- int r = rgw_init_ioctx(get_rados_handle(), svc.zone->get_zone_params().log_pool, io_ctx);
- if (r < 0)
- return r;
-
- librados::ObjectReadOperation op;
-
- cls_log_info(op, header);
-
- int ret = io_ctx.aio_operate(oid, completion, &op, NULL);
- if (ret < 0)
- return ret;
-
- return 0;
- }
-
- int RGWRados::time_log_trim(const string& oid, const real_time& start_time, const real_time& end_time,
- const string& from_marker, const string& to_marker,
- librados::AioCompletion *completion)
- {
- librados::IoCtx io_ctx;
-
- int r = rgw_init_ioctx(get_rados_handle(), svc.zone->get_zone_params().log_pool, io_ctx);
- if (r < 0)
- return r;
-
- utime_t st(start_time);
- utime_t et(end_time);
-
- ObjectWriteOperation op;
- cls_log_trim(op, st, et, from_marker, to_marker);
-
- if (!completion) {
- r = io_ctx.operate(oid, &op);
- } else {
- r = io_ctx.aio_operate(oid, completion, &op);
- }
- return r;
- }
-
- string RGWRados::objexp_hint_get_shardname(int shard_num)
- {
- char buf[32];
- snprintf(buf, sizeof(buf), "%010u", (unsigned)shard_num);
-
- string objname("obj_delete_at_hint.");
- return objname + buf;
- }
-
- int RGWRados::objexp_key_shard(const rgw_obj_index_key& key)
- {
- string obj_key = key.name + key.instance;
- int num_shards = cct->_conf->rgw_objexp_hints_num_shards;
- return rgw_bucket_shard_index(obj_key, num_shards);
- }
-
- static string objexp_hint_get_keyext(const string& tenant_name,
- const string& bucket_name,
- const string& bucket_id,
- const rgw_obj_key& obj_key)
- {
- return tenant_name + (tenant_name.empty() ? "" : ":") + bucket_name + ":" + bucket_id +
- ":" + obj_key.name + ":" + obj_key.instance;
- }
-
- int RGWRados::objexp_hint_add(const ceph::real_time& delete_at,
- const string& tenant_name,
- const string& bucket_name,
- const string& bucket_id,
- const rgw_obj_index_key& obj_key)
- {
- const string keyext = objexp_hint_get_keyext(tenant_name, bucket_name,
- bucket_id, obj_key);
- objexp_hint_entry he = {
- .tenant = tenant_name,
- .bucket_name = bucket_name,
- .bucket_id = bucket_id,
- .obj_key = obj_key,
- .exp_time = delete_at };
- bufferlist hebl;
- encode(he, hebl);
- ObjectWriteOperation op;
- cls_timeindex_add(op, utime_t(delete_at), keyext, hebl);
-
- string shard_name = objexp_hint_get_shardname(objexp_key_shard(obj_key));
- return objexp_pool_ctx.operate(shard_name, &op);
- }
-
- void RGWRados::objexp_get_shard(int shard_num,
- string& shard) /* out */
- {
- shard = objexp_hint_get_shardname(shard_num);
- }
-
- int RGWRados::objexp_hint_list(const string& oid,
- const ceph::real_time& start_time,
- const ceph::real_time& end_time,
- const int max_entries,
- const string& marker,
- list<cls_timeindex_entry>& entries, /* out */
- string *out_marker, /* out */
- bool *truncated) /* out */
- {
- librados::ObjectReadOperation op;
- cls_timeindex_list(op, utime_t(start_time), utime_t(end_time), marker, max_entries, entries,
- out_marker, truncated);
-
- bufferlist obl;
- int ret = objexp_pool_ctx.operate(oid, &op, &obl);
-
- if ((ret < 0 ) && (ret != -ENOENT)) {
- return ret;
- }
-
- if ((ret == -ENOENT) && truncated) {
- *truncated = false;
- }
-
- return 0;
- }
-
- int RGWRados::objexp_hint_parse(cls_timeindex_entry &ti_entry, /* in */
- objexp_hint_entry& hint_entry) /* out */
- {
- try {
- auto iter = ti_entry.value.cbegin();
- decode(hint_entry, iter);
- } catch (buffer::error& err) {
- ldout(cct, 0) << "ERROR: couldn't decode avail_pools" << dendl;
- }
-
- return 0;
- }
-
- int RGWRados::objexp_hint_trim(const string& oid,
- const ceph::real_time& start_time,
- const ceph::real_time& end_time,
- const string& from_marker,
- const string& to_marker)
- {
- int ret = cls_timeindex_trim(objexp_pool_ctx, oid, utime_t(start_time), utime_t(end_time),
- from_marker, to_marker);
- if ((ret < 0 ) && (ret != -ENOENT)) {
- return ret;
- }
-
- return 0;
- }
-
- int RGWRados::lock_exclusive(const rgw_pool& pool, const string& oid, timespan& duration,
- string& zone_id, string& owner_id) {
- librados::IoCtx io_ctx;
-
- int r = rgw_init_ioctx(get_rados_handle(), pool, io_ctx);
- if (r < 0) {
- return r;
- }
- uint64_t msec = std::chrono::duration_cast<std::chrono::milliseconds>(duration).count();
- utime_t ut(msec / 1000, msec % 1000);
-
- rados::cls::lock::Lock l(log_lock_name);
- l.set_duration(ut);
- l.set_cookie(owner_id);
- l.set_tag(zone_id);
- l.set_may_renew(true);
-
- return l.lock_exclusive(&io_ctx, oid);
- }
-
- int RGWRados::unlock(const rgw_pool& pool, const string& oid, string& zone_id, string& owner_id) {
- librados::IoCtx io_ctx;
-
- int r = rgw_init_ioctx(get_rados_handle(), pool, io_ctx);
- if (r < 0) {
- return r;
- }
-
- rados::cls::lock::Lock l(log_lock_name);
- l.set_tag(zone_id);
- l.set_cookie(owner_id);
-
- return l.unlock(&io_ctx, oid);
- }
-
- int RGWRados::decode_policy(bufferlist& bl, ACLOwner *owner)
- {
- auto i = bl.cbegin();
- RGWAccessControlPolicy policy(cct);
- try {
- policy.decode_owner(i);
- } catch (buffer::error& err) {
- ldout(cct, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl;
- return -EIO;
- }
- *owner = policy.get_owner();
- return 0;
- }
-
- int rgw_policy_from_attrset(CephContext *cct, map<string, bufferlist>& attrset, RGWAccessControlPolicy *policy)
- {
- map<string, bufferlist>::iterator aiter = attrset.find(RGW_ATTR_ACL);
- if (aiter == attrset.end())
- return -EIO;
-
- bufferlist& bl = aiter->second;
- auto iter = bl.cbegin();
- try {
- policy->decode(iter);
- } catch (buffer::error& err) {
- ldout(cct, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl;
- return -EIO;
- }
- if (cct->_conf->subsys.should_gather<ceph_subsys_rgw, 15>()) {
- RGWAccessControlPolicy_S3 *s3policy = static_cast<RGWAccessControlPolicy_S3 *>(policy);
- ldout(cct, 15) << __func__ << " Read AccessControlPolicy";
- s3policy->to_xml(*_dout);
- *_dout << dendl;
- }
- return 0;
- }
-
-
- int RGWRados::Bucket::update_bucket_id(const string& new_bucket_id)
- {
- rgw_bucket bucket = bucket_info.bucket;
- bucket.update_bucket_id(new_bucket_id);
-
- auto obj_ctx = store->svc.sysobj->init_obj_ctx();
-
- bucket_info.objv_tracker.clear();
- int ret = store->get_bucket_instance_info(obj_ctx, bucket, bucket_info, nullptr, nullptr, null_yield);
- if (ret < 0) {
- return ret;
- }
-
- return 0;
- }
-
-
- static inline std::string after_delim(std::string_view delim)
- {
- // assert: ! delim.empty()
- std::string result{delim.data(), delim.length()};
- result += char(255);
- return result;
- }
++ // assert: ! delim.empty()
++ std::string result{delim.data(), delim.length()};
++ result += char(255);
++ return result;
++}
+
+
/**
* Get ordered listing of the objects in a bucket.
*
}
};
- RWLock lock{"RGWObjectCtx"};
+ class RGWObjectCtx {
+ RGWRados *store;
++ ceph::shared_mutex lock = ceph::make_shared_mutex("RGWObjectCtx");
+ void *s{nullptr};
+
+ std::map<rgw_obj, RGWObjState> objs_state;
+ public:
+ explicit RGWObjectCtx(RGWRados *_store) : store(_store) {}
+ explicit RGWObjectCtx(RGWRados *_store, void *_s) : store(_store), s(_s) {}
+
+ void *get_private() {
+ return s;
+ }
+
+ RGWRados *get_store() {
+ return store;
+ }
+
+ RGWObjState *get_state(const rgw_obj& obj);
+
+ void set_atomic(rgw_obj& obj);
+ void set_prefetch_data(const rgw_obj& obj);
+ void invalidate(const rgw_obj& obj);
+ };
+
+
struct RGWRawObjState {
rgw_raw_obj obj;
bool has_attrs{false};
int open_pool_ctx(const rgw_pool& pool, librados::IoCtx& io_ctx,
bool mostly_omap);
- int open_bucket_index_ctx(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx);
- int open_bucket_index(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx, string& bucket_oid);
- int open_bucket_index_base(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx,
- string& bucket_oid_base);
- int open_bucket_index_shard(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx,
- const string& obj_key, string *bucket_obj, int *shard_id);
- int open_bucket_index_shard(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx,
- int shard_id, string *bucket_obj);
- int open_bucket_index(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx,
- map<int, string>& bucket_objs, int shard_id = -1, map<int, string> *bucket_instance_ids = NULL);
- template<typename T>
- int open_bucket_index(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx,
- map<int, string>& oids, map<int, T>& bucket_objs,
- int shard_id = -1, map<int, string> *bucket_instance_ids = NULL);
- void build_bucket_index_marker(const string& shard_id_str, const string& shard_marker,
- string *marker);
-
- void get_bucket_instance_ids(const RGWBucketInfo& bucket_info, int shard_id, map<int, string> *result);
std::atomic<int64_t> max_req_id = { 0 };
- Mutex lock;
+ ceph::mutex lock = ceph::make_mutex("rados_timer_lock");
SafeTimer *timer;
RGWGC *gc;
bool use_cache{false};
public:
- RGWRados(): lock("rados_timer_lock"), timer(NULL),
+ RGWRados(): timer(NULL),
gc(NULL), lc(NULL), obj_expirer(NULL), use_gc_thread(false), use_lc_thread(false), quota_threads(false),
- run_sync_thread(false), run_reshard_thread(false), async_rados(nullptr), meta_notifier(NULL),
+ run_sync_thread(false), run_reshard_thread(false), meta_notifier(NULL),
data_notifier(NULL), meta_sync_processor_thread(NULL),
- meta_sync_thread_lock("meta_sync_thread_lock"), data_sync_thread_lock("data_sync_thread_lock"),
- bucket_id_lock("rados_bucket_id"),
bucket_index_max_shards(0),
max_bucket_id(0), cct(NULL),
binfo_cache(NULL), obj_tombstone_cache(nullptr),
#include <functional>
#include <boost/intrusive/list.hpp>
+ #include <boost/asio/basic_waitable_timer.hpp>
#include "include/rados/librados.hpp"
-#include "common/Mutex.h"
-#include "common/Cond.h"
#include "common/ceph_time.h"
+ #include "common/async/yield_context.h"
#include "cls/rgw/cls_rgw_types.h"
#include "cls/lock/cls_lock_client.h"
- #include "rgw_bucket.h"
+
+ #include "rgw_common.h"
class CephContext;
class DefaultStrategy : public rgw::auth::Strategy,
public rgw::auth::TokenExtractor,
public rgw::auth::WebIdentityApplier::Factory {
- RGWRados* const store;
+ RGWCtl* const ctl;
+ ImplicitTenants& implicit_tenant_context;
/* The engine. */
const WebTokenEngine web_token_engine;
public:
DefaultStrategy(CephContext* const cct,
- RGWRados* const store)
- : store(store),
+ ImplicitTenants& implicit_tenant_context,
+ RGWCtl* const ctl)
+ : ctl(ctl),
+ implicit_tenant_context(implicit_tenant_context),
web_token_engine(cct,
static_cast<rgw::auth::TokenExtractor*>(this),
static_cast<rgw::auth::WebIdentityApplier::Factory*>(this)) {
public rgw::auth::RemoteApplier::Factory,
public rgw::auth::LocalApplier::Factory,
public rgw::auth::swift::TempURLApplier::Factory {
- RGWRados* const store;
+ RGWCtl* const ctl;
+ ImplicitTenants& implicit_tenant_context;
/* The engines. */
const rgw::auth::swift::TempURLEngine tempurl_engine;
acl_strategy_t&& extra_acl_strategy,
const rgw::auth::RemoteApplier::AuthInfo &info) const override {
auto apl = \
- rgw::auth::add_3rdparty(store, s->account_name,
- rgw::auth::add_sysreq(cct, store, s,
- rgw::auth::RemoteApplier(cct, store, std::move(extra_acl_strategy), info,
+ rgw::auth::add_3rdparty(ctl, s->account_name,
+ rgw::auth::add_sysreq(cct, ctl, s,
+ rgw::auth::RemoteApplier(cct, ctl, std::move(extra_acl_strategy), info,
- cct->_conf->rgw_keystone_implicit_tenants)));
+ implicit_tenant_context,
+ rgw::auth::ImplicitTenants::IMPLICIT_TENANTS_SWIFT)));
/* TODO(rzarzynski): replace with static_ptr. */
return aplptr_t(new decltype(apl)(std::move(apl)));
}
public:
DefaultStrategy(CephContext* const cct,
- RGWRados* const store)
- : store(store),
+ ImplicitTenants& implicit_tenant_context,
+ RGWCtl* const ctl)
+ : ctl(ctl),
+ implicit_tenant_context(implicit_tenant_context),
tempurl_engine(cct,
- store,
+ ctl,
static_cast<rgw::auth::swift::TempURLApplier::Factory*>(this)),
signed_engine(cct,
- store,
+ ctl,
static_cast<rgw::auth::TokenExtractor*>(this),
static_cast<rgw::auth::LocalApplier::Factory*>(this)),
external_engine(cct,
return 0;
}
- ret = rgw_store_user_info(store, stub_user_info, nullptr, &objv, real_time(), exclusive);
+// update swift_keys with new user id
+static void rename_swift_keys(const rgw_user& user,
+ std::map<std::string, RGWAccessKey>& keys)
+{
+ std::string user_id;
+ user.to_str(user_id);
+
+ auto modify_keys = std::move(keys);
+ for (auto& [k, key] : modify_keys) {
+ std::string id = user_id + ":" + key.subuser;
+ key.id = id;
+ keys[id] = std::move(key);
+ }
+}
+
+int RGWUser::execute_rename(RGWUserAdminOpState& op_state, std::string *err_msg)
+{
+ int ret;
+ bool populated = op_state.is_populated();
+
+ if (!op_state.has_existing_user() && !populated) {
+ set_err_msg(err_msg, "user not found");
+ return -ENOENT;
+ }
+
+ if (!populated) {
+ ret = init(op_state);
+ if (ret < 0) {
+ set_err_msg(err_msg, "unable to retrieve user info");
+ return ret;
+ }
+ }
+
+ rgw_user& old_uid = op_state.get_user_id();
+ RGWUserInfo old_user_info = op_state.get_user_info();
+
+ rgw_user& uid = op_state.get_new_uid();
+ if (old_uid.tenant != uid.tenant) {
+ set_err_msg(err_msg, "users have to be under the same tenant namespace "
+ + old_uid.tenant + " != " + uid.tenant);
+ return -EINVAL;
+ }
+
+ // create a stub user and write only the uid index and buckets object
+ RGWUserInfo stub_user_info;
+ stub_user_info.user_id = uid;
+
+ RGWObjVersionTracker objv;
+ const bool exclusive = !op_state.get_overwrite_new_user(); // overwrite if requested
+
- int ret = rgw_read_user_buckets(store, old_uid, buckets, marker, string(),
- max_buckets, false, &is_truncated);
++ ret = user_ctl->store_info(stub_user_info, null_yield,
++ RGWUserCtl::PutParams()
++ .set_objv_tracker(&objv)
++ .set_exclusive(exclusive));
+ if (ret == -EEXIST) {
+ set_err_msg(err_msg, "user name given by --new-uid already exists");
+ return ret;
+ }
+ if (ret < 0) {
+ set_err_msg(err_msg, "unable to store new user info");
+ return ret;
+ }
+
+ ACLOwner owner;
+ RGWAccessControlPolicy policy_instance;
+ policy_instance.create_default(uid, old_user_info.display_name);
+ owner = policy_instance.get_owner();
+ bufferlist aclbl;
+ policy_instance.encode(aclbl);
+
+ //unlink and link buckets to new user
+ bool is_truncated = false;
+ string marker;
+ string obj_marker;
+ CephContext *cct = store->ctx();
+ size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk;
++ RGWBucketCtl* bucket_ctl = store->ctl.bucket;
+
+ do {
+ RGWUserBuckets buckets;
- set_err_msg(err_msg, "unable to read bucket info of user");
++ ret = user_ctl->list_buckets(old_uid, marker, "", max_buckets,
++ false, &buckets, &is_truncated);
+ if (ret < 0) {
- RGWSysObjectCtx sys_ctx = store->svc.sysobj->init_obj_ctx();
-
- ret = store->get_bucket_info(sys_ctx, obj.bucket.tenant, obj.bucket.name,
- bucket_info, NULL, null_yield, &attrs);
++ set_err_msg(err_msg, "unable to list user buckets");
+ return ret;
+ }
+
+ map<string, bufferlist> attrs;
+ map<std::string, RGWBucketEnt>& m = buckets.get_buckets();
+ std::map<std::string, RGWBucketEnt>::iterator it;
+
+ for (it = m.begin(); it != m.end(); ++it) {
+ RGWBucketEnt obj = it->second;
+ marker = it->first;
+
+ RGWBucketInfo bucket_info;
- ret = rgw_set_bucket_acl(store, owner, obj.bucket, bucket_info, aclbl);
++ ret = bucket_ctl->read_bucket_info(obj.bucket, &bucket_info, null_yield,
++ RGWBucketCtl::BucketInstance::GetParams()
++ .set_attrs(&attrs));
+ if (ret < 0) {
+ set_err_msg(err_msg, "failed to fetch bucket info for bucket=" + obj.bucket.name);
+ return ret;
+ }
+
- ret = rgw_link_bucket(store, uid, bucket_info.bucket,
- ceph::real_time(), true, &ep_data);
++ ret = bucket_ctl->set_acl(owner, obj.bucket, bucket_info, aclbl, null_yield);
+ if (ret < 0) {
+ set_err_msg(err_msg, "failed to set acl on bucket " + obj.bucket.name);
+ return ret;
+ }
+
+ RGWBucketEntryPoint ep;
+ ep.bucket = bucket_info.bucket;
+ ep.owner = uid;
+ ep.creation_time = bucket_info.creation_time;
+ ep.linked = true;
+ map<string, bufferlist> ep_attrs;
+ rgw_ep_info ep_data{ep, ep_attrs};
+
- RGWBucketInfo new_bucket_info;
- ret = store->get_bucket_info(sys_ctx, obj.bucket.tenant, obj.bucket.name,
- new_bucket_info, NULL, null_yield);
- if (ret < 0) {
- set_err_msg(err_msg, "failed to fetch bucket info for bucket= " + obj.bucket.name);
- return ret;
- }
-
- ret = rgw_bucket_chown(store, new_bucket_info, uid,
- old_user_info.display_name, obj_marker);
++ ret = bucket_ctl->link_bucket(uid, bucket_info.bucket, ceph::real_time(),
++ null_yield, true, &ep_data);
+ if (ret < 0) {
+ set_err_msg(err_msg, "failed to link bucket " + obj.bucket.name + " to new user");
+ return ret;
+ }
+
++ ret = bucket_ctl->chown(store, bucket_info, uid, old_user_info.display_name,
++ obj_marker, null_yield);
+ if (ret < 0) {
+ set_err_msg(err_msg, "failed to run bucket chown" + cpp_strerror(-ret));
+ return ret;
+ }
+ }
+
+ } while (is_truncated);
+
+ // update the 'stub user' with all of the other fields and rewrite all of the
+ // associated index objects
+ RGWUserInfo& user_info = op_state.get_user_info();
+ user_info.user_id = uid;
+ op_state.objv = objv;
+
+ rename_swift_keys(uid, user_info.swift_keys);
+
+ return update(op_state, err_msg);
+}
+
int RGWUser::execute_add(RGWUserAdminOpState& op_state, std::string *err_msg)
{
std::string subprocess_msg;
--- /dev/null
-#include "common/Mutex.h"
-#include "common/Cond.h"
+
+
+ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+ // vim: ts=8 sw=2 smarttab
+
+ /*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2019 Red Hat, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+
+ #pragma once
+
+ #include <atomic>
+
+ #include "common/Thread.h"
- Mutex lock;
- Cond cond;
++#include "common/ceph_mutex.h"
+
+ class CephContext;
+ class RGWRados;
+
+ class RGWRadosThread {
+ class Worker : public Thread {
+ CephContext *cct;
+ RGWRadosThread *processor;
- Mutex::Locker l(lock);
- cond.Wait(lock);
++ ceph::mutex lock = ceph::make_mutex("RGWRadosThread::Worker");
++ ceph::condition_variable cond;
+
+ void wait() {
- void wait_interval(const utime_t& wait_time) {
- Mutex::Locker l(lock);
- cond.WaitInterval(lock, wait_time);
++ std::unique_lock l{lock};
++ cond.wait(l);
+ };
+
- Worker(CephContext *_cct, RGWRadosThread *_p) : cct(_cct), processor(_p), lock("RGWRadosThread::Worker") {}
++ void wait_interval(const ceph::real_clock::duration& wait_time) {
++ std::unique_lock l{lock};
++ cond.wait_for(l, wait_time);
+ }
+
+ public:
- Mutex::Locker l(lock);
- cond.Signal();
++ Worker(CephContext *_cct, RGWRadosThread *_p) : cct(_cct), processor(_p) {}
+ void *entry() override;
+ void signal() {
- std::string thread_name;
++ std::lock_guard l{lock};
++ cond.notify_all();
+ }
+ };
+
+ Worker *worker;
+
+ protected:
+ CephContext *cct;
+ RGWRados *store;
+
+ std::atomic<bool> down_flag = { false };
+
- RGWRadosThread(RGWRados *_store, const std::string& thread_name = "radosgw")
++ string thread_name;
+
+ virtual uint64_t interval_msec() = 0;
+ virtual void stop_process() {}
+ public:
++ RGWRadosThread(RGWRados *_store, const string& thread_name = "radosgw")
+ : worker(NULL), cct(_store->ctx()), store(_store), thread_name(thread_name) {}
+ virtual ~RGWRadosThread() {
+ stop();
+ }
+
+ virtual int init() { return 0; }
+ virtual int process() = 0;
+
+ bool going_down() { return down_flag; }
+
+ void start();
+ void stop();
+
+ void signal() {
+ if (worker) {
+ worker->signal();
+ }
+ }
+ };
+
--- /dev/null
- RWLock lock;
+
+
+ #pragma once
+
+
+ #include "rgw/rgw_service.h"
+
+ #include "svc_rados.h"
+ #include "svc_sys_obj_types.h"
+
+
+
+ struct RGWSI_SysObj_Core_GetObjState : public RGWSI_SysObj_Obj_GetObjState {
+ RGWSI_RADOS::Obj rados_obj;
+ bool has_rados_obj{false};
+ uint64_t last_ver{0};
+
+ RGWSI_SysObj_Core_GetObjState() {}
+
+ int get_rados_obj(RGWSI_RADOS *rados_svc,
+ RGWSI_Zone *zone_svc,
+ const rgw_raw_obj& obj,
+ RGWSI_RADOS::Obj **pobj);
+ };
+
+ struct RGWSI_SysObj_Core_PoolListImplInfo : public RGWSI_SysObj_Pool_ListInfo {
+ RGWSI_RADOS::Pool pool;
+ RGWSI_RADOS::Pool::List op;
+ RGWAccessListFilterPrefix filter;
+
+ RGWSI_SysObj_Core_PoolListImplInfo(const string& prefix) : op(pool.op()), filter(prefix) {}
+ };
+
+ struct RGWSysObjState {
+ rgw_raw_obj obj;
+ bool has_attrs{false};
+ bool exists{false};
+ uint64_t size{0};
+ ceph::real_time mtime;
+ uint64_t epoch{0};
+ bufferlist obj_tag;
+ bool has_data{false};
+ bufferlist data;
+ bool prefetch_data{false};
+ uint64_t pg_ver{0};
+
+ /* important! don't forget to update copy constructor */
+
+ RGWObjVersionTracker objv_tracker;
+
+ map<string, bufferlist> attrset;
+ RGWSysObjState() {}
+ RGWSysObjState(const RGWSysObjState& rhs) : obj (rhs.obj) {
+ has_attrs = rhs.has_attrs;
+ exists = rhs.exists;
+ size = rhs.size;
+ mtime = rhs.mtime;
+ epoch = rhs.epoch;
+ if (rhs.obj_tag.length()) {
+ obj_tag = rhs.obj_tag;
+ }
+ has_data = rhs.has_data;
+ if (rhs.data.length()) {
+ data = rhs.data;
+ }
+ prefetch_data = rhs.prefetch_data;
+ pg_ver = rhs.pg_ver;
+ objv_tracker = rhs.objv_tracker;
+ }
+ };
+
+
+ class RGWSysObjectCtxBase {
+ std::map<rgw_raw_obj, RGWSysObjState> objs_state;
- explicit RGWSysObjectCtxBase() : lock("RGWSysObjectCtxBase") {}
++ ceph::shared_mutex lock = ceph::make_shared_mutex("RGWSysObjectCtxBase");
+
+ public:
- RGWSysObjectCtxBase(const RGWSysObjectCtxBase& rhs) : objs_state(rhs.objs_state),
- lock("RGWSysObjectCtxBase") {}
- RGWSysObjectCtxBase(const RGWSysObjectCtxBase&& rhs) : objs_state(std::move(rhs.objs_state)),
- lock("RGWSysObjectCtxBase") {}
++ RGWSysObjectCtxBase() = default;
+
- lock.get_read();
++ RGWSysObjectCtxBase(const RGWSysObjectCtxBase& rhs) : objs_state(rhs.objs_state) {}
++ RGWSysObjectCtxBase(RGWSysObjectCtxBase&& rhs) : objs_state(std::move(rhs.objs_state)) {}
+
+ RGWSysObjState *get_state(const rgw_raw_obj& obj) {
+ RGWSysObjState *result;
+ std::map<rgw_raw_obj, RGWSysObjState>::iterator iter;
- lock.unlock();
++ lock.lock_shared();
+ assert (!obj.empty());
+ iter = objs_state.find(obj);
+ if (iter != objs_state.end()) {
+ result = &iter->second;
- lock.unlock();
- lock.get_write();
++ lock.unlock_shared();
+ } else {
- RWLock::WLocker wl(lock);
++ lock.unlock_shared();
++ lock.lock();
+ result = &objs_state[obj];
+ lock.unlock();
+ }
+ return result;
+ }
+
+ void set_prefetch_data(rgw_raw_obj& obj) {
- RWLock::WLocker wl(lock);
++ std::unique_lock wl{lock};
+ assert (!obj.empty());
+ objs_state[obj].prefetch_data = true;
+ }
+ void invalidate(const rgw_raw_obj& obj) {
++ std::unique_lock wl{lock};
+ auto iter = objs_state.find(obj);
+ if (iter == objs_state.end()) {
+ return;
+ }
+ objs_state.erase(iter);
+ }
+ };
+
--- /dev/null
- if (r >= 0 && inf.user_id != info.user_id) {
+
+
+ #include "svc_user.h"
+ #include "svc_user_rados.h"
+ #include "svc_zone.h"
+ #include "svc_sys_obj.h"
+ #include "svc_sys_obj_cache.h"
+ #include "svc_meta.h"
+ #include "svc_meta_be_sobj.h"
+ #include "svc_sync_modules.h"
+
+ #include "rgw/rgw_user.h"
+ #include "rgw/rgw_bucket.h"
+ #include "rgw/rgw_tools.h"
+ #include "rgw/rgw_zone.h"
+ #include "rgw/rgw_rados.h"
+
+ #include "cls/user/cls_user_client.h"
+
+ #define dout_subsys ceph_subsys_rgw
+
+ #define RGW_BUCKETS_OBJ_SUFFIX ".buckets"
+
+ class RGWSI_User_Module : public RGWSI_MBSObj_Handler_Module {
+ RGWSI_User_RADOS::Svc& svc;
+
+ const string prefix;
+ public:
+ RGWSI_User_Module(RGWSI_User_RADOS::Svc& _svc) : RGWSI_MBSObj_Handler_Module("user"),
+ svc(_svc) {}
+
+ void get_pool_and_oid(const string& key, rgw_pool *pool, string *oid) override {
+ if (pool) {
+ *pool = svc.zone->get_zone_params().user_uid_pool;
+ }
+ if (oid) {
+ *oid = key;
+ }
+ }
+
+ const string& get_oid_prefix() override {
+ return prefix;
+ }
+
+ bool is_valid_oid(const string& oid) override {
+ return true;
+ }
+
+ string key_to_oid(const string& key) override {
+ return key;
+ }
+
+ string oid_to_key(const string& oid) override {
+ return oid;
+ }
+ };
+
+ RGWSI_User_RADOS::RGWSI_User_RADOS(CephContext *cct): RGWSI_User(cct) {
+ }
+
+ RGWSI_User_RADOS::~RGWSI_User_RADOS() {
+ }
+
+ void RGWSI_User_RADOS::init(RGWSI_RADOS *_rados_svc,
+ RGWSI_Zone *_zone_svc, RGWSI_SysObj *_sysobj_svc,
+ RGWSI_SysObj_Cache *_cache_svc, RGWSI_Meta *_meta_svc,
+ RGWSI_MetaBackend *_meta_be_svc,
+ RGWSI_SyncModules *_sync_modules_svc)
+ {
+ svc.user = this;
+ svc.rados = _rados_svc;
+ svc.zone = _zone_svc;
+ svc.sysobj = _sysobj_svc;
+ svc.cache = _cache_svc;
+ svc.meta = _meta_svc;
+ svc.meta_be = _meta_be_svc;
+ svc.sync_modules = _sync_modules_svc;
+ }
+
+ int RGWSI_User_RADOS::do_start()
+ {
+ uinfo_cache.reset(new RGWChainedCacheImpl<user_info_cache_entry>);
+ uinfo_cache->init(svc.cache);
+
+ int r = svc.meta->create_be_handler(RGWSI_MetaBackend::Type::MDBE_SOBJ, &be_handler);
+ if (r < 0) {
+ ldout(ctx(), 0) << "ERROR: failed to create be handler: r=" << r << dendl;
+ return r;
+ }
+
+ RGWSI_MetaBackend_Handler_SObj *bh = static_cast<RGWSI_MetaBackend_Handler_SObj *>(be_handler);
+
+ auto module = new RGWSI_User_Module(svc);
+ be_module.reset(module);
+ bh->set_module(module);
+ return 0;
+ }
+
+ rgw_raw_obj RGWSI_User_RADOS::get_buckets_obj(const rgw_user& user) const
+ {
+ string oid = user.to_str() + RGW_BUCKETS_OBJ_SUFFIX;
+ return rgw_raw_obj(svc.zone->get_zone_params().user_uid_pool, oid);
+ }
+
+ int RGWSI_User_RADOS::read_user_info(RGWSI_MetaBackend::Context *ctx,
+ const rgw_user& user,
+ RGWUserInfo *info,
+ RGWObjVersionTracker * const objv_tracker,
+ real_time * const pmtime,
+ rgw_cache_entry_info * const cache_info,
+ map<string, bufferlist> * const pattrs,
+ optional_yield y)
+ {
+ bufferlist bl;
+ RGWUID user_id;
+
+ RGWSI_MBSObj_GetParams params(&bl, pattrs, pmtime);
+ params.set_cache_info(cache_info);
+
+ int ret = svc.meta_be->get_entry(ctx, get_meta_key(user), params, objv_tracker, y);
+ if (ret < 0) {
+ return ret;
+ }
+
+ auto iter = bl.cbegin();
+ try {
+ decode(user_id, iter);
+ if (user_id.user_id != user) {
+ lderr(svc.meta_be->ctx()) << "ERROR: rgw_get_user_info_by_uid(): user id mismatch: " << user_id.user_id << " != " << user << dendl;
+ return -EIO;
+ }
+ if (!iter.end()) {
+ decode(*info, iter);
+ }
+ } catch (buffer::error& err) {
+ ldout(svc.meta_be->ctx(), 0) << "ERROR: failed to decode user info, caught buffer::error" << dendl;
+ return -EIO;
+ }
+
+ return 0;
+ }
+
+ class PutOperation
+ {
+ RGWSI_User_RADOS::Svc& svc;
+ RGWSI_MetaBackend_SObj::Context_SObj *ctx;
+ RGWUID ui;
+ const RGWUserInfo& info;
+ RGWUserInfo *old_info;
+ RGWObjVersionTracker *objv_tracker;
+ const real_time& mtime;
+ bool exclusive;
+ map<string, bufferlist> *pattrs;
+ RGWObjVersionTracker ot;
+ string err_msg;
+ optional_yield y;
+
+ void set_err_msg(string msg) {
+ if (!err_msg.empty()) {
+ err_msg = std::move(msg);
+ }
+ }
+
+ public:
+ PutOperation(RGWSI_User_RADOS::Svc& svc,
+ RGWSI_MetaBackend::Context *_ctx,
+ const RGWUserInfo& info,
+ RGWUserInfo *old_info,
+ RGWObjVersionTracker *objv_tracker,
+ const real_time& mtime,
+ bool exclusive,
+ map<string, bufferlist> *pattrs,
+ optional_yield y) :
+ svc(svc), info(info), old_info(old_info),
+ objv_tracker(objv_tracker), mtime(mtime),
+ exclusive(exclusive), pattrs(pattrs), y(y) {
+ ctx = static_cast<RGWSI_MetaBackend_SObj::Context_SObj *>(_ctx);
+ ui.user_id = info.user_id;
+ }
+
+ int prepare() {
+ if (objv_tracker) {
+ ot = *objv_tracker;
+ }
+
+ if (ot.write_version.tag.empty()) {
+ if (ot.read_version.tag.empty()) {
+ ot.generate_new_write_ver(svc.meta_be->ctx());
+ } else {
+ ot.write_version = ot.read_version;
+ ot.write_version.ver++;
+ }
+ }
+
+ for (auto iter = info.swift_keys.begin(); iter != info.swift_keys.end(); ++iter) {
+ if (old_info && old_info->swift_keys.count(iter->first) != 0)
+ continue;
+ auto& k = iter->second;
+ /* check if swift mapping exists */
+ RGWUserInfo inf;
+ int r = svc.user->get_user_info_by_swift(ctx, k.id, &inf, nullptr, nullptr, y);
- if (r >= 0 && inf.user_id != info.user_id) {
++ if (r >= 0 && inf.user_id != info.user_id &&
++ (!old_info || inf.user_id != old_info->user_id)) {
+ ldout(svc.meta_be->ctx(), 0) << "WARNING: can't store user info, swift id (" << k.id
+ << ") already mapped to another user (" << info.user_id << ")" << dendl;
+ return -EEXIST;
+ }
+ }
+
+ /* check if access keys already exist */
+ for (auto iter = info.access_keys.begin(); iter != info.access_keys.end(); ++iter) {
+ if (old_info && old_info->access_keys.count(iter->first) != 0)
+ continue;
+ auto& k = iter->second;
+ RGWUserInfo inf;
+ int r = svc.user->get_user_info_by_access_key(ctx, k.id, &inf, nullptr, nullptr, y);
- if (old_info && old_info->access_keys.count(iter->first) != 0)
++ if (r >= 0 && inf.user_id != info.user_id &&
++ (!old_info || inf.user_id != old_info->user_id)) {
+ ldout(svc.meta_be->ctx(), 0) << "WARNING: can't store user info, access key already mapped to another user" << dendl;
+ return -EEXIST;
+ }
+ }
+
+ return 0;
+ }
+
+ int put() {
+ bufferlist data_bl;
+ encode(ui, data_bl);
+ encode(info, data_bl);
+
+ RGWSI_MBSObj_PutParams params(data_bl, pattrs, mtime, exclusive);
+
+ int ret = svc.meta_be->put(ctx, RGWSI_User::get_meta_key(info.user_id), params, &ot, y);
+ if (ret < 0)
+ return ret;
+
+ return 0;
+ }
+
+ int complete() {
+ int ret;
+
+ bufferlist link_bl;
+ encode(ui, link_bl);
+
+ auto& obj_ctx = *ctx->obj_ctx;
+
+ if (!info.user_email.empty()) {
+ if (!old_info ||
+ old_info->user_email.compare(info.user_email) != 0) { /* only if new index changed */
+ ret = rgw_put_system_obj(obj_ctx, svc.zone->get_zone_params().user_email_pool, info.user_email,
+ link_bl, exclusive, NULL, real_time(), y);
+ if (ret < 0)
+ return ret;
+ }
+ }
+
++ const bool renamed = old_info && old_info->user_id != info.user_id;
+ for (auto iter = info.access_keys.begin(); iter != info.access_keys.end(); ++iter) {
+ auto& k = iter->second;
- if (old_info && old_info->swift_keys.count(siter->first) != 0)
++ if (old_info && old_info->access_keys.count(iter->first) != 0 && !renamed)
+ continue;
+
+ ret = rgw_put_system_obj(obj_ctx, svc.zone->get_zone_params().user_keys_pool, k.id,
+ link_bl, exclusive, NULL, real_time(), y);
+ if (ret < 0)
+ return ret;
+ }
+
+ for (auto siter = info.swift_keys.begin(); siter != info.swift_keys.end(); ++siter) {
+ auto& k = siter->second;
++ if (old_info && old_info->swift_keys.count(siter->first) != 0 && !renamed)
+ continue;
+
+ ret = rgw_put_system_obj(obj_ctx, svc.zone->get_zone_params().user_swift_pool, k.id,
+ link_bl, exclusive, NULL, real_time(), y);
+ if (ret < 0)
+ return ret;
+ }
+
+ if (old_info) {
+ ret = remove_old_indexes(*old_info, info, y);
+ if (ret < 0) {
+ return ret;
+ }
+ }
+
+ return 0;
+ }
+
+ int remove_old_indexes(const RGWUserInfo& old_info, const RGWUserInfo& new_info, optional_yield y) {
+ int ret;
+
+ if (!old_info.user_id.empty() &&
+ old_info.user_id != new_info.user_id) {
+ if (old_info.user_id.tenant != new_info.user_id.tenant) {
+ ldout(svc.user->ctx(), 0) << "ERROR: tenant mismatch: " << old_info.user_id.tenant << " != " << new_info.user_id.tenant << dendl;
+ return -EINVAL;
+ }
+ ret = svc.user->remove_uid_index(ctx, old_info, nullptr, y);
+ if (ret < 0 && ret != -ENOENT) {
+ set_err_msg("ERROR: could not remove index for uid " + old_info.user_id.to_str());
+ return ret;
+ }
+ }
+
+ if (!old_info.user_email.empty() &&
+ old_info.user_email != new_info.user_email) {
+ ret = svc.user->remove_email_index(ctx, old_info.user_email, y);
+ if (ret < 0 && ret != -ENOENT) {
+ set_err_msg("ERROR: could not remove index for email " + old_info.user_email);
+ return ret;
+ }
+ }
+
+ for (const auto& [name, access_key] : old_info.access_keys) {
+ if (!new_info.access_keys.count(access_key.id)) {
+ ret = svc.user->remove_key_index(ctx, access_key, y);
+ if (ret < 0 && ret != -ENOENT) {
+ set_err_msg("ERROR: could not remove index for key " + access_key.id);
+ return ret;
+ }
+ }
+ }
+
+ for (auto old_iter = old_info.swift_keys.begin(); old_iter != old_info.swift_keys.end(); ++old_iter) {
+ const auto& swift_key = old_iter->second;
+ auto new_iter = new_info.swift_keys.find(swift_key.id);
+ if (new_iter == new_info.swift_keys.end()) {
+ ret = svc.user->remove_swift_name_index(ctx, swift_key.id, y);
+ if (ret < 0 && ret != -ENOENT) {
+ set_err_msg("ERROR: could not remove index for swift_name " + swift_key.id);
+ return ret;
+ }
+ }
+ }
+
+ return 0;
+ }
+
+ const string& get_err_msg() {
+ return err_msg;
+ }
+ };
+
+ int RGWSI_User_RADOS::store_user_info(RGWSI_MetaBackend::Context *ctx,
+ const RGWUserInfo& info,
+ RGWUserInfo *old_info,
+ RGWObjVersionTracker *objv_tracker,
+ const real_time& mtime,
+ bool exclusive,
+ map<string, bufferlist> *attrs,
+ optional_yield y)
+ {
+ PutOperation op(svc, ctx,
+ info, old_info,
+ objv_tracker,
+ mtime, exclusive,
+ attrs,
+ y);
+
+ int r = op.prepare();
+ if (r < 0) {
+ return r;
+ }
+
+ r = op.put();
+ if (r < 0) {
+ return r;
+ }
+
+ r = op.complete();
+ if (r < 0) {
+ return r;
+ }
+
+ return 0;
+ }
+
+ int RGWSI_User_RADOS::remove_key_index(RGWSI_MetaBackend::Context *_ctx,
+ const RGWAccessKey& access_key,
+ optional_yield y)
+ {
+ RGWSI_MetaBackend_SObj::Context_SObj *ctx = static_cast<RGWSI_MetaBackend_SObj::Context_SObj *>(_ctx);
+ rgw_raw_obj obj(svc.zone->get_zone_params().user_keys_pool, access_key.id);
+ auto sysobj = ctx->obj_ctx->get_obj(obj);
+ return sysobj.wop().remove(y);
+ }
+
+ int RGWSI_User_RADOS::remove_email_index(RGWSI_MetaBackend::Context *_ctx,
+ const string& email,
+ optional_yield y)
+ {
+ if (email.empty()) {
+ return 0;
+ }
+ RGWSI_MetaBackend_SObj::Context_SObj *ctx = static_cast<RGWSI_MetaBackend_SObj::Context_SObj *>(_ctx);
+ rgw_raw_obj obj(svc.zone->get_zone_params().user_email_pool, email);
+ auto sysobj = ctx->obj_ctx->get_obj(obj);
+ return sysobj.wop().remove(y);
+ }
+
+ int RGWSI_User_RADOS::remove_swift_name_index(RGWSI_MetaBackend::Context *_ctx, const string& swift_name,
+ optional_yield y)
+ {
+ RGWSI_MetaBackend_SObj::Context_SObj *ctx = static_cast<RGWSI_MetaBackend_SObj::Context_SObj *>(_ctx);
+ rgw_raw_obj obj(svc.zone->get_zone_params().user_swift_pool, swift_name);
+ auto sysobj = ctx->obj_ctx->get_obj(obj);
+ return sysobj.wop().remove(y);
+ }
+
+ /**
+ * delete a user's presence from the RGW system.
+ * First remove their bucket ACLs, then delete them
+ * from the user and user email pools. This leaves the pools
+ * themselves alone, as well as any ACLs embedded in object xattrs.
+ */
+ int RGWSI_User_RADOS::remove_user_info(RGWSI_MetaBackend::Context *_ctx,
+ const RGWUserInfo& info,
+ RGWObjVersionTracker *objv_tracker,
+ optional_yield y)
+ {
+ int ret;
+
+ auto cct = svc.meta_be->ctx();
+
+ auto kiter = info.access_keys.begin();
+ for (; kiter != info.access_keys.end(); ++kiter) {
+ ldout(cct, 10) << "removing key index: " << kiter->first << dendl;
+ ret = remove_key_index(_ctx, kiter->second, y);
+ if (ret < 0 && ret != -ENOENT) {
+ ldout(cct, 0) << "ERROR: could not remove " << kiter->first << " (access key object), should be fixed (err=" << ret << ")" << dendl;
+ return ret;
+ }
+ }
+
+ auto siter = info.swift_keys.begin();
+ for (; siter != info.swift_keys.end(); ++siter) {
+ auto& k = siter->second;
+ ldout(cct, 10) << "removing swift subuser index: " << k.id << dendl;
+ /* check if swift mapping exists */
+ ret = remove_swift_name_index(_ctx, k.id, y);
+ if (ret < 0 && ret != -ENOENT) {
+ ldout(cct, 0) << "ERROR: could not remove " << k.id << " (swift name object), should be fixed (err=" << ret << ")" << dendl;
+ return ret;
+ }
+ }
+
+ ldout(cct, 10) << "removing email index: " << info.user_email << dendl;
+ ret = remove_email_index(_ctx, info.user_email, y);
+ if (ret < 0 && ret != -ENOENT) {
+ ldout(cct, 0) << "ERROR: could not remove email index object for "
+ << info.user_email << ", should be fixed (err=" << ret << ")" << dendl;
+ return ret;
+ }
+
+ rgw_raw_obj uid_bucks = get_buckets_obj(info.user_id);
+ ldout(cct, 10) << "removing user buckets index" << dendl;
+ RGWSI_MetaBackend_SObj::Context_SObj *ctx = static_cast<RGWSI_MetaBackend_SObj::Context_SObj *>(_ctx);
+ auto sysobj = ctx->obj_ctx->get_obj(uid_bucks);
+ ret = sysobj.wop().remove(y);
+ if (ret < 0 && ret != -ENOENT) {
+ ldout(cct, 0) << "ERROR: could not remove " << info.user_id << ":" << uid_bucks << ", should be fixed (err=" << ret << ")" << dendl;
+ return ret;
+ }
+
+ ret = remove_uid_index(ctx, info, objv_tracker, y);
+ if (ret < 0 && ret != -ENOENT) {
+ return ret;
+ }
+
+ return 0;
+ }
+
+ int RGWSI_User_RADOS::remove_uid_index(RGWSI_MetaBackend::Context *ctx, const RGWUserInfo& user_info, RGWObjVersionTracker *objv_tracker,
+ optional_yield y)
+ {
+ ldout(cct, 10) << "removing user index: " << user_info.user_id << dendl;
+
+ RGWSI_MBSObj_RemoveParams params;
+ int ret = svc.meta_be->remove(ctx, get_meta_key(user_info.user_id), params, objv_tracker, y);
+ if (ret < 0 && ret != -ENOENT && ret != -ECANCELED) {
+ string key;
+ user_info.user_id.to_str(key);
+ rgw_raw_obj uid_obj(svc.zone->get_zone_params().user_uid_pool, key);
+ ldout(cct, 0) << "ERROR: could not remove " << user_info.user_id << ":" << uid_obj << ", should be fixed (err=" << ret << ")" << dendl;
+ return ret;
+ }
+
+ return 0;
+ }
+
+ int RGWSI_User_RADOS::get_user_info_from_index(RGWSI_MetaBackend::Context *_ctx,
+ const string& key,
+ const rgw_pool& pool,
+ RGWUserInfo *info,
+ RGWObjVersionTracker * const objv_tracker,
+ real_time * const pmtime, optional_yield y)
+ {
+ RGWSI_MetaBackend_SObj::Context_SObj *ctx = static_cast<RGWSI_MetaBackend_SObj::Context_SObj *>(_ctx);
+
+ string cache_key = pool.to_str() + "/" + key;
+
+ if (auto e = uinfo_cache->find(cache_key)) {
+ *info = e->info;
+ if (objv_tracker)
+ *objv_tracker = e->objv_tracker;
+ if (pmtime)
+ *pmtime = e->mtime;
+ return 0;
+ }
+
+ user_info_cache_entry e;
+ bufferlist bl;
+ RGWUID uid;
+
+ int ret = rgw_get_system_obj(*ctx->obj_ctx, pool, key, bl, nullptr, &e.mtime, y);
+ if (ret < 0)
+ return ret;
+
+ rgw_cache_entry_info cache_info;
+
+ auto iter = bl.cbegin();
+ try {
+ decode(uid, iter);
+
+ int ret = read_user_info(ctx, uid.user_id,
+ &e.info, &e.objv_tracker, nullptr, &cache_info, nullptr,
+ y);
+ if (ret < 0) {
+ return ret;
+ }
+ } catch (buffer::error& err) {
+ ldout(svc.meta_be->ctx(), 0) << "ERROR: failed to decode user info, caught buffer::error" << dendl;
+ return -EIO;
+ }
+
+ uinfo_cache->put(svc.cache, cache_key, &e, { &cache_info });
+
+ *info = e.info;
+ if (objv_tracker)
+ *objv_tracker = e.objv_tracker;
+ if (pmtime)
+ *pmtime = e.mtime;
+
+ return 0;
+ }
+
+ /**
+ * Given an email, finds the user info associated with it.
+ * returns: 0 on success, -ERR# on failure (including nonexistence)
+ */
+ int RGWSI_User_RADOS::get_user_info_by_email(RGWSI_MetaBackend::Context *ctx,
+ const string& email, RGWUserInfo *info,
+ RGWObjVersionTracker *objv_tracker,
+ real_time *pmtime, optional_yield y)
+ {
+ return get_user_info_from_index(ctx, email, svc.zone->get_zone_params().user_email_pool,
+ info, objv_tracker, pmtime, y);
+ }
+
+ /**
+ * Given an swift username, finds the user_info associated with it.
+ * returns: 0 on success, -ERR# on failure (including nonexistence)
+ */
+ int RGWSI_User_RADOS::get_user_info_by_swift(RGWSI_MetaBackend::Context *ctx,
+ const string& swift_name,
+ RGWUserInfo *info, /* out */
+ RGWObjVersionTracker * const objv_tracker,
+ real_time * const pmtime, optional_yield y)
+ {
+ return get_user_info_from_index(ctx,
+ swift_name,
+ svc.zone->get_zone_params().user_swift_pool,
+ info, objv_tracker, pmtime, y);
+ }
+
+ /**
+ * Given an access key, finds the user info associated with it.
+ * returns: 0 on success, -ERR# on failure (including nonexistence)
+ */
+ int RGWSI_User_RADOS::get_user_info_by_access_key(RGWSI_MetaBackend::Context *ctx,
+ const std::string& access_key,
+ RGWUserInfo *info,
+ RGWObjVersionTracker* objv_tracker,
+ real_time *pmtime, optional_yield y)
+ {
+ return get_user_info_from_index(ctx,
+ access_key,
+ svc.zone->get_zone_params().user_keys_pool,
+ info, objv_tracker, pmtime, y);
+ }
+
+ int RGWSI_User_RADOS::cls_user_update_buckets(rgw_raw_obj& obj, list<cls_user_bucket_entry>& entries, bool add)
+ {
+ auto rados_obj = svc.rados->obj(obj);
+ int r = rados_obj.open();
+ if (r < 0) {
+ return r;
+ }
+
+ librados::ObjectWriteOperation op;
+ cls_user_set_buckets(op, entries, add);
+ r = rados_obj.operate(&op, null_yield);
+ if (r < 0) {
+ return r;
+ }
+
+ return 0;
+ }
+
+ int RGWSI_User_RADOS::cls_user_add_bucket(rgw_raw_obj& obj, const cls_user_bucket_entry& entry)
+ {
+ list<cls_user_bucket_entry> l;
+ l.push_back(entry);
+
+ return cls_user_update_buckets(obj, l, true);
+ }
+
+ int RGWSI_User_RADOS::cls_user_remove_bucket(rgw_raw_obj& obj, const cls_user_bucket& bucket)
+ {
+ auto rados_obj = svc.rados->obj(obj);
+ int r = rados_obj.open();
+ if (r < 0) {
+ return r;
+ }
+
+ librados::ObjectWriteOperation op;
+ ::cls_user_remove_bucket(op, bucket);
+ r = rados_obj.operate(&op, null_yield);
+ if (r < 0)
+ return r;
+
+ return 0;
+ }
+
+ int RGWSI_User_RADOS::add_bucket(RGWSI_MetaBackend::Context *ctx,
+ const rgw_user& user,
+ const rgw_bucket& bucket,
+ ceph::real_time creation_time)
+ {
+ int ret;
+
+ cls_user_bucket_entry new_bucket;
+
+ bucket.convert(&new_bucket.bucket);
+ new_bucket.size = 0;
+ if (real_clock::is_zero(creation_time))
+ new_bucket.creation_time = real_clock::now();
+ else
+ new_bucket.creation_time = creation_time;
+
+ rgw_raw_obj obj = get_buckets_obj(user);
+ ret = cls_user_add_bucket(obj, new_bucket);
+ if (ret < 0) {
+ ldout(cct, 0) << "ERROR: error adding bucket to user: ret=" << ret << dendl;
+ return ret;
+ }
+
+ return 0;
+ }
+
+
+ int RGWSI_User_RADOS::remove_bucket(RGWSI_MetaBackend::Context *ctx,
+ const rgw_user& user,
+ const rgw_bucket& _bucket)
+ {
+ cls_user_bucket bucket;
+ bucket.name = _bucket.name;
+ rgw_raw_obj obj = get_buckets_obj(user);
+ int ret = cls_user_remove_bucket(obj, bucket);
+ if (ret < 0) {
+ ldout(cct, 0) << "ERROR: error removing bucket from user: ret=" << ret << dendl;
+ }
+
+ return 0;
+ }
+
+ int RGWSI_User_RADOS::cls_user_flush_bucket_stats(rgw_raw_obj& user_obj,
+ const RGWBucketEnt& ent)
+ {
+ cls_user_bucket_entry entry;
+ ent.convert(&entry);
+
+ list<cls_user_bucket_entry> entries;
+ entries.push_back(entry);
+
+ int r = cls_user_update_buckets(user_obj, entries, false);
+ if (r < 0) {
+ ldout(cct, 20) << "cls_user_update_buckets() returned " << r << dendl;
+ return r;
+ }
+
+ return 0;
+ }
+
+ int RGWSI_User_RADOS::cls_user_list_buckets(rgw_raw_obj& obj,
+ const string& in_marker,
+ const string& end_marker,
+ const int max_entries,
+ list<cls_user_bucket_entry>& entries,
+ string * const out_marker,
+ bool * const truncated)
+ {
+ auto rados_obj = svc.rados->obj(obj);
+ int r = rados_obj.open();
+ if (r < 0) {
+ return r;
+ }
+
+ librados::ObjectReadOperation op;
+ int rc;
+
+ cls_user_bucket_list(op, in_marker, end_marker, max_entries, entries, out_marker, truncated, &rc);
+ bufferlist ibl;
+ r = rados_obj.operate(&op, &ibl, null_yield);
+ if (r < 0)
+ return r;
+ if (rc < 0)
+ return rc;
+
+ return 0;
+ }
+
+ int RGWSI_User_RADOS::list_buckets(RGWSI_MetaBackend::Context *ctx,
+ const rgw_user& user,
+ const string& marker,
+ const string& end_marker,
+ uint64_t max,
+ RGWUserBuckets *buckets,
+ bool *is_truncated)
+ {
+ int ret;
+
+ buckets->clear();
+
+ rgw_raw_obj obj = get_buckets_obj(user);
+
+ bool truncated = false;
+ string m = marker;
+
+ uint64_t total = 0;
+
+ do {
+ std::list<cls_user_bucket_entry> entries;
+ ret = cls_user_list_buckets(obj, m, end_marker, max - total, entries, &m, &truncated);
+ if (ret == -ENOENT) {
+ ret = 0;
+ }
+
+ if (ret < 0) {
+ return ret;
+ }
+
+ for (auto& entry : entries) {
+ buckets->add(RGWBucketEnt(user, std::move(entry)));
+ total++;
+ }
+
+ } while (truncated && total < max);
+
+ if (is_truncated) {
+ *is_truncated = truncated;
+ }
+
+ return 0;
+ }
+
+ int RGWSI_User_RADOS::flush_bucket_stats(RGWSI_MetaBackend::Context *ctx,
+ const rgw_user& user,
+ const RGWBucketEnt& ent)
+ {
+ rgw_raw_obj obj = get_buckets_obj(user);
+
+ return cls_user_flush_bucket_stats(obj, ent);
+ }
+
+ int RGWSI_User_RADOS::reset_bucket_stats(RGWSI_MetaBackend::Context *ctx,
+ const rgw_user& user)
+ {
+ return cls_user_reset_stats(user);
+ }
+
+ int RGWSI_User_RADOS::cls_user_reset_stats(const rgw_user& user)
+ {
+ rgw_raw_obj obj = get_buckets_obj(user);
+ auto rados_obj = svc.rados->obj(obj);
+ int r = rados_obj.open();
+ if (r < 0) {
+ return r;
+ }
+ librados::ObjectWriteOperation op;
+ ::cls_user_reset_stats(op);
+ return rados_obj.operate(&op, null_yield);
+ }
+
+ int RGWSI_User_RADOS::complete_flush_stats(RGWSI_MetaBackend::Context *ctx,
+ const rgw_user& user)
+ {
+ rgw_raw_obj obj = get_buckets_obj(user);
+ auto rados_obj = svc.rados->obj(obj);
+ int r = rados_obj.open();
+ if (r < 0) {
+ return r;
+ }
+ librados::ObjectWriteOperation op;
+ ::cls_user_complete_stats_sync(op);
+ return rados_obj.operate(&op, null_yield);
+ }
+
+ int RGWSI_User_RADOS::cls_user_get_header(const rgw_user& user, cls_user_header *header)
+ {
+ rgw_raw_obj obj = get_buckets_obj(user);
+ auto rados_obj = svc.rados->obj(obj);
+ int r = rados_obj.open();
+ if (r < 0) {
+ return r;
+ }
+ int rc;
+ bufferlist ibl;
+ librados::ObjectReadOperation op;
+ ::cls_user_get_header(op, header, &rc);
+ return rados_obj.operate(&op, &ibl, null_yield);
+ }
+
+ int RGWSI_User_RADOS::cls_user_get_header_async(const string& user, RGWGetUserHeader_CB *cb)
+ {
+ rgw_raw_obj obj = get_buckets_obj(user);
+ auto rados_obj = svc.rados->obj(obj);
+ int r = rados_obj.open();
+ if (r < 0) {
+ return r;
+ }
+
+ auto& ref = rados_obj.get_ref();
+
+ r = ::cls_user_get_header_async(ref.pool.ioctx(), ref.obj.oid, cb);
+ if (r < 0) {
+ return r;
+ }
+
+ return 0;
+ }
+
+ int RGWSI_User_RADOS::read_stats(RGWSI_MetaBackend::Context *ctx,
+ const rgw_user& user, RGWStorageStats *stats,
+ ceph::real_time *last_stats_sync,
+ ceph::real_time *last_stats_update)
+ {
+ string user_str = user.to_str();
+
+ cls_user_header header;
+ int r = cls_user_get_header(user_str, &header);
+ if (r < 0)
+ return r;
+
+ const cls_user_stats& hs = header.stats;
+
+ stats->size = hs.total_bytes;
+ stats->size_rounded = hs.total_bytes_rounded;
+ stats->num_objects = hs.total_entries;
+
+ if (last_stats_sync) {
+ *last_stats_sync = header.last_stats_sync;
+ }
+
+ if (last_stats_update) {
+ *last_stats_update = header.last_stats_update;
+ }
+
+ return 0;
+ }
+
+ class RGWGetUserStatsContext : public RGWGetUserHeader_CB {
+ RGWGetUserStats_CB *cb;
+
+ public:
+ explicit RGWGetUserStatsContext(RGWGetUserStats_CB * const cb)
+ : cb(cb) {}
+
+ void handle_response(int r, cls_user_header& header) override {
+ const cls_user_stats& hs = header.stats;
+ if (r >= 0) {
+ RGWStorageStats stats;
+
+ stats.size = hs.total_bytes;
+ stats.size_rounded = hs.total_bytes_rounded;
+ stats.num_objects = hs.total_entries;
+
+ cb->set_response(stats);
+ }
+
+ cb->handle_response(r);
+
+ cb->put();
+ }
+ };
+
+ int RGWSI_User_RADOS::read_stats_async(RGWSI_MetaBackend::Context *ctx,
+ const rgw_user& user, RGWGetUserStats_CB *_cb)
+ {
+ string user_str = user.to_str();
+
+ RGWGetUserStatsContext *cb = new RGWGetUserStatsContext(_cb);
+ int r = cls_user_get_header_async(user_str, cb);
+ if (r < 0) {
+ _cb->put();
+ delete cb;
+ return r;
+ }
+
+ return 0;
+ }
+
projects / ceph.git / commitdiff