From b7d022f61a7007e940cb7d1d347b566586751fe0 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Fri, 19 Feb 2016 18:34:46 -0800 Subject: [PATCH] rgw: indexless buckets Signed-off-by: Yehuda Sadeh --- src/rgw/rgw_admin.cc | 4 ++-- src/rgw/rgw_bucket.cc | 12 ++++++++++-- src/rgw/rgw_op.cc | 18 +++++++++++------- src/rgw/rgw_orphan.cc | 2 +- src/rgw/rgw_rados.cc | 39 +++++++++++++++++++++++++++++++++++---- src/rgw/rgw_rados.h | 11 ++++++++--- 6 files changed, 67 insertions(+), 19 deletions(-) diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 40581ddc73eac..86f920e04417c 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -1246,7 +1246,7 @@ int do_check_object_locator(const string& tenant_name, const string& bucket_name map common_prefixes; string ns; - RGWRados::Bucket target(store, bucket); + RGWRados::Bucket target(store, bucket_info); RGWRados::Bucket::List list_op(&target); string marker; @@ -3486,7 +3486,7 @@ int main(int argc, char **argv) map common_prefixes; string ns; - RGWRados::Bucket target(store, bucket); + RGWRados::Bucket target(store, bucket_info); RGWRados::Bucket::List list_op(&target); list_op.params.prefix = prefix; diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index 77830d16cad8a..e77ecfbd174e0 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -480,7 +480,7 @@ int rgw_remove_bucket(RGWRados *store, rgw_bucket& bucket, bool delete_children) return ret; - RGWRados::Bucket target(store, bucket); + RGWRados::Bucket target(store, info); RGWRados::Bucket::List list_op(&target); list_op.params.list_versions = true; @@ -763,7 +763,15 @@ int RGWBucket::check_bad_index_multipart(RGWBucketAdminOpState& op_state, map meta_objs; map all_objs; - RGWRados::Bucket target(store, bucket); + RGWBucketInfo bucket_info; + RGWObjectCtx obj_ctx(store); + int r = store->get_bucket_instance_info(obj_ctx, bucket, bucket_info, nullptr, nullptr); + if (r < 0) { + ldout(store->ctx(), 0) << "ERROR: " << __func__ << "(): get_bucket_instance_info(bucket=" << bucket << ") returned r=" << r << dendl; + return r; + } + + RGWRados::Bucket target(store, bucket_info); RGWRados::Bucket::List list_op(&target); list_op.params.list_versions = true; diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 84fcd6539f5cc..f3b5a362dcb21 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -730,7 +730,7 @@ static int iterate_user_manifest_parts(CephContext * const cct, RGWRados * const store, const off_t ofs, const off_t end, - rgw_bucket& bucket, + RGWBucketInfo *pbucket_info, const string& obj_prefix, RGWAccessControlPolicy * const bucket_policy, uint64_t * const ptotal_len, @@ -744,6 +744,7 @@ static int iterate_user_manifest_parts(CephContext * const cct, void *param), void * const cb_param) { + rgw_bucket& bucket = pbucket_info->bucket; uint64_t obj_ofs = 0, len_count = 0; bool found_start = false, found_end = false, handled_end = false; string delim; @@ -752,7 +753,7 @@ static int iterate_user_manifest_parts(CephContext * const cct, utime_t start_time = ceph_clock_now(cct); - RGWRados::Bucket target(store, bucket); + RGWRados::Bucket target(store, *pbucket_info); RGWRados::Bucket::List list_op(&target); list_op.params.prefix = obj_prefix; @@ -928,9 +929,10 @@ int RGWGetObj::handle_user_manifest(const char *prefix) RGWAccessControlPolicy _bucket_policy(s->cct); RGWAccessControlPolicy *bucket_policy; + RGWBucketInfo bucket_info; + RGWBucketInfo *pbucket_info; if (bucket_name.compare(s->bucket.name) != 0) { - RGWBucketInfo bucket_info; map bucket_attrs; RGWObjectCtx obj_ctx(store); int r = store->get_bucket_info(obj_ctx, s->user->user_id.tenant, @@ -942,6 +944,7 @@ int RGWGetObj::handle_user_manifest(const char *prefix) return r; } bucket = bucket_info.bucket; + pbucket_info = &bucket_info; rgw_obj_key no_obj; bucket_policy = &_bucket_policy; r = read_policy(store, s, bucket_info, bucket_attrs, bucket_policy, bucket, no_obj); @@ -951,6 +954,7 @@ int RGWGetObj::handle_user_manifest(const char *prefix) } } else { bucket = s->bucket; + pbucket_info = &s->bucket_info; bucket_policy = s->bucket_acl; } @@ -959,7 +963,7 @@ int RGWGetObj::handle_user_manifest(const char *prefix) * - overall DLO's content size, * - md5 sum of overall DLO's content (for etag of Swift API). */ int r = iterate_user_manifest_parts(s->cct, store, ofs, end, - bucket, obj_prefix, bucket_policy, + pbucket_info, obj_prefix, bucket_policy, &total_len, &s->obj_size, &lo_etag, nullptr /* cb */, nullptr /* cb arg */); if (r < 0) { @@ -973,7 +977,7 @@ int RGWGetObj::handle_user_manifest(const char *prefix) } r = iterate_user_manifest_parts(s->cct, store, ofs, end, - bucket, obj_prefix, bucket_policy, + pbucket_info, obj_prefix, bucket_policy, nullptr, nullptr, nullptr, get_obj_user_manifest_iterate_cb, (void *)this); if (r < 0) { @@ -1646,7 +1650,7 @@ void RGWListBucket::execute() } } - RGWRados::Bucket target(store, s->bucket); + RGWRados::Bucket target(store, s->bucket_info); if (shard_id >= 0) { target.set_shard_id(shard_id); } @@ -4097,7 +4101,7 @@ void RGWListBucketMultiparts::execute() } marker_meta = marker.get_meta(); - RGWRados::Bucket target(store, s->bucket); + RGWRados::Bucket target(store, s->bucket_info); RGWRados::Bucket::List list_op(&target); list_op.params.prefix = prefix; diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc index 861b89c84d278..52dc8d7e1e4df 100644 --- a/src/rgw/rgw_orphan.cc +++ b/src/rgw/rgw_orphan.cc @@ -454,7 +454,7 @@ int RGWOrphanSearch::build_linked_oids_for_bucket(const string& bucket_instance_ return ret; } - RGWRados::Bucket target(store, bucket_info.bucket); + RGWRados::Bucket target(store, bucket_info); RGWRados::Bucket::List list_op(&target); string marker; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 9d7f568fd76cf..7cd7acbc00861 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -4708,6 +4708,7 @@ int rgw_policy_from_attrset(CephContext *cct, map& attrset, return 0; } + /** * get listing of the objects in a bucket. * bucket: bucket to list contents of @@ -5774,7 +5775,7 @@ int RGWRados::Object::Write::write_meta(uint64_t size, bool versioned_op = (target->versioning_enabled() || is_olh || versioned_target); - RGWRados::Bucket bop(store, bucket); + RGWRados::Bucket bop(store, target->get_bucket_info()); RGWRados::Bucket::UpdateIndex index_op(&bop, obj, state); if (versioned_op) { @@ -7357,7 +7358,7 @@ int RGWRados::Object::Delete::delete_obj() bool ret_not_existed = (!state->exists); - RGWRados::Bucket bop(store, bucket); + RGWRados::Bucket bop(store, target->get_bucket_info()); RGWRados::Bucket::UpdateIndex index_op(&bop, obj, state); index_op.set_bilog_flags(params.bilog_flags); @@ -7456,7 +7457,16 @@ int RGWRados::delete_obj_index(rgw_obj& obj) std::string oid, key; get_obj_bucket_and_oid_loc(obj, bucket, oid, key); - RGWRados::Bucket bop(this, bucket); + RGWObjectCtx obj_ctx(this); + + RGWBucketInfo bucket_info; + int ret = get_bucket_instance_info(obj_ctx, bucket, bucket_info, NULL, NULL); + if (ret < 0) { + ldout(cct, 0) << "ERROR: " << __func__ << "() get_bucket_instance_info(bucket=" << bucket << ") returned ret=" << ret << dendl; + return ret; + } + + RGWRados::Bucket bop(this, bucket_info); RGWRados::Bucket::UpdateIndex index_op(&bop, obj, NULL); int r = index_op.complete_del(-1 /* pool */, 0, NULL); @@ -8069,8 +8079,17 @@ int RGWRados::set_attrs(void *ctx, rgw_obj& obj, if (!op.size()) return 0; + RGWObjectCtx obj_ctx(this); + + RGWBucketInfo bucket_info; + int ret = get_bucket_instance_info(obj_ctx, bucket, bucket_info, NULL, NULL); + if (ret < 0) { + ldout(cct, 0) << "ERROR: " << __func__ << "() get_bucket_instance_info(bucket=" << bucket << ") returned ret=" << ret << dendl; + return ret; + } + bufferlist bl; - RGWRados::Bucket bop(this, bucket); + RGWRados::Bucket bop(this, bucket_info); RGWRados::Bucket::UpdateIndex index_op(&bop, obj, state); if (state) { @@ -8321,6 +8340,9 @@ int RGWRados::SystemObject::Read::stat(RGWObjVersionTracker *objv_tracker) int RGWRados::Bucket::UpdateIndex::prepare(RGWModifyOp op) { + if (blind) { + return 0; + } RGWRados *store = target->get_store(); BucketShard *bs; int ret = get_bucket_shard(&bs); @@ -8351,6 +8373,9 @@ int RGWRados::Bucket::UpdateIndex::complete(int64_t poolid, uint64_t epoch, uint utime_t& ut, string& etag, string& content_type, bufferlist *acl_bl, RGWObjCategory category, list *remove_objs) { + if (blind) { + return 0; + } RGWRados *store = target->get_store(); BucketShard *bs; int ret = get_bucket_shard(&bs); @@ -8383,6 +8408,9 @@ int RGWRados::Bucket::UpdateIndex::complete(int64_t poolid, uint64_t epoch, uint int RGWRados::Bucket::UpdateIndex::complete_del(int64_t poolid, uint64_t epoch, list *remove_objs) { + if (blind) { + return 0; + } RGWRados *store = target->get_store(); BucketShard *bs; int ret = get_bucket_shard(&bs); @@ -8396,6 +8424,9 @@ int RGWRados::Bucket::UpdateIndex::complete_del(int64_t poolid, uint64_t epoch, int RGWRados::Bucket::UpdateIndex::cancel() { + if (blind) { + return 0; + } RGWRados *store = target->get_store(); BucketShard *bs; int ret = get_bucket_shard(&bs); diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 9d45fa75ebe68..18f1b917f3e72 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -2298,14 +2298,16 @@ public: class Bucket { RGWRados *store; + RGWBucketInfo bucket_info; rgw_bucket& bucket; int shard_id; public: - Bucket(RGWRados *_store, rgw_bucket& _bucket) : store(_store), bucket(_bucket), shard_id(RGW_NO_SHARD) {} - + Bucket(RGWRados *_store, RGWBucketInfo& _bucket_info) : store(_store), bucket_info(_bucket_info), bucket(bucket_info.bucket), + shard_id(RGW_NO_SHARD) {} RGWRados *get_store() { return store; } rgw_bucket& get_bucket() { return bucket; } + RGWBucketInfo& get_bucket_info() { return bucket_info; } int get_shard_id() { return shard_id; } void set_shard_id(int id) { @@ -2320,10 +2322,13 @@ public: uint16_t bilog_flags; BucketShard bs; bool bs_initialized; + bool blind; public: UpdateIndex(RGWRados::Bucket *_target, rgw_obj& _obj, RGWObjState *_state) : target(_target), obj(_obj), obj_state(_state), bilog_flags(0), - bs(target->get_store()), bs_initialized(false) {} + bs(target->get_store()), bs_initialized(false) { + blind = (target->get_bucket_info().index_type == RGWBIType_Indexless); + } int get_bucket_shard(BucketShard **pbs) { if (!bs_initialized) { -- 2.39.5