snprintf(buf, sizeof(buf), ".%d", i);
bucket->name = string("buck") + buf;
- bucket->data_pool = string(".data.pool") + buf;
- bucket->index_pool = string(".index.pool") + buf;
bucket->marker = string("mark") + buf;
bucket->bucket_id = string("bucket.id") + buf;
}
void cls_user_bucket::dump(Formatter *f) const
{
encode_json("name", name, f);
- encode_json("data_pool", data_pool,f);
- encode_json("data_extra_pool", data_extra_pool,f);
- encode_json("index_pool", index_pool,f);
encode_json("marker", marker,f);
encode_json("bucket_id", bucket_id,f);
}
*/
struct cls_user_bucket {
std::string name;
- std::string data_pool;
- std::string index_pool;
std::string marker;
std::string bucket_id;
- std::string data_extra_pool;
+ std::string placement_id;
+ struct {
+ std::string data_pool;
+ std::string index_pool;
+ std::string data_extra_pool;
+ } explicit_placement;
void encode(bufferlist& bl) const {
- ENCODE_START(7, 3, bl);
+ ENCODE_START(8, 8, bl);
::encode(name, bl);
- ::encode(data_pool, bl);
::encode(marker, bl);
::encode(bucket_id, bl);
- ::encode(index_pool, bl);
- ::encode(data_extra_pool, bl);
+ ::encode(placement_id, bl);
+ if (placement_id.empty()) {
+ ::encode(explicit_placement.data_pool, bl);
+ ::encode(explicit_placement.index_pool, bl);
+ ::encode(explicit_placement.data_extra_pool, bl);
+ }
ENCODE_FINISH(bl);
}
void decode(bufferlist::iterator& bl) {
- DECODE_START_LEGACY_COMPAT_LEN(7, 3, 3, bl);
+ DECODE_START_LEGACY_COMPAT_LEN(8, 3, 3, bl);
::decode(name, bl);
- ::decode(data_pool, bl);
+ if (struct_v < 8) {
+ ::decode(explicit_placement.data_pool, bl);
+ }
if (struct_v >= 2) {
::decode(marker, bl);
if (struct_v <= 3) {
::decode(bucket_id, bl);
}
}
- if (struct_v >= 5) {
- ::decode(index_pool, bl);
+ if (struct_v < 8) {
+ if (struct_v >= 5) {
+ ::decode(explicit_placement.index_pool, bl);
+ } else {
+ explicit_placement.index_pool = explicit_placement.data_pool;
+ }
+ if (struct_v >= 7) {
+ ::decode(explicit_placement.data_extra_pool, bl);
+ }
} else {
- index_pool = data_pool;
- }
- if (struct_v >= 7) {
- ::decode(data_extra_pool, bl);
+ ::decode(placement_id, bl);
+ if (placement_id.empty()) {
+ ::decode(explicit_placement.data_pool, bl);
+ ::decode(explicit_placement.index_pool, bl);
+ ::decode(explicit_placement.data_extra_pool, bl);
+ }
}
DECODE_FINISH(bl);
}
map<RGWObjCategory, RGWStorageStats> stats;
string bucket_ver, master_ver;
string max_marker;
- int ret = store->get_bucket_stats(bucket, shard_id, &bucket_ver, &master_ver, stats, &max_marker);
+ int ret = store->get_bucket_stats(bucket_info, shard_id, &bucket_ver, &master_ver, stats, &max_marker);
if (ret < 0) {
cerr << "error getting bucket stats ret=" << ret << std::endl;
return ret;
}
formatter->open_object_section("stats");
formatter->dump_string("bucket", bucket.name);
- ::encode_json("placement", bucket.placement, formatter);
+ ::encode_json("explicit_placement", bucket.explicit_placement, formatter);
formatter->dump_string("id", bucket.bucket_id);
formatter->dump_string("marker", bucket.marker);
formatter->open_array_section("objects");
while (is_truncated) {
map<string, RGWObjEnt> result;
- int r = store->cls_bucket_list(bucket, RGW_NO_SHARD, marker, prefix, 1000, true,
+ int r = store->cls_bucket_list(bucket_info, RGW_NO_SHARD, marker, prefix, 1000, true,
result, &is_truncated, &marker,
bucket_object_check_filter);
list<rgw_obj_key> oid_list;
rgw_obj_key key(object, object_version);
oid_list.push_back(key);
- ret = store->remove_objs_from_index(bucket, oid_list);
+ ret = store->remove_objs_from_index(bucket_info, oid_list);
if (ret < 0) {
cerr << "ERROR: remove_obj_from_index() returned error: " << cpp_strerror(-ret) << std::endl;
return 1;
do {
list<rgw_bi_log_entry> entries;
- ret = store->list_bi_log_entries(bucket, shard_id, marker, max_entries - count, entries, &truncated);
+ ret = store->list_bi_log_entries(bucket_info, shard_id, marker, max_entries - count, entries, &truncated);
if (ret < 0) {
cerr << "ERROR: list_bi_log_entries(): " << cpp_strerror(-ret) << std::endl;
return -ret;
cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
return -ret;
}
- ret = store->trim_bi_log_entries(bucket, shard_id, start_marker, end_marker);
+ ret = store->trim_bi_log_entries(bucket_info, shard_id, start_marker, end_marker);
if (ret < 0) {
cerr << "ERROR: trim_bi_log_entries(): " << cpp_strerror(-ret) << std::endl;
return -ret;
return -ret;
}
map<int, string> markers;
- ret = store->get_bi_log_status(bucket, shard_id, markers);
+ ret = store->get_bi_log_status(bucket_info, shard_id, markers);
if (ret < 0) {
cerr << "ERROR: trim_bi_log_entries(): " << cpp_strerror(-ret) << std::endl;
return -ret;
return 0;
}
-int rgw_bucket_sync_user_stats(RGWRados *store, const rgw_user& user_id, rgw_bucket& bucket)
+int rgw_bucket_sync_user_stats(RGWRados *store, const rgw_user& user_id, const RGWBucketInfo& bucket_info)
{
string buckets_obj_id;
rgw_get_buckets_obj(user_id, buckets_obj_id);
rgw_raw_obj obj(store->get_zone_params().user_uid_pool, buckets_obj_id);
- return store->cls_user_sync_bucket_stats(obj, bucket);
+ return store->cls_user_sync_bucket_stats(obj, bucket_info);
}
int rgw_bucket_sync_user_stats(RGWRados *store, const string& tenant_name, const string& bucket_name)
return ret;
}
- ret = rgw_bucket_sync_user_stats(store, bucket_info.owner, bucket_info.bucket);
+ ret = rgw_bucket_sync_user_stats(store, bucket_info.owner, bucket_info);
if (ret < 0) {
ldout(store->ctx(), 0) << "ERROR: could not sync user stats for bucket " << bucket_name << ": ret=" << ret << dendl;
return ret;
if (actual_bucket.name.compare(bucket.name) != 0 ||
actual_bucket.tenant.compare(bucket.tenant) != 0 ||
- actual_bucket.placement.compare(bucket.placement) != 0 ||
actual_bucket.marker.compare(bucket.marker) != 0 ||
actual_bucket.bucket_id.compare(bucket.bucket_id) != 0) {
cout << "bucket info mismatch: expected " << actual_bucket << " got " << bucket << std::endl;
string bucket_ver, master_ver;
- ret = store->get_bucket_stats(bucket, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, NULL);
+ ret = store->get_bucket_info(obj_ctx, bucket.tenant, bucket.name, info, NULL);
if (ret < 0)
return ret;
- ret = store->get_bucket_info(obj_ctx, bucket.tenant, bucket.name, info, NULL);
+ ret = store->get_bucket_stats(info, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, NULL);
if (ret < 0)
return ret;
}
- ret = rgw_bucket_sync_user_stats(store, bucket.tenant, bucket.name);
+ ret = rgw_bucket_sync_user_stats(store, bucket.tenant, info);
if ( ret < 0) {
dout(1) << "WARNING: failed sync user stats before bucket delete. ret=" << ret << dendl;
}
RGWObjVersionTracker objv_tracker;
- ret = store->delete_bucket(bucket, objv_tracker);
+ ret = store->delete_bucket(info, objv_tracker);
if (ret < 0) {
lderr(store->ctx()) << "ERROR: could not remove bucket " << bucket.name << dendl;
return ret;
string bucket_ver, master_ver;
- ret = store->get_bucket_stats(bucket, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, NULL);
+ ret = store->get_bucket_info(obj_ctx, bucket.tenant, bucket.name, info, NULL);
if (ret < 0)
return ret;
- ret = store->get_bucket_info(obj_ctx, bucket.tenant, bucket.name, info, NULL);
+ ret = store->get_bucket_stats(info, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, NULL);
if (ret < 0)
return ret;
RGWObjManifest::obj_iterator miter = manifest.obj_begin();
rgw_obj head_obj = manifest.get_obj();
rgw_raw_obj raw_head_obj;
- RGWRados::obj_to_raw(head_obj, &raw_head_obj);
+ store->obj_to_raw(head_obj, &raw_head_obj);
for (; miter != manifest.obj_end() && max_aio--; ++miter) {
max_aio = concurrent_max;
}
- rgw_raw_obj last_obj = miter.get_location();
+ rgw_raw_obj last_obj = miter.get_location().get_raw_obj(store);
if (last_obj == raw_head_obj) {
// have the head obj deleted at the end
continue;
}
} // for all shadow objs
- ret = store->delete_obj_aio(head_obj, bucket, info, astate, handles, keep_index_consistent);
+ ret = store->delete_obj_aio(head_obj, info, astate, handles, keep_index_consistent);
if (ret < 0) {
lderr(store->ctx()) << "ERROR: delete obj aio failed with " << ret << dendl;
return ret;
return ret;
}
- ret = rgw_bucket_sync_user_stats(store, bucket.tenant, bucket.name);
+ ret = rgw_bucket_sync_user_stats(store, bucket.tenant, info);
if (ret < 0) {
dout(1) << "WARNING: failed sync user stats before bucket delete. ret=" << ret << dendl;
}
return 0;
if (fix_index) {
- int r = store->remove_objs_from_index(bucket, objs_to_unlink);
+ int r = store->remove_objs_from_index(bucket_info, objs_to_unlink);
if (r < 0) {
set_err_msg(err_msg, "ERROR: remove_obj_from_index() returned error: " +
cpp_strerror(-r));
{
bool fix_index = op_state.will_fix_index();
+
rgw_bucket bucket = op_state.get_bucket();
if (!fix_index) {
return -EINVAL;
}
-/*
- dout(0) << "Checking objects, decreasing bucket 2-phase commit timeout.\n"\
- << "** Note that timeout will reset only when operation completes successfully **" << dendl;
-*/
- store->cls_obj_set_bucket_tag_timeout(bucket, BUCKET_TAG_TIMEOUT);
+ store->cls_obj_set_bucket_tag_timeout(bucket_info, BUCKET_TAG_TIMEOUT);
string prefix;
rgw_obj_key marker;
while (is_truncated) {
map<string, RGWObjEnt> result;
- int r = store->cls_bucket_list(bucket, RGW_NO_SHARD, marker, prefix, 1000, true,
+ int r = store->cls_bucket_list(bucket_info, RGW_NO_SHARD, marker, prefix, 1000, true,
result, &is_truncated, &marker,
bucket_object_check_filter);
if (r == -ENOENT) {
formatter->close_section();
- store->cls_obj_set_bucket_tag_timeout(bucket, 0);
+ store->cls_obj_set_bucket_tag_timeout(bucket_info, 0);
return 0;
}
rgw_bucket bucket = op_state.get_bucket();
bool fix_index = op_state.will_fix_index();
- int r = store->bucket_check_index(bucket, &existing_stats, &calculated_stats);
+ int r = store->bucket_check_index(bucket_info, &existing_stats, &calculated_stats);
if (r < 0) {
set_err_msg(err_msg, "failed to check index error=" + cpp_strerror(-r));
return r;
}
if (fix_index) {
- r = store->bucket_rebuild_index(bucket);
+ r = store->bucket_rebuild_index(bucket_info);
if (r < 0) {
set_err_msg(err_msg, "failed to rebuild index err=" + cpp_strerror(-r));
return r;
static int bucket_stats(RGWRados *store, const std::string& tenant_name, std::string& bucket_name, Formatter *formatter)
{
RGWBucketInfo bucket_info;
- rgw_bucket bucket;
map<RGWObjCategory, RGWStorageStats> stats;
real_time mtime;
if (r < 0)
return r;
- bucket = bucket_info.bucket;
+ rgw_bucket& bucket = bucket_info.bucket;
string bucket_ver, master_ver;
string max_marker;
- int ret = store->get_bucket_stats(bucket, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, &max_marker);
+ int ret = store->get_bucket_stats(bucket_info, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, &max_marker);
if (ret < 0) {
cerr << "error getting bucket stats ret=" << ret << std::endl;
return ret;
formatter->open_object_section("stats");
formatter->dump_string("bucket", bucket.name);
- encode_json("placement", bucket.placement, formatter);
formatter->dump_string("id", bucket.bucket_id);
formatter->dump_string("marker", bucket.marker);
::encode_json("owner", bucket_info.owner, formatter);
string bucket_name;
parse_bucket(key, tenant_name, bucket_name);
- rgw_bucket bucket;
RGWZonePlacementInfo rule_info;
- ret = store->set_bucket_location_by_rule(bci.info.placement_rule,
- tenant_name, bucket_name, bucket, &rule_info);
+ bci.info.bucket.name = bucket_name;
+ bci.info.bucket.tenant = tenant_name;
+ ret = store->select_bucket_location_by_rule(bci.info.placement_rule, bci.info.bucket, &rule_info);
if (ret < 0) {
ldout(store->ctx(), 0) << "ERROR: select_bucket_placement() returned " << ret << dendl;
return ret;
}
- bci.info.bucket.tenant = bucket.tenant;
- bci.info.bucket.placement = bucket.placement;
bci.info.index_type = rule_info.index_type;
} else {
- /* existing bucket, keep its placement pools */
- bci.info.bucket.placement = old_bci.info.bucket.placement;
- bci.info.index_type = old_bci.info.index_type;
+ /* existing bucket, keep its placement */
+ bci.info.bucket.explicit_placement = old_bci.info.bucket.explicit_placement;
+ bci.info.placement_rule = old_bci.info.placement_rule;
}
// are we actually going to perform this put, or is it too old?
objv_tracker = bci.info.objv_tracker;
- ret = store->init_bucket_index(bci.info.bucket, bci.info.num_shards);
+ ret = store->init_bucket_index(bci.info, bci.info.num_shards);
if (ret < 0)
return ret;
const string& bucket_name,
RGWObjVersionTracker& objv_tracker);
-extern int rgw_bucket_sync_user_stats(RGWRados *store, const rgw_user& user_id, rgw_bucket& bucket);
+extern int rgw_bucket_sync_user_stats(RGWRados *store, const rgw_user& user_id, const RGWBucketInfo& bucket_info);
extern int rgw_bucket_sync_user_stats(RGWRados *store, const string& tenant_name, const string& bucket_name);
extern void rgw_make_bucket_entry_name(const string& tenant_name,
rgw_obj old_obj;
::decode(old_obj, bl);
- RGWRados::obj_to_raw(old_obj, this);
+ get_obj_bucket_and_oid_loc(old_obj, oid, loc);
+ pool = old_obj.get_explicit_data_pool();
}
std::string rgw_bucket::get_key(char tenant_delim, char id_delim) const
std::string name;
std::string marker;
std::string bucket_id;
- rgw_data_placement_target placement;
+ rgw_data_placement_target explicit_placement;
std::string oid; /*
* runtime in-memory only info. If not empty, points to the bucket instance object
name(b.name),
marker(b.marker),
bucket_id(b.bucket_id),
- placement(b.data_pool, b.data_extra_pool, b.index_pool) {}
+ explicit_placement(b.explicit_placement.data_pool,
+ b.explicit_placement.data_extra_pool,
+ b.explicit_placement.index_pool) {}
- void convert(cls_user_bucket *b) {
+ void convert(cls_user_bucket *b) const {
b->name = name;
- b->data_pool = placement.data_pool.to_str();
- b->data_extra_pool = placement.data_extra_pool.to_str();
- b->index_pool = placement.index_pool.to_str();
b->marker = marker;
b->bucket_id = bucket_id;
+ b->explicit_placement.data_pool = explicit_placement.data_pool.to_str();
+ b->explicit_placement.data_extra_pool = explicit_placement.data_extra_pool.to_str();
+ b->explicit_placement.index_pool = explicit_placement.index_pool.to_str();
}
void encode(bufferlist& bl) const {
- ENCODE_START(9, 3, bl);
+ ENCODE_START(10, 10, bl);
::encode(name, bl);
- ::encode(placement.data_pool.name, bl);
::encode(marker, bl);
::encode(bucket_id, bl);
- ::encode(placement.index_pool.name, bl);
- ::encode(placement.data_extra_pool.name, bl);
::encode(tenant, bl);
+ bool encode_explicit = !explicit_placement.data_pool.empty();
+ ::encode(encode_explicit, bl);
+ if (encode_explicit) {
+ ::encode(explicit_placement.data_pool, bl);
+ ::encode(explicit_placement.data_extra_pool, bl);
+ ::encode(explicit_placement.index_pool, bl);
+ }
ENCODE_FINISH(bl);
}
void decode(bufferlist::iterator& bl) {
- DECODE_START_LEGACY_COMPAT_LEN(9, 3, 3, bl);
+ DECODE_START_LEGACY_COMPAT_LEN(10, 3, 3, bl);
::decode(name, bl);
- ::decode(placement.data_pool.name, bl);
+ if (struct_v < 10) {
+ ::decode(explicit_placement.data_pool.name, bl);
+ }
if (struct_v >= 2) {
::decode(marker, bl);
if (struct_v <= 3) {
::decode(bucket_id, bl);
}
}
- if (struct_v >= 5) {
- ::decode(placement.index_pool.name, bl);
- } else {
- placement.index_pool = placement.data_pool;
- }
- if (struct_v >= 7) {
- ::decode(placement.data_extra_pool.name, bl);
+ if (struct_v < 10) {
+ if (struct_v >= 5) {
+ ::decode(explicit_placement.index_pool.name, bl);
+ } else {
+ explicit_placement.index_pool = explicit_placement.data_pool;
+ }
+ if (struct_v >= 7) {
+ ::decode(explicit_placement.data_extra_pool.name, bl);
+ }
}
if (struct_v >= 8) {
::decode(tenant, bl);
}
+ if (struct_v >= 10) {
+ bool decode_explicit = !explicit_placement.data_pool.empty();
+ ::decode(decode_explicit, bl);
+ if (decode_explicit) {
+ ::decode(explicit_placement.data_pool, bl);
+ ::decode(explicit_placement.data_extra_pool, bl);
+ ::decode(explicit_placement.index_pool, bl);
+ }
+ }
DECODE_FINISH(bl);
}
char id_delim = ':') const;
const rgw_pool& get_data_extra_pool() const {
- return placement.get_data_extra_pool();
+ return explicit_placement.get_data_extra_pool();
}
void dump(Formatter *f) const;
count(e.count) {
}
- void convert(cls_user_bucket_entry *b) {
+ void convert(cls_user_bucket_entry *b) const {
bucket.convert(&b->bucket);
b->size = size;
b->size_rounded = size_rounded;
const std::string& get_loc() const { return loc; }
const std::string& get_instance() const { return instance; }
rgw_bucket bucket;
+ std::string placement_id;
std::string ns;
bool in_extra_data; /* in-memory only member, does not serialize */
return in_extra_data;
}
- const rgw_pool& get_data_pool() const {
- if (!in_extra_data) {
- return bucket.placement.data_pool;
- } else {
- return bucket.placement.data_extra_pool;
- }
- }
-
void encode(bufferlist& bl) const {
ENCODE_START(5, 3, bl);
::encode(bucket.name, bl);
return (r < 0);
}
+
+ const rgw_pool& get_explicit_data_pool() {
+ if (!in_extra_data || bucket.explicit_placement.data_extra_pool.empty()) {
+ return bucket.explicit_placement.data_pool;
+ }
+ return bucket.explicit_placement.data_extra_pool;
+ }
};
WRITE_CLASS_ENCODER(rgw_obj)
b->name = n;
b->marker = m;
b->bucket_id = id;
- b->placement.data_pool = rgw_pool(dp);
- b->placement.index_pool = rgw_pool(ip);
+ b->explicit_placement.data_pool = rgw_pool(dp);
+ b->explicit_placement.index_pool = rgw_pool(ip);
}
void RGWObjManifestPart::generate_test_instances(std::list<RGWObjManifestPart*>& o)
void RGWObjManifest::obj_iterator::update_location()
{
if (manifest->explicit_objs) {
- RGWRados::obj_to_raw(explicit_iter->second.loc, &location);
+ location = explicit_iter->second.loc;
return;
}
- const rgw_raw_obj& head = manifest->get_head();
-
if (ofs < manifest->get_head_size()) {
- location = head;
+ location = manifest->get_obj();
return;
}
o.push_back(new RGWObjManifest);
}
-void RGWObjManifest::get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, string *override_prefix, rgw_raw_obj *location)
+void RGWObjManifest::get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, string *override_prefix, rgw_obj_select *location)
{
string oid;
if (!override_prefix || override_prefix->empty()) {
if (!cur_part_id) {
if (ofs < max_head_size) {
- *location = head_obj;
+ *location = obj;
return;
} else {
char buf[16];
// to get the right shadow object location
loc.set_instance(tail_instance);
- RGWRados::obj_to_raw(loc, location);
+ *location = loc;
}
f->close_section();
f->dump_unsigned("obj_size", obj_size);
::encode_json("explicit_objs", explicit_objs, f);
- ::encode_json("head_obj", head_obj, f);
::encode_json("head_size", head_size, f);
::encode_json("max_head_size", max_head_size, f);
::encode_json("prefix", prefix, f);
encode_json("marker", marker, f);
encode_json("bucket_id", bucket_id, f);
encode_json("tenant", tenant, f);
- encode_json("placement", placement, f);
+ encode_json("explicit_placement", explicit_placement, f);
}
void rgw_bucket::decode_json(JSONObj *obj) {
JSONDecoder::decode_json("marker", marker, obj);
JSONDecoder::decode_json("bucket_id", bucket_id, obj);
JSONDecoder::decode_json("tenant", tenant, obj);
- JSONDecoder::decode_json("placement", placement, obj);
- if (placement.data_pool.empty()) {
+ JSONDecoder::decode_json("explicit_placement", explicit_placement, obj);
+ if (explicit_placement.data_pool.empty()) {
/* decoding old format */
- JSONDecoder::decode_json("pool", placement.data_pool, obj);
- JSONDecoder::decode_json("data_extra_pool", placement.data_extra_pool, obj);
- JSONDecoder::decode_json("index_pool", placement.index_pool, obj);
+ JSONDecoder::decode_json("pool", explicit_placement.data_pool, obj);
+ JSONDecoder::decode_json("data_extra_pool", explicit_placement.data_extra_pool, obj);
+ JSONDecoder::decode_json("index_pool", explicit_placement.index_pool, obj);
}
}
if (s->bucket_exists) {
string selected_placement_rule;
rgw_bucket bucket;
+ bucket.tenant = s->bucket_tenant;
+ bucket.name = s->bucket_name;
op_ret = store->select_bucket_placement(*(s->user), zonegroup_id,
placement_rule,
- s->bucket_tenant, s->bucket_name,
bucket, &selected_placement_rule, nullptr);
if (selected_placement_rule != s->bucket_info.placement_rule) {
op_ret = -EEXIST;
}
}
- op_ret = rgw_bucket_sync_user_stats(store, s->user->user_id, s->bucket);
+ op_ret = rgw_bucket_sync_user_stats(store, s->user->user_id, s->bucket_info);
if ( op_ret < 0) {
ldout(s->cct, 1) << "WARNING: failed to sync user stats before bucket delete: op_ret= " << op_ret << dendl;
}
}
}
- op_ret = store->delete_bucket(s->bucket, ot);
+ op_ret = store->delete_bucket(s->bucket_info, ot);
if (op_ret == -ECANCELED) {
// lost a race, either with mdlog sync or another delete bucket operation.
int RGWPutObjProcessor_Multipart::prepare(RGWRados *store, string *oid_rand)
{
- int r = prepare_init(store, NULL);
- if (r < 0) {
- return r;
- }
-
string oid = obj_str;
upload_id = s->info.args.get("uploadId");
if (!oid_rand) {
manifest.set_multipart_part_rule(store->ctx()->_conf->rgw_obj_stripe_size, num);
- r = manifest_gen.create_begin(store->ctx(), &manifest, bucket, target_obj);
+ int r = manifest_gen.create_begin(store->ctx(), &manifest, bucket, target_obj);
if (r < 0) {
return r;
}
- cur_obj = manifest_gen.get_cur_obj();
+ cur_obj = manifest_gen.get_cur_obj(store);
rgw_raw_obj_to_obj(bucket, cur_obj, &head_obj);
head_obj.index_hash_source = obj_str;
+ r = prepare_init(store, NULL);
+ if (r < 0) {
+ return r;
+ }
+
return 0;
}
op_ret = -ERR_INVALID_PART;
return;
} else {
- manifest.append(obj_part.manifest);
+ manifest.append(obj_part.manifest, store);
}
if (obj_part.cs_info.compression_type != "none") {
RGWObjManifest::obj_iterator oiter = obj_part.manifest.obj_begin();
if (oiter != obj_part.manifest.obj_end()) {
rgw_obj head;
- rgw_raw_obj raw_head = oiter.get_location();
+ rgw_raw_obj raw_head = oiter.get_location().get_raw_obj(store);
rgw_raw_obj_to_obj(s->bucket, raw_head, &head);
rgw_obj_key key;
RGWObjVersionTracker ot;
ot.read_version = binfo.ep_objv;
- ret = store->delete_bucket(binfo.bucket, ot);
+ ret = store->delete_bucket(binfo, ot);
if (0 == ret) {
ret = rgw_unlink_bucket(store, binfo.owner, binfo.bucket.tenant,
binfo.bucket.name, false);
RGWObjManifest::obj_iterator miter;
for (miter = manifest.obj_begin(); miter != manifest.obj_end(); ++miter) {
- const rgw_raw_obj& loc = miter.get_location();
+ const rgw_raw_obj& loc = miter.get_location().get_raw_obj(store);
string s = loc.oid;
obj_oids.insert(obj_fingerprint(s));
}
int BucketAsyncRefreshHandler::init_fetch()
{
+ RGWBucketInfo bucket_info;
+
+ RGWObjectCtx obj_ctx(store);
+
+ int r = store->get_bucket_instance_info(obj_ctx, bucket, bucket_info, NULL, NULL);
+ if (r < 0) {
+ ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket << " r=" << r << dendl;
+ return r;
+ }
+
ldout(store->ctx(), 20) << "initiating async quota refresh for bucket=" << bucket << dendl;
- int r = store->get_bucket_stats_async(bucket, RGW_NO_SHARD, this);
+ r = store->get_bucket_stats_async(bucket_info, RGW_NO_SHARD, this);
if (r < 0) {
ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket.name << dendl;
{
RGWBucketInfo bucket_info;
+ RGWObjectCtx obj_ctx(store);
+
+ int r = store->get_bucket_instance_info(obj_ctx, bucket, bucket_info, NULL, NULL);
+ if (r < 0) {
+ ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket << " r=" << r << dendl;
+ return r;
+ }
+
string bucket_ver;
string master_ver;
map<RGWObjCategory, RGWStorageStats> bucket_stats;
- int r = store->get_bucket_stats(bucket, RGW_NO_SHARD, &bucket_ver,
+ r = store->get_bucket_stats(bucket_info, RGW_NO_SHARD, &bucket_ver,
&master_ver, bucket_stats, nullptr);
if (r < 0) {
- ldout(store->ctx(), 0) << "could not get bucket info for bucket="
+ ldout(store->ctx(), 0) << "could not get bucket stats for bucket="
<< bucket.name << dendl;
return r;
}
int RGWUserStatsCache::sync_bucket(const rgw_user& user, rgw_bucket& bucket)
{
- int r = rgw_bucket_sync_user_stats(store, user, bucket);
+ RGWBucketInfo bucket_info;
+
+ RGWObjectCtx obj_ctx(store);
+
+ int r = store->get_bucket_instance_info(obj_ctx, bucket, bucket_info, NULL, NULL);
+ if (r < 0) {
+ ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket << " r=" << r << dendl;
+ return r;
+ }
+
+ r = rgw_bucket_sync_user_stats(store, user, bucket_info);
if (r < 0) {
ldout(store->ctx(), 0) << "ERROR: rgw_bucket_sync_user_stats() for user=" << user << ", bucket=" << bucket << " returned " << r << dendl;
return r;
#define dout_subsys ceph_subsys_rgw
+
+static bool rgw_get_obj_data_pool(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params,
+ const rgw_obj& obj, rgw_pool *pool)
+{
+ if (!zone_params.get_data_pool(obj, pool)) {
+ RGWZonePlacementInfo placement;
+ if (!zone_params.get_placement(zonegroup.default_placement, &placement)) {
+ return false;
+ }
+
+ if (!obj.in_extra_data) {
+ *pool = placement.data_pool;
+ } else {
+ *pool = placement.data_extra_pool;
+ }
+ }
+
+ return true;
+}
+
+bool rgw_obj_to_raw(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params,
+ const rgw_obj& obj, rgw_raw_obj *raw_obj)
+{
+ get_obj_bucket_and_oid_loc(obj, raw_obj->oid, raw_obj->loc);
+
+ return rgw_get_obj_data_pool(zonegroup, zone_params, obj, &raw_obj->pool);
+}
+
+rgw_raw_obj rgw_obj_select::get_raw_obj(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params) const
+{
+ if (!is_raw) {
+ rgw_raw_obj r;
+ rgw_obj_to_raw(zonegroup, zone_params, obj, &r);
+ return r;
+ }
+ return raw_obj;
+}
+
+rgw_raw_obj rgw_obj_select::get_raw_obj(RGWRados *store) const
+{
+ if (!is_raw) {
+ rgw_raw_obj r;
+ store->obj_to_raw(obj, &r);
+ return r;
+ }
+ return raw_obj;
+}
+
void RGWDefaultZoneGroupInfo::dump(Formatter *f) const {
encode_json("default_zonegroup", default_zonegroup, f);
}
return iter;
}
-int RGWObjManifest::append(RGWObjManifest& m)
+int RGWObjManifest::append(RGWObjManifest& m, RGWZoneGroup& zonegroup, RGWZoneParams& zone_params)
{
if (explicit_objs || m.explicit_objs) {
- return append_explicit(m);
+ return append_explicit(m, zonegroup, zone_params);
}
if (rules.empty()) {
map<uint64_t, RGWObjManifestRule>::iterator miter = m.rules.begin();
if (miter == m.rules.end()) {
- return append_explicit(m);
+ return append_explicit(m, zonegroup, zone_params);
}
for (; miter != m.rules.end(); ++miter) {
return 0;
}
+int RGWObjManifest::append(RGWObjManifest& m, RGWRados *store)
+{
+ return append(m, store->get_zonegroup(), store->get_zone_params());
+}
+
void RGWObjManifest::append_rules(RGWObjManifest& m, map<uint64_t, RGWObjManifestRule>::iterator& miter,
string *override_prefix)
{
}
}
-void RGWObjManifest::convert_to_explicit()
+void RGWObjManifest::convert_to_explicit(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params)
{
if (explicit_objs) {
return;
while (iter != obj_end()) {
RGWObjManifestPart& part = objs[iter.get_stripe_ofs()];
- const rgw_raw_obj& raw_loc = iter.get_location();
+ const rgw_obj_select& os = iter.get_location();
+ const rgw_raw_obj& raw_loc = os.get_raw_obj(zonegroup, zone_params);
part.loc_ofs = 0;
uint64_t ofs = iter.get_stripe_ofs();
prefix.clear();
}
-int RGWObjManifest::append_explicit(RGWObjManifest& m)
+int RGWObjManifest::append_explicit(RGWObjManifest& m, const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params)
{
if (!explicit_objs) {
- convert_to_explicit();
+ convert_to_explicit(zonegroup, zone_params);
}
if (!m.explicit_objs) {
- m.convert_to_explicit();
+ m.convert_to_explicit(zonegroup, zone_params);
}
map<uint64_t, RGWObjManifestPart>::iterator iter;
uint64_t base = obj_size;
rgw_raw_obj raw_head;
if (!head_obj.empty()) {
- RGWRados::obj_to_raw(head_obj, &raw_head);
+ store->obj_to_raw(head_obj, &raw_head);
}
/**
{
RGWPutObjProcessor_Aio::prepare(store, oid_rand);
- int r = store->get_max_chunk_size(bucket, &max_chunk_size);
+ int r = store->get_max_chunk_size(head_obj, &max_chunk_size);
if (r < 0) {
return r;
}
int RGWPutObjProcessor_Atomic::prepare(RGWRados *store, string *oid_rand)
{
+ head_obj.init(bucket, obj_str);
+
int r = prepare_init(store, oid_rand);
if (r < 0) {
return r;
}
- head_obj.init(bucket, obj_str);
if (!version_id.empty()) {
head_obj.set_instance(version_id);
}
cur_part_ofs = ofs;
next_part_ofs = ofs + manifest_gen.cur_stripe_max_size();
- cur_obj = manifest_gen.get_cur_obj();
+ cur_obj = manifest_gen.get_cur_obj(store);
return 0;
}
return 0;
}
-int RGWRados::get_required_alignment(const rgw_bucket& bucket, uint64_t *alignment)
-{
- return get_required_alignment(bucket.placement.data_pool, alignment);
-}
-
int RGWRados::get_max_chunk_size(const rgw_pool& pool, uint64_t *max_chunk_size)
{
uint64_t alignment;
return 0;
}
-int RGWRados::get_max_chunk_size(const rgw_bucket& bucket, uint64_t *max_chunk_size)
+int RGWRados::get_max_chunk_size(const rgw_obj& obj, uint64_t *max_chunk_size)
{
- return get_max_chunk_size(bucket.placement.data_pool, max_chunk_size);
+ rgw_pool pool;
+ if (!get_obj_data_pool(obj, &pool)) {
+ ldout(cct, 0) << "ERROR: failed to get data pool for object " << obj << dendl;
+ return -EIO;
+ }
+ return get_max_chunk_size(pool, max_chunk_size);
}
void RGWRados::finalize()
}
}
-int RGWRados::open_bucket_index_ctx(rgw_bucket& bucket, librados::IoCtx& index_ctx)
+int RGWRados::open_bucket_index_ctx(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx)
{
- int r = open_pool_ctx(bucket.placement.index_pool, index_ctx);
+ const string *rule = &bucket_info.placement_rule;
+ if (rule->empty()) {
+ rule = &zonegroup.default_placement;
+ }
+ auto iter = zone_params.placement_pools.find(*rule);
+ if (iter == zone_params.placement_pools.end()) {
+ ldout(cct, 0) << "could not find placement rule " << *rule << " within zonegroup " << dendl;
+ return -EINVAL;
+ }
+
+ int r = open_pool_ctx(iter->second.index_pool, index_ctx);
if (r < 0)
return r;
{
RGWRados *store = target->get_store();
CephContext *cct = store->ctx();
- rgw_bucket& bucket = target->get_bucket();
int shard_id = target->get_shard_id();
int count = 0;
ldout(cct, 20) << "setting cur_marker=" << cur_marker.name << "[" << cur_marker.instance << "]" << dendl;
}
std::map<string, RGWObjEnt> ent_map;
- int r = store->cls_bucket_list(bucket, shard_id, cur_marker, cur_prefix, read_ahead + 1 - count, params.list_versions, ent_map,
- &truncated, &cur_marker);
+ int r = store->cls_bucket_list(target->get_bucket_info(), shard_id, cur_marker, cur_prefix,
+ read_ahead + 1 - count, params.list_versions, ent_map,
+ &truncated, &cur_marker);
if (r < 0)
return r;
return 0;
}
-int RGWRados::init_bucket_index(rgw_bucket& bucket, int num_shards)
+int RGWRados::init_bucket_index(RGWBucketInfo& bucket_info, int num_shards)
{
librados::IoCtx index_ctx; // context for new bucket
- int r = open_bucket_index_ctx(bucket, index_ctx);
+ int r = open_bucket_index_ctx(bucket_info, index_ctx);
if (r < 0)
return r;
string dir_oid = dir_oid_prefix;
- dir_oid.append(bucket.bucket_id);
+ dir_oid.append(bucket_info.bucket.bucket_id);
map<int, string> bucket_objs;
get_bucket_index_objects(dir_oid, num_shards, bucket_objs);
for (int i = 0; i < MAX_CREATE_RETRIES; i++) {
int ret = 0;
- ret = select_bucket_placement(owner, zonegroup_id, placement_rule,
- bucket.tenant, bucket.name, bucket,
+ ret = select_bucket_placement(owner, zonegroup_id, placement_rule, bucket,
&selected_placement_rule_name, &rule_info);
if (ret < 0)
return ret;
bucket.bucket_id = pmaster_bucket->bucket_id;
}
- int r = init_bucket_index(bucket, bucket_index_max_shards);
- if (r < 0)
- return r;
-
RGWObjVersionTracker& objv_tracker = info.objv_tracker;
if (pobjv) {
if (pquota_info) {
info.quota = *pquota_info;
}
+
+ int r = init_bucket_index(info, bucket_index_max_shards);
+ if (r < 0) {
+ return r;
+ }
+
ret = put_linked_bucket_info(info, exclusive, ceph::real_time(), pep_objv, &attrs, true);
if (ret == -EEXIST) {
/* we need to reread the info and return it, caller will have a use for it */
/* remove bucket index */
librados::IoCtx index_ctx; // context for new bucket
map<int, string> bucket_objs;
- int r = open_bucket_index(bucket, index_ctx, bucket_objs);
+ int r = open_bucket_index(info, index_ctx, bucket_objs);
if (r < 0)
return r;
}
int RGWRados::select_new_bucket_location(RGWUserInfo& user_info, const string& zonegroup_id, const string& request_rule,
- const string& tenant_name, const string& bucket_name, rgw_bucket& bucket, string *pselected_rule_name,
- RGWZonePlacementInfo *rule_info)
+ rgw_bucket& bucket, string *pselected_rule_name, RGWZonePlacementInfo *rule_info)
{
/* first check that rule exists within the specific zonegroup */
if (pselected_rule_name)
*pselected_rule_name = rule;
- return set_bucket_location_by_rule(rule, tenant_name, bucket_name, bucket, rule_info);
+ return select_bucket_location_by_rule(rule, bucket, rule_info);
}
-int RGWRados::set_bucket_location_by_rule(const string& location_rule, const string& tenant_name, const string& bucket_name, rgw_bucket& bucket,
- RGWZonePlacementInfo *rule_info)
+int RGWRados::select_bucket_location_by_rule(const string& location_rule, rgw_bucket& bucket, RGWZonePlacementInfo *rule_info)
{
- bucket.tenant = tenant_name;
- bucket.name = bucket_name;
-
if (location_rule.empty()) {
/* we can only reach here if we're trying to set a bucket location from a bucket
* created on a different zone, using a legacy / default pool configuration
*/
- return select_legacy_bucket_placement(tenant_name, bucket_name, bucket, rule_info);
+ return select_legacy_bucket_placement(bucket, rule_info);
}
/*
RGWZonePlacementInfo& placement_info = piter->second;
- bucket.placement.data_pool = placement_info.data_pool;
- bucket.placement.data_extra_pool = placement_info.data_extra_pool;
- bucket.placement.index_pool = placement_info.index_pool;
-
if (rule_info) {
*rule_info = placement_info;
}
}
int RGWRados::select_bucket_placement(RGWUserInfo& user_info, const string& zonegroup_id, const string& placement_rule,
- const string& tenant_name, const string& bucket_name, rgw_bucket& bucket,
- string *pselected_rule_name, RGWZonePlacementInfo *rule_info)
+ rgw_bucket& bucket, string *pselected_rule_name, RGWZonePlacementInfo *rule_info)
{
if (!get_zone_params().placement_pools.empty()) {
return select_new_bucket_location(user_info, zonegroup_id, placement_rule,
- tenant_name, bucket_name, bucket, pselected_rule_name, rule_info);
+ bucket, pselected_rule_name, rule_info);
}
if (pselected_rule_name) {
pselected_rule_name->clear();
}
- return select_legacy_bucket_placement(tenant_name, bucket_name, bucket, rule_info);
+ return select_legacy_bucket_placement(bucket, rule_info);
}
-int RGWRados::select_legacy_bucket_placement(const string& tenant_name, const string& bucket_name, rgw_bucket& bucket,
- RGWZonePlacementInfo *rule_info)
+int RGWRados::select_legacy_bucket_placement(rgw_bucket& bucket, RGWZonePlacementInfo *rule_info)
{
bufferlist map_bl;
map<string, bufferlist> m;
miter = m.begin();
pool_name = miter->first;
}
- bucket.placement.data_pool = pool_name;
- bucket.placement.index_pool = pool_name;
rule_info->data_pool = pool_name;
rule_info->data_extra_pool = pool_name;
return 0;
}
+bool RGWRados::get_obj_data_pool(const rgw_obj& obj, rgw_pool *pool)
+{
+ return rgw_get_obj_data_pool(zonegroup, zone_params, obj, pool);
+}
+
+bool RGWRados::obj_to_raw(const rgw_obj& obj, rgw_raw_obj *raw_obj)
+{
+ get_obj_bucket_and_oid_loc(obj, raw_obj->oid, raw_obj->loc);
+
+ return get_obj_data_pool(obj, &raw_obj->pool);
+}
+
int RGWRados::update_placement_map()
{
bufferlist header;
int RGWRados::get_obj_ioctx(const rgw_obj& obj, librados::IoCtx *ioctx)
{
- const rgw_bucket& bucket = obj.bucket;
string oid, key;
get_obj_bucket_and_oid_loc(obj, oid, key);
- int r;
-
- if (!obj.is_in_extra_data()) {
- r = open_pool_ctx(bucket.placement.data_pool, *ioctx);
- } else {
- r = open_pool_ctx(bucket.placement.get_data_extra_pool(), *ioctx);
+ rgw_pool pool;
+ if (!get_obj_data_pool(obj, &pool)) {
+ ldout(cct, 0) << "ERROR: cannot get data pool for obj=" << obj << ", probably misconfiguration" << dendl;
+ return -EIO;
}
- if (r < 0)
+
+ int r = open_pool_ctx(pool, *ioctx);
+ if (r < 0) {
return r;
+ }
ioctx->locator_set_key(key);
{
get_obj_bucket_and_oid_loc(obj, ref->oid, ref->key);
- int r;
-
- if (!obj.is_in_extra_data()) {
- r = open_pool_ctx(obj.bucket.placement.data_pool, ref->ioctx);
- } else {
- r = open_pool_ctx(obj.bucket.placement.get_data_extra_pool(), ref->ioctx);
+ rgw_pool pool;
+ if (!get_obj_data_pool(obj, &pool)) {
+ ldout(cct, 0) << "ERROR: cannot get data pool for obj=" << obj << ", probably misconfiguration" << dendl;
+ return -EIO;
}
- if (r < 0)
+
+ int r = open_pool_ctx(pool, ref->ioctx);
+ if (r < 0) {
return r;
+ }
ref->ioctx.locator_set_key(ref->key);
return 0;
}
-void RGWRados::obj_to_raw(const rgw_obj& obj, rgw_raw_obj *raw_obj)
-{
- rgw_obj_to_raw(obj, raw_obj);
-}
-
int RGWRados::get_system_obj_ref(const rgw_raw_obj& obj, rgw_rados_ref *ref, rgw_pool *pool)
{
return get_raw_obj_ref(obj, ref, pool);
RGWObjManifest::obj_iterator miter;
RGWObjManifest& manifest = astate->manifest;
for (miter = manifest.obj_begin(); miter != manifest.obj_end(); ++miter) {
- rgw_raw_obj raw_loc = miter.get_location();
+ rgw_raw_obj raw_loc = miter.get_location().get_raw_obj(this);
rgw_obj loc;
string oid;
string locator;
{
bucket = _bucket;
- int ret = store->open_bucket_index_shard(bucket, index_ctx, obj.get_hash_object(), &bucket_obj, &shard_id);
+ RGWObjectCtx obj_ctx(store);
+
+ RGWBucketInfo bucket_info;
+ int ret = store->get_bucket_instance_info(obj_ctx, bucket, bucket_info, NULL, NULL);
+ if (ret < 0) {
+ return ret;
+ }
+
+ ret = store->open_bucket_index_shard(bucket_info, index_ctx, obj.get_hash_object(), &bucket_obj, &shard_id);
if (ret < 0) {
ldout(store->ctx(), 0) << "ERROR: open_bucket_index_shard() returned ret=" << ret << dendl;
return ret;
bucket = _bucket;
shard_id = sid;
- int ret = store->open_bucket_index_shard(bucket, index_ctx, shard_id, &bucket_obj);
+ RGWObjectCtx obj_ctx(store);
+
+ RGWBucketInfo bucket_info;
+ int ret = store->get_bucket_instance_info(obj_ctx, bucket, bucket_info, NULL, NULL);
+ if (ret < 0) {
+ return ret;
+ }
+
+ ret = store->open_bucket_index_shard(bucket_info, index_ctx, shard_id, &bucket_obj);
if (ret < 0) {
ldout(store->ctx(), 0) << "ERROR: open_bucket_index_shard() returned ret=" << ret << dendl;
return ret;
uint64_t max_chunk_size;
- ret = get_max_chunk_size(obj.bucket, &max_chunk_size);
+ ret = get_max_chunk_size(obj, &max_chunk_size);
if (ret < 0) {
ldout(cct, 0) << "ERROR: failed to get max_chunk_size() for bucket " << obj.bucket << dendl;
return ret;
}
uint64_t max_chunk_size;
- ret = get_max_chunk_size(dest_obj.bucket, &max_chunk_size);
+ ret = get_max_chunk_size(dest_obj, &max_chunk_size);
if (ret < 0) {
ldout(cct, 0) << "ERROR: failed to get max_chunk_size() for bucket " << dest_obj.bucket << dendl;
return ret;
}
- bool copy_data = !astate->has_manifest || (src_obj.bucket.placement.data_pool != dest_obj.bucket.placement.data_pool);
+ rgw_pool src_pool;
+ rgw_pool dest_pool;
+ if (!get_obj_data_pool(src_obj, &src_pool)) {
+ ldout(cct, 0) << "ERROR: failed to locate data pool for " << src_obj << dendl;
+ return -EIO;
+ }
+ if (!get_obj_data_pool(dest_obj, &dest_pool)) {
+ ldout(cct, 0) << "ERROR: failed to locate data pool for " << dest_obj << dendl;
+ return -EIO;
+ }
+
+
+ bool copy_data = !astate->has_manifest || (src_pool != dest_pool);
bool copy_first = false;
if (astate->has_manifest) {
if (!astate->manifest.has_tail()) {
}
rgw_rados_ref ref;
- ret = get_raw_obj_ref(miter.get_location(), &ref);
+ ret = get_raw_obj_ref(miter.get_location().get_raw_obj(this), &ref);
if (ret < 0) {
return ret;
}
for (; miter != astate->manifest.obj_end(); ++miter) {
ObjectWriteOperation op;
cls_refcount_get(op, tag, true);
- const rgw_raw_obj& loc = miter.get_location();
+ const rgw_raw_obj& loc = miter.get_location().get_raw_obj(this);
ref.ioctx.locator_set_key(loc.loc);
ret = ref.ioctx.operate(loc.oid, &op);
* bucket: the bucket to check
* Returns false is the bucket is not synced
*/
-bool RGWRados::is_syncing_bucket_meta(rgw_bucket& bucket)
+bool RGWRados::is_syncing_bucket_meta(const rgw_bucket& bucket)
{
/* no current period */
* bucket: the name of the bucket to delete
* Returns 0 on success, -ERR# otherwise.
*/
-int RGWRados::delete_bucket(rgw_bucket& bucket, RGWObjVersionTracker& objv_tracker)
+int RGWRados::delete_bucket(RGWBucketInfo& bucket_info, RGWObjVersionTracker& objv_tracker)
{
+ const rgw_bucket& bucket = bucket_info.bucket;
librados::IoCtx index_ctx;
map<int, string> bucket_objs;
- int r = open_bucket_index(bucket, index_ctx, bucket_objs);
+ int r = open_bucket_index(bucket_info, index_ctx, bucket_objs);
if (r < 0)
return r;
do {
#define NUM_ENTRIES 1000
- r = cls_bucket_list(bucket, RGW_NO_SHARD, marker, prefix, NUM_ENTRIES, true, ent_map,
+ r = cls_bucket_list(bucket_info, RGW_NO_SHARD, marker, prefix, NUM_ENTRIES, true, ent_map,
&is_truncated, &marker);
if (r < 0)
return r;
rgw_raw_obj raw_head;
obj_to_raw(head_obj, &raw_head);
for (iter = manifest.obj_begin(); iter != manifest.obj_end(); ++iter) {
- const rgw_raw_obj& mobj = iter.get_location();
+ const rgw_raw_obj& mobj = iter.get_location().get_raw_obj(this);
if (mobj == raw_head)
continue;
cls_rgw_obj_key key(mobj.oid);
return gc->send_chain(chain, tag, sync);
}
-int RGWRados::open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx, string& bucket_oid)
+int RGWRados::open_bucket_index(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx, string& bucket_oid)
{
- int r = open_bucket_index_ctx(bucket, index_ctx);
+ const rgw_bucket& bucket = bucket_info.bucket;
+ int r = open_bucket_index_ctx(bucket_info, index_ctx);
if (r < 0)
return r;
return 0;
}
-int RGWRados::open_bucket_index_base(rgw_bucket& bucket, librados::IoCtx& index_ctx,
+int RGWRados::open_bucket_index_base(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx,
string& bucket_oid_base) {
- int r = open_bucket_index_ctx(bucket, index_ctx);
+ const rgw_bucket& bucket = bucket_info.bucket;
+ int r = open_bucket_index_ctx(bucket_info, index_ctx);
if (r < 0)
return r;
}
-int RGWRados::open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx,
+int RGWRados::open_bucket_index(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx,
map<int, string>& bucket_objs, int shard_id, map<int, string> *bucket_instance_ids) {
string bucket_oid_base;
- int ret = open_bucket_index_base(bucket, index_ctx, bucket_oid_base);
- if (ret < 0)
- return ret;
-
- RGWObjectCtx obj_ctx(this);
-
- // Get the bucket info
- RGWBucketInfo binfo;
- ret = get_bucket_instance_info(obj_ctx, bucket, binfo, NULL, NULL);
- if (ret < 0)
+ int ret = open_bucket_index_base(bucket_info, index_ctx, bucket_oid_base);
+ if (ret < 0) {
return ret;
+ }
- get_bucket_index_objects(bucket_oid_base, binfo.num_shards, bucket_objs, shard_id);
+ get_bucket_index_objects(bucket_oid_base, bucket_info.num_shards, bucket_objs, shard_id);
if (bucket_instance_ids) {
- get_bucket_instance_ids(binfo, shard_id, bucket_instance_ids);
+ get_bucket_instance_ids(bucket_info, shard_id, bucket_instance_ids);
}
return 0;
}
template<typename T>
-int RGWRados::open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx,
+int RGWRados::open_bucket_index(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx,
map<int, string>& oids, map<int, T>& bucket_objs,
int shard_id, map<int, string> *bucket_instance_ids)
{
- int ret = open_bucket_index(bucket, index_ctx, oids, shard_id, bucket_instance_ids);
+ int ret = open_bucket_index(bucket_info, index_ctx, oids, shard_id, bucket_instance_ids);
if (ret < 0)
return ret;
return 0;
}
-int RGWRados::open_bucket_index_shard(rgw_bucket& bucket, librados::IoCtx& index_ctx,
+int RGWRados::open_bucket_index_shard(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx,
const string& obj_key, string *bucket_obj, int *shard_id)
{
string bucket_oid_base;
- int ret = open_bucket_index_base(bucket, index_ctx, bucket_oid_base);
+ int ret = open_bucket_index_base(bucket_info, index_ctx, bucket_oid_base);
if (ret < 0)
return ret;
RGWObjectCtx obj_ctx(this);
- // Get the bucket info
- RGWBucketInfo binfo;
- ret = get_bucket_instance_info(obj_ctx, bucket, binfo, NULL, NULL);
- if (ret < 0)
- return ret;
-
- ret = get_bucket_index_object(bucket_oid_base, obj_key, binfo.num_shards,
- (RGWBucketInfo::BIShardsHashType)binfo.bucket_index_shard_hash_type, bucket_obj, shard_id);
+ ret = get_bucket_index_object(bucket_oid_base, obj_key, bucket_info.num_shards,
+ (RGWBucketInfo::BIShardsHashType)bucket_info.bucket_index_shard_hash_type, bucket_obj, shard_id);
if (ret < 0) {
ldout(cct, 10) << "get_bucket_index_object() returned ret=" << ret << dendl;
return ret;
return 0;
}
-int RGWRados::open_bucket_index_shard(rgw_bucket& bucket, librados::IoCtx& index_ctx,
+int RGWRados::open_bucket_index_shard(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx,
int shard_id, string *bucket_obj)
{
string bucket_oid_base;
- int ret = open_bucket_index_base(bucket, index_ctx, bucket_oid_base);
+ int ret = open_bucket_index_base(bucket_info, index_ctx, bucket_oid_base);
if (ret < 0)
return ret;
RGWObjectCtx obj_ctx(this);
- // Get the bucket info
- RGWBucketInfo binfo;
- ret = get_bucket_instance_info(obj_ctx, bucket, binfo, NULL, NULL);
- if (ret < 0)
- return ret;
-
- get_bucket_index_object(bucket_oid_base, binfo.num_shards,
+ get_bucket_index_object(bucket_oid_base, bucket_info.num_shards,
shard_id, bucket_obj);
return 0;
}
}
}
-int RGWRados::bucket_check_index(rgw_bucket& bucket,
+int RGWRados::bucket_check_index(RGWBucketInfo& bucket_info,
map<RGWObjCategory, RGWStorageStats> *existing_stats,
map<RGWObjCategory, RGWStorageStats> *calculated_stats)
{
// value - bucket index check OP returned result with the given bucket index object (shard)
map<int, string> oids;
map<int, struct rgw_cls_check_index_ret> bucket_objs_ret;
- int ret = open_bucket_index(bucket, index_ctx, oids, bucket_objs_ret);
+ int ret = open_bucket_index(bucket_info, index_ctx, oids, bucket_objs_ret);
if (ret < 0)
return ret;
return 0;
}
-int RGWRados::bucket_rebuild_index(rgw_bucket& bucket)
+int RGWRados::bucket_rebuild_index(RGWBucketInfo& bucket_info)
{
librados::IoCtx index_ctx;
map<int, string> bucket_objs;
- int r = open_bucket_index(bucket, index_ctx, bucket_objs);
+ int r = open_bucket_index(bucket_info, index_ctx, bucket_objs);
if (r < 0)
return r;
return r;
}
-static void generate_fake_tag(CephContext *cct, map<string, bufferlist>& attrset, RGWObjManifest& manifest, bufferlist& manifest_bl, bufferlist& tag_bl)
+static void generate_fake_tag(RGWRados *store, map<string, bufferlist>& attrset, RGWObjManifest& manifest, bufferlist& manifest_bl, bufferlist& tag_bl)
{
string tag;
if (mi != manifest.obj_end()) {
if (manifest.has_tail()) // first object usually points at the head, let's skip to a more unique part
++mi;
- tag = mi.get_location().oid;
+ tag = mi.get_location().get_raw_obj(store).oid;
tag.append("_");
}
buf_to_hex(md5, CEPH_CRYPTO_MD5_DIGESTSIZE, md5_str);
tag.append(md5_str);
- ldout(cct, 10) << "generate_fake_tag new tag=" << tag << dendl;
+ ldout(store->ctx(), 10) << "generate_fake_tag new tag=" << tag << dendl;
tag_bl.append(tag.c_str(), tag.size() + 1);
}
if (cct->_conf->subsys.should_gather(ceph_subsys_rgw, 20) && s->manifest.has_explicit_objs()) {
RGWObjManifest::obj_iterator mi;
for (mi = s->manifest.obj_begin(); mi != s->manifest.obj_end(); ++mi) {
- ldout(cct, 20) << "manifest: ofs=" << mi.get_ofs() << " loc=" << mi.get_location() << dendl;
+ ldout(cct, 20) << "manifest: ofs=" << mi.get_ofs() << " loc=" << mi.get_location().get_raw_obj(this) << dendl;
}
}
* Uh oh, something's wrong, object with manifest should have tag. Let's
* create one out of the manifest, would be unique
*/
- generate_fake_tag(cct, s->attrset, s->manifest, manifest_bl, s->obj_tag);
+ generate_fake_tag(this, s->attrset, s->manifest, manifest_bl, s->obj_tag);
s->fake_tag = true;
}
}
}
state.obj = astate->obj;
- RGWRados::obj_to_raw(state.obj, &state.head_obj);
+ store->obj_to_raw(state.obj, &state.head_obj);
r = store->get_obj_ioctx(state.obj, &state.io_ctx);
if (r < 0) {
RGWObjManifest::obj_iterator iter = astate->manifest.obj_find(ofs);
uint64_t stripe_ofs = iter.get_stripe_ofs();
- read_obj = iter.get_location();
+ read_obj = iter.get_location().get_raw_obj(store);
len = min(len, iter.get_stripe_size() - (ofs - stripe_ofs));
read_ofs = iter.location_ofs() + (ofs - stripe_ofs);
reading_from_head = (read_obj == state.head_obj);
off_t next_stripe_ofs = stripe_ofs + iter.get_stripe_size();
while (ofs < next_stripe_ofs && ofs <= end) {
- read_obj = iter.get_location();
+ read_obj = iter.get_location().get_raw_obj(this);
uint64_t read_len = min(len, iter.get_stripe_size() - (ofs - stripe_ofs));
read_ofs = iter.location_ofs() + (ofs - stripe_ofs);
return 0;
}
-int RGWRados::get_bucket_stats(rgw_bucket& bucket, int shard_id, string *bucket_ver, string *master_ver,
+int RGWRados::get_bucket_stats(RGWBucketInfo& bucket_info, int shard_id, string *bucket_ver, string *master_ver,
map<RGWObjCategory, RGWStorageStats>& stats, string *max_marker)
{
map<string, rgw_bucket_dir_header> headers;
map<int, string> bucket_instance_ids;
- int r = cls_bucket_head(bucket, shard_id, headers, &bucket_instance_ids);
- if (r < 0)
+ int r = cls_bucket_head(bucket_info, shard_id, headers, &bucket_instance_ids);
+ if (r < 0) {
return r;
+ }
assert(headers.size() == bucket_instance_ids.size());
return 0;
}
-int RGWRados::get_bi_log_status(rgw_bucket& bucket, int shard_id,
+int RGWRados::get_bi_log_status(RGWBucketInfo& bucket_info, int shard_id,
map<int, string>& markers)
{
map<string, rgw_bucket_dir_header> headers;
map<int, string> bucket_instance_ids;
- int r = cls_bucket_head(bucket, shard_id, headers, &bucket_instance_ids);
+ int r = cls_bucket_head(bucket_info, shard_id, headers, &bucket_instance_ids);
if (r < 0)
return r;
}
};
-int RGWRados::get_bucket_stats_async(rgw_bucket& bucket, int shard_id, RGWGetBucketStats_CB *ctx)
+int RGWRados::get_bucket_stats_async(RGWBucketInfo& bucket_info, int shard_id, RGWGetBucketStats_CB *ctx)
{
- RGWBucketInfo binfo;
- RGWObjectCtx obj_ctx(this);
-
- int r = get_bucket_instance_info(obj_ctx, bucket, binfo, NULL, NULL);
- if (r < 0)
- return r;
-
int num_aio = 0;
- RGWGetBucketStatsContext *get_ctx = new RGWGetBucketStatsContext(ctx, binfo.num_shards);
+ RGWGetBucketStatsContext *get_ctx = new RGWGetBucketStatsContext(ctx, bucket_info.num_shards);
assert(get_ctx);
- r = cls_bucket_head_async(bucket, shard_id, get_ctx, &num_aio);
+ int r = cls_bucket_head_async(bucket_info, shard_id, get_ctx, &num_aio);
get_ctx->put();
if (r < 0) {
ctx->put();
int RGWRados::update_containers_stats(map<string, RGWBucketEnt>& m)
{
+ RGWObjectCtx obj_ctx(this);
+
map<string, RGWBucketEnt>::iterator iter;
for (iter = m.begin(); iter != m.end(); ++iter) {
RGWBucketEnt& ent = iter->second;
ent.size_rounded = 0;
map<string, rgw_bucket_dir_header> headers;
- int r = cls_bucket_head(bucket, RGW_NO_SHARD, headers);
+
+ RGWBucketInfo bucket_info;
+ int ret = get_bucket_instance_info(obj_ctx, bucket, bucket_info, NULL, NULL);
+ if (ret < 0) {
+ return ret;
+ }
+
+ int r = cls_bucket_head(bucket_info, RGW_NO_SHARD, headers);
if (r < 0)
return r;
return oids.size();
}
-int RGWRados::list_bi_log_entries(rgw_bucket& bucket, int shard_id, string& marker, uint32_t max,
+int RGWRados::list_bi_log_entries(RGWBucketInfo& bucket_info, int shard_id, string& marker, uint32_t max,
std::list<rgw_bi_log_entry>& result, bool *truncated)
{
- ldout(cct, 20) << __func__ << ": " << bucket << " marker " << marker << " shard_id=" << shard_id << " max " << max << dendl;
+ ldout(cct, 20) << __func__ << ": " << bucket_info.bucket << " marker " << marker << " shard_id=" << shard_id << " max " << max << dendl;
result.clear();
librados::IoCtx index_ctx;
map<int, string> oids;
map<int, cls_rgw_bi_log_list_ret> bi_log_lists;
map<int, string> bucket_instance_ids;
- int r = open_bucket_index(bucket, index_ctx, oids, shard_id, &bucket_instance_ids);
+ int r = open_bucket_index(bucket_info, index_ctx, oids, shard_id, &bucket_instance_ids);
if (r < 0)
return r;
return 0;
}
-int RGWRados::trim_bi_log_entries(rgw_bucket& bucket, int shard_id, string& start_marker, string& end_marker)
+int RGWRados::trim_bi_log_entries(RGWBucketInfo& bucket_info, int shard_id, string& start_marker, string& end_marker)
{
librados::IoCtx index_ctx;
map<int, string> bucket_objs;
- int r = open_bucket_index(bucket, index_ctx, bucket_objs, shard_id);
+ int r = open_bucket_index(bucket_info, index_ctx, bucket_objs, shard_id);
if (r < 0)
return r;
return cls_obj_complete_op(bs, CLS_RGW_OP_CANCEL, tag, -1 /* pool id */, 0, ent, RGW_OBJ_CATEGORY_NONE, NULL, bilog_flags);
}
-int RGWRados::cls_obj_set_bucket_tag_timeout(rgw_bucket& bucket, uint64_t timeout)
+int RGWRados::cls_obj_set_bucket_tag_timeout(RGWBucketInfo& bucket_info, uint64_t timeout)
{
librados::IoCtx index_ctx;
map<int, string> bucket_objs;
- int r = open_bucket_index(bucket, index_ctx, bucket_objs);
+ int r = open_bucket_index(bucket_info, index_ctx, bucket_objs);
if (r < 0)
return r;
return CLSRGWIssueSetTagTimeout(index_ctx, bucket_objs, cct->_conf->rgw_bucket_index_max_aio, timeout)();
}
-int RGWRados::cls_bucket_list(rgw_bucket& bucket, int shard_id, rgw_obj_key& start, const string& prefix,
+int RGWRados::cls_bucket_list(RGWBucketInfo& bucket_info, int shard_id, rgw_obj_key& start, const string& prefix,
uint32_t num_entries, bool list_versions, map<string, RGWObjEnt>& m,
bool *is_truncated, rgw_obj_key *last_entry,
bool (*force_check_filter)(const string& name))
{
- ldout(cct, 10) << "cls_bucket_list " << bucket << " start " << start.name << "[" << start.instance << "] num_entries " << num_entries << dendl;
+ ldout(cct, 10) << "cls_bucket_list " << bucket_info.bucket << " start " << start.name << "[" << start.instance << "] num_entries " << num_entries << dendl;
librados::IoCtx index_ctx;
// key - oid (for different shards if there is any)
// value - list result for the corresponding oid (shard), it is filled by the AIO callback
map<int, string> oids;
map<int, struct rgw_cls_list_ret> list_results;
- int r = open_bucket_index(bucket, index_ctx, oids, shard_id);
+ int r = open_bucket_index(bucket_info, index_ctx, oids, shard_id);
if (r < 0)
return r;
* and if the tags are old we need to do cleanup as well. */
librados::IoCtx sub_ctx;
sub_ctx.dup(index_ctx);
- r = check_disk_state(sub_ctx, bucket, dirent, e, updates[vnames[pos]]);
+ r = check_disk_state(sub_ctx, bucket_info.bucket, dirent, e, updates[vnames[pos]]);
if (r < 0 && r != -ENOENT) {
return r;
}
return r;
}
-int RGWRados::remove_objs_from_index(rgw_bucket& bucket, list<rgw_obj_key>& oid_list)
+int RGWRados::remove_objs_from_index(RGWBucketInfo& bucket_info, list<rgw_obj_key>& oid_list)
{
librados::IoCtx index_ctx;
string dir_oid;
uint8_t suggest_flag = (get_zone().log_data ? CEPH_RGW_DIR_SUGGEST_LOG_OP : 0);
- int r = open_bucket_index(bucket, index_ctx, dir_oid);
+ int r = open_bucket_index(bucket_info, index_ctx, dir_oid);
if (r < 0)
return r;
for (iter = oid_list.begin(); iter != oid_list.end(); ++iter) {
rgw_obj_key& key = *iter;
- dout(2) << "RGWRados::remove_objs_from_index bucket=" << bucket << " obj=" << key.name << ":" << key.instance << dendl;
+ dout(2) << "RGWRados::remove_objs_from_index bucket=" << bucket_info.bucket << " obj=" << key.name << ":" << key.instance << dendl;
rgw_bucket_dir_entry entry;
entry.ver.epoch = (uint64_t)-1; // ULLONG_MAX, needed to that objclass doesn't skip out request
key.transform(&entry.key);
RGWObjManifest::obj_iterator miter;
RGWObjManifest& manifest = astate->manifest;
for (miter = manifest.obj_begin(); miter != manifest.obj_end(); ++miter) {
- const rgw_raw_obj& raw_loc = miter.get_location();
+ const rgw_raw_obj& raw_loc = miter.get_location().get_raw_obj(this);
rgw_obj loc;
rgw_raw_obj_to_obj(manifest.get_obj().bucket, raw_loc, &loc);
return 0;
}
-int RGWRados::cls_bucket_head(rgw_bucket& bucket, int shard_id, map<string, struct rgw_bucket_dir_header>& headers, map<int, string> *bucket_instance_ids)
+int RGWRados::cls_bucket_head(const RGWBucketInfo& bucket_info, int shard_id, map<string, struct rgw_bucket_dir_header>& headers, map<int, string> *bucket_instance_ids)
{
librados::IoCtx index_ctx;
map<int, string> oids;
map<int, struct rgw_cls_list_ret> list_results;
- int r = open_bucket_index(bucket, index_ctx, oids, list_results, shard_id, bucket_instance_ids);
+ int r = open_bucket_index(bucket_info, index_ctx, oids, list_results, shard_id, bucket_instance_ids);
if (r < 0)
return r;
return 0;
}
-int RGWRados::cls_bucket_head_async(rgw_bucket& bucket, int shard_id, RGWGetDirHeader_CB *ctx, int *num_aio)
+int RGWRados::cls_bucket_head_async(const RGWBucketInfo& bucket_info, int shard_id, RGWGetDirHeader_CB *ctx, int *num_aio)
{
librados::IoCtx index_ctx;
map<int, string> bucket_objs;
- int r = open_bucket_index(bucket, index_ctx, bucket_objs, shard_id);
+ int r = open_bucket_index(bucket_info, index_ctx, bucket_objs, shard_id);
if (r < 0)
return r;
return 0;
}
-int RGWRados::cls_user_sync_bucket_stats(rgw_raw_obj& user_obj, rgw_bucket& bucket)
+int RGWRados::cls_user_sync_bucket_stats(rgw_raw_obj& user_obj, const RGWBucketInfo& bucket_info)
{
map<string, struct rgw_bucket_dir_header> headers;
- int r = cls_bucket_head(bucket, RGW_NO_SHARD, headers);
+ int r = cls_bucket_head(bucket_info, RGW_NO_SHARD, headers);
if (r < 0) {
ldout(cct, 20) << "cls_bucket_header() returned " << r << dendl;
return r;
cls_user_bucket_entry entry;
- bucket.convert(&entry.bucket);
+ bucket_info.bucket.convert(&entry.bucket);
map<string, struct rgw_bucket_dir_header>::iterator hiter = headers.begin();
for (; hiter != headers.end(); ++hiter) {
return 0;
}
-int RGWRados::update_user_bucket_stats(const string& user_id, rgw_bucket& bucket, RGWStorageStats& stats)
-{
- cls_user_bucket_entry entry;
-
- entry.size = stats.size;
- entry.size_rounded = stats.size_rounded;
- entry.count += stats.num_objects;
-
- list<cls_user_bucket_entry> entries;
- entries.push_back(entry);
-
- string buckets_obj_id;
- rgw_get_buckets_obj(user_id, buckets_obj_id);
- rgw_raw_obj obj(get_zone_params().user_uid_pool, buckets_obj_id);
-
- int r = cls_user_update_buckets(obj, entries, false);
- if (r < 0) {
- ldout(cct, 20) << "cls_user_update_buckets() returned " << r << dendl;
- return r;
- }
-
- return 0;
-}
-
int RGWRados::cls_user_list_buckets(rgw_raw_obj& obj,
const string& in_marker,
const string& end_marker,
}
}
-void RGWRados::get_bucket_instance_ids(RGWBucketInfo& bucket_info, int shard_id, map<int, string> *result)
+void RGWRados::get_bucket_instance_ids(const RGWBucketInfo& bucket_info, int shard_id, map<int, string> *result)
{
- rgw_bucket& bucket = bucket_info.bucket;
+ const rgw_bucket& bucket = bucket_info.bucket;
string plain_id = bucket.name + ":" + bucket.bucket_id;
if (!bucket_info.num_shards) {
(*result)[0] = plain_id;
return 0;
}
-int RGWRados::delete_obj_aio(const rgw_obj& obj, rgw_bucket& bucket,
+int RGWRados::delete_obj_aio(const rgw_obj& obj,
RGWBucketInfo& bucket_info, RGWObjState *astate,
list<librados::AioCompletion *>& handles, bool keep_index_consistent)
{
class RGWDataSyncProcessorThread;
class RGWSyncLogTrimThread;
class RGWRESTConn;
+struct RGWZoneGroup;
+struct RGWZoneParams;
/* flags for put_obj_meta() */
#define PUT_OBJ_CREATE 0x01
int rgw_policy_from_attrset(CephContext *cct, map<string, bufferlist>& attrset, RGWAccessControlPolicy *policy);
-static inline void rgw_obj_to_raw(const rgw_obj& obj, rgw_raw_obj *raw_obj)
-{
- get_obj_bucket_and_oid_loc(obj, raw_obj->oid, raw_obj->loc);
-
- if (!obj.is_in_extra_data()) {
- raw_obj->pool = obj.bucket.placement.data_pool;
- } else {
- raw_obj->pool = obj.bucket.placement.get_data_extra_pool();
- }
-}
-
static inline bool rgw_raw_obj_to_obj(const rgw_bucket& bucket, const rgw_raw_obj& raw_obj, rgw_obj *obj)
{
string name;
return true;
}
+bool rgw_obj_to_raw(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params,
+ const rgw_obj& obj, rgw_raw_obj *raw_obj);
+
+class rgw_obj_select {
+ rgw_obj obj;
+ rgw_raw_obj raw_obj;
+ bool is_raw;
+
+public:
+ rgw_obj_select() : is_raw(false) {}
+ rgw_obj_select(const rgw_obj& _obj) : obj(_obj), is_raw(false) {}
+ rgw_obj_select(const rgw_raw_obj& _raw_obj) : raw_obj(_raw_obj), is_raw(true) {}
+ rgw_obj_select(const rgw_obj_select& rhs) {
+ is_raw = rhs.is_raw;
+ if (is_raw) {
+ raw_obj = rhs.raw_obj;
+ } else {
+ obj = rhs.obj;
+ }
+ }
+
+ rgw_raw_obj get_raw_obj(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params) const;
+ rgw_raw_obj get_raw_obj(RGWRados *store) const;
+
+ rgw_obj_select& operator=(const rgw_obj& rhs) {
+ obj = rhs;
+ is_raw = false;
+ return *this;
+ }
+
+ rgw_obj_select& operator=(const rgw_raw_obj& rhs) {
+ raw_obj = rhs;
+ is_raw = true;
+ return *this;
+ }
+};
+
struct compression_block {
uint64_t old_ofs;
uint64_t new_ofs;
};
WRITE_CLASS_ENCODER(RGWObjManifestRule)
+
class RGWObjManifest {
protected:
bool explicit_objs; /* old manifest? */
uint64_t obj_size;
rgw_obj obj;
- rgw_raw_obj head_obj; /* in-memory only, calculated from obj */
uint64_t head_size;
uint64_t max_head_size;
string tail_instance; /* tail object's instance */
- void convert_to_explicit();
- int append_explicit(RGWObjManifest& m);
+ void convert_to_explicit(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params);
+ int append_explicit(RGWObjManifest& m, const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params);
void append_rules(RGWObjManifest& m, map<uint64_t, RGWObjManifestRule>::iterator& iter, string *override_prefix);
void update_iterators() {
objs = rhs.objs;
obj_size = rhs.obj_size;
obj = rhs.obj;
- head_obj = rhs.head_obj;
head_size = rhs.head_size;
max_head_size = rhs.max_head_size;
prefix = rhs.prefix;
objs.swap(_objs);
}
- void get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, string *override_prefix, rgw_raw_obj *location);
+ void get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, string *override_prefix, rgw_obj_select *location);
void set_trivial_rule(uint64_t tail_ofs, uint64_t stripe_max_size) {
RGWObjManifestRule rule(0, tail_ofs, 0, stripe_max_size);
tail_instance = obj.get_instance();
}
- rgw_obj_to_raw(obj, &head_obj);
-
update_iterators();
DECODE_FINISH(bl);
}
void dump(Formatter *f) const;
static void generate_test_instances(list<RGWObjManifest*>& o);
- int append(RGWObjManifest& m);
+ int append(RGWObjManifest& m, RGWZoneGroup& zonegroup, RGWZoneParams& zone_params);
+ int append(RGWObjManifest& m, RGWRados *store);
bool get_rule(uint64_t ofs, RGWObjManifestRule *rule);
void set_head(const rgw_obj& _o, uint64_t _s) {
obj = _o;
- rgw_obj_to_raw(obj, &head_obj);
head_size = _s;
if (explicit_objs && head_size > 0) {
return obj;
}
- const rgw_raw_obj& get_head() {
- return head_obj;
- }
-
void set_tail_bucket(const rgw_bucket& _b) {
tail_bucket = _b;
}
int cur_stripe;
string cur_override_prefix;
- rgw_raw_obj location;
+ rgw_obj_select location;
map<uint64_t, RGWObjManifestRule>::iterator rule_iter;
map<uint64_t, RGWObjManifestRule>::iterator next_rule_iter;
bool operator!=(const obj_iterator& rhs) {
return (ofs != rhs.ofs);
}
- const rgw_raw_obj& get_location() {
+ const rgw_obj_select& get_location() {
return location;
}
string oid_prefix;
- rgw_raw_obj cur_obj;
+ rgw_obj_select cur_obj;
rgw_pool pool;
int create_next(uint64_t ofs);
- const rgw_raw_obj& get_cur_obj() { return cur_obj; }
+ rgw_raw_obj get_cur_obj(RGWZoneGroup& zonegroup, RGWZoneParams& zone_params) { return cur_obj.get_raw_obj(zonegroup, zone_params); }
+ rgw_raw_obj get_cur_obj(RGWRados *store) { return cur_obj.get_raw_obj(store); }
/* total max size of current stripe (including head obj) */
uint64_t cur_stripe_max_size() {
};
WRITE_CLASS_ENCODER(RGWSystemMetaObj)
-struct RGWZoneGroup;
-
struct RGWZonePlacementInfo {
string index_pool;
string data_pool;
void dump(Formatter *f) const;
void decode_json(JSONObj *obj);
static void generate_test_instances(list<RGWZoneParams*>& o);
+
+ bool find_placement(const rgw_data_placement_target& placement, string *placement_id) {
+ for (const auto& pp : placement_pools) {
+ const RGWZonePlacementInfo& info = pp.second;
+ if (info.index_pool == placement.index_pool.to_str() &&
+ info.data_pool == placement.data_pool.to_str() &&
+ info.data_extra_pool == placement.data_extra_pool.to_str()) {
+ *placement_id = pp.first;
+ return true;
+ }
+ }
+ return false;
+ }
+
+ bool get_placement(const string& placement_id, RGWZonePlacementInfo *placement) const {
+ auto iter = placement_pools.find(placement_id);
+ if (iter == placement_pools.end()) {
+ return false;
+ }
+ *placement = iter->second;
+ return true;
+ }
+
+ bool get_index_pool(const rgw_obj& obj, rgw_pool *pool) {
+ if (!obj.bucket.explicit_placement.index_pool.empty()) {
+ *pool = obj.bucket.explicit_placement.index_pool;
+ return true;
+ }
+ if (obj.placement_id.empty()) {
+ return false;
+ }
+ auto iter = placement_pools.find(obj.placement_id);
+ if (iter == placement_pools.end()) {
+ return false;
+ }
+ pool->init(iter->second.index_pool);
+ return true;
+ }
+
+ bool get_data_pool(const rgw_obj& obj, rgw_pool *pool) const {
+ const rgw_data_placement_target& explicit_placement = obj.bucket.explicit_placement;
+ if (!explicit_placement.data_pool.empty()) {
+ if (!obj.in_extra_data) {
+ *pool = explicit_placement.data_pool;
+ } else {
+ *pool = explicit_placement.get_data_extra_pool();
+ }
+ return true;
+ }
+ if (obj.placement_id.empty()) {
+ return false;
+ }
+ auto iter = placement_pools.find(obj.placement_id);
+ if (iter == placement_pools.end()) {
+ return false;
+ }
+ if (!obj.in_extra_data) {
+ pool->init(iter->second.data_pool);
+ } else {
+ pool->init(iter->second.data_extra_pool);
+ }
+ return true;
+ }
};
WRITE_CLASS_ENCODER(RGWZoneParams)
int open_objexp_pool_ctx();
int open_pool_ctx(const rgw_pool& pool, librados::IoCtx& io_ctx);
- int open_bucket_index_ctx(rgw_bucket& bucket, librados::IoCtx& index_ctx);
- int open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx, string& bucket_oid);
- int open_bucket_index_base(rgw_bucket& bucket, librados::IoCtx& index_ctx,
+ int open_bucket_index_ctx(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx);
+ int open_bucket_index(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx, string& bucket_oid);
+ int open_bucket_index_base(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx,
string& bucket_oid_base);
- int open_bucket_index_shard(rgw_bucket& bucket, librados::IoCtx& index_ctx,
+ int open_bucket_index_shard(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx,
const string& obj_key, string *bucket_obj, int *shard_id);
- int open_bucket_index_shard(rgw_bucket& bucket, librados::IoCtx& index_ctx,
+ int open_bucket_index_shard(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx,
int shard_id, string *bucket_obj);
- int open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx,
+ int open_bucket_index(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx,
map<int, string>& bucket_objs, int shard_id = -1, map<int, string> *bucket_instance_ids = NULL);
template<typename T>
- int open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx,
+ int open_bucket_index(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx,
map<int, string>& oids, map<int, T>& bucket_objs,
int shard_id = -1, map<int, string> *bucket_instance_ids = NULL);
void build_bucket_index_marker(const string& shard_id_str, const string& shard_marker,
string *marker);
- void get_bucket_instance_ids(RGWBucketInfo& bucket_info, int shard_id, map<int, string> *result);
+ void get_bucket_instance_ids(const RGWBucketInfo& bucket_info, int shard_id, map<int, string> *result);
atomic64_t max_req_id;
Mutex lock;
}
int get_required_alignment(const rgw_pool& pool, uint64_t *alignment);
- int get_required_alignment(const rgw_bucket& bucket, uint64_t *alignment);
int get_max_chunk_size(const rgw_pool& pool, uint64_t *max_chunk_size);
- int get_max_chunk_size(const rgw_bucket& bucket, uint64_t *max_chunk_size);
+ int get_max_chunk_size(const rgw_obj& obj, uint64_t *max_chunk_size);
uint32_t get_max_bucket_shards() {
return MAX_BUCKET_INDEX_SHARDS_PRIME;
}
int get_raw_obj_ref(const rgw_raw_obj& obj, rgw_rados_ref *ref, rgw_pool *pool = NULL);
- static void obj_to_raw(const rgw_obj& obj, rgw_raw_obj *raw_obj);
int list_raw_objects(const rgw_pool& pool, const string& prefix_filter, int max,
RGWListRawObjsCtx& ctx, list<string>& oids,
* create a bucket with name bucket and the given list of attrs
* returns 0 on success, -ERR# otherwise.
*/
- int init_bucket_index(rgw_bucket& bucket, int num_shards);
+ int init_bucket_index(RGWBucketInfo& bucket_info, int num_shards);
int select_bucket_placement(RGWUserInfo& user_info, const string& zonegroup_id, const string& rule,
- const string& tenant_name, const string& bucket_name, rgw_bucket& bucket, string *pselected_rule_name,
- RGWZonePlacementInfo *rule_info);
- int select_legacy_bucket_placement(const string& tenant_name, const string& bucket_name, rgw_bucket& bucket,
- RGWZonePlacementInfo *rule_info);
+ rgw_bucket& bucket, string *pselected_rule_name, RGWZonePlacementInfo *rule_info);
+ int select_legacy_bucket_placement(rgw_bucket& bucket, RGWZonePlacementInfo *rule_info);
int select_new_bucket_location(RGWUserInfo& user_info, const string& zonegroup_id, const string& rule,
- const string& tenant_name, const string& bucket_name, rgw_bucket& bucket, string *pselected_rule_name,
- RGWZonePlacementInfo *rule_info);
- int set_bucket_location_by_rule(const string& location_rule, const string& tenant_name, const string& bucket_name, rgw_bucket& bucket,
- RGWZonePlacementInfo *rule_info);
+ rgw_bucket& bucket, string *pselected_rule_name, RGWZonePlacementInfo *rule_info);
+ int select_bucket_location_by_rule(const string& location_rule, rgw_bucket& bucket, RGWZonePlacementInfo *rule_info);
void create_bucket_id(string *bucket_id);
+
+ bool get_obj_data_pool(const rgw_obj& obj, rgw_pool *pool);
+ bool obj_to_raw(const rgw_obj& obj, rgw_raw_obj *raw_obj);
+
int create_bucket(RGWUserInfo& owner, rgw_bucket& bucket,
const string& zonegroup_id,
const string& placement_rule,
* bucket: the name of the bucket to delete
* Returns 0 on success, -ERR# otherwise.
*/
- int delete_bucket(rgw_bucket& bucket, RGWObjVersionTracker& objv_tracker);
+ int delete_bucket(RGWBucketInfo& bucket_info, RGWObjVersionTracker& objv_tracker);
bool is_meta_master();
/**
* Check to see if the bucket metadata is synced
*/
- bool is_syncing_bucket_meta(rgw_bucket& bucket);
+ bool is_syncing_bucket_meta(const rgw_bucket& bucket);
void wakeup_meta_sync_shards(set<int>& shard_ids);
void wakeup_data_sync_shards(const string& source_zone, map<int, set<string> >& shard_ids);
}
int decode_policy(bufferlist& bl, ACLOwner *owner);
- int get_bucket_stats(rgw_bucket& bucket, int shard_id, string *bucket_ver, string *master_ver,
+ int get_bucket_stats(RGWBucketInfo& bucket_info, int shard_id, string *bucket_ver, string *master_ver,
map<RGWObjCategory, RGWStorageStats>& stats, string *max_marker);
- int get_bucket_stats_async(rgw_bucket& bucket, int shard_id, RGWGetBucketStats_CB *cb);
+ int get_bucket_stats_async(RGWBucketInfo& bucket_info, int shard_id, RGWGetBucketStats_CB *cb);
int get_user_stats(const rgw_user& user, RGWStorageStats& stats);
int get_user_stats_async(const rgw_user& user, RGWGetUserStats_CB *cb);
void get_bucket_instance_obj(const rgw_bucket& bucket, rgw_raw_obj& obj);
int cls_obj_complete_del(BucketShard& bs, string& tag, int64_t pool, uint64_t epoch, rgw_obj& obj,
ceph::real_time& removed_mtime, list<rgw_obj_key> *remove_objs, uint16_t bilog_flags);
int cls_obj_complete_cancel(BucketShard& bs, string& tag, rgw_obj& obj, uint16_t bilog_flags);
- int cls_obj_set_bucket_tag_timeout(rgw_bucket& bucket, uint64_t timeout);
- int cls_bucket_list(rgw_bucket& bucket, int shard_id, rgw_obj_key& start, const string& prefix,
+ int cls_obj_set_bucket_tag_timeout(RGWBucketInfo& bucket_info, uint64_t timeout);
+ int cls_bucket_list(RGWBucketInfo& bucket_info, int shard_id, rgw_obj_key& start, const string& prefix,
uint32_t num_entries, bool list_versions, map<string, RGWObjEnt>& m,
bool *is_truncated, rgw_obj_key *last_entry,
bool (*force_check_filter)(const string& name) = NULL);
- int cls_bucket_head(rgw_bucket& bucket, int shard_id, map<string, struct rgw_bucket_dir_header>& headers, map<int, string> *bucket_instance_ids = NULL);
- int cls_bucket_head_async(rgw_bucket& bucket, int shard_id, RGWGetDirHeader_CB *ctx, int *num_aio);
- int list_bi_log_entries(rgw_bucket& bucket, int shard_id, string& marker, uint32_t max, std::list<rgw_bi_log_entry>& result, bool *truncated);
- int trim_bi_log_entries(rgw_bucket& bucket, int shard_id, string& marker, string& end_marker);
- int get_bi_log_status(rgw_bucket& bucket, int shard_id, map<int, string>& max_marker);
+ int cls_bucket_head(const RGWBucketInfo& bucket_info, int shard_id, map<string, struct rgw_bucket_dir_header>& headers, map<int, string> *bucket_instance_ids = NULL);
+ int cls_bucket_head_async(const RGWBucketInfo& bucket_info, int shard_id, RGWGetDirHeader_CB *ctx, int *num_aio);
+ int list_bi_log_entries(RGWBucketInfo& bucket_info, int shard_id, string& marker, uint32_t max, std::list<rgw_bi_log_entry>& result, bool *truncated);
+ int trim_bi_log_entries(RGWBucketInfo& bucket_info, int shard_id, string& marker, string& end_marker);
+ int get_bi_log_status(RGWBucketInfo& bucket_info, int shard_id, map<int, string>& max_marker);
int bi_get_instance(rgw_obj& obj, rgw_bucket_dir_entry *dirent);
int bi_get(rgw_bucket& bucket, rgw_obj& obj, BIIndexType index_type, rgw_cls_bi_entry *entry);
int process_lc();
int list_lc_progress(const string& marker, uint32_t max_entries, map<string, int> *progress_map);
- int bucket_check_index(rgw_bucket& bucket,
+ int bucket_check_index(RGWBucketInfo& bucket_info,
map<RGWObjCategory, RGWStorageStats> *existing_stats,
map<RGWObjCategory, RGWStorageStats> *calculated_stats);
- int bucket_rebuild_index(rgw_bucket& bucket);
- int remove_objs_from_index(rgw_bucket& bucket, list<rgw_obj_key>& oid_list);
+ int bucket_rebuild_index(RGWBucketInfo& bucket_info);
+ int remove_objs_from_index(RGWBucketInfo& bucket_info, list<rgw_obj_key>& oid_list);
int move_rados_obj(librados::IoCtx& src_ioctx,
const string& src_oid, const string& src_locator,
librados::IoCtx& dst_ioctx,
int cls_user_get_header(const string& user_id, cls_user_header *header);
int cls_user_get_header_async(const string& user_id, RGWGetUserHeader_CB *ctx);
- int cls_user_sync_bucket_stats(rgw_raw_obj& user_obj, rgw_bucket& bucket);
- int update_user_bucket_stats(const string& user_id, rgw_bucket& bucket, RGWStorageStats& stats);
+ int cls_user_sync_bucket_stats(rgw_raw_obj& user_obj, const RGWBucketInfo& bucket_info);
int cls_user_list_buckets(rgw_raw_obj& obj,
const string& in_marker,
const string& end_marker,
librados::Rados* get_rados_handle();
int delete_raw_obj_aio(const rgw_raw_obj& obj, list<librados::AioCompletion *>& handles);
- int delete_obj_aio(const rgw_obj& obj, rgw_bucket& bucket, RGWBucketInfo& info, RGWObjState *astate,
+ int delete_obj_aio(const rgw_obj& obj, RGWBucketInfo& info, RGWObjState *astate,
list<librados::AioCompletion *>& handles, bool keep_index_consistent);
private:
/**
send_response();
do {
list<rgw_bi_log_entry> entries;
- int ret = store->list_bi_log_entries(bucket_info.bucket, shard_id,
+ int ret = store->list_bi_log_entries(bucket_info, shard_id,
marker, max_entries - count,
entries, &truncated);
if (ret < 0) {
}
}
map<RGWObjCategory, RGWStorageStats> stats;
- int ret = store->get_bucket_stats(bucket_info.bucket, shard_id, &bucket_ver, &master_ver, stats, &max_marker);
+ int ret = store->get_bucket_stats(bucket_info, shard_id, &bucket_ver, &master_ver, stats, &max_marker);
if (ret < 0 && ret != -ENOENT) {
http_ret = ret;
return;
return;
}
}
- http_ret = store->trim_bi_log_entries(bucket_info.bucket, shard_id, start_marker, end_marker);
+ http_ret = store->trim_bi_log_entries(bucket_info, shard_id, start_marker, end_marker);
if (http_ret < 0) {
dout(5) << "ERROR: trim_bi_log_entries() " << dendl;
}
bool is_truncated;
string marker;
int ret;
+ RGWObjectCtx obj_ctx(store);
do {
RGWUserBuckets user_buckets;
marker = i->first;
RGWBucketEnt& bucket_ent = i->second;
- ret = rgw_bucket_sync_user_stats(store, user_id, bucket_ent.bucket);
+ RGWBucketInfo bucket_info;
+
+ ret = store->get_bucket_instance_info(obj_ctx, bucket_ent.bucket, bucket_info, NULL, NULL);
+ if (ret < 0) {
+ ldout(cct, 0) << "ERROR: could not read bucket info: bucket=" << bucket_ent.bucket << " ret=" << ret << dendl;
+ continue;
+ }
+ ret = rgw_bucket_sync_user_stats(store, user_id, bucket_info);
if (ret < 0) {
ldout(cct, 0) << "ERROR: could not sync bucket stats: ret=" << ret << dendl;
return ret;
projects / ceph.git / commitdiff