OPTION(rgw_get_obj_window_size, OPT_INT, 16 << 20) // window size in bytes for single get obj request
OPTION(rgw_get_obj_max_req_size, OPT_INT, 4 << 20) // max length of a single get obj rados op
OPTION(rgw_relaxed_s3_bucket_names, OPT_BOOL, false) // enable relaxed bucket name rules for US region buckets
+OPTION(rgw_list_buckets_max_chunk, OPT_INT, 1000) // max buckets to retrieve in a single op when listing user buckets
OPTION(mutex_perf_counter, OPT_BOOL, false) // enable/disable mutex perf counter
buckets_obj_id += RGW_BUCKETS_OBJ_PREFIX;
}
-static int rgw_read_buckets_from_attr(RGWRados *store, string& user_id, RGWUserBuckets& buckets)
-{
- bufferlist bl;
- rgw_obj obj(store->zone.user_uid_pool, user_id);
- int ret = store->get_attr(NULL, obj, RGW_ATTR_BUCKETS, bl);
- if (ret)
- return ret;
-
- bufferlist::iterator iter = bl.begin();
- try {
- buckets.decode(iter);
- } catch (buffer::error& err) {
- ldout(store->ctx(), 0) << "ERROR: failed to decode buckets info, caught buffer::error" << dendl;
- return -EIO;
- }
- return 0;
-}
-
/**
* Get all the buckets owned by a user and fill up an RGWUserBuckets with them.
* Returns: 0 on success, -ERR# on failure.
*/
-int rgw_read_user_buckets(RGWRados *store, string user_id, RGWUserBuckets& buckets, bool need_stats)
+int rgw_read_user_buckets(RGWRados *store, string user_id, RGWUserBuckets& buckets,
+ const string& marker, uint64_t max, bool need_stats)
{
int ret;
buckets.clear();
- if (store->supports_omap()) {
- string buckets_obj_id;
- rgw_get_buckets_obj(user_id, buckets_obj_id);
- bufferlist bl;
- rgw_obj obj(store->zone.user_uid_pool, buckets_obj_id);
- bufferlist header;
- map<string,bufferlist> m;
-
- ret = store->omap_get_all(obj, header, m);
- if (ret == -ENOENT)
- ret = 0;
+ string buckets_obj_id;
+ rgw_get_buckets_obj(user_id, buckets_obj_id);
+ bufferlist bl;
+ rgw_obj obj(store->zone.user_uid_pool, buckets_obj_id);
+ bufferlist header;
+ map<string,bufferlist> m;
- if (ret < 0)
- return ret;
+ ret = store->omap_get_vals(obj, header, marker, max, m);
+ if (ret == -ENOENT)
+ ret = 0;
- for (map<string,bufferlist>::iterator q = m.begin(); q != m.end(); ++q) {
- bufferlist::iterator iter = q->second.begin();
- RGWBucketEnt bucket;
- ::decode(bucket, iter);
- buckets.add(bucket);
- }
- } else {
- ret = rgw_read_buckets_from_attr(store, user_id, buckets);
- switch (ret) {
- case 0:
- break;
- case -ENODATA:
- ret = 0;
- return 0;
- default:
- return ret;
- }
+ if (ret < 0)
+ return ret;
+
+ for (map<string,bufferlist>::iterator q = m.begin(); q != m.end(); ++q) {
+ bufferlist::iterator iter = q->second.begin();
+ RGWBucketEnt bucket;
+ ::decode(bucket, iter);
+ buckets.add(bucket);
}
if (need_stats) {
int ret;
string& bucket_name = bucket.name;
- if (store->supports_omap()) {
- bufferlist bl;
-
- RGWBucketEnt new_bucket;
- new_bucket.bucket = bucket;
- new_bucket.size = 0;
- time(&new_bucket.mtime);
- ::encode(new_bucket, bl);
+ bufferlist bl;
- string buckets_obj_id;
- rgw_get_buckets_obj(user_id, buckets_obj_id);
+ RGWBucketEnt new_bucket;
+ new_bucket.bucket = bucket;
+ new_bucket.size = 0;
+ time(&new_bucket.mtime);
+ ::encode(new_bucket, bl);
- rgw_obj obj(store->zone.user_uid_pool, buckets_obj_id);
- ret = store->omap_set(obj, bucket_name, bl);
- if (ret < 0) {
- ldout(store->ctx(), 0) << "ERROR: error adding bucket to directory: "
- << cpp_strerror(-ret)<< dendl;
- }
- } else {
- RGWUserBuckets buckets;
+ string buckets_obj_id;
+ rgw_get_buckets_obj(user_id, buckets_obj_id);
- ret = rgw_read_user_buckets(store, user_id, buckets, false);
- RGWBucketEnt new_bucket;
-
- switch (ret) {
- case 0:
- case -ENOENT:
- case -ENODATA:
- new_bucket.bucket = bucket;
- new_bucket.size = 0;
- time(&new_bucket.mtime);
- buckets.add(new_bucket);
- ret = rgw_write_buckets_attr(store, user_id, buckets);
- break;
- default:
- ldout(store->ctx(), 10) << "rgw_write_buckets_attr returned " << ret << dendl;
- break;
- }
+ rgw_obj obj(store->zone.user_uid_pool, buckets_obj_id);
+ ret = store->omap_set(obj, bucket_name, bl);
+ if (ret < 0) {
+ ldout(store->ctx(), 0) << "ERROR: error adding bucket to directory: "
+ << cpp_strerror(-ret)<< dendl;
}
return ret;
{
int ret;
- if (store->supports_omap()) {
- bufferlist bl;
-
- string buckets_obj_id;
- rgw_get_buckets_obj(user_id, buckets_obj_id);
-
- rgw_obj obj(store->zone.user_uid_pool, buckets_obj_id);
- ret = store->omap_del(obj, bucket.name);
- if (ret < 0) {
- ldout(store->ctx(), 0) << "ERROR: error removing bucket from directory: "
- << cpp_strerror(-ret)<< dendl;
- }
- } else {
- RGWUserBuckets buckets;
+ bufferlist bl;
- ret = rgw_read_user_buckets(store, user_id, buckets, false);
+ string buckets_obj_id;
+ rgw_get_buckets_obj(user_id, buckets_obj_id);
- if (ret == 0 || ret == -ENOENT) {
- buckets.remove(bucket.name);
- ret = rgw_write_buckets_attr(store, user_id, buckets);
- }
+ rgw_obj obj(store->zone.user_uid_pool, buckets_obj_id);
+ ret = store->omap_del(obj, bucket.name);
+ if (ret < 0) {
+ ldout(store->ctx(), 0) << "ERROR: error removing bucket from directory: "
+ << cpp_strerror(-ret)<< dendl;
}
return ret;
void check_bad_user_bucket_mapping(RGWRados *store, const string& user_id, bool fix)
{
RGWUserBuckets user_buckets;
- int ret = rgw_read_user_buckets(store, user_id, user_buckets, false);
- if (ret < 0) {
- ldout(store->ctx(), 0) << "failed to read user buckets: " << cpp_strerror(-ret) << dendl;
- return;
- }
+ bool done;
+ string marker;
- map<string, RGWBucketEnt>& buckets = user_buckets.get_buckets();
- for (map<string, RGWBucketEnt>::iterator i = buckets.begin();
- i != buckets.end();
- ++i) {
- RGWBucketEnt& bucket_ent = i->second;
- rgw_bucket& bucket = bucket_ent.bucket;
+ CephContext *cct = store->ctx();
- RGWBucketInfo bucket_info;
- int r = store->get_bucket_info(NULL, bucket.name, bucket_info);
- if (r < 0) {
- ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket << dendl;
- continue;
+ size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk;
+
+ do {
+ int ret = rgw_read_user_buckets(store, user_id, user_buckets, marker, max_entries, false);
+ if (ret < 0) {
+ ldout(store->ctx(), 0) << "failed to read user buckets: " << cpp_strerror(-ret) << dendl;
+ return;
}
- rgw_bucket& actual_bucket = bucket_info.bucket;
-
- if (actual_bucket.name.compare(bucket.name) != 0 ||
- actual_bucket.pool.compare(bucket.pool) != 0 ||
- actual_bucket.marker.compare(bucket.marker) != 0 ||
- actual_bucket.bucket_id.compare(bucket.bucket_id) != 0) {
- cout << "bucket info mismatch: expected " << actual_bucket << " got " << bucket << std::endl;
- if (fix) {
- cout << "fixing" << std::endl;
- r = rgw_add_bucket(store, user_id, actual_bucket);
- if (r < 0) {
- cerr << "failed to fix bucket: " << cpp_strerror(-r) << std::endl;
+ map<string, RGWBucketEnt>& buckets = user_buckets.get_buckets();
+ for (map<string, RGWBucketEnt>::iterator i = buckets.begin();
+ i != buckets.end();
+ ++i) {
+ marker = i->first;
+
+ RGWBucketEnt& bucket_ent = i->second;
+ rgw_bucket& bucket = bucket_ent.bucket;
+
+ RGWBucketInfo bucket_info;
+ int r = store->get_bucket_info(NULL, bucket.name, bucket_info);
+ if (r < 0) {
+ ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket << dendl;
+ continue;
+ }
+
+ rgw_bucket& actual_bucket = bucket_info.bucket;
+
+ if (actual_bucket.name.compare(bucket.name) != 0 ||
+ actual_bucket.pool.compare(bucket.pool) != 0 ||
+ actual_bucket.marker.compare(bucket.marker) != 0 ||
+ actual_bucket.bucket_id.compare(bucket.bucket_id) != 0) {
+ cout << "bucket info mismatch: expected " << actual_bucket << " got " << bucket << std::endl;
+ if (fix) {
+ cout << "fixing" << std::endl;
+ r = rgw_add_bucket(store, user_id, actual_bucket);
+ if (r < 0) {
+ cerr << "failed to fix bucket: " << cpp_strerror(-r) << std::endl;
+ }
}
}
}
- }
+ done = (buckets.size() < max_entries);
+ } while (!done);
}
static bool bucket_object_check_filter(const string& name)
if (r < 0)
return r;
- r = rgw_read_user_buckets(store, user_id, user_buckets, true);
- if (r < 0)
- return r;
-
- op_state.set_user_buckets(user_buckets);
op_state.display_name = info.display_name;
}
Formatter *formatter = flusher.get_formatter();
flusher.start(0);
+ CephContext *cct = store->ctx();
+
+ size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk;
+
bool show_stats = op_state.will_fetch_stats();
if (op_state.is_user_op()) {
formatter->open_array_section("buckets");
- RGWUserBuckets buckets = op_state.get_user_buckets();
- map<string, RGWBucketEnt>& m = buckets.get_buckets();
- map<string, RGWBucketEnt>::iterator iter;
-
- for (iter = m.begin(); iter != m.end(); ++iter) {
- std::string obj_name = iter->first;
- if (show_stats)
- bucket_stats(store, obj_name, formatter);
- else
- formatter->dump_string("bucket", obj_name);
- }
+ RGWUserBuckets buckets;
+ string marker;
+ bool done;
+
+ do {
+ ret = rgw_read_user_buckets(store, op_state.get_user_id(), buckets, marker, max_entries, false);
+ if (ret < 0)
+ return ret;
+
+ map<string, RGWBucketEnt>& m = buckets.get_buckets();
+ map<string, RGWBucketEnt>::iterator iter;
+
+ for (iter = m.begin(); iter != m.end(); ++iter) {
+ std::string obj_name = iter->first;
+ if (show_stats)
+ bucket_stats(store, obj_name, formatter);
+ else
+ formatter->dump_string("bucket", obj_name);
+
+ marker = obj_name;
+ }
+
+ flusher.flush();
+ done = (m.size() < max_entries);
+ } while (!done);
formatter->close_section();
} else if (!bucket_name.empty()) {
* Get all the buckets owned by a user and fill up an RGWUserBuckets with them.
* Returns: 0 on success, -ERR# on failure.
*/
-extern int rgw_read_user_buckets(RGWRados *store, string user_id, RGWUserBuckets& buckets, bool need_stats);
+extern int rgw_read_user_buckets(RGWRados *store, string user_id, RGWUserBuckets& buckets,
+ const string& marker, uint64_t max, bool need_stats);
/**
* Store the set of buckets associated with a user.
bool bucket_stored;
rgw_bucket bucket;
- RGWUserBuckets buckets;
void set_fetch_stats(bool value) { stat_buckets = value; }
void set_check_objects(bool value) { check_objects = value; }
bucket_stored = true;
}
- RGWUserBuckets& get_user_buckets() { return buckets; };
- void set_user_buckets(RGWUserBuckets& _buckets) { buckets = _buckets; };
-
bool will_fetch_stats() { return stat_buckets; };
bool will_fix_index() { return fix_index; };
bool will_delete_children() { return delete_child_objects; };
if (ret < 0)
return;
- ret = rgw_read_user_buckets(store, s->user.user_id, buckets, !!(s->prot_flags & RGW_REST_SWIFT));
- if (ret < 0) {
- /* hmm.. something wrong here.. the user was authenticated, so it
- should exist, just try to recreate */
- ldout(s->cct, 10) << "WARNING: failed on rgw_get_user_buckets uid=" << s->user.user_id << dendl;
+ bool done;
- /*
+ bool started = false;
+ uint64_t total_count = 0;
- on a second thought, this is probably a bug and we should fail
+ size_t max_buckets = s->cct->_conf->rgw_list_buckets_max_chunk;
- rgw_put_user_buckets(s->user.user_id, buckets);
- ret = 0;
+ do {
+ RGWUserBuckets buckets;
+ uint64_t read_count = min(limit - total_count, max_buckets);
+ ret = rgw_read_user_buckets(store, s->user.user_id, buckets,
+ marker, read_count, !!(s->prot_flags & RGW_REST_SWIFT));
- */
- }
+ if (!started) {
+ send_response_begin();
+ started = true;
+ }
+
+ if (ret < 0) {
+ /* hmm.. something wrong here.. the user was authenticated, so it
+ should exist, just try to recreate */
+ ldout(s->cct, 10) << "WARNING: failed on rgw_get_user_buckets uid=" << s->user.user_id << dendl;
+
+ /*
+
+ on a second thought, this is probably a bug and we should fail
+
+ rgw_put_user_buckets(s->user.user_id, buckets);
+ ret = 0;
+
+ */
+ break;
+ }
+ map<string, RGWBucketEnt>& m = buckets.get_buckets();
+
+ total_count += m.size();
+
+ done = (m.size() < read_count);
+
+ if (m.size()) {
+ send_response_data(buckets);
+
+ map<string, RGWBucketEnt>::reverse_iterator riter = m.rbegin();
+ marker = riter->first;
+ }
+ } while (!done);
+
+ send_response_end();
}
int RGWStatAccount::verify_permission()
void RGWStatAccount::execute()
{
- RGWUserBuckets buckets;
+ string marker;
+ bool done;
+ size_t max_buckets = s->cct->_conf->rgw_list_buckets_max_chunk;
- ret = rgw_read_user_buckets(store, s->user.user_id, buckets, true);
- if (ret < 0) {
- /* hmm.. something wrong here.. the user was authenticated, so it
- should exist, just try to recreate */
- ldout(s->cct, 10) << "WARNING: failed on rgw_get_user_buckets uid=" << s->user.user_id << dendl;
+ do {
+ RGWUserBuckets buckets;
- /*
+ ret = rgw_read_user_buckets(store, s->user.user_id, buckets, marker, max_buckets, true);
+ if (ret < 0) {
+ /* hmm.. something wrong here.. the user was authenticated, so it
+ should exist, just try to recreate */
+ ldout(s->cct, 10) << "WARNING: failed on rgw_get_user_buckets uid=" << s->user.user_id << dendl;
- on a second thought, this is probably a bug and we should fail
+ /*
- rgw_put_user_buckets(s->user.user_id, buckets);
- ret = 0;
+ on a second thought, this is probably a bug and we should fail
- */
- } else {
- map<string, RGWBucketEnt>& m = buckets.get_buckets();
- map<string, RGWBucketEnt>::iterator iter;
- for (iter = m.begin(); iter != m.end(); ++iter) {
- RGWBucketEnt& bucket = iter->second;
- buckets_size += bucket.size;
- buckets_size_rounded += bucket.size_rounded;
- buckets_objcount += bucket.count;
+ rgw_put_user_buckets(s->user.user_id, buckets);
+ ret = 0;
+
+ */
+
+ break;
+ } else {
+ map<string, RGWBucketEnt>& m = buckets.get_buckets();
+ map<string, RGWBucketEnt>::iterator iter;
+ for (iter = m.begin(); iter != m.end(); ++iter) {
+ RGWBucketEnt& bucket = iter->second;
+ buckets_size += bucket.size;
+ buckets_size_rounded += bucket.size_rounded;
+ buckets_objcount += bucket.count;
+ }
+ buckets_count = m.size();
+
+ done = (m.size() < max_buckets);
}
- buckets_count = m.size();
- }
+ } while (!done);
}
int RGWStatBucket::verify_permission()
if (s->user.max_buckets) {
RGWUserBuckets buckets;
- int ret = rgw_read_user_buckets(store, s->user.user_id, buckets, false);
+ string marker;
+ int ret = rgw_read_user_buckets(store, s->user.user_id, buckets, marker, s->user.max_buckets, false);
if (ret < 0)
return ret;
- if (buckets.count() >= s->user.max_buckets) {
+ map<string, RGWBucketEnt>& m = buckets.get_buckets();
+ if (m.size() >= s->user.max_buckets) {
return -ERR_TOO_MANY_BUCKETS;
}
}
class RGWListBuckets : public RGWOp {
protected:
int ret;
- RGWUserBuckets buckets;
+ bool sent_data;
+ string marker;
+ uint64_t limit;
+ uint64_t limit_max;
public:
- RGWListBuckets() : ret(0) {}
+ RGWListBuckets() : ret(0), sent_data(false) {
+ limit = limit_max = 10000;
+ }
int verify_permission();
void execute();
virtual int get_params() = 0;
- virtual void send_response() = 0;
+ virtual void send_response_begin() = 0;
+ virtual void send_response_data(RGWUserBuckets& buckets) = 0;
+ virtual void send_response_end() = 0;
+ virtual void send_response() {}
virtual const char *name() { return "list_buckets"; }
};
return ret;
}
-int RGWRados::omap_get_all(rgw_obj& obj, bufferlist& header, std::map<string, bufferlist>& m)
+int RGWRados::omap_get_vals(rgw_obj& obj, bufferlist& header, const string& marker, uint64_t count, std::map<string, bufferlist>& m)
{
bufferlist bl;
librados::IoCtx io_ctx;
io_ctx.locator_set_key(key);
- string start_after;
- r = io_ctx.omap_get_vals(oid, start_after, -1, &m);
+ r = io_ctx.omap_get_vals(oid, marker, count, &m);
if (r < 0)
return r;
}
+int RGWRados::omap_get_all(rgw_obj& obj, bufferlist& header, std::map<string, bufferlist>& m)
+{
+ string start_after;
+
+ return omap_get_vals(obj, header, start_after, (uint64_t)-1, m);
+}
+
int RGWRados::omap_set(rgw_obj& obj, std::string& key, bufferlist& bl)
{
rgw_bucket bucket;
uint64_t *epoch, map<string, bufferlist> *attrs, bufferlist *first_chunk);
virtual bool supports_omap() { return true; }
+ int omap_get_vals(rgw_obj& obj, bufferlist& header, const std::string& marker, uint64_t count, std::map<string, bufferlist>& m);
virtual int omap_get_all(rgw_obj& obj, bufferlist& header, std::map<string, bufferlist>& m);
virtual int omap_set(rgw_obj& obj, std::string& key, bufferlist& bl);
virtual int omap_set(rgw_obj& obj, map<std::string, bufferlist>& m);
return 0;
}
-void RGWListBuckets_ObjStore_S3::send_response()
+void RGWListBuckets_ObjStore_S3::send_response_begin()
{
if (ret)
set_req_state_err(s, ret);
dump_errno(s);
dump_start(s);
+ end_header(s, "application/xml");
- list_all_buckets_start(s);
- dump_owner(s, s->user.user_id, s->user.display_name);
+ if (!ret) {
+ list_all_buckets_start(s);
+ dump_owner(s, s->user.user_id, s->user.display_name);
+ s->formatter->open_array_section("Buckets");
+ sent_data = true;
+ }
+}
+void RGWListBuckets_ObjStore_S3::send_response_data(RGWUserBuckets& buckets)
+{
map<string, RGWBucketEnt>& m = buckets.get_buckets();
map<string, RGWBucketEnt>::iterator iter;
- s->formatter->open_array_section("Buckets");
for (iter = m.begin(); iter != m.end(); ++iter) {
RGWBucketEnt obj = iter->second;
dump_bucket(s, obj);
}
- s->formatter->close_section();
- list_all_buckets_end(s);
- dump_content_length(s, s->formatter->get_len());
- end_header(s, "application/xml");
+ rgw_flush_formatter(s, s->formatter);
+}
+
+void RGWListBuckets_ObjStore_S3::send_response_end()
+{
+ if (sent_data) {
+ s->formatter->close_section();
+ list_all_buckets_end(s);
+ }
rgw_flush_formatter_and_reset(s, s->formatter);
}
~RGWListBuckets_ObjStore_S3() {}
int get_params() { return 0; }
- void send_response();
+ virtual void send_response_begin();
+ virtual void send_response_data(RGWUserBuckets& buckets);
+ virtual void send_response_end();
};
class RGWListBucket_ObjStore_S3 : public RGWListBucket_ObjStore {
marker = s->args.get("marker");
string limit_str;
limit_str = s->args.get("limit");
- limit = strtol(limit_str.c_str(), NULL, 10);
- if (limit > limit_max || limit < 0)
+ long l = strtol(limit_str.c_str(), NULL, 10);
+ if (l > (long)limit_max || l < 0)
return -ERR_PRECONDITION_FAILED;
+ limit = (uint64_t)l;
+
if (limit == 0)
limit = limit_max;
return 0;
}
-void RGWListBuckets_ObjStore_SWIFT::send_response()
+void RGWListBuckets_ObjStore_SWIFT::send_response_begin()
{
- map<string, RGWBucketEnt>& m = buckets.get_buckets();
- map<string, RGWBucketEnt>::iterator iter;
-
- if (ret < 0)
- goto done;
+ if (ret)
+ set_req_state_err(s, ret);
+ dump_errno(s);
+ end_header(s);
- dump_start(s);
+ if (!ret) {
+ dump_start(s);
+ s->formatter->open_array_section("account");
- s->formatter->open_array_section("account");
+ sent_data = true;
+ }
+}
- if (marker.empty())
- iter = m.begin();
- else
- iter = m.upper_bound(marker);
+void RGWListBuckets_ObjStore_SWIFT::send_response_data(RGWUserBuckets& buckets)
+{
+ map<string, RGWBucketEnt>& m = buckets.get_buckets();
+ map<string, RGWBucketEnt>::iterator iter;
- for (int i = 0; i < limit && iter != m.end(); ++iter, ++i) {
+ for (iter = m.begin(); iter != m.end(); ++iter) {
RGWBucketEnt obj = iter->second;
s->formatter->open_object_section("container");
s->formatter->dump_string("name", obj.bucket.name);
s->formatter->dump_int("count", obj.count);
s->formatter->dump_int("bytes", obj.size);
s->formatter->close_section();
+ rgw_flush_formatter(s, s->formatter);
}
- s->formatter->close_section();
-
- if (!ret && s->formatter->get_len() == 0)
- ret = STATUS_NO_CONTENT;
-done:
- set_req_state_err(s, ret);
- dump_errno(s);
- end_header(s);
+}
- if (ret < 0) {
- return;
+void RGWListBuckets_ObjStore_SWIFT::send_response_end()
+{
+ if (sent_data) {
+ s->formatter->close_section();
}
rgw_flush_formatter_and_reset(s, s->formatter);
};
class RGWListBuckets_ObjStore_SWIFT : public RGWListBuckets_ObjStore {
- int limit_max;
- int limit;
- string marker;
public:
- RGWListBuckets_ObjStore_SWIFT() {
- limit_max = 10000;
- limit = limit_max;
- }
+ RGWListBuckets_ObjStore_SWIFT() {}
~RGWListBuckets_ObjStore_SWIFT() {}
int get_params();
- void send_response();
+ void send_response_begin();
+ void send_response_data(RGWUserBuckets& buckets);
+ void send_response_end();
};
class RGWListBucket_ObjStore_SWIFT : public RGWListBucket_ObjStore {
* themselves alone, as well as any ACLs embedded in object xattrs.
*/
int rgw_delete_user(RGWRados *store, RGWUserInfo& info) {
- RGWUserBuckets user_buckets;
- int ret = rgw_read_user_buckets(store, info.user_id, user_buckets, false);
- if (ret < 0)
- return ret;
-
- map<string, RGWBucketEnt>& buckets = user_buckets.get_buckets();
+ string marker;
vector<rgw_bucket> buckets_vec;
- for (map<string, RGWBucketEnt>::iterator i = buckets.begin();
- i != buckets.end();
- ++i) {
- RGWBucketEnt& bucket = i->second;
- buckets_vec.push_back(bucket.bucket);
- }
+
+ bool done;
+ int ret;
+ CephContext *cct = store->ctx();
+ size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk;
+
+ do {
+ RGWUserBuckets user_buckets;
+ ret = rgw_read_user_buckets(store, info.user_id, user_buckets, marker, max_buckets, false);
+ if (ret < 0)
+ return ret;
+
+ map<string, RGWBucketEnt>& buckets = user_buckets.get_buckets();
+ for (map<string, RGWBucketEnt>::iterator i = buckets.begin();
+ i != buckets.end();
+ ++i) {
+ RGWBucketEnt& bucket = i->second;
+ buckets_vec.push_back(bucket.bucket);
+
+ marker = i->first;
+ }
+
+ done = (buckets.size() < max_buckets);
+ } while (!done);
+
map<string, RGWAccessKey>::iterator kiter = info.access_keys.begin();
for (; kiter != info.access_keys.end(); ++kiter) {
ldout(store->ctx(), 10) << "removing key index: " << kiter->first << dendl;
return -EINVAL;
}
- RGWUserBuckets buckets;
- ret = rgw_read_user_buckets(store, uid, buckets, false);
- if (ret < 0) {
- set_err_msg(err_msg, "unable to read user bucket info");
- return ret;
- }
+ bool done;
+ string marker;
+ CephContext *cct = store->ctx();
+ size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk;
+ do {
+ RGWUserBuckets buckets;
+ ret = rgw_read_user_buckets(store, uid, buckets, marker, max_buckets, false);
+ if (ret < 0) {
+ set_err_msg(err_msg, "unable to read user bucket info");
+ return ret;
+ }
- map<std::string, RGWBucketEnt>& m = buckets.get_buckets();
- if (!m.empty() && !purge_data) {
- set_err_msg(err_msg, "must specify purge data to remove user with buckets");
- return -EEXIST; // change to code that maps to 409: conflict
- }
+ map<std::string, RGWBucketEnt>& m = buckets.get_buckets();
+ if (!m.empty() && !purge_data) {
+ set_err_msg(err_msg, "must specify purge data to remove user with buckets");
+ return -EEXIST; // change to code that maps to 409: conflict
+ }
- if (!m.empty()) {
std::map<std::string, RGWBucketEnt>::iterator it;
for (it = m.begin(); it != m.end(); ++it) {
ret = rgw_remove_bucket(store, ((*it).second).bucket, true);
set_err_msg(err_msg, "unable to delete user data");
return ret;
}
+
+ marker = it->first;
}
- }
+
+ done = (m.size() < max_buckets);
+ } while (!done);
ret = rgw_delete_user(store, user_info);
if (ret < 0) {
return -EINVAL;
}
- ret = rgw_read_user_buckets(store, user_id, buckets, false);
- if (ret < 0) {
- set_err_msg(err_msg, "could not get buckets for uid: " + user_id);
- return ret;
- }
+ bool done;
+ string marker;
+ CephContext *cct = store->ctx();
+ size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk;
+ do {
+ ret = rgw_read_user_buckets(store, user_id, buckets, marker, max_buckets, false);
+ if (ret < 0) {
+ set_err_msg(err_msg, "could not get buckets for uid: " + user_id);
+ return ret;
+ }
- map<string, RGWBucketEnt>& m = buckets.get_buckets();
- map<string, RGWBucketEnt>::iterator iter;
+ map<string, RGWBucketEnt>& m = buckets.get_buckets();
+ map<string, RGWBucketEnt>::iterator iter;
- vector<rgw_bucket> bucket_names;
- for (iter = m.begin(); iter != m.end(); ++iter) {
- RGWBucketEnt obj = iter->second;
- bucket_names.push_back(obj.bucket);
- }
+ vector<rgw_bucket> bucket_names;
+ for (iter = m.begin(); iter != m.end(); ++iter) {
+ RGWBucketEnt obj = iter->second;
+ bucket_names.push_back(obj.bucket);
- ret = store->set_buckets_enabled(bucket_names, !suspended);
- if (ret < 0) {
- set_err_msg(err_msg, "failed to change pool");
- return ret;
- }
+ marker = iter->first;
+ }
+
+ ret = store->set_buckets_enabled(bucket_names, !suspended);
+ if (ret < 0) {
+ set_err_msg(err_msg, "failed to modify bucket");
+ return ret;
+ }
+
+ done = (m.size() < max_buckets);
+ } while (!done);
}
op_state.set_user_info(user_info);
projects / ceph.git / commitdiff