From e9e3feeaf5b5ebc8fa905e7e0eb26557ce953cb4 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Wed, 15 Jun 2011 17:06:55 -0700 Subject: [PATCH] rgw: implement list multiparts doesn't work completely, getting there --- src/rgw/rgw_access.h | 8 +++++- src/rgw/rgw_fs.cc | 17 ++++++++---- src/rgw/rgw_fs.h | 2 +- src/rgw/rgw_op.cc | 62 ++++++++++++++++++++++++++++++++++++++++-- src/rgw/rgw_op.h | 25 ++++++++++++++--- src/rgw/rgw_rados.cc | 49 +++++++++++++++++++++------------ src/rgw/rgw_rados.h | 2 +- src/rgw/rgw_rest.cc | 8 ++++-- src/rgw/rgw_rest.h | 2 +- src/rgw/rgw_rest_s3.cc | 33 +++++++++++++--------- 10 files changed, 159 insertions(+), 49 deletions(-) diff --git a/src/rgw/rgw_access.h b/src/rgw/rgw_access.h index b10120c1dcf14..7c8944e51d007 100644 --- a/src/rgw/rgw_access.h +++ b/src/rgw/rgw_access.h @@ -11,6 +11,12 @@ struct md_config_t; +class RGWAccessListFilter { +public: + virtual ~RGWAccessListFilter() {} + virtual bool filter(string& name, string& key) = 0; +}; + /** * Abstract class defining the interface for storage devices used by RGW. */ @@ -40,7 +46,7 @@ public: */ virtual int list_objects(std::string& id, std::string& bucket, int max, std::string& prefix, std::string& delim, std::string& marker, std::vector& result, map& common_prefixes, - bool get_content_type, std::string& ns) = 0; + bool get_content_type, std::string& ns, bool *is_truncated, RGWAccessListFilter *filter) = 0; /** Create a new bucket*/ virtual int create_bucket(std::string& id, std::string& bucket, map& attrs, uint64_t auid=0) = 0; diff --git a/src/rgw/rgw_fs.cc b/src/rgw/rgw_fs.cc index f1eb79f3b33d7..84b2cd3bac036 100644 --- a/src/rgw/rgw_fs.cc +++ b/src/rgw/rgw_fs.cc @@ -96,9 +96,9 @@ int RGWFS::obj_stat(rgw_obj& obj, uint64_t *psize, time_t *pmtime) int RGWFS::list_objects(string& id, string& bucket, int max, string& prefix, string& delim, string& marker, vector& result, map& common_prefixes, - bool get_content_type, string& ns) + bool get_content_type, string& ns, bool *is_truncated, RGWAccessListFilter *filter) { - map dir_map; + map dir_map; char path[BUF_SIZE]; snprintf(path, BUF_SIZE, "%s/%s", DIR_NAME, bucket.c_str()); @@ -123,14 +123,19 @@ int RGWFS::list_objects(string& id, string& bucket, int max, string& prefix, str if (!rgw_obj::translate_raw_obj(obj, ns)) continue; - dir_map[obj] = true; + string key = obj; + + if (filter && !filter->filter(obj, key)) + continue; + + dir_map[obj] = key; } } closedir(dir); - map::iterator iter; + map::iterator iter; if (!marker.empty()) iter = dir_map.lower_bound(marker); else @@ -159,8 +164,10 @@ int RGWFS::list_objects(string& id, string& bucket, int max, string& prefix, str } result.push_back(obj); } + if (is_truncated) + *is_truncated = (iter != dir_map.end()); - return i; + return result.size(); } diff --git a/src/rgw/rgw_fs.h b/src/rgw/rgw_fs.h index 14000bb001c15..682bd653dd4e2 100644 --- a/src/rgw/rgw_fs.h +++ b/src/rgw/rgw_fs.h @@ -16,7 +16,7 @@ public: int list_objects(std::string& id, std::string& bucket, int max, std::string& prefix, std::string& delim, std::string& marker, std::vector& result, map& common_prefixes, - bool get_content_type, string& ns); + bool get_content_type, string& ns, bool *is_truncated, RGWAccessListFilter *filter); int create_bucket(std::string& id, std::string& bucket, map& attrs, uint64_t auid=0); int put_obj_meta(std::string& id, rgw_obj& obj, time_t *mtime, diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 5a996338e3c49..1d7902036be54 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -20,6 +20,30 @@ using ceph::crypto::MD5; static string mp_ns = "multipart"; +class MultipartMetaFilter : public RGWAccessListFilter { +public: + MultipartMetaFilter() {} + bool filter(string& name, string& key) { + int len = name.size(); + if (len < 6) + return false; + + int pos = name.find(".meta", len - 5); + if (pos <= 0) + return false; + + pos = name.rfind('.', pos - 1); + if (pos < 0) + return false; + + key = name.substr(0, pos); + + return true; + } +}; + +static MultipartMetaFilter mp_filter; + static int parse_range(const char *range, off_t& ofs, off_t& end) { int r = -ERANGE; @@ -117,6 +141,7 @@ int read_acls(struct req_state *s, RGWAccessControlPolicy *policy, string& bucke if (!oid.empty() && !upload_id.empty()) { oid.append("."); oid.append(upload_id); + oid.append(".meta"); obj.set_ns(mp_ns); } obj.init(bucket, oid, object); @@ -315,7 +340,8 @@ void RGWListBucket::execute() } ret = rgwstore->list_objects(s->user.user_id, s->bucket_str, max, prefix, delimiter, marker, objs, common_prefixes, - !!(s->prot_flags & RGW_REST_OPENSTACK), no_ns); + !!(s->prot_flags & RGW_REST_OPENSTACK), no_ns, NULL, NULL); + done: send_response(); } @@ -444,6 +470,7 @@ void RGWPutObj::execute() oid.append("."); oid.append(s->args.get("uploadId")); multipart_meta_obj = oid; + multipart_meta_obj.append(".meta"); part_num = s->args.get("partNumber"); if (part_num.empty()) { ret = -EINVAL; @@ -889,6 +916,7 @@ void RGWInitMultipart::execute() string tmp_obj_name = s->object_str; tmp_obj_name.append("."); tmp_obj_name.append(upload_id); + tmp_obj_name.append(".meta"); obj.init(s->bucket_str, tmp_obj_name, s->object_str, mp_ns); ret = rgwstore->put_obj_meta(s->user.user_id, obj, NULL, attrs, true); @@ -985,6 +1013,7 @@ void RGWCompleteMultipart::execute() oid.append("."); oid.append(upload_id); meta_oid = oid; + meta_oid.append(".meta"); prefix = oid; prefix.append("."); @@ -1079,6 +1108,7 @@ void RGWAbortMultipart::execute() oid.append("."); oid.append(upload_id); meta_oid = oid; + meta_oid.append(".meta"); prefix = oid; prefix.append("."); @@ -1118,6 +1148,7 @@ void RGWListMultipart::execute() obj.append("."); obj.append(upload_id); + obj.append(".meta"); ret = get_multiparts_info(s, obj, parts, policy, xattrs); done: @@ -1126,13 +1157,20 @@ done: void RGWListBucketMultiparts::execute() { + vector objs; + string marker; + if (!verify_permission(s, RGW_PERM_READ)) { ret = -EACCES; goto done; } url_decode(s->args.get("prefix"), prefix); - marker = s->args.get("marker"); + key_marker = s->args.get("key-marker"); + uploadid_marker = s->args.get("upload-id-marker-marker"); + marker = key_marker; + marker.append("."); + marker.append(uploadid_marker); max_keys = s->args.get(limit_opt_name); if (!max_keys.empty()) { max = atoi(max_keys.c_str()); @@ -1154,7 +1192,25 @@ void RGWListBucketMultiparts::execute() } ret = rgwstore->list_objects(s->user.user_id, s->bucket_str, max, prefix, delimiter, marker, objs, common_prefixes, - !!(s->prot_flags & RGW_REST_OPENSTACK), mp_ns); + !!(s->prot_flags & RGW_REST_OPENSTACK), mp_ns, &is_truncated, &mp_filter); + if (objs.size()) { + vector::iterator iter; + RGWMultipartUploadEntry entry; + for (iter = objs.begin(); iter != objs.end(); ++iter) { + string name = iter->name; + int pos = name.rfind('.'); // search for '.meta' + if (pos < 0) + continue; + pos = name.rfind('.', pos - 1); // . + if (pos < 0) + continue; + entry.key = name.substr(0, pos); + entry.upload_id = name.substr(pos + 1); + entry.obj = *iter; + uploads.push_back(entry); + } + next_marker = entry; + } done: send_response(); } diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 7e7d2d0095747..6a34996f2916f 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -416,19 +416,33 @@ public: virtual void send_response() = 0; }; +struct RGWMultipartUploadEntry { + string key; + string upload_id; + RGWObjEnt obj; + + void clear() { + key = ""; + upload_id = ""; + } +}; + class RGWListBucketMultiparts : public RGWOp { protected: string prefix; - string marker; + string key_marker; + string uploadid_marker; + RGWMultipartUploadEntry next_marker; string max_keys; string delimiter; int max; int ret; - vector objs; + vector uploads; map common_prefixes; string limit_opt_name; int default_max; + bool is_truncated; public: RGWListBucketMultiparts() {} @@ -436,12 +450,15 @@ public: virtual void init(struct req_state *s) { RGWOp::init(s); prefix.clear(); - marker.clear(); + key_marker.clear(); + uploadid_marker.clear(); + next_marker.clear(); max_keys.clear(); delimiter.clear(); max = 0; ret = 0; - objs.clear(); + uploads.clear(); + is_truncated = false; common_prefixes.clear(); } void execute(); diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 45f03031447cc..fea0d113fc7cb 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -151,77 +151,92 @@ int RGWRados::list_buckets_next(std::string& id, RGWObjEnt& obj, RGWAccessHandle */ int RGWRados::list_objects(string& id, string& bucket, int max, string& prefix, string& delim, string& marker, vector& result, map& common_prefixes, - bool get_content_type, string& ns) + bool get_content_type, string& ns, bool *is_truncated, RGWAccessListFilter *filter) { librados::IoCtx io_ctx; int r = open_bucket_ctx(bucket, io_ctx); if (r < 0) return r; - set dir_set; + std::map dir_map; { librados::ObjectIterator i_end = io_ctx.objects_end(); for (librados::ObjectIterator i = io_ctx.objects_begin(); i != i_end; ++i) { string obj = *i; + string key = obj; if (!rgw_obj::translate_raw_obj(obj, ns)) continue; + if (filter && !filter->filter(obj, key)) + continue; + if (prefix.empty() || ((obj).compare(0, prefix.size(), prefix) == 0)) { - dir_set.insert(obj); + dir_map[obj] = key; } } } - set::iterator p; + std::map::iterator p; if (!marker.empty()) - p = dir_set.lower_bound(marker); + p = dir_map.lower_bound(marker); else - p = dir_set.begin(); + p = dir_map.begin(); if (max < 0) { - max = dir_set.size(); + max = dir_map.size(); } result.clear(); - int i, count = 0; - for (i=0; ifirst; + string key = p->second; + obj.name = name; if (!delim.empty()) { - int delim_pos = obj.name.find(delim, prefix.size()); + int delim_pos = name.find(delim, prefix.size()); if (delim_pos >= 0) { - common_prefixes[obj.name.substr(0, delim_pos + 1)] = true; + common_prefixes[name.substr(0, delim_pos + 1)] = true; continue; } } + string oid = name; + if (!ns.empty()) { + oid = "_"; + oid.append(ns); + oid.append("_"); + oid.append(name); + } uint64_t s; - string p_str = *p; - if (io_ctx.stat(*p, &s, &obj.mtime) < 0) + io_ctx.locator_set_key(key); + if (io_ctx.stat(oid, &s, &obj.mtime) < 0) continue; obj.size = s; bufferlist bl; obj.etag[0] = '\0'; - if (io_ctx.getxattr(*p, RGW_ATTR_ETAG, bl) >= 0) { + if (io_ctx.getxattr(oid, RGW_ATTR_ETAG, bl) >= 0) { strncpy(obj.etag, bl.c_str(), sizeof(obj.etag)); obj.etag[sizeof(obj.etag)-1] = '\0'; } if (get_content_type) { bl.clear(); obj.content_type = ""; - if (io_ctx.getxattr(*p, RGW_ATTR_CONTENT_TYPE, bl) >= 0) { + if (io_ctx.getxattr(oid, RGW_ATTR_CONTENT_TYPE, bl) >= 0) { obj.content_type = bl.c_str(); } } result.push_back(obj); } + if (is_truncated) + *is_truncated = (p != dir_map.end()); - return count; + return 0; } /** diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index aea5be318b314..148b8d727f180 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -33,7 +33,7 @@ public: /** get listing of the objects in a bucket */ virtual int list_objects(std::string& id, std::string& bucket, int max, std::string& prefix, std::string& delim, std::string& marker, std::vector& result, map& common_prefixes, - bool get_content_type, string& ns); + bool get_content_type, string& ns, bool *is_truncated, RGWAccessListFilter *filter); /** * create a bucket with name bucket and the given list of attrs diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index f9218e7d9ee4b..00f9004160b8a 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -138,12 +138,14 @@ void dump_time(struct req_state *s, const char *name, time_t *t) s->formatter->dump_value_str(name, buf); } -void dump_owner(struct req_state *s, string& id, string& name) +void dump_owner(struct req_state *s, string& id, string& name, const char *section) { - s->formatter->open_obj_section("Owner"); + if (!section) + section = "Owner"; + s->formatter->open_obj_section(section); s->formatter->dump_value_str("ID", id.c_str()); s->formatter->dump_value_str("DisplayName", name.c_str()); - s->formatter->close_section("Owner"); + s->formatter->close_section(section); } void dump_start(struct req_state *s) diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h index a50ca9da64612..f2431ebff89af 100644 --- a/src/rgw/rgw_rest.h +++ b/src/rgw/rgw_rest.h @@ -153,7 +153,7 @@ extern void dump_errno(struct req_state *s, int ret); extern void end_header(struct req_state *s, const char *content_type = NULL); extern void dump_start(struct req_state *s); extern void list_all_buckets_start(struct req_state *s); -extern void dump_owner(struct req_state *s, string& id, string& name); +extern void dump_owner(struct req_state *s, string& id, string& name, const char *section = NULL); extern void dump_content_length(struct req_state *s, size_t len); extern void dump_etag(struct req_state *s, const char *etag); extern void dump_last_modified(struct req_state *s, time_t t); diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index c209c99c2dcce..be2d4587a5152 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -357,37 +357,44 @@ void RGWListBucketMultiparts_REST_S3::send_response() s->formatter->open_obj_section("ListMultipartUploadsResult"); s->formatter->dump_value_str("Bucket", s->bucket); if (!prefix.empty()) - s->formatter->dump_value_str("Prefix", prefix.c_str()); - if (!marker.empty()) - s->formatter->dump_value_str("KeyMarker", marker.c_str()); + s->formatter->dump_value_str("ListMultipartUploadsResult.Prefix", prefix.c_str()); + if (!key_marker.empty()) + s->formatter->dump_value_str("KeyMarker", key_marker.c_str()); + if (!uploadid_marker.empty()) + s->formatter->dump_value_str("UploadIdMarker", uploadid_marker.c_str()); + if (!next_marker.key.empty()) + s->formatter->dump_value_str("NextKeyMarker", next_marker.key.c_str()); + if (!next_marker.upload_id.empty()) + s->formatter->dump_value_str("NextUploadIdMarker", next_marker.upload_id.c_str()); if (!max_keys.empty()) { - s->formatter->dump_value_str("MaxKeys", max_keys.c_str()); + s->formatter->dump_value_str("MaxUploads", max_keys.c_str()); } if (!delimiter.empty()) s->formatter->dump_value_str("Delimiter", delimiter.c_str()); + s->formatter->dump_value_str("IsTruncated", (is_truncated ? "true" : "false")); if (ret >= 0) { - vector::iterator iter; - for (iter = objs.begin(); iter != objs.end(); ++iter) { + vector::iterator iter; + for (iter = uploads.begin(); iter != uploads.end(); ++iter) { s->formatter->open_array_section("Upload"); - s->formatter->dump_value_str("Key", iter->name.c_str()); - dump_time(s, "LastModified", &iter->mtime); - s->formatter->dump_value_str("ETag", "\"%s\"", iter->etag); - s->formatter->dump_value_int("Size", "%lld", iter->size); - s->formatter->dump_value_str("StorageClass", "STANDARD"); + s->formatter->dump_value_str("Key", iter->key.c_str()); + s->formatter->dump_value_str("UploadId", iter->upload_id.c_str()); + dump_owner(s, s->user.user_id, s->user.display_name, "Initiator"); dump_owner(s, s->user.user_id, s->user.display_name); + s->formatter->dump_value_str("StorageClass", "STANDARD"); + dump_time(s, "Initiated", &iter->obj.mtime); s->formatter->close_section("Upload"); } if (common_prefixes.size() > 0) { s->formatter->open_array_section("CommonPrefixes"); map::iterator pref_iter; for (pref_iter = common_prefixes.begin(); pref_iter != common_prefixes.end(); ++pref_iter) { - s->formatter->dump_value_str("Prefix", pref_iter->first.c_str()); + s->formatter->dump_value_str("CommonPrefixes.Prefix", pref_iter->first.c_str()); } s->formatter->close_section("CommonPrefixes"); } } - s->formatter->close_section("ListBucketResult"); + s->formatter->close_section("ListMultipartUploadsResult"); s->formatter->flush(); } -- 2.39.5