From 29bbcee9e68c7f031c441724a0982ffa33e62c59 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Fri, 23 Sep 2011 15:46:00 -0700 Subject: [PATCH] rgw: update index when doing put_obj_meta this fixes multipart upload listing --- src/rgw/rgw_access.h | 9 +-- src/rgw/rgw_fs.cc | 2 +- src/rgw/rgw_fs.h | 2 +- src/rgw/rgw_op.cc | 7 ++- src/rgw/rgw_rados.cc | 127 +++++++++++++++++++++++++++---------------- src/rgw/rgw_rados.h | 8 ++- 6 files changed, 95 insertions(+), 60 deletions(-) diff --git a/src/rgw/rgw_access.h b/src/rgw/rgw_access.h index 849cfadd2f20c..cc3d037ff77fc 100644 --- a/src/rgw/rgw_access.h +++ b/src/rgw/rgw_access.h @@ -61,25 +61,20 @@ public: virtual int create_pools(std::string& id, vector& names, vector& retcodes, int auid = 0) { return -ENOTSUP; } /** write an object to the storage device in the appropriate pool with the given stats */ - virtual int put_obj_meta(void *ctx, std::string& id, rgw_obj& obj, time_t *mtime, + virtual int put_obj_meta(void *ctx, std::string& id, rgw_obj& obj, uint64_t size, time_t *mtime, map& attrs, string& category, bool exclusive) = 0; virtual int put_obj_data(void *ctx, std::string& id, rgw_obj& obj, const char *data, off_t ofs, size_t len) = 0; virtual int aio_put_obj_data(void *ctx, std::string& id, rgw_obj& obj, const char *data, off_t ofs, size_t len, void **handle) { return -ENOTSUP; } - virtual int complete_put_obj(void *ctx, std::string& id, rgw_obj& obj, size_t len) { return 0; } - /* note that put_obj doesn't set category on an object, only use it for none user objects */ int put_obj(void *ctx, std::string& id, rgw_obj& obj, const char *data, size_t len, time_t *mtime, map& attrs) { int ret = put_obj_data(ctx, id, obj, data, -1, len); if (ret >= 0) { string category; - ret = put_obj_meta(ctx, id, obj, mtime, attrs, category, false); - } - if (ret >= 0) { - ret = complete_put_obj(ctx, id, obj, len); + ret = put_obj_meta(ctx, id, obj, len, mtime, attrs, category, false); } return ret; } diff --git a/src/rgw/rgw_fs.cc b/src/rgw/rgw_fs.cc index 1431b90668b88..d90820ce0ca5a 100644 --- a/src/rgw/rgw_fs.cc +++ b/src/rgw/rgw_fs.cc @@ -203,7 +203,7 @@ int RGWFS::create_bucket(std::string& id, rgw_bucket& bucket, map& attrs, + uint64_t size, time_t *mtime, map& attrs, string& category, bool exclusive) { rgw_bucket& bucket = obj.bucket; diff --git a/src/rgw/rgw_fs.h b/src/rgw/rgw_fs.h index 09fe24c758b5b..04262e8f40fd8 100644 --- a/src/rgw/rgw_fs.h +++ b/src/rgw/rgw_fs.h @@ -19,7 +19,7 @@ public: bool get_content_type, string& ns, bool *is_truncated, RGWAccessListFilter *filter); int create_bucket(std::string& id, rgw_bucket& bucket, map& attrs, bool create_pool, bool assign_marker, bool exclusive, uint64_t auid=0); - int put_obj_meta(void *ctx, std::string& id, rgw_obj& obj, time_t *mtime, + int put_obj_meta(void *ctx, std::string& id, rgw_obj& obj, uint64_t size, time_t *mtime, map& attrs, string& category, bool exclusive); int put_obj_data(void *ctx, std::string& id, rgw_obj& obj, const char *data, off_t ofs, size_t size); diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 42fa61af9c502..9a428363acf68 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -746,7 +746,7 @@ void RGWPutObj::execute() goto done; } } else { - ret = rgwstore->put_obj_meta(s->obj_ctx, s->user.user_id, obj, NULL, attrs, rgw_obj_category_main, false); + ret = rgwstore->put_obj_meta(s->obj_ctx, s->user.user_id, obj, s->obj_size, NULL, attrs, rgw_obj_category_main, false); if (ret < 0) goto done_err; @@ -1182,7 +1182,8 @@ void RGWInitMultipart::execute() tmp_obj_name = mp.get_meta(); obj.init(s->bucket, tmp_obj_name, s->object_str, mp_ns); - ret = rgwstore->put_obj_meta(s->obj_ctx, s->user.user_id, obj, NULL, attrs, rgw_obj_category_multimeta, true); + // the meta object will be indexed with 0 size, we c + ret = rgwstore->put_obj_meta(s->obj_ctx, s->user.user_id, obj, 0, NULL, attrs, rgw_obj_category_multimeta, true); } while (ret == -EEXIST); done: send_response(); @@ -1335,7 +1336,7 @@ void RGWCompleteMultipart::execute() target_obj.init(s->bucket, s->object_str); rgwstore->set_atomic(s->obj_ctx, target_obj); - ret = rgwstore->put_obj_meta(s->obj_ctx, s->user.user_id, target_obj, NULL, attrs, rgw_obj_category_main, false); + ret = rgwstore->put_obj_meta(s->obj_ctx, s->user.user_id, target_obj, 0, NULL, attrs, rgw_obj_category_main, false); if (ret < 0) goto done; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 680d7d8c6418e..ee3a23d120561 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -392,7 +392,7 @@ int RGWRados::create_pools(std::string& id, vector& names, vector& * exclusive: create object exclusively * Returns: 0 on success, -ERR# otherwise. */ -int RGWRados::put_obj_meta(void *ctx, std::string& id, rgw_obj& obj, +int RGWRados::put_obj_meta(void *ctx, std::string& id, rgw_obj& obj, uint64_t size, time_t *mtime, map& attrs, string& category, bool exclusive) { rgw_bucket bucket; @@ -413,23 +413,45 @@ int RGWRados::put_obj_meta(void *ctx, std::string& id, rgw_obj& obj, else op.create(exclusive); + string etag; + bufferlist acl_bl; + map::iterator iter; for (iter = attrs.begin(); iter != attrs.end(); ++iter) { const string& name = iter->first; bufferlist& bl = iter->second; - if (bl.length()) { - op.setxattr(name.c_str(), bl); + if (!bl.length()) + continue; + + op.setxattr(name.c_str(), bl); + + if (name.compare(RGW_ATTR_ETAG) == 0) { + etag = bl.c_str(); + } else if (name.compare(RGW_ATTR_ACL) == 0) { + acl_bl = bl; } } if (!op.size()) return 0; + string tag; + r = prepare_update_index(NULL, bucket, obj.object, tag); + if (r < 0) + return r; + r = io_ctx.operate(oid, &op); if (r < 0) return r; + uint64_t epoch = io_ctx.get_last_version(); + + uint8_t index_category = 0; + utime_t ut = ceph_clock_now(g_ceph_context); + r = complete_update_index(bucket, obj.object, tag, epoch, size, + ut, etag, &acl_bl, index_category); + if (mtime) { r = io_ctx.stat(oid, NULL, mtime); if (r < 0) @@ -781,18 +803,9 @@ int RGWRados::delete_obj_impl(void *ctx, std::string& id, rgw_obj& obj, bool syn string tag; op.remove(); if (sync) { - if (state && state->obj_tag.length()) { - int len = state->obj_tag.length(); - char buf[len + 1]; - memcpy(buf, state->obj_tag.c_str(), len); - buf[len] = '\0'; - tag = buf; - } else { - append_rand_alpha(tag, tag, 32); - } - int ret = cls_obj_prepare_op(bucket, CLS_RGW_OP_ADD, tag, obj.object); - if (ret < 0) - return ret; + r = prepare_update_index(state, bucket, obj.object, tag); + if (r < 0) + return r; r = io_ctx.operate(oid, &op); } else { librados::AioCompletion *completion = rados->aio_create_completion(NULL, NULL, NULL); @@ -804,7 +817,7 @@ int RGWRados::delete_obj_impl(void *ctx, std::string& id, rgw_obj& obj, bool syn if (r >= 0 && bucket.marker.size()) { uint64_t epoch = io_ctx.get_last_version(); - r = cls_obj_complete_del(bucket, tag, epoch, obj.object); + r = complete_update_index_del(bucket, obj.object, tag, epoch); } if (r < 0) @@ -1253,6 +1266,49 @@ done_err: return r; } +int RGWRados::prepare_update_index(RGWObjState *state, rgw_bucket& bucket, string& oid, string& tag) +{ + if (state && state->obj_tag.length()) { + int len = state->obj_tag.length(); + char buf[len + 1]; + memcpy(buf, state->obj_tag.c_str(), len); + buf[len] = '\0'; + tag = buf; + } else { + append_rand_alpha(tag, tag, 32); + } + int ret = cls_obj_prepare_op(bucket, CLS_RGW_OP_ADD, tag, oid); + + return ret; +} + +int RGWRados::complete_update_index(rgw_bucket& bucket, string& oid, string& tag, uint64_t epoch, uint64_t size, + utime_t& ut, string& etag, bufferlist *acl_bl, uint8_t category) +{ + if (bucket.marker.empty()) + return 0; + + RGWObjEnt ent; + ent.name = oid; + ent.size = size; + ent.mtime = ut; + ent.etag = etag; + ACLOwner owner; + if (acl_bl && acl_bl->length()) { + int ret = decode_policy(*acl_bl, &owner); + if (ret < 0) { + RGW_LOG(0) << "WARNING: could not decode policy ret=" << ret << dendl; + } + } + ent.owner = owner.get_id(); + ent.owner_display_name = owner.get_display_name(); + + int ret = cls_obj_complete_add(bucket, tag, epoch, ent, category); + + return ret; +} + + int RGWRados::clone_objs_impl(void *ctx, rgw_obj& dst_obj, vector& ranges, map attrs, @@ -1344,46 +1400,23 @@ int RGWRados::clone_objs_impl(void *ctx, rgw_obj& dst_obj, op.mtime(&mt); } - bufferlist outbl; string tag; - if (state && state->obj_tag.length()) { - int len = state->obj_tag.length(); - char buf[len + 1]; - memcpy(buf, state->obj_tag.c_str(), len); - buf[len] = '\0'; - tag = buf; - } else { - append_rand_alpha(tag, tag, 32); - } - int ret = cls_obj_prepare_op(bucket, CLS_RGW_OP_ADD, tag, dst_obj.object); + uint64_t epoch; + int ret = prepare_update_index(state, bucket, dst_obj.object, tag); if (ret < 0) goto done; ret = io_ctx.operate(dst_oid, &op); + epoch = io_ctx.get_last_version(); + done: atomic_write_finish(state, ret); - if (ret >= 0 && bucket.marker.size()) { - uint64_t epoch = io_ctx.get_last_version(); - - RGWObjEnt ent; - ent.name = dst_obj.object; - ent.size = size; - ent.mtime = ut; - ent.etag = etag; - ACLOwner owner; - if (acl_bl.length()) { - ret = decode_policy(acl_bl, &owner); - if (ret < 0) { - RGW_LOG(0) << "WARNING: could not decode policy ret=" << ret << dendl; - } - } - ent.owner = owner.get_id(); - ent.owner_display_name = owner.get_display_name(); - - uint8_t category = 0; - ret = cls_obj_complete_add(bucket, tag, epoch, ent, category); + if (ret >= 0) { + uint8_t category = 0; + ret = complete_update_index(bucket, dst_obj.object, tag, epoch, size, + ut, etag, &acl_bl, category); } return ret; diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 1ce685240fa1d..97bb67ea9ea9c 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -150,7 +150,7 @@ public: virtual int create_pools(std::string& id, vector& names, vector& retcodes, int auid = 0); /** Write/overwrite an object to the bucket storage. */ - virtual int put_obj_meta(void *ctx, std::string& id, rgw_obj& obj, time_t *mtime, + virtual int put_obj_meta(void *ctx, std::string& id, rgw_obj& obj, uint64_t size, time_t *mtime, map& attrs, string& category, bool exclusive); virtual int put_obj_data(void *ctx, std::string& id, rgw_obj& obj, const char *data, off_t ofs, size_t len); @@ -288,6 +288,12 @@ public: int cls_obj_complete_del(rgw_bucket& bucket, string& tag, uint64_t epoch, string& name); int cls_bucket_list(rgw_bucket& bucket, string start, uint32_t num, map& m, bool *is_truncated); int cls_bucket_head(rgw_bucket& bucket, struct rgw_bucket_dir_header& header); + int prepare_update_index(RGWObjState *state, rgw_bucket& bucket, string& oid, string& tag); + int complete_update_index(rgw_bucket& bucket, string& oid, string& tag, uint64_t epoch, uint64_t size, + utime_t& ut, string& etag, bufferlist *acl_bl, uint8_t category); + int complete_update_index_del(rgw_bucket& bucket, string& oid, string& tag, uint64_t epoch) { + return cls_obj_complete_del(bucket, tag, epoch, oid); + } }; #endif -- 2.39.5