From: Yehuda Sadeh Date: Thu, 24 Dec 2015 00:47:20 +0000 (-0800) Subject: rgw: fetch remote obj depends on zone id and pg ver X-Git-Tag: v10.1.0~354^2~98 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=b9da8db726301731ba249b7d7789ed5d130fc192;p=ceph.git rgw: fetch remote obj depends on zone id and pg ver This provides a consistent objects view on zones when changes happen on the same objects at the same time. The zone id and pg ver are used for break even decisions when mtime is equal. Signed-off-by: Yehuda Sadeh --- diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index e3677f1a791..173e1f65acd 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -1195,6 +1195,8 @@ void RGWGetObj::execute() read_op.conds.mod_ptr = mod_ptr; read_op.conds.unmod_ptr = unmod_ptr; + read_op.conds.mod_zone_id = mod_zone_id; + read_op.conds.mod_pg_ver = mod_pg_ver; read_op.conds.if_match = if_match; read_op.conds.if_nomatch = if_nomatch; read_op.params.attrs = &attrs; diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 3de4aafcef5..0feb0139e30 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -133,6 +133,8 @@ protected: const char *if_unmod; const char *if_match; const char *if_nomatch; + uint32_t mod_zone_id; + uint64_t mod_pg_ver; off_t ofs; uint64_t total_len; off_t start; @@ -159,6 +161,8 @@ public: if_unmod = NULL; if_match = NULL; if_nomatch = NULL; + mod_zone_id = 0; + mod_pg_ver = 0; start = 0; ofs = 0; total_len = 0; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index acfc49ab84e..f955661bb99 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -6113,6 +6113,49 @@ int RGWRados::rewrite_obj(RGWBucketInfo& dest_bucket_info, rgw_obj& obj) return copy_obj_data(rctx, dest_bucket_info, read_op, end, obj, obj, max_chunk_size, NULL, mtime, attrset, RGW_OBJ_CATEGORY_MAIN, 0, 0, NULL, NULL, NULL, NULL); } +struct obj_time_weight { + time_t mtime; + uint32_t zone_short_id; + uint64_t pg_ver; + + obj_time_weight() : mtime(0), zone_short_id(0), pg_ver(0) {} + + bool operator<(const obj_time_weight& rhs) { + if (mtime > rhs.mtime) { + return false; + } + if (mtime < rhs.mtime) { + return true; + } + if (zone_short_id != rhs.zone_short_id) { + return (zone_short_id < rhs.zone_short_id); + } + return (pg_ver < rhs.pg_ver); + } + + void init(const time_t& _mtime, uint32_t _short_id, uint64_t _pg_ver) { + mtime = _mtime; + zone_short_id = _short_id; + pg_ver = _pg_ver; + } + + void init(RGWObjState *state) { + mtime = state->mtime; + zone_short_id = state->zone_short_id; + pg_ver = state->pg_ver; + } +}; + +inline ostream& operator<<(ostream& out, const obj_time_weight &o) { + out << o.mtime; + + if (o.zone_short_id != 0 || o.pg_ver != 0) { + out << "[zid=" << o.zone_short_id << ", pgv=" << o.pg_ver << "]"; + } + + return out; +} + int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx, const rgw_user& user_id, const string& client_id, @@ -6149,6 +6192,7 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx, map src_attrs; int i; append_rand_alpha(cct, tag, tag, 32); + obj_time_weight set_mtime_weight; RGWPutObjProcessor_Atomic processor(obj_ctx, dest_bucket_info, dest_obj.bucket, dest_obj.get_orig_obj(), @@ -6201,9 +6245,10 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx, RGWObjState *dest_state = NULL; - time_t dest_mtime; const time_t *pmod = mod_ptr; + obj_time_weight dest_mtime_weight; + if (copy_if_newer) { /* need to get mtime for destination */ ret = get_obj_state(&obj_ctx, dest_obj, &dest_state, NULL); @@ -6211,13 +6256,15 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx, return ret; if (dest_state->exists) { - dest_mtime = dest_state->mtime; - pmod = &dest_mtime; + dest_mtime_weight.init(dest_state); + pmod = &dest_mtime_weight.mtime; } } - ret = conn->get_obj(user_id, info, src_obj, pmod, unmod_ptr, true, &cb, &in_stream_req); + ret = conn->get_obj(user_id, info, src_obj, pmod, unmod_ptr, + dest_mtime_weight.zone_short_id, dest_mtime_weight.pg_ver, + true, &cb, &in_stream_req); if (ret < 0) { goto set_err_state; } @@ -6274,6 +6321,21 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx, attrs = src_attrs; } + if (copy_if_newer) { + uint64_t pg_ver = 0; + auto i = attrs.find(RGW_ATTR_PG_VER); + if (i != attrs.end() && i->second.length() > 0) { + bufferlist::iterator iter = i->second.begin(); + try { + ::decode(pg_ver, iter); + } catch (buffer::error& err) { + ldout(ctx(), 0) << "ERROR: failed to decode pg ver attribute, ignoring" << dendl; + /* non critical error */ + } + } + set_mtime_weight.init(set_mtime, get_zone_short_id(), pg_ver); + } + #define MAX_COMPLETE_RETRY 100 for (i = 0; i < MAX_COMPLETE_RETRY; i++) { ret = cb.complete(etag, mtime, set_mtime, attrs, delete_at); @@ -6288,8 +6350,9 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx, ldout(cct, 0) << "ERROR: " << __func__ << ": get_err_state() returned ret=" << ret << dendl; goto set_err_state; } + dest_mtime_weight.init(dest_state); if (!dest_state->exists || - dest_state->mtime < set_mtime) { + dest_mtime_weight < set_mtime_weight) { ldout(cct, 20) << "retrying writing object mtime=" << set_mtime << " dest_state->mtime=" << dest_state->mtime << " dest_state->exists=" << dest_state->exists << dendl; continue; } else { @@ -7492,13 +7555,28 @@ int RGWRados::get_obj_state_impl(RGWObjectCtx *rctx, rgw_obj& obj, RGWObjState * s->fake_tag = true; } } - bufferlist pg_ver_bl = s->attrset[RGW_ATTR_PG_VER]; - if (pg_ver_bl.length()) { - bufferlist::iterator pgbl = pg_ver_bl.begin(); - try { - ::decode(s->pg_ver, pgbl); - } catch (buffer::error& err) { - ldout(cct, 0) << "ERROR: couldn't decode pg ver attr for object " << s->obj << ", non-critical error, ignoring" << dendl; + map::iterator aiter = s->attrset.find(RGW_ATTR_PG_VER); + if (aiter != s->attrset.end()) { + bufferlist& pg_ver_bl = aiter->second; + if (pg_ver_bl.length()) { + bufferlist::iterator pgbl = pg_ver_bl.begin(); + try { + ::decode(s->pg_ver, pgbl); + } catch (buffer::error& err) { + ldout(cct, 0) << "ERROR: couldn't decode pg ver attr for object " << s->obj << ", non-critical error, ignoring" << dendl; + } + } + } + aiter = s->attrset.find(RGW_ATTR_SOURCE_ZONE); + if (aiter != s->attrset.end()) { + bufferlist& zone_short_id_bl = aiter->second; + if (zone_short_id_bl.length()) { + bufferlist::iterator zbl = zone_short_id_bl.begin(); + try { + ::decode(s->zone_short_id, zbl); + } catch (buffer::error& err) { + ldout(cct, 0) << "ERROR: couldn't decode zone short id attr for object " << s->obj << ", non-critical error, ignoring" << dendl; + } } } if (s->obj_tag.length()) @@ -8029,18 +8107,23 @@ int RGWRados::Object::Read::prepare(int64_t *pofs, int64_t *pend) /* Convert all times go GMT to make them compatible */ if (conds.mod_ptr || conds.unmod_ptr) { - time_t ctime = astate->mtime; + obj_time_weight src_weight; + src_weight.init(astate); + + obj_time_weight dest_weight; if (conds.mod_ptr) { - ldout(cct, 10) << "If-Modified-Since: " << *conds.mod_ptr << " Last-Modified: " << ctime << dendl; - if (ctime <= *conds.mod_ptr) { + dest_weight.init(*conds.mod_ptr, conds.mod_zone_id, conds.mod_pg_ver); + ldout(cct, 10) << "If-Modified-Since: " << dest_weight << " Last-Modified: " << src_weight << dendl; + if (!(dest_weight < src_weight)) { return -ERR_NOT_MODIFIED; } } if (conds.unmod_ptr) { - ldout(cct, 10) << "If-UnModified-Since: " << *conds.unmod_ptr << " Last-Modified: " << ctime << dendl; - if (ctime > *conds.unmod_ptr) { + dest_weight.init(*conds.unmod_ptr, conds.mod_zone_id, conds.mod_pg_ver); + ldout(cct, 10) << "If-UnModified-Since: " << dest_weight << " Last-Modified: " << src_weight << dendl; + if (dest_weight < src_weight) { return -ERR_PRECONDITION_FAILED; } } diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 05928cef58b..f807c9979aa 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -628,6 +628,7 @@ struct RGWObjState { bool is_olh; bufferlist olh_tag; uint64_t pg_ver; + uint32_t zone_short_id; /* important! don't forget to update copy constructor */ @@ -637,7 +638,7 @@ struct RGWObjState { RGWObjState() : is_atomic(false), has_attrs(0), exists(false), size(0), mtime(0), epoch(0), fake_tag(false), has_manifest(false), has_data(false), prefetch_data(false), keep_tail(false), is_olh(false), - pg_ver(0) {} + pg_ver(0), zone_short_id(0) {} RGWObjState(const RGWObjState& rhs) : obj (rhs.obj) { is_atomic = rhs.is_atomic; has_attrs = rhs.has_attrs; @@ -2132,11 +2133,14 @@ public: struct ConditionParams { const time_t *mod_ptr; const time_t *unmod_ptr; + uint32_t mod_zone_id; + uint64_t mod_pg_ver; const char *if_match; const char *if_nomatch; ConditionParams() : - mod_ptr(NULL), unmod_ptr(NULL), if_match(NULL), if_nomatch(NULL) {} + mod_ptr(NULL), unmod_ptr(NULL), mod_zone_id(0), mod_pg_ver(0), + if_match(NULL), if_nomatch(NULL) {} } conds; struct Params { diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index 51294047d9b..10984e15bbe 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -744,6 +744,11 @@ int RGWGetObj_ObjStore::get_params() if_match = s->info.env->get("HTTP_IF_MATCH"); if_nomatch = s->info.env->get("HTTP_IF_NONE_MATCH"); + if (s->system_request) { + mod_zone_id = s->info.env->get_int("HTTP_DEST_ZONE_SHORT_ID", 0); + mod_pg_ver = s->info.env->get_int("HTTP_DEST_PG_VER", 0); + } + return 0; } diff --git a/src/rgw/rgw_rest_conn.cc b/src/rgw/rgw_rest_conn.cc index 203097f6914..9893faf3ebc 100644 --- a/src/rgw/rgw_rest_conn.cc +++ b/src/rgw/rgw_rest_conn.cc @@ -106,9 +106,18 @@ static void set_date_header(const time_t *t, map& headers, const headers["HTTP_IF_MODIFIED_SINCE"] = s.str(); } +template +static void set_header(T val, map& headers, const string& header_name) +{ + stringstream s; + s << val; + headers[header_name] = s.str(); +} + int RGWRESTConn::get_obj(const rgw_user& uid, req_info *info /* optional */, rgw_obj& obj, const time_t *mod_ptr, const time_t *unmod_ptr, + uint32_t mod_zone_id, uint64_t mod_pg_ver, bool prepend_metadata, RGWGetDataCB *cb, RGWRESTStreamReadRequest **req) { string url; @@ -147,6 +156,12 @@ int RGWRESTConn::get_obj(const rgw_user& uid, req_info *info /* optional */, rgw set_date_header(mod_ptr, extra_headers, "HTTP_IF_MODIFIED_SINCE"); set_date_header(unmod_ptr, extra_headers, "HTTP_IF_UNMODIFIED_SINCE"); + if (mod_zone_id != 0) { + set_header(mod_zone_id, extra_headers, "HTTP_DEST_ZONE_SHORT_ID"); + } + if (mod_pg_ver != 0) { + set_header(mod_pg_ver, extra_headers, "HTTP_DEST_PG_VER"); + } return (*req)->get_obj(key, extra_headers, obj); } diff --git a/src/rgw/rgw_rest_conn.h b/src/rgw/rgw_rest_conn.h index d4b286b2f42..a1ccb893c81 100644 --- a/src/rgw/rgw_rest_conn.h +++ b/src/rgw/rgw_rest_conn.h @@ -92,6 +92,7 @@ public: int get_obj(const rgw_user& uid, req_info *info /* optional */, rgw_obj& obj, const time_t *mod_ptr, const time_t *unmod_ptr, + uint32_t mod_zone_id, uint64_t mod_pg_ver, bool prepend_metadata, RGWGetDataCB *cb, RGWRESTStreamReadRequest **req); int complete_request(RGWRESTStreamReadRequest *req, string& etag, time_t *mtime, map& attrs);