From: zhang Shaowen Date: Tue, 28 May 2019 12:04:51 +0000 (+0800) Subject: rgw: fix some bugs in object lock feature X-Git-Tag: v15.1.0~2406^2~7 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=07f70d780c20ebfe415c502deb3ab5a94b821101;p=ceph.git rgw: fix some bugs in object lock feature Signed-off-by: zhang Shaowen --- diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc index 1a34b9750be5..6638d61bc82a 100644 --- a/src/rgw/rgw_common.cc +++ b/src/rgw/rgw_common.cc @@ -74,6 +74,7 @@ rgw_http_errors rgw_http_s3_errors({ { ERR_INVALID_CORS_RULES_ERROR, {400, "InvalidRequest" }}, { ERR_INVALID_WEBSITE_ROUTING_RULES_ERROR, {400, "InvalidRequest" }}, { ERR_INVALID_ENCRYPTION_ALGORITHM, {400, "InvalidEncryptionAlgorithmError" }}, + { ERR_INVALID_RETENTION_PERIOD,{400, "InvalidRetentionPeriod"}}, { ERR_LIMIT_EXCEEDED, {400, "LimitExceeded" }}, { ERR_LENGTH_REQUIRED, {411, "MissingContentLength" }}, { EACCES, {403, "AccessDenied" }}, diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index be2e9ee91934..652ca49e88f7 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -217,6 +217,7 @@ using ceph::crypto::MD5; #define ERR_MFA_REQUIRED 2044 #define ERR_NO_SUCH_CORS_CONFIGURATION 2045 #define ERR_NO_SUCH_OBJECT_LOCK_CONFIGURATION 2046 +#define ERR_INVALID_RETENTION_PERIOD 2047 #define ERR_USER_SUSPENDED 2100 #define ERR_INTERNAL_ERROR 2200 #define ERR_NOT_IMPLEMENTED 2201 diff --git a/src/rgw/rgw_iam_policy.h b/src/rgw/rgw_iam_policy.h index e2854bd9ebe9..a087ceead775 100644 --- a/src/rgw/rgw_iam_policy.h +++ b/src/rgw/rgw_iam_policy.h @@ -169,6 +169,7 @@ inline int op_to_perm(std::uint64_t op) { case s3RestoreObject: case s3PutObjectRetention: case s3PutObjectLegalHold: + case s3BypassGovernanceRetention: return RGW_PERM_WRITE; case s3GetAccelerateConfiguration: diff --git a/src/rgw/rgw_object_lock.h b/src/rgw/rgw_object_lock.h index e7430105d98d..5f5149d8e153 100644 --- a/src/rgw/rgw_object_lock.h +++ b/src/rgw/rgw_object_lock.h @@ -189,6 +189,8 @@ class RGWObjectLegalHold protected: string status; public: + RGWObjectLegalHold() {} + RGWObjectLegalHold(string _status): status(_status) {} void set_status(string _status) { status = _status; } diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 3d3497c170a5..ca8dee2662ad 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3118,6 +3118,7 @@ void RGWCreateBucket::execute() creation_time = master_info.creation_time; pmaster_num_shards = &master_info.num_shards; pobjv = &objv; + obj_lock_enabled = master_info.obj_lock_enabled(); } else { pmaster_bucket = NULL; pmaster_num_shards = NULL; @@ -3983,6 +3984,16 @@ void RGWPutObj::execute() slo_userindicator_bl.append("True", 4); emplace_attr(RGW_ATTR_SLO_UINDICATOR, std::move(slo_userindicator_bl)); } + if (obj_legal_hold) { + bufferlist obj_legal_hold_bl; + obj_legal_hold->encode(obj_legal_hold_bl); + emplace_attr(RGW_ATTR_OBJECT_LEGAL_HOLD, std::move(obj_legal_hold_bl)); + } + if (obj_retention) { + bufferlist obj_retention_bl; + obj_retention->encode(obj_retention_bl); + emplace_attr(RGW_ATTR_OBJECT_RETENTION, std::move(obj_retention_bl)); + } tracepoint(rgw_op, processor_complete_enter, s->req_id.c_str()); op_ret = processor->complete(s->obj_size, etag, &mtime, real_time(), attrs, @@ -4574,13 +4585,13 @@ int RGWDeleteObj::verify_permission() if (s->bucket_info.obj_lock_enabled() && bypass_governance_mode) { auto r = eval_user_policies(s->iam_user_policies, s->env, boost::none, rgw::IAM::s3BypassGovernanceRetention, ARN(s->bucket, s->object.name)); - if (r == Effect::Allow) { - bypass_perm = true; + if (r == Effect::Deny) { + bypass_perm = false; } else if (r == Effect::Pass && s->iam_policy) { r = s->iam_policy->eval(s->env, *s->auth.identity, rgw::IAM::s3BypassGovernanceRetention, ARN(s->bucket, s->object.name)); - if (r == Effect::Allow) { - bypass_perm = true; + if (r == Effect::Deny) { + bypass_perm = false; } } } @@ -4642,7 +4653,7 @@ void RGWDeleteObj::execute() bool check_obj_lock = obj.key.have_instance() && s->bucket_info.obj_lock_enabled(); if (!s->object.empty()) { - if (need_object_expiration() || multipart_delete || check_obj_lock) { + if (need_object_expiration() || multipart_delete) { /* check if obj exists, read orig attrs */ op_ret = get_obj_attrs(store, s, obj, attrs); if (op_ret < 0) { @@ -4650,11 +4661,26 @@ void RGWDeleteObj::execute() } } + if (check_obj_lock) { + /* check if obj exists, read orig attrs */ + op_ret = get_obj_attrs(store, s, obj, attrs); + if (op_ret < 0) { + /* object maybe delete_marker, skip check_obj_lock*/ + check_obj_lock = false; + } + } + if (check_obj_lock) { auto aiter = attrs.find(RGW_ATTR_OBJECT_RETENTION); if (aiter != attrs.end()) { RGWObjectRetention obj_retention; - decode(obj_retention, aiter->second); + try { + decode(obj_retention, aiter->second); + } catch (buffer::error& err) { + ldpp_dout(this, 0) << "ERROR: failed to decode RGWObjectRetention" << dendl; + op_ret = -EIO; + return; + } if (ceph::real_clock::to_time_t(obj_retention.get_retain_until_date()) > ceph_clock_now()) { if (obj_retention.get_mode().compare("GOVERNANCE") != 0 || !bypass_perm || !bypass_governance_mode) { op_ret = -EACCES; @@ -4665,7 +4691,13 @@ void RGWDeleteObj::execute() aiter = attrs.find(RGW_ATTR_OBJECT_LEGAL_HOLD); if (aiter != attrs.end()) { RGWObjectLegalHold obj_legal_hold; - decode(obj_legal_hold, aiter->second); + try { + decode(obj_legal_hold, aiter->second); + } catch (buffer::error& err) { + ldpp_dout(this, 0) << "ERROR: failed to decode RGWObjectLegalHold" << dendl; + op_ret = -EIO; + return; + } if (obj_legal_hold.is_enabled()) { op_ret = -EACCES; return; @@ -7489,10 +7521,6 @@ void RGWPutBucketObjectLock::execute() return; } - op_ret = get_params(); - if (op_ret < 0) - return; - if (!parser.parse(data.c_str(), data.length(), 1)) { op_ret = -ERR_MALFORMED_XML; return; @@ -7507,7 +7535,7 @@ void RGWPutBucketObjectLock::execute() } if (obj_lock.has_rule() && !obj_lock.retention_period_valid()) { ldpp_dout(this, 0) << "ERROR: retention period must be a positive integer value" << dendl; - op_ret = -EINVAL; + op_ret = -ERR_INVALID_RETENTION_PERIOD; return; } @@ -7597,6 +7625,12 @@ void RGWPutObjRetention::execute() op_ret = -ERR_MALFORMED_XML; return; } + + if (ceph::real_clock::to_time_t(obj_retention.get_retain_until_date()) < ceph_clock_now()) { + ldpp_dout(this, 0) << "ERROR: the retain until date must be in the future" << dendl; + op_ret = -EINVAL; + return; + } bufferlist bl; obj_retention.encode(bl); rgw_obj obj(s->bucket, s->object); @@ -7611,16 +7645,23 @@ void RGWPutObjRetention::execute() auto aiter = attrs.find(RGW_ATTR_OBJECT_RETENTION); if (aiter != attrs.end()) { RGWObjectRetention old_obj_retention; - decode(old_obj_retention, aiter->second); - if (obj_retention.get_mode().compare("GOVERNANCE") != 0 || !bypass_perm || !bypass_governance_mode) { - op_ret = -EACCES; + try { + decode(old_obj_retention, aiter->second); + } catch (buffer::error& err) { + ldpp_dout(this, 0) << "ERROR: failed to decode RGWObjectRetention" << dendl; + op_ret = -EIO; return; } + if (ceph::real_clock::to_time_t(obj_retention.get_retain_until_date()) < ceph::real_clock::to_time_t(old_obj_retention.get_retain_until_date())) { + if (old_obj_retention.get_mode().compare("GOVERNANCE") != 0 || !bypass_perm || !bypass_governance_mode) { + op_ret = -EACCES; + return; + } + } } - store->set_atomic(s->obj_ctx, obj); - attrs[RGW_ATTR_OBJECT_RETENTION] = bl; - op_ret = store->set_attrs(s->obj_ctx, s->bucket_info, obj, attrs, NULL); + op_ret = modify_obj_attr(store, s, obj, RGW_ATTR_OBJECT_RETENTION, bl); + return; } @@ -7715,7 +7756,6 @@ void RGWPutObjLegalHold::execute() { bufferlist bl; obj_legal_hold.encode(bl); rgw_obj obj(s->bucket, s->object); - store->set_atomic(s->obj_ctx, obj); //if instance is empty, we should modify the latest object op_ret = modify_obj_attr(store, s, obj, RGW_ATTR_OBJECT_LEGAL_HOLD, bl); return; diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 1a4af91582df..aa4b8e2e7357 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -1106,6 +1106,10 @@ protected: uint64_t position; uint64_t cur_accounted_size; + //object lock + RGWObjectRetention *obj_retention; + RGWObjectLegalHold *obj_legal_hold; + public: RGWPutObj() : ofs(0), supplied_md5_b64(NULL), @@ -1121,10 +1125,14 @@ public: olh_epoch(0), append(false), position(0), - cur_accounted_size(0) {} + cur_accounted_size(0), + obj_retention(nullptr), + obj_legal_hold(nullptr) {} ~RGWPutObj() override { delete slo_info; + delete obj_retention; + delete obj_legal_hold; } void init(RGWRados *store, struct req_state *s, RGWHandler *h) override { @@ -1337,7 +1345,7 @@ public: multipart_delete(false), no_precondition_error(false), deleter(nullptr), - bypass_perm(false), + bypass_perm(true), bypass_governance_mode(false) { } @@ -2225,6 +2233,7 @@ protected: bool bypass_perm; bool bypass_governance_mode; public: + RGWPutObjRetention():bypass_perm(true), bypass_governance_mode(false) {} int verify_permission() override; void pre_exec() override; void execute() override; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index a53e403d8dde..f4b2a383ae96 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -3589,12 +3589,15 @@ int RGWRados::Object::Write::_do_write_meta(uint64_t size, uint64_t accounted_si } if (target->bucket_info.obj_lock_enabled() && target->bucket_info.obj_lock.has_rule() && meta.flags == PUT_OBJ_CREATE) { - real_time lock_until_date = target->bucket_info.obj_lock.get_lock_until_date(meta.set_mtime); - string mode = target->bucket_info.obj_lock.get_mode(); - RGWObjectRetention obj_retention(mode, lock_until_date); - bufferlist bl; - obj_retention.encode(bl); - op.setxattr(RGW_ATTR_OBJECT_RETENTION, bl); + auto iter = attrs.find(RGW_ATTR_OBJECT_RETENTION); + if (iter == attrs.end()) { + real_time lock_until_date = target->bucket_info.obj_lock.get_lock_until_date(meta.set_mtime); + string mode = target->bucket_info.obj_lock.get_mode(); + RGWObjectRetention obj_retention(mode, lock_until_date); + bufferlist bl; + obj_retention.encode(bl); + op.setxattr(RGW_ATTR_OBJECT_RETENTION, bl); + } } if (state->is_olh) { diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index c08ef1158e66..cd7e17384f27 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -331,9 +331,21 @@ int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs, dump_header(s, RGW_AMZ_TAG_COUNT, obj_tags.count()); } else if (iter->first.compare(RGW_ATTR_OBJECT_RETENTION) == 0 && get_retention){ RGWObjectRetention retention; - decode(retention, iter->second); - dump_header(s, "x-amz-object-lock-mode", retention.get_mode()); - dump_time_header(s, "x-amz-object-lock-retain-until-date", retention.get_retain_until_date()); + try { + decode(retention, iter->second); + dump_header(s, "x-amz-object-lock-mode", retention.get_mode()); + dump_time_header(s, "x-amz-object-lock-retain-until-date", retention.get_retain_until_date()); + } catch (buffer::error& err) { + ldpp_dout(this, 0) << "ERROR: failed to decode RGWObjectRetention" << dendl; + } + } else if (iter->first.compare(RGW_ATTR_OBJECT_LEGAL_HOLD) == 0 && get_legal_hold) { + RGWObjectLegalHold legal_hold; + try { + decode(legal_hold, iter->second); + dump_header(s, "x-amz-object-lock-legal-hold",legal_hold.get_status()); + } catch (buffer::error& err) { + ldpp_dout(this, 0) << "ERROR: failed to decode RGWObjectLegalHold" << dendl; + } } } } @@ -1549,6 +1561,41 @@ int RGWPutObj_ObjStore_S3::get_params() } } + //handle object lock + auto obj_lock_mode_str = s->info.env->get("HTTP_X_AMZ_OBJECT_LOCK_MODE"); + auto obj_lock_date_str = s->info.env->get("HTTP_X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE"); + auto obj_legal_hold_str = s->info.env->get("HTTP_X_AMZ_OBJECT_LOCK_LEGAL_HOLD"); + if (obj_lock_mode_str && obj_lock_date_str) { + boost::optional date = ceph::from_iso_8601(obj_lock_date_str); + if (boost::none == date || ceph::real_clock::to_time_t(*date) <= ceph_clock_now()) { + ret = -EINVAL; + ldpp_dout(this,0) << "invalid x-amz-object-lock-retain-until-date value" << dendl; + return ret; + } + if (strcmp(obj_lock_mode_str, "GOVERNANCE") != 0 && strcmp(obj_lock_mode_str, "COMPLIANCE") != 0) { + ret = -EINVAL; + ldpp_dout(this,0) << "invalid x-amz-object-lock-mode value" << dendl; + return ret; + } + obj_retention = new RGWObjectRetention(obj_lock_mode_str, *date); + } else if ((obj_lock_mode_str && !obj_lock_date_str) || (!obj_lock_mode_str && obj_lock_date_str)) { + ret = -EINVAL; + ldpp_dout(this,0) << "need both x-amz-object-lock-mode and x-amz-object-lock-retain-until-date " << dendl; + return ret; + } + if (obj_legal_hold_str) { + if (strcmp(obj_legal_hold_str, "ON") != 0 && strcmp(obj_legal_hold_str, "OFF") != 0) { + ret = -EINVAL; + ldpp_dout(this,0) << "invalid x-amz-object-lock-legal-hold value" << dendl; + return ret; + } + obj_legal_hold = new RGWObjectLegalHold(obj_legal_hold_str); + } + if (!s->bucket_info.obj_lock_enabled() && (obj_retention || obj_legal_hold)) { + ldpp_dout(this, 0) << "ERROR: object retention or legal hold can't be set if bucket object lock not configured" << dendl; + ret = -ERR_INVALID_REQUEST; + return ret; + } multipart_upload_id = s->info.args.get("uploadId"); multipart_part_str = s->info.args.get("partNumber"); if (!multipart_part_str.empty()) { @@ -2272,7 +2319,7 @@ int RGWDeleteObj_ObjStore_S3::get_params() const char *bypass_gov_header = s->info.env->get("HTTP_X_AMZ_BYPASS_GOVERNANCE_RETENTION"); if (bypass_gov_header) { std::string bypass_gov_decoded = url_decode(bypass_gov_header); - bypass_governance_mode = bypass_gov_decoded.compare("true") == 0; + bypass_governance_mode = boost::algorithm::iequals(bypass_gov_decoded, "true"); } return 0; @@ -3206,7 +3253,7 @@ int RGWPutObjRetention_ObjStore_S3::get_params() const char *bypass_gov_header = s->info.env->get("HTTP_X_AMZ_BYPASS_GOVERNANCE_RETENTION"); if (bypass_gov_header) { std::string bypass_gov_decoded = url_decode(bypass_gov_header); - bypass_governance_mode = bypass_gov_decoded.compare("true") == 0; + bypass_governance_mode = boost::algorithm::iequals(bypass_gov_decoded, "true"); } const auto max_size = s->cct->_conf->rgw_max_put_param_size;