From: daijufang Date: Mon, 4 Dec 2023 07:46:29 +0000 (+0000) Subject: src/rgw: fix for the multipart interface in the WORM function X-Git-Tag: testing/wip-root-testing-20240411.174241~136^2~2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=ef5178272fa6738fb11dc2582bf34474245ca942;p=ceph-ci.git src/rgw: fix for the multipart interface in the WORM function 1. Save the WORM configuration information in the initialization chunk information for use when merging chunks. 2. Support x-amz-bypass-governance-retention when merging chunks. Fixes: https://tracker.ceph.com/issues/63724 Signed-off-by: daijufang (cherry picked from commit 3a65a0ad14fb3c4d836441a417d594826fb6e711) --- diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc index 299f46e3c9d..b1f1ce8f7ed 100644 --- a/src/rgw/driver/rados/rgw_sal_rados.cc +++ b/src/rgw/driver/rados/rgw_sal_rados.cc @@ -2299,6 +2299,15 @@ int RadosMultipartUpload::init(const DoutPrefixProvider *dpp, optional_yield y, multipart_upload_info upload_info; upload_info.dest_placement = dest_placement; + + if (obj_legal_hold) { + upload_info.obj_legal_hold_exist = true; + upload_info.obj_legal_hold = (*obj_legal_hold); + } + if (obj_retention) { + upload_info.obj_retention_exist = true; + upload_info.obj_retention = (*obj_retention); + } bufferlist bl; encode(upload_info, bl); @@ -2562,6 +2571,20 @@ int RadosMultipartUpload::complete(const DoutPrefixProvider *dpp, attrs[RGW_ATTR_ETAG] = etag_bl; + rgw_placement_rule* ru; + ru = &placement; + ret = RadosMultipartUpload::get_info(dpp, y, &ru, &attrs); + if (upload_information.obj_retention_exist) { + bufferlist obj_retention_bl; + upload_information.obj_retention.encode(obj_retention_bl); + attrs[RGW_ATTR_OBJECT_RETENTION] = std::move(obj_retention_bl); + } + if (upload_information.obj_legal_hold_exist) { + bufferlist obj_legal_hold_bl; + upload_information.obj_legal_hold.encode(obj_legal_hold_bl); + attrs[RGW_ATTR_OBJECT_LEGAL_HOLD] = std::move(obj_legal_hold_bl); + } + if (compressed) { // write compression attribute to full object bufferlist tmp; @@ -2666,6 +2689,7 @@ int RadosMultipartUpload::get_info(const DoutPrefixProvider *dpp, optional_yield return -EIO; } placement = upload_info.dest_placement; + upload_information = upload_info; *rule = &placement; return 0; diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h index d4bd19a55fe..095ea959889 100644 --- a/src/rgw/driver/rados/rgw_sal_rados.h +++ b/src/rgw/driver/rados/rgw_sal_rados.h @@ -602,6 +602,7 @@ class RadosMultipartUpload : public StoreMultipartUpload { ceph::real_time mtime; rgw_placement_rule placement; RGWObjManifest manifest; + multipart_upload_info upload_information; public: RadosMultipartUpload(RadosStore* _store, Bucket* _bucket, const std::string& oid, diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index ea91859f5e6..0843bbc267a 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -1359,16 +1359,34 @@ inline std::ostream& operator<<(std::ostream& out, const rgw_obj &o) { struct multipart_upload_info { rgw_placement_rule dest_placement; + //object lock + bool obj_retention_exist{false}; + bool obj_legal_hold_exist{false}; + RGWObjectRetention obj_retention; + RGWObjectLegalHold obj_legal_hold; void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); + ENCODE_START(2, 1, bl); encode(dest_placement, bl); + encode(obj_retention_exist, bl); + encode(obj_legal_hold_exist, bl); + encode(obj_retention, bl); + encode(obj_legal_hold, bl); ENCODE_FINISH(bl); } void decode(bufferlist::const_iterator& bl) { - DECODE_START(1, bl); + DECODE_START(2, bl); decode(dest_placement, bl); + if (struct_v >= 2) { + decode(obj_retention_exist, bl); + decode(obj_legal_hold_exist, bl); + decode(obj_retention, bl); + decode(obj_legal_hold, bl); + } else { + obj_retention_exist = false; + obj_legal_hold_exist = false; + } DECODE_FINISH(bl); } diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index f6011dbe5c9..cdbd53639e8 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -5265,7 +5265,6 @@ void RGWDeleteObj::execute(optional_yield y) { RGWObjState* astate = nullptr; bool check_obj_lock = s->object->have_instance() && s->bucket->get_info().obj_lock_enabled(); - op_ret = s->object->get_obj_state(this, &astate, s->yield, true); if (op_ret < 0) { if (need_object_expiration() || multipart_delete) { @@ -5288,7 +5287,6 @@ void RGWDeleteObj::execute(optional_yield y) // ignore return value from get_obj_attrs in all other cases op_ret = 0; - if (check_obj_lock) { ceph_assert(astate); int object_lock_response = verify_object_lock(this, astate->attrset, bypass_perm, bypass_governance_mode); @@ -6525,6 +6523,8 @@ void RGWInitMultipart::execute(optional_yield y) std::unique_ptr upload; upload = s->bucket->get_multipart_upload(s->object->get_name(), upload_id); + upload->obj_legal_hold = obj_legal_hold; + upload->obj_retention = obj_retention; op_ret = upload->init(this, s->yield, s->owner, s->dest_placement, attrs); if (op_ret == 0) { @@ -6545,6 +6545,28 @@ int RGWCompleteMultipart::verify_permission(optional_yield y) rgw_iam_add_crypt_attrs(s->env, s->info.crypt_attribute_map); if (s->iam_policy || ! s->iam_user_policies.empty() || ! s->session_policies.empty()) { + if (s->bucket->get_info().obj_lock_enabled() && bypass_governance_mode) { + auto r = eval_identity_or_session_policies(this, s->iam_user_policies, s->env, + rgw::IAM::s3BypassGovernanceRetention, ARN(s->bucket->get_key(), s->object->get_name())); + if (r == Effect::Deny) { + bypass_perm = false; + } else if (r == Effect::Pass && s->iam_policy) { + ARN arn(s->bucket->get_key(), s->object->get_name()); + r = s->iam_policy->eval(s->env, *s->auth.identity, rgw::IAM::s3BypassGovernanceRetention, arn); + if (r == Effect::Deny) { + bypass_perm = false; + } + } else if (r == Effect::Pass && !s->session_policies.empty()) { + r = eval_identity_or_session_policies(this, s->session_policies, s->env, + rgw::IAM::s3BypassGovernanceRetention, ARN(s->bucket->get_key(), s->object->get_name())); + if (r == Effect::Deny) { + bypass_perm = false; + } + } else if (r == Effect::Pass) { + bypass_perm = false; + } + bypass_governance_mode &= bypass_perm; + } auto identity_policy_res = eval_identity_or_session_policies(this, s->iam_user_policies, s->env, rgw::IAM::s3PutObject, s->object->get_obj()); diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index fcfb24786e8..6e962de4f47 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -1811,6 +1811,9 @@ protected: RGWAccessControlPolicy policy; ceph::real_time mtime; jspan_ptr multipart_trace; + //object lock + std::optional obj_retention = std::nullopt; + std::optional obj_legal_hold = std::nullopt; public: RGWInitMultipart() {} @@ -1840,6 +1843,9 @@ protected: std::unique_ptr res; std::unique_ptr meta_obj; off_t ofs = 0; + //object lock + bool bypass_perm = true; + bool bypass_governance_mode = false; public: RGWCompleteMultipart() {} diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index a7c35f3773c..d830fd40caa 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -1572,6 +1572,12 @@ int RGWCompleteMultipart_ObjStore::get_params(optional_yield y) std::tie(op_ret, data) = read_all_input(s, max_size); if (op_ret < 0) return op_ret; + + const char *bypass_gov_header = s->info.env->get("HTTP_X_AMZ_BYPASS_GOVERNANCE_RETENTION"); + if (bypass_gov_header) { + std::string bypass_gov_decoded = url_decode(bypass_gov_header); + bypass_governance_mode = boost::algorithm::iequals(bypass_gov_decoded, "true"); + } return 0; } diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index fddb3f092c1..1b964742a68 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -3924,7 +3924,42 @@ int RGWInitMultipart_ObjStore_S3::get_params(optional_yield y) return ret; } - return create_s3_policy(s, driver, policy, s->owner); + ret = create_s3_policy(s, driver, policy, s->owner); + if (ret < 0) + return ret; + + //handle object lock + auto obj_lock_mode_str = s->info.env->get("HTTP_X_AMZ_OBJECT_LOCK_MODE"); + auto obj_lock_date_str = s->info.env->get("HTTP_X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE"); + auto obj_legal_hold_str = s->info.env->get("HTTP_X_AMZ_OBJECT_LOCK_LEGAL_HOLD"); + if (obj_lock_mode_str && obj_lock_date_str) { + boost::optional date = ceph::from_iso_8601(obj_lock_date_str); + if (boost::none == date || ceph::real_clock::to_time_t(*date) <= ceph_clock_now()) { + ldpp_dout(this,0) << "invalid x-amz-object-lock-retain-until-date value" << dendl; + return -EINVAL;; + } + if (strcmp(obj_lock_mode_str, "GOVERNANCE") != 0 && strcmp(obj_lock_mode_str, "COMPLIANCE") != 0) { + ldpp_dout(this,0) << "invalid x-amz-object-lock-mode value" << dendl; + return -EINVAL; + } + obj_retention = RGWObjectRetention(obj_lock_mode_str, *date); + } else if ((obj_lock_mode_str && !obj_lock_date_str) || (!obj_lock_mode_str && obj_lock_date_str)) { + ldpp_dout(this,0) << "need both x-amz-object-lock-mode and x-amz-object-lock-retain-until-date " << dendl; + return -EINVAL; + } + if (obj_legal_hold_str) { + if (strcmp(obj_legal_hold_str, "ON") != 0 && strcmp(obj_legal_hold_str, "OFF") != 0) { + ldpp_dout(this,0) << "invalid x-amz-object-lock-legal-hold value" << dendl; + return -EINVAL; + } + obj_legal_hold = RGWObjectLegalHold(obj_legal_hold_str); + } + if (!s->bucket->get_info().obj_lock_enabled() && (obj_retention || obj_legal_hold)) { + ldpp_dout(this, 0) << "ERROR: object retention or legal hold can't be set if bucket object lock not configured" << dendl; + return -ERR_INVALID_REQUEST; + } + + return 0; } void RGWInitMultipart_ObjStore_S3::send_response() diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index 6ee02be0b67..abeb197152f 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -1117,6 +1117,10 @@ public: */ class MultipartUpload { public: + //object lock + std::optional obj_retention = std::nullopt; + std::optional obj_legal_hold = std::nullopt; + MultipartUpload() = default; virtual ~MultipartUpload() = default;