From b5fe51379ddd368a2237bdbe1c23c88cf28321fd Mon Sep 17 00:00:00 2001 From: imtzw Date: Fri, 8 Dec 2023 06:11:13 +0000 Subject: [PATCH] rgw: fix list obj v1 nextmarker when encoding-type=url given, aws sdk will try to decode next marker returned in a list obj response. if it is not actually encoded and the marker obj just have an encoding symbol in its name, aws sdk may get error when trying to decode it. Signed-off-by: imtzw --- src/rgw/rgw_rest.h | 10 ++++++ src/rgw/rgw_rest_s3.cc | 76 +++++++----------------------------------- 2 files changed, 22 insertions(+), 64 deletions(-) diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h index 434de99e9c2ec..0b6cf62ed2c73 100644 --- a/src/rgw/rgw_rest.h +++ b/src/rgw/rgw_rest.h @@ -706,6 +706,16 @@ extern void dump_start(req_state *s); extern void list_all_buckets_start(req_state *s); extern void dump_owner(req_state *s, const rgw_user& id, const std::string& name, const char *section = NULL); +inline void dump_urlsafe(req_state *s, bool encode_key, const char* key, const std::string& val, bool encode_slash = true) { + if (encode_key) { + std::string _val; + url_encode(val, _val, encode_slash); + s->formatter->dump_string(key, _val); + } + else { + s->formatter->dump_string(key, val); + } +} extern void dump_header(req_state* s, const std::string_view& name, const std::string_view& val); diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 6c3a8c8adc168..511adb990ad12 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -1707,11 +1707,7 @@ void RGWListBucket_ObjStore_S3::send_common_versioned_response() for (pref_iter = common_prefixes.begin(); pref_iter != common_prefixes.end(); ++pref_iter) { s->formatter->open_array_section("CommonPrefixes"); - if (encode_key) { - s->formatter->dump_string("Prefix", url_encode(pref_iter->first, false)); - } else { - s->formatter->dump_string("Prefix", pref_iter->first); - } + dump_urlsafe(s, encode_key, "Prefix", pref_iter->first, false); s->formatter->close_section(); } @@ -1729,7 +1725,7 @@ void RGWListBucket_ObjStore_S3::send_versioned_response() s->formatter->dump_string("KeyMarker", marker.name); s->formatter->dump_string("VersionIdMarker", marker.instance); if (is_truncated && !next_marker.empty()) { - s->formatter->dump_string("NextKeyMarker", next_marker.name); + dump_urlsafe(s ,encode_key, "NextKeyMarker", next_marker.name); if (next_marker.instance.empty()) { s->formatter->dump_string("NextVersionIdMarker", "null"); } @@ -1752,14 +1748,7 @@ void RGWListBucket_ObjStore_S3::send_versioned_response() s->formatter->dump_bool("IsDeleteMarker", iter->is_delete_marker()); } rgw_obj_key key(iter->key); - if (encode_key) { - string key_name; - url_encode(key.name, key_name); - s->formatter->dump_string("Key", key_name); - } - else { - s->formatter->dump_string("Key", key.name); - } + dump_urlsafe(s ,encode_key, "Key", key.name); string version_id = key.instance; if (version_id.empty()) { version_id = "null"; @@ -1807,11 +1796,7 @@ void RGWListBucket_ObjStore_S3::send_common_response() s->formatter->dump_string("Prefix", prefix); s->formatter->dump_int("MaxKeys", max); if (!delimiter.empty()) { - if (encode_key) { - s->formatter->dump_string("Delimiter", url_encode(delimiter, false)); - } else { - s->formatter->dump_string("Delimiter", delimiter); - } + dump_urlsafe(s, encode_key, "Delimiter", delimiter, false); } s->formatter->dump_string("IsTruncated", (max && is_truncated ? "true" : "false")); @@ -1821,11 +1806,7 @@ void RGWListBucket_ObjStore_S3::send_common_response() for (pref_iter = common_prefixes.begin(); pref_iter != common_prefixes.end(); ++pref_iter) { s->formatter->open_array_section("CommonPrefixes"); - if (encode_key) { - s->formatter->dump_string("Prefix", url_encode(pref_iter->first, false)); - } else { - s->formatter->dump_string("Prefix", pref_iter->first); - } + dump_urlsafe(s, encode_key, "Prefix", pref_iter->first, false); s->formatter->close_section(); } } @@ -1866,13 +1847,6 @@ void RGWListBucket_ObjStore_S3::send_response() for (iter = objs.begin(); iter != objs.end(); ++iter) { rgw_obj_key key(iter->key); - std::string key_name; - - if (encode_key) { - url_encode(key.name, key_name); - } else { - key_name = key.name; - } /* conditionally format JSON in the obvious way--I'm unsure if * AWS actually does this */ if (s->format == RGWFormat::XML) { @@ -1881,7 +1855,7 @@ void RGWListBucket_ObjStore_S3::send_response() // json s->formatter->open_object_section("dummy"); } - s->formatter->dump_string("Key", key_name); + dump_urlsafe(s ,encode_key, "Key", key.name); dump_time(s, "LastModified", iter->meta.mtime); s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str()); s->formatter->dump_int("Size", iter->meta.accounted_size); @@ -1905,7 +1879,7 @@ void RGWListBucket_ObjStore_S3::send_response() } s->formatter->dump_string("Marker", marker.name); if (is_truncated && !next_marker.empty()) { - s->formatter->dump_string("NextMarker", next_marker.name); + dump_urlsafe(s, encode_key, "NextMarker", next_marker.name); } s->formatter->close_section(); rgw_flush_formatter_and_reset(s, s->formatter); @@ -1941,14 +1915,7 @@ void RGWListBucket_ObjStore_S3v2::send_versioned_response() s->formatter->dump_bool("IsDeleteContinuationToken", iter->is_delete_marker()); } rgw_obj_key key(iter->key); - if (encode_key) { - string key_name; - url_encode(key.name, key_name); - s->formatter->dump_string("Key", key_name); - } - else { - s->formatter->dump_string("Key", key.name); - } + dump_urlsafe(s, encode_key, "Key", key.name); string version_id = key.instance; if (version_id.empty()) { version_id = "null"; @@ -1986,11 +1953,7 @@ void RGWListBucket_ObjStore_S3v2::send_versioned_response() for (pref_iter = common_prefixes.begin(); pref_iter != common_prefixes.end(); ++pref_iter) { s->formatter->open_array_section("CommonPrefixes"); - if (encode_key) { - s->formatter->dump_string("Prefix", url_encode(pref_iter->first, false)); - } else { - s->formatter->dump_string("Prefix", pref_iter->first); - } + dump_urlsafe(s, encode_key, "Prefix", pref_iter->first, false); s->formatter->dump_int("KeyCount",objs.size()); if (start_after_exist) { @@ -2036,14 +1999,7 @@ void RGWListBucket_ObjStore_S3v2::send_response() for (iter = objs.begin(); iter != objs.end(); ++iter) { rgw_obj_key key(iter->key); s->formatter->open_array_section("Contents"); - if (encode_key) { - string key_name; - url_encode(key.name, key_name); - s->formatter->dump_string("Key", key_name); - } - else { - s->formatter->dump_string("Key", key.name); - } + dump_urlsafe(s, encode_key, "Key", key.name); dump_time(s, "LastModified", iter->meta.mtime); s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str()); s->formatter->dump_int("Size", iter->meta.accounted_size); @@ -4147,11 +4103,7 @@ void RGWListBucketMultiparts_ObjStore_S3::send_response() for (iter = uploads.begin(); iter != uploads.end(); ++iter) { rgw::sal::MultipartUpload* upload = iter->get(); s->formatter->open_array_section("Upload"); - if (encode_url) { - s->formatter->dump_string("Key", url_encode(upload->get_key(), false)); - } else { - s->formatter->dump_string("Key", upload->get_key()); - } + dump_urlsafe(s, encode_url, "Key", upload->get_key(), false); s->formatter->dump_string("UploadId", upload->get_upload_id()); const ACLOwner& owner = upload->get_owner(); dump_owner(s, owner.get_id(), owner.get_display_name(), "Initiator"); @@ -4163,11 +4115,7 @@ void RGWListBucketMultiparts_ObjStore_S3::send_response() if (!common_prefixes.empty()) { s->formatter->open_array_section("CommonPrefixes"); for (const auto& kv : common_prefixes) { - if (encode_url) { - s->formatter->dump_string("Prefix", url_encode(kv.first, false)); - } else { - s->formatter->dump_string("Prefix", kv.first); - } + dump_urlsafe(s, encode_url, "Prefix", kv.first, false); } s->formatter->close_section(); } -- 2.39.5