From: imtzw Date: Fri, 8 Dec 2023 06:11:13 +0000 (+0000) Subject: rgw: fix list obj v1 nextmarker X-Git-Tag: v19.3.0~90^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=b5fe51379ddd368a2237bdbe1c23c88cf28321fd;p=ceph.git rgw: fix list obj v1 nextmarker when encoding-type=url given, aws sdk will try to decode next marker returned in a list obj response. if it is not actually encoded and the marker obj just have an encoding symbol in its name, aws sdk may get error when trying to decode it. Signed-off-by: imtzw --- diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h index 434de99e9c2ec..0b6cf62ed2c73 100644 --- a/src/rgw/rgw_rest.h +++ b/src/rgw/rgw_rest.h @@ -706,6 +706,16 @@ extern void dump_start(req_state *s); extern void list_all_buckets_start(req_state *s); extern void dump_owner(req_state *s, const rgw_user& id, const std::string& name, const char *section = NULL); +inline void dump_urlsafe(req_state *s, bool encode_key, const char* key, const std::string& val, bool encode_slash = true) { + if (encode_key) { + std::string _val; + url_encode(val, _val, encode_slash); + s->formatter->dump_string(key, _val); + } + else { + s->formatter->dump_string(key, val); + } +} extern void dump_header(req_state* s, const std::string_view& name, const std::string_view& val); diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 6c3a8c8adc168..511adb990ad12 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -1707,11 +1707,7 @@ void RGWListBucket_ObjStore_S3::send_common_versioned_response() for (pref_iter = common_prefixes.begin(); pref_iter != common_prefixes.end(); ++pref_iter) { s->formatter->open_array_section("CommonPrefixes"); - if (encode_key) { - s->formatter->dump_string("Prefix", url_encode(pref_iter->first, false)); - } else { - s->formatter->dump_string("Prefix", pref_iter->first); - } + dump_urlsafe(s, encode_key, "Prefix", pref_iter->first, false); s->formatter->close_section(); } @@ -1729,7 +1725,7 @@ void RGWListBucket_ObjStore_S3::send_versioned_response() s->formatter->dump_string("KeyMarker", marker.name); s->formatter->dump_string("VersionIdMarker", marker.instance); if (is_truncated && !next_marker.empty()) { - s->formatter->dump_string("NextKeyMarker", next_marker.name); + dump_urlsafe(s ,encode_key, "NextKeyMarker", next_marker.name); if (next_marker.instance.empty()) { s->formatter->dump_string("NextVersionIdMarker", "null"); } @@ -1752,14 +1748,7 @@ void RGWListBucket_ObjStore_S3::send_versioned_response() s->formatter->dump_bool("IsDeleteMarker", iter->is_delete_marker()); } rgw_obj_key key(iter->key); - if (encode_key) { - string key_name; - url_encode(key.name, key_name); - s->formatter->dump_string("Key", key_name); - } - else { - s->formatter->dump_string("Key", key.name); - } + dump_urlsafe(s ,encode_key, "Key", key.name); string version_id = key.instance; if (version_id.empty()) { version_id = "null"; @@ -1807,11 +1796,7 @@ void RGWListBucket_ObjStore_S3::send_common_response() s->formatter->dump_string("Prefix", prefix); s->formatter->dump_int("MaxKeys", max); if (!delimiter.empty()) { - if (encode_key) { - s->formatter->dump_string("Delimiter", url_encode(delimiter, false)); - } else { - s->formatter->dump_string("Delimiter", delimiter); - } + dump_urlsafe(s, encode_key, "Delimiter", delimiter, false); } s->formatter->dump_string("IsTruncated", (max && is_truncated ? "true" : "false")); @@ -1821,11 +1806,7 @@ void RGWListBucket_ObjStore_S3::send_common_response() for (pref_iter = common_prefixes.begin(); pref_iter != common_prefixes.end(); ++pref_iter) { s->formatter->open_array_section("CommonPrefixes"); - if (encode_key) { - s->formatter->dump_string("Prefix", url_encode(pref_iter->first, false)); - } else { - s->formatter->dump_string("Prefix", pref_iter->first); - } + dump_urlsafe(s, encode_key, "Prefix", pref_iter->first, false); s->formatter->close_section(); } } @@ -1866,13 +1847,6 @@ void RGWListBucket_ObjStore_S3::send_response() for (iter = objs.begin(); iter != objs.end(); ++iter) { rgw_obj_key key(iter->key); - std::string key_name; - - if (encode_key) { - url_encode(key.name, key_name); - } else { - key_name = key.name; - } /* conditionally format JSON in the obvious way--I'm unsure if * AWS actually does this */ if (s->format == RGWFormat::XML) { @@ -1881,7 +1855,7 @@ void RGWListBucket_ObjStore_S3::send_response() // json s->formatter->open_object_section("dummy"); } - s->formatter->dump_string("Key", key_name); + dump_urlsafe(s ,encode_key, "Key", key.name); dump_time(s, "LastModified", iter->meta.mtime); s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str()); s->formatter->dump_int("Size", iter->meta.accounted_size); @@ -1905,7 +1879,7 @@ void RGWListBucket_ObjStore_S3::send_response() } s->formatter->dump_string("Marker", marker.name); if (is_truncated && !next_marker.empty()) { - s->formatter->dump_string("NextMarker", next_marker.name); + dump_urlsafe(s, encode_key, "NextMarker", next_marker.name); } s->formatter->close_section(); rgw_flush_formatter_and_reset(s, s->formatter); @@ -1941,14 +1915,7 @@ void RGWListBucket_ObjStore_S3v2::send_versioned_response() s->formatter->dump_bool("IsDeleteContinuationToken", iter->is_delete_marker()); } rgw_obj_key key(iter->key); - if (encode_key) { - string key_name; - url_encode(key.name, key_name); - s->formatter->dump_string("Key", key_name); - } - else { - s->formatter->dump_string("Key", key.name); - } + dump_urlsafe(s, encode_key, "Key", key.name); string version_id = key.instance; if (version_id.empty()) { version_id = "null"; @@ -1986,11 +1953,7 @@ void RGWListBucket_ObjStore_S3v2::send_versioned_response() for (pref_iter = common_prefixes.begin(); pref_iter != common_prefixes.end(); ++pref_iter) { s->formatter->open_array_section("CommonPrefixes"); - if (encode_key) { - s->formatter->dump_string("Prefix", url_encode(pref_iter->first, false)); - } else { - s->formatter->dump_string("Prefix", pref_iter->first); - } + dump_urlsafe(s, encode_key, "Prefix", pref_iter->first, false); s->formatter->dump_int("KeyCount",objs.size()); if (start_after_exist) { @@ -2036,14 +1999,7 @@ void RGWListBucket_ObjStore_S3v2::send_response() for (iter = objs.begin(); iter != objs.end(); ++iter) { rgw_obj_key key(iter->key); s->formatter->open_array_section("Contents"); - if (encode_key) { - string key_name; - url_encode(key.name, key_name); - s->formatter->dump_string("Key", key_name); - } - else { - s->formatter->dump_string("Key", key.name); - } + dump_urlsafe(s, encode_key, "Key", key.name); dump_time(s, "LastModified", iter->meta.mtime); s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str()); s->formatter->dump_int("Size", iter->meta.accounted_size); @@ -4147,11 +4103,7 @@ void RGWListBucketMultiparts_ObjStore_S3::send_response() for (iter = uploads.begin(); iter != uploads.end(); ++iter) { rgw::sal::MultipartUpload* upload = iter->get(); s->formatter->open_array_section("Upload"); - if (encode_url) { - s->formatter->dump_string("Key", url_encode(upload->get_key(), false)); - } else { - s->formatter->dump_string("Key", upload->get_key()); - } + dump_urlsafe(s, encode_url, "Key", upload->get_key(), false); s->formatter->dump_string("UploadId", upload->get_upload_id()); const ACLOwner& owner = upload->get_owner(); dump_owner(s, owner.get_id(), owner.get_display_name(), "Initiator"); @@ -4163,11 +4115,7 @@ void RGWListBucketMultiparts_ObjStore_S3::send_response() if (!common_prefixes.empty()) { s->formatter->open_array_section("CommonPrefixes"); for (const auto& kv : common_prefixes) { - if (encode_url) { - s->formatter->dump_string("Prefix", url_encode(kv.first, false)); - } else { - s->formatter->dump_string("Prefix", kv.first); - } + dump_urlsafe(s, encode_url, "Prefix", kv.first, false); } s->formatter->close_section(); }