rgw: fix list obj v1 nextmarker

when encoding-type=url given, aws sdk will try to decode
next marker returned in a list obj response.
if it is not actually encoded and the marker obj just
have an encoding symbol in its name, aws sdk may get
error when trying to decode it.

Signed-off-by: imtzw <tongzhiwei_yewu@cmss.chinamobile.com>

diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h
index 434de99e9c2ec3c13153d6f2618d6920aec66944..0b6cf62ed2c732ddcacf5eb5850de8376c0d96c9 100644 (file)
--- a/src/rgw/rgw_rest.h
+++ b/src/rgw/rgw_rest.h
@@ -706,6 +706,16 @@ extern void dump_start(req_state *s);
 extern void list_all_buckets_start(req_state *s);
 extern void dump_owner(req_state *s, const rgw_user& id,
                        const std::string& name, const char *section = NULL);
+inline void dump_urlsafe(req_state *s, bool encode_key, const char* key, const std::string& val, bool encode_slash = true) {
+  if (encode_key) {
+    std::string _val;
+    url_encode(val, _val, encode_slash);
+    s->formatter->dump_string(key, _val);
+  }
+  else {
+    s->formatter->dump_string(key, val);
+  }
+}
 extern void dump_header(req_state* s,
                         const std::string_view& name,
                         const std::string_view& val);

diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 6c3a8c8adc1682ee1305aa0b92d4c3e874f7a6c0..511adb990ad126a3ba90795a76108a05533fa72d 100644 (file)
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -1707,11 +1707,7 @@ void RGWListBucket_ObjStore_S3::send_common_versioned_response()
       for (pref_iter = common_prefixes.begin();
       pref_iter != common_prefixes.end(); ++pref_iter) {
       s->formatter->open_array_section("CommonPrefixes");
-      if (encode_key) {
-        s->formatter->dump_string("Prefix", url_encode(pref_iter->first, false));
-      } else {
-        s->formatter->dump_string("Prefix", pref_iter->first);
-      }
+      dump_urlsafe(s, encode_key, "Prefix", pref_iter->first, false);
 
       s->formatter->close_section();
       }
@@ -1729,7 +1725,7 @@ void RGWListBucket_ObjStore_S3::send_versioned_response()
   s->formatter->dump_string("KeyMarker", marker.name);
   s->formatter->dump_string("VersionIdMarker", marker.instance);
   if (is_truncated && !next_marker.empty()) {
-    s->formatter->dump_string("NextKeyMarker", next_marker.name);
+    dump_urlsafe(s ,encode_key, "NextKeyMarker", next_marker.name);
     if (next_marker.instance.empty()) {
       s->formatter->dump_string("NextVersionIdMarker", "null");
     }
@@ -1752,14 +1748,7 @@ void RGWListBucket_ObjStore_S3::send_versioned_response()
         s->formatter->dump_bool("IsDeleteMarker", iter->is_delete_marker());
       }
       rgw_obj_key key(iter->key);
-      if (encode_key) {
-        string key_name;
-        url_encode(key.name, key_name);
-        s->formatter->dump_string("Key", key_name);
-      }
-      else {
-        s->formatter->dump_string("Key", key.name);
-      }
+      dump_urlsafe(s ,encode_key, "Key", key.name);
       string version_id = key.instance;
       if (version_id.empty()) {
         version_id = "null";
@@ -1807,11 +1796,7 @@ void RGWListBucket_ObjStore_S3::send_common_response()
   s->formatter->dump_string("Prefix", prefix);
   s->formatter->dump_int("MaxKeys", max);
   if (!delimiter.empty()) {
-    if (encode_key) {
-      s->formatter->dump_string("Delimiter", url_encode(delimiter, false));
-    } else {
-      s->formatter->dump_string("Delimiter", delimiter);
-    }
+    dump_urlsafe(s, encode_key, "Delimiter", delimiter, false);
   }
   s->formatter->dump_string("IsTruncated", (max && is_truncated ? "true"
               : "false"));
@@ -1821,11 +1806,7 @@ void RGWListBucket_ObjStore_S3::send_common_response()
       for (pref_iter = common_prefixes.begin();
       pref_iter != common_prefixes.end(); ++pref_iter) {
       s->formatter->open_array_section("CommonPrefixes");
-      if (encode_key) {
-        s->formatter->dump_string("Prefix", url_encode(pref_iter->first, false));
-      } else {
-        s->formatter->dump_string("Prefix", pref_iter->first);
-      }
+      dump_urlsafe(s, encode_key, "Prefix", pref_iter->first, false);
       s->formatter->close_section();
       }
     }
@@ -1866,13 +1847,6 @@ void RGWListBucket_ObjStore_S3::send_response()
     for (iter = objs.begin(); iter != objs.end(); ++iter) {
 
       rgw_obj_key key(iter->key);
-      std::string key_name;
-
-      if (encode_key) {
-       url_encode(key.name, key_name);
-      } else {
-       key_name = key.name;
-      }
       /* conditionally format JSON in the obvious way--I'm unsure if
        * AWS actually does this */
       if (s->format == RGWFormat::XML) {
@@ -1881,7 +1855,7 @@ void RGWListBucket_ObjStore_S3::send_response()
        // json
        s->formatter->open_object_section("dummy");
       }
-      s->formatter->dump_string("Key", key_name);
+      dump_urlsafe(s ,encode_key, "Key", key.name);
       dump_time(s, "LastModified", iter->meta.mtime);
       s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str());
       s->formatter->dump_int("Size", iter->meta.accounted_size);
@@ -1905,7 +1879,7 @@ void RGWListBucket_ObjStore_S3::send_response()
   }
   s->formatter->dump_string("Marker", marker.name);
   if (is_truncated && !next_marker.empty()) {
-    s->formatter->dump_string("NextMarker", next_marker.name);
+    dump_urlsafe(s, encode_key, "NextMarker", next_marker.name);
   }
   s->formatter->close_section();
   rgw_flush_formatter_and_reset(s, s->formatter);
@@ -1941,14 +1915,7 @@ void RGWListBucket_ObjStore_S3v2::send_versioned_response()
         s->formatter->dump_bool("IsDeleteContinuationToken", iter->is_delete_marker());
       }
       rgw_obj_key key(iter->key);
-      if (encode_key) {
-        string key_name;
-        url_encode(key.name, key_name);
-        s->formatter->dump_string("Key", key_name);
-      }
-      else {
-        s->formatter->dump_string("Key", key.name);
-      }
+      dump_urlsafe(s, encode_key, "Key", key.name);
       string version_id = key.instance;
       if (version_id.empty()) {
         version_id = "null";
@@ -1986,11 +1953,7 @@ void RGWListBucket_ObjStore_S3v2::send_versioned_response()
       for (pref_iter = common_prefixes.begin();
       pref_iter != common_prefixes.end(); ++pref_iter) {
       s->formatter->open_array_section("CommonPrefixes");
-      if (encode_key) {
-        s->formatter->dump_string("Prefix", url_encode(pref_iter->first, false));
-      } else {
-        s->formatter->dump_string("Prefix", pref_iter->first);
-      }
+      dump_urlsafe(s, encode_key, "Prefix", pref_iter->first, false);
 
       s->formatter->dump_int("KeyCount",objs.size());
       if (start_after_exist) {
@@ -2036,14 +1999,7 @@ void RGWListBucket_ObjStore_S3v2::send_response()
     for (iter = objs.begin(); iter != objs.end(); ++iter) {
       rgw_obj_key key(iter->key);
       s->formatter->open_array_section("Contents");
-      if (encode_key) {
-        string key_name;
-        url_encode(key.name, key_name);
-        s->formatter->dump_string("Key", key_name);
-      }
-      else {
-        s->formatter->dump_string("Key", key.name);
-      }
+      dump_urlsafe(s, encode_key, "Key", key.name);
       dump_time(s, "LastModified", iter->meta.mtime);
       s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str());
       s->formatter->dump_int("Size", iter->meta.accounted_size);
@@ -4147,11 +4103,7 @@ void RGWListBucketMultiparts_ObjStore_S3::send_response()
     for (iter = uploads.begin(); iter != uploads.end(); ++iter) {
       rgw::sal::MultipartUpload* upload = iter->get();
       s->formatter->open_array_section("Upload");
-      if (encode_url) {
-        s->formatter->dump_string("Key", url_encode(upload->get_key(), false));
-      } else {
-        s->formatter->dump_string("Key", upload->get_key());
-      }
+      dump_urlsafe(s, encode_url, "Key", upload->get_key(), false);
       s->formatter->dump_string("UploadId", upload->get_upload_id());
       const ACLOwner& owner = upload->get_owner();
       dump_owner(s, owner.get_id(), owner.get_display_name(), "Initiator");
@@ -4163,11 +4115,7 @@ void RGWListBucketMultiparts_ObjStore_S3::send_response()
     if (!common_prefixes.empty()) {
       s->formatter->open_array_section("CommonPrefixes");
       for (const auto& kv : common_prefixes) {
-        if (encode_url) {
-          s->formatter->dump_string("Prefix", url_encode(kv.first, false));
-        } else {
-          s->formatter->dump_string("Prefix", kv.first);
-        }
+        dump_urlsafe(s, encode_url, "Prefix", kv.first, false);
       }
       s->formatter->close_section();
     }