]> git-server-git.apps.pok.os.sepia.ceph.com Git - ceph.git/commitdiff
rgw: rework rest client to handle meta attributes
authorYehuda Sadeh <yehuda@redhat.com>
Wed, 11 Apr 2018 03:53:13 +0000 (20:53 -0700)
committerYehuda Sadeh <yehuda@redhat.com>
Wed, 25 Apr 2018 16:03:53 +0000 (09:03 -0700)
Some cleanup around the rest client, and fix cloud sync code to
send the meta attributes to the remote endpoint. Earlier we didn't
do it correctly when dealing with sync that uses multipart upload.

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
src/rgw/rgw_cr_rest.h
src/rgw/rgw_rados.cc
src/rgw/rgw_rest_client.cc
src/rgw/rgw_rest_client.h
src/rgw/rgw_rest_conn.cc
src/rgw/rgw_rest_conn.h
src/rgw/rgw_sync_module.h
src/rgw/rgw_sync_module_aws.cc

index 43448836af4bd9f31a413006bc276c72343a565c..7d4156fa382528fb020c1165f50aa262b8d34877 100644 (file)
@@ -15,6 +15,10 @@ struct rgw_rest_obj {
   std::map<string, string> attrs;
   std::map<string, string> custom_attrs;
   RGWAccessControlPolicy acls;
+
+  void init(const rgw_obj_key& _key) {
+    key = _key;
+  }
 };
 
 class RGWReadRawRESTResourceCR : public RGWSimpleCoroutine {
@@ -119,6 +123,7 @@ class RGWSendRawRESTResourceCR: public RGWSimpleCoroutine {
   string path;
   param_vec_t params;
   param_vec_t headers;
+  map<string, string> *attrs;
   T *result;
   bufferlist input_bl;
   bool send_content_length=false;
@@ -129,23 +134,19 @@ class RGWSendRawRESTResourceCR: public RGWSimpleCoroutine {
                           RGWHTTPManager *_http_manager,
                           const string& _method, const string& _path,
                           rgw_http_param_pair *_params,
-                          map<string, string> *_headers,
+                          map<string, string> *_attrs,
                           bufferlist& _input, T *_result, bool _send_content_length)
    : RGWSimpleCoroutine(_cct), conn(_conn), http_manager(_http_manager),
-    method(_method), path(_path), params(make_param_list(_params)), headers(make_param_list(_headers)), result(_result),
-    input_bl(_input), send_content_length(_send_content_length)
-    {}
+    method(_method), path(_path), params(make_param_list(_params)), headers(make_param_list(_attrs)), attrs(_attrs), result(_result),
+    input_bl(_input), send_content_length(_send_content_length) {}
 
- RGWSendRawRESTResourceCR(CephContext *_cct, RGWRESTConn *_conn,
 RGWSendRawRESTResourceCR(CephContext *_cct, RGWRESTConn *_conn,
                           RGWHTTPManager *_http_manager,
                           const string& _method, const string& _path,
-                          rgw_http_param_pair *_params, map<string, string> *_headers,
+                          rgw_http_param_pair *_params, map<string, string> *_attrs,
                           T *_result)
    : RGWSimpleCoroutine(_cct), conn(_conn), http_manager(_http_manager),
-    method(_method), path(_path), params(make_param_list(_params)), headers(make_param_list(_headers)), result(_result)
-    {}
-
-
+    method(_method), path(_path), params(make_param_list(_params)), headers(make_param_list(_attrs)), attrs(_attrs), result(_result) {}
 
   ~RGWSendRawRESTResourceCR() override {
     request_cleanup();
@@ -202,9 +203,9 @@ class RGWSendRESTResourceCR : public RGWSendRawRESTResourceCR<T> {
   RGWSendRESTResourceCR(CephContext *_cct, RGWRESTConn *_conn,
                            RGWHTTPManager *_http_manager,
                            const string& _method, const string& _path,
-                        rgw_http_param_pair *_params, map<string, string> *_headers,
+                        rgw_http_param_pair *_params, map<string, string> *_attrs,
                         S& _input, T *_result)
-    : RGWSendRawRESTResourceCR<T>(_cct, _conn, _http_manager, _method, _path, _params, _headers, _result) {
+    : RGWSendRawRESTResourceCR<T>(_cct, _conn, _http_manager, _method, _path, _params, _attrs, _result) {
 
     JSONFormatter jf;
     encode_json("data", _input, &jf);
@@ -246,9 +247,9 @@ class RGWPostRawRESTResourceCR: public RGWSendRawRESTResourceCR <T> {
                           RGWHTTPManager *_http_manager,
                           const string& _path,
                           rgw_http_param_pair *_params,
-                          map<string, string> * _headers,
+                          map<string, string> * _attrs,
                           bufferlist& _input, T *_result)
-    : RGWSendRawRESTResourceCR<T>(_cct, _conn, _http_manager, "POST", _path, _params, _headers, _input, _result, true){}
+    : RGWSendRawRESTResourceCR<T>(_cct, _conn, _http_manager, "POST", _path, _params, _attrs, _input, _result, true){}
 
 };
 
@@ -427,7 +428,7 @@ public:
                                                                 env(_env),
                                                                 caller(_caller),
                                                                 http_manager(_http_manager) {
-    rest_obj.key = _src_key;
+    rest_obj.init(_src_key);
   }
 
   int init() override;
index 9eb866a98698908c683da9284900e3b4c1a6d7b9..51c6ef990dfd766bf4ccee7d6a53e5c55d7f211a 100644 (file)
@@ -7851,7 +7851,6 @@ int RGWRados::stat_remote_obj(RGWObjectCtx& obj_ctx,
   }
 
   RGWGetExtraDataCB cb;
-  string etag;
   map<string, string> req_headers;
   real_time set_mtime;
 
@@ -7873,7 +7872,7 @@ int RGWRados::stat_remote_obj(RGWObjectCtx& obj_ctx,
     return ret;
   }
 
-  ret = conn->complete_request(in_stream_req, etag, &set_mtime, psize, req_headers);
+  ret = conn->complete_request(in_stream_req, nullptr, &set_mtime, psize, nullptr, pheaders);
   if (ret < 0) {
     return ret;
   }
@@ -7910,10 +7909,6 @@ int RGWRados::stat_remote_obj(RGWObjectCtx& obj_ctx,
     *pattrs = std::move(src_attrs);
   }
 
-  if (pheaders) {
-    *pheaders = std::move(req_headers);
-  }
-
   return 0;
 }
 
@@ -8022,7 +8017,6 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
   RGWRadosPutObj cb(cct, plugin, compressor, &processor, opstate, progress_cb, progress_data);
 
   string etag;
-  map<string, string> req_headers;
   real_time set_mtime;
 
   RGWObjState *dest_state = NULL;
@@ -8058,7 +8052,7 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
     goto set_err_state;
   }
 
-  ret = conn->complete_request(in_stream_req, etag, &set_mtime, nullptr, req_headers);
+  ret = conn->complete_request(in_stream_req, &etag, &set_mtime, nullptr, nullptr, nullptr);
   if (ret < 0) {
     goto set_err_state;
   }
index b56e633838e3bbfd350db20ded53e44d82991039..cd8b8e08122e154cddfeaabdae51947b0d2fa069 100644 (file)
@@ -190,10 +190,9 @@ int RGWHTTPSimpleRequest::receive_data(void *ptr, size_t len, bool *pause)
   response.append(p);
 
   return 0;
-
 }
 
-void RGWHTTPSimpleRequest::append_param(string& dest, const string& name, const string& val)
+static void append_param(string& dest, const string& name, const string& val)
 {
   if (dest.empty()) {
     dest.append("?");
@@ -212,18 +211,22 @@ void RGWHTTPSimpleRequest::append_param(string& dest, const string& name, const
   }
 }
 
-void RGWHTTPSimpleRequest::get_params_str(map<string, string>& extra_args, string& dest)
+static void do_get_params_str(const param_vec_t& params, map<string, string>& extra_args, string& dest)
 {
   map<string, string>::iterator miter;
   for (miter = extra_args.begin(); miter != extra_args.end(); ++miter) {
     append_param(dest, miter->first, miter->second);
   }
-  param_vec_t::iterator iter;
-  for (iter = params.begin(); iter != params.end(); ++iter) {
+  for (auto iter = params.begin(); iter != params.end(); ++iter) {
     append_param(dest, iter->first, iter->second);
   }
 }
 
+void RGWHTTPSimpleRequest::get_params_str(map<string, string>& extra_args, string& dest)
+{
+  do_get_params_str(params, extra_args, dest);
+}
+
 void RGWHTTPSimpleRequest::get_out_headers(map<string, string> *pheaders)
 {
   unique_lock guard(out_headers_lock);
@@ -436,58 +439,51 @@ static void add_grants_headers(map<int, string>& grants, RGWEnv& env, map<string
   }
 }
 
-void RGWRESTStreamS3PutObj::send_init(rgw_obj& obj)
+void RGWRESTGenerateHTTPHeaders::init(const string& _method, const string& _url, const string& resource, const param_vec_t& params)
 {
-  string resource_str;
-  string resource;
-  string new_url = url;
+  string params_str;
+  map<string, string>& args = new_info->args.get_params();
+  do_get_params_str(params, args, params_str);
 
-  if (host_style == VirtualStyle) {
-    resource_str = obj.get_oid();
-    new_url = obj.bucket.name + "."  + new_url;
-  } else {
-    resource_str = obj.bucket.name + "/" + obj.get_oid();
+  /* merge params with extra args so that we can sign correctly */
+  for (auto iter = params.begin(); iter != params.end(); ++iter) {
+    new_info->args.append(iter->first, iter->second);
   }
 
-  //do not encode slash in object key name
-  url_encode(resource_str, resource, false);
-
-  string uri;
-  //do not encode slash in object key name
-  url_encode(obj.bucket.name + "/" + obj.get_oid(), uri, false);
-
-  if (new_url[new_url.size() - 1] != '/')
-    new_url.append("/");
+  url = _url + resource + params_str;
 
   string date_str;
   get_gmt_date_str(date_str);
 
-  string params_str;
-  map<string, string>& args = new_info.args.get_params();
-  get_params_str(args, params_str);
-
-  /* merge params with extra args so that we can sign correctly */
-  for (param_vec_t::iterator iter = params.begin(); iter != params.end(); ++iter) {
-    new_info.args.append(iter->first, iter->second);
-  }
+  new_env->set("HTTP_DATE", date_str.c_str());
 
-  new_url.append(resource + params_str);
-
-  new_env.set("HTTP_DATE", date_str.c_str());
+  method = _method;
+  new_info->method = method.c_str();
 
-  new_info.method = "PUT";
+  new_info->script_uri = "/";
+  new_info->script_uri.append(resource);
+  new_info->request_uri = new_info->script_uri;
+}
 
-  new_info.script_uri = "/";
-  new_info.script_uri.append(uri);
-  new_info.request_uri = new_info.script_uri;
+static bool is_x_amz(const string& s) {
+  return boost::algorithm::starts_with(s, "x-amz-");
+}
 
-  method = new_info.method;
-  url = new_url;
+void RGWRESTGenerateHTTPHeaders::set_extra_headers(const map<string, string>& extra_headers)
+{
+  for (auto iter : extra_headers) {
+    const string& name = lowercase_dash_http_attr(iter.first);
+    new_env->set(name, iter.second.c_str());
+    if (is_x_amz(name)) {
+      new_info->x_meta_map[name] = iter.second;
+    }
+  }
 }
 
-int RGWRESTStreamS3PutObj::send_ready(RGWAccessKey& key, map<string, bufferlist>& rgw_attrs, bool send)
+int RGWRESTGenerateHTTPHeaders::set_obj_attrs(map<string, bufferlist>& rgw_attrs)
 {
   map<string, string> new_attrs;
+
   /* merge send headers */
   for (auto& attr: rgw_attrs) {
     bufferlist& bl = attr.second;
@@ -507,7 +503,10 @@ int RGWRESTStreamS3PutObj::send_ready(RGWAccessKey& key, map<string, bufferlist>
     return ret;
   }
 
-  return send_ready(key, new_attrs, policy, send);
+  set_http_attrs(new_attrs);
+  set_policy(policy);
+
+  return 0;
 }
 
 static std::set<string> keep_headers = { "content-type",
@@ -515,25 +514,25 @@ static std::set<string> keep_headers = { "content-type",
                                          "content-disposition",
                                          "content-language" };
 
-int RGWRESTStreamS3PutObj::send_ready(RGWAccessKey& key, const map<string, string>& http_attrs,
-                                      RGWAccessControlPolicy& policy, bool send)
+void RGWRESTGenerateHTTPHeaders::set_http_attrs(const map<string, string>& http_attrs)
 {
-  map<string, string> other_headers;
-
   /* merge send headers */
   for (auto& attr: http_attrs) {
     const string& val = attr.second;
     const string& name = lowercase_dash_http_attr(attr.first);
-    if (name.compare(0, sizeof(RGW_AMZ_PREFIX) - 1, RGW_AMZ_PREFIX) == 0) {
-      new_env.set(name, val);
-      new_info.x_meta_map[name] = val;
-    } else if (keep_headers.find(name) != keep_headers.end()) {
-      new_env.set(attr.first, val); /* Ugh, using the uppercase representation,
+    if (is_x_amz(name)) {
+      new_env->set(name, val);
+      new_info->x_meta_map[name] = val;
+    } else {
+      new_env->set(attr.first, val); /* Ugh, using the uppercase representation,
                                        as the signing function calls info.env.get("CONTENT_TYPE").
                                        This needs to be cleaned up! */
     }
   }
+}
 
+void RGWRESTGenerateHTTPHeaders::set_policy(RGWAccessControlPolicy& policy)
+{
   /* update acl headers */
   RGWAccessControlList& acl = policy.get_acl();
   multimap<string, ACLGrant>& grant_map = acl.get_grant_map();
@@ -544,13 +543,65 @@ int RGWRESTStreamS3PutObj::send_ready(RGWAccessKey& key, const map<string, strin
     ACLPermission& perm = grant.get_permission();
     grants_by_type_add_perm(grants_by_type, perm.get_permissions(), grant);
   }
-  add_grants_headers(grants_by_type, new_env, new_info.x_meta_map);
-  int ret = sign_request(cct, key, new_env, new_info);
+  add_grants_headers(grants_by_type, *new_env, new_info->x_meta_map);
+}
+
+int RGWRESTGenerateHTTPHeaders::sign(RGWAccessKey& key)
+{
+  int ret = sign_request(cct, key, *new_env, *new_info);
   if (ret < 0) {
     ldout(cct, 0) << "ERROR: failed to sign request" << dendl;
     return ret;
   }
 
+  return 0;
+}
+
+void RGWRESTStreamS3PutObj::send_init(rgw_obj& obj)
+{
+  string resource_str;
+  string resource;
+  string new_url = url;
+
+  if (host_style == VirtualStyle) {
+    resource_str = obj.get_oid();
+    new_url = obj.bucket.name + "."  + new_url;
+  } else {
+    resource_str = obj.bucket.name + "/" + obj.get_oid();
+  }
+
+  //do not encode slash in object key name
+  url_encode(resource_str, resource, false);
+
+  if (new_url[new_url.size() - 1] != '/')
+    new_url.append("/");
+
+  method = "PUT";
+  headers_gen.init(method, new_url, resource, params);
+
+  url = headers_gen.get_url();
+}
+
+int RGWRESTStreamS3PutObj::send_ready(RGWAccessKey& key, map<string, bufferlist>& rgw_attrs, bool send)
+{
+  headers_gen.set_obj_attrs(rgw_attrs);
+
+  return send_ready(key, send);
+}
+
+int RGWRESTStreamS3PutObj::send_ready(RGWAccessKey& key, const map<string, string>& http_attrs,
+                                      RGWAccessControlPolicy& policy, bool send)
+{
+  headers_gen.set_http_attrs(http_attrs);
+  headers_gen.set_policy(policy);
+
+  return send_ready(key, send);
+}
+
+int RGWRESTStreamS3PutObj::send_ready(RGWAccessKey& key, bool send)
+{
+  headers_gen.sign(key);
+
   for (const auto& kv: new_env.get_map()) {
     headers.emplace_back(kv);
   }
@@ -654,21 +705,9 @@ int RGWRESTStreamRWRequest::do_send_prepare(RGWAccessKey *key, map<string, strin
   if (new_url[new_url.size() - 1] != '/')
     new_url.append("/");
   
-  string date_str;
-  get_gmt_date_str(date_str);
-
   RGWEnv new_env;
   req_info new_info(cct, &new_env);
   
-  string params_str;
-  map<string, string>& args = new_info.args.get_params();
-  get_params_str(args, params_str);
-
-  /* merge params with extra args so that we can sign correctly */
-  for (param_vec_t::iterator iter = params.begin(); iter != params.end(); ++iter) {
-    new_info.args.append(iter->first, iter->second);
-  }
-
   string new_resource;
   string bucket_name;
   string old_resource = resource;
@@ -679,14 +718,12 @@ int RGWRESTStreamRWRequest::do_send_prepare(RGWAccessKey *key, map<string, strin
     new_resource = resource;
   }
 
-  string uri = new_resource;
-
   size_t pos = new_resource.find("/");
   bucket_name = new_resource.substr(0, pos);
 
   //when dest is a bucket with out other params, uri should end up with '/'
   if(pos == string::npos && params.size() == 0 && host_style == VirtualStyle) {
-    uri.append("/");
+    new_resource.append("/");
   }
 
   if (host_style == VirtualStyle) {
@@ -698,25 +735,18 @@ int RGWRESTStreamRWRequest::do_send_prepare(RGWAccessKey *key, map<string, strin
     }
   }
 
-  new_url.append(new_resource + params_str);
-
-  new_env.set("HTTP_DATE", date_str.c_str());
-
-  for (map<string, string>::iterator iter = extra_headers.begin();
-       iter != extra_headers.end(); ++iter) {
-    new_env.set(iter->first.c_str(), iter->second.c_str());
-  }
+  RGWRESTGenerateHTTPHeaders headers_gen(cct, &new_env, &new_info);
 
-  new_info.method = method.c_str();
+  headers_gen.init(method, new_url, new_resource, params);
 
-  new_info.script_uri = "/";
-  new_info.script_uri.append(uri);
-  new_info.request_uri = new_info.script_uri;
-
-  new_info.init_meta_info(NULL);
+  headers_gen.set_http_attrs(extra_headers);
 
   if (key) {
-    int ret = sign_request(cct, *key, new_env, new_info);
+#if 0
+    new_info.init_meta_info(nullptr);
+#endif
+
+    int ret = headers_gen.sign(*key);
     if (ret < 0) {
       ldout(cct, 0) << "ERROR: failed to sign request" << dendl;
       return ret;
@@ -735,7 +765,7 @@ int RGWRESTStreamRWRequest::do_send_prepare(RGWAccessKey *key, map<string, strin
   
 
   method = new_info.method;
-  url = new_url;
+  url = headers_gen.get_url();
 
   return 0;
 }
@@ -765,7 +795,11 @@ int RGWRESTStreamRWRequest::send(RGWHTTPManager *mgr)
   return 0;
 }
 
-int RGWRESTStreamRWRequest::complete_request(string& etag, real_time *mtime, uint64_t *psize, map<string, string>& attrs)
+int RGWRESTStreamRWRequest::complete_request(string *etag,
+                                             real_time *mtime,
+                                             uint64_t *psize,
+                                             map<string, string> *pattrs,
+                                             map<string, string> *pheaders)
 {
   int ret = wait();
   if (ret < 0) {
@@ -774,7 +808,9 @@ int RGWRESTStreamRWRequest::complete_request(string& etag, real_time *mtime, uin
 
   unique_lock guard(out_headers_lock);
 
-  set_str_from_headers(out_headers, "ETAG", etag);
+  if (etag) {
+    set_str_from_headers(out_headers, "ETAG", *etag);
+  }
   if (status >= 0) {
     if (mtime) {
       string mtime_str;
@@ -800,8 +836,7 @@ int RGWRESTStreamRWRequest::complete_request(string& etag, real_time *mtime, uin
     }
   }
 
-  map<string, string>::iterator iter;
-  for (iter = out_headers.begin(); iter != out_headers.end(); ++iter) {
+  for (auto iter = out_headers.begin(); pattrs && iter != out_headers.end(); ++iter) {
     const string& attr_name = iter->first;
     if (attr_name.compare(0, sizeof(RGW_HTTP_RGWX_ATTR_PREFIX) - 1, RGW_HTTP_RGWX_ATTR_PREFIX) == 0) {
       string name = attr_name.substr(sizeof(RGW_HTTP_RGWX_ATTR_PREFIX) - 1);
@@ -818,9 +853,13 @@ int RGWRESTStreamRWRequest::complete_request(string& etag, real_time *mtime, uin
         }
       }
       *dest = '\0';
-      attrs[buf] = iter->second;
+      (*pattrs)[buf] = iter->second;
     }
   }
+
+  if (pheaders) {
+    *pheaders = std::move(out_headers);
+  }
   return status;
 }
 
index 27842e0bda771de2c2dbfaf51bea5eafccc7e9ed..4ce188857a6ac0e28b93950ca5417b72a66d6286 100644 (file)
@@ -25,7 +25,6 @@ protected:
   bufferlist response;
 
   virtual int handle_header(const string& name, const string& val);
-  void append_param(string& dest, const string& name, const string& val);
   void get_params_str(map<string, string>& extra_args, string& dest);
 
 public:
@@ -76,6 +75,25 @@ public:
   virtual void notify(uint64_t pending_size) = 0;
 };
 
+class RGWRESTGenerateHTTPHeaders {
+  CephContext *cct;
+  RGWEnv *new_env;
+  req_info *new_info;
+  string method;
+  string url;
+  string resource;
+
+public:
+  RGWRESTGenerateHTTPHeaders(CephContext *_cct, RGWEnv *_env, req_info *_info) : cct(_cct), new_env(_env), new_info(_info) {}
+  void init(const string& method, const string& url, const string& resource, const param_vec_t& params);
+  void set_extra_headers(const map<string, string>& extra_headers);
+  int set_obj_attrs(map<string, bufferlist>& rgw_attrs);
+  void set_http_attrs(const map<string, string>& http_attrs);
+  void set_policy(RGWAccessControlPolicy& policy);
+  int sign(RGWAccessKey& key);
+
+  const string& get_url() { return url; }
+};
 
 class RGWHTTPStreamRWRequest : public RGWHTTPSimpleRequest {
 public:
@@ -159,7 +177,11 @@ public:
   int send_request(RGWAccessKey& key, map<string, string>& extra_headers, const rgw_obj& obj, RGWHTTPManager *mgr);
   int send_request(RGWAccessKey *key, map<string, string>& extra_headers, const string& resource, RGWHTTPManager *mgr, bufferlist *send_data = nullptr /* optional input data */);
 
-  int complete_request(string& etag, real_time *mtime, uint64_t *psize, map<string, string>& attrs);
+  int complete_request(string *etag = nullptr,
+                       real_time *mtime = nullptr,
+                       uint64_t *psize = nullptr,
+                       map<string, string> *pattrs = nullptr,
+                       map<string, string> *pheaders = nullptr);
 
   void add_params(param_vec_t *params);
 
@@ -183,16 +205,18 @@ class RGWRESTStreamS3PutObj : public RGWRESTStreamRWRequest {
   RGWGetDataCB *out_cb;
   RGWEnv new_env;
   req_info new_info;
+  RGWRESTGenerateHTTPHeaders headers_gen;
 public:
   RGWRESTStreamS3PutObj(CephContext *_cct, const string& _method, const string& _url, param_vec_t *_headers,
                param_vec_t *_params, HostStyle _host_style) : RGWRESTStreamRWRequest(_cct, _method, _url, nullptr, _headers, _params, _host_style),
-                out_cb(NULL), new_info(cct, &new_env) {}
+                out_cb(NULL), new_info(cct, &new_env), headers_gen(_cct, &new_env, &new_info) {}
   ~RGWRESTStreamS3PutObj() override;
 
   void send_init(rgw_obj& obj);
   int send_ready(RGWAccessKey& key, map<string, bufferlist>& rgw_attrs, bool send);
   int send_ready(RGWAccessKey& key, const map<string, string>& http_attrs,
                  RGWAccessControlPolicy& policy, bool send);
+  int send_ready(RGWAccessKey& key, bool send);
 
   int put_obj_init(RGWAccessKey& key, rgw_obj& obj, uint64_t obj_size, map<string, bufferlist>& attrs, bool send);
 
index 1b0712f9c7ec2cd382d9e84fdfff583718c7f952..4257e26fde7225c89ca8bad7377078a69d38ac40 100644 (file)
@@ -157,8 +157,7 @@ int RGWRESTConn::put_obj_async(const rgw_user& uid, rgw_obj& obj, uint64_t obj_s
 
 int RGWRESTConn::complete_request(RGWRESTStreamS3PutObj *req, string& etag, real_time *mtime)
 {
-  map<string, string> attrs;
-  int ret = req->complete_request(etag, mtime, nullptr, attrs);
+  int ret = req->complete_request(&etag, mtime);
   delete req;
 
   return ret;
@@ -292,10 +291,14 @@ done_err:
   return r;
 }
 
-int RGWRESTConn::complete_request(RGWRESTStreamRWRequest *req, string& etag, real_time *mtime,
-                                  uint64_t *psize, map<string, string>& attrs)
+int RGWRESTConn::complete_request(RGWRESTStreamRWRequest *req,
+                                  string *etag,
+                                  real_time *mtime,
+                                  uint64_t *psize,
+                                  map<string, string> *pattrs,
+                                  map<string, string> *pheaders)
 {
-  int ret = req->complete_request(etag, mtime, psize, attrs);
+  int ret = req->complete_request(etag, mtime, psize, pattrs, pheaders);
   delete req;
 
   return ret;
@@ -336,9 +339,7 @@ int RGWRESTConn::get_resource(const string& resource,
     return ret;
   }
 
-  string etag;
-  map<string, string> attrs;
-  return req.complete_request(etag, NULL, NULL, attrs);
+  return req.complete_request();
 }
 
 RGWRESTReadResource::RGWRESTReadResource(RGWRESTConn *_conn,
@@ -383,9 +384,7 @@ int RGWRESTReadResource::read()
     return ret;
   }
 
-  string etag;
-  map<string, string> attrs;
-  return req.complete_request(etag, NULL, NULL, attrs);
+  return req.complete_request();
 }
 
 int RGWRESTReadResource::aio_read()
@@ -446,9 +445,7 @@ int RGWRESTSendResource::send(bufferlist& outbl)
     return ret;
   }
 
-  string etag;
-  map<string, string> attrs;
-  return req.complete_request(etag, NULL, NULL, attrs);
+  return req.complete_request();
 }
 
 int RGWRESTSendResource::aio_send(bufferlist& outbl)
index 00a24d2d4d3584e28fd70efccb1755120cbdc1f1..4ae17760a49447204aa29e06e42c4fc7aadfb7c0 100644 (file)
@@ -152,7 +152,12 @@ public:
               uint32_t mod_zone_id, uint64_t mod_pg_ver,
               bool prepend_metadata, bool get_op, bool rgwx_stat, bool sync_manifest,
               bool skip_decrypt, bool send, RGWHTTPStreamRWRequest::ReceiveCB *cb, RGWRESTStreamRWRequest **req);
-  int complete_request(RGWRESTStreamRWRequest *req, string& etag, ceph::real_time *mtime, uint64_t *psize, map<string, string>& attrs);
+  int complete_request(RGWRESTStreamRWRequest *req,
+                       string *etag,
+                       ceph::real_time *mtime,
+                       uint64_t *psize,
+                       map<string, string> *pattrs,
+                       map<string, string> *pheaders);
 
   int get_resource(const string& resource,
                   param_vec_t *extra_params,
index dfc26214041f5d57a14164401c07659758b929cb..78e9858ab3f95a7bd02ed70dec502c9c56aee376 100644 (file)
@@ -141,6 +141,7 @@ public:
     size = _size;
     etag = _etag;
     attrs = std::move(_attrs);
+    headers = std::move(_headers);
   }
 };
 
index 56731489238e5012a6638c6c8f18cc703b05daa1..9231df5f1849da371621a5f93b313d2ce8a98a22 100644 (file)
@@ -676,7 +676,7 @@ struct AWSSyncInstanceEnv {
   }
 };
 
-int do_decode_rest_obj(CephContext *cct, map<string, bufferlist>& attrs, map<string, string>& headers, rgw_rest_obj *info)
+static int do_decode_rest_obj(CephContext *cct, map<string, bufferlist>& attrs, map<string, string>& headers, rgw_rest_obj *info)
 {
   for (auto header : headers) {
     const string& val = header.second;
@@ -778,6 +778,11 @@ public:
   }
 };
 
+static std::set<string> keep_headers = { "CONTENT_TYPE",
+                                         "CONTENT_ENCODING",
+                                         "CONTENT_DISPOSITION",
+                                         "CONTENT_LANGUAGE" };
+
 class RGWAWSStreamPutCRF : public RGWStreamWriteHTTPResourceCRF
 {
   RGWDataSyncEnv *sync_env;
@@ -816,6 +821,11 @@ public:
     return RGWStreamWriteHTTPResourceCRF::init();
   }
 
+  static bool keep_attr(const string& h) {
+    return (keep_headers.find(h) != keep_headers.end() ||
+            boost::algorithm::starts_with(h, "X_AMZ_"));
+  }
+
   static void init_send_attrs(CephContext *cct,
                               const rgw_rest_obj& rest_obj,
                               const rgw_sync_aws_src_obj_properties& src_properties,
@@ -823,7 +833,13 @@ public:
                               map<string, string> *attrs) {
     auto& new_attrs = *attrs;
 
-    new_attrs = rest_obj.attrs;
+    new_attrs.clear();
+
+    for (auto& hi : rest_obj.attrs) {
+      if (keep_attr(hi.first)) {
+        new_attrs.insert(hi);
+      }
+    }
 
     auto acl = rest_obj.acls.get_acl();
 
@@ -938,8 +954,6 @@ public:
     map<string, string> new_attrs;
     if (!multipart.is_multipart) {
       init_send_attrs(sync_env->cct, rest_obj, src_properties, target.get(), &new_attrs);
-    } else {
-      new_attrs = rest_obj.attrs;
     }
 
     r->set_send_length(rest_obj.content_len);
@@ -1131,7 +1145,7 @@ class RGWAWSInitMultipartCR : public RGWCoroutine {
   rgw_obj dest_obj;
 
   uint64_t obj_size;
-  map<string, string> obj_headers;
+  map<string, string> attrs;
 
   bufferlist out_bl;
 
@@ -1154,13 +1168,13 @@ public:
                         RGWRESTConn *_dest_conn,
                         const rgw_obj& _dest_obj,
                         uint64_t _obj_size,
-                        const map<string, string>& _obj_headers,
+                        const map<string, string>& _attrs,
                         string *_upload_id) : RGWCoroutine(_sync_env->cct),
                                                    sync_env(_sync_env),
                                                    dest_conn(_dest_conn),
                                                    dest_obj(_dest_obj),
                                                    obj_size(_obj_size),
-                                                   obj_headers(_obj_headers),
+                                                   attrs(_attrs),
                                                    upload_id(_upload_id) {}
 
   int operate() {
@@ -1170,7 +1184,7 @@ public:
         rgw_http_param_pair params[] = { { "uploads", nullptr }, {nullptr, nullptr} };
         bufferlist bl;
         call(new RGWPostRawRESTResourceCR <bufferlist> (sync_env->cct, dest_conn, sync_env->http_manager,
-                                                 obj_to_aws_path(dest_obj), params, &obj_headers, bl, &out_bl));
+                                                 obj_to_aws_path(dest_obj), params, &attrs, bl, &out_bl));
       }
 
       if (retcode < 0) {
@@ -1379,7 +1393,7 @@ class RGWAWSStreamObjToCloudMultipartCR : public RGWCoroutine {
 
   rgw_sync_aws_multipart_upload_info status;
 
-  map<string, string> obj_headers;
+  map<string, string> new_attrs;
 
   rgw_sync_aws_multipart_part_info *pcur_part_info{nullptr};
 
@@ -1432,9 +1446,9 @@ public:
       }
 
       if (retcode == -ENOENT) {
-        RGWAWSStreamPutCRF::init_send_attrs(sync_env->cct, rest_obj, src_properties, target.get(), &obj_headers);
+        RGWAWSStreamPutCRF::init_send_attrs(sync_env->cct, rest_obj, src_properties, target.get(), &new_attrs);
 
-        yield call(new RGWAWSInitMultipartCR(sync_env, target->conn.get(), dest_obj, status.obj_size, std::move(obj_headers), &status.upload_id));
+        yield call(new RGWAWSInitMultipartCR(sync_env, target->conn.get(), dest_obj, status.obj_size, std::move(new_attrs), &status.upload_id));
         if (retcode < 0) {
           return set_cr_error(retcode);
         }
@@ -1655,6 +1669,7 @@ public:
                                                  dest_obj));
         } else {
           rgw_rest_obj rest_obj;
+          rest_obj.init(key);
           if (do_decode_rest_obj(sync_env->cct, attrs, headers, &rest_obj)) {
             ldout(sync_env->cct, 0) << "ERROR: failed to decode rest obj out of headers=" << headers << ", attrs=" << attrs << dendl;
             return set_cr_error(-EINVAL);