From: Yehuda Sadeh <yehuda@inktank.com>
Date: Wed, 18 Dec 2013 21:10:21 +0000 (-0800)
Subject: rgw: don't return data within the librados cb
X-Git-Tag: v0.67.5~2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=7587ee52161d74a41bf89dc741e8ece9c964fb67;p=ceph.git

rgw: don't return data within the librados cb

Fixes: #7030
The callback is running within a single Finisher thread, thus we
shouldn't block there. Append read data to a list and flush it within
the iterate context.

Reviewed-by: Sage Weil <sage@inktank.com>
Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
(cherry picked from commit d6a4f6adfaa75c3140d07d6df7be03586cc16183)
---

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index cc020ddbfa13..70af545db93d 100644
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -4097,6 +4097,7 @@ struct get_obj_data : public RefCountedObject {
   atomic_t cancelled;
   atomic_t err_code;
   Throttle throttle;
+  list<bufferlist> read_list;
 
   get_obj_data(CephContext *_cct)
     : cct(_cct),
@@ -4284,20 +4285,44 @@ void RGWRados::get_obj_aio_completion_cb(completion_t c, void *arg)
     goto done_unlock;
   }
 
-  for (iter = bl_list.begin(); iter != bl_list.end(); ++iter) {
+  d->read_list.splice(d->read_list.end(), bl_list);
+
+done_unlock:
+  d->data_lock.Unlock();
+done:
+  d->put();
+  return;
+}
+
+int RGWRados::flush_read_list(struct get_obj_data *d)
+{
+  d->data_lock.Lock();
+  list<bufferlist> l;
+  l.swap(d->read_list);
+  d->get();
+  d->read_list.clear();
+
+  d->data_lock.Unlock();
+
+  int r = 0;
+
+  list<bufferlist>::iterator iter;
+  for (iter = l.begin(); iter != l.end(); ++iter) {
     bufferlist& bl = *iter;
-    int r = d->client_cb->handle_data(bl, 0, bl.length());
+    r = d->client_cb->handle_data(bl, 0, bl.length());
     if (r < 0) {
-      d->set_cancelled(r);
+      dout(0) << "ERROR: flush_read_list(): d->client_c->handle_data() returned " << r << dendl;
       break;
     }
   }
 
-done_unlock:
-  d->data_lock.Unlock();
-done:
+  d->data_lock.Lock();
   d->put();
-  return;
+  if (r < 0) {
+    d->set_cancelled(r);
+  }
+  d->data_lock.Unlock();
+  return r;
 }
 
 int RGWRados::get_obj_iterate_cb(void *ctx, RGWObjState *astate,
@@ -4344,6 +4369,10 @@ int RGWRados::get_obj_iterate_cb(void *ctx, RGWObjState *astate,
     }
   }
 
+  r = flush_read_list(d);
+  if (r < 0)
+    return r;
+
   get_obj_bucket_and_oid_key(obj, bucket, oid, key);
 
   d->throttle.get(len);
@@ -4404,6 +4433,12 @@ int RGWRados::get_obj_iterate(void *ctx, void **handle, rgw_obj& obj,
       data->cancel_all_io();
       break;
     }
+    r = flush_read_list(data);
+    if (r < 0) {
+      dout(10) << "get_obj_iterate() r=" << r << ", canceling all io" << dendl;
+      data->cancel_all_io();
+      break;
+    }
   }
 
 done:
diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index fa5954c9019c..e9e3791f1b78 100644
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -1232,6 +1232,8 @@ public:
                       off_t ofs, off_t end,
 	              RGWGetDataCB *cb);
 
+  int flush_read_list(struct get_obj_data *d);
+
   int get_obj_iterate_cb(void *ctx, RGWObjState *astate,
                          rgw_obj& obj,
                          off_t obj_ofs, off_t read_ofs, off_t len,