rgw: don't return data within the librados cb

Fixes: #7030
The callback is running within a single Finisher thread, thus we
shouldn't block there. Append read data to a list and flush it within
the iterate context.

Reviewed-by: Sage Weil <sage@inktank.com>
Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
(cherry picked from commit d6a4f6adfaa75c3140d07d6df7be03586cc16183)

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 37eacc0533ae5d42d911d963381bddc6b4fda42a..a5247ffca294703b3cf451c47f85adb9c5c627b5 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -4151,6 +4151,7 @@ struct get_obj_data : public RefCountedObject {
   atomic_t cancelled;
   atomic_t err_code;
   Throttle throttle;
+  list<bufferlist> read_list;
 
   get_obj_data(CephContext *_cct)
     : cct(_cct),
@@ -4338,20 +4339,44 @@ void RGWRados::get_obj_aio_completion_cb(completion_t c, void *arg)
     goto done_unlock;
   }
 
-  for (iter = bl_list.begin(); iter != bl_list.end(); ++iter) {
+  d->read_list.splice(d->read_list.end(), bl_list);
+
+done_unlock:
+  d->data_lock.Unlock();
+done:
+  d->put();
+  return;
+}
+
+int RGWRados::flush_read_list(struct get_obj_data *d)
+{
+  d->data_lock.Lock();
+  list<bufferlist> l;
+  l.swap(d->read_list);
+  d->get();
+  d->read_list.clear();
+
+  d->data_lock.Unlock();
+
+  int r = 0;
+
+  list<bufferlist>::iterator iter;
+  for (iter = l.begin(); iter != l.end(); ++iter) {
     bufferlist& bl = *iter;
-    int r = d->client_cb->handle_data(bl, 0, bl.length());
+    r = d->client_cb->handle_data(bl, 0, bl.length());
     if (r < 0) {
-      d->set_cancelled(r);
+      dout(0) << "ERROR: flush_read_list(): d->client_c->handle_data() returned " << r << dendl;
       break;
     }
   }
 
-done_unlock:
-  d->data_lock.Unlock();
-done:
+  d->data_lock.Lock();
   d->put();
-  return;
+  if (r < 0) {
+    d->set_cancelled(r);
+  }
+  d->data_lock.Unlock();
+  return r;
 }
 
 int RGWRados::get_obj_iterate_cb(void *ctx, RGWObjState *astate,
@@ -4398,6 +4423,10 @@ int RGWRados::get_obj_iterate_cb(void *ctx, RGWObjState *astate,
     }
   }
 
+  r = flush_read_list(d);
+  if (r < 0)
+    return r;
+
   get_obj_bucket_and_oid_key(obj, bucket, oid, key);
 
   d->throttle.get(len);
@@ -4458,6 +4487,12 @@ int RGWRados::get_obj_iterate(void *ctx, void **handle, rgw_obj& obj,
       data->cancel_all_io();
       break;
     }
+    r = flush_read_list(data);
+    if (r < 0) {
+      dout(10) << "get_obj_iterate() r=" << r << ", canceling all io" << dendl;
+      data->cancel_all_io();
+      break;
+    }
   }
 
 done:

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 476572ce3f6c9c69a6de74aae3d8ad4522d68b1c..b0b39ecc2fd80c2e0737020be773d25a57c5468d 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -1263,6 +1263,8 @@ public:
                       off_t ofs, off_t end,
                      RGWGetDataCB *cb);
 
+  int flush_read_list(struct get_obj_data *d);
+
   int get_obj_iterate_cb(void *ctx, RGWObjState *astate,
                          rgw_obj& obj,
                          off_t obj_ofs, off_t read_ofs, off_t len,