From: Adam C. Emerson <aemerson@redhat.com>
Date: Wed, 30 Jan 2019 19:26:33 +0000 (-0500)
Subject: rgw: Get rid of num_rados_handles infrastructure in RGWSI_RADOS
X-Git-Tag: v14.2.3~17^2~3
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=57b072ccaefee458998e277a0e61af66ad098b8e;p=ceph.git

rgw: Get rid of num_rados_handles infrastructure in RGWSI_RADOS

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
(cherry picked from commit 5666b4c41a70bac75e26b44a2cec1bd895d8375d)
---

diff --git a/src/rgw/services/svc_notify.cc b/src/rgw/services/svc_notify.cc
index 27a5a169fa9d..e557fa0fb0b9 100644
--- a/src/rgw/services/svc_notify.cc
+++ b/src/rgw/services/svc_notify.cc
@@ -179,7 +179,7 @@ int RGWSI_Notify::init_watch()
       notify_oid = notify_oid_prefix;
     }
 
-    notify_objs[i] = rados_svc->handle(0).obj({control_pool, notify_oid});
+    notify_objs[i] = rados_svc->handle().obj({control_pool, notify_oid});
     auto& notify_obj = notify_objs[i];
 
     int r = notify_obj.open();
@@ -295,7 +295,7 @@ int RGWSI_Notify::unwatch(RGWSI_RADOS::Obj& obj, uint64_t watch_handle)
     ldout(cct, 0) << "ERROR: rados->unwatch2() returned r=" << r << dendl;
     return r;
   }
-  r = rados_svc->handle(0).watch_flush();
+  r = rados_svc->handle().watch_flush();
   if (r < 0) {
     ldout(cct, 0) << "ERROR: rados->watch_flush() returned r=" << r << dendl;
     return r;
diff --git a/src/rgw/services/svc_rados.cc b/src/rgw/services/svc_rados.cc
index 4e95fe278cc3..30bf849f7ef3 100644
--- a/src/rgw/services/svc_rados.cc
+++ b/src/rgw/services/svc_rados.cc
@@ -44,63 +44,31 @@ static int init_ioctx(CephContext *cct, librados::Rados *rados, const rgw_pool&
 
 int RGWSI_RADOS::do_start()
 {
-  auto handles = std::vector<librados::Rados>{static_cast<size_t>(1)};
-
-  for (auto& r : handles) {
-    int ret = r.init_with_context(cct);
-    if (ret < 0) {
-      return ret;
-    }
-    ret = r.connect();
-    if (ret < 0) {
-      return ret;
-    }
+  int ret = rados.init_with_context(cct);
+  if (ret < 0) {
+    return ret;
+  }
+  ret = rados.connect();
+  if (ret < 0) {
+    return ret;
   }
-  std::swap(handles, rados);
   return 0;
 }
 
-librados::Rados* RGWSI_RADOS::get_rados_handle(int rados_handle)
+librados::Rados* RGWSI_RADOS::get_rados_handle()
 {
-  if (rados.size() == 1) {
-    return &rados[0];
-  }
-
-  if (rados_handle >= 0) {
-    if (rados_handle >= (int)rados.size()) {
-      rados_handle = 0;
-    }
-    return &rados[rados_handle];
-  }
-
-  handle_lock.get_read();
-  pthread_t id = pthread_self();
-  std::map<pthread_t, int>:: iterator it = rados_map.find(id);
-
-  if (it != rados_map.end()) {
-    handle_lock.put_read();
-    return &rados[it->second];
-  }
-  handle_lock.put_read();
-  handle_lock.get_write();
-  const uint32_t handle = next_rados_handle;
-  rados_map[id] = handle;
-  if (++next_rados_handle == rados.size()) {
-    next_rados_handle = 0;
-  }
-  handle_lock.put_write();
-  return &rados[handle];
+  return &rados;
 }
 
 uint64_t RGWSI_RADOS::instance_id()
 {
-  return get_rados_handle(-1)->get_instance_id();
+  return get_rados_handle()->get_instance_id();
 }
 
-int RGWSI_RADOS::open_pool_ctx(const rgw_pool& pool, librados::IoCtx& io_ctx, int rados_handle)
+int RGWSI_RADOS::open_pool_ctx(const rgw_pool& pool, librados::IoCtx& io_ctx)
 {
   constexpr bool create = true; // create the pool if it doesn't exist
-  return init_ioctx(cct, get_rados_handle(rados_handle), pool, io_ctx, create);
+  return init_ioctx(cct, get_rados_handle(), pool, io_ctx, create);
 }
 
 int RGWSI_RADOS::pool_iterate(librados::IoCtx& io_ctx,
@@ -141,7 +109,7 @@ void RGWSI_RADOS::Obj::init(const rgw_raw_obj& obj)
 
 int RGWSI_RADOS::Obj::open()
 {
-  int r = rados_svc->open_pool_ctx(ref.obj.pool, ref.ioctx, rados_handle);
+  int r = rados_svc->open_pool_ctx(ref.obj.pool, ref.ioctx);
   if (r < 0) {
     return r;
   }
@@ -210,7 +178,7 @@ uint64_t RGWSI_RADOS::Obj::get_last_version()
 
 int RGWSI_RADOS::Pool::create()
 {
-  librados::Rados *rad = rados_svc->get_rados_handle(rados_handle);
+  librados::Rados *rad = rados_svc->get_rados_handle();
   int r = rad->pool_create(pool.name.c_str());
   if (r < 0) {
     ldout(rados_svc->cct, 0) << "WARNING: pool_create returned " << r << dendl;
@@ -235,7 +203,7 @@ int RGWSI_RADOS::Pool::create(const vector<rgw_pool>& pools, vector<int> *retcod
   vector<librados::PoolAsyncCompletion *> completions;
   vector<int> rets;
 
-  librados::Rados *rad = rados_svc->get_rados_handle(rados_handle);
+  librados::Rados *rad = rados_svc->get_rados_handle();
   for (auto iter = pools.begin(); iter != pools.end(); ++iter) {
     librados::PoolAsyncCompletion *c = librados::Rados::pool_async_create_completion();
     completions.push_back(c);
@@ -311,7 +279,7 @@ int RGWSI_RADOS::Pool::create(const vector<rgw_pool>& pools, vector<int> *retcod
 
 int RGWSI_RADOS::Pool::lookup()
 {
-  librados::Rados *rad = rados_svc->get_rados_handle(rados_handle);
+  librados::Rados *rad = rados_svc->get_rados_handle();
   int ret = rad->pool_lookup(pool.name.c_str());
   if (ret < 0) {
     return ret;
@@ -326,7 +294,7 @@ int RGWSI_RADOS::Pool::List::init(const string& marker, RGWAccessListFilter *fil
     return -EINVAL;
   }
 
-  int r = pool.rados_svc->open_pool_ctx(pool.pool, ctx.ioctx, pool.rados_handle);
+  int r = pool.rados_svc->open_pool_ctx(pool.pool, ctx.ioctx);
   if (r < 0) {
     return r;
   }
@@ -370,7 +338,6 @@ int RGWSI_RADOS::Pool::List::get_next(int max,
 
 int RGWSI_RADOS::Handle::watch_flush()
 {
-  librados::Rados *rad = rados_svc->get_rados_handle(rados_handle);
+  librados::Rados *rad = rados_svc->get_rados_handle();
   return rad->watch_flush();
 }
-
diff --git a/src/rgw/services/svc_rados.h b/src/rgw/services/svc_rados.h
index a90558696cf9..0453eb0cded3 100644
--- a/src/rgw/services/svc_rados.h
+++ b/src/rgw/services/svc_rados.h
@@ -29,15 +29,12 @@ struct rgw_rados_ref {
 
 class RGWSI_RADOS : public RGWServiceInstance
 {
-  std::vector<librados::Rados> rados;
-  uint32_t next_rados_handle{0};
-  RWLock handle_lock;
-  std::map<pthread_t, int> rados_map;
+  librados::Rados rados;
 
   int do_start() override;
 
-  librados::Rados* get_rados_handle(int rados_handle);
-  int open_pool_ctx(const rgw_pool& pool, librados::IoCtx& io_ctx, int rados_handle);
+  librados::Rados* get_rados_handle();
+  int open_pool_ctx(const rgw_pool& pool, librados::IoCtx& io_ctx);
   int pool_iterate(librados::IoCtx& ioctx,
                    librados::NObjectIterator& iter,
                    uint32_t num, vector<rgw_bucket_dir_entry>& objs,
@@ -45,8 +42,7 @@ class RGWSI_RADOS : public RGWServiceInstance
                    bool *is_truncated);
 
 public:
-  RGWSI_RADOS(CephContext *cct): RGWServiceInstance(cct),
-                                 handle_lock("rados_handle_lock") {}
+  RGWSI_RADOS(CephContext *cct) : RGWServiceInstance(cct) {}
 
   void init() {}
 
@@ -56,15 +52,15 @@ public:
 
   class Obj {
     friend class RGWSI_RADOS;
-    friend class Handle;
+    friend Handle;
 
     RGWSI_RADOS *rados_svc{nullptr};
-    int rados_handle{-1};
     rgw_rados_ref ref;
 
     void init(const rgw_raw_obj& obj);
 
-    Obj(RGWSI_RADOS *_rados_svc, const rgw_raw_obj& _obj, int _rados_handle) : rados_svc(_rados_svc), rados_handle(_rados_handle) {
+    Obj(RGWSI_RADOS *_rados_svc, const rgw_raw_obj& _obj)
+      : rados_svc(_rados_svc) {
       init(_obj);
     }
 
@@ -98,17 +94,14 @@ public:
 
   class Pool {
     friend class RGWSI_RADOS;
-    friend class Handle;
+    friend Handle;
 
     RGWSI_RADOS *rados_svc{nullptr};
-    int rados_handle{-1};
     rgw_pool pool;
 
     Pool(RGWSI_RADOS *_rados_svc,
-         const rgw_pool& _pool,
-         int _rados_handle) : rados_svc(_rados_svc),
-                              rados_handle(_rados_handle),
-                              pool(_pool) {}
+         const rgw_pool& _pool) : rados_svc(_rados_svc),
+                                  pool(_pool) {}
 
     Pool(RGWSI_RADOS *_rados_svc) : rados_svc(_rados_svc) {}
   public:
@@ -140,35 +133,33 @@ public:
       return List(*this);
     }
 
-    friend class List;
+    friend List;
   };
 
   class Handle {
     friend class RGWSI_RADOS;
 
     RGWSI_RADOS *rados_svc{nullptr};
-    int rados_handle{-1};
 
-    Handle(RGWSI_RADOS *_rados_svc, int _rados_handle) : rados_svc(_rados_svc),
-                                                         rados_handle(_rados_handle) {}
+    Handle(RGWSI_RADOS *_rados_svc) : rados_svc(_rados_svc) {}
   public:
     Obj obj(const rgw_raw_obj& o) {
-      return Obj(rados_svc, o, rados_handle);
+      return Obj(rados_svc, o);
     }
 
     Pool pool(const rgw_pool& p) {
-      return Pool(rados_svc, p, rados_handle);
+      return Pool(rados_svc, p);
     }
 
     int watch_flush();
   };
 
-  Handle handle(int rados_handle) {
-    return Handle(this, rados_handle);
+  Handle handle() {
+    return Handle(this);
   }
 
   Obj obj(const rgw_raw_obj& o) {
-    return Obj(this, o, -1);
+    return Obj(this, o);
   }
 
   Pool pool() {
@@ -176,12 +167,12 @@ public:
   }
 
   Pool pool(const rgw_pool& p) {
-    return Pool(this, p, -1);
+    return Pool(this, p);
   }
 
-  friend class Obj;
-  friend class Pool;
-  friend class Pool::List;
+  friend Obj;
+  friend Pool;
+  friend Pool::List;
 };
 
 #endif