From: Yehuda Sadeh <yehuda@redhat.com>
Date: Tue, 14 Aug 2018 00:28:43 +0000 (-0700)
Subject: rgw: more refactoring work, initial work on svc_rados
X-Git-Tag: v14.1.0~965^2~43
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=ca8a636d080297502a8743f5a3fe192795761958;p=ceph.git

rgw: more refactoring work, initial work on svc_rados

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
---

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 42d334204427..a51ef9cc355b 100644
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -2400,7 +2400,7 @@ int RGWRados::init_complete()
     return ret;
   }
 
-  if (is_meta_master()) {
+  if (zone_svc->is_meta_master()) {
     auto md_log = meta_mgr->get_log(current_period.get_id());
     meta_notifier = new RGWMetaNotifier(this, md_log);
     meta_notifier->start();
@@ -2524,6 +2524,17 @@ int RGWRados::initialize()
     cct->_conf.get_val<double>("rgw_inject_notify_timeout_probability");
   max_notify_retries = cct->_conf.get_val<uint64_t>("rgw_max_notify_retries");
 
+  svc_registry = std::make_unique<RGWServiceRegistry>();
+  svc_registry->register_all();
+
+  JSONFormattable zone_svc_conf;
+  ret = svc_registry->get_instance("zone", zone_svc_conf, &zone_svc);
+  if (ret < 0) {
+    return ret;
+  }
+
+  host_id = zone_svc->gen_host_id();
+
   ret = init_rados();
   if (ret < 0)
     return ret;
@@ -6320,15 +6331,6 @@ int RGWRados::copy_obj_data(RGWObjectCtx& obj_ctx,
                             nullptr, nullptr, nullptr, nullptr, nullptr);
 }
 
-bool RGWRados::is_meta_master()
-{
-  if (!get_zonegroup().is_master_zonegroup()) {
-    return false;
-  }
-
-  return (get_zonegroup().master_zone == zone_public_config.id);
-}
-
 /**
   * Check to see if the bucket metadata could be synced
   * bucket: the bucket to check
@@ -11867,7 +11869,6 @@ int RGWRados::get_bucket_index_object(const string& bucket_oid_base, const strin
   return r;
 }
 
-
 uint64_t RGWRados::instance_id()
 {
   return get_rados_handle()->get_instance_id();
diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 7f7c7413e50a..aa87942cadac 100644
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -28,6 +28,7 @@
 #include "rgw_sync_log_trim.h"
 
 #include "services/svc_zone.h"
+#include "services/svc_rados.h"
 
 class RGWWatcher;
 class SafeTimer;
@@ -995,45 +996,39 @@ struct RGWListRawObjsCtx {
   RGWListRawObjsCtx() : initialized(false) {}
 };
 
-struct RGWDefaultSystemMetaObjInfo {
-  string default_id;
+struct objexp_hint_entry {
+  string tenant;
+  string bucket_name;
+  string bucket_id;
+  rgw_obj_key obj_key;
+  ceph::real_time exp_time;
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(1, 1, bl);
-    encode(default_id, bl);
-    ENCODE_FINISH(bl);
-  }
-
-  void decode(bufferlist::const_iterator& bl) {
-    DECODE_START(1, bl);
-    decode(default_id, bl);
-    DECODE_FINISH(bl);
-  }
-
-  void dump(Formatter *f) const;
-  void decode_json(JSONObj *obj);
-};
-WRITE_CLASS_ENCODER(RGWDefaultSystemMetaObjInfo)
-
-struct RGWNameToId {
-  string obj_id;
-
-  void encode(bufferlist& bl) const {
-    ENCODE_START(1, 1, bl);
-    encode(obj_id, bl);
+    ENCODE_START(2, 1, bl);
+    encode(bucket_name, bl);
+    encode(bucket_id, bl);
+    encode(obj_key, bl);
+    encode(exp_time, bl);
+    encode(tenant, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::const_iterator& bl) {
-    DECODE_START(1, bl);
-    decode(obj_id, bl);
+    // XXX Do we want DECODE_START_LEGACY_COMPAT_LEN(2, 1, 1, bl); ?
+    DECODE_START(2, bl);
+    decode(bucket_name, bl);
+    decode(bucket_id, bl);
+    decode(obj_key, bl);
+    decode(exp_time, bl);
+    if (struct_v >= 2) {
+      decode(tenant, bl);
+    } else {
+      tenant.clear();
+    }
     DECODE_FINISH(bl);
   }
-
-  void dump(Formatter *f) const;
-  void decode_json(JSONObj *obj);
 };
-WRITE_CLASS_ENCODER(RGWNameToId)
+WRITE_CLASS_ENCODER(objexp_hint_entry)
 
 class RGWDataChangesLog;
 class RGWMetaSyncStatusManager;
@@ -1069,13 +1064,6 @@ public:
 class RGWGetDirHeader_CB;
 class RGWGetUserHeader_CB;
 
-struct rgw_rados_ref {
-  rgw_pool pool;
-  string oid;
-  string key;
-  librados::IoCtx ioctx;
-};
-
 class RGWChainedCache {
 public:
   virtual ~RGWChainedCache() {}
@@ -1328,7 +1316,7 @@ protected:
   RGWSyncModuleInstanceRef sync_module;
   bool writeable_zone{false};
 
-
+  RGWServiceRegistryRef svc_registry;
   std::shared_ptr<RGWSI_Zone> zone_svc;
 
   RGWIndexCompletionManager *index_completion_manager{nullptr};
@@ -1368,20 +1356,8 @@ public:
    * AmazonS3 errors contain a HostId string, but is an opaque base64 blob; we
    * try to be more transparent. This has a wrapper so we can update it when zonegroup/zone are changed.
    */
-  void init_host_id() {
-    /* uint64_t needs 16, two '-' separators and a trailing null */
-    const string& zone_name = get_zone().name;
-    const string& zonegroup_name = zonegroup.get_name();
-    char charbuf[16 + zone_name.size() + zonegroup_name.size() + 2 + 1];
-    snprintf(charbuf, sizeof(charbuf), "%llx-%s-%s", (unsigned long long)instance_id(), zone_name.c_str(), zonegroup_name.c_str());
-    string s(charbuf);
-    host_id = s;
-  }
-
   string host_id;
 
-  RGWRealm realm;
-
   RGWRESTConn *rest_master_conn;
   map<string, RGWRESTConn *> zone_conn_map;
   map<string, RGWRESTConn *> zone_data_sync_from_map;
@@ -1419,26 +1395,11 @@ public:
   }
 
   const RGWQuotaInfo& get_bucket_quota() {
-    return current_period.get_config().bucket_quota;
+    return zone_svc->get_current_period().get_config().bucket_quota;
   }
 
   const RGWQuotaInfo& get_user_quota() {
-    return current_period.get_config().user_quota;
-  }
-
-  const string& get_current_period_id() {
-    return current_period.get_id();
-  }
-
-  bool has_zonegroup_api(const std::string& api) const {
-    if (!current_period.get_id().empty()) {
-      const auto& zonegroups_by_api = current_period.get_map().zonegroups_by_api;
-      if (zonegroups_by_api.find(api) != zonegroups_by_api.end())
-        return true;
-    } else if (zonegroup.api_name == api) {
-        return true;
-    }
-    return false;
+    return zone_svc->current_period.get_config().user_quota;
   }
 
   // pulls missing periods for period_history
@@ -1555,12 +1516,6 @@ public:
   int create_pool(const rgw_pool& pool);
 
   int init_bucket_index(RGWBucketInfo& bucket_info, int num_shards);
-  int select_bucket_placement(RGWUserInfo& user_info, const string& zonegroup_id, const string& rule,
-                              string *pselected_rule_name, RGWZonePlacementInfo *rule_info);
-  int select_legacy_bucket_placement(RGWZonePlacementInfo *rule_info);
-  int select_new_bucket_location(RGWUserInfo& user_info, const string& zonegroup_id, const string& rule,
-                                 string *pselected_rule_name, RGWZonePlacementInfo *rule_info);
-  int select_bucket_location_by_rule(const string& location_rule, RGWZonePlacementInfo *rule_info);
   void create_bucket_id(string *bucket_id);
 
   bool get_obj_data_pool(const string& placement_rule, const rgw_obj& obj, rgw_pool *pool);
@@ -2166,8 +2121,6 @@ public:
    */
   int delete_bucket(RGWBucketInfo& bucket_info, RGWObjVersionTracker& objv_tracker, bool check_empty = true);
 
-  bool is_meta_master();
-
   /**
    * Check to see if the bucket metadata is synced
    */
@@ -2602,10 +2555,7 @@ public:
   uint64_t instance_id();
 
   string unique_id(uint64_t unique_num) {
-    char buf[32];
-    snprintf(buf, sizeof(buf), ".%llu.%llu", (unsigned long long)instance_id(), (unsigned long long)unique_num);
-    string s = get_zone_params().get_id() + buf;
-    return s;
+    return zone_svc->unique_id(unique_num);
   }
 
   void init_unique_trans_id_deps() {
@@ -2637,24 +2587,6 @@ public:
     return string(buf) + trans_id_suffix;
   }
 
-  void get_log_pool(rgw_pool& pool) {
-    pool = get_zone_params().log_pool;
-  }
-
-  bool need_to_log_data() {
-    return get_zone().log_data;
-  }
-
-  bool need_to_log_metadata() {
-    return is_meta_master() &&
-      (get_zonegroup().zones.size() > 1 || current_period.is_multi_zonegroups_with_zones());
-  }
-
-  bool can_reshard() const {
-    return current_period.get_id().empty() ||
-      (zonegroup.zones.size() == 1 && current_period.is_single_zonegroup());
-  }
-
   librados::Rados* get_rados_handle();
 
   int delete_raw_obj_aio(const rgw_raw_obj& obj, list<librados::AioCompletion *>& handles);
diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc
index 39d98425c4e5..0a93150fa96c 100644
--- a/src/rgw/rgw_rest.cc
+++ b/src/rgw/rgw_rest.cc
@@ -171,8 +171,6 @@ static set<string> hostnames_s3website_set;
 
 void rgw_rest_init(CephContext *cct, RGWRados *store, RGWZoneGroup& zone_group)
 {
-  store->init_host_id();
-
   for (const auto& rgw2http : base_rgw_to_http_attrs)  {
     rgw_to_http_attrs[rgw2http.rgw_attr] = rgw2http.http_attr;
   }
diff --git a/src/rgw/rgw_service.cc b/src/rgw/rgw_service.cc
index 5a4da5903b04..d59361b251e2 100644
--- a/src/rgw/rgw_service.cc
+++ b/src/rgw/rgw_service.cc
@@ -2,7 +2,7 @@
 
 #include "services/svc_rados.h"
 
-
+#define dout_subsys ceph_subsys_rgw
 
 RGWServiceInstance::~RGWServiceInstance()
 {
@@ -27,9 +27,24 @@ bool RGWServiceRegistry::find(const string& name, RGWServiceRef *svc)
   return true;
 }
 
-int RGWServiceRegistry::instantiate(RGWServiceRegistryRef& registry, RGWServiceRef& svc, JSONFormattable& conf) {
+string RGWServiceRegistry::get_conf_id(const string& service_type, const string& conf)
+{
+  return service_type + ":" + conf;
+}
+
+int RGWServiceRegistry::get_instance(RGWServiceRef& svc,
+                                     const string& conf,
+                                     RGWServiceInstanceRef *ref) {
   auto self_ref = shared_from_this();
   RGWServiceInstanceRef instance_ref;
+
+  string conf_id = get_conf_id(svc->type(), conf);
+
+  auto iter = instances_by_conf.find(conf_id);
+  if (iter != instances_by_conf.end()) {
+    *ref = iter->second.ref;
+    return 0;
+  }
   int r = svc->create_instance(conf, &instance_ref);
   if (r < 0) {
     return r;
@@ -37,7 +52,31 @@ int RGWServiceRegistry::instantiate(RGWServiceRegistryRef& registry, RGWServiceR
   instance_ref->svc = svc;
   instance_ref->svc_id = ++max_registry_id;
 
-  r = instance_ref->init(conf);
+  map<string, RGWServiceInstanceRef> dep_refs;
+
+  instance_info& iinfo = instances[instance_ref->svc_id];
+  iinfo.conf_id = get_conf_id(svc->type(), conf);
+  iinfo.id = instance_ref->svc_id;
+  iinfo.title = instance_ref->get_title();
+  iinfo.conf = conf;
+  iinfo.ref = instance_ref;
+
+  instances_by_conf[iinfo.conf_id] = iinfo;
+
+  auto deps = instance_ref->get_deps();
+  for (auto iter : deps) {
+    auto& dep_id = iter.first;
+    auto& dep = iter.second;
+    RGWServiceInstanceRef dep_ref;
+    r = get_instance(dep.name, dep.conf, &dep_ref);
+    if (r < 0) {
+      ldout(cct, 0) << "ERROR: cannot satisfy dependency for service " << svc->type() << ": " << dep.name << dendl;
+      return r;
+    }
+    dep_refs[dep_id] = dep_ref;
+  }
+
+  r = instance_ref->init(conf, dep_refs);
   if (r < 0) {
     return r;
   }
@@ -48,14 +87,16 @@ int RGWServiceRegistry::instantiate(RGWServiceRegistryRef& registry, RGWServiceR
     instance_ref->svc_instance = buf;
   }
 
-  instance_info& iinfo = instances[instance_ref->svc_id];
-  iinfo.id = instance_ref->svc_id;
-  iinfo.title = instance_ref->get_title();
-  iinfo.conf = conf;
+  *ref = iinfo.ref;
 
   return 0;
 }
 
 void RGWServiceRegistry::remove_instance(RGWServiceInstance *instance) {
-  instances.erase(instance->svc_id);
+  auto iter = instances.find(instance->svc_id);
+  if (iter == instances.end()) {
+    return;
+  }
+  instances_by_conf.erase(iter->second.conf_id);
+  instances.erase(iter);
 }
diff --git a/src/rgw/rgw_service.h b/src/rgw/rgw_service.h
index f77a14e949e0..1681e923a863 100644
--- a/src/rgw/rgw_service.h
+++ b/src/rgw/rgw_service.h
@@ -10,7 +10,6 @@
 
 
 class CephContext;
-class JSONFormattable;
 class RGWServiceInstance;
 class RGWServiceRegistry;
 
@@ -35,7 +34,7 @@ public:
   const std::string& type() {
     return svc_type;
   }
-  virtual int create_instance(JSONFormattable& conf, RGWServiceInstanceRef *instance) = 0;
+  virtual int create_instance(const string& conf, RGWServiceInstanceRef *instance) = 0;
 };
 
 
@@ -51,10 +50,15 @@ protected:
   string svc_instance;
   uint64_t svc_id{0};
 
-  virtual std::vector<std::string> get_deps() {
-    return vector<std::string>();
+  struct dependency {
+    string name;
+    string conf;
+  };
+
+  virtual std::map<std::string, dependency> get_deps() {
+    return std::map<std::string, dependency>();
   }
-  virtual int init(JSONFormattable& conf) = 0;
+  virtual int init(const string& conf, std::map<std::string, RGWServiceInstanceRef>& dep_refs) = 0;
 public:
   RGWServiceInstance(RGWService *svc, CephContext *_cct) : cct(_cct) {}
 
@@ -66,26 +70,42 @@ public:
 };
 
 class RGWServiceRegistry : std::enable_shared_from_this<RGWServiceRegistry> {
+  CephContext *cct;
+
   map<string, RGWServiceRef> services;
 
   struct instance_info {
+    string conf_id;
     uint64_t id;
     string title;
-    JSONFormattable conf;
+    string conf;
     RGWServiceInstanceRef ref;
   };
   map<uint64_t, instance_info> instances; /* registry_id -> instance */
+  map<string, instance_info> instances_by_conf; /* conf_id -> instance */
 
   std::atomic<uint64_t> max_registry_id;
 
+  string get_conf_id(const string& service_type, const string& conf);
   void register_all(CephContext *cct);
 public:
-  RGWServiceRegistry(CephContext *cct) {
+  RGWServiceRegistry(CephContext *_cct) : cct(_cct) {
     register_all(cct);
   }
   bool find(const string& name, RGWServiceRef *svc);
 
-  int instantiate(RGWServiceRegistryRef& registry, RGWServiceRef& svc, JSONFormattable& conf);
+  int get_instance(RGWServiceRef& svc,
+                   const string& conf,
+                   RGWServiceInstanceRef *ref); /* returns existing or creates a new one */
+  int get_instance(const string& svc_name,
+                   const string& conf,
+                   RGWServiceInstanceRef *ref) {
+    auto iter = services.find(svc_name);
+    if (iter == services.end()) {
+      return -ENOENT;
+    }
+    return get_instance(iter->second, conf, ref);
+  }
   void remove_instance(RGWServiceInstance *instance);
 };
 
diff --git a/src/rgw/rgw_zone.h b/src/rgw/rgw_zone.h
index e66d4ab945b8..94708fb72649 100644
--- a/src/rgw/rgw_zone.h
+++ b/src/rgw/rgw_zone.h
@@ -1,6 +1,46 @@
 #ifndef CEPH_RGW_ZONE_H
 #define CEPH_RGW_ZONE_H
 
+struct RGWNameToId {
+  string obj_id;
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    encode(obj_id, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::const_iterator& bl) {
+    DECODE_START(1, bl);
+    decode(obj_id, bl);
+    DECODE_FINISH(bl);
+  }
+
+  void dump(Formatter *f) const;
+  void decode_json(JSONObj *obj);
+};
+WRITE_CLASS_ENCODER(RGWNameToId)
+
+struct RGWDefaultSystemMetaObjInfo {
+  string default_id;
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    encode(default_id, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::const_iterator& bl) {
+    DECODE_START(1, bl);
+    decode(default_id, bl);
+    DECODE_FINISH(bl);
+  }
+
+  void dump(Formatter *f) const;
+  void decode_json(JSONObj *obj);
+};
+WRITE_CLASS_ENCODER(RGWDefaultSystemMetaObjInfo)
+
 class RGWSystemMetaObj {
 protected:
   string id;
@@ -656,41 +696,6 @@ struct RGWZoneGroupMap {
 WRITE_CLASS_ENCODER(RGWZoneGroupMap)
 
 class RGWRealm;
-
-struct objexp_hint_entry {
-  string tenant;
-  string bucket_name;
-  string bucket_id;
-  rgw_obj_key obj_key;
-  ceph::real_time exp_time;
-
-  void encode(bufferlist& bl) const {
-    ENCODE_START(2, 1, bl);
-    encode(bucket_name, bl);
-    encode(bucket_id, bl);
-    encode(obj_key, bl);
-    encode(exp_time, bl);
-    encode(tenant, bl);
-    ENCODE_FINISH(bl);
-  }
-
-  void decode(bufferlist::const_iterator& bl) {
-    // XXX Do we want DECODE_START_LEGACY_COMPAT_LEN(2, 1, 1, bl); ?
-    DECODE_START(2, bl);
-    decode(bucket_name, bl);
-    decode(bucket_id, bl);
-    decode(obj_key, bl);
-    decode(exp_time, bl);
-    if (struct_v >= 2) {
-      decode(tenant, bl);
-    } else {
-      tenant.clear();
-    }
-    DECODE_FINISH(bl);
-  }
-};
-WRITE_CLASS_ENCODER(objexp_hint_entry)
-
 class RGWPeriod;
 
 class RGWRealm : public RGWSystemMetaObj
diff --git a/src/rgw/services/svc_rados.cc b/src/rgw/services/svc_rados.cc
index fed1e20fbe52..0ab76758b80c 100644
--- a/src/rgw/services/svc_rados.cc
+++ b/src/rgw/services/svc_rados.cc
@@ -1 +1,133 @@
 #include "svc_rados.h"
+
+#include "include/rados/librados.hpp"
+#include "common/errno.h"
+#include "osd/osd_types.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+int RGWS_RADOS::create_instance(const string& conf, RGWServiceInstanceRef *instance)
+{
+  *instance = std::make_shared<RGWServiceInstance>();
+  return 0;
+}
+
+static int init_ioctx(CephContext *cct, librados::Rados *rados, const rgw_pool& pool, librados::IoCtx& ioctx, bool create)
+{
+  int r = rados->ioctx_create(pool.name.c_str(), ioctx);
+  if (r == -ENOENT && create) {
+    r = rados->pool_create(pool.name.c_str());
+    if (r == -ERANGE) {
+      ldout(cct, 0)
+        << __func__
+        << " ERROR: librados::Rados::pool_create returned " << cpp_strerror(-r)
+        << " (this can be due to a pool or placement group misconfiguration, e.g."
+        << " pg_num < pgp_num or mon_max_pg_per_osd exceeded)"
+        << dendl;
+    }
+    if (r < 0 && r != -EEXIST) {
+      return r;
+    }
+
+    r = rados->ioctx_create(pool.name.c_str(), ioctx);
+    if (r < 0) {
+      return r;
+    }
+
+    r = ioctx.application_enable(pg_pool_t::APPLICATION_NAME_RGW, false);
+    if (r < 0 && r != -EOPNOTSUPP) {
+      return r;
+    }
+  } else if (r < 0) {
+    return r;
+  }
+  if (!pool.ns.empty()) {
+    ioctx.set_namespace(pool.ns);
+  }
+  return 0;
+}
+
+int RGWSI_RADOS::init(const string& conf, map<string, RGWServiceInstanceRef>& deps)
+{
+  auto handles = std::vector<librados::Rados>{static_cast<size_t>(cct->_conf->rgw_num_rados_handles)};
+
+  for (auto& r : handles) {
+    int ret = r.init_with_context(cct);
+    if (ret < 0) {
+      return ret;
+    }
+    ret = r.connect();
+    if (ret < 0) {
+      return ret;
+    }
+  }
+  return 0;
+}
+
+librados::Rados* RGWSI_RADOS::get_rados_handle()
+{
+  if (rados.size() == 1) {
+    return &rados[0];
+  }
+  handle_lock.get_read();
+  pthread_t id = pthread_self();
+  std::map<pthread_t, int>:: iterator it = rados_map.find(id);
+
+  if (it != rados_map.end()) {
+    handle_lock.put_read();
+    return &rados[it->second];
+  }
+  handle_lock.put_read();
+  handle_lock.get_write();
+  const uint32_t handle = next_rados_handle;
+  rados_map[id] = handle;
+  if (++next_rados_handle == rados.size()) {
+    next_rados_handle = 0;
+  }
+  handle_lock.put_write();
+  return &rados[handle];
+}
+
+uint64_t RGWSI_RADOS::instance_id()
+{
+  return get_rados_handle()->get_instance_id();
+}
+
+int RGWSI_RADOS::open_pool_ctx(const rgw_pool& pool, librados::IoCtx& io_ctx)
+{
+  constexpr bool create = true; // create the pool if it doesn't exist
+  return init_ioctx(cct, get_rados_handle(), pool, io_ctx, create);
+}
+
+void RGWSI_RADOS::Obj::init(const rgw_raw_obj& obj)
+{
+  ref.oid = obj.oid;
+  ref.key = obj.loc;
+  ref.pool = obj.pool;
+}
+
+int RGWSI_RADOS::Obj::open()
+{
+  int r = rados_svc->open_pool_ctx(ref.pool, ref.ioctx);
+  if (r < 0)
+    return r;
+
+  ref.ioctx.locator_set_key(ref.key);
+
+  return 0;
+}
+
+int RGWSI_RADOS::Obj::operate(librados::ObjectWriteOperation *op)
+{
+  return ref.ioctx.operate(ref.oid, op);
+}
+
+int RGWSI_RADOS::Obj::operate(librados::ObjectReadOperation *op, bufferlist *pbl)
+{
+  return ref.ioctx.operate(ref.oid, op, pbl);
+}
+
+int RGWSI_RADOS::Obj::aio_operate(librados::AioCompletion *c, librados::ObjectWriteOperation *op)
+{
+  return ref.ioctx.aio_operate(ref.oid, c, op);
+}
diff --git a/src/rgw/services/svc_rados.h b/src/rgw/services/svc_rados.h
index 15e68cf3613b..29b1752965bc 100644
--- a/src/rgw/services/svc_rados.h
+++ b/src/rgw/services/svc_rados.h
@@ -1,9 +1,10 @@
-#ifndef CEPH_RGW_SERVICES_ZONE_H
-#define CEPH_RGW_SERVICES_ZONE_H
+#ifndef CEPH_RGW_SERVICES_RADOS_H
+#define CEPH_RGW_SERVICES_RADOS_H
 
 
 #include "rgw/rgw_service.h"
 
+#include "include/rados/librados.hpp"
 
 class RGWS_RADOS : public RGWService
 {
@@ -11,8 +12,64 @@ class RGWS_RADOS : public RGWService
 public:
   RGWS_RADOS(CephContext *cct) : RGWService(cct, "rados") {}
 
-  int create_instance(JSONFormattable& conf, RGWServiceInstanceRef *instance);
+  int create_instance(const string& conf, RGWServiceInstanceRef *instance);
 };
 
+struct rgw_rados_ref {
+  rgw_pool pool;
+  string oid;
+  string key;
+  librados::IoCtx ioctx;
+};
+
+class RGWSI_RADOS : public RGWServiceInstance
+{
+  RGWServiceInstanceRef svc_rados;
+
+  std::vector<librados::Rados> rados;
+  uint32_t next_rados_handle{0};
+  RWLock handle_lock;
+  std::map<pthread_t, int> rados_map;
+
+  int init(const string& conf, std::map<std::string, RGWServiceInstanceRef>& deps);
+  librados::Rados* get_rados_handle();
+  int open_pool_ctx(const rgw_pool& pool, librados::IoCtx& io_ctx);
+public:
+  RGWSI_RADOS(RGWService *svc, CephContext *cct): RGWServiceInstance(svc, cct),
+                                                  handle_lock("rados_handle_lock") {}
+
+  uint64_t instance_id();
+
+  class Obj {
+    friend class RGWSI_RADOS;
+
+    RGWSI_RADOS *rados_svc;
+    rgw_rados_ref ref;
+
+    void init(const rgw_raw_obj& obj);
+
+    Obj(RGWSI_RADOS *_rados_svc, const rgw_raw_obj& _obj) : rados_svc(_rados_svc) {
+      init(_obj);
+    }
+
+  public:
+    Obj(const Obj& o) : rados_svc(o.rados_svc),
+                        ref(o.ref) {}
+
+    Obj(const Obj&& o) : rados_svc(o.rados_svc),
+                         ref(std::move(o.ref)) {}
+
+    int open();
+
+    int operate(librados::ObjectWriteOperation *op);
+    int operate(librados::ObjectReadOperation *op, bufferlist *pbl);
+    int aio_operate(librados::AioCompletion *c, librados::ObjectWriteOperation *op);
+  };
+
+  Obj obj(const rgw_raw_obj& o) {
+    return Obj(this, o);
+  }
+
+};
 
 #endif
diff --git a/src/rgw/services/svc_zone.cc b/src/rgw/services/svc_zone.cc
index 73e099e2340f..fd8f4d3e160e 100644
--- a/src/rgw/services/svc_zone.cc
+++ b/src/rgw/services/svc_zone.cc
@@ -1,5 +1,23 @@
 #include "svc_zone.h"
-#include "rgw_zone.h"
+#include "svc_rados.h"
+
+#include "rgw/rgw_zone.h"
+
+std::map<string, RGWServiceInstance::dependency> RGWSI_Zone::get_deps()
+{
+  RGWServiceInstance::dependency dep = { .name = "rados",
+                                         .conf = "{}" };
+  map<string, RGWServiceInstance::dependency> deps;
+  deps["rados_dep"] = dep;
+  return deps;
+}
+
+int RGWSI_Zone::init(const string& conf, std::map<std::string, RGWServiceInstanceRef>& dep_refs)
+{
+  rados_svc = static_pointer_cast<RGWSI_RADOS>(dep_refs["rados_dep"]);
+  assert(rados_svc);
+  return 0;
+}
 
 RGWZoneParams& RGWSI_Zone::get_zone_params()
 {
@@ -29,7 +47,46 @@ int RGWSI_Zone::get_zonegroup(const string& id, RGWZoneGroup& zonegroup)
 
 RGWRealm& RGWSI_Zone::get_realm()
 {
-  return realm;
+  return *realm;
+}
+
+RGWPeriod& RGWSI_Zone::get_current_period()
+{
+  return *current_period;
+}
+
+const string& RGWSI_Zone::get_current_period_id()
+{
+  return current_period->get_id();
+}
+
+bool RGWSI_Zone::has_zonegroup_api(const std::string& api) const
+{
+  if (!current_period->get_id().empty()) {
+    const auto& zonegroups_by_api = current_period->get_map().zonegroups_by_api;
+    if (zonegroups_by_api.find(api) != zonegroups_by_api.end())
+      return true;
+  } else if (zonegroup->api_name == api) {
+    return true;
+  }
+  return false;
+}
+
+string RGWSI_Zone::gen_host_id() {
+  /* uint64_t needs 16, two '-' separators and a trailing null */
+  const string& zone_name = zone_public_config->name;
+  const string& zonegroup_name = zonegroup->get_name();
+  char charbuf[16 + zone_name.size() + zonegroup_name.size() + 2 + 1];
+  snprintf(charbuf, sizeof(charbuf), "%llx-%s-%s", (unsigned long long)rados_svc->instance_id(), zone_name.c_str(), zonegroup_name.c_str());
+  return string(charbuf);
+}
+
+string RGWSI_Zone::unique_id(uint64_t unique_num)
+{
+  char buf[32];
+  snprintf(buf, sizeof(buf), ".%llu.%llu", (unsigned long long)rados_svc->instance_id(), (unsigned long long)unique_num);
+  string s = zone_params->get_id() + buf;
+  return s;
 }
 
 bool RGWSI_Zone::zone_is_writeable()
@@ -50,3 +107,30 @@ const string& RGWSI_Zone::zone_id()
 {
   return get_zone_params().get_id();
 }
+
+bool RGWSI_Zone::need_to_log_data() const
+{
+  return zone_public_config->log_data;
+}
+
+bool RGWSI_Zone::is_meta_master() const
+{
+  if (!zonegroup->is_master_zonegroup()) {
+    return false;
+  }
+
+  return (zonegroup->master_zone == zone_public_config->id);
+}
+
+bool RGWSI_Zone::need_to_log_metadata() const
+{
+  return is_meta_master() &&
+    (zonegroup->zones.size() > 1 || current_period->is_multi_zonegroups_with_zones());
+}
+
+bool RGWSI_Zone::can_reshard() const
+{
+  return current_period->get_id().empty() ||
+    (zonegroup->zones.size() == 1 && current_period->is_single_zonegroup());
+}
+
diff --git a/src/rgw/services/svc_zone.h b/src/rgw/services/svc_zone.h
index f3c041067950..66fa65b35340 100644
--- a/src/rgw/services/svc_zone.h
+++ b/src/rgw/services/svc_zone.h
@@ -9,46 +9,70 @@ struct RGWZoneGroup;
 struct RGWZone;
 struct RGWZoneParams;
 struct RGWPeriod;
+struct RGWRealm;
+struct RGWZonePlacementInfo;
+
+class RGWSI_RADOS;
 
 class RGWS_Zone : public RGWService
 {
 public:
   RGWS_Zone(CephContext *cct) : RGWService(cct, "zone") {}
 
-  int create_instance(JSONFormattable& conf, RGWServiceInstanceRef *instance);
+  int create_instance(const std::string& conf, RGWServiceInstanceRef *instance);
 };
 
 class RGWSI_Zone : public RGWServiceInstance
 {
+  std::shared_ptr<RGWSI_RADOS> rados_svc;
+
+  std::unique_ptr<RGWRealm> realm;
   std::unique_ptr<RGWZoneGroup> zonegroup;
   std::unique_ptr<RGWZone> zone_public_config; /* external zone params, e.g., entrypoints, log flags, etc. */  
   std::unique_ptr<RGWZoneParams> zone_params; /* internal zone params, e.g., rados pools */
   std::unique_ptr<RGWPeriod> current_period;
   uint32_t zone_short_id{0};
+  bool writeable_zone{false};
 
-  std::vector<std::string> svci_deps = { "sys_obj" };
-  std::vector<std::string> get_deps() {
-    return svci_deps;
-  }
+  std::map<std::string, RGWServiceInstance::dependency> get_deps();
+  int init(const std::string& conf, std::map<std::string, RGWServiceInstanceRef>& dep_refs);
 
-  RGWServiceInstanceRef svc_rados;
 public:
   RGWSI_Zone(RGWService *svc, CephContext *cct): RGWServiceInstance(svc, cct) {}
 
   RGWZoneParams& get_zone_params();
+  RGWPeriod& get_current_period();
   RGWRealm& get_realm();
   RGWZoneGroup& get_zonegroup();
+  int get_zonegroup(const string& id, RGWZoneGroup& zonegroup);
   RGWZone& get_zone();
 
   const string& zone_name();
   const string& zone_id();
   uint32_t get_zone_short_id() const;
 
-  int get_zonegroup(const string& id, RGWZoneGroup& zonegroup);
+  const string& get_current_period_id();
+  bool has_zonegroup_api(const std::string& api) const;
+
+  string gen_host_id();
+  string unique_id(uint64_t unique_num);
 
   bool zone_is_writeable();
   bool zone_syncs_from(RGWZone& target_zone, RGWZone& source_zone);
   bool get_redirect_zone_endpoint(string *endpoint);
+
+  int select_bucket_placement(RGWUserInfo& user_info, const string& zonegroup_id, const string& rule,
+                              string *pselected_rule_name, RGWZonePlacementInfo *rule_info);
+  int select_legacy_bucket_placement(RGWZonePlacementInfo *rule_info);
+  int select_new_bucket_location(RGWUserInfo& user_info, const string& zonegroup_id, const string& rule,
+                                 string *pselected_rule_name, RGWZonePlacementInfo *rule_info);
+  int select_bucket_location_by_rule(const string& location_rule, RGWZonePlacementInfo *rule_info);
+
+  bool is_meta_master() const;
+
+  bool need_to_log_data() const;
+  bool need_to_log_metadata() const;
+  bool can_reshard() const;
 };
 
 #endif