From: Pete Zaitcev <zaitcev@kotori.zaitcev.us>
Date: Wed, 11 Nov 2015 15:26:33 +0000 (-0700)
Subject: rgw: make APIs to work with tenants
X-Git-Tag: v10.0.2~127^2~9
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=f7ca00a6e97b82787becbc15d7e7143e96b3e96d;p=ceph.git

rgw: make APIs to work with tenants

Previous commits introduced the tenant infrastructure such as
rgw_user, but did not allow anyone to access it productively.

Done:
 - radosgw-admin
 - bucket creation, listing, deletion with non-empty tenant
 - COPY
 - ACLs
 - Using colon for S3 with URL addressing

TODO:
 - Fix S3 remapping of DNS so that period turns into colon in buckets,
   possibly using typed endpoint domains, some assuming tenants
 - Have Swift authentication to set tenant into URL, then pick it there
 - Resolve leftover XXX

Signed-off-by: Pete Zaitcev <zaitcev@redhat.com>
---

diff --git a/doc/radosgw/layout.rst b/doc/radosgw/layout.rst
index 0ebb54017902..f8027a09e45b 100644
--- a/doc/radosgw/layout.rst
+++ b/doc/radosgw/layout.rst
@@ -34,7 +34,7 @@ $ radosgw-admin metadata list user
 
 $ radosgw-admin metadata get bucket:<bucket>
 $ radosgw-admin metadata get bucket.instance:<bucket>:<bucket_id>
-$ radosgw-admin metadata get user:<user>   # or set
+$ radosgw-admin metadata get user:<user>   # get or set
 
 user: Holds user information
 bucket: Holds a mapping between bucket name and bucket instance id
@@ -149,14 +149,30 @@ Known pools:
 
 .rgw
   <bucket>
-  .bucket.meta.<bucket>:<marker>
+  .bucket.meta.<bucket>:<marker>   # see put_bucket_instance_info()
+
+  The tenant is used to disambiguate buckets, but not bucket instances.
+  Example:
+
+  .bucket.meta.prodtx:test%25star:default.84099.6
+  .bucket.meta.testcont:default.4126.1
+  .bucket.meta.prodtx:testcont:default.84099.4
+  prodtx/testcont
+  prodtx/test%25star
+  testcont
 
 .rgw.gc
   gc.<N>
 
 .users.uid
   Contains _both_ per-user information (RGWUserInfo) in "<user>" objects
-  and per-user bucket lists in omaps of "<user>.buckets" objects.
+  and per-user lists of buckets in omaps of "<user>.buckets" objects.
+  The "<user>" may contain the tenant if non-empty, for example:
+
+  prodtx$prodt
+  test2.buckets
+  prodtx$prodt.buckets
+  test2
 
 .users.email
   Unimportant
@@ -170,9 +186,14 @@ Known pools:
 
 .rgw.buckets.index
   Objects are named ".dir.<marker>", each contains a bucket index.
+  If the index is sharded, each shard appends the shard index after
+  the marker.
 
 .rgw.buckets
   default.7593.4__shadow_.488urDFerTYXavx4yAd-Op8mxehnvTI_1
   <marker>_<key>
 
 An example of a marker would be "default.16004.1" or "default.7593.4".
+The current format is "<zone>.<instance_id>.<bucket_id>". But once
+generated, a marker is not parsed again, so its format may change
+freely in the future.
diff --git a/src/rgw/rgw_acl.h b/src/rgw/rgw_acl.h
index b7471d234b80..fc2a7ef28de2 100644
--- a/src/rgw/rgw_acl.h
+++ b/src/rgw/rgw_acl.h
@@ -119,7 +119,7 @@ public:
   bool get_id(rgw_user& _id) {
     switch(type.get_type()) {
     case ACL_TYPE_EMAIL_USER:
-      _id = email;
+      _id = email; // implies from_str() that parses the 't:u' syntax
       return true;
     case ACL_TYPE_GROUP:
       return false;
@@ -133,9 +133,11 @@ public:
   ACLGroupTypeEnum get_group() { return group; }
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(3, 3, bl);
+    ENCODE_START(4, 3, bl);
     ::encode(type, bl);
-    ::encode(id, bl);
+    string s;
+    id.to_str(s);
+    ::encode(s, bl);
     string uri;
     ::encode(uri, bl);
     ::encode(email, bl);
@@ -146,9 +148,11 @@ public:
     ENCODE_FINISH(bl);
   }
   void decode(bufferlist::iterator& bl) {
-    DECODE_START_LEGACY_COMPAT_LEN(3, 3, 3, bl);
+    DECODE_START_LEGACY_COMPAT_LEN(4, 3, 3, bl);
     ::decode(type, bl);
-    ::decode(id, bl);
+    string s;
+    ::decode(s, bl);
+    id.from_str(s);
     string uri;
     ::decode(uri, bl);
     ::decode(email, bl);
@@ -168,7 +172,7 @@ public:
 
   ACLGroupTypeEnum uri_to_group(string& uri);
   
-  void set_canon(rgw_user& _id, string& _name, int perm) {
+  void set_canon(const rgw_user& _id, string& _name, int perm) {
     type.set(ACL_TYPE_CANON_USER);
     id = _id;
     name = _name;
@@ -235,7 +239,7 @@ public:
 
   multimap<string, ACLGrant>& get_grant_map() { return grant_map; }
 
-  void create_default(rgw_user& id, string name) {
+  void create_default(const rgw_user& id, string name) {
     acl_user_map.clear();
     acl_group_map.clear();
 
@@ -273,7 +277,7 @@ public:
   }
   void dump(Formatter *f) const;
   static void generate_test_instances(list<ACLOwner*>& o);
-  void set_id(rgw_user& _id) { id = _id; }
+  void set_id(const rgw_user& _id) { id = _id; }
   void set_name(string& name) { display_name = name; }
 
   rgw_user& get_id() { return id; }
@@ -327,7 +331,7 @@ public:
     return owner;
   }
 
-  void create_default(rgw_user& id, string& name) {
+  void create_default(const rgw_user& id, string& name) {
     acl.create_default(id, name);
     owner.set_id(id);
     owner.set_name(name);
diff --git a/src/rgw/rgw_acl_s3.cc b/src/rgw/rgw_acl_s3.cc
index aecb7a3e66a9..d9130e0d94a3 100644
--- a/src/rgw/rgw_acl_s3.cc
+++ b/src/rgw/rgw_acl_s3.cc
@@ -516,7 +516,7 @@ int RGWAccessControlPolicy_S3::rebuild(RGWRados *store, ACLOwner *owner, RGWAcce
           ldout(cct, 0) << "ERROR: src_grant.get_id() failed" << dendl;
           return -EINVAL;
         }
-        email = u.id;
+        email = u.id;  // XXX none of that .to_str() stuff here, but why?
         ldout(cct, 10) << "grant user email=" << email << dendl;
         if (rgw_get_user_info_by_email(store, email, grant_user) < 0) {
           ldout(cct, 10) << "grant user email not found or other error" << dendl;
diff --git a/src/rgw/rgw_acl_swift.h b/src/rgw/rgw_acl_swift.h
index 45900cba1529..cbadfaa6884a 100644
--- a/src/rgw/rgw_acl_swift.h
+++ b/src/rgw/rgw_acl_swift.h
@@ -4,10 +4,9 @@
 #ifndef CEPH_RGW_ACL_SWIFT_H
 #define CEPH_RGW_ACL_SWIFT_H
 
+#include <list>
 #include <map>
 #include <string>
-#include <iostream>
-#include <list>
 #include <include/types.h>
 
 #include "rgw_acl.h"
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index a68a20b3d7ca..4eee6edb0272 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -628,14 +628,14 @@ public:
   }
 };
 
-static int init_bucket(const string& tenant, const string& bucket_name, const string& bucket_id,
+static int init_bucket(const string& tenant_name, const string& bucket_name, const string& bucket_id,
                        RGWBucketInfo& bucket_info, rgw_bucket& bucket)
 {
   if (!bucket_name.empty()) {
     RGWObjectCtx obj_ctx(store);
     int r;
     if (bucket_id.empty()) {
-      r = store->get_bucket_info(obj_ctx, tenant, bucket_name, bucket_info, NULL);
+      r = store->get_bucket_info(obj_ctx, tenant_name, bucket_name, bucket_info, NULL);
     } else {
       string bucket_instance_id = bucket_name + ":" + bucket_id;
       r = store->get_bucket_instance_info(obj_ctx, bucket_instance_id, bucket_info, NULL, NULL);
@@ -809,13 +809,15 @@ void set_quota_info(RGWQuotaInfo& quota, int opt_cmd, int64_t max_size, int64_t
   }
 }
 
-int set_bucket_quota(RGWRados *store, int opt_cmd, string& bucket_name, int64_t max_size, int64_t max_objects,
+int set_bucket_quota(RGWRados *store, int opt_cmd,
+                     const string& tenant_name, const string& bucket_name,
+                     int64_t max_size, int64_t max_objects,
                      bool have_max_size, bool have_max_objects)
 {
   RGWBucketInfo bucket_info;
   map<string, bufferlist> attrs;
   RGWObjectCtx obj_ctx(store);
-  int r = store->get_bucket_info(obj_ctx, bucket_name, bucket_info, NULL, &attrs);
+  int r = store->get_bucket_info(obj_ctx, tenant_name, bucket_name, bucket_info, NULL, &attrs);
   if (r < 0) {
     cerr << "could not get bucket info for bucket=" << bucket_name << ": " << cpp_strerror(-r) << std::endl;
     return -r;
@@ -1003,7 +1005,8 @@ int check_obj_tail_locator_underscore(RGWBucketInfo& bucket_info, rgw_obj& obj,
   return 0;
 }
 
-int do_check_object_locator(const string& bucket_name, bool fix, bool remove_bad, Formatter *f)
+int do_check_object_locator(const string& tenant_name, const string& bucket_name,
+                            bool fix, bool remove_bad, Formatter *f)
 {
   if (remove_bad && !fix) {
     cerr << "ERROR: can't have remove_bad specified without fix" << std::endl;
@@ -1016,7 +1019,7 @@ int do_check_object_locator(const string& bucket_name, bool fix, bool remove_bad
 
   f->open_object_section("bucket");
   f->dump_string("bucket", bucket_name);
-  int ret = init_bucket(bucket_name, bucket_id, bucket_info, bucket);
+  int ret = init_bucket(tenant_name, bucket_name, bucket_id, bucket_info, bucket);
   if (ret < 0) {
     cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
     return ret;
@@ -1077,7 +1080,6 @@ int do_check_object_locator(const string& bucket_name, bool fix, bool remove_bad
   return 0;
 }
 
-
 int main(int argc, char **argv) 
 {
   vector<const char*> args;
@@ -1179,12 +1181,7 @@ int main(int argc, char **argv)
       usage();
       return 0;
     } else if (ceph_argparse_witharg(args, i, &val, "-i", "--uid", (char*)NULL)) {
-      string s = val;
-      if (!tenant.empty()) {
-        s = tenant + ":" + val;
-      }
-      user_id.from_str(s);
-      tenant = user_id.tenant;
+      user_id.from_str(val);
     } else if (ceph_argparse_witharg(args, i, &val, "--tenant", (char*)NULL)) {
       tenant = val;
     } else if (ceph_argparse_witharg(args, i, &val, "--access-key", (char*)NULL)) {
@@ -1395,6 +1392,15 @@ int main(int argc, char **argv)
       ++i;
     }
   }
+  if (tenant.empty()) {
+    tenant = user_id.tenant;
+  } else {
+    if (user_id.empty()) {
+      cerr << "ERROR: --tennant is set, but there's no user ID" << std::endl;
+      return EINVAL;
+    }
+    user_id.tenant = tenant;
+  }
 
   if (args.empty()) {
     return usage();
@@ -1768,7 +1774,6 @@ int main(int argc, char **argv)
 
   /* populate bucket operation */
   bucket_op.set_bucket_name(bucket_name);
-  bucket_op.set_tenant(tenant);
   bucket_op.set_object(object);
   bucket_op.set_check_objects(check_objects);
   bucket_op.set_delete_children(delete_child_objects);
@@ -1901,7 +1906,7 @@ int main(int argc, char **argv)
       RGWBucketAdminOp::info(store, bucket_op, f);
     } else {
       RGWBucketInfo bucket_info;
-      int ret = init_bucket(bucket_name, bucket_id, bucket_info, bucket);
+      int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
       if (ret < 0) {
         cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
         return -ret;
@@ -2223,7 +2228,7 @@ next:
       return EINVAL;
     }
     RGWBucketInfo bucket_info;
-    int ret = init_bucket(bucket_name, tenant, bucket_id, bucket_info, bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -2270,7 +2275,7 @@ next:
 
   if (opt_cmd == OPT_BI_GET) {
     RGWBucketInfo bucket_info;
-    int ret = init_bucket(bucket_name, bucket_id, bucket_info, bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -2294,7 +2299,7 @@ next:
 
   if (opt_cmd == OPT_BI_PUT) {
     RGWBucketInfo bucket_info;
-    int ret = init_bucket(bucket_name, bucket_id, bucket_info, bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -2319,7 +2324,7 @@ next:
 
   if (opt_cmd == OPT_BI_LIST) {
     RGWBucketInfo bucket_info;
-    int ret = init_bucket(bucket_name, bucket_id, bucket_info, bucket);
+    int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
     if (ret < 0) {
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
@@ -2596,7 +2601,7 @@ next:
         cerr << "ERROR: need to specify bucket name" << std::endl;
         return EINVAL;
       }
-      do_check_object_locator(bucket_name, fix, remove_bad, formatter);
+      do_check_object_locator(tenant, bucket_name, fix, remove_bad, formatter);
     } else {
       RGWBucketAdminOp::check_index(store, bucket_op, f);
     }
@@ -2706,7 +2711,7 @@ next:
   if (opt_cmd == OPT_USER_STATS) {
     if (sync_stats) {
       if (!bucket_name.empty()) {
-        int ret = rgw_bucket_sync_user_stats(store, bucket_name);
+        int ret = rgw_bucket_sync_user_stats(store, tenant, bucket_name);
         if (ret < 0) {
           cerr << "ERROR: could not sync bucket stats: " << cpp_strerror(-ret) << std::endl;
           return -ret;
@@ -3243,7 +3248,8 @@ next:
         cerr << "ERROR: invalid quota scope specification." << std::endl;
         return EINVAL;
       }
-      set_bucket_quota(store, opt_cmd, bucket_name, max_size, max_objects, have_max_size, have_max_objects);
+      set_bucket_quota(store, opt_cmd, tenant, bucket_name,
+                       max_size, max_objects, have_max_size, have_max_objects);
     } else if (!user_id.empty()) {
       if (quota_scope == "bucket") {
         set_user_bucket_quota(opt_cmd, user, user_op, max_size, max_objects, have_max_size, have_max_objects);
diff --git a/src/rgw/rgw_basic_types.h b/src/rgw/rgw_basic_types.h
index 0685c0eeee7b..4a9c42cbe103 100644
--- a/src/rgw/rgw_basic_types.h
+++ b/src/rgw/rgw_basic_types.h
@@ -29,7 +29,7 @@ struct rgw_user {
 
   void to_str(std::string& str) const {
     if (!tenant.empty()) {
-      str = tenant + ':' + id;
+      str = tenant + '$' + id;
     } else {
       str = id;
     }
@@ -51,7 +51,7 @@ struct rgw_user {
   }
 
   void from_str(const std::string& str) {
-    ssize_t pos = str.find(':');
+    ssize_t pos = str.find('$');
     if (pos >= 0) {
       tenant = str.substr(0, pos);
       id = str.substr(pos + 1);
diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index fbe630233e59..304448c79524 100644
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -38,6 +38,32 @@ void rgw_get_buckets_obj(const rgw_user& user_id, string& buckets_obj_id)
   buckets_obj_id += RGW_BUCKETS_OBJ_SUFFIX;
 }
 
+/*
+ * Note that this is not a reversal of parse_bucket(). That one deals
+ * with the syntax we need in metadata and such. This one deals with
+ * the representation in RADOS pools. We chose '/' because it's not
+ * acceptable in bucket names and thus qualified buckets cannot conflict
+ * with the legacy or S3 buckets.
+ */
+void rgw_make_bucket_entry_name(const string& tenant_name, const string& bucket_name, string& bucket_entry) {
+  if (tenant_name.empty()) {
+    bucket_entry = bucket_name;
+  } else {
+    bucket_entry = tenant_name + "/" + bucket_name;
+  }
+}
+
+/*
+ * Tenants are separated from buckets in URLs by a colon in S3.
+ * This function is not to be used on Swift URLs, not even for COPY arguments.
+ */
+void rgw_parse_url_bucket(const string &bucket,
+                          string &tenant_name, string &bucket_name) {
+  int pos = bucket.find(':');
+  tenant_name = bucket.substr(0, pos);
+  bucket_name = bucket.substr(pos + 1);
+}
+
 /**
  * Get all the buckets owned by a user and fill up an RGWUserBuckets with them.
  * Returns: 0 on success, -ERR# on failure.
@@ -104,11 +130,11 @@ int rgw_bucket_sync_user_stats(RGWRados *store, const rgw_user& user_id, rgw_buc
   return store->cls_user_sync_bucket_stats(obj, bucket);
 }
 
-int rgw_bucket_sync_user_stats(RGWRados *store, const string& bucket_name)
+int rgw_bucket_sync_user_stats(RGWRados *store, const string& tenant_name, const string& bucket_name)
 {
   RGWBucketInfo bucket_info;
   RGWObjectCtx obj_ctx(store);
-  int ret = store->get_bucket_info(obj_ctx, bucket_name, bucket_info, NULL);
+  int ret = store->get_bucket_info(obj_ctx, tenant_name, bucket_name, bucket_info, NULL);
   if (ret < 0) {
     ldout(store->ctx(), 0) << "ERROR: could not fetch bucket info: ret=" << ret << dendl;
     return ret;
@@ -126,6 +152,8 @@ int rgw_bucket_sync_user_stats(RGWRados *store, const string& bucket_name)
 int rgw_link_bucket(RGWRados *store, const rgw_user& user_id, rgw_bucket& bucket, time_t creation_time, bool update_entrypoint)
 {
   int ret;
+  // XXX Actually, should we use user_id.tenant when creating bucket?
+  string& tenant_name = bucket.tenant;
   string& bucket_name = bucket.name;
 
   cls_user_bucket_entry new_bucket;
@@ -144,7 +172,7 @@ int rgw_link_bucket(RGWRados *store, const rgw_user& user_id, rgw_bucket& bucket
   RGWObjectCtx obj_ctx(store);
 
   if (update_entrypoint) {
-    ret = store->get_bucket_entrypoint_info(obj_ctx, bucket_name, ep, &ot, NULL, &attrs);
+    ret = store->get_bucket_entrypoint_info(obj_ctx, tenant_name, bucket_name, ep, &ot, NULL, &attrs);
     if (ret < 0 && ret != -ENOENT) {
       ldout(store->ctx(), 0) << "ERROR: store->get_bucket_entrypoint_info() returned " << ret << dendl;
     } else if (ret >= 0 && ep.linked && ep.owner != user_id) {
@@ -169,20 +197,20 @@ int rgw_link_bucket(RGWRados *store, const rgw_user& user_id, rgw_bucket& bucket
 
   ep.linked = true;
   ep.owner = user_id;
-  ret = store->put_bucket_entrypoint_info(bucket_name, ep, false, ot, 0, &attrs);
+  ret = store->put_bucket_entrypoint_info(tenant_name, bucket_name, ep, false, ot, 0, &attrs);
   if (ret < 0)
     goto done_err;
 
   return 0;
 done_err:
-  int r = rgw_unlink_bucket(store, user_id, bucket.name);
+  int r = rgw_unlink_bucket(store, user_id, bucket.tenant, bucket.name);
   if (r < 0) {
     ldout(store->ctx(), 0) << "ERROR: failed unlinking bucket on error cleanup: " << cpp_strerror(-r) << dendl;
   }
   return ret;
 }
 
-int rgw_unlink_bucket(RGWRados *store, const rgw_user& user_id, const string& bucket_name, bool update_entrypoint)
+int rgw_unlink_bucket(RGWRados *store, const rgw_user& user_id, const string& tenant_name, const string& bucket_name, bool update_entrypoint)
 {
   int ret;
 
@@ -207,7 +235,7 @@ int rgw_unlink_bucket(RGWRados *store, const rgw_user& user_id, const string& bu
   RGWObjVersionTracker ot;
   map<string, bufferlist> attrs;
   RGWObjectCtx obj_ctx(store);
-  ret = store->get_bucket_entrypoint_info(obj_ctx, user_id.tenant, bucket_name, ep, &ot, NULL, &attrs);
+  ret = store->get_bucket_entrypoint_info(obj_ctx, tenant_name, bucket_name, ep, &ot, NULL, &attrs);
   if (ret == -ENOENT)
     return 0;
   if (ret < 0)
@@ -222,7 +250,7 @@ int rgw_unlink_bucket(RGWRados *store, const rgw_user& user_id, const string& bu
   }
 
   ep.linked = false;
-  ret = store->put_bucket_entrypoint_info(bucket_name, ep, false, ot, 0, &attrs);
+  ret = store->put_bucket_entrypoint_info(tenant_name, bucket_name, ep, false, ot, 0, &attrs);
   if (ret < 0)
     return ret;
 
@@ -281,7 +309,7 @@ int rgw_bucket_set_attrs(RGWRados *store, RGWBucketInfo& bucket_info,
   if (!bucket_info.has_instance_obj) {
     /* an old bucket object, need to convert it */
     RGWObjectCtx obj_ctx(store);
-    int ret = store->convert_old_bucket_info(obj_ctx, bucket.name);
+    int ret = store->convert_old_bucket_info(obj_ctx, bucket.tenant, bucket.name);
     if (ret < 0) {
       ldout(store->ctx(), 0) << "ERROR: failed converting old bucket info: " << ret << dendl;
       return ret;
@@ -412,7 +440,7 @@ int rgw_remove_bucket(RGWRados *store, rgw_bucket& bucket, bool delete_children)
 
   obj.bucket = bucket;
 
-  ret = store->get_bucket_info(obj_ctx, bucket.name, info, NULL);
+  ret = store->get_bucket_info(obj_ctx, bucket.tenant, bucket.name, info, NULL);
   if (ret < 0)
     return ret;
 
@@ -451,7 +479,7 @@ int rgw_remove_bucket(RGWRados *store, rgw_bucket& bucket, bool delete_children)
     return ret;
   }
 
-  ret = rgw_unlink_bucket(store, info.owner, bucket.name);
+  ret = rgw_unlink_bucket(store, info.owner, bucket.tenant, bucket.name);
   if (ret < 0) {
     lderr(store->ctx()) << "ERROR: unable to remove user bucket information" << dendl;
   }
@@ -459,9 +487,15 @@ int rgw_remove_bucket(RGWRados *store, rgw_bucket& bucket, bool delete_children)
   return ret;
 }
 
-int rgw_bucket_delete_bucket_obj(RGWRados *store, string& bucket_name, RGWObjVersionTracker& objv_tracker)
+int rgw_bucket_delete_bucket_obj(RGWRados *store,
+                                 const string& tenant_name,
+                                 const string& bucket_name,
+                                 RGWObjVersionTracker& objv_tracker)
 {
-  return store->meta_mgr->remove_entry(bucket_meta_handler, bucket_name, &objv_tracker);
+  string key;
+
+  rgw_make_bucket_entry_name(tenant_name, bucket_name, key);
+  return store->meta_mgr->remove_entry(bucket_meta_handler, key, &objv_tracker);
 }
 
 static void set_err_msg(std::string *sink, std::string msg)
@@ -478,8 +512,8 @@ int RGWBucket::init(RGWRados *storage, RGWBucketAdminOpState& op_state)
   store = storage;
 
   rgw_user user_id = op_state.get_user_id();
+  tenant = user_id.tenant;
   bucket_name = op_state.get_bucket_name();
-  tenant = op_state.get_tenant();
   RGWUserBuckets user_buckets;
   RGWObjectCtx obj_ctx(store);
 
@@ -553,7 +587,7 @@ int RGWBucket::link(RGWBucketAdminOpState& op_state, std::string *err_msg)
       return -EIO;
     }
 
-    r = rgw_unlink_bucket(store, owner.get_id(), bucket.name);
+    r = rgw_unlink_bucket(store, owner.get_id(), bucket.tenant, bucket.name);
     if (r < 0) {
       set_err_msg(err_msg, "could not unlink policy from user " + owner.get_id().to_str());
       return r;
@@ -597,7 +631,7 @@ int RGWBucket::unlink(RGWBucketAdminOpState& op_state, std::string *err_msg)
     return -EINVAL;
   }
 
-  int r = rgw_unlink_bucket(store, user_info.user_id, bucket.name);
+  int r = rgw_unlink_bucket(store, user_info.user_id, bucket.tenant, bucket.name);
   if (r < 0) {
     set_err_msg(err_msg, "error unlinking bucket" + cpp_strerror(-r));
   }
@@ -849,7 +883,7 @@ int RGWBucket::get_policy(RGWBucketAdminOpState& op_state, ostream& o)
 
   RGWBucketInfo bucket_info;
   map<string, bufferlist> attrs;
-  int ret = store->get_bucket_info(obj_ctx, bucket.name, bucket_info, NULL, &attrs);
+  int ret = store->get_bucket_info(obj_ctx, bucket.tenant, bucket.name, bucket_info, NULL, &attrs);
   if (ret < 0) {
     return ret;
   }
@@ -880,7 +914,7 @@ int RGWBucket::get_policy(RGWBucketAdminOpState& op_state, ostream& o)
 int RGWBucketAdminOp::get_policy(RGWRados *store, RGWBucketAdminOpState& op_state,
                   ostream& os)
 {
-   RGWBucket bucket;
+  RGWBucket bucket;
 
   int ret = bucket.init(store, op_state);
   if (ret < 0)
@@ -1003,7 +1037,7 @@ int RGWBucketAdminOp::remove_object(RGWRados *store, RGWBucketAdminOpState& op_s
   return bucket.remove_object(op_state);
 }
 
-static int bucket_stats(RGWRados *store, std::string& tenant, std::string&  bucket_name, Formatter *formatter)
+static int bucket_stats(RGWRados *store, const std::string& tenant_name, std::string&  bucket_name, Formatter *formatter)
 {
   RGWBucketInfo bucket_info;
   rgw_bucket bucket;
@@ -1011,7 +1045,7 @@ static int bucket_stats(RGWRados *store, std::string& tenant, std::string&  buck
 
   time_t mtime;
   RGWObjectCtx obj_ctx(store);
-  int r = store->get_bucket_info(obj_ctx, tenant, bucket_name, bucket_info, &mtime);
+  int r = store->get_bucket_info(obj_ctx, tenant_name, bucket_name, bucket_info, &mtime);
   if (r < 0)
     return r;
 
@@ -1068,7 +1102,7 @@ int RGWBucketAdminOp::info(RGWRados *store, RGWBucketAdminOpState& op_state,
   size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk;
 
   bool show_stats = op_state.will_fetch_stats();
-  string tenant = op_state.get_tenant();
+  rgw_user user_id = op_state.get_user_id();
   if (op_state.is_user_op()) {
     formatter->open_array_section("buckets");
 
@@ -1077,7 +1111,7 @@ int RGWBucketAdminOp::info(RGWRados *store, RGWBucketAdminOpState& op_state,
     bool done;
 
     do {
-      ret = rgw_read_user_buckets(store, op_state.get_user_id(), buckets, marker, max_entries, false);
+      ret = rgw_read_user_buckets(store, user_id, buckets, marker, max_entries, false);
       if (ret < 0)
         return ret;
 
@@ -1087,7 +1121,7 @@ int RGWBucketAdminOp::info(RGWRados *store, RGWBucketAdminOpState& op_state,
       for (iter = m.begin(); iter != m.end(); ++iter) {
         std::string  obj_name = iter->first;
         if (show_stats)
-          bucket_stats(store, tenant, obj_name, formatter);
+          bucket_stats(store, user_id.tenant, obj_name, formatter);
         else
           formatter->dump_string("bucket", obj_name);
 
@@ -1100,7 +1134,7 @@ int RGWBucketAdminOp::info(RGWRados *store, RGWBucketAdminOpState& op_state,
 
     formatter->close_section();
   } else if (!bucket_name.empty()) {
-    bucket_stats(store, tenant, bucket_name, formatter);
+    bucket_stats(store, user_id.tenant, bucket_name, formatter);
   } else {
     RGWAccessHandle handle;
 
@@ -1110,7 +1144,7 @@ int RGWBucketAdminOp::info(RGWRados *store, RGWBucketAdminOpState& op_state,
       while (store->list_buckets_next(obj, &handle) >= 0) {
 	formatter->dump_string("bucket", obj.key.name);
         if (show_stats)
-          bucket_stats(store, tenant, obj.key.name, formatter);
+          bucket_stats(store, user_id.tenant, obj.key.name, formatter);
       }
     }
 
@@ -1540,7 +1574,9 @@ public:
     map<string, bufferlist> attrs;
     RGWObjectCtx obj_ctx(store);
 
-    int ret = store->get_bucket_entrypoint_info(obj_ctx, entry, be, &ot, &mtime, &attrs);
+    string tenant_name, bucket_name;
+    parse_bucket(entry, tenant_name, bucket_name);
+    int ret = store->get_bucket_entrypoint_info(obj_ctx, tenant_name, bucket_name, be, &ot, &mtime, &attrs);
     if (ret < 0)
       return ret;
 
@@ -1562,7 +1598,9 @@ public:
     RGWObjVersionTracker old_ot;
     RGWObjectCtx obj_ctx(store);
 
-    int ret = store->get_bucket_entrypoint_info(obj_ctx, entry, old_be, &old_ot, &orig_mtime, &attrs);
+    string tenant_name, bucket_name;
+    parse_bucket(entry, tenant_name, bucket_name);
+    int ret = store->get_bucket_entrypoint_info(obj_ctx, tenant_name, bucket_name, old_be, &old_ot, &orig_mtime, &attrs);
     if (ret < 0 && ret != -ENOENT)
       return ret;
 
@@ -1575,7 +1613,7 @@ public:
 
     objv_tracker.read_version = old_ot.read_version; /* maintain the obj version we just read */
 
-    ret = store->put_bucket_entrypoint_info(entry, be, false, objv_tracker, mtime, &attrs);
+    ret = store->put_bucket_entrypoint_info(tenant_name, bucket_name, be, false, objv_tracker, mtime, &attrs);
     if (ret < 0)
       return ret;
 
@@ -1583,7 +1621,7 @@ public:
     if (be.linked) {
       ret = rgw_link_bucket(store, be.owner, be.bucket, be.creation_time, false);
     } else {
-      ret = rgw_unlink_bucket(store, be.owner, be.bucket.name, false);
+      ret = rgw_unlink_bucket(store, be.owner, be.bucket.tenant, be.bucket.name, false);
     }
 
     return ret;
@@ -1598,7 +1636,9 @@ public:
     RGWBucketEntryPoint be;
     RGWObjectCtx obj_ctx(store);
 
-    int ret = store->get_bucket_entrypoint_info(obj_ctx, entry, be, &objv_tracker, NULL, NULL);
+    string tenant_name, bucket_name;
+    parse_bucket(entry, tenant_name, bucket_name);
+    int ret = store->get_bucket_entrypoint_info(obj_ctx, tenant_name, bucket_name, be, &objv_tracker, NULL, NULL);
     if (ret < 0)
       return ret;
 
@@ -1607,12 +1647,12 @@ public:
      * it immediately and don't want to invalidate our cached objv_version or the bucket obj removal
      * will incorrectly fail.
      */
-    ret = rgw_unlink_bucket(store, be.owner, entry, false);
+    ret = rgw_unlink_bucket(store, be.owner, tenant_name, bucket_name, false);
     if (ret < 0) {
       lderr(store->ctx()) << "could not unlink bucket=" << entry << " owner=" << be.owner << dendl;
     }
 
-    ret = rgw_bucket_delete_bucket_obj(store, entry, objv_tracker);
+    ret = rgw_bucket_delete_bucket_obj(store, tenant_name, bucket_name, objv_tracker);
     if (ret < 0) {
       lderr(store->ctx()) << "could not delete bucket=" << entry << dendl;
     }
@@ -1698,7 +1738,7 @@ public:
     return 0;
   }
 
-  int put(RGWRados *store, string& oid, RGWObjVersionTracker& objv_tracker,
+  int put(RGWRados *store, string& entry, RGWObjVersionTracker& objv_tracker,
           time_t mtime, JSONObj *obj, sync_type_t sync_type) {
     RGWBucketCompleteInfo bci, old_bci;
     decode_json_obj(bci, obj);
@@ -1706,16 +1746,22 @@ public:
     time_t orig_mtime;
     RGWObjectCtx obj_ctx(store);
 
-    int ret = store->get_bucket_instance_info(obj_ctx, oid, old_bci.info, &orig_mtime, &old_bci.attrs);
+    int ret = store->get_bucket_instance_info(obj_ctx, entry, old_bci.info,
+            &orig_mtime, &old_bci.attrs);
     bool exists = (ret != -ENOENT);
     if (ret < 0 && exists)
       return ret;
 
-
     if (!exists || old_bci.info.bucket.bucket_id != bci.info.bucket.bucket_id) {
       /* a new bucket, we need to select a new bucket placement for it */
+      // XXX not sure if this is correct -- stolen from Radoslaw; what about get()?
+      string tenant_name;
+      string bucket_name;
+      parse_bucket(entry, tenant_name, bucket_name);
+
       rgw_bucket bucket;
-      ret = store->set_bucket_location_by_rule(bci.info.placement_rule, oid, bucket);
+      ret = store->set_bucket_location_by_rule(bci.info.placement_rule,
+                                           tenant_name, bucket_name, bucket);
       if (ret < 0) {
         ldout(store->ctx(), 0) << "ERROR: select_bucket_placement() returned " << ret << dendl;
         return ret;
diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h
index 3d274c55d083..e818f0a6b5c9 100644
--- a/src/rgw/rgw_bucket.h
+++ b/src/rgw/rgw_bucket.h
@@ -36,10 +36,19 @@ extern int rgw_bucket_parse_bucket_instance(const string& bucket_instance, strin
 
 extern int rgw_bucket_instance_remove_entry(RGWRados *store, string& entry, RGWObjVersionTracker *objv_tracker);
 
-extern int rgw_bucket_delete_bucket_obj(RGWRados *store, string& bucket_name, RGWObjVersionTracker& objv_tracker);
+extern int rgw_bucket_delete_bucket_obj(RGWRados *store,
+                                        const string& tenant_name,
+                                        const string& bucket_name,
+                                        RGWObjVersionTracker& objv_tracker);
 
 extern int rgw_bucket_sync_user_stats(RGWRados *store, const rgw_user& user_id, rgw_bucket& bucket);
-extern int rgw_bucket_sync_user_stats(RGWRados *store, const string& bucket_name);
+extern int rgw_bucket_sync_user_stats(RGWRados *store, const string& tenant_name, const string& bucket_name);
+
+extern void rgw_make_bucket_entry_name(const string& tenant_name,
+                                       const string& bucket_name,
+                                       string& bucket_entry);
+extern void rgw_parse_url_bucket(const string &bucket,
+                                 string &tenant_name, string &bucket_name);
 
 /**
  * Store a list of the user's buckets, with associated functinos.
@@ -113,7 +122,8 @@ extern int rgw_read_user_buckets(RGWRados *store,
                                  uint64_t default_amount = 1000);
 
 extern int rgw_link_bucket(RGWRados *store, const rgw_user& user_id, rgw_bucket& bucket, time_t creation_time, bool update_entrypoint = true);
-extern int rgw_unlink_bucket(RGWRados *store, const rgw_user& user_id, const string& bucket_name, bool update_entrypoint = true);
+extern int rgw_unlink_bucket(RGWRados *store, const rgw_user& user_id,
+                             const string& tenant_name, const string& bucket_name, bool update_entrypoint = true);
 
 extern int rgw_remove_object(RGWRados *store, RGWBucketInfo& bucket_info, rgw_bucket& bucket, rgw_obj_key& key);
 extern int rgw_remove_bucket(RGWRados *store, rgw_bucket& bucket, bool delete_children);
@@ -127,7 +137,6 @@ extern void check_bad_user_bucket_mapping(RGWRados *store, const rgw_user& user_
 
 struct RGWBucketAdminOpState {
   rgw_user uid;
-  std::string tenant;
   std::string display_name;
   std::string bucket_name;
   std::string bucket_id;
@@ -150,10 +159,6 @@ struct RGWBucketAdminOpState {
   void set_user_id(rgw_user& user_id) {
     if (!user_id.empty())
       uid = user_id;
-    tenant = uid.tenant;
-  }
-  void set_tenant(string& t) {
-    tenant = t;
   }
   void set_bucket_name(std::string& bucket_str) {
     bucket_name = bucket_str; 
@@ -166,7 +171,6 @@ struct RGWBucketAdminOpState {
   std::string& get_user_display_name() { return display_name; }
   std::string& get_bucket_name() { return bucket_name; }
   std::string& get_object_name() { return object_name; }
-  std::string& get_tenant() { return tenant; };
 
   rgw_bucket& get_bucket() { return bucket; }
   void set_bucket(rgw_bucket& _bucket) {
diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc
index 9f59459c109b..bb8ecf384e0e 100644
--- a/src/rgw/rgw_common.cc
+++ b/src/rgw/rgw_common.cc
@@ -171,7 +171,6 @@ req_state::req_state(CephContext *_cct, class RGWEnv *e) : cct(_cct), cio(NULL),
   bucket_exists = false;
   has_bad_meta = false;
   length = NULL;
-  copy_source = NULL;
   http_auth = NULL;
   local_source = false;
 
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 6bc4ac96f9cf..bff070a06ae4 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -28,6 +28,7 @@
 #include "include/types.h"
 #include "include/utime.h"
 #include "rgw_acl.h"
+#include "rgw_basic_types.h"
 #include "rgw_cors.h"
 #include "rgw_quota.h"
 #include "rgw_string.h"
@@ -35,7 +36,6 @@
 #include "cls/user/cls_user_types.h"
 #include "cls/rgw/cls_rgw_types.h"
 #include "include/rados/librados.hpp"
-#include "rgw_basic_types.h"
 
 using namespace std;
 
@@ -506,7 +506,7 @@ struct RGWUserInfo
      ::encode(bucket_quota, bl);
      ::encode(temp_url_keys, bl);
      ::encode(user_quota, bl);
-     ::encode(user_id, bl);
+     ::encode(user_id.tenant, bl);
      ENCODE_FINISH(bl);
   }
   void decode(bufferlist::iterator& bl) {
@@ -525,13 +525,11 @@ struct RGWUserInfo
     }
     ::decode(display_name, bl);
     ::decode(user_email, bl);
+    /* We populate swift_keys map later nowadays, but we have to decode. */
     string swift_name;
     string swift_key;
     if (struct_v >= 3) ::decode(swift_name, bl);
     if (struct_v >= 4) ::decode(swift_key, bl);
-    if (struct_v < 13) {
-      user_id.tenant.clear();
-    }
     if (struct_v >= 5)
       ::decode(user_id.id, bl);
     else
@@ -575,7 +573,9 @@ struct RGWUserInfo
       ::decode(user_quota, bl);
     }
     if (struct_v >= 17) {
-      ::decode(user_id, bl);
+      ::decode(user_id.tenant, bl);
+    } else {
+      user_id.tenant.clear();
     }
     DECODE_FINISH(bl);
   }
@@ -613,8 +613,8 @@ struct rgw_bucket {
     data_pool = index_pool = n;
     marker = "";
   }
-  rgw_bucket(const char *n, const char *dp, const char *ip, const char *m, const char *id, const char *h) :
-    name(n), data_pool(dp), index_pool(ip), marker(m), bucket_id(id) {}
+  rgw_bucket(const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id, const char *h) :
+    tenant(t), name(n), data_pool(dp), index_pool(ip), marker(m), bucket_id(id) {}
 
   void convert(cls_user_bucket *b) {
     b->name = name;
@@ -625,8 +625,6 @@ struct rgw_bucket {
     b->bucket_id = bucket_id;
   }
 
-  rgw_bucket(const char *t, const char *n, const char *p, const char *m, const char *id) :
-    tenant(t), name(n), pool(p), marker(m), bucket_id(id) {}
   void encode(bufferlist& bl) const {
      ENCODE_START(8, 3, bl);
     ::encode(name, bl);
@@ -815,7 +813,7 @@ struct RGWBucketInfo
      ::encode(num_shards, bl);
      ::encode(bucket_index_shard_hash_type, bl);
      ::encode(requester_pays, bl);
-     ::encode(owner, bl);
+     ::encode(owner.tenant, bl);
      ENCODE_FINISH(bl);
   }
   void decode(bufferlist::iterator& bl) {
@@ -848,7 +846,7 @@ struct RGWBucketInfo
      if (struct_v >= 12)
        ::decode(requester_pays, bl);
      if (struct_v >= 13)
-       ::decode(owner, bl);
+       ::decode(owner.tenant, bl);
      DECODE_FINISH(bl);
   }
   void dump(Formatter *f) const;
@@ -1049,9 +1047,13 @@ struct req_state {
    uint32_t perm_mask;
    utime_t header_time;
 
+   /* Set once when req_state is initialized and not violated thereafter */
+   string bucket_tenant;
+   string bucket_name;
+
    rgw_bucket bucket;
-   string bucket_name_str;
    rgw_obj_key object;
+   string src_tenant_name;
    string src_bucket_name;
    rgw_obj_key src_object;
    ACLOwner bucket_owner;
@@ -1067,7 +1069,6 @@ struct req_state {
    bool has_bad_meta;
 
    RGWUserInfo user; 
-   string tenant;
    RGWAccessControlPolicy *bucket_acl;
    RGWAccessControlPolicy *object_acl;
 
@@ -1075,7 +1076,6 @@ struct req_state {
 
    string canned_acl;
    bool has_acl_header;
-   const char *copy_source;
    const char *http_auth;
    bool local_source; /* source is local */
 
diff --git a/src/rgw/rgw_log.cc b/src/rgw/rgw_log.cc
index b897fc61e7e2..0e565cb2c858 100644
--- a/src/rgw/rgw_log.cc
+++ b/src/rgw/rgw_log.cc
@@ -7,6 +7,7 @@
 #include "common/OutputDataSocket.h"
 #include "common/Formatter.h"
 
+#include "rgw_bucket.h"
 #include "rgw_log.h"
 #include "rgw_acl.h"
 #include "rgw_rados.h"
@@ -177,7 +178,7 @@ static void log_usage(struct req_state *s, const string& op_name)
 
   rgw_user user;
 
-  if (!s->bucket_name_str.empty())
+  if (!s->bucket_name.empty())
     user = s->bucket_owner.get_id();
   else
     user = s->user.user_id;
@@ -275,7 +276,7 @@ int rgw_log_op(RGWRados *store, struct req_state *s, const string& op_name, OpsL
   if (!s->enable_ops_log)
     return 0;
 
-  if (s->bucket_name_str.empty()) {
+  if (s->bucket_name.empty()) {
     ldout(s->cct, 5) << "nothing to log for operation" << dendl;
     return -EINVAL;
   }
@@ -288,9 +289,11 @@ int rgw_log_op(RGWRados *store, struct req_state *s, const string& op_name, OpsL
   } else {
     bucket_id = s->bucket.bucket_id;
   }
-  entry.bucket = s->bucket_name_str;
+  string bucket_log;
+  rgw_make_bucket_entry_name(s->bucket_tenant, s->bucket_name, bucket_log);
+  entry.bucket = bucket_log;
 
-  if (check_utf8(s->bucket_name_str.c_str(), entry.bucket.size()) != 0) {
+  if (check_utf8(s->bucket_name.c_str(), entry.bucket.size()) != 0) {
     ldout(s->cct, 5) << "not logging op on bucket with non-utf8 name" << dendl;
     return 0;
   }
diff --git a/src/rgw/rgw_log.h b/src/rgw/rgw_log.h
index 8fd279ce9ba4..51acfdf8f9d4 100644
--- a/src/rgw/rgw_log.h
+++ b/src/rgw/rgw_log.h
@@ -60,7 +60,7 @@ struct rgw_log_entry {
     DECODE_START_LEGACY_COMPAT_LEN(8, 5, 5, p);
     ::decode(object_owner.id, p);
     if (struct_v > 3)
-      ::decode(bucket_owner, p);
+      ::decode(bucket_owner.id, p);
     ::decode(bucket, p);
     ::decode(time, p);
     ::decode(remote_addr, p);
diff --git a/src/rgw/rgw_metadata.h b/src/rgw/rgw_metadata.h
index 8063cf7ac384..ed21dc59487c 100644
--- a/src/rgw/rgw_metadata.h
+++ b/src/rgw/rgw_metadata.h
@@ -108,6 +108,17 @@ protected:
     }
     return true;
   }
+
+  /*
+   * The tenant_name is always returned on purpose. May be empty, of course.
+   */
+  static void parse_bucket(const string &bucket,
+                           string &tenant_name, string &bucket_name)
+  {
+    int pos = bucket.find('/');
+    tenant_name = bucket.substr(0, pos);
+    bucket_name = bucket.substr(pos + 1);
+  }
 };
 
 #define META_LOG_OBJ_PREFIX "meta.log."
diff --git a/src/rgw/rgw_object_expirer_core.cc b/src/rgw/rgw_object_expirer_core.cc
index 59035111db2a..14957d7448ca 100644
--- a/src/rgw/rgw_object_expirer_core.cc
+++ b/src/rgw/rgw_object_expirer_core.cc
@@ -39,16 +39,22 @@ using namespace std;
 
 static string objexp_lock_name = "gc_process";
 
-int RGWObjectExpirer::init_bucket_info(const string& bucket_name,
-                                    const string& bucket_id,
-                                    RGWBucketInfo& bucket_info)
+int RGWObjectExpirer::init_bucket_info(const string& tenant_name,
+                                       const string& bucket_name,
+                                       const string& bucket_id,
+                                       RGWBucketInfo& bucket_info)
 {
   RGWObjectCtx obj_ctx(store);
-  const string bucket_instance_id = bucket_name + ":" + bucket_id;
 
+  /*
+   * XXX Here's where it gets tricky. We went to all the trouble of
+   * punching the tenant through the objexp_hint_entry, but now we
+   * find that our instances do not actually have tenants. They are
+   * unique thanks to IDs. So the tenant string is not needed...
+   */
+  const string bucket_instance_id = bucket_name + ":" + bucket_id;
   int ret = store->get_bucket_instance_info(obj_ctx, bucket_instance_id,
           bucket_info, NULL, NULL);
-
   return ret;
 }
 
@@ -56,7 +62,8 @@ int RGWObjectExpirer::garbage_single_object(objexp_hint_entry& hint)
 {
   RGWBucketInfo bucket_info;
 
-  int ret = init_bucket_info(hint.bucket_name, hint.bucket_id, bucket_info);
+  int ret = init_bucket_info(hint.tenant, hint.bucket_name,
+          hint.bucket_id, bucket_info);
   if (-ENOENT == ret) {
     ldout(store->ctx(), 15) << "NOTICE: cannot find bucket = " \
         << hint.bucket_name << ". The object must be already removed" << dendl;
diff --git a/src/rgw/rgw_object_expirer_core.h b/src/rgw/rgw_object_expirer_core.h
index bd137fa6bd29..c9d56da2eeb9 100644
--- a/src/rgw/rgw_object_expirer_core.h
+++ b/src/rgw/rgw_object_expirer_core.h
@@ -41,7 +41,8 @@ class RGWObjectExpirer {
 protected:
   RGWRados *store;
 
-  int init_bucket_info(const string& bucket_name,
+  int init_bucket_info(const string& tenant_name,
+                       const string& bucket_name,
                        const string& bucket_id,
                        RGWBucketInfo& bucket_info);
 
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 127d8618c7a0..93391098aa73 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -365,35 +365,30 @@ static int rgw_build_policies(RGWRados *store, struct req_state *s, bool only_bu
     s->bucket_acl = new RGWAccessControlPolicy(s->cct);
   }
 
-  if (s->copy_source) { /* check if copy source is within the current domain */
-    const char *src = s->copy_source;
-    if (*src == '/')
-      ++src;
-    string copy_source_str(src);
-
-    int pos = copy_source_str.find('/');
-    if (pos > 0)
-      copy_source_str = copy_source_str.substr(0, pos);
-
+  /* check if copy source is within the current domain */
+  if (!s->src_bucket_name.empty()) {
     RGWBucketInfo source_info;
 
-    ret = store->get_bucket_info(obj_ctx, copy_source_str, source_info, NULL);
+    ret = store->get_bucket_info(obj_ctx,
+        s->src_tenant_name, s->src_bucket_name, source_info, NULL);
     if (ret == 0) {
       string& region = source_info.region;
       s->local_source = store->region.equals(region);
     }
   }
 
-  if (!s->bucket_name_str.empty()) {
+  if (!s->bucket_name.empty()) {
     s->bucket_exists = true;
     if (s->bucket_instance_id.empty()) {
-      ret = store->get_bucket_info(obj_ctx, s->bucket_name_str, s->bucket_info, NULL, &s->bucket_attrs);
+      ret = store->get_bucket_info(obj_ctx, s->bucket_tenant, s->bucket_name, s->bucket_info, NULL, &s->bucket_attrs);
     } else {
       ret = store->get_bucket_instance_info(obj_ctx, s->bucket_instance_id, s->bucket_info, NULL, &s->bucket_attrs);
     }
     if (ret < 0) {
       if (ret != -ENOENT) {
-        ldout(s->cct, 0) << "NOTICE: couldn't get bucket from bucket_name (name=" << s->bucket_name_str << ")" << dendl;
+        string bucket_log;
+        rgw_make_bucket_entry_name(s->bucket_tenant, s->bucket_name, bucket_log);
+        ldout(s->cct, 0) << "NOTICE: couldn't get bucket from bucket_name (name=" << bucket_log << ")" << dendl;
         return ret;
       }
       s->bucket_exists = false;
@@ -832,7 +827,8 @@ int RGWGetObj::handle_user_manifest(const char *prefix)
     RGWBucketInfo bucket_info;
     map<string, bufferlist> bucket_attrs;
     RGWObjectCtx obj_ctx(store);
-    int r = store->get_bucket_info(obj_ctx, s->tenant, bucket_name, bucket_info, NULL, &bucket_attrs);
+    int r = store->get_bucket_info(obj_ctx, s->user.user_id.tenant, bucket_name,
+                                   bucket_info, NULL, &bucket_attrs);
     if (r < 0) {
       ldout(s->cct, 0) << "could not get bucket info for bucket=" << bucket_name << dendl;
       return r;
@@ -1342,6 +1338,11 @@ int RGWCreateBucket::verify_permission()
   if (!rgw_user_is_authenticated(s->user))
     return -EACCES;
 
+  /* XXX: maybe we need to check ACLs here! */
+  // if ((s->perm_mask & RGW_PERM_WRITE) == 0) {
+  //   return -EACCES;
+  // }
+
   if (s->user.max_buckets) {
     RGWUserBuckets buckets;
     string marker;
@@ -1394,7 +1395,9 @@ void RGWCreateBucket::execute()
   bufferlist aclbl;
   bufferlist corsbl;
   bool existed;
-  rgw_obj obj(store->zone.domain_root, s->bucket_name_str);
+  string bucket_name;
+  rgw_make_bucket_entry_name(s->bucket_tenant, s->bucket_name, bucket_name);
+  rgw_obj obj(store->zone.domain_root, bucket_name);
   obj_version objv, *pobjv = NULL;
 
   ret = get_params();
@@ -1410,7 +1413,8 @@ void RGWCreateBucket::execute()
 
   /* we need to make sure we read bucket info, it's not read before for this specific request */
   RGWObjectCtx& obj_ctx = *static_cast<RGWObjectCtx *>(s->obj_ctx);
-  ret = store->get_bucket_info(obj_ctx, s->bucket_name_str, s->bucket_info, NULL, &s->bucket_attrs);
+  ret = store->get_bucket_info(obj_ctx, s->bucket_tenant, s->bucket_name,
+                               s->bucket_info, NULL, &s->bucket_attrs);
   if (ret < 0 && ret != -ENOENT)
     return;
   s->bucket_exists = (ret != -ENOENT);
@@ -1465,7 +1469,9 @@ void RGWCreateBucket::execute()
   if (s->bucket_exists) {
     string selected_placement_rule;
     rgw_bucket bucket;
-    ret = store->select_bucket_placement(s->user, region_name, placement_rule, s->bucket_name_str, bucket, &selected_placement_rule);
+    ret = store->select_bucket_placement(s->user, region_name, placement_rule,
+                                         s->bucket_tenant, s->bucket_name, bucket,
+                                         &selected_placement_rule);
     if (selected_placement_rule != s->bucket_info.placement_rule) {
       ret = -EEXIST;
       return;
@@ -1480,7 +1486,8 @@ void RGWCreateBucket::execute()
     cors_config.encode(corsbl);
     attrs[RGW_ATTR_CORS] = corsbl;
   }
-  s->bucket.name = s->bucket_name_str;
+  s->bucket.tenant = s->bucket_tenant; /* ignored if bucket exists */
+  s->bucket.name = s->bucket_name;
   ret = store->create_bucket(s->user, s->bucket, region_name, placement_rule, attrs, info, pobjv,
                              &ep_objv, creation_time, pmaster_bucket, true);
   /* continue if EEXIST and create_bucket will fail below.  this way we can recover
@@ -1508,7 +1515,7 @@ void RGWCreateBucket::execute()
 
   ret = rgw_link_bucket(store, s->user.user_id, s->bucket, info.creation_time, false);
   if (ret && !existed && ret != -EEXIST) {  /* if it exists (or previously existed), don't remove it! */
-    ret = rgw_unlink_bucket(store, s->user.user_id, s->bucket.name);
+    ret = rgw_unlink_bucket(store, s->user.user_id, s->bucket.tenant, s->bucket.name);
     if (ret < 0) {
       ldout(s->cct, 0) << "WARNING: failed to unlink bucket: ret=" << ret << dendl;
     }
@@ -1534,7 +1541,7 @@ void RGWDeleteBucket::execute()
 {
   ret = -EINVAL;
 
-  if (s->bucket_name_str.empty())
+  if (s->bucket_name.empty())
     return;
 
   RGWObjVersionTracker ot;
@@ -1560,7 +1567,7 @@ void RGWDeleteBucket::execute()
   ret = store->delete_bucket(s->bucket, ot);
 
   if (ret == 0) {
-    ret = rgw_unlink_bucket(store, s->user.user_id, s->bucket.name, false);
+    ret = rgw_unlink_bucket(store, s->user.user_id, s->bucket.tenant, s->bucket.name, false);
     if (ret < 0) {
       ldout(s->cct, 0) << "WARNING: failed to unlink bucket: ret=" << ret << dendl;
     }
@@ -2244,6 +2251,9 @@ int RGWPutMetadataAccount::verify_permission()
   if (!rgw_user_is_authenticated(s->user)) {
     return -EACCES;
   }
+  // if ((s->perm_mask & RGW_PERM_WRITE) == 0) {
+  //   return -EACCES;
+  // }
   return 0;
 }
 
@@ -2280,15 +2290,9 @@ void RGWPutMetadataAccount::filter_out_temp_url(map<string, bufferlist>& add_att
 
 void RGWPutMetadataAccount::execute()
 {
-  rgw_obj obj;
   map<string, bufferlist> attrs, orig_attrs, rmattrs;
   RGWObjVersionTracker acct_op_tracker;
 
-  /* Get the name of raw object which stores the metadata in its xattrs. */
-  string buckets_obj_id;
-  rgw_get_buckets_obj(s->user.user_id, buckets_obj_id);
-  obj = rgw_obj(store->zone.user_uid_pool, buckets_obj_id);
-
   ret = get_params();
   if (ret < 0) {
     return;
@@ -2309,7 +2313,8 @@ void RGWPutMetadataAccount::execute()
     }
   }
 
-  ret = rgw_store_user_attrs(store, s->user.user_id, attrs, &rmattrs, &acct_op_tracker);
+  /* XXX tenant needed? */
+  ret = rgw_store_user_attrs(store, s->user.user_id.id, attrs, &rmattrs, &acct_op_tracker);
   if (ret < 0) {
     return;
   }
@@ -2493,7 +2498,6 @@ bool RGWCopyObj::parse_copy_location(const string& url_src, string& bucket_name,
     params_str = url_src.substr(pos + 1);
   }
 
-
   string dec_src;
 
   url_decode(name_str, dec_src);
@@ -2503,7 +2507,7 @@ bool RGWCopyObj::parse_copy_location(const string& url_src, string& bucket_name,
 
   string str(src);
 
-  pos = str.find("/");
+  pos = str.find('/');
   if (pos <= 0)
     return false;
 
@@ -2540,7 +2544,8 @@ int RGWCopyObj::verify_permission()
 
   RGWObjectCtx& obj_ctx = *static_cast<RGWObjectCtx *>(s->obj_ctx);
 
-  ret = store->get_bucket_info(obj_ctx, s->tenant, src_bucket_name, src_bucket_info, NULL, &src_attrs);
+  ret = store->get_bucket_info(obj_ctx, src_tenant_name, src_bucket_name,
+                               src_bucket_info, NULL, &src_attrs);
   if (ret < 0)
     return ret;
 
@@ -2569,7 +2574,8 @@ int RGWCopyObj::verify_permission()
     dest_bucket_info = src_bucket_info;
     dest_attrs = src_attrs;
   } else {
-    ret = store->get_bucket_info(obj_ctx, s->tenant, dest_bucket_name, dest_bucket_info, NULL, &dest_attrs);
+    ret = store->get_bucket_info(obj_ctx, dest_tenant_name, dest_bucket_name,
+                                 dest_bucket_info, NULL, &dest_attrs);
     if (ret < 0)
       return ret;
   }
diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index ee6cc2a5f1a7..0252ff8a65d2 100644
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -647,10 +647,10 @@ protected:
   time_t *unmod_ptr;
   int ret;
   map<string, bufferlist> attrs;
-  string src_bucket_name;
+  string src_tenant_name, src_bucket_name;
   rgw_bucket src_bucket;
   rgw_obj_key src_object;
-  string dest_bucket_name;
+  string dest_tenant_name, dest_bucket_name;
   rgw_bucket dest_bucket;
   string dest_object;
   time_t src_mtime;
@@ -694,7 +694,9 @@ public:
     delete_at = 0;
   }
 
-  static bool parse_copy_location(const string& src, string& bucket_name, rgw_obj_key& object);
+  static bool parse_copy_location(const string& src,
+                                  string& bucket_name,
+                                  rgw_obj_key& object);
 
   virtual void init(RGWRados *store, struct req_state *s, RGWHandler *h) {
     RGWOp::init(store, s, h);
@@ -1090,7 +1092,6 @@ protected:
   int max_to_delete;
   size_t len;
   char *data;
-  string bucket_name;
   rgw_bucket bucket;
   bool quiet;
   bool status_dumped;
diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index b800fd4da304..dd152d0c533b 100644
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -1879,7 +1879,7 @@ void RGWRados::pick_control_oid(const string& key, string& notify_oid)
   notify_oid.append(buf);
 }
 
-int RGWRados::open_bucket_pool_ctx(const string& tenant, const string& bucket_name, const string& pool, librados::IoCtx&  io_ctx)
+int RGWRados::open_bucket_pool_ctx(const string& pool, librados::IoCtx&  io_ctx)
 {
   librados::Rados *rad = get_rados_handle();
   int r = rad->ioctx_create(pool.c_str(), io_ctx);
@@ -1900,7 +1900,7 @@ int RGWRados::open_bucket_pool_ctx(const string& tenant, const string& bucket_na
 
 int RGWRados::open_bucket_data_ctx(rgw_bucket& bucket, librados::IoCtx& data_ctx)
 {
-  int r = open_bucket_pool_ctx(bucket.name, bucket.data_pool, data_ctx);
+  int r = open_bucket_pool_ctx(bucket.data_pool, data_ctx);
   if (r < 0)
     return r;
 
@@ -1910,7 +1910,7 @@ int RGWRados::open_bucket_data_ctx(rgw_bucket& bucket, librados::IoCtx& data_ctx
 int RGWRados::open_bucket_data_extra_ctx(rgw_bucket& bucket, librados::IoCtx& data_ctx)
 {
   string& pool = (!bucket.data_extra_pool.empty() ? bucket.data_extra_pool : bucket.data_pool);
-  int r = open_bucket_pool_ctx(bucket.name, pool, data_ctx);
+  int r = open_bucket_pool_ctx(pool, data_ctx);
   if (r < 0)
     return r;
 
@@ -1928,7 +1928,7 @@ void RGWRados::build_bucket_index_marker(const string& shard_id_str, const strin
 
 int RGWRados::open_bucket_index_ctx(rgw_bucket& bucket, librados::IoCtx& index_ctx)
 {
-  int r = open_bucket_pool_ctx(bucket.name, bucket.index_pool, index_ctx);
+  int r = open_bucket_pool_ctx(bucket.index_pool, index_ctx);
   if (r < 0)
     return r;
 
@@ -2394,21 +2394,25 @@ int RGWRados::objexp_key_shard(const rgw_obj_key& key)
   return sid % num_shards;
 }
 
-static string objexp_hint_get_keyext(const string& bucket_name,
+static string objexp_hint_get_keyext(const string& tenant_name,
+                                     const string& bucket_name,
                                      const string& bucket_id,
                                      const rgw_obj_key& obj_key)
 {
-  return bucket_name + ":" + bucket_id + ":" + obj_key.name + ":" + obj_key.instance;
+  return tenant_name + ":" + bucket_name + ":" + bucket_id +
+      ":" + obj_key.name + ":" + obj_key.instance;
 }
 
 int RGWRados::objexp_hint_add(const utime_t& delete_at,
+                              const string& tenant_name,
                               const string& bucket_name,
                               const string& bucket_id,
                               const rgw_obj_key& obj_key)
 {
-  const string keyext = objexp_hint_get_keyext(bucket_name,
+  const string keyext = objexp_hint_get_keyext(tenant_name, bucket_name,
           bucket_id, obj_key);
   objexp_hint_entry he = {
+      .tenant = tenant_name,
       .bucket_name = bucket_name,
       .bucket_id = bucket_id,
       .obj_key = obj_key,
@@ -2805,7 +2809,9 @@ int RGWRados::create_bucket(RGWUserInfo& owner, rgw_bucket& bucket,
   string selected_placement_rule;
   for (int i = 0; i < MAX_CREATE_RETRIES; i++) {
     int ret = 0;
-    ret = select_bucket_placement(owner, region_name, placement_rule, bucket.name, bucket, &selected_placement_rule);
+    ret = select_bucket_placement(owner, region_name, placement_rule,
+                                  bucket.tenant, bucket.name, bucket,
+                                  &selected_placement_rule);
     if (ret < 0)
       return ret;
     bufferlist bl;
@@ -2832,9 +2838,6 @@ int RGWRados::create_bucket(RGWUserInfo& owner, rgw_bucket& bucket,
       bucket.bucket_id = pmaster_bucket->bucket_id;
     }
 
-    string dir_oid =  dir_oid_prefix;
-    dir_oid.append(bucket.marker);
-
     r = init_bucket_index(bucket, bucket_index_max_shards);
     if (r < 0)
       return r;
@@ -2864,7 +2867,7 @@ int RGWRados::create_bucket(RGWUserInfo& owner, rgw_bucket& bucket,
       RGWObjVersionTracker instance_ver = info.objv_tracker;
       info.objv_tracker.clear();
       RGWObjectCtx obj_ctx(this);
-      r = get_bucket_info(obj_ctx, bucket.name, info, NULL, NULL);
+      r = get_bucket_info(obj_ctx, bucket.tenant, bucket.name, info, NULL, NULL);
       if (r < 0) {
         if (r == -ENOENT) {
           continue;
@@ -2906,7 +2909,7 @@ int RGWRados::create_bucket(RGWUserInfo& owner, rgw_bucket& bucket,
 }
 
 int RGWRados::select_new_bucket_location(RGWUserInfo& user_info, const string& region_name, const string& request_rule,
-                                         const string& bucket_name, rgw_bucket& bucket, string *pselected_rule)
+                                         const string& tenant_name, const string& bucket_name, rgw_bucket& bucket, string *pselected_rule)
 {
   /* first check that rule exists within the specific region */
   map<string, RGWRegion>::iterator riter = region_map.regions.find(region_name);
@@ -2948,18 +2951,19 @@ int RGWRados::select_new_bucket_location(RGWUserInfo& user_info, const string& r
   if (pselected_rule)
     *pselected_rule = rule;
   
-  return set_bucket_location_by_rule(rule, bucket_name, bucket);
+  return set_bucket_location_by_rule(rule, tenant_name, bucket_name, bucket);
 }
 
-int RGWRados::set_bucket_location_by_rule(const string& location_rule, const std::string& bucket_name, rgw_bucket& bucket)
+int RGWRados::set_bucket_location_by_rule(const string& location_rule, const string& tenant_name, const string& bucket_name, rgw_bucket& bucket)
 {
+  bucket.tenant = tenant_name;
   bucket.name = bucket_name;
 
   if (location_rule.empty()) {
     /* we can only reach here if we're trying to set a bucket location from a bucket
      * created on a different zone, using a legacy / default pool configuration
      */
-    return select_legacy_bucket_placement(bucket_name, bucket);
+    return select_legacy_bucket_placement(tenant_name, bucket_name, bucket);
   }
 
   /*
@@ -2986,24 +2990,29 @@ int RGWRados::set_bucket_location_by_rule(const string& location_rule, const std
   bucket.data_extra_pool = placement_info.data_extra_pool;
   bucket.index_pool = placement_info.index_pool;
 
-  return 0;
+  // XXX The select_legacy_bucket_placement() does this, but here we don't?
+  // bucket.tenant = tenant_name;
+  // bucket.name = bucket_name;
 
+  return 0;
 }
 
 int RGWRados::select_bucket_placement(RGWUserInfo& user_info, const string& region_name, const string& placement_rule,
-                                      const string& bucket_name, rgw_bucket& bucket, string *pselected_rule)
+                                      const string& tenant_name, const string& bucket_name, rgw_bucket& bucket,
+                                      string *pselected_rule)
 {
   if (!zone.placement_pools.empty()) {
-    return select_new_bucket_location(user_info, region_name, placement_rule, bucket_name, bucket, pselected_rule);
+    return select_new_bucket_location(user_info, region_name, placement_rule,
+                                      tenant_name, bucket_name, bucket, pselected_rule);
   }
 
   if (pselected_rule)
     pselected_rule->clear();
 
-  return select_legacy_bucket_placement(bucket_name, bucket);
+  return select_legacy_bucket_placement(tenant_name, bucket_name, bucket);
 }
 
-int RGWRados::select_legacy_bucket_placement(const string& bucket_name, rgw_bucket& bucket)
+int RGWRados::select_legacy_bucket_placement(const string& tenant_name, const string& bucket_name, rgw_bucket& bucket)
 {
   bufferlist map_bl;
   map<string, bufferlist> m;
@@ -3076,6 +3085,7 @@ read_omap:
   }
   bucket.data_pool = pool_name;
   bucket.index_pool = pool_name;
+  bucket.tenant = tenant_name;
   bucket.name = bucket_name;
 
   return 0;
@@ -3638,7 +3648,8 @@ int RGWRados::Object::Write::write_meta(uint64_t size,
     rgw_obj_key obj_key;
     obj.get_index_key(&obj_key);
 
-    r = store->objexp_hint_add(utime_t(meta.delete_at, 0), bucket.name, bucket.bucket_id, obj_key);
+    r = store->objexp_hint_add(utime_t(meta.delete_at, 0),
+            bucket.tenant, bucket.name, bucket.bucket_id, obj_key);
     if (r < 0) {
       ldout(store->ctx(), 0) << "ERROR: objexp_hint_add() returned r=" << r << ", object will not get removed" << dendl;
       /* ignoring error, nothing we can do at this point */
@@ -4538,7 +4549,7 @@ int RGWRados::delete_bucket(rgw_bucket& bucket, RGWObjVersionTracker& objv_track
     }
   } while (is_truncated);
 
-  r = rgw_bucket_delete_bucket_obj(this, bucket.name, objv_tracker);
+  r = rgw_bucket_delete_bucket_obj(this, bucket.tenant, bucket.name, objv_tracker);
   if (r < 0)
     return r;
 
@@ -4561,7 +4572,7 @@ int RGWRados::set_bucket_owner(rgw_bucket& bucket, ACLOwner& owner)
   RGWBucketInfo info;
   map<string, bufferlist> attrs;
   RGWObjectCtx obj_ctx(this);
-  int r = get_bucket_info(obj_ctx, bucket.name, info, NULL, &attrs);
+  int r = get_bucket_info(obj_ctx, bucket.tenant, bucket.name, info, NULL, &attrs);
   if (r < 0) {
     ldout(cct, 0) << "NOTICE: get_bucket_info on bucket=" << bucket.name << " returned err=" << r << dendl;
     return r;
@@ -4595,7 +4606,7 @@ int RGWRados::set_buckets_enabled(vector<rgw_bucket>& buckets, bool enabled)
     RGWBucketInfo info;
     map<string, bufferlist> attrs;
     RGWObjectCtx obj_ctx(this);
-    int r = get_bucket_info(obj_ctx, bucket.name, info, NULL, &attrs);
+    int r = get_bucket_info(obj_ctx, bucket.tenant, bucket.name, info, NULL, &attrs);
     if (r < 0) {
       ldout(cct, 0) << "NOTICE: get_bucket_info on bucket=" << bucket.name << " returned err=" << r << ", skipping bucket" << dendl;
       ret = r;
@@ -4621,7 +4632,7 @@ int RGWRados::bucket_suspended(rgw_bucket& bucket, bool *suspended)
 {
   RGWBucketInfo bucket_info;
   RGWObjectCtx obj_ctx(this);
-  int ret = get_bucket_info(obj_ctx, bucket.name, bucket_info, NULL);
+  int ret = get_bucket_info(obj_ctx, bucket.tenant, bucket.name, bucket_info, NULL);
   if (ret < 0) {
     return ret;
   }
@@ -5551,7 +5562,7 @@ int RGWRados::set_attrs(void *ctx, rgw_obj& obj,
         rgw_obj_key obj_key;
         obj.get_index_key(&obj_key);
 
-        objexp_hint_add(ts, bucket.name, bucket.bucket_id, obj_key);
+        objexp_hint_add(ts, bucket.tenant, bucket.name, bucket.bucket_id, obj_key);
       } catch (buffer::error& err) {
 	ldout(cct, 0) << "ERROR: failed to decode " RGW_ATTR_DELETE_AT << " attr" << dendl;
       }
@@ -5641,6 +5652,7 @@ int RGWRados::set_attrs(void *ctx, rgw_obj& obj,
  *          (if get_data==true) length of read data,
  *          (if get_data==false) length of the object
  */
+// P3 XXX get_data is not seen used anywhere.
 int RGWRados::Object::Read::prepare(int64_t *pofs, int64_t *pend)
 {
   RGWRados *store = source->get_store();
@@ -7541,7 +7553,9 @@ int RGWRados::get_bucket_instance_from_oid(RGWObjectCtx& obj_ctx, string& oid, R
   return 0;
 }
 
-int RGWRados::get_bucket_entrypoint_info(RGWObjectCtx& obj_ctx, const string& bucket_name,
+int RGWRados::get_bucket_entrypoint_info(RGWObjectCtx& obj_ctx,
+                                         const string& tenant_name,
+                                         const string& bucket_name,
                                          RGWBucketEntryPoint& entry_point,
                                          RGWObjVersionTracker *objv_tracker,
                                          time_t *pmtime,
@@ -7549,8 +7563,10 @@ int RGWRados::get_bucket_entrypoint_info(RGWObjectCtx& obj_ctx, const string& bu
                                          rgw_cache_entry_info *cache_info)
 {
   bufferlist bl;
+  string bucket_entry;
 
-  int ret = rgw_get_system_obj(this, obj_ctx, zone.domain_root, bucket_name, bl, objv_tracker, pmtime, pattrs, cache_info);
+  rgw_make_bucket_entry_name(tenant_name, bucket_name, bucket_entry);
+  int ret = rgw_get_system_obj(this, obj_ctx, zone.domain_root, bucket_entry, bl, objv_tracker, pmtime, pattrs, cache_info);
   if (ret < 0) {
     return ret;
   }
@@ -7565,7 +7581,9 @@ int RGWRados::get_bucket_entrypoint_info(RGWObjectCtx& obj_ctx, const string& bu
   return 0;
 }
 
-int RGWRados::convert_old_bucket_info(RGWObjectCtx& obj_ctx, string& bucket_name)
+int RGWRados::convert_old_bucket_info(RGWObjectCtx& obj_ctx,
+                                      const string& tenant_name,
+                                      const string& bucket_name)
 {
   RGWBucketEntryPoint entry_point;
   time_t ep_mtime;
@@ -7575,7 +7593,7 @@ int RGWRados::convert_old_bucket_info(RGWObjectCtx& obj_ctx, string& bucket_name
 
   ldout(cct, 10) << "RGWRados::convert_old_bucket_info(): bucket=" << bucket_name << dendl;
 
-  int ret = get_bucket_entrypoint_info(obj_ctx, bucket_name, entry_point, &ot, &ep_mtime, &attrs);
+  int ret = get_bucket_entrypoint_info(obj_ctx, tenant_name, bucket_name, entry_point, &ot, &ep_mtime, &attrs);
   if (ret < 0) {
     ldout(cct, 0) << "ERROR: get_bucket_entrypont_info() returned " << ret << " bucket=" << bucket_name << dendl;
     return ret;
@@ -7600,19 +7618,15 @@ int RGWRados::convert_old_bucket_info(RGWObjectCtx& obj_ctx, string& bucket_name
   return 0;
 }
 
-struct bucket_info_entry {
-  RGWBucketInfo info;
-  time_t mtime;
-  map<string, bufferlist> attrs;
-};
-
-static RGWChainedCacheImpl<bucket_info_entry> binfo_cache;
-
-int RGWRados::get_bucket_info(RGWObjectCtx& obj_ctx, const string& tenant, const string& bucket_name, RGWBucketInfo& info,
+int RGWRados::get_bucket_info(RGWObjectCtx& obj_ctx,
+                              const string& tenant, const string& bucket_name, RGWBucketInfo& info,
                               time_t *pmtime, map<string, bufferlist> *pattrs)
 {
   bucket_info_entry e;
-  if (binfo_cache.find(bucket_name, &e)) {
+  string bucket_entry;
+  rgw_make_bucket_entry_name(tenant, bucket_name, bucket_entry);
+
+  if (binfo_cache.find(bucket_entry, &e)) {
     info = e.info;
     if (pattrs)
       *pattrs = e.attrs;
@@ -7629,7 +7643,9 @@ int RGWRados::get_bucket_info(RGWObjectCtx& obj_ctx, const string& tenant, const
   rgw_cache_entry_info entry_cache_info;
   int ret = get_bucket_entrypoint_info(obj_ctx, tenant, bucket_name, entry_point, &ot, &ep_mtime, pattrs, &entry_cache_info);
   if (ret < 0) {
-    info.bucket.name = bucket_name; /* only init this field */
+    /* only init these fields */
+    info.bucket.tenant = tenant;
+    info.bucket.name = bucket_name;
     return ret;
   }
 
@@ -7665,7 +7681,9 @@ int RGWRados::get_bucket_info(RGWObjectCtx& obj_ctx, const string& tenant, const
   e.info.ep_objv = ot.read_version;
   info = e.info;
   if (ret < 0) {
+    info.bucket.tenant = tenant;
     info.bucket.name = bucket_name;
+    // XXX and why return anything in case of an error anyway?
     return ret;
   }
 
@@ -7680,20 +7698,22 @@ int RGWRados::get_bucket_info(RGWObjectCtx& obj_ctx, const string& tenant, const
 
 
   /* chain to both bucket entry point and bucket instance */
-  if (!binfo_cache.put(this, bucket_name, &e, cache_info_entries)) {
+  if (!binfo_cache.put(this, bucket_entry, &e, cache_info_entries)) {
     ldout(cct, 20) << "couldn't put binfo cache entry, might have raced with data changes" << dendl;
   }
 
   return 0;
 }
 
-int RGWRados::put_bucket_entrypoint_info(const string& tenant, const string& bucket_name, RGWBucketEntryPoint& entry_point,
+int RGWRados::put_bucket_entrypoint_info(const string& tenant_name, const string& bucket_name, RGWBucketEntryPoint& entry_point,
                                          bool exclusive, RGWObjVersionTracker& objv_tracker, time_t mtime,
                                          map<string, bufferlist> *pattrs)
 {
   bufferlist epbl;
   ::encode(entry_point, epbl);
-  return rgw_bucket_store_info(this, bucket_name, epbl, exclusive, pattrs, &objv_tracker, mtime);
+  string bucket_entry;
+  rgw_make_bucket_entry_name(tenant_name, bucket_name, bucket_entry);
+  return rgw_bucket_store_info(this, bucket_entry, epbl, exclusive, pattrs, &objv_tracker, mtime);
 }
 
 int RGWRados::put_bucket_instance_info(RGWBucketInfo& info, bool exclusive,
@@ -7738,7 +7758,7 @@ int RGWRados::put_linked_bucket_info(RGWBucketInfo& info, bool exclusive, time_t
       *pep_objv = ot.write_version;
     }
   }
-  ret = put_bucket_entrypoint_info(info.bucket.name, entry_point, exclusive, ot, mtime, NULL); 
+  ret = put_bucket_entrypoint_info(info.bucket.tenant, info.bucket.name, entry_point, exclusive, ot, mtime, NULL); 
   if (ret < 0)
     return ret;
 
diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 577e7d617226..67adc60e4c73 100644
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -51,7 +51,7 @@ static inline void get_obj_bucket_and_oid_loc(const rgw_obj& obj, rgw_bucket& bu
   prepend_bucket_marker(bucket, obj.get_object(), oid);
   const string& loc = obj.get_loc();
   if (!loc.empty()) {
-    prepend_bucket_marker(bucket, obj.get_loc(), locator);
+    prepend_bucket_marker(bucket, obj.get_loc(), locator); // XXX get_loc twice
   } else {
     locator.clear();
   }
@@ -996,26 +996,34 @@ struct RGWRegionMap {
 WRITE_CLASS_ENCODER(RGWRegionMap)
 
 struct objexp_hint_entry {
+  string tenant;
   string bucket_name;
   string bucket_id;
   rgw_obj_key obj_key;
   utime_t exp_time;
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(1, 1, bl);
+    ENCODE_START(2, 1, bl);
     ::encode(bucket_name, bl);
     ::encode(bucket_id, bl);
     ::encode(obj_key, bl);
     ::encode(exp_time, bl);
+    ::encode(tenant, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::iterator& bl) {
-    DECODE_START(1, bl);
+    // XXX Do we want DECODE_START_LEGACY_COMPAT_LEN(2, 1, 1, bl); ?
+    DECODE_START(2, bl);
     ::decode(bucket_name, bl);
     ::decode(bucket_id, bl);
     ::decode(obj_key, bl);
     ::decode(exp_time, bl);
+    if (struct_v >= 2) {
+      ::decode(tenant, bl);
+    } else {
+      tenant.clear();
+    }
     DECODE_FINISH(bl);
   }
 };
@@ -1208,7 +1216,7 @@ class RGWRados
   int open_gc_pool_ctx();
   int open_objexp_pool_ctx();
 
-  int open_bucket_pool_ctx(const string& bucket_name, const string& pool, librados::IoCtx&  io_ctx);
+  int open_bucket_pool_ctx(const string& pool, librados::IoCtx&  io_ctx);
   int open_bucket_index_ctx(rgw_bucket& bucket, librados::IoCtx&  index_ctx);
   int open_bucket_data_ctx(rgw_bucket& bucket, librados::IoCtx&  io_ctx);
   int open_bucket_data_extra_ctx(rgw_bucket& bucket, librados::IoCtx&  io_ctx);
@@ -1417,12 +1425,12 @@ public:
    * returns 0 on success, -ERR# otherwise.
    */
   virtual int init_bucket_index(rgw_bucket& bucket, int num_shards);
-  int select_bucket_placement(RGWUserInfo& user_info, const string& region_name, const std::string& rule,
-                              const std::string& bucket_name, rgw_bucket& bucket, string *pselected_rule);
-  int select_legacy_bucket_placement(const string& bucket_name, rgw_bucket& bucket);
+  int select_bucket_placement(RGWUserInfo& user_info, const string& region_name, const string& rule,
+                              const string& tenant_name, const string& bucket_name, rgw_bucket& bucket, string *pselected_rule);
+  int select_legacy_bucket_placement(const string& tenant_name, const string& bucket_name, rgw_bucket& bucket);
   int select_new_bucket_location(RGWUserInfo& user_info, const string& region_name, const string& rule,
-                                 const std::string& bucket_name, rgw_bucket& bucket, string *pselected_rule);
-  int set_bucket_location_by_rule(const string& location_rule, const std::string& bucket_name, rgw_bucket& bucket);
+                                 const string& tenant_name, const string& bucket_name, rgw_bucket& bucket, string *pselected_rule);
+  int set_bucket_location_by_rule(const string& location_rule, const string& tenant_name, const string& bucket_name, rgw_bucket& bucket);
   virtual int create_bucket(RGWUserInfo& owner, rgw_bucket& bucket,
                             const string& region_name,
                             const string& placement_rule,
@@ -2055,18 +2063,23 @@ public:
   void get_bucket_instance_entry(rgw_bucket& bucket, string& entry);
   void get_bucket_meta_oid(rgw_bucket& bucket, string& oid);
 
-  int put_bucket_entrypoint_info(const string& bucket_name, RGWBucketEntryPoint& entry_point, bool exclusive, RGWObjVersionTracker& objv_tracker, time_t mtime,
+  int put_bucket_entrypoint_info(const string& tenant_name, const string& bucket_name, RGWBucketEntryPoint& entry_point,
+                                 bool exclusive, RGWObjVersionTracker& objv_tracker, time_t mtime,
                                  map<string, bufferlist> *pattrs);
   int put_bucket_instance_info(RGWBucketInfo& info, bool exclusive, time_t mtime, map<string, bufferlist> *pattrs);
-  int get_bucket_entrypoint_info(RGWObjectCtx& obj_ctx, const string& tenant, const string& bucket_name, RGWBucketEntryPoint& entry_point, RGWObjVersionTracker *objv_tracker, time_t *pmtime,
-                                 map<string, bufferlist> *pattrs, rgw_cache_entry_info *cache_info = NULL);
+  int get_bucket_entrypoint_info(RGWObjectCtx& obj_ctx, const string& tenant_name, const string& bucket_name,
+                                 RGWBucketEntryPoint& entry_point, RGWObjVersionTracker *objv_tracker,
+                                 time_t *pmtime, map<string, bufferlist> *pattrs, rgw_cache_entry_info *cache_info = NULL);
   int get_bucket_instance_info(RGWObjectCtx& obj_ctx, const string& meta_key, RGWBucketInfo& info, time_t *pmtime, map<string, bufferlist> *pattrs);
   int get_bucket_instance_info(RGWObjectCtx& obj_ctx, rgw_bucket& bucket, RGWBucketInfo& info, time_t *pmtime, map<string, bufferlist> *pattrs);
   int get_bucket_instance_from_oid(RGWObjectCtx& obj_ctx, string& oid, RGWBucketInfo& info, time_t *pmtime, map<string, bufferlist> *pattrs,
                                    rgw_cache_entry_info *cache_info = NULL);
 
-  int convert_old_bucket_info(RGWObjectCtx& obj_ctx, string& bucket_name);
-  virtual int get_bucket_info(RGWObjectCtx& obj_ctx, const string& bucket_name, RGWBucketInfo& info,
+  int convert_old_bucket_info(RGWObjectCtx& obj_ctx, const string& tenant_name, const string& bucket_name);
+  static void make_bucket_entry_name(const string& tenant_name, const string& bucket_name, string& bucket_entry);
+  virtual int get_bucket_info(RGWObjectCtx& obj_ctx,
+                              const string& tenant_name, const string& bucket_name,
+                              RGWBucketInfo& info,
                               time_t *pmtime, map<string, bufferlist> *pattrs = NULL);
   virtual int put_linked_bucket_info(RGWBucketInfo& info, bool exclusive, time_t mtime, obj_version *pep_objv,
                                      map<string, bufferlist> *pattrs, bool create_entry_point);
@@ -2118,6 +2131,7 @@ public:
   void objexp_get_shard(int shard_num,
                         string& shard);                       /* out */
   int objexp_hint_add(const utime_t& delete_at,
+                      const string& tenant_name,
                       const string& bucket_name,
                       const string& bucket_id,
                       const rgw_obj_key& obj_key);
diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc
index 0bc4ca5b20c5..cb687656153f 100644
--- a/src/rgw/rgw_rest.cc
+++ b/src/rgw/rgw_rest.cc
@@ -366,9 +366,14 @@ void dump_bucket_from_state(struct req_state *s)
 {
   int expose_bucket = g_conf->rgw_expose_bucket;
   if (expose_bucket) {
-    if (!s->bucket_name_str.empty()) {
+    if (!s->bucket_name.empty()) {
       string b;
-      url_encode(s->bucket_name_str, b);
+      if (!s->bucket_tenant.empty()) {
+        string g = s->bucket_tenant + "/" + s->bucket_name;
+        url_encode(g, b);
+      } else {
+        url_encode(s->bucket_name, b);
+      }
       s->cio->print("Bucket: %s\r\n", b.c_str());
     }
   }
@@ -382,8 +387,12 @@ void dump_uri_from_state(struct req_state *s)
     string server = s->info.env->get("SERVER_NAME", "<SERVER_NAME>");
     location.append(server);
     location += "/";
-    if (!s->bucket_name_str.empty()) {
-      location += s->bucket_name_str;
+    if (!s->bucket_name.empty()) {
+      if (!s->bucket_tenant.empty()) {
+        location += s->bucket_tenant;
+        location += ":";
+      }
+      location += s->bucket_name;
       location += "/";
       if (!s->object.empty()) {
         location += s->object.name;
@@ -1072,9 +1081,8 @@ int RGWListBucketMultiparts_ObjStore::get_params()
 
 int RGWDeleteMultiObj_ObjStore::get_params()
 {
-  bucket_name = s->bucket_name_str;
 
-  if (bucket_name.empty()) {
+  if (s->bucket_name.empty()) {
     ret = -EINVAL;
     return ret;
   }
@@ -1168,6 +1176,17 @@ int RGWHandler_ObjStore::allocate_formatter(struct req_state *s, int default_typ
   return 0;
 }
 
+int RGWHandler_ObjStore::validate_tenant_name(string const& t)
+{
+  struct tench {
+    static bool is_good(char ch) {
+      return isalnum(ch) || ch == '_';
+    }
+  };
+  std::string::const_iterator it = std::find_if(t.begin(), t.end(), tench::is_good);
+  return (it == t.end())? 0: -ERR_INVALID_BUCKET_NAME;
+}
+
 // This function enforces Amazon's spec for bucket names.
 // (The requirements, not the recommendations.)
 int RGWHandler_ObjStore::validate_bucket_name(const string& bucket)
@@ -1392,7 +1411,7 @@ int RGWREST::preprocess(struct req_state *s, RGWClientIO *cio)
       string encoded_bucket = "/";
       encoded_bucket.append(subdomain);
       if (s->info.request_uri[0] != '/')
-        encoded_bucket.append("/'");
+        encoded_bucket.append("/'");  // XXX What in the world is this apostrophe?
       encoded_bucket.append(s->info.request_uri);
       s->info.request_uri = encoded_bucket;
     }
diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h
index 55d3f5c895e6..b9355e6457b0 100644
--- a/src/rgw/rgw_rest.h
+++ b/src/rgw/rgw_rest.h
@@ -304,6 +304,7 @@ protected:
   virtual RGWOp *op_copy() { return NULL; }
   virtual RGWOp *op_options() { return NULL; }
 
+  virtual int validate_tenant_name(const string& bucket);
   virtual int validate_bucket_name(const string& bucket);
   virtual int validate_object_name(const string& object);
 
diff --git a/src/rgw/rgw_rest_log.cc b/src/rgw/rgw_rest_log.cc
index 7e3707f8ad06..2ba2ee772f07 100644
--- a/src/rgw/rgw_rest_log.cc
+++ b/src/rgw/rgw_rest_log.cc
@@ -269,7 +269,8 @@ void RGWOp_MDLog_Unlock::execute() {
 }
 
 void RGWOp_BILog_List::execute() {
-  string bucket_name = s->info.args.get("bucket"),
+  string tenant_name = s->info.args.get("tenant"),
+         bucket_name = s->info.args.get("bucket"),
          marker = s->info.args.get("marker"),
          max_entries_str = s->info.args.get("max-entries"),
          bucket_instance = s->info.args.get("bucket-instance");
@@ -297,7 +298,7 @@ void RGWOp_BILog_List::execute() {
       return;
     }
   } else { /* !bucket_name.empty() */
-    http_ret = store->get_bucket_info(obj_ctx, bucket_name, bucket_info, NULL, NULL);
+    http_ret = store->get_bucket_info(obj_ctx, tenant_name, bucket_name, bucket_info, NULL, NULL);
     if (http_ret < 0) {
       dout(5) << "could not get bucket info for bucket=" << bucket_name << dendl;
       return;
@@ -364,7 +365,8 @@ void RGWOp_BILog_List::send_response_end() {
 }
       
 void RGWOp_BILog_Info::execute() {
-  string bucket_name = s->info.args.get("bucket"),
+  string tenant_name = s->info.args.get("tenant"),
+         bucket_name = s->info.args.get("bucket"),
          bucket_instance = s->info.args.get("bucket-instance");
   RGWBucketInfo bucket_info;
 
@@ -383,7 +385,7 @@ void RGWOp_BILog_Info::execute() {
       return;
     }
   } else { /* !bucket_name.empty() */
-    http_ret = store->get_bucket_info(obj_ctx, bucket_name, bucket_info, NULL, NULL);
+    http_ret = store->get_bucket_info(obj_ctx, tenant_name, bucket_name, bucket_info, NULL, NULL);
     if (http_ret < 0) {
       dout(5) << "could not get bucket info for bucket=" << bucket_name << dendl;
       return;
@@ -415,7 +417,8 @@ void RGWOp_BILog_Info::send_response() {
 }
 
 void RGWOp_BILog_Delete::execute() {
-  string bucket_name = s->info.args.get("bucket"),
+  string tenant_name = s->info.args.get("tenant"),
+         bucket_name = s->info.args.get("bucket"),
          start_marker = s->info.args.get("start-marker"),
          end_marker = s->info.args.get("end-marker"),
          bucket_instance = s->info.args.get("bucket-instance");
@@ -445,7 +448,7 @@ void RGWOp_BILog_Delete::execute() {
       return;
     }
   } else { /* !bucket_name.empty() */
-    http_ret = store->get_bucket_info(obj_ctx, bucket_name, bucket_info, NULL, NULL);
+    http_ret = store->get_bucket_info(obj_ctx, tenant_name, bucket_name, bucket_info, NULL, NULL);
     if (http_ret < 0) {
       dout(5) << "could not get bucket info for bucket=" << bucket_name << dendl;
       return;
diff --git a/src/rgw/rgw_rest_metadata.cc b/src/rgw/rgw_rest_metadata.cc
index 1068076a7761..f97fdb4a7226 100644
--- a/src/rgw/rgw_rest_metadata.cc
+++ b/src/rgw/rgw_rest_metadata.cc
@@ -32,8 +32,8 @@ static inline void frame_metadata_key(req_state *s, string& out) {
   string key = s->info.args.get("key", &exists);
 
   string section;
-  if (!s->bucket_name_str.empty()) {
-    section = s->bucket_name_str;
+  if (!s->bucket_name.empty()) {
+    section = s->bucket_name;
   } else {
     section = key;
     key.clear();
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index b80ea3b40725..a7268627f544 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -257,7 +257,9 @@ void RGWListBucket_ObjStore_S3::send_versioned_response()
 {
   s->formatter->open_object_section_in_ns("ListVersionsResult",
 					  "http://s3.amazonaws.com/doc/2006-03-01/");
-  s->formatter->dump_string("Name", s->bucket_name_str);
+  if (!s->bucket_tenant.empty())
+    s->formatter->dump_string("Tenant", s->bucket_tenant);
+  s->formatter->dump_string("Name", s->bucket_name);
   s->formatter->dump_string("Prefix", prefix);
   s->formatter->dump_string("KeyMarker", marker.name);
   if (is_truncated && !next_marker.empty())
@@ -334,7 +336,9 @@ void RGWListBucket_ObjStore_S3::send_response()
 
   s->formatter->open_object_section_in_ns("ListBucketResult",
 					  "http://s3.amazonaws.com/doc/2006-03-01/");
-  s->formatter->dump_string("Name", s->bucket_name_str);
+  if (!s->bucket_tenant.empty())
+    s->formatter->dump_string("Tenant", s->bucket_tenant);
+  s->formatter->dump_string("Name", s->bucket_name);
   s->formatter->dump_string("Prefix", prefix);
   s->formatter->dump_string("Marker", marker.name);
   if (is_truncated && !next_marker.empty())
@@ -1347,6 +1351,7 @@ void RGWPostObj_ObjStore_S3::send_response()
 
     part_str("success_action_redirect", &redirect);
 
+    string tenant;
     string bucket;
     string key;
     string etag_str = "\"";
@@ -1356,12 +1361,28 @@ void RGWPostObj_ObjStore_S3::send_response()
 
     string etag_url;
 
-    url_encode(s->bucket_name_str, bucket);
+    url_encode(s->bucket_tenant, tenant); /* surely overkill, but cheap */
+    url_encode(s->bucket_name, bucket);
     url_encode(s->object.name, key);
     url_encode(etag_str, etag_url);
 
-    redirect.append("?bucket=");
-    redirect.append(bucket);
+    if (!s->bucket_tenant.empty()) {
+      /*
+       * What we really would like is to quaily the bucket name, so
+       * that the client could simply copy it and paste into next request.
+       * Unfortunately, in S3 we cannot know if the client will decide
+       * to come through DNS, with "bucket.tenant" sytanx, or through
+       * URL with "tenant\bucket" syntax. Therefore, we provide the
+       * tenant separately.
+       */
+      redirect.append("?tenant=");
+      redirect.append(tenant);
+      redirect.append("&bucket=");
+      redirect.append(bucket);
+    } else {
+      redirect.append("?bucket=");
+      redirect.append(bucket);
+    }
     redirect.append("&key=");
     redirect.append(key);
     redirect.append("&etag=");
@@ -1405,7 +1426,9 @@ done:
     s->formatter->open_object_section("PostResponse");
     if (g_conf->rgw_dns_name.length())
       s->formatter->dump_format("Location", "%s/%s", s->info.script_uri.c_str(), s->object.name.c_str());
-    s->formatter->dump_string("Bucket", s->bucket_name_str);
+    if (!s->bucket_tenant.empty())
+      s->formatter->dump_string("Tenant", s->bucket_tenant);
+    s->formatter->dump_string("Bucket", s->bucket_name);
     s->formatter->dump_string("Key", s->object.name);
     s->formatter->close_section();
   }
@@ -1467,8 +1490,10 @@ int RGWCopyObj_ObjStore_S3::get_params()
   if_match = s->info.env->get("HTTP_X_AMZ_COPY_IF_MATCH");
   if_nomatch = s->info.env->get("HTTP_X_AMZ_COPY_IF_NONE_MATCH");
 
+  src_tenant_name = s->src_tenant_name;
   src_bucket_name = s->src_bucket_name;
   src_object = s->src_object;
+  dest_tenant_name = s->bucket.tenant;
   dest_bucket_name = s->bucket.name;
   dest_object = s->object.name;
 
@@ -1500,6 +1525,7 @@ int RGWCopyObj_ObjStore_S3::get_params()
   }
 
   if (source_zone.empty() &&
+      (dest_tenant_name.compare(src_tenant_name) == 0) &&
       (dest_bucket_name.compare(src_bucket_name) == 0) &&
       (dest_object.compare(src_object.name) == 0) &&
       src_object.instance.empty() &&
@@ -1816,7 +1842,9 @@ void RGWInitMultipart_ObjStore_S3::send_response()
     dump_start(s);
     s->formatter->open_object_section_in_ns("InitiateMultipartUploadResult",
 		  "http://s3.amazonaws.com/doc/2006-03-01/");
-    s->formatter->dump_string("Bucket", s->bucket_name_str);
+    if (!s->bucket_tenant.empty())
+      s->formatter->dump_string("Tenant", s->bucket_tenant);
+    s->formatter->dump_string("Bucket", s->bucket_name);
     s->formatter->dump_string("Key", s->object.name);
     s->formatter->dump_string("UploadId", upload_id);
     s->formatter->close_section();
@@ -1834,9 +1862,22 @@ void RGWCompleteMultipart_ObjStore_S3::send_response()
     dump_start(s);
     s->formatter->open_object_section_in_ns("CompleteMultipartUploadResult",
 			  "http://s3.amazonaws.com/doc/2006-03-01/");
-    if (s->info.domain.length())
-      s->formatter->dump_format("Location", "%s.%s", s->bucket_name_str.c_str(), s->info.domain.c_str());
-    s->formatter->dump_string("Bucket", s->bucket_name_str);
+    if (!s->bucket_tenant.empty()) {
+      if (s->info.domain.length()) {
+        s->formatter->dump_format("Location", "%s.%s.%s",
+          s->bucket_name.c_str(),
+          s->bucket_tenant.c_str(),
+          s->info.domain.c_str());
+      }
+      s->formatter->dump_string("Tenant", s->bucket_tenant);
+    } else {
+      if (s->info.domain.length()) {
+        s->formatter->dump_format("Location", "%s.%s",
+          s->bucket_name.c_str(),
+          s->info.domain.c_str());
+      }
+    }
+    s->formatter->dump_string("Bucket", s->bucket_name);
     s->formatter->dump_string("Key", s->object.name);
     s->formatter->dump_string("ETag", etag);
     s->formatter->close_section();
@@ -1875,7 +1916,9 @@ void RGWListMultipart_ObjStore_S3::send_response()
     if (test_iter != parts.rend()) {
       cur_max = test_iter->first;
     }
-    s->formatter->dump_string("Bucket", s->bucket_name_str);
+    if (!s->bucket_tenant.empty())
+      s->formatter->dump_string("Tenant", s->bucket_tenant);
+    s->formatter->dump_string("Bucket", s->bucket_name);
     s->formatter->dump_string("Key", s->object.name);
     s->formatter->dump_string("UploadId", upload_id);
     s->formatter->dump_string("StorageClass", "STANDARD");
@@ -1923,7 +1966,9 @@ void RGWListBucketMultiparts_ObjStore_S3::send_response()
     return;
 
   s->formatter->open_object_section("ListMultipartUploadsResult");
-  s->formatter->dump_string("Bucket", s->bucket_name_str);
+  if (!s->bucket_tenant.empty())
+    s->formatter->dump_string("Tenant", s->bucket_tenant);
+  s->formatter->dump_string("Bucket", s->bucket_name);
   if (!prefix.empty())
     s->formatter->dump_string("ListMultipartUploadsResult.Prefix", prefix);
   string& key_marker = marker.get_key();
@@ -2161,7 +2206,7 @@ RGWOp *RGWHandler_ObjStore_Obj_S3::op_put()
   if (is_acl_op()) {
     return new RGWPutACLs_ObjStore_S3;
   }
-  if (!s->copy_source)
+  if (s->src_bucket_name.empty())
     return new RGWPutObj_ObjStore_S3;
   else
     return new RGWCopyObj_ObjStore_S3;
@@ -2231,8 +2276,20 @@ int RGWHandler_ObjStore_S3::init_from_header(struct req_state *s, int default_fo
     first = req;
   }
 
-  if (s->bucket_name_str.empty()) {
-    s->bucket_name_str = first;
+  /*
+   * XXX The intent of the check for empty is apparently to let the bucket
+   * name from DNS to be set ahead. However, we currently take the DNS
+   * bucket and re-insert it into URL in rgw_rest.cc:RGWREST::preprocess().
+   * So, this check is meaningless.
+   *
+   * Rather than dropping this, the code needs to be changed into putting
+   * the bucket (and its tenant) from DNS and Host: header (HTTP_HOST)
+   * into req_status.bucket_name directly.
+   */
+  if (s->bucket_name.empty()) {
+    rgw_parse_url_bucket(first, s->bucket_tenant, s->bucket_name);
+    if (s->bucket_tenant.empty())
+      s->bucket_tenant = s->user.user_id.tenant;
 
     if (pos >= 0) {
       string encoded_obj_str = req.substr(pos+1);
@@ -2304,10 +2361,16 @@ int RGWHandler_ObjStore_S3::validate_bucket_name(const string& bucket, bool rela
 
 int RGWHandler_ObjStore_S3::init(RGWRados *store, struct req_state *s, RGWClientIO *cio)
 {
-  dout(10) << "s->object=" << (!s->object.empty() ? s->object : rgw_obj_key("<NULL>")) << " s->bucket=" << (!s->bucket_name_str.empty() ? s->bucket_name_str : "<NULL>") << dendl;
+  string bucket_log;
+  rgw_make_bucket_entry_name(s->bucket_tenant, s->bucket_name, bucket_log);
+  dout(10) << "s->object=" << (!s->object.empty() ? s->object : rgw_obj_key("<NULL>")) << " s->bucket=" << bucket_log << dendl;
 
+  int ret;
+  ret = validate_tenant_name(s->bucket_tenant);
+  if (ret)
+    return ret;
   bool relaxed_names = s->cct->_conf->rgw_relaxed_s3_bucket_names;
-  int ret = validate_bucket_name(s->bucket_name_str, relaxed_names);
+  ret = validate_bucket_name(s->bucket_name, relaxed_names);
   if (ret)
     return ret;
   ret = validate_object_name(s->object.name);
@@ -2320,13 +2383,17 @@ int RGWHandler_ObjStore_S3::init(RGWRados *store, struct req_state *s, RGWClient
 
   s->has_acl_header = s->info.env->exists_prefix("HTTP_X_AMZ_GRANT");
 
-  s->copy_source = s->info.env->get("HTTP_X_AMZ_COPY_SOURCE");
-  if (s->copy_source) {
-    ret = RGWCopyObj::parse_copy_location(s->copy_source, s->src_bucket_name, s->src_object);
+  const char *copy_source = s->info.env->get("HTTP_X_AMZ_COPY_SOURCE");
+  if (copy_source) {
+    string src_bucket_str;
+    ret = RGWCopyObj::parse_copy_location(copy_source, src_bucket_str, s->src_object);
     if (!ret) {
       ldout(s->cct, 0) << "failed to parse copy location" << dendl;
-      return -EINVAL;
+      return -EINVAL; // XXX why not -ERR_INVALID_BUCKET_NAME or -ERR_BAD_URL?
     }
+    rgw_parse_url_bucket(src_bucket_str, s->src_tenant_name, s->src_bucket_name);
+    if (s->src_tenant_name.empty())
+      s->src_tenant_name = s->user.user_id.tenant;
   }
 
   s->dialect = "s3";
@@ -2583,7 +2650,6 @@ int RGW_Auth_S3::authorize(RGWRados *store, struct req_state *s)
   } /* if keystone_result < 0 */
 
   // populate the owner info
-  s->tenant = s->user.user_id.tenant;
   s->owner.set_id(s->user.user_id);
   s->owner.set_name(s->user.display_name);
 
@@ -2606,7 +2672,7 @@ RGWHandler *RGWRESTMgr_S3::get_handler(struct req_state *s)
   if (ret < 0)
     return NULL;
 
-  if (s->bucket_name_str.empty())
+  if (s->bucket_name.empty())
     return new RGWHandler_ObjStore_Service_S3;
 
   if (s->object.empty())
diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index 35aa14669ea4..0d3aeeb9afd1 100644
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -766,9 +766,12 @@ int RGWCopyObj_ObjStore_SWIFT::get_params()
   if_match = s->info.env->get("HTTP_COPY_IF_MATCH");
   if_nomatch = s->info.env->get("HTTP_COPY_IF_NONE_MATCH");
 
+  /* XXX why copy this? just use req_state in rgw_op.cc:verify_permission */
+  src_tenant_name = s->src_tenant_name;
   src_bucket_name = s->src_bucket_name;
   src_object = s->src_object;
-  dest_bucket_name = s->bucket_name_str;
+  dest_tenant_name = s->bucket_tenant;
+  dest_bucket_name = s->bucket_name;
   dest_object = s->object.name;
 
   const char * const fresh_meta = s->info.env->get("HTTP_X_FRESH_METADATA");
@@ -819,7 +822,7 @@ void RGWCopyObj_ObjStore_SWIFT::dump_copy_info()
 
   /* Dump X-Copied-From-Account */
   string account_name;
-  url_encode(s->user.user_id, account_name);
+  url_encode(s->user.user_id.id, account_name); // XXX tenant
   s->cio->print("X-Copied-From-Account: %s\r\n", account_name.c_str());
 
   /* Dump X-Copied-From-Last-Modified. */
@@ -1197,9 +1200,11 @@ int RGWHandler_ObjStore_SWIFT::init_from_header(struct req_state *s)
   if (first.size() == 0)
     return 0;
 
-  s->bucket_name_str = first;
+  s->info.effective_uri = "/" + first;
 
-  s->info.effective_uri = "/" + s->bucket_name_str;
+  /* XXX Temporarily not parsing URL until Auth puts something in there. */
+  s->bucket_tenant = s->user.user_id.tenant;
+  s->bucket_name = first;
 
   if (req.size()) {
     s->object = rgw_obj_key(req, s->info.env->get("HTTP_X_OBJECT_VERSION_ID", "")); /* rgw swift extension */
@@ -1211,20 +1216,28 @@ int RGWHandler_ObjStore_SWIFT::init_from_header(struct req_state *s)
 
 int RGWHandler_ObjStore_SWIFT::init(RGWRados *store, struct req_state *s, RGWClientIO *cio)
 {
-  dout(10) << "s->object=" << (!s->object.empty() ? s->object : rgw_obj_key("<NULL>")) << " s->bucket=" << (!s->bucket_name_str.empty() ? s->bucket_name_str : "<NULL>") << dendl;
+  string bucket_log;
+  rgw_make_bucket_entry_name(s->bucket_tenant, s->bucket_name, bucket_log);
+  dout(10) << "s->object=" << (!s->object.empty() ? s->object : rgw_obj_key("<NULL>")) << " s->bucket=" << bucket_log << dendl;
 
-  int ret = validate_bucket_name(s->bucket_name_str.c_str());
+  int ret;
+  ret = validate_tenant_name(s->bucket_tenant);
+  if (ret)
+    return ret;
+  /* XXX The c_str is redundant here, isn't it? We do one inside. */
+  ret = validate_bucket_name(s->bucket_name.c_str());
   if (ret)
     return ret;
   ret = validate_object_name(s->object.name);
   if (ret)
     return ret;
 
-  s->copy_source = s->info.env->get("HTTP_X_COPY_FROM");
-  if (s->copy_source) {
-    bool result = RGWCopyObj::parse_copy_location(s->copy_source, s->src_bucket_name, s->src_object);
+  const char *copy_source = s->info.env->get("HTTP_X_COPY_FROM");
+  if (copy_source) {
+    bool result = RGWCopyObj::parse_copy_location(copy_source, s->src_bucket_name, s->src_object);
     if (!result)
        return -ERR_BAD_URL;
+    s->src_tenant_name = s->user.user_id.tenant;
   }
 
   s->dialect = "swift";
@@ -1234,23 +1247,30 @@ int RGWHandler_ObjStore_SWIFT::init(RGWRados *store, struct req_state *s, RGWCli
     if (!req_dest)
       return -ERR_BAD_URL;
 
-    string dest_bucket_name;
+    string dest_tenant_name, dest_bucket_name;
     rgw_obj_key dest_obj_key;
     bool result = RGWCopyObj::parse_copy_location(req_dest, dest_bucket_name, dest_obj_key);
     if (!result)
        return -ERR_BAD_URL;
+    dest_tenant_name = s->user.user_id.tenant;
 
     string dest_object = dest_obj_key.name;
-    if (dest_bucket_name != s->bucket_name_str) {
+    if (dest_bucket_name != s->bucket_name) {
       ret = validate_bucket_name(dest_bucket_name.c_str());
       if (ret < 0)
         return ret;
     }
 
+    ret = validate_tenant_name(dest_tenant_name);
+    if (ret < 0)
+      return ret;
+
     /* convert COPY operation into PUT */
-    s->src_bucket_name = s->bucket_name_str;
+    s->src_tenant_name = s->bucket_tenant;
+    s->src_bucket_name = s->bucket_name;
     s->src_object = s->object;
-    s->bucket_name_str = dest_bucket_name;
+    s->bucket_tenant = dest_tenant_name;
+    s->bucket_name = dest_bucket_name;
     s->object = rgw_obj_key(dest_object);
     s->op = OP_PUT;
   }
@@ -1265,7 +1285,7 @@ RGWHandler *RGWRESTMgr_SWIFT::get_handler(struct req_state *s)
   if (ret < 0)
     return NULL;
 
-  if (s->bucket_name_str.empty())
+  if (s->bucket_name.empty())
     return new RGWHandler_ObjStore_Service_SWIFT;
   if (s->object.empty())
     return new RGWHandler_ObjStore_Bucket_SWIFT;
diff --git a/src/rgw/rgw_swift.cc b/src/rgw/rgw_swift.cc
index f9f15e78cc88..8d73602858bd 100644
--- a/src/rgw/rgw_swift.cc
+++ b/src/rgw/rgw_swift.cc
@@ -542,7 +542,7 @@ int RGWSwift::validate_keystone_token(RGWRados *store, const string& token, stru
 int authenticate_temp_url(RGWRados *store, req_state *s)
 {
   /* temp url requires bucket and object specified in the requets */
-  if (s->bucket_name_str.empty())
+  if (s->bucket_name.empty())
     return -EPERM;
 
   if (s->object.empty())
@@ -559,7 +559,9 @@ int authenticate_temp_url(RGWRados *store, req_state *s)
   /* need to get user info of bucket owner */
   RGWBucketInfo bucket_info;
 
-  int ret = store->get_bucket_info(*static_cast<RGWObjectCtx *>(s->obj_ctx), s->bucket_name_str, bucket_info, NULL);
+  int ret = store->get_bucket_info(*static_cast<RGWObjectCtx *>(s->obj_ctx),
+                                   s->bucket_tenant, s->bucket_name,
+                                   bucket_info, NULL);
   if (ret < 0)
     return -EPERM;
 
@@ -638,8 +640,8 @@ bool RGWSwift::verify_swift_token(RGWRados *store, req_state *s)
     s->perm_mask = 0;
     map<string, RGWSubUser>::iterator iter = s->user.subusers.find(subuser);
     if (iter != s->user.subusers.end()) {
-      RGWSubUser& subuser = iter->second;
-      s->perm_mask = subuser.perm_mask;
+      RGWSubUser& subuser_ = iter->second;
+      s->perm_mask = subuser_.perm_mask;
     }
   } else {
     s->perm_mask = RGW_PERM_FULL_CONTROL;
diff --git a/src/rgw/rgw_user.cc b/src/rgw/rgw_user.cc
index f6d5c75a3f8b..5063cd0ccfc5 100644
--- a/src/rgw/rgw_user.cc
+++ b/src/rgw/rgw_user.cc
@@ -148,6 +148,8 @@ int rgw_store_user_info(RGWRados *store,
 
   RGWUID ui;
   ui.user_id = info.user_id;
+  // P3
+  ldout(store->ctx(), 0) << "DEBUG: rgw_store_user_info: user_id " << ui.user_id << dendl;
 
   bufferlist link_bl;
   ::encode(ui, link_bl);
@@ -158,7 +160,7 @@ int rgw_store_user_info(RGWRados *store,
 
   string key;
   info.user_id.to_str(key);
-  
+
   ret = store->meta_mgr->put_entry(user_meta_handler, key, data_bl, exclusive, &ot, mtime, pattrs);
   if (ret < 0)
     return ret;
@@ -344,12 +346,12 @@ extern int rgw_get_user_info_by_access_key(RGWRados *store, string& access_key,
 }
 
 int rgw_get_user_attrs_by_uid(RGWRados *store,
-                              const string& user_id,
+                              const rgw_user& user_id,
                               map<string, bufferlist>& attrs,
                               RGWObjVersionTracker *objv_tracker)
 {
   RGWObjectCtx obj_ctx(store);
-  rgw_obj obj(store->zone.user_uid_pool, user_id);
+  rgw_obj obj(store->zone.user_uid_pool, user_id.to_str());
   RGWRados::SystemObject src(store, obj_ctx, obj);
   RGWRados::SystemObject::Read rop(&src);
 
@@ -1839,7 +1841,6 @@ int RGWUser::execute_add(RGWUserAdminOpState& op_state, std::string *err_msg)
     } else {
       set_err_msg(err_msg, "user: " + op_state.user_id.to_str() + " exists");
     }
-
     return -EEXIST;
   }
 
diff --git a/src/rgw/rgw_user.h b/src/rgw/rgw_user.h
index d4992ba1dece..6e877fe80287 100644
--- a/src/rgw/rgw_user.h
+++ b/src/rgw/rgw_user.h
@@ -112,7 +112,7 @@ extern int rgw_get_user_info_by_access_key(RGWRados *store, string& access_key,
  * Returns: 0 on success, -ERR# on failure.
  */
 extern int rgw_get_user_attrs_by_uid(RGWRados *store,
-                                     const string& user_id,
+                                     const rgw_user& user_id,
                                      map<string, bufferlist>& attrs,
                                      RGWObjVersionTracker *objv_tracker = NULL);
 /**
@@ -269,7 +269,7 @@ struct RGWUserAdminOpState {
     size_t pos = _subuser.find(":");
 
     if (pos != string::npos) {
-      user_id = _subuser.substr(0, pos);
+      user_id.id = _subuser.substr(0, pos);
       subuser = _subuser.substr(pos+1);
     } else {
       subuser = _subuser;
@@ -411,10 +411,10 @@ struct RGWUserAdminOpState {
   RGWUserCaps *get_caps_obj() { return &info.caps; }
 
   std::string build_default_swift_kid() {
-    if (user_id.id.empty() || subuser.empty())
+    if (user_id.empty() || subuser.empty())
       return "";
 
-    string kid;
+    std::string kid;
     user_id.to_str(kid);
     kid.append(":");
     kid.append(subuser);
@@ -423,7 +423,7 @@ struct RGWUserAdminOpState {
   }
 
   std::string generate_subuser() {
-    if (user_id.id.empty())
+    if (user_id.empty())
       return "";
 
     std::string generated_subuser;
diff --git a/src/test/rgw/test_rgw_manifest.cc b/src/test/rgw/test_rgw_manifest.cc
index 4fb80694ed33..086e7d25f648 100644
--- a/src/test/rgw/test_rgw_manifest.cc
+++ b/src/test/rgw/test_rgw_manifest.cc
@@ -29,9 +29,9 @@
 #endif
 using namespace std;
 
-static void init_bucket(rgw_bucket *bucket, const char *name)
+static void init_bucket(rgw_bucket *bucket, const char *ten, const char *name)
 {
-  *bucket = rgw_bucket(name, ".data-pool", ".index-pool", "marker.", "bucket-id", NULL);
+  *bucket = rgw_bucket(ten, name, ".data-pool", ".index-pool", "marker.", "bucket-id", NULL);
 }
 
 void append_head(list<rgw_obj> *objs, rgw_obj& head)
@@ -62,7 +62,7 @@ static void gen_obj(uint64_t obj_size, uint64_t head_max_size, uint64_t stripe_s
 {
   manifest->set_trivial_rule(head_max_size, stripe_size);
 
-  init_bucket(bucket, "buck");
+  init_bucket(bucket, "", "buck");
 
   *head = rgw_obj(*bucket, "oid");
   gen->create_begin(g_ceph_context, manifest, *bucket, *head);
diff --git a/src/test/rgw/test_rgw_obj.cc b/src/test/rgw/test_rgw_obj.cc
index 18696a6cb36c..79159cacf05d 100644
--- a/src/test/rgw/test_rgw_obj.cc
+++ b/src/test/rgw/test_rgw_obj.cc
@@ -32,7 +32,7 @@ using namespace std;
 
 static void init_bucket(rgw_bucket *bucket, const char *name)
 {
-  *bucket = rgw_bucket(name, ".data-pool", ".index-pool", "marker", "bucket-id", NULL);
+  *bucket = rgw_bucket("", name, ".data-pool", ".index-pool", "marker", "bucket-id", NULL);
 }
 
 void check_parsed_correctly(rgw_obj& obj, const string& name, const string& ns, const string& instance)