obsync: fix bucket creation through rgw target

The rgw: target can now create buckets. Add a test.

Signed-off-by: Colin McCabe <colin.mccabe@dreamhost.com>

diff --git a/src/obsync/obsync b/src/obsync/obsync
index b91f2c86f0f071396e965e6d42bbe855af53bba9..0d58719ae0b2e2f7ecc8536c61a5a63d1ea9236e 100755 (executable)
--- a/src/obsync/obsync
+++ b/src/obsync/obsync
@@ -835,7 +835,7 @@ rgw:/path/to/ceph/conf:pool:key_prefix. Failed to find the bucket.")
                 self.create_rgw_bucket(self.rgw_bucket_name)
             else:
                 raise NonexistentStore()
-        else:
+        elif self.owner == None:
             # Figure out what owner we should use when creating objects.
             # We use the owner of the destination bucket
             ioctx = self.rados.open_ioctx(RGW_META_BUCKET_NAME)
@@ -843,9 +843,9 @@ rgw:/path/to/ceph/conf:pool:key_prefix. Failed to find the bucket.")
                 bin_ = ioctx.get_xattr(self.rgw_bucket_name, RGW_META_ACL)
                 xml = lrgw.acl_bin2xml(bin_)
                 acl = AclPolicy.from_xml(xml)
-                self.bucket_owner = acl.owner_id
+                self.owner = acl.owner_id
                 if (opts.more_verbose):
-                    print "using owner \"%s\"" % self.bucket_owner
+                    print "using owner \"%s\"" % self.owner
             finally:
                ioctx.close()
         self.ioctx = self.rados.open_ioctx(self.rgw_bucket_name)
@@ -853,7 +853,7 @@ rgw:/path/to/ceph/conf:pool:key_prefix. Failed to find the bucket.")
     def create_rgw_bucket(self, rgw_bucket_name):
         global lrgw
         """ Create an rgw bucket named 'rgw_bucket_name' """
-        if (self.bucket_owner == None):
+        if (self.owner == None):
             raise Exception("Can't create a bucket without knowing who " +
                     "should own it. Please set DST_OWNER")
         self.rados.create_pool(self.rgw_bucket_name)

diff --git a/src/obsync/test-obsync.py b/src/obsync/test-obsync.py
index f9237efdf11c9361bcbc3d772439c9a1d369e6cc..8aebf6dc10ce52a0dd716800f229f958ffae6430 100755 (executable)
--- a/src/obsync/test-obsync.py
+++ b/src/obsync/test-obsync.py
@@ -49,6 +49,16 @@ def get_nonce():
     else:
         return random.randint(9999, 99999)
 
+def get_s3_connection(conf):
+    return boto.s3.connection.S3Connection(
+            aws_access_key_id = conf["access_key"],
+            aws_secret_access_key = conf["secret_key"],
+            host = conf["host"],
+            # TODO support & test all variations
+            calling_format=boto.s3.connection.OrdinaryCallingFormat(),
+            is_secure=False,
+            )
+
 def read_s3_config(cfg, section, sconfig, name):
     # TODO: support 'port', 'is_secure'
     sconfig[name] = {}
@@ -60,13 +70,7 @@ def read_s3_config(cfg, section, sconfig, name):
             pass
     # Make sure connection works
     try:
-        conn = boto.s3.connection.S3Connection(
-            aws_access_key_id = sconfig[name]["access_key"],
-            aws_secret_access_key = sconfig[name]["secret_key"],
-            host = sconfig[name]["host"],
-            # TODO support & test all variations
-            calling_format=boto.s3.connection.OrdinaryCallingFormat(),
-            )
+        conn = get_s3_connection(sconfig[name])
     except Exception, e:
         print >>stderr, "error initializing connection!"
         raise
@@ -269,12 +273,13 @@ def assert_xattr(file_name, meta):
 
 ###### ObSyncTestBucket #######
 class ObSyncTestBucket(object):
-    def __init__(self, name, url, akey, skey, consistency):
-        self.name = name
-        self.url = url
-        self.akey = akey
-        self.skey = skey
-        self.consistency = consistency
+    def __init__(self, conf):
+        self.conf = conf
+        self.name = conf["bucket_name"]
+        self.url = "s3://" + conf["host"] + "/" + conf["bucket_name"]
+        self.akey = conf["access_key"]
+        self.skey = conf["secret_key"]
+        self.consistency = get_optional(conf, "consistency")
     def to_src(self, env, args):
         env["SRC_AKEY"] = self.akey
         env["SRC_SKEY"] = self.skey
@@ -327,14 +332,8 @@ if (opts.more_verbose):
 # parse configuration file
 sconfig, rconfig = read_config()
 opts.buckets = []
-opts.buckets.append(ObSyncTestBucket(sconfig["main"]["bucket_name"], \
-    "s3://" + sconfig["main"]["host"] + "/" + sconfig["main"]["bucket_name"], \
-    sconfig["main"]["access_key"], sconfig["main"]["secret_key"],
-    get_optional(sconfig["main"], "consistency")))
-opts.buckets.append(ObSyncTestBucket(sconfig["alt"]["bucket_name"], \
-    "s3://" + sconfig["alt"]["host"] + "/" + sconfig["alt"]["bucket_name"], \
-    sconfig["alt"]["access_key"], sconfig["alt"]["secret_key"],
-    get_optional(sconfig["alt"], "consistency")))
+opts.buckets.append(ObSyncTestBucket(sconfig["main"]))
+opts.buckets.append(ObSyncTestBucket(sconfig["alt"]))
 
 opts.pools = []
 if (rconfig.has_key("main")):
@@ -548,9 +547,14 @@ if (opts.verbose):
 obsync_check("file://%s/empty1" % tdir, opts.buckets[0],
             ["--delete-after"])
 
+def rmbucket(bucket):
+    conn = get_s3_connection(bucket.conf)
+    bucket = conn.get_bucket(bucket.name)
+    bucket.delete()
+
 # rgw target tests
 if len(opts.pools) > 0:
-    print "testing rgw target"
+    rmbucket(opts.buckets[0])
     os.mkdir("%s/rgw1" % tdir)
     f = open("%s/rgw1/aaa" % tdir, 'w')
     f.write("aaa")
@@ -562,8 +566,16 @@ if len(opts.pools) > 0:
     f = open("%s/rgw1/brick" % tdir, 'w')
     f.write("br\0ick")
     f.close()
-    obsync_check("%s/rgw1" % tdir, opts.pools[0], [])
-    print "testing rgw source"
+    # we should fail here, because we didn't supply -c, and the bucket
+    # doesn't exist
+    ret = obsync("%s/rgw1" % tdir, opts.pools[0], [])
+    if (ret == 0):
+        raise RuntimeError("expected this call to obsync to fail, because \
+    we didn't supply -c. But it succeeded.")
+    if (opts.verbose):
+        print "first rgw: call failed as expected."
+    print "testing rgw target with --create"
+    obsync_check("%s/rgw1" % tdir, opts.pools[0], ["--create"])
     obsync_check(opts.pools[0], "%s/rgw2" % tdir, ["-c"])
     compare_directories("%s/rgw1" % tdir, "%s/rgw2" % tdir, compare_xattr = False)
     # some tests with xattrs
@@ -574,9 +586,6 @@ if len(opts.pools) > 0:
     obsync_check(opts.pools[0], "%s/rgw3" % tdir, ["-c"])
     compare_directories("%s/rgw2" % tdir, "%s/rgw3" % tdir, compare_xattr = True)
 
-#    print "testing rgw target with --create"
-#    obsync_check("%s/rgw1" % tdir, opts.pools[0], ["--create"])
-
 # test escaping
 os.mkdir("%s/escape_dir1" % tdir)
 f = open("%s/escape_dir1/$$foo" % tdir, 'w')