obsync: add DST_CONSISTENCY

The DST_CONSISTENCY variable allows us to specify that the destination
is expected to use read-after-write consistency. If that is the case, we
don't have to do slow retries if certain operations fail.

Signed-off-by: Colin McCabe <colin.mccabe@dreamhost.com>

diff --git a/src/obsync/obsync.py b/src/obsync/obsync.py
index da0b6ace98fa302ac205c96efd985adcd2fd900e..7a5d433da2c6a258e69c003257b6eb54415c7c3f 100755 (executable)
--- a/src/obsync/obsync.py
+++ b/src/obsync/obsync.py
@@ -544,6 +544,7 @@ s3://host/bucket/key_prefix. Failed to find the bucket.")
         #k.set_metadata("Content-Type", mime)
         k.set_contents_from_filename(local_copy.path)
         if (src_acl.acl_policy != None):
+            ex = None
             for retry_num in S3RetryIterator.create():
                 try:
                     xml = src_acl.acl_policy.to_xml()
@@ -583,6 +584,9 @@ class S3RetryIterator(object):
         return self
     def next(self):
         t = self.cur_try
+        if (os.environ.has_key("DST_CONSISTENCY") and \
+                os.environ["DST_CONSISTENCY"] == "read_after_write"):
+            raise StopIteration
         if (self.cur_try >= len(self.try_times)):
             raise StopIteration
         sleep_time = self.try_times[self.cur_try]

diff --git a/src/obsync/test-obsync.py b/src/obsync/test-obsync.py
index a97d7b30cba763a2d18e8e675b2e78bd69965429..c5f00ff3db5a5c9002422ee542f467edfeb426f2 100755 (executable)
--- a/src/obsync/test-obsync.py
+++ b/src/obsync/test-obsync.py
@@ -66,7 +66,7 @@ def read_config():
 
         config[name] = {}
         for var in [ 'access_key', 'host', 'secret_key', 'user_id',
-                     'display_name', 'email', ]:
+                     'display_name', 'email', 'consistency', ]:
             try:
                 config[name][var] = cfg.get(section, var)
             except ConfigParser.NoOptionError:
@@ -111,6 +111,8 @@ def obsync(src, dst, misc):
         full.append(dst.url)
         e["DST_AKEY"] = dst.akey
         e["DST_SKEY"] = dst.skey
+        if (dst.consistency != None):
+            e["DST_CONSISTENCY"] = dst.consistency
     else:
         full.append(dst)
     full.extend(misc)
@@ -162,13 +164,22 @@ def count_obj_in_dir(d):
 def xuser(src, dst):
     return [ "--xuser", config[src]["user_id"] + "=" + config[dst]["user_id"]]
 
+def get_optional(h, k):
+    if (h.has_key(k)):
+        print "found " + str(h[k])
+        return h[k]
+    else:
+        print "found nothing"
+        return None
+
 ###### ObSyncTestBucket #######
 class ObSyncTestBucket(object):
-    def __init__(self, name, url, akey, skey):
+    def __init__(self, name, url, akey, skey, consistency):
         self.name = name
         self.url = url
         self.akey = akey
         self.skey = skey
+        self.consistency = consistency
 
 ###### Main #######
 # change directory to obsync directory
@@ -201,10 +212,12 @@ config = read_config()
 opts.buckets = []
 opts.buckets.append(ObSyncTestBucket(config["main"]["bucket_name"], \
     "s3://" + config["main"]["host"] + "/" + config["main"]["bucket_name"], \
-    config["main"]["access_key"], config["main"]["secret_key"]))
+    config["main"]["access_key"], config["main"]["secret_key"],
+    get_optional(config["main"], "consistency")))
 opts.buckets.append(ObSyncTestBucket(config["alt"]["bucket_name"], \
     "s3://" + config["alt"]["host"] + "/" + config["alt"]["bucket_name"], \
-    config["alt"]["access_key"], config["alt"]["secret_key"]))
+    config["alt"]["access_key"], config["alt"]["secret_key"],
+    get_optional(config["alt"], "consistency")))
 
 if not config["main"]["user_id"]:
     raise Exception("You must specify a user_id for the main section.")