From: Colin Patrick McCabe Date: Wed, 23 Mar 2011 16:29:57 +0000 (-0700) Subject: Rename objsync -> obsync X-Git-Tag: v0.26~62^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=6788c3c0e3b0f515fc9bcfdc85b53ea2b8f70d1e;p=ceph.git Rename objsync -> obsync Signed-off-by: Colin McCabe --- diff --git a/src/objsync/boto_del.py b/src/objsync/boto_del.py deleted file mode 100755 index 14e790544ec3..000000000000 --- a/src/objsync/boto_del.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/python - -# -# Ceph - scalable distributed file system -# -# Copyright (C) 2011 New Dream Network -# -# This is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License version 2.1, as published by the Free Software -# Foundation. See file COPYING. -# - -""" -boto_del.py: simple bucket deletion program - -A lot of common s3 clients can't delete weirdly named buckets. -But this little script can do it! -""" - -from boto.s3.connection import OrdinaryCallingFormat -from boto.s3.connection import S3Connection -from boto.s3.key import Key -from sys import stderr -import boto -import os -import sys - -bucket_name = sys.argv[1] -conn = S3Connection(calling_format=OrdinaryCallingFormat(), is_secure=False, - aws_access_key_id=os.environ["AKEY"], - aws_secret_access_key=os.environ["SKEY"]) -bucket = conn.lookup(bucket_name) -if (bucket == None): - print "bucket '%s' no longer exists" % bucket_name - sys.exit(0) - -print "deleting bucket '%s' ..." % bucket_name -bucket.delete() -print "done." -sys.exit(0) diff --git a/src/objsync/objsync.py b/src/objsync/objsync.py deleted file mode 100755 index 7347c29652e2..000000000000 --- a/src/objsync/objsync.py +++ /dev/null @@ -1,444 +0,0 @@ -#!/usr/bin/env python - -# -# Ceph - scalable distributed file system -# -# Copyright (C) 2011 New Dream Network -# -# This is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License version 2.1, as published by the Free Software -# Foundation. See file COPYING. -# - -""" -objsync.py: the object synchronizer -""" - -from boto.s3.connection import OrdinaryCallingFormat -from boto.s3.connection import S3Connection -from boto.s3.key import Key -from optparse import OptionParser -from sys import stderr -import boto -import base64 -import errno -import hashlib -import mimetypes -import os -import shutil -import string -import sys -import tempfile -import traceback - -global opts - -###### Helper functions ####### -def mkdir_p(path): - try: - os.makedirs(path) - except OSError as exc: - if exc.errno != errno.EEXIST: - raise - if (not os.path.isdir(path)): - raise - -def bytes_to_str(b): - return ''.join(["%02x"% ord(x) for x in b]).strip() - -def get_md5(f, block_size=2**20): - md5 = hashlib.md5() - while True: - data = f.read(block_size) - if not data: - break - md5.update(data) - return "%s" % md5.hexdigest() - -def strip_prefix(prefix, s): - if not (s[0:len(prefix)] == prefix): - return None - return s[len(prefix):] - -def etag_to_md5(etag): - if (etag[:1] == '"'): - start = 1 - else: - start = 0 - if (etag[-1:] == '"'): - end = -1 - else: - end = None - return etag[start:end] - -def getenv(a, b): - if os.environ.has_key(a): - return os.environ[a] - elif os.environ.has_key(b): - return os.environ[b] - else: - return None - -###### NonexistentStore ####### -class NonexistentStore(Exception): - pass - -###### Object ####### -class Object(object): - def __init__(self, name, md5, size): - self.name = name - self.md5 = md5 - self.size = int(size) - def equals(self, rhs): - if (self.name != rhs.name): - return False - if (self.md5 != rhs.md5): - return False - if (self.size != rhs.size): - return False - return True - @staticmethod - def from_file(obj_name, path): - f = open(path) - try: - md5 = get_md5(f) - finally: - f.close() - size = os.path.getsize(path) - #print "Object.from_file: path="+path+",md5=" + bytes_to_str(md5) +",size=" + str(size) - return Object(obj_name, md5, size) - -###### Store ####### -class Store(object): - @staticmethod - def make_store(url, create, akey, skey): - s3_url = strip_prefix("s3://", url) - if (s3_url): - return S3Store(s3_url, create, akey, skey) - file_url = strip_prefix("file://", url) - if (file_url): - return FileStore(file_url, create) - if (url[0:1] == "/"): - return FileStore(url, create) - if (url[0:2] == "./"): - return FileStore(url, create) - raise Exception("Failed to find a prefix of s3://, file://, /, or ./ \ -Cannot handle this URL.") - def __init__(self, url): - self.url = url - -###### S3 store ####### -class S3StoreLocalCopy(object): - def __init__(self, path): - self.path = path - def __del__(self): - self.remove() - def remove(self): - if (self.path): - os.unlink(self.path) - self.path = None - -class S3StoreIterator(object): - """S3Store iterator""" - def __init__(self, blrs): - self.blrs = blrs - def __iter__(self): - return self - def next(self): - # This will raise StopIteration when there are no more objects to - # iterate on - key = self.blrs.next() - ret = Object(key.name, etag_to_md5(key.etag), key.size) - return ret - -class S3Store(Store): - def __init__(self, url, create, akey, skey): - # Parse the s3 url - host_end = string.find(url, "/") - if (host_end == -1): - raise Exception("S3Store URLs are of the form \ -s3://host/bucket/key_prefix. Failed to find the host.") - self.host = url[0:host_end] - bucket_end = url.find("/", host_end+1) - if (bucket_end == -1): - self.bucket_name = url[host_end+1:] - self.key_prefix = "" - else: - self.bucket_name = url[host_end+1:bucket_end] - self.key_prefix = url[bucket_end+1:] - if (self.bucket_name == ""): - raise Exception("S3Store URLs are of the form \ -s3://host/bucket/key_prefix. Failed to find the bucket.") - if (opts.more_verbose): - print "self.host = '" + self.host + "', ", - print "self.bucket_name = '" + self.bucket_name + "' ", - print "self.key_prefix = '" + self.key_prefix + "'" - self.conn = S3Connection(calling_format=OrdinaryCallingFormat(), - host=self.host, is_secure=False, - aws_access_key_id=akey, aws_secret_access_key=skey) - self.bucket = self.conn.lookup(self.bucket_name) - if (self.bucket == None): - if (create): - if (opts.dry_run): - raise Exception("logic error: this should be unreachable.") - self.bucket = self.conn.create_bucket(bucket_name = self.bucket_name) - else: - raise RuntimeError("%s: no such bucket as %s" % \ - (url, self.bucket_name)) - Store.__init__(self, "s3://" + url) - def __str__(self): - return "s3://" + self.host + "/" + self.bucket_name + "/" + self.key_prefix - def make_local_copy(self, obj): - k = Key(self.bucket) - k.key = obj.name - temp_file = tempfile.NamedTemporaryFile(mode='w+b', delete=False) - try: - k.get_contents_to_filename(temp_file.name) - except: - os.unlink(temp_file.name) - raise - return S3StoreLocalCopy(temp_file.name) - def all_objects(self): - blrs = self.bucket.list(prefix = self.key_prefix) - return S3StoreIterator(blrs.__iter__()) - def locate_object(self, obj): - k = self.bucket.get_key(obj.name) - if (k == None): - return None - return Object(obj.name, etag_to_md5(k.etag), k.size) - def upload(self, local_copy, obj): - if (opts.more_verbose): - print "UPLOAD: local_copy.path='" + local_copy.path + "' " + \ - "obj='" + obj.name + "'" - if (opts.dry_run): - return -# mime = mimetypes.guess_type(local_copy.path)[0] -# if (mime == NoneType): -# mime = "application/octet-stream" - k = Key(self.bucket) - k.key = obj.name - #k.set_metadata("Content-Type", mime) - k.set_contents_from_filename(local_copy.path) - def remove(self, obj): - if (opts.dry_run): - return - self.bucket.delete_key(obj.name) - if (opts.more_verbose): - print "S3Store: removed %s" % obj.name - -###### FileStore ####### -class FileStoreIterator(object): - """FileStore iterator""" - def __init__(self, base): - self.base = base - self.generator = os.walk(base) - self.path = "" - self.files = [] - def __iter__(self): - return self - def next(self): - while True: - if (len(self.files) == 0): - self.path, dirs, self.files = self.generator.next() - continue - path = self.path + "/" + self.files[0] - self.files = self.files[1:] - obj_name = path[len(self.base)+1:] - if (not os.path.isfile(path)): - continue - return Object.from_file(obj_name, path) - -class FileStoreLocalCopy(object): - def __init__(self, path): - self.path = path - def remove(self): - self.path = None - -class FileStore(Store): - def __init__(self, url, create): - # Parse the file url - self.base = url - if (self.base[-1:] == '/'): - self.base = self.base[:-1] - if (create): - if (opts.dry_run): - raise Exception("logic error: this should be unreachable.") - mkdir_p(self.base) - elif (not os.path.isdir(self.base)): - raise NonexistentStore() - Store.__init__(self, "file://" + url) - def __str__(self): - return "file://" + self.base - def make_local_copy(self, obj): - return FileStoreLocalCopy(self.base + "/" + obj.name) - def all_objects(self): - return FileStoreIterator(self.base) - def locate_object(self, obj): - path = self.base + "/" + obj.name - found = os.path.isfile(path) - if (opts.more_verbose): - if (found): - print "FileStore::locate_object: found object '" + \ - obj.name + "'" - else: - print "FileStore::locate_object: did not find object '" + \ - obj.name + "'" - if (not found): - return None - return Object.from_file(obj.name, path) - def upload(self, local_copy, obj): - if (opts.dry_run): - return - s = local_copy.path - d = self.base + "/" + obj.name - #print "s='" + s +"', d='" + d + "'" - mkdir_p(os.path.dirname(d)) - shutil.copy(s, d) - def remove(self, obj): - if (opts.dry_run): - return - os.unlink(self.base + "/" + obj.name) - if (opts.more_verbose): - print "FileStore: removed %s" % obj.name - -###### Functions ####### -def delete_unreferenced(src, dst): - """ delete everything from dst that is not referenced in src """ - if (opts.more_verbose): - print "handling deletes." - for dobj in dst.all_objects(): - sobj = src.locate_object(dobj) - if (sobj == None): - dst.remove(dobj) - -USAGE = """ -objsync synchronizes objects. The source and destination can both be local or -both remote. - -Examples: -# copy contents of mybucket to disk -objsync -v s3://myhost/mybucket file://mydir - -# copy contents of mydir to an S3 bucket -objsync -v file://mydir s3://myhost/mybucket - -# synchronize two S3 buckets -SRC_AKEY=foo SRC_SKEY=foo \ -DST_AKEY=foo DST_SKEY=foo \ -objsync -v s3://myhost/mybucket1 s3://myhost/mybucket2 - -Note: You must specify an AWS access key and secret access key when accessing -S3. objsync honors these environment variables: -SRC_AKEY Access key for the source URL -SRC_SKEY Secret access key for the source URL -DST_AKEY Access key for the destination URL -DST_SKEY Secret access key for the destination URL -AKEY Access key for both source and dest -SKEY Secret access key for both source and dest - -If these environment variables are not given, we will fall back on libboto -defaults. - -objsync (options) [source] [destination]""" - -parser = OptionParser(USAGE) -parser.add_option("-n", "--dry-run", action="store_true", \ - dest="dry_run", default=False) -parser.add_option("-S", "--source-config", - dest="source_config", help="boto configuration file to use for the S3 source") -parser.add_option("-D", "--dest-config", - dest="dest_config", help="boto configuration file to use for the S3 destination") -parser.add_option("-c", "--create-dest", action="store_true", \ - dest="create", help="create the destination if it doesn't already exist") -parser.add_option("--delete-before", action="store_true", \ - dest="delete_before", help="delete objects that aren't in SOURCE from \ -DESTINATION before transferring any objects") -parser.add_option("-d", "--delete-after", action="store_true", \ - dest="delete_after", help="delete objects that aren't in SOURCE from \ -DESTINATION after doing all transfers.") -parser.add_option("-v", "--verbose", action="store_true", \ - dest="verbose", help="be verbose") -parser.add_option("-V", "--more-verbose", action="store_true", \ - dest="more_verbose", help="be really, really verbose (developer mode)") -(opts, args) = parser.parse_args() -if (opts.create and opts.dry_run): - raise Exception("You can't run with both --create-dest and --dry-run! \ -By definition, a dry run never changes anything.") -if (len(args) < 2): - print >>stderr, "Expected two positional arguments: source and destination" - print >>stderr, USAGE - sys.exit(1) -elif (len(args) > 2): - print >>stderr, "Too many positional arguments." - print >>stderr, USAGE - sys.exit(1) -if (opts.more_verbose): - opts.verbose = True - boto.set_stream_logger("stdout") - boto.log.info("Enabling verbose boto logging.") -if (opts.delete_before and opts.delete_after): - print >>stderr, "It doesn't make sense to specify both --delete-before \ -and --delete-after." - sys.exit(1) -src_name = args[0] -dst_name = args[1] -try: - if (opts.more_verbose): - print "SOURCE: " + src_name - src = Store.make_store(src_name, False, - getenv("SRC_AKEY", "AKEY"), getenv("SRC_SKEY", "SKEY")) -except NonexistentStore as e: - print >>stderr, "Fatal error: Source " + src_name + " does not exist." - sys.exit(1) -except Exception as e: - print >>stderr, "error creating source: " + str(e) - traceback.print_exc(100000, stderr) - sys.exit(1) -try: - if (opts.more_verbose): - print "DESTINATION: " + dst_name - dst = Store.make_store(dst_name, opts.create, - getenv("DST_AKEY", "AKEY"), getenv("DST_SKEY", "SKEY")) -except NonexistentStore as e: - print >>stderr, "Fatal error: Destination " + dst_name + " does " +\ - "not exist. Run with -c or --create-dest to create it automatically." - sys.exit(1) -except Exception as e: - print >>stderr, "error creating destination: " + str(e) - traceback.print_exc(100000, stderr) - sys.exit(1) - -if (opts.delete_before): - delete_unreferenced(src, dst) - -for sobj in src.all_objects(): - if (opts.more_verbose): - print "handling " + sobj.name - dobj = dst.locate_object(sobj) - upload = False - if (dobj == None): - if (opts.verbose): - print "+ " + sobj.name - upload = True - elif not sobj.equals(dobj): - if (opts.verbose): - print "> " + sobj.name - upload = True - else: - if (opts.verbose): - print ". " + sobj.name - if (upload): - local_copy = src.make_local_copy(sobj) - try: - dst.upload(local_copy, sobj) - finally: - local_copy.remove() - -if (opts.delete_after): - delete_unreferenced(src, dst) - -if (opts.more_verbose): - print "finished." - -sys.exit(0) diff --git a/src/objsync/test-objsync.py b/src/objsync/test-objsync.py deleted file mode 100755 index 6b990a726c06..000000000000 --- a/src/objsync/test-objsync.py +++ /dev/null @@ -1,280 +0,0 @@ -#!/usr/bin/env python - -# -# Ceph - scalable distributed file system -# -# Copyright (C) 2011 New Dream Network -# -# This is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License version 2.1, as published by the Free Software -# Foundation. See file COPYING. -# - -""" -objsync_test.py: a system test for objsync -""" - -from optparse import OptionParser -import atexit -import os -import tempfile -import shutil -import subprocess -import sys - -global opts -global tdir - -###### Helper functions ####### -def getenv(e): - if os.environ.has_key(e): - return os.environ[e] - else: - return None - -def objsync(src, dst, misc): - full = ["./objsync.py"] - e = {} - if (isinstance(src, ObjSyncTestBucket)): - full.append(src.url) - e["SRC_AKEY"] = src.akey - e["SRC_SKEY"] = src.skey - else: - full.append(src) - if (isinstance(dst, ObjSyncTestBucket)): - full.append(dst.url) - e["DST_AKEY"] = dst.akey - e["DST_SKEY"] = dst.skey - else: - full.append(dst) - full.extend(misc) - return subprocess.call(full, stderr=opts.error_out, env=e) - -def objsync_check(src, dst, opts): - ret = objsync(src, dst, opts) - if (ret != 0): - raise RuntimeError("call to objsync failed!") - -def cleanup_tempdir(): - if tdir != None and opts.keep_tempdir == False: - shutil.rmtree(tdir) - -def compare_directories(dir_a, dir_b): - if (opts.verbose): - print "comparing directories %s and %s" % (dir_a, dir_b) - subprocess.check_call(["diff", "-r", dir_a, dir_b]) - -def count_obj_in_dir(d): - """counts the number of objects in a directory (WITHOUT recursing)""" - num_objects = 0 - for f in os.listdir(d): - num_objects = num_objects + 1 - return num_objects - -###### ObjSyncTestBucket ####### -class ObjSyncTestBucket(object): - def __init__(self, url, akey, skey): - self.url = url - self.akey = akey - self.skey = skey - -###### Main ####### -# change directory to osync directory -os.chdir(os.path.dirname(os.path.abspath(__file__))) - -# parse options -parser = OptionParser("""osync-test.sh -A system test for osync. - -Important environment variables: -URL1, SKEY1, AKEY1: to set up bucket1 (optional) -URL2, SKEY2, AKEY2: to set up bucket2 (optional)""") -parser.add_option("-k", "--keep-tempdir", action="store_true", - dest="keep_tempdir", default=False, - help="create the destination if it doesn't already exist") -parser.add_option("-v", "--verbose", action="store_true", - dest="verbose", default=False, - help="run verbose") -parser.add_option("-V", "--more-verbose", action="store_true", \ - dest="more_verbose", help="be really, really verbose (developer mode)") -(opts, args) = parser.parse_args() -if (opts.more_verbose): - opts.verbose = True - -# parse environment -opts.buckets = [] -if (not os.environ.has_key("URL1")): - if (opts.verbose): - print "no bucket urls were given. Running local tests only." -elif (not os.environ.has_key("URL2")): - opts.buckets.append(ObjSyncTestBucket(getenv("URL1"), getenv("AKEY1"), - getenv("SKEY1"))) - if (opts.verbose): - print "have scratch1_url: will test bucket transfers" -else: - opts.buckets.append(ObjSyncTestBucket(getenv("URL1"), getenv("AKEY1"), - getenv("SKEY1"))) - opts.buckets.append(ObjSyncTestBucket(getenv("URL2"), getenv("AKEY2"), - getenv("SKEY2"))) - if (opts.verbose): - print "have both scratch1_url and scratch2_url: will test \ -bucket-to-bucket transfers." - -# set up temporary directory -tdir = tempfile.mkdtemp() -if (opts.verbose): - print "created temporary directory: %s" % tdir -atexit.register(cleanup_tempdir) - -# set up a little tree of files -os.mkdir("%s/dir1" % tdir) -os.mkdir("%s/dir1/c" % tdir) -os.mkdir("%s/dir1/c/g" % tdir) -f = open("%s/dir1/a" % tdir, 'w') -f.write("a") -f.close() -f = open("%s/dir1/b" % tdir, 'w') -f.close() -f = open("%s/dir1/c/d" % tdir, 'w') -f.write("file d!") -f.close() -f = open("%s/dir1/c/e" % tdir, 'w') -f.write("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee") -f.close() -f = open("%s/dir1/c/f" % tdir, 'w') -f.write("file f.") -f.close() -f = open("%s/dir1/c/g/h" % tdir, 'w') -for i in range(0, 1000): - f.write("%d." % i) -f.close() - -if (opts.more_verbose): - opts.error_out = sys.stderr -else: - opts.error_out = open("/dev/null", 'w') - -# copy this tree to somewhere else -subprocess.check_call(["cp", "-r", "%s/dir1" % tdir, "%s/dir1a" % tdir]) - -# make sure it's still the same -compare_directories("%s/dir1" % tdir, "%s/dir1a" % tdir) - -# we should fail here, because we didn't supply -c -ret = subprocess.call(["./osync.py", "file://%s/dir1" % tdir, - "file://%s/dir2" % tdir], stderr=opts.error_out) -if (ret == 0): - raise RuntimeError("expected this call to osync to fail, because \ -we didn't supply -c. But it succeeded.") -if (opts.verbose): - print "first call failed as expected." - -# now supply -c and it should work -ret = subprocess.check_call(["./osync.py", "-c", "file://%s/dir1" % tdir, - "file://%s/dir2" % tdir], stderr=opts.error_out) -compare_directories("%s/dir1" % tdir, "%s/dir2" % tdir) - -# test the alternate syntax where we leave off the file://, and it is assumed -# because the url begins with / or ./ -ret = subprocess.check_call(["./osync.py", "-c", "file://%s/dir1" % tdir, - "/%s/dir2" % tdir], stderr=opts.error_out) -compare_directories("%s/dir1" % tdir, "%s/dir2" % tdir) - -if (opts.verbose): - print "successfully created dir2 from dir1" - -if (opts.verbose): - print "test a dry run between local directories" -os.mkdir("%s/dir1b" % tdir) -osync_check("file://%s/dir1" % tdir, "file://%s/dir1b" % tdir, ["-n"]) -if (count_obj_in_dir("/%s/dir1b" % tdir) != 0): - raise RuntimeError("error! the dry run copied some files!") - -if (opts.verbose): - print "dry run didn't do anything. good." -osync_check("file://%s/dir1" % tdir, "file://%s/dir1b" % tdir, []) -compare_directories("%s/dir1" % tdir, "%s/dir1b" % tdir) -if (opts.verbose): - print "regular run synchronized the directories." - -if (opts.verbose): - print "test running without --delete-after or --delete-before..." -osync_check("file://%s/dir1b" % tdir, "file://%s/dir1c" % tdir, ["-c"]) -os.unlink("%s/dir1b/a" % tdir) -osync_check("/%s/dir1b" % tdir, "file://%s/dir1c" % tdir, []) -if not os.path.exists("/%s/dir1c/a" % tdir): - raise RuntimeError("error: running without --delete-after or \ ---delete-before still deleted files from the destination!") -if (opts.verbose): - print "test running _with_ --delete-after..." -osync_check("/%s/dir1b" % tdir, "file://%s/dir1c" % tdir, ["--delete-after"]) -if os.path.exists("/%s/dir1c/a" % tdir): - raise RuntimeError("error: running with --delete-after \ -failed to delete files from the destination!") - -if (len(opts.buckets) >= 1): - # first, let's empty out the S3 bucket - os.mkdir("%s/empty1" % tdir) - if (opts.verbose): - print "emptying out bucket1..." - osync_check("file://%s/empty1" % tdir, opts.buckets[0], - ["-c", "--delete-after"]) - - # make sure that the empty worked - osync_check(opts.buckets[0], "file://%s/empty2" % tdir, ["-c"]) - compare_directories("%s/empty1" % tdir, "%s/empty2" % tdir) - if (opts.verbose): - print "successfully emptied out the bucket." - - if (opts.verbose): - print "copying the sample directory to the test bucket..." - # now copy the sample files to the test bucket - osync_check("file://%s/dir1" % tdir, opts.buckets[0], []) - - # make sure that the copy worked - osync_check(opts.buckets[0], "file://%s/dir3" % tdir, ["-c"]) - compare_directories("%s/dir1" % tdir, "%s/dir3" % tdir) - if (opts.verbose): - print "successfully copied the sample directory to the test bucket." - -if (len(opts.buckets) >= 2): - if (opts.verbose): - print "copying dir1 to bucket0..." - osync_check("file://%s/dir1" % tdir, opts.buckets[0], ["--delete-before"]) - if (opts.verbose): - print "copying bucket0 to bucket1..." - osync_check(opts.buckets[0], opts.buckets[1], ["-c", "--delete-after"]) - if (opts.verbose): - print "copying bucket1 to dir4..." - osync_check(opts.buckets[1], "file://%s/dir4" % tdir, ["-c"]) - compare_directories("%s/dir1" % tdir, "%s/dir4" % tdir) - if (opts.verbose): - print "successfully copied one bucket to another." - if (opts.verbose): - print "adding another object to bucket1..." - os.mkdir("%s/small" % tdir) - f = open("%s/small/new_thing" % tdir, 'w') - f.write("a new object!!!") - f.close() - osync_check("%s/small" % tdir, opts.buckets[1], []) - osync_check(opts.buckets[0], "%s/bucket0_out" % tdir, ["-c"]) - osync_check(opts.buckets[1], "%s/bucket1_out" % tdir, ["-c"]) - bucket0_count = count_obj_in_dir("/%s/bucket0_out" % tdir) - bucket1_count = count_obj_in_dir("/%s/bucket1_out" % tdir) - if (bucket1_count != bucket0_count + 1): - raise RuntimeError("error! expected one extra object in bucket1! \ -bucket0_count=%d, bucket1_count=%d" % (bucket0_count, bucket1_count)) - if (opts.verbose): - print "copying bucket0 to bucket1..." - osync_check(opts.buckets[0], opts.buckets[1], ["-c", "--delete-before"]) - osync_check(opts.buckets[0], "%s/bucket0_out" % tdir, ["--delete-after"]) - osync_check(opts.buckets[1], "%s/bucket1_out" % tdir, ["--delete-after"]) - bucket0_count = count_obj_in_dir("/%s/bucket0_out" % tdir) - bucket1_count = count_obj_in_dir("/%s/bucket1_out" % tdir) - if (bucket0_count != bucket1_count): - raise RuntimeError("error! expected the same number of objects \ -in bucket0 and bucket1. bucket0_count=%d, bucket1_count=%d" \ -% (bucket0_count, bucket1_count)) - -sys.exit(0) diff --git a/src/obsync/boto_del.py b/src/obsync/boto_del.py new file mode 100755 index 000000000000..14e790544ec3 --- /dev/null +++ b/src/obsync/boto_del.py @@ -0,0 +1,41 @@ +#!/usr/bin/python + +# +# Ceph - scalable distributed file system +# +# Copyright (C) 2011 New Dream Network +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1, as published by the Free Software +# Foundation. See file COPYING. +# + +""" +boto_del.py: simple bucket deletion program + +A lot of common s3 clients can't delete weirdly named buckets. +But this little script can do it! +""" + +from boto.s3.connection import OrdinaryCallingFormat +from boto.s3.connection import S3Connection +from boto.s3.key import Key +from sys import stderr +import boto +import os +import sys + +bucket_name = sys.argv[1] +conn = S3Connection(calling_format=OrdinaryCallingFormat(), is_secure=False, + aws_access_key_id=os.environ["AKEY"], + aws_secret_access_key=os.environ["SKEY"]) +bucket = conn.lookup(bucket_name) +if (bucket == None): + print "bucket '%s' no longer exists" % bucket_name + sys.exit(0) + +print "deleting bucket '%s' ..." % bucket_name +bucket.delete() +print "done." +sys.exit(0) diff --git a/src/obsync/obsync.py b/src/obsync/obsync.py new file mode 100755 index 000000000000..f4ddb40c6322 --- /dev/null +++ b/src/obsync/obsync.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python + +# +# Ceph - scalable distributed file system +# +# Copyright (C) 2011 New Dream Network +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1, as published by the Free Software +# Foundation. See file COPYING. +# + +""" +obsync.py: the object synchronizer +""" + +from boto.s3.connection import OrdinaryCallingFormat +from boto.s3.connection import S3Connection +from boto.s3.key import Key +from optparse import OptionParser +from sys import stderr +import boto +import base64 +import errno +import hashlib +import mimetypes +import os +import shutil +import string +import sys +import tempfile +import traceback + +global opts + +###### Helper functions ####### +def mkdir_p(path): + try: + os.makedirs(path) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + if (not os.path.isdir(path)): + raise + +def bytes_to_str(b): + return ''.join(["%02x"% ord(x) for x in b]).strip() + +def get_md5(f, block_size=2**20): + md5 = hashlib.md5() + while True: + data = f.read(block_size) + if not data: + break + md5.update(data) + return "%s" % md5.hexdigest() + +def strip_prefix(prefix, s): + if not (s[0:len(prefix)] == prefix): + return None + return s[len(prefix):] + +def etag_to_md5(etag): + if (etag[:1] == '"'): + start = 1 + else: + start = 0 + if (etag[-1:] == '"'): + end = -1 + else: + end = None + return etag[start:end] + +def getenv(a, b): + if os.environ.has_key(a): + return os.environ[a] + elif os.environ.has_key(b): + return os.environ[b] + else: + return None + +###### NonexistentStore ####### +class NonexistentStore(Exception): + pass + +###### Object ####### +class Object(object): + def __init__(self, name, md5, size): + self.name = name + self.md5 = md5 + self.size = int(size) + def equals(self, rhs): + if (self.name != rhs.name): + return False + if (self.md5 != rhs.md5): + return False + if (self.size != rhs.size): + return False + return True + @staticmethod + def from_file(obj_name, path): + f = open(path) + try: + md5 = get_md5(f) + finally: + f.close() + size = os.path.getsize(path) + #print "Object.from_file: path="+path+",md5=" + bytes_to_str(md5) +",size=" + str(size) + return Object(obj_name, md5, size) + +###### Store ####### +class Store(object): + @staticmethod + def make_store(url, create, akey, skey): + s3_url = strip_prefix("s3://", url) + if (s3_url): + return S3Store(s3_url, create, akey, skey) + file_url = strip_prefix("file://", url) + if (file_url): + return FileStore(file_url, create) + if (url[0:1] == "/"): + return FileStore(url, create) + if (url[0:2] == "./"): + return FileStore(url, create) + raise Exception("Failed to find a prefix of s3://, file://, /, or ./ \ +Cannot handle this URL.") + def __init__(self, url): + self.url = url + +###### S3 store ####### +class S3StoreLocalCopy(object): + def __init__(self, path): + self.path = path + def __del__(self): + self.remove() + def remove(self): + if (self.path): + os.unlink(self.path) + self.path = None + +class S3StoreIterator(object): + """S3Store iterator""" + def __init__(self, blrs): + self.blrs = blrs + def __iter__(self): + return self + def next(self): + # This will raise StopIteration when there are no more objects to + # iterate on + key = self.blrs.next() + ret = Object(key.name, etag_to_md5(key.etag), key.size) + return ret + +class S3Store(Store): + def __init__(self, url, create, akey, skey): + # Parse the s3 url + host_end = string.find(url, "/") + if (host_end == -1): + raise Exception("S3Store URLs are of the form \ +s3://host/bucket/key_prefix. Failed to find the host.") + self.host = url[0:host_end] + bucket_end = url.find("/", host_end+1) + if (bucket_end == -1): + self.bucket_name = url[host_end+1:] + self.key_prefix = "" + else: + self.bucket_name = url[host_end+1:bucket_end] + self.key_prefix = url[bucket_end+1:] + if (self.bucket_name == ""): + raise Exception("S3Store URLs are of the form \ +s3://host/bucket/key_prefix. Failed to find the bucket.") + if (opts.more_verbose): + print "self.host = '" + self.host + "', ", + print "self.bucket_name = '" + self.bucket_name + "' ", + print "self.key_prefix = '" + self.key_prefix + "'" + self.conn = S3Connection(calling_format=OrdinaryCallingFormat(), + host=self.host, is_secure=False, + aws_access_key_id=akey, aws_secret_access_key=skey) + self.bucket = self.conn.lookup(self.bucket_name) + if (self.bucket == None): + if (create): + if (opts.dry_run): + raise Exception("logic error: this should be unreachable.") + self.bucket = self.conn.create_bucket(bucket_name = self.bucket_name) + else: + raise RuntimeError("%s: no such bucket as %s" % \ + (url, self.bucket_name)) + Store.__init__(self, "s3://" + url) + def __str__(self): + return "s3://" + self.host + "/" + self.bucket_name + "/" + self.key_prefix + def make_local_copy(self, obj): + k = Key(self.bucket) + k.key = obj.name + temp_file = tempfile.NamedTemporaryFile(mode='w+b', delete=False) + try: + k.get_contents_to_filename(temp_file.name) + except: + os.unlink(temp_file.name) + raise + return S3StoreLocalCopy(temp_file.name) + def all_objects(self): + blrs = self.bucket.list(prefix = self.key_prefix) + return S3StoreIterator(blrs.__iter__()) + def locate_object(self, obj): + k = self.bucket.get_key(obj.name) + if (k == None): + return None + return Object(obj.name, etag_to_md5(k.etag), k.size) + def upload(self, local_copy, obj): + if (opts.more_verbose): + print "UPLOAD: local_copy.path='" + local_copy.path + "' " + \ + "obj='" + obj.name + "'" + if (opts.dry_run): + return +# mime = mimetypes.guess_type(local_copy.path)[0] +# if (mime == NoneType): +# mime = "application/octet-stream" + k = Key(self.bucket) + k.key = obj.name + #k.set_metadata("Content-Type", mime) + k.set_contents_from_filename(local_copy.path) + def remove(self, obj): + if (opts.dry_run): + return + self.bucket.delete_key(obj.name) + if (opts.more_verbose): + print "S3Store: removed %s" % obj.name + +###### FileStore ####### +class FileStoreIterator(object): + """FileStore iterator""" + def __init__(self, base): + self.base = base + self.generator = os.walk(base) + self.path = "" + self.files = [] + def __iter__(self): + return self + def next(self): + while True: + if (len(self.files) == 0): + self.path, dirs, self.files = self.generator.next() + continue + path = self.path + "/" + self.files[0] + self.files = self.files[1:] + obj_name = path[len(self.base)+1:] + if (not os.path.isfile(path)): + continue + return Object.from_file(obj_name, path) + +class FileStoreLocalCopy(object): + def __init__(self, path): + self.path = path + def remove(self): + self.path = None + +class FileStore(Store): + def __init__(self, url, create): + # Parse the file url + self.base = url + if (self.base[-1:] == '/'): + self.base = self.base[:-1] + if (create): + if (opts.dry_run): + raise Exception("logic error: this should be unreachable.") + mkdir_p(self.base) + elif (not os.path.isdir(self.base)): + raise NonexistentStore() + Store.__init__(self, "file://" + url) + def __str__(self): + return "file://" + self.base + def make_local_copy(self, obj): + return FileStoreLocalCopy(self.base + "/" + obj.name) + def all_objects(self): + return FileStoreIterator(self.base) + def locate_object(self, obj): + path = self.base + "/" + obj.name + found = os.path.isfile(path) + if (opts.more_verbose): + if (found): + print "FileStore::locate_object: found object '" + \ + obj.name + "'" + else: + print "FileStore::locate_object: did not find object '" + \ + obj.name + "'" + if (not found): + return None + return Object.from_file(obj.name, path) + def upload(self, local_copy, obj): + if (opts.dry_run): + return + s = local_copy.path + d = self.base + "/" + obj.name + #print "s='" + s +"', d='" + d + "'" + mkdir_p(os.path.dirname(d)) + shutil.copy(s, d) + def remove(self, obj): + if (opts.dry_run): + return + os.unlink(self.base + "/" + obj.name) + if (opts.more_verbose): + print "FileStore: removed %s" % obj.name + +###### Functions ####### +def delete_unreferenced(src, dst): + """ delete everything from dst that is not referenced in src """ + if (opts.more_verbose): + print "handling deletes." + for dobj in dst.all_objects(): + sobj = src.locate_object(dobj) + if (sobj == None): + dst.remove(dobj) + +USAGE = """ +obsync synchronizes objects. The source and destination can both be local or +both remote. + +Examples: +# copy contents of mybucket to disk +obsync -v s3://myhost/mybucket file://mydir + +# copy contents of mydir to an S3 bucket +obsync -v file://mydir s3://myhost/mybucket + +# synchronize two S3 buckets +SRC_AKEY=foo SRC_SKEY=foo \ +DST_AKEY=foo DST_SKEY=foo \ +obsync -v s3://myhost/mybucket1 s3://myhost/mybucket2 + +Note: You must specify an AWS access key and secret access key when accessing +S3. obsync honors these environment variables: +SRC_AKEY Access key for the source URL +SRC_SKEY Secret access key for the source URL +DST_AKEY Access key for the destination URL +DST_SKEY Secret access key for the destination URL +AKEY Access key for both source and dest +SKEY Secret access key for both source and dest + +If these environment variables are not given, we will fall back on libboto +defaults. + +obsync (options) [source] [destination]""" + +parser = OptionParser(USAGE) +parser.add_option("-n", "--dry-run", action="store_true", \ + dest="dry_run", default=False) +parser.add_option("-S", "--source-config", + dest="source_config", help="boto configuration file to use for the S3 source") +parser.add_option("-D", "--dest-config", + dest="dest_config", help="boto configuration file to use for the S3 destination") +parser.add_option("-c", "--create-dest", action="store_true", \ + dest="create", help="create the destination if it doesn't already exist") +parser.add_option("--delete-before", action="store_true", \ + dest="delete_before", help="delete objects that aren't in SOURCE from \ +DESTINATION before transferring any objects") +parser.add_option("-d", "--delete-after", action="store_true", \ + dest="delete_after", help="delete objects that aren't in SOURCE from \ +DESTINATION after doing all transfers.") +parser.add_option("-v", "--verbose", action="store_true", \ + dest="verbose", help="be verbose") +parser.add_option("-V", "--more-verbose", action="store_true", \ + dest="more_verbose", help="be really, really verbose (developer mode)") +(opts, args) = parser.parse_args() +if (opts.create and opts.dry_run): + raise Exception("You can't run with both --create-dest and --dry-run! \ +By definition, a dry run never changes anything.") +if (len(args) < 2): + print >>stderr, "Expected two positional arguments: source and destination" + print >>stderr, USAGE + sys.exit(1) +elif (len(args) > 2): + print >>stderr, "Too many positional arguments." + print >>stderr, USAGE + sys.exit(1) +if (opts.more_verbose): + opts.verbose = True + boto.set_stream_logger("stdout") + boto.log.info("Enabling verbose boto logging.") +if (opts.delete_before and opts.delete_after): + print >>stderr, "It doesn't make sense to specify both --delete-before \ +and --delete-after." + sys.exit(1) +src_name = args[0] +dst_name = args[1] +try: + if (opts.more_verbose): + print "SOURCE: " + src_name + src = Store.make_store(src_name, False, + getenv("SRC_AKEY", "AKEY"), getenv("SRC_SKEY", "SKEY")) +except NonexistentStore as e: + print >>stderr, "Fatal error: Source " + src_name + " does not exist." + sys.exit(1) +except Exception as e: + print >>stderr, "error creating source: " + str(e) + traceback.print_exc(100000, stderr) + sys.exit(1) +try: + if (opts.more_verbose): + print "DESTINATION: " + dst_name + dst = Store.make_store(dst_name, opts.create, + getenv("DST_AKEY", "AKEY"), getenv("DST_SKEY", "SKEY")) +except NonexistentStore as e: + print >>stderr, "Fatal error: Destination " + dst_name + " does " +\ + "not exist. Run with -c or --create-dest to create it automatically." + sys.exit(1) +except Exception as e: + print >>stderr, "error creating destination: " + str(e) + traceback.print_exc(100000, stderr) + sys.exit(1) + +if (opts.delete_before): + delete_unreferenced(src, dst) + +for sobj in src.all_objects(): + if (opts.more_verbose): + print "handling " + sobj.name + dobj = dst.locate_object(sobj) + upload = False + if (dobj == None): + if (opts.verbose): + print "+ " + sobj.name + upload = True + elif not sobj.equals(dobj): + if (opts.verbose): + print "> " + sobj.name + upload = True + else: + if (opts.verbose): + print ". " + sobj.name + if (upload): + local_copy = src.make_local_copy(sobj) + try: + dst.upload(local_copy, sobj) + finally: + local_copy.remove() + +if (opts.delete_after): + delete_unreferenced(src, dst) + +if (opts.more_verbose): + print "finished." + +sys.exit(0) diff --git a/src/obsync/test-obsync.py b/src/obsync/test-obsync.py new file mode 100755 index 000000000000..df77658e30db --- /dev/null +++ b/src/obsync/test-obsync.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python + +# +# Ceph - scalable distributed file system +# +# Copyright (C) 2011 New Dream Network +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1, as published by the Free Software +# Foundation. See file COPYING. +# + +""" +obsync_test.py: a system test for obsync +""" + +from optparse import OptionParser +import atexit +import os +import tempfile +import shutil +import subprocess +import sys + +global opts +global tdir + +###### Helper functions ####### +def getenv(e): + if os.environ.has_key(e): + return os.environ[e] + else: + return None + +def obsync(src, dst, misc): + full = ["./obsync.py"] + e = {} + if (isinstance(src, ObSyncTestBucket)): + full.append(src.url) + e["SRC_AKEY"] = src.akey + e["SRC_SKEY"] = src.skey + else: + full.append(src) + if (isinstance(dst, ObSyncTestBucket)): + full.append(dst.url) + e["DST_AKEY"] = dst.akey + e["DST_SKEY"] = dst.skey + else: + full.append(dst) + full.extend(misc) + return subprocess.call(full, stderr=opts.error_out, env=e) + +def obsync_check(src, dst, opts): + ret = obsync(src, dst, opts) + if (ret != 0): + raise RuntimeError("call to obsync failed!") + +def cleanup_tempdir(): + if tdir != None and opts.keep_tempdir == False: + shutil.rmtree(tdir) + +def compare_directories(dir_a, dir_b): + if (opts.verbose): + print "comparing directories %s and %s" % (dir_a, dir_b) + subprocess.check_call(["diff", "-r", dir_a, dir_b]) + +def count_obj_in_dir(d): + """counts the number of objects in a directory (WITHOUT recursing)""" + num_objects = 0 + for f in os.listdir(d): + num_objects = num_objects + 1 + return num_objects + +###### ObSyncTestBucket ####### +class ObSyncTestBucket(object): + def __init__(self, url, akey, skey): + self.url = url + self.akey = akey + self.skey = skey + +###### Main ####### +# change directory to osync directory +os.chdir(os.path.dirname(os.path.abspath(__file__))) + +# parse options +parser = OptionParser("""osync-test.sh +A system test for osync. + +Important environment variables: +URL1, SKEY1, AKEY1: to set up bucket1 (optional) +URL2, SKEY2, AKEY2: to set up bucket2 (optional)""") +parser.add_option("-k", "--keep-tempdir", action="store_true", + dest="keep_tempdir", default=False, + help="create the destination if it doesn't already exist") +parser.add_option("-v", "--verbose", action="store_true", + dest="verbose", default=False, + help="run verbose") +parser.add_option("-V", "--more-verbose", action="store_true", \ + dest="more_verbose", help="be really, really verbose (developer mode)") +(opts, args) = parser.parse_args() +if (opts.more_verbose): + opts.verbose = True + +# parse environment +opts.buckets = [] +if (not os.environ.has_key("URL1")): + if (opts.verbose): + print "no bucket urls were given. Running local tests only." +elif (not os.environ.has_key("URL2")): + opts.buckets.append(ObSyncTestBucket(getenv("URL1"), getenv("AKEY1"), + getenv("SKEY1"))) + if (opts.verbose): + print "have scratch1_url: will test bucket transfers" +else: + opts.buckets.append(ObSyncTestBucket(getenv("URL1"), getenv("AKEY1"), + getenv("SKEY1"))) + opts.buckets.append(ObSyncTestBucket(getenv("URL2"), getenv("AKEY2"), + getenv("SKEY2"))) + if (opts.verbose): + print "have both scratch1_url and scratch2_url: will test \ +bucket-to-bucket transfers." + +# set up temporary directory +tdir = tempfile.mkdtemp() +if (opts.verbose): + print "created temporary directory: %s" % tdir +atexit.register(cleanup_tempdir) + +# set up a little tree of files +os.mkdir("%s/dir1" % tdir) +os.mkdir("%s/dir1/c" % tdir) +os.mkdir("%s/dir1/c/g" % tdir) +f = open("%s/dir1/a" % tdir, 'w') +f.write("a") +f.close() +f = open("%s/dir1/b" % tdir, 'w') +f.close() +f = open("%s/dir1/c/d" % tdir, 'w') +f.write("file d!") +f.close() +f = open("%s/dir1/c/e" % tdir, 'w') +f.write("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee") +f.close() +f = open("%s/dir1/c/f" % tdir, 'w') +f.write("file f.") +f.close() +f = open("%s/dir1/c/g/h" % tdir, 'w') +for i in range(0, 1000): + f.write("%d." % i) +f.close() + +if (opts.more_verbose): + opts.error_out = sys.stderr +else: + opts.error_out = open("/dev/null", 'w') + +# copy this tree to somewhere else +subprocess.check_call(["cp", "-r", "%s/dir1" % tdir, "%s/dir1a" % tdir]) + +# make sure it's still the same +compare_directories("%s/dir1" % tdir, "%s/dir1a" % tdir) + +# we should fail here, because we didn't supply -c +ret = subprocess.call(["./osync.py", "file://%s/dir1" % tdir, + "file://%s/dir2" % tdir], stderr=opts.error_out) +if (ret == 0): + raise RuntimeError("expected this call to osync to fail, because \ +we didn't supply -c. But it succeeded.") +if (opts.verbose): + print "first call failed as expected." + +# now supply -c and it should work +ret = subprocess.check_call(["./osync.py", "-c", "file://%s/dir1" % tdir, + "file://%s/dir2" % tdir], stderr=opts.error_out) +compare_directories("%s/dir1" % tdir, "%s/dir2" % tdir) + +# test the alternate syntax where we leave off the file://, and it is assumed +# because the url begins with / or ./ +ret = subprocess.check_call(["./osync.py", "-c", "file://%s/dir1" % tdir, + "/%s/dir2" % tdir], stderr=opts.error_out) +compare_directories("%s/dir1" % tdir, "%s/dir2" % tdir) + +if (opts.verbose): + print "successfully created dir2 from dir1" + +if (opts.verbose): + print "test a dry run between local directories" +os.mkdir("%s/dir1b" % tdir) +osync_check("file://%s/dir1" % tdir, "file://%s/dir1b" % tdir, ["-n"]) +if (count_obj_in_dir("/%s/dir1b" % tdir) != 0): + raise RuntimeError("error! the dry run copied some files!") + +if (opts.verbose): + print "dry run didn't do anything. good." +osync_check("file://%s/dir1" % tdir, "file://%s/dir1b" % tdir, []) +compare_directories("%s/dir1" % tdir, "%s/dir1b" % tdir) +if (opts.verbose): + print "regular run synchronized the directories." + +if (opts.verbose): + print "test running without --delete-after or --delete-before..." +osync_check("file://%s/dir1b" % tdir, "file://%s/dir1c" % tdir, ["-c"]) +os.unlink("%s/dir1b/a" % tdir) +osync_check("/%s/dir1b" % tdir, "file://%s/dir1c" % tdir, []) +if not os.path.exists("/%s/dir1c/a" % tdir): + raise RuntimeError("error: running without --delete-after or \ +--delete-before still deleted files from the destination!") +if (opts.verbose): + print "test running _with_ --delete-after..." +osync_check("/%s/dir1b" % tdir, "file://%s/dir1c" % tdir, ["--delete-after"]) +if os.path.exists("/%s/dir1c/a" % tdir): + raise RuntimeError("error: running with --delete-after \ +failed to delete files from the destination!") + +if (len(opts.buckets) >= 1): + # first, let's empty out the S3 bucket + os.mkdir("%s/empty1" % tdir) + if (opts.verbose): + print "emptying out bucket1..." + osync_check("file://%s/empty1" % tdir, opts.buckets[0], + ["-c", "--delete-after"]) + + # make sure that the empty worked + osync_check(opts.buckets[0], "file://%s/empty2" % tdir, ["-c"]) + compare_directories("%s/empty1" % tdir, "%s/empty2" % tdir) + if (opts.verbose): + print "successfully emptied out the bucket." + + if (opts.verbose): + print "copying the sample directory to the test bucket..." + # now copy the sample files to the test bucket + osync_check("file://%s/dir1" % tdir, opts.buckets[0], []) + + # make sure that the copy worked + osync_check(opts.buckets[0], "file://%s/dir3" % tdir, ["-c"]) + compare_directories("%s/dir1" % tdir, "%s/dir3" % tdir) + if (opts.verbose): + print "successfully copied the sample directory to the test bucket." + +if (len(opts.buckets) >= 2): + if (opts.verbose): + print "copying dir1 to bucket0..." + osync_check("file://%s/dir1" % tdir, opts.buckets[0], ["--delete-before"]) + if (opts.verbose): + print "copying bucket0 to bucket1..." + osync_check(opts.buckets[0], opts.buckets[1], ["-c", "--delete-after"]) + if (opts.verbose): + print "copying bucket1 to dir4..." + osync_check(opts.buckets[1], "file://%s/dir4" % tdir, ["-c"]) + compare_directories("%s/dir1" % tdir, "%s/dir4" % tdir) + if (opts.verbose): + print "successfully copied one bucket to another." + if (opts.verbose): + print "adding another object to bucket1..." + os.mkdir("%s/small" % tdir) + f = open("%s/small/new_thing" % tdir, 'w') + f.write("a new object!!!") + f.close() + osync_check("%s/small" % tdir, opts.buckets[1], []) + osync_check(opts.buckets[0], "%s/bucket0_out" % tdir, ["-c"]) + osync_check(opts.buckets[1], "%s/bucket1_out" % tdir, ["-c"]) + bucket0_count = count_obj_in_dir("/%s/bucket0_out" % tdir) + bucket1_count = count_obj_in_dir("/%s/bucket1_out" % tdir) + if (bucket1_count != bucket0_count + 1): + raise RuntimeError("error! expected one extra object in bucket1! \ +bucket0_count=%d, bucket1_count=%d" % (bucket0_count, bucket1_count)) + if (opts.verbose): + print "copying bucket0 to bucket1..." + osync_check(opts.buckets[0], opts.buckets[1], ["-c", "--delete-before"]) + osync_check(opts.buckets[0], "%s/bucket0_out" % tdir, ["--delete-after"]) + osync_check(opts.buckets[1], "%s/bucket1_out" % tdir, ["--delete-after"]) + bucket0_count = count_obj_in_dir("/%s/bucket0_out" % tdir) + bucket1_count = count_obj_in_dir("/%s/bucket1_out" % tdir) + if (bucket0_count != bucket1_count): + raise RuntimeError("error! expected the same number of objects \ +in bucket0 and bucket1. bucket0_count=%d, bucket1_count=%d" \ +% (bucket0_count, bucket1_count)) + +sys.exit(0)