+++ /dev/null
-#!/usr/bin/python
-
-#
-# Ceph - scalable distributed file system
-#
-# Copyright (C) 2011 New Dream Network
-#
-# This is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License version 2.1, as published by the Free Software
-# Foundation. See file COPYING.
-#
-
-"""
-boto_del.py: simple bucket deletion program
-
-A lot of common s3 clients can't delete weirdly named buckets.
-But this little script can do it!
-"""
-
-from boto.s3.connection import OrdinaryCallingFormat
-from boto.s3.connection import S3Connection
-from boto.s3.key import Key
-from sys import stderr
-import boto
-import os
-import sys
-
-bucket_name = sys.argv[1]
-conn = S3Connection(calling_format=OrdinaryCallingFormat(), is_secure=False,
- aws_access_key_id=os.environ["AKEY"],
- aws_secret_access_key=os.environ["SKEY"])
-bucket = conn.lookup(bucket_name)
-if (bucket == None):
- print "bucket '%s' no longer exists" % bucket_name
- sys.exit(0)
-
-print "deleting bucket '%s' ..." % bucket_name
-bucket.delete()
-print "done."
-sys.exit(0)
+++ /dev/null
-#!/usr/bin/env python
-
-#
-# Ceph - scalable distributed file system
-#
-# Copyright (C) 2011 New Dream Network
-#
-# This is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License version 2.1, as published by the Free Software
-# Foundation. See file COPYING.
-#
-
-"""
-objsync.py: the object synchronizer
-"""
-
-from boto.s3.connection import OrdinaryCallingFormat
-from boto.s3.connection import S3Connection
-from boto.s3.key import Key
-from optparse import OptionParser
-from sys import stderr
-import boto
-import base64
-import errno
-import hashlib
-import mimetypes
-import os
-import shutil
-import string
-import sys
-import tempfile
-import traceback
-
-global opts
-
-###### Helper functions #######
-def mkdir_p(path):
- try:
- os.makedirs(path)
- except OSError as exc:
- if exc.errno != errno.EEXIST:
- raise
- if (not os.path.isdir(path)):
- raise
-
-def bytes_to_str(b):
- return ''.join(["%02x"% ord(x) for x in b]).strip()
-
-def get_md5(f, block_size=2**20):
- md5 = hashlib.md5()
- while True:
- data = f.read(block_size)
- if not data:
- break
- md5.update(data)
- return "%s" % md5.hexdigest()
-
-def strip_prefix(prefix, s):
- if not (s[0:len(prefix)] == prefix):
- return None
- return s[len(prefix):]
-
-def etag_to_md5(etag):
- if (etag[:1] == '"'):
- start = 1
- else:
- start = 0
- if (etag[-1:] == '"'):
- end = -1
- else:
- end = None
- return etag[start:end]
-
-def getenv(a, b):
- if os.environ.has_key(a):
- return os.environ[a]
- elif os.environ.has_key(b):
- return os.environ[b]
- else:
- return None
-
-###### NonexistentStore #######
-class NonexistentStore(Exception):
- pass
-
-###### Object #######
-class Object(object):
- def __init__(self, name, md5, size):
- self.name = name
- self.md5 = md5
- self.size = int(size)
- def equals(self, rhs):
- if (self.name != rhs.name):
- return False
- if (self.md5 != rhs.md5):
- return False
- if (self.size != rhs.size):
- return False
- return True
- @staticmethod
- def from_file(obj_name, path):
- f = open(path)
- try:
- md5 = get_md5(f)
- finally:
- f.close()
- size = os.path.getsize(path)
- #print "Object.from_file: path="+path+",md5=" + bytes_to_str(md5) +",size=" + str(size)
- return Object(obj_name, md5, size)
-
-###### Store #######
-class Store(object):
- @staticmethod
- def make_store(url, create, akey, skey):
- s3_url = strip_prefix("s3://", url)
- if (s3_url):
- return S3Store(s3_url, create, akey, skey)
- file_url = strip_prefix("file://", url)
- if (file_url):
- return FileStore(file_url, create)
- if (url[0:1] == "/"):
- return FileStore(url, create)
- if (url[0:2] == "./"):
- return FileStore(url, create)
- raise Exception("Failed to find a prefix of s3://, file://, /, or ./ \
-Cannot handle this URL.")
- def __init__(self, url):
- self.url = url
-
-###### S3 store #######
-class S3StoreLocalCopy(object):
- def __init__(self, path):
- self.path = path
- def __del__(self):
- self.remove()
- def remove(self):
- if (self.path):
- os.unlink(self.path)
- self.path = None
-
-class S3StoreIterator(object):
- """S3Store iterator"""
- def __init__(self, blrs):
- self.blrs = blrs
- def __iter__(self):
- return self
- def next(self):
- # This will raise StopIteration when there are no more objects to
- # iterate on
- key = self.blrs.next()
- ret = Object(key.name, etag_to_md5(key.etag), key.size)
- return ret
-
-class S3Store(Store):
- def __init__(self, url, create, akey, skey):
- # Parse the s3 url
- host_end = string.find(url, "/")
- if (host_end == -1):
- raise Exception("S3Store URLs are of the form \
-s3://host/bucket/key_prefix. Failed to find the host.")
- self.host = url[0:host_end]
- bucket_end = url.find("/", host_end+1)
- if (bucket_end == -1):
- self.bucket_name = url[host_end+1:]
- self.key_prefix = ""
- else:
- self.bucket_name = url[host_end+1:bucket_end]
- self.key_prefix = url[bucket_end+1:]
- if (self.bucket_name == ""):
- raise Exception("S3Store URLs are of the form \
-s3://host/bucket/key_prefix. Failed to find the bucket.")
- if (opts.more_verbose):
- print "self.host = '" + self.host + "', ",
- print "self.bucket_name = '" + self.bucket_name + "' ",
- print "self.key_prefix = '" + self.key_prefix + "'"
- self.conn = S3Connection(calling_format=OrdinaryCallingFormat(),
- host=self.host, is_secure=False,
- aws_access_key_id=akey, aws_secret_access_key=skey)
- self.bucket = self.conn.lookup(self.bucket_name)
- if (self.bucket == None):
- if (create):
- if (opts.dry_run):
- raise Exception("logic error: this should be unreachable.")
- self.bucket = self.conn.create_bucket(bucket_name = self.bucket_name)
- else:
- raise RuntimeError("%s: no such bucket as %s" % \
- (url, self.bucket_name))
- Store.__init__(self, "s3://" + url)
- def __str__(self):
- return "s3://" + self.host + "/" + self.bucket_name + "/" + self.key_prefix
- def make_local_copy(self, obj):
- k = Key(self.bucket)
- k.key = obj.name
- temp_file = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
- try:
- k.get_contents_to_filename(temp_file.name)
- except:
- os.unlink(temp_file.name)
- raise
- return S3StoreLocalCopy(temp_file.name)
- def all_objects(self):
- blrs = self.bucket.list(prefix = self.key_prefix)
- return S3StoreIterator(blrs.__iter__())
- def locate_object(self, obj):
- k = self.bucket.get_key(obj.name)
- if (k == None):
- return None
- return Object(obj.name, etag_to_md5(k.etag), k.size)
- def upload(self, local_copy, obj):
- if (opts.more_verbose):
- print "UPLOAD: local_copy.path='" + local_copy.path + "' " + \
- "obj='" + obj.name + "'"
- if (opts.dry_run):
- return
-# mime = mimetypes.guess_type(local_copy.path)[0]
-# if (mime == NoneType):
-# mime = "application/octet-stream"
- k = Key(self.bucket)
- k.key = obj.name
- #k.set_metadata("Content-Type", mime)
- k.set_contents_from_filename(local_copy.path)
- def remove(self, obj):
- if (opts.dry_run):
- return
- self.bucket.delete_key(obj.name)
- if (opts.more_verbose):
- print "S3Store: removed %s" % obj.name
-
-###### FileStore #######
-class FileStoreIterator(object):
- """FileStore iterator"""
- def __init__(self, base):
- self.base = base
- self.generator = os.walk(base)
- self.path = ""
- self.files = []
- def __iter__(self):
- return self
- def next(self):
- while True:
- if (len(self.files) == 0):
- self.path, dirs, self.files = self.generator.next()
- continue
- path = self.path + "/" + self.files[0]
- self.files = self.files[1:]
- obj_name = path[len(self.base)+1:]
- if (not os.path.isfile(path)):
- continue
- return Object.from_file(obj_name, path)
-
-class FileStoreLocalCopy(object):
- def __init__(self, path):
- self.path = path
- def remove(self):
- self.path = None
-
-class FileStore(Store):
- def __init__(self, url, create):
- # Parse the file url
- self.base = url
- if (self.base[-1:] == '/'):
- self.base = self.base[:-1]
- if (create):
- if (opts.dry_run):
- raise Exception("logic error: this should be unreachable.")
- mkdir_p(self.base)
- elif (not os.path.isdir(self.base)):
- raise NonexistentStore()
- Store.__init__(self, "file://" + url)
- def __str__(self):
- return "file://" + self.base
- def make_local_copy(self, obj):
- return FileStoreLocalCopy(self.base + "/" + obj.name)
- def all_objects(self):
- return FileStoreIterator(self.base)
- def locate_object(self, obj):
- path = self.base + "/" + obj.name
- found = os.path.isfile(path)
- if (opts.more_verbose):
- if (found):
- print "FileStore::locate_object: found object '" + \
- obj.name + "'"
- else:
- print "FileStore::locate_object: did not find object '" + \
- obj.name + "'"
- if (not found):
- return None
- return Object.from_file(obj.name, path)
- def upload(self, local_copy, obj):
- if (opts.dry_run):
- return
- s = local_copy.path
- d = self.base + "/" + obj.name
- #print "s='" + s +"', d='" + d + "'"
- mkdir_p(os.path.dirname(d))
- shutil.copy(s, d)
- def remove(self, obj):
- if (opts.dry_run):
- return
- os.unlink(self.base + "/" + obj.name)
- if (opts.more_verbose):
- print "FileStore: removed %s" % obj.name
-
-###### Functions #######
-def delete_unreferenced(src, dst):
- """ delete everything from dst that is not referenced in src """
- if (opts.more_verbose):
- print "handling deletes."
- for dobj in dst.all_objects():
- sobj = src.locate_object(dobj)
- if (sobj == None):
- dst.remove(dobj)
-
-USAGE = """
-objsync synchronizes objects. The source and destination can both be local or
-both remote.
-
-Examples:
-# copy contents of mybucket to disk
-objsync -v s3://myhost/mybucket file://mydir
-
-# copy contents of mydir to an S3 bucket
-objsync -v file://mydir s3://myhost/mybucket
-
-# synchronize two S3 buckets
-SRC_AKEY=foo SRC_SKEY=foo \
-DST_AKEY=foo DST_SKEY=foo \
-objsync -v s3://myhost/mybucket1 s3://myhost/mybucket2
-
-Note: You must specify an AWS access key and secret access key when accessing
-S3. objsync honors these environment variables:
-SRC_AKEY Access key for the source URL
-SRC_SKEY Secret access key for the source URL
-DST_AKEY Access key for the destination URL
-DST_SKEY Secret access key for the destination URL
-AKEY Access key for both source and dest
-SKEY Secret access key for both source and dest
-
-If these environment variables are not given, we will fall back on libboto
-defaults.
-
-objsync (options) [source] [destination]"""
-
-parser = OptionParser(USAGE)
-parser.add_option("-n", "--dry-run", action="store_true", \
- dest="dry_run", default=False)
-parser.add_option("-S", "--source-config",
- dest="source_config", help="boto configuration file to use for the S3 source")
-parser.add_option("-D", "--dest-config",
- dest="dest_config", help="boto configuration file to use for the S3 destination")
-parser.add_option("-c", "--create-dest", action="store_true", \
- dest="create", help="create the destination if it doesn't already exist")
-parser.add_option("--delete-before", action="store_true", \
- dest="delete_before", help="delete objects that aren't in SOURCE from \
-DESTINATION before transferring any objects")
-parser.add_option("-d", "--delete-after", action="store_true", \
- dest="delete_after", help="delete objects that aren't in SOURCE from \
-DESTINATION after doing all transfers.")
-parser.add_option("-v", "--verbose", action="store_true", \
- dest="verbose", help="be verbose")
-parser.add_option("-V", "--more-verbose", action="store_true", \
- dest="more_verbose", help="be really, really verbose (developer mode)")
-(opts, args) = parser.parse_args()
-if (opts.create and opts.dry_run):
- raise Exception("You can't run with both --create-dest and --dry-run! \
-By definition, a dry run never changes anything.")
-if (len(args) < 2):
- print >>stderr, "Expected two positional arguments: source and destination"
- print >>stderr, USAGE
- sys.exit(1)
-elif (len(args) > 2):
- print >>stderr, "Too many positional arguments."
- print >>stderr, USAGE
- sys.exit(1)
-if (opts.more_verbose):
- opts.verbose = True
- boto.set_stream_logger("stdout")
- boto.log.info("Enabling verbose boto logging.")
-if (opts.delete_before and opts.delete_after):
- print >>stderr, "It doesn't make sense to specify both --delete-before \
-and --delete-after."
- sys.exit(1)
-src_name = args[0]
-dst_name = args[1]
-try:
- if (opts.more_verbose):
- print "SOURCE: " + src_name
- src = Store.make_store(src_name, False,
- getenv("SRC_AKEY", "AKEY"), getenv("SRC_SKEY", "SKEY"))
-except NonexistentStore as e:
- print >>stderr, "Fatal error: Source " + src_name + " does not exist."
- sys.exit(1)
-except Exception as e:
- print >>stderr, "error creating source: " + str(e)
- traceback.print_exc(100000, stderr)
- sys.exit(1)
-try:
- if (opts.more_verbose):
- print "DESTINATION: " + dst_name
- dst = Store.make_store(dst_name, opts.create,
- getenv("DST_AKEY", "AKEY"), getenv("DST_SKEY", "SKEY"))
-except NonexistentStore as e:
- print >>stderr, "Fatal error: Destination " + dst_name + " does " +\
- "not exist. Run with -c or --create-dest to create it automatically."
- sys.exit(1)
-except Exception as e:
- print >>stderr, "error creating destination: " + str(e)
- traceback.print_exc(100000, stderr)
- sys.exit(1)
-
-if (opts.delete_before):
- delete_unreferenced(src, dst)
-
-for sobj in src.all_objects():
- if (opts.more_verbose):
- print "handling " + sobj.name
- dobj = dst.locate_object(sobj)
- upload = False
- if (dobj == None):
- if (opts.verbose):
- print "+ " + sobj.name
- upload = True
- elif not sobj.equals(dobj):
- if (opts.verbose):
- print "> " + sobj.name
- upload = True
- else:
- if (opts.verbose):
- print ". " + sobj.name
- if (upload):
- local_copy = src.make_local_copy(sobj)
- try:
- dst.upload(local_copy, sobj)
- finally:
- local_copy.remove()
-
-if (opts.delete_after):
- delete_unreferenced(src, dst)
-
-if (opts.more_verbose):
- print "finished."
-
-sys.exit(0)
+++ /dev/null
-#!/usr/bin/env python
-
-#
-# Ceph - scalable distributed file system
-#
-# Copyright (C) 2011 New Dream Network
-#
-# This is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License version 2.1, as published by the Free Software
-# Foundation. See file COPYING.
-#
-
-"""
-objsync_test.py: a system test for objsync
-"""
-
-from optparse import OptionParser
-import atexit
-import os
-import tempfile
-import shutil
-import subprocess
-import sys
-
-global opts
-global tdir
-
-###### Helper functions #######
-def getenv(e):
- if os.environ.has_key(e):
- return os.environ[e]
- else:
- return None
-
-def objsync(src, dst, misc):
- full = ["./objsync.py"]
- e = {}
- if (isinstance(src, ObjSyncTestBucket)):
- full.append(src.url)
- e["SRC_AKEY"] = src.akey
- e["SRC_SKEY"] = src.skey
- else:
- full.append(src)
- if (isinstance(dst, ObjSyncTestBucket)):
- full.append(dst.url)
- e["DST_AKEY"] = dst.akey
- e["DST_SKEY"] = dst.skey
- else:
- full.append(dst)
- full.extend(misc)
- return subprocess.call(full, stderr=opts.error_out, env=e)
-
-def objsync_check(src, dst, opts):
- ret = objsync(src, dst, opts)
- if (ret != 0):
- raise RuntimeError("call to objsync failed!")
-
-def cleanup_tempdir():
- if tdir != None and opts.keep_tempdir == False:
- shutil.rmtree(tdir)
-
-def compare_directories(dir_a, dir_b):
- if (opts.verbose):
- print "comparing directories %s and %s" % (dir_a, dir_b)
- subprocess.check_call(["diff", "-r", dir_a, dir_b])
-
-def count_obj_in_dir(d):
- """counts the number of objects in a directory (WITHOUT recursing)"""
- num_objects = 0
- for f in os.listdir(d):
- num_objects = num_objects + 1
- return num_objects
-
-###### ObjSyncTestBucket #######
-class ObjSyncTestBucket(object):
- def __init__(self, url, akey, skey):
- self.url = url
- self.akey = akey
- self.skey = skey
-
-###### Main #######
-# change directory to osync directory
-os.chdir(os.path.dirname(os.path.abspath(__file__)))
-
-# parse options
-parser = OptionParser("""osync-test.sh
-A system test for osync.
-
-Important environment variables:
-URL1, SKEY1, AKEY1: to set up bucket1 (optional)
-URL2, SKEY2, AKEY2: to set up bucket2 (optional)""")
-parser.add_option("-k", "--keep-tempdir", action="store_true",
- dest="keep_tempdir", default=False,
- help="create the destination if it doesn't already exist")
-parser.add_option("-v", "--verbose", action="store_true",
- dest="verbose", default=False,
- help="run verbose")
-parser.add_option("-V", "--more-verbose", action="store_true", \
- dest="more_verbose", help="be really, really verbose (developer mode)")
-(opts, args) = parser.parse_args()
-if (opts.more_verbose):
- opts.verbose = True
-
-# parse environment
-opts.buckets = []
-if (not os.environ.has_key("URL1")):
- if (opts.verbose):
- print "no bucket urls were given. Running local tests only."
-elif (not os.environ.has_key("URL2")):
- opts.buckets.append(ObjSyncTestBucket(getenv("URL1"), getenv("AKEY1"),
- getenv("SKEY1")))
- if (opts.verbose):
- print "have scratch1_url: will test bucket transfers"
-else:
- opts.buckets.append(ObjSyncTestBucket(getenv("URL1"), getenv("AKEY1"),
- getenv("SKEY1")))
- opts.buckets.append(ObjSyncTestBucket(getenv("URL2"), getenv("AKEY2"),
- getenv("SKEY2")))
- if (opts.verbose):
- print "have both scratch1_url and scratch2_url: will test \
-bucket-to-bucket transfers."
-
-# set up temporary directory
-tdir = tempfile.mkdtemp()
-if (opts.verbose):
- print "created temporary directory: %s" % tdir
-atexit.register(cleanup_tempdir)
-
-# set up a little tree of files
-os.mkdir("%s/dir1" % tdir)
-os.mkdir("%s/dir1/c" % tdir)
-os.mkdir("%s/dir1/c/g" % tdir)
-f = open("%s/dir1/a" % tdir, 'w')
-f.write("a")
-f.close()
-f = open("%s/dir1/b" % tdir, 'w')
-f.close()
-f = open("%s/dir1/c/d" % tdir, 'w')
-f.write("file d!")
-f.close()
-f = open("%s/dir1/c/e" % tdir, 'w')
-f.write("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee")
-f.close()
-f = open("%s/dir1/c/f" % tdir, 'w')
-f.write("file f.")
-f.close()
-f = open("%s/dir1/c/g/h" % tdir, 'w')
-for i in range(0, 1000):
- f.write("%d." % i)
-f.close()
-
-if (opts.more_verbose):
- opts.error_out = sys.stderr
-else:
- opts.error_out = open("/dev/null", 'w')
-
-# copy this tree to somewhere else
-subprocess.check_call(["cp", "-r", "%s/dir1" % tdir, "%s/dir1a" % tdir])
-
-# make sure it's still the same
-compare_directories("%s/dir1" % tdir, "%s/dir1a" % tdir)
-
-# we should fail here, because we didn't supply -c
-ret = subprocess.call(["./osync.py", "file://%s/dir1" % tdir,
- "file://%s/dir2" % tdir], stderr=opts.error_out)
-if (ret == 0):
- raise RuntimeError("expected this call to osync to fail, because \
-we didn't supply -c. But it succeeded.")
-if (opts.verbose):
- print "first call failed as expected."
-
-# now supply -c and it should work
-ret = subprocess.check_call(["./osync.py", "-c", "file://%s/dir1" % tdir,
- "file://%s/dir2" % tdir], stderr=opts.error_out)
-compare_directories("%s/dir1" % tdir, "%s/dir2" % tdir)
-
-# test the alternate syntax where we leave off the file://, and it is assumed
-# because the url begins with / or ./
-ret = subprocess.check_call(["./osync.py", "-c", "file://%s/dir1" % tdir,
- "/%s/dir2" % tdir], stderr=opts.error_out)
-compare_directories("%s/dir1" % tdir, "%s/dir2" % tdir)
-
-if (opts.verbose):
- print "successfully created dir2 from dir1"
-
-if (opts.verbose):
- print "test a dry run between local directories"
-os.mkdir("%s/dir1b" % tdir)
-osync_check("file://%s/dir1" % tdir, "file://%s/dir1b" % tdir, ["-n"])
-if (count_obj_in_dir("/%s/dir1b" % tdir) != 0):
- raise RuntimeError("error! the dry run copied some files!")
-
-if (opts.verbose):
- print "dry run didn't do anything. good."
-osync_check("file://%s/dir1" % tdir, "file://%s/dir1b" % tdir, [])
-compare_directories("%s/dir1" % tdir, "%s/dir1b" % tdir)
-if (opts.verbose):
- print "regular run synchronized the directories."
-
-if (opts.verbose):
- print "test running without --delete-after or --delete-before..."
-osync_check("file://%s/dir1b" % tdir, "file://%s/dir1c" % tdir, ["-c"])
-os.unlink("%s/dir1b/a" % tdir)
-osync_check("/%s/dir1b" % tdir, "file://%s/dir1c" % tdir, [])
-if not os.path.exists("/%s/dir1c/a" % tdir):
- raise RuntimeError("error: running without --delete-after or \
---delete-before still deleted files from the destination!")
-if (opts.verbose):
- print "test running _with_ --delete-after..."
-osync_check("/%s/dir1b" % tdir, "file://%s/dir1c" % tdir, ["--delete-after"])
-if os.path.exists("/%s/dir1c/a" % tdir):
- raise RuntimeError("error: running with --delete-after \
-failed to delete files from the destination!")
-
-if (len(opts.buckets) >= 1):
- # first, let's empty out the S3 bucket
- os.mkdir("%s/empty1" % tdir)
- if (opts.verbose):
- print "emptying out bucket1..."
- osync_check("file://%s/empty1" % tdir, opts.buckets[0],
- ["-c", "--delete-after"])
-
- # make sure that the empty worked
- osync_check(opts.buckets[0], "file://%s/empty2" % tdir, ["-c"])
- compare_directories("%s/empty1" % tdir, "%s/empty2" % tdir)
- if (opts.verbose):
- print "successfully emptied out the bucket."
-
- if (opts.verbose):
- print "copying the sample directory to the test bucket..."
- # now copy the sample files to the test bucket
- osync_check("file://%s/dir1" % tdir, opts.buckets[0], [])
-
- # make sure that the copy worked
- osync_check(opts.buckets[0], "file://%s/dir3" % tdir, ["-c"])
- compare_directories("%s/dir1" % tdir, "%s/dir3" % tdir)
- if (opts.verbose):
- print "successfully copied the sample directory to the test bucket."
-
-if (len(opts.buckets) >= 2):
- if (opts.verbose):
- print "copying dir1 to bucket0..."
- osync_check("file://%s/dir1" % tdir, opts.buckets[0], ["--delete-before"])
- if (opts.verbose):
- print "copying bucket0 to bucket1..."
- osync_check(opts.buckets[0], opts.buckets[1], ["-c", "--delete-after"])
- if (opts.verbose):
- print "copying bucket1 to dir4..."
- osync_check(opts.buckets[1], "file://%s/dir4" % tdir, ["-c"])
- compare_directories("%s/dir1" % tdir, "%s/dir4" % tdir)
- if (opts.verbose):
- print "successfully copied one bucket to another."
- if (opts.verbose):
- print "adding another object to bucket1..."
- os.mkdir("%s/small" % tdir)
- f = open("%s/small/new_thing" % tdir, 'w')
- f.write("a new object!!!")
- f.close()
- osync_check("%s/small" % tdir, opts.buckets[1], [])
- osync_check(opts.buckets[0], "%s/bucket0_out" % tdir, ["-c"])
- osync_check(opts.buckets[1], "%s/bucket1_out" % tdir, ["-c"])
- bucket0_count = count_obj_in_dir("/%s/bucket0_out" % tdir)
- bucket1_count = count_obj_in_dir("/%s/bucket1_out" % tdir)
- if (bucket1_count != bucket0_count + 1):
- raise RuntimeError("error! expected one extra object in bucket1! \
-bucket0_count=%d, bucket1_count=%d" % (bucket0_count, bucket1_count))
- if (opts.verbose):
- print "copying bucket0 to bucket1..."
- osync_check(opts.buckets[0], opts.buckets[1], ["-c", "--delete-before"])
- osync_check(opts.buckets[0], "%s/bucket0_out" % tdir, ["--delete-after"])
- osync_check(opts.buckets[1], "%s/bucket1_out" % tdir, ["--delete-after"])
- bucket0_count = count_obj_in_dir("/%s/bucket0_out" % tdir)
- bucket1_count = count_obj_in_dir("/%s/bucket1_out" % tdir)
- if (bucket0_count != bucket1_count):
- raise RuntimeError("error! expected the same number of objects \
-in bucket0 and bucket1. bucket0_count=%d, bucket1_count=%d" \
-% (bucket0_count, bucket1_count))
-
-sys.exit(0)
--- /dev/null
+#!/usr/bin/python
+
+#
+# Ceph - scalable distributed file system
+#
+# Copyright (C) 2011 New Dream Network
+#
+# This is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1, as published by the Free Software
+# Foundation. See file COPYING.
+#
+
+"""
+boto_del.py: simple bucket deletion program
+
+A lot of common s3 clients can't delete weirdly named buckets.
+But this little script can do it!
+"""
+
+from boto.s3.connection import OrdinaryCallingFormat
+from boto.s3.connection import S3Connection
+from boto.s3.key import Key
+from sys import stderr
+import boto
+import os
+import sys
+
+bucket_name = sys.argv[1]
+conn = S3Connection(calling_format=OrdinaryCallingFormat(), is_secure=False,
+ aws_access_key_id=os.environ["AKEY"],
+ aws_secret_access_key=os.environ["SKEY"])
+bucket = conn.lookup(bucket_name)
+if (bucket == None):
+ print "bucket '%s' no longer exists" % bucket_name
+ sys.exit(0)
+
+print "deleting bucket '%s' ..." % bucket_name
+bucket.delete()
+print "done."
+sys.exit(0)
--- /dev/null
+#!/usr/bin/env python
+
+#
+# Ceph - scalable distributed file system
+#
+# Copyright (C) 2011 New Dream Network
+#
+# This is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1, as published by the Free Software
+# Foundation. See file COPYING.
+#
+
+"""
+obsync.py: the object synchronizer
+"""
+
+from boto.s3.connection import OrdinaryCallingFormat
+from boto.s3.connection import S3Connection
+from boto.s3.key import Key
+from optparse import OptionParser
+from sys import stderr
+import boto
+import base64
+import errno
+import hashlib
+import mimetypes
+import os
+import shutil
+import string
+import sys
+import tempfile
+import traceback
+
+global opts
+
+###### Helper functions #######
+def mkdir_p(path):
+ try:
+ os.makedirs(path)
+ except OSError as exc:
+ if exc.errno != errno.EEXIST:
+ raise
+ if (not os.path.isdir(path)):
+ raise
+
+def bytes_to_str(b):
+ return ''.join(["%02x"% ord(x) for x in b]).strip()
+
+def get_md5(f, block_size=2**20):
+ md5 = hashlib.md5()
+ while True:
+ data = f.read(block_size)
+ if not data:
+ break
+ md5.update(data)
+ return "%s" % md5.hexdigest()
+
+def strip_prefix(prefix, s):
+ if not (s[0:len(prefix)] == prefix):
+ return None
+ return s[len(prefix):]
+
+def etag_to_md5(etag):
+ if (etag[:1] == '"'):
+ start = 1
+ else:
+ start = 0
+ if (etag[-1:] == '"'):
+ end = -1
+ else:
+ end = None
+ return etag[start:end]
+
+def getenv(a, b):
+ if os.environ.has_key(a):
+ return os.environ[a]
+ elif os.environ.has_key(b):
+ return os.environ[b]
+ else:
+ return None
+
+###### NonexistentStore #######
+class NonexistentStore(Exception):
+ pass
+
+###### Object #######
+class Object(object):
+ def __init__(self, name, md5, size):
+ self.name = name
+ self.md5 = md5
+ self.size = int(size)
+ def equals(self, rhs):
+ if (self.name != rhs.name):
+ return False
+ if (self.md5 != rhs.md5):
+ return False
+ if (self.size != rhs.size):
+ return False
+ return True
+ @staticmethod
+ def from_file(obj_name, path):
+ f = open(path)
+ try:
+ md5 = get_md5(f)
+ finally:
+ f.close()
+ size = os.path.getsize(path)
+ #print "Object.from_file: path="+path+",md5=" + bytes_to_str(md5) +",size=" + str(size)
+ return Object(obj_name, md5, size)
+
+###### Store #######
+class Store(object):
+ @staticmethod
+ def make_store(url, create, akey, skey):
+ s3_url = strip_prefix("s3://", url)
+ if (s3_url):
+ return S3Store(s3_url, create, akey, skey)
+ file_url = strip_prefix("file://", url)
+ if (file_url):
+ return FileStore(file_url, create)
+ if (url[0:1] == "/"):
+ return FileStore(url, create)
+ if (url[0:2] == "./"):
+ return FileStore(url, create)
+ raise Exception("Failed to find a prefix of s3://, file://, /, or ./ \
+Cannot handle this URL.")
+ def __init__(self, url):
+ self.url = url
+
+###### S3 store #######
+class S3StoreLocalCopy(object):
+ def __init__(self, path):
+ self.path = path
+ def __del__(self):
+ self.remove()
+ def remove(self):
+ if (self.path):
+ os.unlink(self.path)
+ self.path = None
+
+class S3StoreIterator(object):
+ """S3Store iterator"""
+ def __init__(self, blrs):
+ self.blrs = blrs
+ def __iter__(self):
+ return self
+ def next(self):
+ # This will raise StopIteration when there are no more objects to
+ # iterate on
+ key = self.blrs.next()
+ ret = Object(key.name, etag_to_md5(key.etag), key.size)
+ return ret
+
+class S3Store(Store):
+ def __init__(self, url, create, akey, skey):
+ # Parse the s3 url
+ host_end = string.find(url, "/")
+ if (host_end == -1):
+ raise Exception("S3Store URLs are of the form \
+s3://host/bucket/key_prefix. Failed to find the host.")
+ self.host = url[0:host_end]
+ bucket_end = url.find("/", host_end+1)
+ if (bucket_end == -1):
+ self.bucket_name = url[host_end+1:]
+ self.key_prefix = ""
+ else:
+ self.bucket_name = url[host_end+1:bucket_end]
+ self.key_prefix = url[bucket_end+1:]
+ if (self.bucket_name == ""):
+ raise Exception("S3Store URLs are of the form \
+s3://host/bucket/key_prefix. Failed to find the bucket.")
+ if (opts.more_verbose):
+ print "self.host = '" + self.host + "', ",
+ print "self.bucket_name = '" + self.bucket_name + "' ",
+ print "self.key_prefix = '" + self.key_prefix + "'"
+ self.conn = S3Connection(calling_format=OrdinaryCallingFormat(),
+ host=self.host, is_secure=False,
+ aws_access_key_id=akey, aws_secret_access_key=skey)
+ self.bucket = self.conn.lookup(self.bucket_name)
+ if (self.bucket == None):
+ if (create):
+ if (opts.dry_run):
+ raise Exception("logic error: this should be unreachable.")
+ self.bucket = self.conn.create_bucket(bucket_name = self.bucket_name)
+ else:
+ raise RuntimeError("%s: no such bucket as %s" % \
+ (url, self.bucket_name))
+ Store.__init__(self, "s3://" + url)
+ def __str__(self):
+ return "s3://" + self.host + "/" + self.bucket_name + "/" + self.key_prefix
+ def make_local_copy(self, obj):
+ k = Key(self.bucket)
+ k.key = obj.name
+ temp_file = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
+ try:
+ k.get_contents_to_filename(temp_file.name)
+ except:
+ os.unlink(temp_file.name)
+ raise
+ return S3StoreLocalCopy(temp_file.name)
+ def all_objects(self):
+ blrs = self.bucket.list(prefix = self.key_prefix)
+ return S3StoreIterator(blrs.__iter__())
+ def locate_object(self, obj):
+ k = self.bucket.get_key(obj.name)
+ if (k == None):
+ return None
+ return Object(obj.name, etag_to_md5(k.etag), k.size)
+ def upload(self, local_copy, obj):
+ if (opts.more_verbose):
+ print "UPLOAD: local_copy.path='" + local_copy.path + "' " + \
+ "obj='" + obj.name + "'"
+ if (opts.dry_run):
+ return
+# mime = mimetypes.guess_type(local_copy.path)[0]
+# if (mime == NoneType):
+# mime = "application/octet-stream"
+ k = Key(self.bucket)
+ k.key = obj.name
+ #k.set_metadata("Content-Type", mime)
+ k.set_contents_from_filename(local_copy.path)
+ def remove(self, obj):
+ if (opts.dry_run):
+ return
+ self.bucket.delete_key(obj.name)
+ if (opts.more_verbose):
+ print "S3Store: removed %s" % obj.name
+
+###### FileStore #######
+class FileStoreIterator(object):
+ """FileStore iterator"""
+ def __init__(self, base):
+ self.base = base
+ self.generator = os.walk(base)
+ self.path = ""
+ self.files = []
+ def __iter__(self):
+ return self
+ def next(self):
+ while True:
+ if (len(self.files) == 0):
+ self.path, dirs, self.files = self.generator.next()
+ continue
+ path = self.path + "/" + self.files[0]
+ self.files = self.files[1:]
+ obj_name = path[len(self.base)+1:]
+ if (not os.path.isfile(path)):
+ continue
+ return Object.from_file(obj_name, path)
+
+class FileStoreLocalCopy(object):
+ def __init__(self, path):
+ self.path = path
+ def remove(self):
+ self.path = None
+
+class FileStore(Store):
+ def __init__(self, url, create):
+ # Parse the file url
+ self.base = url
+ if (self.base[-1:] == '/'):
+ self.base = self.base[:-1]
+ if (create):
+ if (opts.dry_run):
+ raise Exception("logic error: this should be unreachable.")
+ mkdir_p(self.base)
+ elif (not os.path.isdir(self.base)):
+ raise NonexistentStore()
+ Store.__init__(self, "file://" + url)
+ def __str__(self):
+ return "file://" + self.base
+ def make_local_copy(self, obj):
+ return FileStoreLocalCopy(self.base + "/" + obj.name)
+ def all_objects(self):
+ return FileStoreIterator(self.base)
+ def locate_object(self, obj):
+ path = self.base + "/" + obj.name
+ found = os.path.isfile(path)
+ if (opts.more_verbose):
+ if (found):
+ print "FileStore::locate_object: found object '" + \
+ obj.name + "'"
+ else:
+ print "FileStore::locate_object: did not find object '" + \
+ obj.name + "'"
+ if (not found):
+ return None
+ return Object.from_file(obj.name, path)
+ def upload(self, local_copy, obj):
+ if (opts.dry_run):
+ return
+ s = local_copy.path
+ d = self.base + "/" + obj.name
+ #print "s='" + s +"', d='" + d + "'"
+ mkdir_p(os.path.dirname(d))
+ shutil.copy(s, d)
+ def remove(self, obj):
+ if (opts.dry_run):
+ return
+ os.unlink(self.base + "/" + obj.name)
+ if (opts.more_verbose):
+ print "FileStore: removed %s" % obj.name
+
+###### Functions #######
+def delete_unreferenced(src, dst):
+ """ delete everything from dst that is not referenced in src """
+ if (opts.more_verbose):
+ print "handling deletes."
+ for dobj in dst.all_objects():
+ sobj = src.locate_object(dobj)
+ if (sobj == None):
+ dst.remove(dobj)
+
+USAGE = """
+obsync synchronizes objects. The source and destination can both be local or
+both remote.
+
+Examples:
+# copy contents of mybucket to disk
+obsync -v s3://myhost/mybucket file://mydir
+
+# copy contents of mydir to an S3 bucket
+obsync -v file://mydir s3://myhost/mybucket
+
+# synchronize two S3 buckets
+SRC_AKEY=foo SRC_SKEY=foo \
+DST_AKEY=foo DST_SKEY=foo \
+obsync -v s3://myhost/mybucket1 s3://myhost/mybucket2
+
+Note: You must specify an AWS access key and secret access key when accessing
+S3. obsync honors these environment variables:
+SRC_AKEY Access key for the source URL
+SRC_SKEY Secret access key for the source URL
+DST_AKEY Access key for the destination URL
+DST_SKEY Secret access key for the destination URL
+AKEY Access key for both source and dest
+SKEY Secret access key for both source and dest
+
+If these environment variables are not given, we will fall back on libboto
+defaults.
+
+obsync (options) [source] [destination]"""
+
+parser = OptionParser(USAGE)
+parser.add_option("-n", "--dry-run", action="store_true", \
+ dest="dry_run", default=False)
+parser.add_option("-S", "--source-config",
+ dest="source_config", help="boto configuration file to use for the S3 source")
+parser.add_option("-D", "--dest-config",
+ dest="dest_config", help="boto configuration file to use for the S3 destination")
+parser.add_option("-c", "--create-dest", action="store_true", \
+ dest="create", help="create the destination if it doesn't already exist")
+parser.add_option("--delete-before", action="store_true", \
+ dest="delete_before", help="delete objects that aren't in SOURCE from \
+DESTINATION before transferring any objects")
+parser.add_option("-d", "--delete-after", action="store_true", \
+ dest="delete_after", help="delete objects that aren't in SOURCE from \
+DESTINATION after doing all transfers.")
+parser.add_option("-v", "--verbose", action="store_true", \
+ dest="verbose", help="be verbose")
+parser.add_option("-V", "--more-verbose", action="store_true", \
+ dest="more_verbose", help="be really, really verbose (developer mode)")
+(opts, args) = parser.parse_args()
+if (opts.create and opts.dry_run):
+ raise Exception("You can't run with both --create-dest and --dry-run! \
+By definition, a dry run never changes anything.")
+if (len(args) < 2):
+ print >>stderr, "Expected two positional arguments: source and destination"
+ print >>stderr, USAGE
+ sys.exit(1)
+elif (len(args) > 2):
+ print >>stderr, "Too many positional arguments."
+ print >>stderr, USAGE
+ sys.exit(1)
+if (opts.more_verbose):
+ opts.verbose = True
+ boto.set_stream_logger("stdout")
+ boto.log.info("Enabling verbose boto logging.")
+if (opts.delete_before and opts.delete_after):
+ print >>stderr, "It doesn't make sense to specify both --delete-before \
+and --delete-after."
+ sys.exit(1)
+src_name = args[0]
+dst_name = args[1]
+try:
+ if (opts.more_verbose):
+ print "SOURCE: " + src_name
+ src = Store.make_store(src_name, False,
+ getenv("SRC_AKEY", "AKEY"), getenv("SRC_SKEY", "SKEY"))
+except NonexistentStore as e:
+ print >>stderr, "Fatal error: Source " + src_name + " does not exist."
+ sys.exit(1)
+except Exception as e:
+ print >>stderr, "error creating source: " + str(e)
+ traceback.print_exc(100000, stderr)
+ sys.exit(1)
+try:
+ if (opts.more_verbose):
+ print "DESTINATION: " + dst_name
+ dst = Store.make_store(dst_name, opts.create,
+ getenv("DST_AKEY", "AKEY"), getenv("DST_SKEY", "SKEY"))
+except NonexistentStore as e:
+ print >>stderr, "Fatal error: Destination " + dst_name + " does " +\
+ "not exist. Run with -c or --create-dest to create it automatically."
+ sys.exit(1)
+except Exception as e:
+ print >>stderr, "error creating destination: " + str(e)
+ traceback.print_exc(100000, stderr)
+ sys.exit(1)
+
+if (opts.delete_before):
+ delete_unreferenced(src, dst)
+
+for sobj in src.all_objects():
+ if (opts.more_verbose):
+ print "handling " + sobj.name
+ dobj = dst.locate_object(sobj)
+ upload = False
+ if (dobj == None):
+ if (opts.verbose):
+ print "+ " + sobj.name
+ upload = True
+ elif not sobj.equals(dobj):
+ if (opts.verbose):
+ print "> " + sobj.name
+ upload = True
+ else:
+ if (opts.verbose):
+ print ". " + sobj.name
+ if (upload):
+ local_copy = src.make_local_copy(sobj)
+ try:
+ dst.upload(local_copy, sobj)
+ finally:
+ local_copy.remove()
+
+if (opts.delete_after):
+ delete_unreferenced(src, dst)
+
+if (opts.more_verbose):
+ print "finished."
+
+sys.exit(0)
--- /dev/null
+#!/usr/bin/env python
+
+#
+# Ceph - scalable distributed file system
+#
+# Copyright (C) 2011 New Dream Network
+#
+# This is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1, as published by the Free Software
+# Foundation. See file COPYING.
+#
+
+"""
+obsync_test.py: a system test for obsync
+"""
+
+from optparse import OptionParser
+import atexit
+import os
+import tempfile
+import shutil
+import subprocess
+import sys
+
+global opts
+global tdir
+
+###### Helper functions #######
+def getenv(e):
+ if os.environ.has_key(e):
+ return os.environ[e]
+ else:
+ return None
+
+def obsync(src, dst, misc):
+ full = ["./obsync.py"]
+ e = {}
+ if (isinstance(src, ObSyncTestBucket)):
+ full.append(src.url)
+ e["SRC_AKEY"] = src.akey
+ e["SRC_SKEY"] = src.skey
+ else:
+ full.append(src)
+ if (isinstance(dst, ObSyncTestBucket)):
+ full.append(dst.url)
+ e["DST_AKEY"] = dst.akey
+ e["DST_SKEY"] = dst.skey
+ else:
+ full.append(dst)
+ full.extend(misc)
+ return subprocess.call(full, stderr=opts.error_out, env=e)
+
+def obsync_check(src, dst, opts):
+ ret = obsync(src, dst, opts)
+ if (ret != 0):
+ raise RuntimeError("call to obsync failed!")
+
+def cleanup_tempdir():
+ if tdir != None and opts.keep_tempdir == False:
+ shutil.rmtree(tdir)
+
+def compare_directories(dir_a, dir_b):
+ if (opts.verbose):
+ print "comparing directories %s and %s" % (dir_a, dir_b)
+ subprocess.check_call(["diff", "-r", dir_a, dir_b])
+
+def count_obj_in_dir(d):
+ """counts the number of objects in a directory (WITHOUT recursing)"""
+ num_objects = 0
+ for f in os.listdir(d):
+ num_objects = num_objects + 1
+ return num_objects
+
+###### ObSyncTestBucket #######
+class ObSyncTestBucket(object):
+ def __init__(self, url, akey, skey):
+ self.url = url
+ self.akey = akey
+ self.skey = skey
+
+###### Main #######
+# change directory to osync directory
+os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+# parse options
+parser = OptionParser("""osync-test.sh
+A system test for osync.
+
+Important environment variables:
+URL1, SKEY1, AKEY1: to set up bucket1 (optional)
+URL2, SKEY2, AKEY2: to set up bucket2 (optional)""")
+parser.add_option("-k", "--keep-tempdir", action="store_true",
+ dest="keep_tempdir", default=False,
+ help="create the destination if it doesn't already exist")
+parser.add_option("-v", "--verbose", action="store_true",
+ dest="verbose", default=False,
+ help="run verbose")
+parser.add_option("-V", "--more-verbose", action="store_true", \
+ dest="more_verbose", help="be really, really verbose (developer mode)")
+(opts, args) = parser.parse_args()
+if (opts.more_verbose):
+ opts.verbose = True
+
+# parse environment
+opts.buckets = []
+if (not os.environ.has_key("URL1")):
+ if (opts.verbose):
+ print "no bucket urls were given. Running local tests only."
+elif (not os.environ.has_key("URL2")):
+ opts.buckets.append(ObSyncTestBucket(getenv("URL1"), getenv("AKEY1"),
+ getenv("SKEY1")))
+ if (opts.verbose):
+ print "have scratch1_url: will test bucket transfers"
+else:
+ opts.buckets.append(ObSyncTestBucket(getenv("URL1"), getenv("AKEY1"),
+ getenv("SKEY1")))
+ opts.buckets.append(ObSyncTestBucket(getenv("URL2"), getenv("AKEY2"),
+ getenv("SKEY2")))
+ if (opts.verbose):
+ print "have both scratch1_url and scratch2_url: will test \
+bucket-to-bucket transfers."
+
+# set up temporary directory
+tdir = tempfile.mkdtemp()
+if (opts.verbose):
+ print "created temporary directory: %s" % tdir
+atexit.register(cleanup_tempdir)
+
+# set up a little tree of files
+os.mkdir("%s/dir1" % tdir)
+os.mkdir("%s/dir1/c" % tdir)
+os.mkdir("%s/dir1/c/g" % tdir)
+f = open("%s/dir1/a" % tdir, 'w')
+f.write("a")
+f.close()
+f = open("%s/dir1/b" % tdir, 'w')
+f.close()
+f = open("%s/dir1/c/d" % tdir, 'w')
+f.write("file d!")
+f.close()
+f = open("%s/dir1/c/e" % tdir, 'w')
+f.write("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee")
+f.close()
+f = open("%s/dir1/c/f" % tdir, 'w')
+f.write("file f.")
+f.close()
+f = open("%s/dir1/c/g/h" % tdir, 'w')
+for i in range(0, 1000):
+ f.write("%d." % i)
+f.close()
+
+if (opts.more_verbose):
+ opts.error_out = sys.stderr
+else:
+ opts.error_out = open("/dev/null", 'w')
+
+# copy this tree to somewhere else
+subprocess.check_call(["cp", "-r", "%s/dir1" % tdir, "%s/dir1a" % tdir])
+
+# make sure it's still the same
+compare_directories("%s/dir1" % tdir, "%s/dir1a" % tdir)
+
+# we should fail here, because we didn't supply -c
+ret = subprocess.call(["./osync.py", "file://%s/dir1" % tdir,
+ "file://%s/dir2" % tdir], stderr=opts.error_out)
+if (ret == 0):
+ raise RuntimeError("expected this call to osync to fail, because \
+we didn't supply -c. But it succeeded.")
+if (opts.verbose):
+ print "first call failed as expected."
+
+# now supply -c and it should work
+ret = subprocess.check_call(["./osync.py", "-c", "file://%s/dir1" % tdir,
+ "file://%s/dir2" % tdir], stderr=opts.error_out)
+compare_directories("%s/dir1" % tdir, "%s/dir2" % tdir)
+
+# test the alternate syntax where we leave off the file://, and it is assumed
+# because the url begins with / or ./
+ret = subprocess.check_call(["./osync.py", "-c", "file://%s/dir1" % tdir,
+ "/%s/dir2" % tdir], stderr=opts.error_out)
+compare_directories("%s/dir1" % tdir, "%s/dir2" % tdir)
+
+if (opts.verbose):
+ print "successfully created dir2 from dir1"
+
+if (opts.verbose):
+ print "test a dry run between local directories"
+os.mkdir("%s/dir1b" % tdir)
+osync_check("file://%s/dir1" % tdir, "file://%s/dir1b" % tdir, ["-n"])
+if (count_obj_in_dir("/%s/dir1b" % tdir) != 0):
+ raise RuntimeError("error! the dry run copied some files!")
+
+if (opts.verbose):
+ print "dry run didn't do anything. good."
+osync_check("file://%s/dir1" % tdir, "file://%s/dir1b" % tdir, [])
+compare_directories("%s/dir1" % tdir, "%s/dir1b" % tdir)
+if (opts.verbose):
+ print "regular run synchronized the directories."
+
+if (opts.verbose):
+ print "test running without --delete-after or --delete-before..."
+osync_check("file://%s/dir1b" % tdir, "file://%s/dir1c" % tdir, ["-c"])
+os.unlink("%s/dir1b/a" % tdir)
+osync_check("/%s/dir1b" % tdir, "file://%s/dir1c" % tdir, [])
+if not os.path.exists("/%s/dir1c/a" % tdir):
+ raise RuntimeError("error: running without --delete-after or \
+--delete-before still deleted files from the destination!")
+if (opts.verbose):
+ print "test running _with_ --delete-after..."
+osync_check("/%s/dir1b" % tdir, "file://%s/dir1c" % tdir, ["--delete-after"])
+if os.path.exists("/%s/dir1c/a" % tdir):
+ raise RuntimeError("error: running with --delete-after \
+failed to delete files from the destination!")
+
+if (len(opts.buckets) >= 1):
+ # first, let's empty out the S3 bucket
+ os.mkdir("%s/empty1" % tdir)
+ if (opts.verbose):
+ print "emptying out bucket1..."
+ osync_check("file://%s/empty1" % tdir, opts.buckets[0],
+ ["-c", "--delete-after"])
+
+ # make sure that the empty worked
+ osync_check(opts.buckets[0], "file://%s/empty2" % tdir, ["-c"])
+ compare_directories("%s/empty1" % tdir, "%s/empty2" % tdir)
+ if (opts.verbose):
+ print "successfully emptied out the bucket."
+
+ if (opts.verbose):
+ print "copying the sample directory to the test bucket..."
+ # now copy the sample files to the test bucket
+ osync_check("file://%s/dir1" % tdir, opts.buckets[0], [])
+
+ # make sure that the copy worked
+ osync_check(opts.buckets[0], "file://%s/dir3" % tdir, ["-c"])
+ compare_directories("%s/dir1" % tdir, "%s/dir3" % tdir)
+ if (opts.verbose):
+ print "successfully copied the sample directory to the test bucket."
+
+if (len(opts.buckets) >= 2):
+ if (opts.verbose):
+ print "copying dir1 to bucket0..."
+ osync_check("file://%s/dir1" % tdir, opts.buckets[0], ["--delete-before"])
+ if (opts.verbose):
+ print "copying bucket0 to bucket1..."
+ osync_check(opts.buckets[0], opts.buckets[1], ["-c", "--delete-after"])
+ if (opts.verbose):
+ print "copying bucket1 to dir4..."
+ osync_check(opts.buckets[1], "file://%s/dir4" % tdir, ["-c"])
+ compare_directories("%s/dir1" % tdir, "%s/dir4" % tdir)
+ if (opts.verbose):
+ print "successfully copied one bucket to another."
+ if (opts.verbose):
+ print "adding another object to bucket1..."
+ os.mkdir("%s/small" % tdir)
+ f = open("%s/small/new_thing" % tdir, 'w')
+ f.write("a new object!!!")
+ f.close()
+ osync_check("%s/small" % tdir, opts.buckets[1], [])
+ osync_check(opts.buckets[0], "%s/bucket0_out" % tdir, ["-c"])
+ osync_check(opts.buckets[1], "%s/bucket1_out" % tdir, ["-c"])
+ bucket0_count = count_obj_in_dir("/%s/bucket0_out" % tdir)
+ bucket1_count = count_obj_in_dir("/%s/bucket1_out" % tdir)
+ if (bucket1_count != bucket0_count + 1):
+ raise RuntimeError("error! expected one extra object in bucket1! \
+bucket0_count=%d, bucket1_count=%d" % (bucket0_count, bucket1_count))
+ if (opts.verbose):
+ print "copying bucket0 to bucket1..."
+ osync_check(opts.buckets[0], opts.buckets[1], ["-c", "--delete-before"])
+ osync_check(opts.buckets[0], "%s/bucket0_out" % tdir, ["--delete-after"])
+ osync_check(opts.buckets[1], "%s/bucket1_out" % tdir, ["--delete-after"])
+ bucket0_count = count_obj_in_dir("/%s/bucket0_out" % tdir)
+ bucket1_count = count_obj_in_dir("/%s/bucket1_out" % tdir)
+ if (bucket0_count != bucket1_count):
+ raise RuntimeError("error! expected the same number of objects \
+in bucket0 and bucket1. bucket0_count=%d, bucket1_count=%d" \
+% (bucket0_count, bucket1_count))
+
+sys.exit(0)
projects / ceph.git / commitdiff