+++ /dev/null
-import bunch
-import collections
-import gevent
-import random
-import time
-
-from ..common import context, get_next_key
-from ..common.results import TransferGreenletResult
-from ..realistic import FileVerifier
-
-
-# Make sure context has somewhere to store what we need
-context.update(bunch.Bunch(
- needs_first_read=collections.deque(),
- all_keys=[],
- files_iter=None,
-))
-
-
-class SafeTransferGreenlet(gevent.Greenlet):
- def __init__(self, timeout=120):
- gevent.Greenlet.__init__(self)
- self.timeout = timeout
- self.result = None
-
- def _run(self):
- result = self.result = TransferGreenletResult(self.type)
- result.markStarted()
-
- try:
- with gevent.Timeout(self.timeout, False):
- result.success = self._doit()
- except gevent.GreenletExit:
- return
- except:
- result.setError(show_traceback=True)
-
- result.markFinished()
-
-
-class ReaderGreenlet(SafeTransferGreenlet):
- type = 'reader'
-
- def _doit(self):
- if context.needs_first_read:
- key = context.needs_first_read.popleft()
- elif context.all_keys:
- key = random.choice(context.all_keys)
- else:
- time.sleep(1)
- return self.result.setError('No available keys to test with reader. Try again later.')
-
- # Copynew the key object
- key = key.bucket.new_key(key.name)
- self.result.setKey(key)
-
- fp = FileVerifier()
-
- key.get_contents_to_file(fp)
-
- self.result.request_finish = time.time()
- self.result.request_start = fp.created_at
- self.result.chunks = fp.chunks
- self.result.size = fp.size
-
- if not fp.valid():
- return self.result.setError('Failed to validate key {name!s}'.format(name=key.name))
-
- return True
-
-
-class WriterGreenlet(SafeTransferGreenlet):
- type = 'writer'
-
- def _doit(self):
- key = get_next_key(context.bucket)
- self.result.setKey(key)
-
- fp = next(context.files_iter)
- self.result.size = fp.size
-
- key.set_contents_from_file(fp)
-
- self.result.request_finish = time.time()
- self.result.request_start = fp.start_time
- self.result.chunks = fp.last_chunks
-
- # And at the end, add to needs_first_read and shuffle
- context.needs_first_read.append(key)
- context.all_keys.append(key)
-
- return True
+++ /dev/null
-import bunch
-import collections
-import gevent
-import time
-import traceback
-import yaml
-
-from ..common import context
-
-context.update(bunch.Bunch(
- result_queue=collections.deque(),
-))
-
-
-class TransferGreenletResult(object):
- """ Generic container object. Weeeeeeeeeeeeeee *short* """
- def __init__(self, type):
- # About the Greenlet
- self.type = type
-
- # About the key
- self.bucket = None
- self.key = None
- self.size = None
-
- # About the job
- self.success = False
- self.error = None
-
- self.start_time = None
- self.finish_time = None
-
- self.duration = None
- self.latency = None
-
- self.request_start = None
- self.request_finish = None
-
- self.chunks = None
-
- def markStarted(self):
- self.start_time = time.time()
-
- def markFinished(self):
- self.finish_time = time.time()
- self.duration = self.finish_time - self.start_time
- context.result_queue.append(self)
-
- def setKey(self, key):
- self.key = key.name
- self.bucket = key.bucket.name
-
- def setError(self, message='Unhandled Exception', show_traceback=False):
- """ Sets an error state in the result, and returns False... example usage:
-
- return self.result.setError('Something happened', traceback=True)
- """
- self.error = dict()
- self.error['msg'] = message
- if show_traceback:
- self.error['traceback'] = traceback.format_exc()
- return False
-
- @classmethod
- def repr_yaml(c, dumper, self):
- data = dict()
- for x in ('type', 'bucket', 'key', 'chunks'):
- data[x] = self.__dict__[x]
-
- # reader => r, writer => w
- data['type'] = data['type'][0]#chunks
-
- # the error key must be present ONLY on failure.
- assert not (self.success and self.error)
- if self.success:
- assert self.error == None
- else:
- assert self.error != None
- data['error'] = self.error
-
- data['start'] = self.request_start
- if self.request_finish:
- data['duration'] = 1000000000 * (self.request_finish - self.request_start)
-
- return dumper.represent_dict(data)
-
-# And a representer for dumping a TransferGreenletResult as a YAML dict()
-yaml.add_representer(TransferGreenletResult, TransferGreenletResult.repr_yaml)
-
-
-class ResultsLogger(gevent.Greenlet):
- """ A quick little greenlet to always run and dump results. """
- def __init__(self):
- gevent.Greenlet.__init__(self)
- self.outfile = context.real_stdout
-
- def _run(self):
- while True:
- try:
- self._doit()
- except:
- print "An exception was encountered while dumping the results... this shouldn't happen!"
- traceback.print_exc()
- time.sleep(0.1)
-
- def _doit(self):
- while context.result_queue:
- result = context.result_queue.popleft()
- yrep = yaml.dump(result)
- self.outfile.write(yrep + "---\n")
-import bunch
import hashlib
import random
import string
break
name = ''.join(rand.choice(charset) for _ in xrange(length))
yield name
-
-def files_varied(groups, unlimited=False):
- """ Yields a weighted-random selection of file-like objects. """
- # Quick data type sanity.
- assert groups and isinstance(groups, (list, tuple))
-
- total_num = 0
- file_sets = []
- rand = random.Random(time.time())
-
- # Build the sets for our yield
- for num, size, stddev in groups:
- assert num and size #TODO
-
- file_sets.append(bunch.Bunch(
- num = num,
- size = size,
- stddev = stddev,
- files = files(size, stddev, time.time()),
- ))
- total_num += num
-
- while True:
- if not total_num:
- raise StopIteration
-
- num = rand.randrange(total_num)
-
- ok = 0
- for file_set in file_sets:
- if num > file_set.num:
- num -= file_set.num
- continue
-
- if not unlimited:
- total_num -= 1
- file_set.num -= 1
-
- # None left in this set!
- if file_set.num == 0:
- file_sets.remove(file_set)
-
- ok = 1
- yield next(file_set.files)
-
- if not ok:
- raise RuntimeError("Couldn't find a match.")
projects / s3-tests.git / commitdiff