# Make sure context has somewhere to store what we need
context.update(bunch.Bunch(
- neads_first_read = collections.deque(),
+ needs_first_read = collections.deque(),
all_keys = [],
files_iter = None,
))
gevent.Greenlet.__init__(self)
self.timeout = timeout
self.result = None
- self.key = None # We store key in case we ned to retry due to gevent being a jerk
def _run(self):
- """ A runner loop... using gevent creates a fun little bug where if two gevents try to
- do the same op (reading, for ex), it raises an AssertionError rather than just switching
- contexts again. Oh joy.
-
- To combat this, we've put the main work to do in _real_run, which handles detecting the
- gevent quirk, and we'll retry as long as _real_run requests that we retry, as indicated
- by _real_run returning True.
- """
- while self._real_run():
- time.sleep(0.1)
-
- def _real_run(self):
- """ Return True if we need to retry, False otherwise. """
result = self.result = TransferGreenletResult(self.type)
result.markStarted()
with gevent.Timeout(self.timeout, False):
result.success = self._doit()
except gevent.GreenletExit:
- # We don't want to retry, as it's time to exit, but we also don't want to count
- # this as a failure.
- return False
+ return
except:
result.setError(show_traceback=True)
result.markFinished()
- return False # don't retry
class ReaderGreenlet(SafeTransferGreenlet):
type = 'reader'
def _doit(self):
- if self.key:
- key = self.key
- elif context.neads_first_read:
- key = context.neads_first_read.popleft()
+ if context.needs_first_read:
+ key = context.needs_first_read.popleft()
elif context.all_keys:
key = random.choice(context.all_keys)
else:
# Copynew the key object
key = key.bucket.new_key(key.name)
- self.key = key
self.result.setKey(key)
fp = FileVerifier()
type = 'writer'
def _doit(self):
- if self.key:
- key = self.key
- else:
- key = get_next_key(context.bucket)
-
- self.key = key
+ key = get_next_key(context.bucket)
self.result.setKey(key)
fp = next(context.files_iter)
self.result.request_start = fp.start_time
self.result.chunks = fp.last_chunks
- # And at the end, add to neads_first_read and shuffle
- context.neads_first_read.append(key)
+ # And at the end, add to needs_first_read and shuffle
+ context.needs_first_read.append(key)
context.all_keys.append(key)
return True