Immediately fail when op state is not found

There's a very narrow window in which our connection sending the copy
request could fail before radosgw records the op state, and we query
the op state in that time. Retrying later at that point is fine, and
avoids a long timeout if there is a bug causing us to reach this point
without the op state ever being recorded. This race hasn't been
observed in practice.

Signed-off-by: Josh Durgin <jdurgin@redhat.com>

diff --git a/radosgw_agent/tests/test_worker.py b/radosgw_agent/tests/test_worker.py
index 5ab9d3182fc320fe2ac546155b8481cce1e99dd2..f8d599f2d47820bd827c7b1656b2cab3c1774904 100644 (file)
--- a/radosgw_agent/tests/test_worker.py
+++ b/radosgw_agent/tests/test_worker.py
@@ -1,5 +1,6 @@
 from mock import Mock, patch
 import py.test
+import time
 
 from radosgw_agent import worker, client
 
@@ -125,3 +126,13 @@ class TestSyncObject(object):
             msg = 'should not have called wait_for_object'
             w.wait_for_object = Mock(side_effect=AssertionError(msg))
             assert w.sync_object('mah-bucket', 'mah-object') is True
+
+    def test_wait_for_object_state_not_found_raises_sync_failed(self):
+        self.client.get_op_state = Mock(side_effect=client.NotFound(404, ''))
+        with patch('radosgw_agent.worker.client', self.client):
+            w = worker.DataWorker(None, None, None, self.src, None, daemon_id=1)
+            with py.test.raises(worker.SyncFailed) as exc:
+                w.wait_for_object(None, None, time.time() + 1000, None)
+
+        exc_message = exc.exconly()
+        assert 'state not found' in exc_message

diff --git a/radosgw_agent/worker.py b/radosgw_agent/worker.py
index eff6c898ae9c9a958082c1045894118dc8329494..40435eaf2a2fca63a49c88bbaf5adfcdd1673954 100644 (file)
--- a/radosgw_agent/worker.py
+++ b/radosgw_agent/worker.py
@@ -233,6 +233,8 @@ class DataWorker(Worker):
                 time.sleep(1)
             except SyncFailed:
                 raise
+            except client.NotFound:
+                raise SyncFailed('object copy state not found')
             except Exception as e:
                 log.debug('error geting op state: %s', e, exc_info=True)
                 time.sleep(1)