s3tests: more thorough multipart resend tests

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
(cherry picked from commit 725bb83e5eddd55633fde0b536a0ae2a302f408d)

Conflicts:
	s3tests/functional/test_s3.py

diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py
index 6183674f9291327f908a0b42a92ccc35d5ecc374..61c08710aae382bf70af421de267c477a8de845e 100644 (file)
--- a/s3tests/functional/test_s3.py
+++ b/s3tests/functional/test_s3.py
@@ -4359,8 +4359,7 @@ def generate_random(size, part_size=5*1024*1024):
         if (x == size):
             return
 
-
-def _multipart_upload(bucket, s3_key_name, size, part_size=5*1024*1024, do_list=None, headers=None, metadata=None, resend_part=-1):
+def _multipart_upload(bucket, s3_key_name, size, part_size=5*1024*1024, do_list=None, headers=None, metadata=None, resend_parts=[]):
     """
     generate a multi-part upload for a random file of specifed size,
     if requested, generate a list of the parts
@@ -4371,7 +4370,7 @@ def _multipart_upload(bucket, s3_key_name, size, part_size=5*1024*1024, do_list=
     for i, part in enumerate(generate_random(size, part_size)):
         s += part
         transfer_part(bucket, upload.id, upload.key_name, i, part)
-        if resend_part == i:
+        if i in resend_parts:
             transfer_part(bucket, upload.id, upload.key_name, i, part)
 
     if do_list is not None:
@@ -4403,6 +4402,17 @@ def test_multipart_upload_small():
     key2 = bucket.get_key(key)
     eq(key2.size, size)
 
+def _check_content_using_range(k, data, step):
+    objlen = k.size
+    for ofs in xrange(0, k.size, step):
+        toread = k.size - ofs
+        if toread > step:
+            toread = step
+        end = ofs + toread - 1
+        read_range = k.get_contents_as_string(headers={'Range': 'bytes={s}-{e}'.format(s=ofs, e=end)})
+        eq(len(read_range), toread)
+        eq(read_range, data[ofs:end+1])
+
 @attr(resource='object')
 @attr(method='put')
 @attr(operation='complete multi-part upload')
@@ -4411,7 +4421,8 @@ def test_multipart_upload():
     bucket = get_new_bucket()
     key="mymultipart"
     content_type='text/bla'
-    (upload, data) = _multipart_upload(bucket, key, 30 * 1024 * 1024, headers={'Content-Type': content_type}, metadata={'foo': 'bar'})
+    objlen = 30 * 1024 * 1024
+    (upload, data) = _multipart_upload(bucket, key, objlen, headers={'Content-Type': content_type}, metadata={'foo': 'bar'})
     upload.complete_upload()
 
     (obj_count, bytes_used) = _head_bucket(bucket)
@@ -4422,27 +4433,20 @@ def test_multipart_upload():
     k=bucket.get_key(key)
     eq(k.metadata['foo'], 'bar')
     eq(k.content_type, content_type)
+    test_string=k.get_contents_as_string()
+    eq(len(test_string), k.size)
+    eq(test_string, data)
 
-@attr(resource='object')
-@attr(method='put')
-@attr(operation='complete multiple multi-part upload with different sizes')
-@attr(resource='object')
-@attr(method='put')
-@attr(operation='complete multi-part upload')
-@attr(assertion='successful')
-def test_multipart_upload_resend_part():
-    bucket = get_new_bucket()
-    key="mymultipart"
+    _check_content_using_range(k, data, 1000000)
+    _check_content_using_range(k, data, 10000000)
+
+def _check_upload_multipart_resend(bucket, key, objlen, resend_parts):
     content_type='text/bla'
-    objlen = 30 * 1024 * 1024
-    (upload, data) = _multipart_upload(bucket, key, objlen, headers={'Content-Type': content_type}, metadata={'foo': 'bar'}, resend_part=1)
+    (upload, data) = _multipart_upload(bucket, key, objlen, headers={'Content-Type': content_type}, metadata={'foo': 'bar'}, resend_parts=resend_parts)
     upload.complete_upload()
 
     (obj_count, bytes_used) = _head_bucket(bucket)
 
-    # eq(obj_count, 1)
-    # eq(bytes_used, 30 * 1024 * 1024)
-
     k=bucket.get_key(key)
     eq(k.metadata['foo'], 'bar')
     eq(k.content_type, content_type)
@@ -4451,6 +4455,27 @@ def test_multipart_upload_resend_part():
     eq(k.size, objlen)
     eq(test_string, data)
 
+    _check_content_using_range(k, data, 1000000)
+    _check_content_using_range(k, data, 10000000)
+
+@attr(resource='object')
+@attr(method='put')
+@attr(operation='complete multiple multi-part upload with different sizes')
+@attr(resource='object')
+@attr(method='put')
+@attr(operation='complete multi-part upload')
+@attr(assertion='successful')
+def test_multipart_upload_resend_part():
+    bucket = get_new_bucket()
+    key="mymultipart"
+    objlen = 30 * 1024 * 1024
+
+    _check_upload_multipart_resend(bucket, key, objlen, [0])
+    _check_upload_multipart_resend(bucket, key, objlen, [1])
+    _check_upload_multipart_resend(bucket, key, objlen, [2])
+    _check_upload_multipart_resend(bucket, key, objlen, [1,2])
+    _check_upload_multipart_resend(bucket, key, objlen, [0,1,2,3,4,5])
+
 @attr(assertion='successful')
 def test_multipart_upload_multiple_sizes():
     bucket = get_new_bucket()