Fetch ceph binary tarball independently on every node.

Avoids shuffling the bytes through the controlling node.
Use sha1 file to make sure everyone gets the same version.

diff --git a/dbench.py b/dbench.py
index d7a71299352f9d2d1b99a7d5b223e624203e37b8..23721c4625be6ef8b8eec596a23099ee3e98fcf0 100644 (file)
--- a/dbench.py
+++ b/dbench.py
@@ -6,7 +6,6 @@ from cStringIO import StringIO
 import json
 import logging
 import os
-import urllib2
 import sys
 import yaml
 
@@ -84,8 +83,23 @@ if __name__ == '__main__':
                     )
 
     log.info('Untarring ceph binaries...')
-    ceph_bin = urllib2.urlopen(teuthology.get_ceph_binary_url())
-    teuthology.untar_to_dir(ceph_bin, '/tmp/cephtest/binary', cluster.remotes.keys())
+    ceph_bindir_url = teuthology.get_ceph_binary_url()
+    cluster.run(
+        args=[
+            'uname', '-m',
+            run.Raw('|'),
+            'sed', '-e', 's/^/ceph./; s/$/.tgz/',
+            run.Raw('|'),
+            'wget',
+            '-nv',
+            '-O-',
+            '--base={url}'.format(url=ceph_bindir_url),
+            # need to use --input-file to make wget respect --base
+            '--input-file=-',
+            run.Raw('|'),
+            'tar', '-xzf', '-', '-C', '/tmp/cephtest/binary',
+            ],
+        )
 
     log.info('Writing configs...')
     ips = [host for (host, port) in (conn.get_transport().getpeername() for conn in connections)]

diff --git a/teuthology/misc.py b/teuthology/misc.py
index 2e5b0621afca3446c49b598ed14927d9aa7c3b18..ca02099d054b2b04e496f609e2804048c2ddd028 100644 (file)
--- a/teuthology/misc.py
+++ b/teuthology/misc.py
@@ -4,19 +4,20 @@ import os
 import logging
 import configobj
 import time
-
-from orchestra import run
+import urllib2
+import urlparse
 
 log = logging.getLogger(__name__)
 
 def get_ceph_binary_url():
-    machine = os.uname()[4]
-    BRANCH = 'master'
-    CEPH_TARBALL_DEFAULT_URL = 'http://ceph.newdream.net/gitbuilder/output/ref/origin_{branch}/ceph.{machine}.tgz'.format(
-        branch=BRANCH,
-        machine=machine,
-        )
-    return CEPH_TARBALL_DEFAULT_URL
+    BRANCH = 'master' #TODO
+    BASE = 'http://ceph.newdream.net/gitbuilder/output/'
+    sha1_url = urlparse.urljoin(BASE, 'ref/origin_{branch}/sha1'.format(branch=BRANCH))
+    sha1_fp = urllib2.urlopen(sha1_url)
+    sha1 = sha1_fp.read().rstrip('\n')
+    sha1_fp.close()
+    bindir_url = urlparse.urljoin(BASE, 'sha1/{sha1}/'.format(sha1=sha1))
+    return bindir_url
 
 def feed_many_stdins(fp, processes):
     while True:
@@ -31,25 +32,6 @@ def feed_many_stdins_and_close(fp, processes):
     for proc in processes:
         proc.stdin.close()
 
-def untar_to_dir(fp, dst, remotes):
-    """
-    Untar a ``.tar.gz`` to given hosts and directories.
-
-    :param fp: a file-like object
-    :param conns_and_dirs: a list of 2-tuples `(client, path)`
-    """
-    untars = [
-        remote.run(
-            logger=log.getChild('cephbin'),
-            args=['tar', '-xzf', '-', '-C', dst],
-            wait=False,
-            stdin=run.PIPE,
-            )
-        for remote in remotes
-        ]
-    feed_many_stdins_and_close(fp, untars)
-    run.wait(untars)
-
 def get_mons(roles, ips):
     mons = {}
     for idx, roles in enumerate(roles):