Download ELFs (regular + debug) from remotes when coredump occurs.

Per issue #5629

Since we need the binaries (regular stripped + non-stripped with
debug symbols) to do debugging this will make it more convinient
for developers so they don't have to track down the files from
the specific build in question and instead they will just be
sitting in the archive folder with the coredump files.

Signed-off-by: Sandon Van Ness <sandon@redhat.com>

diff --git a/teuthology/task/internal.py b/teuthology/task/internal.py
index 31b55030c3915297db657ff50d60d659cb3340d6..cec9de9b92d75dba3b18e3292dbbef76a0263e9d 100644 (file)
--- a/teuthology/task/internal.py
+++ b/teuthology/task/internal.py
@@ -359,6 +359,43 @@ def check_conflict(ctx, config):
     if failed:
         raise RuntimeError('Stale jobs detected, aborting.')
 
+def fetch_binaries_for_coredumps(path, remote):
+    """
+    Pul ELFs (debug and stripped) for each coredump found
+    """
+    #Check for Coredumps:
+    coredump_path = os.path.join(path, 'coredump')
+    if os.path.isdir(coredump_path):
+        log.info('Transferring binaries for coredumps...')
+        for dump in os.listdir(coredump_path):
+            # Pull program from core file
+            dump_path = os.path.join(coredump_path, dump)
+            dump_info = subprocess.Popen(['file', dump_path], stdout=subprocess.PIPE)
+            dump_out = dump_info.communicate()
+
+            # Parse file output to get program, Example output:
+            # 1422917770.7450.core: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, \
+            # from 'radosgw --rgw-socket-path /home/ubuntu/cephtest/apache/tmp.client.0/fastcgi_soc'
+            dump_program = dump_out.split("from '")[1].split(' ')[0]
+
+            # Find path on remote server:
+            r = remote.run(args=['which', dump_program], stdout=StringIO())
+            remote_path = r.stdout.getvalue()
+
+            # Pull remote program into coredump folder:
+            remote._sftp_get_file(remote_path, os.path.join(coredump_path, dump_program))
+
+            # Pull Debug symbols:
+            # RPM distro's append their non-stripped ELF's with .debug
+            # When deb based distro's do not.
+            debug_program = '{dump_program}.debug'.format(dump_program=dump_program)
+            debug_path = os.path.join('/usr/lib/debug', remote_path)
+
+            if remote.system_type == 'rpm':
+                debug_path = '{debug_path}.debug'.format(debug_path=debug_path)
+
+            remote._sftp_get_file(debug_path, os.path.join(coredump_path, debug_program))
+
 @contextlib.contextmanager
 def archive(ctx, config):
     """
@@ -392,6 +429,8 @@ def archive(ctx, config):
             for rem in ctx.cluster.remotes.iterkeys():
                 path = os.path.join(logdir, rem.shortname)
                 misc.pull_directory(rem, archive_dir, path)
+                # Check for coredumps and pull binaries
+                fetch_binaries_for_coredumps(path, rem)
 
         log.info('Removing archive directory...')
         run.wait(