From 119ed6fe10fe5a385198fc55276ca932d7b565b1 Mon Sep 17 00:00:00 2001 From: Sandon Van Ness Date: Tue, 17 Feb 2015 13:21:24 -0800 Subject: [PATCH] Download ELFs (regular + debug) from remotes when coredump occurs. Per issue #5629 Since we need the binaries (regular stripped + non-stripped with debug symbols) to do debugging this will make it more convinient for developers so they don't have to track down the files from the specific build in question and instead they will just be sitting in the archive folder with the coredump files. Signed-off-by: Sandon Van Ness --- teuthology/task/internal.py | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/teuthology/task/internal.py b/teuthology/task/internal.py index 31b55030c3..cec9de9b92 100644 --- a/teuthology/task/internal.py +++ b/teuthology/task/internal.py @@ -359,6 +359,43 @@ def check_conflict(ctx, config): if failed: raise RuntimeError('Stale jobs detected, aborting.') +def fetch_binaries_for_coredumps(path, remote): + """ + Pul ELFs (debug and stripped) for each coredump found + """ + #Check for Coredumps: + coredump_path = os.path.join(path, 'coredump') + if os.path.isdir(coredump_path): + log.info('Transferring binaries for coredumps...') + for dump in os.listdir(coredump_path): + # Pull program from core file + dump_path = os.path.join(coredump_path, dump) + dump_info = subprocess.Popen(['file', dump_path], stdout=subprocess.PIPE) + dump_out = dump_info.communicate() + + # Parse file output to get program, Example output: + # 1422917770.7450.core: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, \ + # from 'radosgw --rgw-socket-path /home/ubuntu/cephtest/apache/tmp.client.0/fastcgi_soc' + dump_program = dump_out.split("from '")[1].split(' ')[0] + + # Find path on remote server: + r = remote.run(args=['which', dump_program], stdout=StringIO()) + remote_path = r.stdout.getvalue() + + # Pull remote program into coredump folder: + remote._sftp_get_file(remote_path, os.path.join(coredump_path, dump_program)) + + # Pull Debug symbols: + # RPM distro's append their non-stripped ELF's with .debug + # When deb based distro's do not. + debug_program = '{dump_program}.debug'.format(dump_program=dump_program) + debug_path = os.path.join('/usr/lib/debug', remote_path) + + if remote.system_type == 'rpm': + debug_path = '{debug_path}.debug'.format(debug_path=debug_path) + + remote._sftp_get_file(debug_path, os.path.join(coredump_path, debug_program)) + @contextlib.contextmanager def archive(ctx, config): """ @@ -392,6 +429,8 @@ def archive(ctx, config): for rem in ctx.cluster.remotes.iterkeys(): path = os.path.join(logdir, rem.shortname) misc.pull_directory(rem, archive_dir, path) + # Check for coredumps and pull binaries + fetch_binaries_for_coredumps(path, rem) log.info('Removing archive directory...') run.wait( -- 2.39.5