nuke: reset syslog configuration after rebooting

Previously we removed a file and rebooted without syncing, so the file
was never deleted.

diff --git a/teuthology/nuke.py b/teuthology/nuke.py
index 8baadf449cb3ca04e9ddee71a589120635236b65..d4d8aec993846dc3c3a0af620166b76b7c86bb0d 100644 (file)
--- a/teuthology/nuke.py
+++ b/teuthology/nuke.py
@@ -66,16 +66,8 @@ def shutdown_daemons(ctx, log):
                 '/tmp/cephtest/binary/usr/local/bin/ceph-fuse',
                 '/tmp/cephtest/binary/usr/local/bin/radosgw',
                 'apache2',
-                run.Raw(';'),
-                'if', 'test', '-e', '/etc/rsyslog.d/80-cephtest.conf',
-                run.Raw(';'),
-                'then',
-                'sudo', 'rm', '-f', '--', '/etc/rsyslog.d/80-cephtest.conf',
-                run.Raw('&&'),
-                'sudo', 'initctl', 'restart', 'rsyslog',
-                run.Raw(';'),
-                'fi',
-                run.Raw(';'),
+                run.Raw('||'),
+                'true', # ignore errors from ceph binaries not being found
                 ],
             wait=False,
             )
@@ -154,6 +146,30 @@ def reboot(ctx, remotes, log):
         time.sleep(5) #if we try and reconnect too quickly, it succeeds!
         reconnect(ctx, 300)     #allow 5 minutes for the reboots
 
+def reset_syslog_dir(ctx, log):
+    from .orchestra import run
+    nodes = {}
+    for remote in ctx.cluster.remotes.iterkeys():
+        proc = remote.run(
+            args=[
+                'if', 'test', '-e', '/etc/rsyslog.d/80-cephtest.conf',
+                run.Raw(';'),
+                'then',
+                'sudo', 'rm', '-f', '--', '/etc/rsyslog.d/80-cephtest.conf',
+                run.Raw('&&'),
+                'sudo', 'initctl', 'restart', 'rsyslog',
+                run.Raw(';'),
+                'fi',
+                run.Raw(';'),
+                ],
+            wait=False,
+            )
+        nodes[remote.name] = proc
+
+    for name, proc in nodes.iteritems():
+        log.info('Waiting for %s to restart syslog...', name)
+        proc.exitstatus.get()
+
 def remove_testing_tree(ctx, log):
     nodes = {}
     for remote in ctx.cluster.remotes.iterkeys():
@@ -239,6 +255,8 @@ def main():
         need_reboot = ctx.cluster.remotes.keys()
     synch_clocks(need_reboot, log)
 
+    log.info('Reseting syslog output locations...')
+    reset_syslog_dir(ctx, log)
     log.info('Clearing filesystem of test data...')
     remove_testing_tree(ctx, log)
     log.info('Filesystem Cleared.')