From: Zack Cerza <zack@redhat.com>
Date: Wed, 29 Jul 2015 15:40:54 +0000 (-0600)
Subject: Add teuthology-prune-logs
X-Git-Tag: 1.1.0~807^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F639%2Fhead;p=teuthology.git

Add teuthology-prune-logs

Signed-off-by: Zack Cerza <zack@redhat.com>
---

diff --git a/scripts/prune_logs.py b/scripts/prune_logs.py
new file mode 100644
index 000000000..d401e288c
--- /dev/null
+++ b/scripts/prune_logs.py
@@ -0,0 +1,33 @@
+import docopt
+
+import teuthology.config
+import teuthology.prune
+
+doc = """
+usage:
+    teuthology-prune-logs -h
+    teuthology-prune-logs [-v] [options]
+
+Prune old logfiles from the archive
+
+optional arguments:
+  -h, --help            Show this help message and exit
+  -v, --verbose         Be more verbose
+  -a ARCHIVE, --archive ARCHIVE
+                        The base archive directory
+                        [default: {archive_base}]
+  --dry-run             Don't actually delete anything; just log what would be
+                        deleted
+  -p DAYS, --pass DAYS  Remove all logs for jobs which passed and are older
+                        than DAYS. Negative values will skip this operation.
+                        [default: 14]
+  -r DAYS, --remotes DAYS
+                        Remove the 'remote' subdir of jobs older than DAYS.
+                        Negative values will skip this operation.
+                        [default: 60]
+""".format(archive_base=teuthology.config.config.archive_base)
+
+
+def main():
+    args = docopt.docopt(doc)
+    teuthology.prune.main(args)
diff --git a/scripts/test/test_prune_logs.py b/scripts/test/test_prune_logs.py
new file mode 100644
index 000000000..8e967522f
--- /dev/null
+++ b/scripts/test/test_prune_logs.py
@@ -0,0 +1,5 @@
+from script import Script
+
+
+class TestPruneLogs(Script):
+    script_name = 'teuthology-prune-logs'
diff --git a/setup.py b/setup.py
index 646ad2d32..332f1124d 100644
--- a/setup.py
+++ b/setup.py
@@ -79,6 +79,7 @@ setup(
             'teuthology-report = scripts.report:main',
             'teuthology-kill = scripts.kill:main',
             'teuthology-queue = scripts.queue:main',
+            'teuthology-prune-logs = scripts.prune_logs:main',
             ],
         },
 
diff --git a/teuthology/prune.py b/teuthology/prune.py
new file mode 100644
index 000000000..d0560beb6
--- /dev/null
+++ b/teuthology/prune.py
@@ -0,0 +1,153 @@
+import logging
+import os
+import shutil
+import time
+
+import teuthology
+from teuthology.contextutil import safe_while
+
+log = logging.getLogger(__name__)
+
+
+# If we see this in any directory, we do not prune it
+PRESERVE_FILE = '.preserve'
+
+
+def main(args):
+    """
+    Main function; parses args and calls prune_archive()
+    """
+    verbose = args['--verbose']
+    if verbose:
+        teuthology.log.setLevel(logging.DEBUG)
+    archive_dir = args['--archive']
+    dry_run = args['--dry-run']
+    pass_days = int(args['--pass'])
+    remotes_days = int(args['--remotes'])
+
+    prune_archive(archive_dir, pass_days, remotes_days, dry_run)
+
+
+def prune_archive(archive_dir, pass_days, remotes_days, dry_run=False):
+    """
+    Walk through the archive_dir, calling the cleanup functions to process
+    directories that might be old enough
+    """
+    max_days = max(pass_days, remotes_days)
+    run_dirs = list()
+    log.debug("Archive {archive} has {count} children".format(
+        archive=archive_dir, count=len(os.listdir(archive_dir))))
+    for child in listdir(archive_dir):
+        item = os.path.join(archive_dir, child)
+        # Ensure that the path is not a symlink, is a directory, and is old
+        # enough to process
+        if (not os.path.islink(item) and os.path.isdir(item) and
+                is_old_enough(item, max_days)):
+            run_dirs.append(item)
+    for run_dir in run_dirs:
+        log.debug("Processing %s ..." % run_dir)
+        maybe_remove_passes(run_dir, pass_days, dry_run)
+        maybe_remove_remotes(run_dir, remotes_days, dry_run)
+
+
+def listdir(path):
+    with safe_while(sleep=1, increment=1, tries=10) as proceed:
+        while proceed():
+            try:
+                return os.listdir(path)
+            except OSError:
+                log.exception("Failed to list %s !" % path)
+
+
+def should_preserve(dir_name):
+    """
+    Should the directory be preserved?
+
+    :returns: True if the directory contains a file named '.preserve'; False
+              otherwise
+    """
+    preserve_path = os.path.join(dir_name, PRESERVE_FILE)
+    if os.path.isdir(dir_name) and os.path.exists(preserve_path):
+        return True
+    return False
+
+
+def is_old_enough(file_name, days):
+    """
+    :returns: True if the file's modification date is earlier than the amount
+              of days specified
+    """
+    now = time.time()
+    secs_to_days = lambda s: s / (60 * 60 * 24)
+    age = now - os.path.getmtime(file_name)
+    if secs_to_days(age) > days:
+        return True
+    return False
+
+
+def remove(path):
+    """
+    Attempt to recursively remove a directory. If an OSError is encountered,
+    log it and continue.
+    """
+    try:
+        shutil.rmtree(path)
+    except OSError:
+        log.exception("Failed to remove %s !" % path)
+
+
+def maybe_remove_passes(run_dir, days, dry_run=False):
+    """
+    Remove entire job log directories if they are old enough and the job passed
+    """
+    if days < 0:
+        return
+    contents = listdir(run_dir)
+    if PRESERVE_FILE in contents:
+        return
+    for child in contents:
+        item = os.path.join(run_dir, child)
+        # Ensure the path isn't marked for preservation, that it is a
+        # directory, and that it is old enough
+        if (should_preserve(item) or not os.path.isdir(item) or not
+                is_old_enough(item, days)):
+            continue
+        # Is it a job dir?
+        summary_path = os.path.join(item, 'summary.yaml')
+        if not os.path.exists(summary_path):
+            continue
+        # Is it a passed job?
+        summary_lines = [line.strip() for line in
+                         file(summary_path).readlines()]
+        if 'success: true' in summary_lines:
+            log.info("{job} is a {days}-day old passed job; removing".format(
+                job=item, days=days))
+            if not dry_run:
+                remove(item)
+
+
+def maybe_remove_remotes(run_dir, days, dry_run=False):
+    """
+    Remove remote logs (not teuthology logs) from job directories if they are
+    old enough
+    """
+    if days < 0:
+        return
+    contents = listdir(run_dir)
+    if PRESERVE_FILE in contents:
+        return
+    for child in contents:
+        item = os.path.join(run_dir, child)
+        # Ensure the path isn't marked for preservation, that it is a
+        # directory, and that it is old enough
+        if (should_preserve(item) or not os.path.isdir(item) or not
+                is_old_enough(item, days)):
+            continue
+        # Does it have a remote subdir?
+        remote_path = os.path.join(item, 'remote')
+        if not os.path.isdir(remote_path):
+            continue
+        log.info("{job} is {days} days old; removing remote logs".format(
+            job=item, days=days))
+        if not dry_run:
+            remove(remote_path)