From 1caf97757b889a58b777a3f4744fe889826242fb Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Tue, 5 Nov 2019 17:42:03 -0800 Subject: [PATCH] Add "quay-pruner" to prune stale quay.io container images Signed-off-by: Dan Mick --- quay-pruner/build/build | 6 + quay-pruner/build/prune-quay.py | 148 ++++++++++++++++++ .../config/definitions/quay-pruner.yml | 48 ++++++ 3 files changed, 202 insertions(+) create mode 100755 quay-pruner/build/build create mode 100755 quay-pruner/build/prune-quay.py create mode 100644 quay-pruner/config/definitions/quay-pruner.yml diff --git a/quay-pruner/build/build b/quay-pruner/build/build new file mode 100755 index 00000000..dbbfbc59 --- /dev/null +++ b/quay-pruner/build/build @@ -0,0 +1,6 @@ +#!/bin/bash -ex +virtualenv -p python3 ./v +./v/bin/pip install requests +./v/bin/python3 ./ceph-build/quay-pruner/build/prune-quay.py -v +rm -rf ./v + diff --git a/quay-pruner/build/prune-quay.py b/quay-pruner/build/prune-quay.py new file mode 100755 index 00000000..093bbdf7 --- /dev/null +++ b/quay-pruner/build/prune-quay.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 + +import argparse +import os +import re +import requests +import sys + +QUAYBASE = "https://quay.io/api/v1" +REPO = "cephci/daemon-base" + + +def get_all_quay_tags(quaytoken): + page = 1 + has_additional = True + ret = list() + + while has_additional: + try: + response = requests.get( + '/'.join((QUAYBASE, 'repository', REPO, 'tag')), + params={'page': page, 'limit': 100, 'onlyActiveTags': 'false'}, + headers={'Authorization': 'Bearer %s' % quaytoken}, + timeout=30, + ) + response.raise_for_status() + except requests.exceptions.RequestException as e: + print( + 'quay.io request', + response.url, + 'failed:', + e, + requests.reason, + file=sys.stderr + ) + break + response = response.json() + ret.extend(response['tags']) + page += 1 + has_additional = response.get('has_additional') + return ret + + +NAME_RE = re.compile(r'(.*)-([0-9a-f]{7})-centos-7-x86_64-devel') + + +def present_in_shaman(tag, verbose): + mo = NAME_RE.match(tag['name']) + if mo is None: + print('Can''t parse name', tag['name'], file=sys.stderr) + return False + ref = mo.group(1) + short_sha1 = mo.group(2) + try: + response = requests.get( + 'https://shaman.ceph.com/api/search/', + params={ + 'ref': ref, + 'distros': 'centos/7/x86_64', + 'flavor': 'default', + 'status': 'ready', + }, + timeout=30 + ) + response.raise_for_status() + except requests.exceptions.RequestException as e: + # err on the side of caution; if there's some error, keep it + print( + 'shaman request', + response.url, + 'failed:', + e, + response.reason, + file=sys.stderr + ) + if not response.ok: + print('shaman request', response.request.url, 'failed:', + response.status_code, response.reason, file=sys.stderr) + return True + + matches = response.json() + if len(matches) == 0: + return False + for match in matches: + if match['sha1'][0:7] == short_sha1: + if verbose: + print('Found matching build: ref %s sha1 %s quayname %s' % + (match['ref'], match['sha1'], tag['name'])) + return True + return False + + +def delete_from_quay(tag, quaytoken, dryrun): + if dryrun: + print('Would delete from quay: ', tag['name']) + return + + try: + response = requests.delete( + '/'.join((QUAYBASE, 'repository', REPO, 'tag', tag['name'])), + headers={'Authorization': 'Bearer %s' % quaytoken}, + timeout=30, + ) + response.raise_for_status() + print('Deleted', tag['name']) + except requests.exceptions.RequestException as e: + print( + 'Problem on delete of tag %s:', + tag['name'], + e, + response.reason, + file=sys.stderr + ) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('-d', '--dryrun', action='store_true', help="don't actually delete") + parser.add_argument('-v', '--verbose', action='store_true', help="say more") + return parser.parse_args() + + +def main(): + args = parse_args() + + quaytoken = None + if not args.dryrun: + if 'QUAYTOKEN' in os.environ: + quaytoken = os.environ['QUAYTOKEN'] + else: + quaytoken = open( + os.path.join(os.environ['HOME'], '.quaytoken'), + 'rb' + ).read().strip().decode() + + quaytags = get_all_quay_tags(quaytoken) + for tag in quaytags: + if 'expiration' in tag or 'end_ts' in tag: + if args.verbose: + print('Skipping already-deleted tag', tag['name']) + continue + if present_in_shaman(tag, args.verbose): + continue + delete_from_quay(tag, quaytoken, args.dryrun) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/quay-pruner/config/definitions/quay-pruner.yml b/quay-pruner/config/definitions/quay-pruner.yml new file mode 100644 index 00000000..63ef4057 --- /dev/null +++ b/quay-pruner/config/definitions/quay-pruner.yml @@ -0,0 +1,48 @@ +- scm: + name: ceph-build + scm: + - git: + url: https://github.com/ceph/ceph-build.git + branches: + - origin/master + browser-url: https://github.com/ceph/ceph-build + timeout: 20 + basedir: "ceph-build" + + +- job: + name: quay-pruner + node: small + project-type: freestyle + defaults: global + display-name: 'Quay: prune container images' + concurrent: true + quiet-period: 5 + block-downstream: false + block-upstream: false + retry-count: 3 + properties: + - build-discarder: + days-to-keep: 15 + artifact-days-to-keep: 15 + + triggers: + - timed: '@daily' + + scm: + - ceph-build + + + builders: + - shell: + !include-raw: + - ../../build/build + + wrappers: + - inject-passwords: + global: true + mask-password-params: true + - credentials-binding: + - text: + credential-id: quay-api-token + variable: QUAYTOKEN -- 2.39.5