From: Dan Mick Date: Sat, 5 Dec 2020 02:21:12 +0000 (+0000) Subject: quay-pruner: look for orphaned SHA1 tags and delete X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F1719%2Fhead;p=ceph-build.git quay-pruner: look for orphaned SHA1 tags and delete This required the ability to query shaman for SHA1 alone as well as the existing ref/sha1/CentOS-minor-version. Add some refactoring to pull out the actual shaman request, and some caching to minimize the actual number of requests. Signed-off-by: Dan Mick --- diff --git a/quay-pruner/build/prune-quay.py b/quay-pruner/build/prune-quay.py index f755a534..5524525a 100755 --- a/quay-pruner/build/prune-quay.py +++ b/quay-pruner/build/prune-quay.py @@ -9,10 +9,17 @@ import sys QUAYBASE = "https://quay.ceph.io/api/v1" REPO = "ceph-ci/ceph" +# cache shaman search results so we only have to ask once +short_sha1_cache = set() +sha1_cache = set() + # quay page ranges to fetch; hackable for testing start_page = 1 page_limit = 100000 +NAME_RE = re.compile(r'(.*)-([0-9a-f]{7})-centos-([78])-x86_64-devel') +SHA1_RE = re.compile(r'([0-9a-f]{40})(-crimson)*') + def get_all_quay_tags(quaytoken): page = start_page @@ -45,9 +52,6 @@ def get_all_quay_tags(quaytoken): return ret -NAME_RE = re.compile(r'(.*)-([0-9a-f]{7})-centos-([78])-x86_64-devel') - - def parse_quay_tag(tag): mo = NAME_RE.match(tag) @@ -59,25 +63,28 @@ def parse_quay_tag(tag): return ref, short_sha1, el -def present_in_shaman(tag, verbose): - ref, short_sha1, el = parse_quay_tag(tag['name']) - if ref is None: - print("Can't parse name", tag['name'], file=sys.stderr) - return False +def query_shaman(ref, sha1, el): + + params = { + 'flavor': 'default', + 'status': 'ready', + } + if el: + params['distros'] = 'centos/{el}/x86_64'.format(el=el) + else: + params['distros'] = 'centos/7/x86_64,centos/8/x86_64,centos/9/x86_64' + if ref: + params['ref'] = ref + if sha1: + params['sha1'] = sha1 try: response = requests.get( 'https://shaman.ceph.com/api/search/', - params={ - 'ref': ref, - 'distros': 'centos/{el}/x86_64'.format(el=el), - 'flavor': 'default', - 'status': 'ready', - }, + params=params, timeout=30 ) response.raise_for_status() except requests.exceptions.RequestException as e: - # err on the side of caution; if there's some error, keep it print( 'shaman request', response.url, @@ -86,9 +93,27 @@ def present_in_shaman(tag, verbose): response.reason, file=sys.stderr ) + return response + + +def ref_present_in_shaman(tag, verbose): + + ref, short_sha1, el = parse_quay_tag(tag['name']) + if ref is None: + print("Can't parse name", tag['name'], file=sys.stderr) + return False + + if short_sha1 in short_sha1_cache: + if verbose: + print('Found %s in in_shaman cache' % short_sha1) + return True + + response = query_shaman(ref, None, el) if not response.ok: print('shaman request', response.request.url, 'failed:', response.status_code, response.reason, file=sys.stderr) + # don't cache, but claim present: + # avoid deletion in case of transient shaman failure return True matches = response.json() @@ -97,8 +122,36 @@ def present_in_shaman(tag, verbose): for match in matches: if match['sha1'][0:7] == short_sha1: if verbose: - print('Found matching build: ref %s sha1 %s quayname %s' % - (match['ref'], match['sha1'], tag['name'])) + print('Found match for %s: sha1 %s quayname %s' % + (ref, match['sha1'], tag['name'])) + short_sha1_cache.add(short_sha1) + return True + return False + + +def sha1_present_in_shaman(sha1, verbose): + + if sha1 in sha1_cache: + if verbose: + print('Found %s in in_shaman cache' % sha1) + return True + + response = query_shaman(None, sha1, None) + if not response.ok: + print('shaman request', response.request.url, 'failed:', + response.status_code, response.reason, file=sys.stderr) + # don't cache, but claim present + # to avoid deleting on transient shaman failure + return True + + matches = response.json() + if len(matches) == 0: + return False + for match in matches: + if match['sha1'] == sha1: + if verbose: + print('Found sha1 %s in shaman' % sha1) + sha1_cache.add(sha1) return True return False @@ -151,7 +204,7 @@ def main(): quaytags = get_all_quay_tags(quaytoken) # find all full tags to delete, put them and ref tag on list - tags_to_delete = list() + tags_to_delete = set() short_sha1s_to_delete = list() for tag in quaytags: if 'expiration' in tag or 'end_ts' in tag: @@ -163,24 +216,41 @@ def main(): if ref is None: continue - if present_in_shaman(tag, args.verbose): + if ref_present_in_shaman(tag, args.verbose): continue # accumulate full and ref tags to delete; keep list of short_sha1s - tags_to_delete.append(tag['name']) + tags_to_delete.add(tag['name']) if ref: - tags_to_delete.append(ref) + tags_to_delete.add(ref) if short_sha1: short_sha1s_to_delete.append(short_sha1) # now find all the full-sha1 tags to delete by making a second # pass and seeing if the tagname starts with a short_sha1 we - # know we want deleted + # know we want deleted, or if it matches SHA1_RE but is gone from + # shaman for tag in quaytags: + if 'expiration' in tag or 'end_ts' in tag: continue + if tag['name'][0:7] in short_sha1s_to_delete: - tags_to_delete.append(tag['name']) + if args.verbose: + print('Selecting %s: matches %s' % + (tag['name'], tag['name'][0:7])) + tags_to_delete.add(tag['name']) + # already selected a SHA1 tag; no point in checking for orphaned + continue + + match = SHA1_RE.match(tag['name']) + if match: + sha1 = match[1] + if sha1_present_in_shaman(sha1, args.verbose): + continue + if args.verbose: + print('Selecting %s: orphaned sha1 tag' % tag['name']) + tags_to_delete.add(tag['name']) if args.verbose: print('Deleting tags:', sorted(tags_to_delete))