From f3c8a59457389573c377e402d0c58e57382783cb Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Fri, 14 Nov 2025 16:00:40 -0800 Subject: [PATCH] review comments: - explain --stragglers a bit in help text - add re match group names and use them - use variables for URLs in messages - dryrun -> dry-run - remove dead code will squash before merge if approved. Signed-off-by: Dan Mick --- quay-pruner/build/delete-tags.py | 15 ++++------ quay-pruner/build/get-tagdates.py | 2 +- quay-pruner/build/prune-quay.py | 8 +++--- quay-pruner/build/util.py | 47 ++++++++++++++++++++----------- 4 files changed, 41 insertions(+), 31 deletions(-) diff --git a/quay-pruner/build/delete-tags.py b/quay-pruner/build/delete-tags.py index d5e5da5a..d947914b 100755 --- a/quay-pruner/build/delete-tags.py +++ b/quay-pruner/build/delete-tags.py @@ -15,8 +15,8 @@ def parse_args(): help="number of pages") parser.add_argument('-S', '--stragglers', type=int, metavar="DAYS_OLD", - help="find stragglers and delete them") - parser.add_argument('-n', '--dryrun', action='store_true', + help="find stragglers and delete them; stragglers are currently \"anything older than the given age in days that doesn't have one of the recent release names in its tag\"; this is a heuristic that will change over time") + parser.add_argument('-n', '--dry-run', action='store_true', help="don't actually delete") parser.add_argument('-v', '--verbose', action='count', default=0, help="say more (-vv for more info)") @@ -29,7 +29,7 @@ def main(): tags_to_delete = list() quaytoken = None - if not args.dryrun: + if not args.dry_run: if 'QUAYTOKEN' in os.environ: quaytoken = os.environ['QUAYTOKEN'] else: @@ -38,7 +38,7 @@ def main(): 'rb' ).read().strip().decode() - print('Getting ceph-ci container tags from quay.ceph.io', file=sys.stderr) + print(f'Getting tags from {util.QUAYBASE}{util.REPO}', file=sys.stderr) quaytags, digest_to_tags = util.get_all_quay_tags(quaytoken, args.start, args.pages) if args.stragglers: @@ -49,11 +49,6 @@ def main(): if any([r in t for t in alltags for r in ('quincy', 'reef', 'squid', 'tentacle')]): print(f'skipping {alltags}, looks distinguished') continue - ''' - if len(alltags) > 1: - print(f'{tag["name"]} has friends {alltags}, leaving for now') - continue - ''' print(f'Marking {tag["name"]}') tags_to_delete.append((alltags, date)) else: @@ -76,7 +71,7 @@ def main(): # and now delete all the ones we found for alltags, date in tags_to_delete: for tagname in alltags: - util.delete_from_quay(tagname, date, quaytoken, args.dryrun) + util.delete_from_quay(tagname, date, quaytoken, args.dry_run) if __name__ == "__main__": diff --git a/quay-pruner/build/get-tagdates.py b/quay-pruner/build/get-tagdates.py index 14f92045..ac638f2d 100755 --- a/quay-pruner/build/get-tagdates.py +++ b/quay-pruner/build/get-tagdates.py @@ -24,7 +24,7 @@ def parse_args(): def main(): args = parse_args() - print('Getting ceph-ci container tags from quay.ceph.io', file=sys.stderr) + print(f'Getting tags from {util.QUAYBASE}{util.REPO}', file=sys.stderr) quaytags, digest_to_tags = util.get_all_quay_tags(None, args.start, args.pages) tagdates = defaultdict(str) diff --git a/quay-pruner/build/prune-quay.py b/quay-pruner/build/prune-quay.py index e195447c..3fb7d671 100755 --- a/quay-pruner/build/prune-quay.py +++ b/quay-pruner/build/prune-quay.py @@ -16,7 +16,7 @@ def parse_args(): help="start page (of 100 tags)") parser.add_argument('-p', '--pages', type=int, default=100000, help="number of pages") - parser.add_argument('-n', '--dryrun', action='store_true', + parser.add_argument('-n', '--dry-run', action='store_true', help="don't actually delete") parser.add_argument('-v', '--verbose', action='count', default=0, help="say more (-vv for more info)") @@ -34,7 +34,7 @@ def main(): args = parse_args() quaytoken = None - if not args.dryrun: + if not args.dry_run: if 'QUAYTOKEN' in os.environ: quaytoken = os.environ['QUAYTOKEN'] else: @@ -43,7 +43,7 @@ def main(): 'rb' ).read().strip().decode() - print('Getting ceph-ci container tags from quay.ceph.io', file=sys.stderr) + print(f'Getting tags from {util.QUAYBASE}{util.REPO}', file=sys.stderr) quaytags, digest_to_tags = util.get_all_quay_tags(quaytoken, args.start, args.pages) tagstat = defaultdict(int) @@ -123,7 +123,7 @@ def main(): # and now delete all the ones we found for (alltags, date) in tags_to_delete: for tagname in alltags: - util.delete_from_quay(tagname, date, quaytoken, args.dryrun) + util.delete_from_quay(tagname, date, quaytoken, args.dry_run) tagstat['deleted'] += 1 tagstat['subtags_deleted'] += len(alltags) diff --git a/quay-pruner/build/util.py b/quay-pruner/build/util.py index dbd102f6..cd3d1dae 100644 --- a/quay-pruner/build/util.py +++ b/quay-pruner/build/util.py @@ -5,20 +5,20 @@ import sys from collections import defaultdict from datetime import datetime, timezone -QUAYBASE = "https://quay.ceph.io/api/v1" +QUAYBASE = "https://quay.ceph.io/" +QUAYAPI = QUAYBASE + "api/v1/" REPO = "ceph-ci/ceph" # cache shaman search results so we only have to ask once sha1_cache = set() tags_done = set() -# XXX why aren't these using named groups # # example full tag: # ceph-nvmeof-mon-5295149-centos-stream8-x86_64-devel # ^^^^ ref ^^sha1 ^dist ^^dver ^arch ^assumed OLD_FULLTAG_RE = re.compile( - r'(.*)-([-2-9a-f]{7})-centos-.*([0-9]+)-(x86_64|aarch64)-devel' + r'(?P.*)-(?P[-2-9a-f]{7})-centos-.*(?P[0-9]+)-(?Px86_64|aarch64)-devel' ) # ^ this was originally true. Now, when dmick introduced # container/build.sh, he forgot about the short sha1 altogether, @@ -27,33 +27,43 @@ OLD_FULLTAG_RE = re.compile( # there could also be more suffixes, but they don't matter to the algorithm # (for that matter, neither do most of these fields) FULLTAG_RE = re.compile( - r'(.*)-(centos|rockylinux)-.*([-2-9]+)-(x86_64|aarch64)-devel' + r'(?P.*)-(?Pcentos|rockylinux)-.*(?P[0-9]+)-(?Px86_64|aarch64)-devel' ) # example sha1 tag: # 1d7b744e98c74bba9acb22262ef14c776a1e8bfe-crimson-{debug,release} # there are also still older tags with just '-crimson' which ought to still # be handled here -SHA1_RE = re.compile(r'([0-9a-f]{40})(-crimson-debug|-crimson-release|-crimson|-aarch64)*') +#SHA1_RE = re.compile(r'(?P[0-9a-f]{40})(?P-crimson-debug|-crimson-release|-crimson|-aarch64)*') # ...but, now that we've added an option fromtag, and apparently # switched to arm62, it's more like -SHA1_RE = re.compile(r'([0-9a-f]{40})([a-z0-9-]+)*(-crimson-debug|-crimson-release|-crimson|-aarch64)*') +SHA1_RE = re.compile(r'(?P[0-9a-f]{40})(?P[a-z0-9-]+)*(?P-crimson-debug|-crimson-release|-crimson|-aarch64)*') def parse_full_quay_tag(tag, old=False): if old: mo = OLD_FULLTAG_RE.match(tag) - # parsed, ref, shortsha1, el, arch if not mo: return False, None, None, None, None - return True, mo[1], mo[2], mo[3], mo[4] - + return ( + True, + mo.group('ref'), + mo.group('shortsha1'), + mo.group('el'), + mo.group('arch'), + ) else: mo = FULLTAG_RE.match(tag) # parsed, ref, distro, el, arch if mo is None: return False, None, None, None, None - return True, mo[1], mo[2], mo[3], mo[4] + return ( + True, + mo.group('ref'), + mo.group('distro'), + mo.group('el'), + mo.group('arch'), + ) def parse_sha1_quay_tag(tag): @@ -61,7 +71,12 @@ def parse_sha1_quay_tag(tag): # parsed, sha1, fromtag, flav_or_arch if mo is None: return False, None, None, None - return True, mo[1], mo[2], mo[3] + return ( + True, + mo.group('sha1'), + mo.group('fromtag'), + mo.group('flav_or_arch'), + ) def get_all_quay_tags(quaytoken, start, npages): @@ -79,7 +94,7 @@ def get_all_quay_tags(quaytoken, start, npages): while has_additional and page < page_limit: try: response = requests.get( - '/'.join((QUAYBASE, 'repository', REPO, 'tag')), + '/'.join((QUAYAPI, 'repository', REPO, 'tag')), params={'page': page, 'limit': 98, 'onlyActiveTags': 'true'}, headers=headers, timeout=28, @@ -87,7 +102,7 @@ def get_all_quay_tags(quaytoken, start, npages): response.raise_for_status() except requests.exceptions.RequestException as e: print( - 'Quay.io request', + 'quay request', response.url, 'failed:', e, @@ -205,14 +220,14 @@ def sha1_present_in_shaman(sha1, verbose): return False -def delete_from_quay(tagname, date, quaytoken, dryrun): - if dryrun: +def delete_from_quay(tagname, date, quaytoken, dry_run): + if dry_run: print('Would delete from quay:', tagname, date) return try: response = requests.delete( - '/'.join((QUAYBASE, 'repository', REPO, 'tag', tagname)), + '/'.join((QUAYAPI, 'repository', REPO, 'tag', tagname)), headers={'Authorization': 'Bearer %s' % quaytoken}, timeout=28, ) -- 2.47.3