quay-pruner: look for orphaned SHA1 tags and delete

This required the ability to query shaman for SHA1 alone as
well as the existing ref/sha1/CentOS-minor-version.  Add some
refactoring to pull out the actual shaman request, and some
caching to minimize the actual number of requests.

Signed-off-by: Dan Mick <dmick@redhat.com>

diff --git a/quay-pruner/build/prune-quay.py b/quay-pruner/build/prune-quay.py
index f755a5343b459a913105e7be566a4ca7e7a7ab5d..5524525acefb71e9ec20c4611f7d33d778456601 100755 (executable)
--- a/quay-pruner/build/prune-quay.py
+++ b/quay-pruner/build/prune-quay.py
@@ -9,10 +9,17 @@ import sys
 QUAYBASE = "https://quay.ceph.io/api/v1"
 REPO = "ceph-ci/ceph"
 
+# cache shaman search results so we only have to ask once
+short_sha1_cache = set()
+sha1_cache = set()
+
 # quay page ranges to fetch; hackable for testing
 start_page = 1
 page_limit = 100000
 
+NAME_RE = re.compile(r'(.*)-([0-9a-f]{7})-centos-([78])-x86_64-devel')
+SHA1_RE = re.compile(r'([0-9a-f]{40})(-crimson)*')
+
 
 def get_all_quay_tags(quaytoken):
     page = start_page
@@ -45,9 +52,6 @@ def get_all_quay_tags(quaytoken):
     return ret
 
 
-NAME_RE = re.compile(r'(.*)-([0-9a-f]{7})-centos-([78])-x86_64-devel')
-
-
 def parse_quay_tag(tag):
 
     mo = NAME_RE.match(tag)
@@ -59,25 +63,28 @@ def parse_quay_tag(tag):
     return ref, short_sha1, el
 
 
-def present_in_shaman(tag, verbose):
-    ref, short_sha1, el = parse_quay_tag(tag['name'])
-    if ref is None:
-        print("Can't parse name", tag['name'], file=sys.stderr)
-        return False
+def query_shaman(ref, sha1, el):
+
+    params = {
+        'flavor': 'default',
+        'status': 'ready',
+    }
+    if el:
+        params['distros'] = 'centos/{el}/x86_64'.format(el=el)
+    else:
+        params['distros'] = 'centos/7/x86_64,centos/8/x86_64,centos/9/x86_64'
+    if ref:
+        params['ref'] = ref
+    if sha1:
+        params['sha1'] = sha1
     try:
         response = requests.get(
             'https://shaman.ceph.com/api/search/',
-            params={
-                'ref': ref,
-                'distros': 'centos/{el}/x86_64'.format(el=el),
-                'flavor': 'default',
-                'status': 'ready',
-            },
+            params=params,
             timeout=30
         )
         response.raise_for_status()
     except requests.exceptions.RequestException as e:
-        # err on the side of caution; if there's some error, keep it
         print(
             'shaman request',
             response.url,
@@ -86,9 +93,27 @@ def present_in_shaman(tag, verbose):
             response.reason,
             file=sys.stderr
         )
+    return response
+
+
+def ref_present_in_shaman(tag, verbose):
+
+    ref, short_sha1, el = parse_quay_tag(tag['name'])
+    if ref is None:
+        print("Can't parse name", tag['name'], file=sys.stderr)
+        return False
+
+    if short_sha1 in short_sha1_cache:
+        if verbose:
+            print('Found %s in in_shaman cache' % short_sha1)
+        return True
+
+    response = query_shaman(ref, None, el)
     if not response.ok:
         print('shaman request', response.request.url, 'failed:',
               response.status_code, response.reason, file=sys.stderr)
+        # don't cache, but claim present:
+        # avoid deletion in case of transient shaman failure
         return True
 
     matches = response.json()
@@ -97,8 +122,36 @@ def present_in_shaman(tag, verbose):
     for match in matches:
         if match['sha1'][0:7] == short_sha1:
             if verbose:
-                print('Found matching build: ref %s sha1 %s quayname %s' %
-                      (match['ref'], match['sha1'], tag['name']))
+                print('Found match for %s: sha1 %s quayname %s' %
+                      (ref, match['sha1'], tag['name']))
+            short_sha1_cache.add(short_sha1)
+            return True
+    return False
+
+
+def sha1_present_in_shaman(sha1, verbose):
+
+    if sha1 in sha1_cache:
+        if verbose:
+            print('Found %s in in_shaman cache' % sha1)
+        return True
+
+    response = query_shaman(None, sha1, None)
+    if not response.ok:
+        print('shaman request', response.request.url, 'failed:',
+              response.status_code, response.reason, file=sys.stderr)
+        # don't cache, but claim present
+        # to avoid deleting on transient shaman failure
+        return True
+
+    matches = response.json()
+    if len(matches) == 0:
+        return False
+    for match in matches:
+        if match['sha1'] == sha1:
+            if verbose:
+                print('Found sha1 %s in shaman' % sha1)
+            sha1_cache.add(sha1)
             return True
     return False
 
@@ -151,7 +204,7 @@ def main():
     quaytags = get_all_quay_tags(quaytoken)
 
     # find all full tags to delete, put them and ref tag on list
-    tags_to_delete = list()
+    tags_to_delete = set()
     short_sha1s_to_delete = list()
     for tag in quaytags:
         if 'expiration' in tag or 'end_ts' in tag:
@@ -163,24 +216,41 @@ def main():
         if ref is None:
             continue
 
-        if present_in_shaman(tag, args.verbose):
+        if ref_present_in_shaman(tag, args.verbose):
             continue
 
         # accumulate full and ref tags to delete; keep list of short_sha1s
-        tags_to_delete.append(tag['name'])
+        tags_to_delete.add(tag['name'])
         if ref:
-            tags_to_delete.append(ref)
+            tags_to_delete.add(ref)
         if short_sha1:
             short_sha1s_to_delete.append(short_sha1)
 
     # now find all the full-sha1 tags to delete by making a second
     # pass and seeing if the tagname starts with a short_sha1 we
-    # know we want deleted
+    # know we want deleted, or if it matches SHA1_RE but is gone from
+    # shaman
     for tag in quaytags:
+
         if 'expiration' in tag or 'end_ts' in tag:
             continue
+
         if tag['name'][0:7] in short_sha1s_to_delete:
-            tags_to_delete.append(tag['name'])
+            if args.verbose:
+                print('Selecting %s: matches %s' %
+                      (tag['name'], tag['name'][0:7]))
+            tags_to_delete.add(tag['name'])
+            # already selected a SHA1 tag; no point in checking for orphaned
+            continue
+
+        match = SHA1_RE.match(tag['name'])
+        if match:
+            sha1 = match[1]
+            if sha1_present_in_shaman(sha1, args.verbose):
+                continue
+            if args.verbose:
+                print('Selecting %s: orphaned sha1 tag' % tag['name'])
+            tags_to_delete.add(tag['name'])
 
     if args.verbose:
         print('Deleting tags:', sorted(tags_to_delete))