From: Sebastian Wagner <sebastian.wagner@suse.com>
Date: Mon, 25 May 2020 16:13:50 +0000 (+0200)
Subject: qa/cephadm: Add local registry mirror
X-Git-Tag: v15.2.4~83^2^2~2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=41ffeed98cf1c52802bc8285241a0d4932e6e561;p=ceph.git

qa/cephadm: Add local registry mirror

Signed-off-by: Sebastian Wagner <sebastian.wagner@suse.com>
(cherry picked from commit d6edd8199f0ad3e34260e3d7eeb05e64ce726ca5)

Conflicts:
	qa/tasks/cephadm.py
	qa/tox.ini

Changes:
  qa/tox.ini converted to py2
---

diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py
index 2c64ca99725e..ee2e4ee6f6c6 100644
--- a/qa/tasks/cephadm.py
+++ b/qa/tasks/cephadm.py
@@ -12,6 +12,7 @@ import json
 import re
 import uuid
 
+import toml
 from ceph_manager import CephManager
 from tarfile import ReadError
 from teuthology import misc as teuthology
@@ -302,6 +303,7 @@ def ceph_bootstrap(ctx, config):
     ctx.cluster.run(args=[
         'sudo', 'chmod', '777', '/etc/ceph',
         ]);
+    add_mirror_to_cluster(ctx, config.get('docker_registry_mirror', 'vossi04.front.sepia.ceph.com:5000'))
     try:
         # write seed config
         log.info('Writing seed config...')
@@ -1110,3 +1112,52 @@ def task(ctx, config):
 
         finally:
             log.info('Teardown begin')
+
+
+def registries_add_mirror_to_docker_io(conf, mirror):
+    config = toml.loads(conf)
+    is_v1 = 'registries' in config
+    if is_v1:
+        search = config.get('registries', {}).get('search', {}).get('registries', [])
+        insecure = config.get('registries', {}).get('search', {}).get('insecure', [])
+        # v2: MutableMapping[str, Any] = { needs Python 3
+        v2 = {
+            'unqualified-search-registries': search,
+            'registry': [
+                {
+                    'prefix': reg,
+                    'location': reg,
+                    'insecure': reg in insecure,
+                    'blocked': False,
+                } for reg in search
+            ]
+        }
+    else:
+        v2 = config  # type: ignore
+    dockers = [r for r in v2['registry'] if r['prefix'] == 'docker.io']
+    if dockers:
+        docker = dockers[0]
+        docker['mirror'] = [{
+            "location": mirror,
+            "insecure": True,
+        }]
+    return v2
+
+
+def add_mirror_to_cluster(ctx, mirror):
+    log.info('Adding local image mirror %s' % mirror)
+    
+    registries_conf = '/etc/containers/registries.conf'
+    
+    for remote in ctx.cluster.remotes.keys():
+        config = teuthology.get_file(
+            remote=remote,
+            path=registries_conf
+        )
+        new_config = toml.dumps(registries_add_mirror_to_docker_io(config.decode('utf-8'), mirror))
+
+        teuthology.sudo_write_file(
+            remote=remote,
+            path=registries_conf,
+            data=new_config,
+        )
diff --git a/qa/tasks/tests/test_cephadm.py b/qa/tasks/tests/test_cephadm.py
new file mode 100644
index 000000000000..403d1915e0f6
--- /dev/null
+++ b/qa/tasks/tests/test_cephadm.py
@@ -0,0 +1,70 @@
+from tasks import cephadm
+
+v1 = """
+[registries.search]
+registries = ['registry.access.redhat.com', 'registry.redhat.io', 'docker.io', 'quay.io']
+
+[registries.insecure]
+registries = []
+"""
+
+v2 = """
+unqualified-search-registries = ["registry.access.redhat.com", "registry.redhat.io", "docker.io", 'quay.io']
+
+[[registry]]
+prefix = "registry.access.redhat.com"
+location = "registry.access.redhat.com"
+insecure = false
+blocked = false
+
+[[registry]]
+prefix = "registry.redhat.io"
+location = "registry.redhat.io"
+insecure = false
+blocked = false
+
+[[registry]]
+prefix = "docker.io"
+location = "docker.io"
+insecure = false
+blocked = false
+
+[[registry.mirror]]
+location = "vossi04.front.sepia.ceph.com:5000"
+insecure = true
+
+[[registry]]
+prefix = "quay.io"
+location = "quay.io"
+insecure = false
+blocked = false
+"""
+
+expected = {
+    'unqualified-search-registries': ['registry.access.redhat.com', 'registry.redhat.io',
+                                      'docker.io', 'quay.io'],
+    'registry': [
+        {'prefix': 'registry.access.redhat.com',
+         'location': 'registry.access.redhat.com',
+         'insecure': False,
+         'blocked': False},
+        {'prefix': 'registry.redhat.io',
+         'location': 'registry.redhat.io',
+         'insecure': False,
+         'blocked': False},
+        {'prefix': 'docker.io',
+         'location': 'docker.io',
+         'insecure': False,
+         'blocked': False,
+         'mirror': [{'location': 'vossi04.front.sepia.ceph.com:5000',
+                     'insecure': True}]},
+        {'prefix': 'quay.io',
+         'location': 'quay.io',
+         'insecure': False,
+         'blocked': False}
+    ]
+}
+
+def test_add_mirror():
+    assert cephadm.registries_add_mirror_to_docker_io(v1, 'vossi04.front.sepia.ceph.com:5000') == expected
+    assert cephadm.registries_add_mirror_to_docker_io(v2, 'vossi04.front.sepia.ceph.com:5000') == expected
diff --git a/qa/tox.ini b/qa/tox.ini
index 746cd1b4f44d..31a1ef4fd43d 100644
--- a/qa/tox.ini
+++ b/qa/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = flake8-py2, flake8-py3, mypy
+envlist = flake8-py2, flake8-py3, mypy, pytest
 skipsdist = True
 
 [testenv:flake8-py2]
@@ -18,3 +18,11 @@ commands=flake8 --select=F,E9 --exclude=venv,.tox
 basepython = python3
 deps = mypy==0.770
 commands = mypy {posargs:.}
+
+[testenv:pytest]
+basepython = python2
+deps =
+  {env:TEUTHOLOGY_GIT:git+https://github.com/ceph/teuthology.git@py2}#egg=teuthology[test]
+  httplib2
+  mock
+commands = pytest -vv tasks/tests