From a63b030ded2f3ea885883530eb06549356655955 Mon Sep 17 00:00:00 2001 From: Adam King Date: Fri, 10 Jul 2020 08:09:39 -0400 Subject: [PATCH] mgr/cephadm: allow use of authenticated registry Add option to use custom authenticated registry during bootstrap as well as a registry-login command in order to let user change authenticated registry login info Fixes: https://tracker.ceph.com/issues/44886 Signed-off-by: Adam King --- doc/cephadm/install.rst | 12 ++- doc/man/8/cephadm.rst | 42 ++++++++- src/cephadm/cephadm | 77 +++++++++++++++++ src/cephadm/tests/test_cephadm.py | 48 +++++++++++ src/pybind/mgr/cephadm/inventory.py | 14 +++ src/pybind/mgr/cephadm/module.py | 91 ++++++++++++++++++++ src/pybind/mgr/cephadm/tests/test_cephadm.py | 46 ++++++++++ 7 files changed, 327 insertions(+), 3 deletions(-) diff --git a/doc/cephadm/install.rst b/doc/cephadm/install.rst index 6b943b1ca93..fe60972923b 100644 --- a/doc/cephadm/install.rst +++ b/doc/cephadm/install.rst @@ -108,8 +108,16 @@ or run ``cephadm bootstrap -h`` to see all available options: * You can choose the ssh user cephadm will use to connect to hosts by using the ``--ssh-user **`` option. The ssh key will be added to ``/home/**/.ssh/authorized_keys``. This user will require - passwordless sudo access. + passwordless sudo access. +* If you are using a container on an authenticated registry that requires + login you may add the three arguments ``--registry-url ``, + ``--registry-username ``, + ``--registry-password `` OR + ``--registry-json ``. Cephadm will attempt + to login to this registry so it may pull your container and then store + the login info in its config database so other hosts added to the cluster + may also make use of the authenticated registry. Enable Ceph CLI =============== @@ -422,4 +430,4 @@ See :ref:`orchestrator-cli-placement-spec` for details of the placement specific Deploying custom containers =========================== -It is also possible to choose different containers than the default containers to deploy Ceph. See :ref:`containers` for information about your options in this regard. +It is also possible to choose different containers than the default containers to deploy Ceph. See :ref:`containers` for information about your options in this regard. diff --git a/doc/man/8/cephadm.rst b/doc/man/8/cephadm.rst index 6c7ddb2508e..60b4535f11b 100644 --- a/doc/man/8/cephadm.rst +++ b/doc/man/8/cephadm.rst @@ -64,7 +64,7 @@ Synopsis | [--dashboard-crt DASHBOARD_CRT] | [--ssh-config SSH_CONFIG] | [--ssh-private-key SSH_PRIVATE_KEY] -| [--ssh-public-key SSH_PUBLIC_KEY] +| [--ssh-public-key SSH_PUBLIC_KEY] | [--ssh-user SSH_USER] [--skip-mon-network] | [--skip-dashboard] [--dashboard-password-noupdate] | [--no-minimize-config] [--skip-ping-check] @@ -72,6 +72,10 @@ Synopsis | [--allow-fqdn-hostname] [--skip-prepare-host] | [--orphan-initial-daemons] [--skip-monitoring-stack] | [--apply-spec APPLY_SPEC] +| [--registry-url REGISTRY_URL] +| [--registry-username REGISTRY_USERNAME] +| [--registry-password REGISTRY_PASSWORD] +| [--registry-json REGISTRY_JSON] @@ -93,6 +97,10 @@ Synopsis | **cephadm** **install** [-h] [packages [packages ...]] +| **cephadm** **registry-login** [-h] [--registry-url REGISTRY_URL] +| [--registry-username REGISTRY_USERNAME] +| [--registry-password REGISTRY_PASSWORD] +| [--registry-json REGISTRY_JSON] [--fsid FSID] @@ -221,6 +229,10 @@ Arguments: * [--orphan-initial-daemons] Do not create initial mon, mgr, and crash service specs * [--skip-monitoring-stack] Do not automatically provision monitoring stack] (prometheus, grafana, alertmanager, node-exporter) * [--apply-spec APPLY_SPEC] Apply cluster spec after bootstrap (copy ssh key, add hosts and apply services) +* [--registry-url REGISTRY_URL] url of custom registry to login to. e.g. docker.io, quay.io +* [--registry-username REGISTRY_USERNAME] username of account to login to on custom registry +* [--registry-password REGISTRY_PASSWORD] password of account to login to on custom registry +* [--registry-json REGISTRY_JSON] JSON file containing registry login info (see registry-login command documentation) ceph-volume ----------- @@ -360,6 +372,34 @@ Pull the ceph image:: cephadm pull +registry-login +-------------- + +Give cephadm login information for an authenticated registry (url, username and password). +Cephadm will attempt to log the calling host into that registry:: + + cephadm registry-login --registry-url [REGISTRY_URL] --registry-username [USERNAME] + --registry-password [PASSWORD] + +Can also use a JSON file containing the login info formatted as:: + + { + "url":"REGISTRY_URL", + "username":"REGISTRY_USERNAME", + "password":"REGISTRY_PASSWORD" + } + +and turn it in with command:: + + cephadm registry-login --registry-json [JSON FILE] + +Arguments: + +* [--registry-url REGISTRY_URL] url of registry to login to. e.g. docker.io, quay.io +* [--registry-username REGISTRY_USERNAME] username of account to login to on registry +* [--registry-password REGISTRY_PASSWORD] password of account to login to on registry +* [--registry-json REGISTRY_JSON] JSON file containing login info for custom registry +* [--fsid FSID] cluster FSID rm-daemon --------- diff --git a/src/cephadm/cephadm b/src/cephadm/cephadm index e3a9cd39d2f..98314900744 100755 --- a/src/cephadm/cephadm +++ b/src/cephadm/cephadm @@ -2563,6 +2563,9 @@ def command_bootstrap(): cp.write(cpf) config = cpf.getvalue() + if args.registry_json or args.registry_url: + command_registry_login() + if not args.skip_pull: _pull_image(args.image) @@ -2875,6 +2878,11 @@ def command_bootstrap(): logger.info('Deploying %s service with default placement...' % t) cli(['orch', 'apply', t]) + if args.registry_url and args.registry_username and args.registry_password: + cli(['config', 'set', 'mgr', 'mgr/cephadm/registry_url', args.registry_url]) + cli(['config', 'set', 'mgr', 'mgr/cephadm/registry_username', args.registry_username]) + cli(['config', 'set', 'mgr', 'mgr/cephadm/registry_password', args.registry_password]) + if not args.skip_dashboard: logger.info('Enabling the dashboard module...') cli(['mgr', 'module', 'enable', 'dashboard']) @@ -2948,6 +2956,43 @@ def command_bootstrap(): ################################## +def command_registry_login(): + if args.registry_json: + logger.info("Pulling custom registry login info from %s." % args.registry_json) + d = get_parm(args.registry_json) + if d.get('url') and d.get('username') and d.get('password'): + args.registry_url = d.get('url') + args.registry_username = d.get('username') + args.registry_password = d.get('password') + registry_login(args.registry_url, args.registry_username, args.registry_password) + else: + raise Error("json provided for custom registry login did not include all necessary fields. " + "Please setup json file as\n" + "{\n" + " \"url\": \"REGISTRY_URL\",\n" + " \"username\": \"REGISTRY_USERNAME\",\n" + " \"password\": \"REGISTRY_PASSWORD\"\n" + "}\n") + elif args.registry_url and args.registry_username and args.registry_password: + registry_login(args.registry_url, args.registry_username, args.registry_password) + else: + raise Error("Invalid custom registry arguments received. To login to a custom registry include " + "--registry-url, --registry-username and --registry-password " + "options or --registry-json option") + return 0 + +def registry_login(url, username, password): + logger.info("Logging into custom registry.") + try: + out, _, _ = call_throws([container_path, 'login', + '-u', username, + '-p', password, + url]) + except: + raise Error("Failed to login to custom registry @ %s as %s with given password" % (args.registry_url, args.registry_username)) + +################################## + def extract_uid_gid_monitoring(daemon_type): # type: (str) -> Tuple[int, int] @@ -4902,6 +4947,19 @@ def _get_parser(): metavar='CEPH_SOURCE_FOLDER', help='Development mode. Several folders in containers are volumes mapped to different sub-folders in the ceph source folder') + parser_bootstrap.add_argument( + '--registry-url', + help='url for custom registry') + parser_bootstrap.add_argument( + '--registry-username', + help='username for custom registry') + parser_bootstrap.add_argument( + '--registry-password', + help='password for custom registry') + parser_bootstrap.add_argument( + '--registry-json', + help='json file with custom registry login info (URL, Username, Password)') + parser_deploy = subparsers.add_parser( 'deploy', help='deploy a daemon') parser_deploy.set_defaults(func=command_deploy) @@ -4992,6 +5050,25 @@ def _get_parser(): default=['cephadm'], help='packages') + parser_registry_login = subparsers.add_parser( + 'registry-login', help='log host into authenticated registry') + parser_registry_login.set_defaults(func=command_registry_login) + parser_registry_login.add_argument( + '--registry-url', + help='url for custom registry') + parser_registry_login.add_argument( + '--registry-username', + help='username for custom registry') + parser_registry_login.add_argument( + '--registry-password', + help='password for custom registry') + parser_registry_login.add_argument( + '--registry-json', + help='json file with custom registry login info (URL, Username, Password)') + parser_registry_login.add_argument( + '--fsid', + help='cluster FSID') + return parser diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py index 83969097a79..e37acc79758 100644 --- a/src/cephadm/tests/test_cephadm.py +++ b/src/cephadm/tests/test_cephadm.py @@ -164,3 +164,51 @@ default via fe80::2480:28ec:5097:3fe2 dev wlp2s0 proto ra metric 20600 pref medi "ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffg", "1:2:3:4:5:6:7:8:9", "fd00::1::1", "[fg::1]"): assert not cd.is_ipv6(bad) + + @mock.patch('cephadm.call_throws') + @mock.patch('cephadm.get_parm') + def test_registry_login(self, get_parm, call_throws): + + # test normal valid login with url, username and password specified + call_throws.return_value = '', '', 0 + args = cd._parse_args(['registry-login', '--registry-url', 'sample-url', '--registry-username', 'sample-user', '--registry-password', 'sample-pass']) + cd.args = args + retval = cd.command_registry_login() + assert retval == 0 + + # test bad login attempt with invalid arguments given + args = cd._parse_args(['registry-login', '--registry-url', 'bad-args-url']) + cd.args = args + with pytest.raises(Exception) as e: + assert cd.command_registry_login() + assert str(e.value) == ('Invalid custom registry arguments received. To login to a custom registry include ' + '--registry-url, --registry-username and --registry-password options or --registry-json option') + + # test normal valid login with json file + get_parm.return_value = {"url": "sample-url", "username": "sample-username", "password": "sample-password"} + args = cd._parse_args(['registry-login', '--registry-json', 'sample-json']) + cd.args = args + retval = cd.command_registry_login() + assert retval == 0 + + # test bad login attempt with bad json file + get_parm.return_value = {"bad-json": "bad-json"} + args = cd._parse_args(['registry-login', '--registry-json', 'sample-json']) + cd.args = args + with pytest.raises(Exception) as e: + assert cd.command_registry_login() + assert str(e.value) == ("json provided for custom registry login did not include all necessary fields. " + "Please setup json file as\n" + "{\n" + " \"url\": \"REGISTRY_URL\",\n" + " \"username\": \"REGISTRY_USERNAME\",\n" + " \"password\": \"REGISTRY_PASSWORD\"\n" + "}\n") + + # test login attempt with valid arguments where login command fails + call_throws.side_effect = Exception + args = cd._parse_args(['registry-login', '--registry-url', 'sample-url', '--registry-username', 'sample-user', '--registry-password', 'sample-pass']) + cd.args = args + with pytest.raises(Exception) as e: + cd.command_registry_login() + assert str(e.value) == "Failed to login to custom registry @ sample-url as sample-user with given password" diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py index 7bfde531d62..d9d6d5825e5 100644 --- a/src/pybind/mgr/cephadm/inventory.py +++ b/src/pybind/mgr/cephadm/inventory.py @@ -179,6 +179,7 @@ class HostCache(): self.last_host_check = {} # type: Dict[str, datetime.datetime] self.loading_osdspec_preview = set() # type: Set[str] self.etc_ceph_ceph_conf_refresh_queue: Set[str] = set() + self.registry_login_queue: Set[str] = set() def load(self): # type: () -> None @@ -221,6 +222,7 @@ class HostCache(): self.last_host_check[host] = datetime.datetime.strptime( j['last_host_check'], DATEFMT) self.etc_ceph_ceph_conf_refresh_queue.add(host) + self.registry_login_queue.add(host) self.mgr.log.debug( 'HostCache.load: host %s has %d daemons, ' '%d devices, %d networks' % ( @@ -266,6 +268,7 @@ class HostCache(): self.device_refresh_queue.append(host) self.osdspec_previews_refresh_queue.append(host) self.etc_ceph_ceph_conf_refresh_queue.add(host) + self.registry_login_queue.add(host) def invalidate_host_daemons(self, host): # type: (str) -> None @@ -283,6 +286,9 @@ class HostCache(): def distribute_new_etc_ceph_ceph_conf(self): self.etc_ceph_ceph_conf_refresh_queue = set(self.mgr.inventory.keys()) + + def distribute_new_registry_login_info(self): + self.registry_login_queue = set(self.mgr.inventory.keys()) def save_host(self, host): # type: (str) -> None @@ -441,6 +447,14 @@ class HostCache(): # self.etc_ceph_ceph_conf_refresh_queue.remove(host) return True return False + + def host_needs_registry_login(self, host): + if host in self.mgr.offline_hosts: + return False + if host in self.registry_login_queue: + self.registry_login_queue.remove(host) + return True + return False def remove_host_needs_new_etc_ceph_ceph_conf(self, host): self.etc_ceph_ceph_conf_refresh_queue.remove(host) diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 125fc0ac0f7..d8832e2aeab 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -230,6 +230,24 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, 'default': False, 'desc': 'Manage and own /etc/ceph/ceph.conf on the hosts.', }, + { + 'name': 'registry_url', + 'type': 'str', + 'default': None, + 'desc': 'Custom repository url' + }, + { + 'name': 'registry_username', + 'type': 'str', + 'default': None, + 'desc': 'Custom repository username' + }, + { + 'name': 'registry_password', + 'type': 'str', + 'default': None, + 'desc': 'Custom repository password' + }, ] def __init__(self, *args, **kwargs): @@ -265,6 +283,9 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, self.migration_current = None self.config_dashboard = True self.manage_etc_ceph_ceph_conf = True + self.registry_url: Optional[str] = None + self.registry_username: Optional[str] = None + self.registry_password: Optional[str] = None self._cons = {} # type: Dict[str, Tuple[remoto.backends.BaseConnection,remoto.backends.LegacyModuleExecute]] @@ -363,6 +384,20 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, self.log.debug('_kick_serve_loop') self.event.set() + # function responsible for logging single host into custom registry + def _registry_login(self, host, url, username, password): + self.log.debug(f"Attempting to log host {host} into custom registry @ {url}") + # want to pass info over stdin rather than through normal list of args + args_str = ("{\"url\": \"" + url + "\", \"username\": \"" + username + "\", " + " \"password\": \"" + password + "\"}") + out, err, code = self._run_cephadm( + host, 'mon', 'registry-login', + ['--registry-json', '-'], stdin=args_str, error_ok=True) + if code: + return f"Host {host} failed to login to {url} as {username} with given password" + return + + def _check_host(self, host): if host not in self.inventory: return @@ -831,6 +866,50 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, self.log.info(msg) return 0, msg, '' + @orchestrator._cli_read_command( + 'cephadm registry-login', + "name=url,type=CephString,req=false " + "name=username,type=CephString,req=false " + "name=password,type=CephString,req=false", + 'Set custom registry login info by providing url, username and password or json file with login info (-i )') + def registry_login(self, url=None, username=None, password=None, inbuf=None): + # if password not given in command line, get it through file input + if not (url and username and password) and (inbuf is None or len(inbuf) == 0): + return -errno.EINVAL, "", ("Invalid arguments. Please provide arguments " + "or -i ") + elif not (url and username and password): + login_info = json.loads(inbuf) + if "url" in login_info and "username" in login_info and "password" in login_info: + url = login_info["url"] + username = login_info["username"] + password = login_info["password"] + else: + return -errno.EINVAL, "", ("json provided for custom registry login did not include all necessary fields. " + "Please setup json file as\n" + "{\n" + " \"url\": \"REGISTRY_URL\",\n" + " \"username\": \"REGISTRY_USERNAME\",\n" + " \"password\": \"REGISTRY_PASSWORD\"\n" + "}\n") + # verify login info works by attempting login on random host + host = None + for host_name in self.inventory.keys(): + host = host_name + break + if not host: + raise OrchestratorError('no hosts defined') + r = self._registry_login(host, url, username, password) + if r is not None: + return 1, '', r + # if logins succeeded, store info + self.log.debug("Host logins successful. Storing login info.") + self.set_module_option('registry_url', url) + self.set_module_option('registry_username', username) + self.set_module_option('registry_password', password) + # distribute new login info to all hosts + self.cache.distribute_new_registry_login_info() + return 0, "registry login scheduled", '' + @orchestrator._cli_read_command( 'cephadm check-host', 'name=host,type=CephString ' @@ -1176,6 +1255,13 @@ you may want to run: r = self._refresh_host_daemons(host) if r: failures.append(r) + + if self.cache.host_needs_registry_login(host) and self.registry_url: + self.log.debug(f"Logging `{host}` into custom registry") + r = self._registry_login(host, self.registry_url, self.registry_username, self.registry_password) + if r: + bad_hosts.append(r) + if self.cache.host_needs_device_refresh(host): self.log.debug('refreshing %s devices' % host) r = self._refresh_host_devices(host) @@ -1739,6 +1825,9 @@ you may want to run: if self.allow_ptrace: daemon_spec.extra_args.append('--allow-ptrace') + if self.cache.host_needs_registry_login(daemon_spec.host) and self.registry_url: + self._registry_login(daemon_spec.host, self.registry_url, self.registry_username, self.registry_password) + self.log.info('%s daemon %s on %s' % ( 'Reconfiguring' if reconfig else 'Deploying', daemon_spec.name(), daemon_spec.host)) @@ -2257,6 +2346,8 @@ you may want to run: break if not host: raise OrchestratorError('no hosts defined') + if self.cache.host_needs_registry_login(host) and self.registry_url: + self._registry_login(host, self.registry_url, self.registry_username, self.registry_password) out, err, code = self._run_cephadm( host, '', 'pull', [], image=image_name, diff --git a/src/pybind/mgr/cephadm/tests/test_cephadm.py b/src/pybind/mgr/cephadm/tests/test_cephadm.py index 15ef2129374..1e46134f508 100644 --- a/src/pybind/mgr/cephadm/tests/test_cephadm.py +++ b/src/pybind/mgr/cephadm/tests/test_cephadm.py @@ -662,3 +662,49 @@ class TestCephadm(object): cephadm_module.notify('mon_map', mock.MagicMock()) assert cephadm_module.cache.host_needs_new_etc_ceph_ceph_conf('test') + + @mock.patch("cephadm.module.CephadmOrchestrator._run_cephadm") + def test_registry_login(self, _run_cephadm, cephadm_module: CephadmOrchestrator): + def check_registry_credentials(url, username, password): + assert cephadm_module.get_module_option('registry_url') == url + assert cephadm_module.get_module_option('registry_username') == username + assert cephadm_module.get_module_option('registry_password') == password + + _run_cephadm.return_value = '{}', '', 0 + with with_host(cephadm_module, 'test'): + # test successful login with valid args + code, out, err = cephadm_module.registry_login('test-url', 'test-user', 'test-password') + assert out == 'registry login scheduled' + assert err == '' + check_registry_credentials('test-url', 'test-user', 'test-password') + + # test bad login attempt with invalid args + code, out, err = cephadm_module.registry_login('bad-args') + assert err == ("Invalid arguments. Please provide arguments " + "or -i ") + check_registry_credentials('test-url', 'test-user', 'test-password') + + # test bad login using invalid json file + code, out, err = cephadm_module.registry_login(None, None, None, '{"bad-json": "bad-json"}') + assert err == ("json provided for custom registry login did not include all necessary fields. " + "Please setup json file as\n" + "{\n" + " \"url\": \"REGISTRY_URL\",\n" + " \"username\": \"REGISTRY_USERNAME\",\n" + " \"password\": \"REGISTRY_PASSWORD\"\n" + "}\n") + check_registry_credentials('test-url', 'test-user', 'test-password') + + # test good login using valid json file + good_json = ("{\"url\": \"" + "json-url" + "\", \"username\": \"" + "json-user" + "\", " + " \"password\": \"" + "json-pass" + "\"}") + code, out, err = cephadm_module.registry_login(None, None, None, good_json) + assert out == 'registry login scheduled' + assert err == '' + check_registry_credentials('json-url', 'json-user', 'json-pass') + + # test bad login where args are valid but login command fails + _run_cephadm.return_value = '{}', 'error', 1 + code, out, err = cephadm_module.registry_login('fail-url', 'fail-user', 'fail-password') + assert err == 'Host test failed to login to fail-url as fail-user with given password' + check_registry_credentials('json-url', 'json-user', 'json-pass') -- 2.39.5