From: Joao Eduardo Luis <joao@suse.com>
Date: Wed, 30 Dec 2020 19:19:03 +0000 (+0000)
Subject: cephadm: split some of bootstrap to other functions
X-Git-Tag: v17.0.0~8^2~14
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=6dec4d500817e14618682b336faba25f2df23fe6;p=ceph.git

cephadm: split some of bootstrap to other functions

Signed-off-by: Joao Eduardo Luis <joao@suse.com>
---

diff --git a/src/cephadm/cephadm b/src/cephadm/cephadm
index f4ae21875112e..0b7ab263c78ee 100755
--- a/src/cephadm/cephadm
+++ b/src/cephadm/cephadm
@@ -3097,60 +3097,9 @@ def is_ipv6(ctx, address):
         return False
 
 
-@default_image
-def command_bootstrap(ctx):
-    # type: (CephadmContext) -> int
-
-    args = ctx.args
-    host: Optional[str] = None
-
-    if not ctx.args.output_config:
-        ctx.args.output_config = os.path.join(ctx.args.output_dir, 'ceph.conf')
-    if not ctx.args.output_keyring:
-        ctx.args.output_keyring = os.path.join(ctx.args.output_dir,
-                                           'ceph.client.admin.keyring')
-    if not ctx.args.output_pub_ssh_key:
-        ctx.args.output_pub_ssh_key = os.path.join(ctx.args.output_dir, 'ceph.pub')
-
-    # verify output files
-    for f in [ctx.args.output_config, ctx.args.output_keyring,
-              ctx.args.output_pub_ssh_key]:
-        if not ctx.args.allow_overwrite:
-            if os.path.exists(f):
-                raise Error('%s already exists; delete or pass '
-                              '--allow-overwrite to overwrite' % f)
-        dirname = os.path.dirname(f)
-        if dirname and not os.path.exists(dirname):
-            fname = os.path.basename(f)
-            logger.info(f"Creating directory {dirname} for {fname}")
-            try:
-                # use makedirs to create intermediate missing dirs
-                os.makedirs(dirname, 0o755)
-            except PermissionError:
-                raise Error(f"Unable to create {dirname} due to permissions failure. Retry with root, or sudo or preallocate the directory.")
-
-
-    if not ctx.args.skip_prepare_host:
-        command_prepare_host(ctx)
-    else:
-        logger.info('Skip prepare_host')
-
-    # initial vars
-    fsid = ctx.args.fsid or make_fsid()
-    hostname = get_hostname()
-    if '.' in hostname and not ctx.args.allow_fqdn_hostname:
-        raise Error('hostname is a fully qualified domain name (%s); either fix (e.g., "sudo hostname %s" or similar) or pass --allow-fqdn-hostname' % (hostname, hostname.split('.')[0]))
-    mon_id = ctx.args.mon_id or hostname
-    mgr_id = ctx.args.mgr_id or generate_service_id()
-    logger.info('Cluster fsid: %s' % fsid)
-    ipv6 = False
-
-    l = FileLock(ctx, fsid)
-    l.acquire()
-
-    # ip
+def obtain_mon_ip(ctx: CephadmContext) -> Tuple[str, str]:
     r = re.compile(r':(\d+)$')
-    base_ip = ''
+    base_ip = ""
     if ctx.args.mon_ip:
         ipv6 = is_ipv6(ctx, ctx.args.mon_ip)
         if ipv6:
@@ -3171,8 +3120,8 @@ def command_bootstrap(ctx):
         else:
             base_ip = ctx.args.mon_ip
             addr_arg = '[v2:%s:3300,v1:%s:6789]' % (ctx.args.mon_ip, ctx.args.mon_ip)
-            check_ip_port(ctx, args.mon_ip, 3300)
-            check_ip_port(ctx, args.mon_ip, 6789)
+            check_ip_port(ctx, ctx.args.mon_ip, 3300)
+            check_ip_port(ctx, ctx.args.mon_ip, 6789)
     elif ctx.args.mon_addrv:
         addr_arg = ctx.args.mon_addrv
         if addr_arg[0] != '[' or addr_arg[-1] != ']':
@@ -3192,59 +3141,34 @@ def command_bootstrap(ctx):
     else:
         raise Error('must specify --mon-ip or --mon-addrv')
     logger.debug('Base mon IP is %s, final addrv is %s' % (base_ip, addr_arg))
+    return (base_ip, addr_arg)
 
-    mon_network = None
-    if not ctx.args.skip_mon_network:
-        # make sure IP is configured locally, and then figure out the
-        # CIDR network
-        for net, ips in list_networks(ctx).items():
-            if ipaddress.ip_address(unicode(unwrap_ipv6(base_ip))) in \
-                    [ipaddress.ip_address(unicode(ip)) for ip in ips]:
-                mon_network = net
-                logger.info('Mon IP %s is in CIDR network %s' % (base_ip,
-                                                                 mon_network))
-                break
-        if not mon_network:
-            raise Error('Failed to infer CIDR network for mon ip %s; pass '
-                        '--skip-mon-network to configure it later' % base_ip)
-
-    # config
-    cp = read_config(ctx.args.config)
-    if not cp.has_section('global'):
-        cp.add_section('global')
-    cp.set('global', 'fsid', fsid);
-    cp.set('global', 'mon host', addr_arg)
-    cp.set('global', 'container_image', ctx.args.image)
-    cpf = StringIO()
-    cp.write(cpf)
-    config = cpf.getvalue()
-
-    if ctx.args.registry_json or ctx.args.registry_url:
-        command_registry_login(ctx)
 
-    if not ctx.args.skip_pull:
-        _pull_image(ctx, ctx.args.image)
+def create_initial_keys(
+    ctx: CephadmContext,
+    uid: int, gid: int,
+    mgr_id: str
+) -> Tuple[str, str, str, Any, Any]: # type: ignore
 
-    logger.info('Extracting ceph user uid/gid from container image...')
-    (uid, gid) = extract_uid_gid(ctx)
+    _image = ctx.args.image
 
     # create some initial keys
     logger.info('Creating initial keys...')
     mon_key = CephContainer(
         ctx,
-        image=ctx.args.image,
+        image=_image,
         entrypoint='/usr/bin/ceph-authtool',
         args=['--gen-print-key'],
     ).run().strip()
     admin_key = CephContainer(
         ctx,
-        image=ctx.args.image,
+        image=_image,
         entrypoint='/usr/bin/ceph-authtool',
         args=['--gen-print-key'],
     ).run().strip()
     mgr_key = CephContainer(
         ctx,
-        image=ctx.args.image,
+        image=_image,
         entrypoint='/usr/bin/ceph-authtool',
         args=['--gen-print-key'],
     ).run().strip()
@@ -3265,12 +3189,24 @@ def command_bootstrap(ctx):
                '\tcaps osd = allow *\n'
                % (mon_key, admin_key, mgr_id, mgr_key))
 
+    admin_keyring = write_tmp('[client.admin]\n'
+                                  '\tkey = ' + admin_key + '\n',
+                                       uid, gid)
+
     # tmp keyring file
-    tmp_bootstrap_keyring = write_tmp(keyring, uid, gid)
+    bootstrap_keyring = write_tmp(keyring, uid, gid)
+    return (mon_key, mgr_key, admin_key,
+            bootstrap_keyring, admin_keyring)
+
 
-    # create initial monmap, tmp monmap file
+def create_initial_monmap(
+    ctx: CephadmContext,
+    uid: int, gid: int,
+    fsid: str,
+    mon_id: str, mon_addr: str
+) -> Any:
     logger.info('Creating initial monmap...')
-    tmp_monmap = write_tmp('', 0, 0)
+    monmap = write_tmp('', 0, 0)
     out = CephContainer(
         ctx,
         image=ctx.args.image,
@@ -3278,18 +3214,27 @@ def command_bootstrap(ctx):
         args=['--create',
               '--clobber',
               '--fsid', fsid,
-              '--addv', mon_id, addr_arg,
+              '--addv', mon_id, mon_addr,
               '/tmp/monmap'
         ],
         volume_mounts={
-            tmp_monmap.name: '/tmp/monmap:z',
+            monmap.name: '/tmp/monmap:z',
         },
     ).run()
+    logger.debug(f"monmaptool for {mon_id} {mon_addr} on {out}")
 
     # pass monmap file to ceph user for use by ceph-mon --mkfs below
-    os.fchown(tmp_monmap.fileno(), uid, gid)
+    os.fchown(monmap.fileno(), uid, gid)
+    return monmap
 
-    # create mon
+
+def create_mon(
+    ctx: CephadmContext,
+    uid: int, gid: int,
+    fsid: str, mon_id: str,
+    bootstrap_keyring_path: str,
+    monmap_path: str
+):
     logger.info('Creating mon...')
     create_daemon_dirs(ctx, fsid, 'mon', mon_id, uid, gid)
     mon_dir = get_data_dir(fsid, ctx.args.data_dir, 'mon', mon_id)
@@ -3308,10 +3253,113 @@ def command_bootstrap(ctx):
         volume_mounts={
             log_dir: '/var/log/ceph:z',
             mon_dir: '/var/lib/ceph/mon/ceph-%s:z' % (mon_id),
-            tmp_bootstrap_keyring.name: '/tmp/keyring:z',
-            tmp_monmap.name: '/tmp/monmap:z',
+            bootstrap_keyring_path: '/tmp/keyring:z',
+            monmap_path: '/tmp/monmap:z',
         },
     ).run()
+    logger.debug(f"create mon.{mon_id} on {out}")
+    return (mon_dir, log_dir)
+
+
+@default_image
+def command_bootstrap(ctx):
+    # type: (CephadmContext) -> int
+
+    args = ctx.args
+    host: Optional[str] = None
+
+    if not ctx.args.output_config:
+        ctx.args.output_config = os.path.join(ctx.args.output_dir, 'ceph.conf')
+    if not ctx.args.output_keyring:
+        ctx.args.output_keyring = os.path.join(ctx.args.output_dir,
+                                           'ceph.client.admin.keyring')
+    if not ctx.args.output_pub_ssh_key:
+        ctx.args.output_pub_ssh_key = os.path.join(ctx.args.output_dir, 'ceph.pub')
+
+    # verify output files
+    for f in [ctx.args.output_config, ctx.args.output_keyring,
+              ctx.args.output_pub_ssh_key]:
+        if not ctx.args.allow_overwrite:
+            if os.path.exists(f):
+                raise Error('%s already exists; delete or pass '
+                              '--allow-overwrite to overwrite' % f)
+        dirname = os.path.dirname(f)
+        if dirname and not os.path.exists(dirname):
+            fname = os.path.basename(f)
+            logger.info(f"Creating directory {dirname} for {fname}")
+            try:
+                # use makedirs to create intermediate missing dirs
+                os.makedirs(dirname, 0o755)
+            except PermissionError:
+                raise Error(f"Unable to create {dirname} due to permissions failure. Retry with root, or sudo or preallocate the directory.")
+
+
+    if not ctx.args.skip_prepare_host:
+        command_prepare_host(ctx)
+    else:
+        logger.info('Skip prepare_host')
+
+    # initial vars
+    fsid = ctx.args.fsid or make_fsid()
+    hostname = get_hostname()
+    if '.' in hostname and not ctx.args.allow_fqdn_hostname:
+        raise Error('hostname is a fully qualified domain name (%s); either fix (e.g., "sudo hostname %s" or similar) or pass --allow-fqdn-hostname' % (hostname, hostname.split('.')[0]))
+    mon_id = ctx.args.mon_id or hostname
+    mgr_id = ctx.args.mgr_id or generate_service_id()
+    logger.info('Cluster fsid: %s' % fsid)
+    ipv6 = False
+
+    l = FileLock(ctx, fsid)
+    l.acquire()
+
+    # ip
+    (base_ip, addr_arg) = obtain_mon_ip(ctx)    
+
+    mon_network = None
+    if not ctx.args.skip_mon_network:
+        # make sure IP is configured locally, and then figure out the
+        # CIDR network
+        for net, ips in list_networks(ctx).items():
+            if ipaddress.ip_address(unicode(unwrap_ipv6(base_ip))) in \
+                    [ipaddress.ip_address(unicode(ip)) for ip in ips]:
+                mon_network = net
+                logger.info('Mon IP %s is in CIDR network %s' % (base_ip,
+                                                                 mon_network))
+                break
+        if not mon_network:
+            raise Error('Failed to infer CIDR network for mon ip %s; pass '
+                        '--skip-mon-network to configure it later' % base_ip)
+
+    # config
+    cp = read_config(ctx.args.config)
+    if not cp.has_section('global'):
+        cp.add_section('global')
+    cp.set('global', 'fsid', fsid);
+    cp.set('global', 'mon host', addr_arg)
+    cp.set('global', 'container_image', ctx.args.image)
+    cpf = StringIO()
+    cp.write(cpf)
+    config = cpf.getvalue()
+
+    if ctx.args.registry_json or ctx.args.registry_url:
+        command_registry_login(ctx)
+
+    if not ctx.args.skip_pull:
+        _pull_image(ctx, ctx.args.image)
+
+    logger.info('Extracting ceph user uid/gid from container image...')
+    (uid, gid) = extract_uid_gid(ctx)
+
+    # create some initial keys
+    (mon_key, mgr_key, admin_key,
+     bootstrap_keyring, admin_keyring
+    ) = \
+        create_initial_keys(ctx, uid, gid, mgr_id)
+
+    monmap = create_initial_monmap(ctx, uid, gid, fsid, mon_id, addr_arg)
+    (mon_dir, log_dir) = \
+        create_mon(ctx, uid, gid, fsid, mon_id,
+                   bootstrap_keyring.name, monmap.name)
 
     with open(mon_dir + '/config', 'w') as f:
         os.fchown(f.fileno(), uid, gid)
@@ -3323,10 +3371,7 @@ def command_bootstrap(ctx):
     deploy_daemon(ctx, fsid, 'mon', mon_id, mon_c, uid, gid,
                   config=None, keyring=None)
 
-    # client.admin key + config to issue various CLI commands
-    tmp_admin_keyring = write_tmp('[client.admin]\n'
-                                  '\tkey = ' + admin_key + '\n',
-                                       uid, gid)
+    # config to issue various CLI commands
     tmp_config = write_tmp(config, uid, gid)
 
     # a CLI helper to reduce our typing
@@ -3334,7 +3379,7 @@ def command_bootstrap(ctx):
         # type: (List[str], Dict[str, str], Optional[int]) -> str
         mounts = {
             log_dir: '/var/log/ceph:z',
-            tmp_admin_keyring.name: '/etc/ceph/ceph.client.admin.keyring:z',
+            admin_keyring.name: '/etc/ceph/ceph.client.admin.keyring:z',
             tmp_config.name: '/etc/ceph/ceph.conf:z',
         }
         for k, v in extra_mounts.items():
@@ -3357,7 +3402,7 @@ def command_bootstrap(ctx):
             'status'],
         volume_mounts={
             mon_dir: '/var/lib/ceph/mon/ceph-%s:z' % (mon_id),
-            tmp_admin_keyring.name: '/etc/ceph/ceph.client.admin.keyring:z',
+            admin_keyring.name: '/etc/ceph/ceph.client.admin.keyring:z',
             tmp_config.name: '/etc/ceph/ceph.conf:z',
         },
     )