cephadm: Set ms bind ipv6 when mon-ip is ipv6

If you use cephadm bootstrap with an ipv6 mon ip then currently you'll
get into a address family split-brain state, where the mon's messenger
connects and binds to ipv6 but the mgr's binds to ipv4 (usually
0.0.0.0). In this state the bootstrap process hangs as it attempts to
talk and get the mgr state.

A work around is to have `ms bind ipv6 = true` in a ceph conf
you can then pass to bootstrap which gets pulled in and set in mon's
config store.

This patch sets `ms bind ipv6 = true` to the global section in the
mon config store when the mon-ip argument is an ipv6 address.

Fixes: https://tracker.ceph.com/issues/45016
Signed-off-by: Matthew Oliver <moliver@suse.com>

diff --git a/src/cephadm/cephadm b/src/cephadm/cephadm
index 7c0cb328ee270687a5e4fa68d95308cda8c70b7d..32a0ba1e8daa1a205a45e1f1d9b5b4df627e9b0d 100755 (executable)
--- a/src/cephadm/cephadm
+++ b/src/cephadm/cephadm
@@ -41,6 +41,7 @@ You can invoke cephadm in two ways:
 import argparse
 import datetime
 import fcntl
+import ipaddress
 import json
 import logging
 import os
@@ -82,6 +83,9 @@ if sys.version_info >= (3, 0):
 else:
     from urllib2 import urlopen, HTTPError
 
+if sys.version_info > (3, 0):
+    unicode = str
+
 container_path = ''
 cached_stdin = None
 
@@ -2267,6 +2271,17 @@ def command_inspect_image():
 
 ##################################
 
+def is_ipv6(address):
+    # type: (str) -> bool
+    if address.startswith('[') and address.endswith(']'):
+        address = address[1:-1]
+    try:
+        return ipaddress.ip_address(unicode(address)).version == 6
+    except ValueError:
+        logger.warning("Address: {} isn't a valid IP address".format(address))
+        return False
+
+
 @default_image
 def command_bootstrap():
     # type: () -> int
@@ -2302,6 +2317,7 @@ def command_bootstrap():
     mon_id = args.mon_id or hostname
     mgr_id = args.mgr_id or generate_service_id()
     logging.info('Cluster fsid: %s' % fsid)
+    ipv6 = False
 
     l = FileLock(fsid)
     l.acquire()
@@ -2310,6 +2326,7 @@ def command_bootstrap():
     r = re.compile(r':(\d+)$')
     base_ip = None
     if args.mon_ip:
+        ipv6 = is_ipv6(args.mon_ip)
         hasport = r.findall(args.mon_ip)
         if hasport:
             port = int(hasport[0])
@@ -2333,6 +2350,7 @@ def command_bootstrap():
         if addr_arg[0] != '[' or addr_arg[-1] != ']':
             raise Error('--mon-addrv value %s must use square backets' %
                         addr_arg)
+        ipv6 = addr_arg.count('[') > 1
         for addr in addr_arg[1:-1].split(','):
             hasport = r.findall(addr)
             if not hasport:
@@ -2546,6 +2564,10 @@ def command_bootstrap():
         logger.info('Setting mon public_network...')
         cli(['config', 'set', 'mon', 'public_network', mon_network])
 
+    if ipv6:
+        logger.info('Enabling IPv6 (ms_bind_ipv6)')
+        cli(['config', 'set', 'global', 'ms_bind_ipv6', 'true'])
+
     # create mgr
     logger.info('Creating mgr...')
     mgr_keyring = '[mgr.%s]\n\tkey = %s\n' % (mgr_id, mgr_key)
@@ -2699,7 +2721,7 @@ def command_bootstrap():
                         get_fqdn(), port,
                         args.initial_dashboard_user,
                         password))
-    
+
     if args.apply_spec:
         logger.info('Applying %s to cluster' % args.apply_spec)
 
@@ -4624,7 +4646,6 @@ def _get_parser():
         '--apply-spec',
         help='Apply cluster spec after bootstrap (copy ssh key, add hosts and apply services)')
 
-
     parser_bootstrap.add_argument(
         '--shared_ceph_folder',
         metavar='CEPH_SOURCE_FOLDER',

diff --git a/src/cephadm/requirements.txt b/src/cephadm/requirements.txt
new file mode 100644 (file)
index 0000000..9271687
--- /dev/null
+++ b/src/cephadm/requirements.txt
@@ -0,0 +1 @@
+ipaddress ; python_version < '3.3'

diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py
index 785aedac7820f102d3b3cff7e4db881f05eb0d8b..b44f0b72219ffa199eae4d066ab68e3459d73370 100644 (file)
--- a/src/cephadm/tests/test_cephadm.py
+++ b/src/cephadm/tests/test_cephadm.py
@@ -92,3 +92,13 @@ default via 10.3.64.1 dev eno1 proto static metric 100
     ])
     def test_parse_ip_route(self, test_input, expected):
         assert cd._parse_ip_route(test_input) == expected
+
+    def test_is_ipv6(self):
+        cd.logger = mock.Mock()
+        for good in ("[::1]", "::1",
+                     "fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"):
+            assert cd.is_ipv6(good)
+        for bad in ("127.0.0.1",
+                    "ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffg",
+                    "1:2:3:4:5:6:7:8:9", "fd00::1::1", "[fg::1]"):
+            assert not cd.is_ipv6(bad)

diff --git a/src/cephadm/tox.ini b/src/cephadm/tox.ini
index 59d94a4c4c0f84c002189fe1031cde27f1ac6d60..579fdef47f1fa7ce073d47cad360db68080ee51f 100644 (file)
--- a/src/cephadm/tox.ini
+++ b/src/cephadm/tox.ini
@@ -7,6 +7,7 @@ skip_install=true
 deps =
   pytest
   mock
+  -r{toxinidir}/requirements.txt
 commands=pytest {posargs}
 
 [testenv:mypy]