From: Sage Weil Date: Thu, 1 Apr 2021 18:14:13 +0000 (-0400) Subject: mgr/cephadm: update list-networks to report interface names too X-Git-Tag: v16.2.2~1^2~55 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=3237e485ef5ffe64d6920215d11786fbd8feaeb0;p=ceph.git mgr/cephadm: update list-networks to report interface names too Also, minor fix in the ipv6 addr reporting: ignore networks that aren't in CIDR form (no /). Signed-off-by: Sage Weil (cherry picked from commit 1897d1cd15af385bd888da0a9ee944cd3a68af07) --- diff --git a/src/cephadm/cephadm b/src/cephadm/cephadm index 25c4be5437bc5..eff5dbbc43b8a 100755 --- a/src/cephadm/cephadm +++ b/src/cephadm/cephadm @@ -3428,7 +3428,10 @@ def prepare_mon_addresses( if not ctx.skip_mon_network: # make sure IP is configured locally, and then figure out the # CIDR network - for net, ips in list_networks(ctx).items(): + for net, ifaces in list_networks(ctx).items(): + ips: List[str] = [] + for iface, ls in ifaces.items(): + ips.extend(ls) if ipaddress.ip_address(unwrap_ipv6(base_ip)) in \ [ipaddress.ip_address(ip) for ip in ips]: mon_network = net @@ -4540,7 +4543,7 @@ def command_logs(ctx): def list_networks(ctx): - # type: (CephadmContext) -> Dict[str,List[str]] + # type: (CephadmContext) -> Dict[str,Dict[str,List[str]]] # sadly, 18.04's iproute2 4.15.0-2ubun doesn't support the -j flag, # so we'll need to use a regex to parse 'ip' command output. @@ -4563,17 +4566,20 @@ def _list_ipv4_networks(ctx: CephadmContext): def _parse_ipv4_route(out): - r = {} # type: Dict[str,List[str]] - p = re.compile(r'^(\S+) (.*)scope link (.*)src (\S+)') + r = {} # type: Dict[str,Dict[str,List[str]]] + p = re.compile(r'^(\S+) dev (\S+) (.*)scope link (.*)src (\S+)') for line in out.splitlines(): m = p.findall(line) if not m: continue net = m[0][0] - ip = m[0][3] + iface = m[0][1] + ip = m[0][4] if net not in r: - r[net] = [] - r[net].append(ip) + r[net] = {} + if iface not in r[net]: + r[net][iface] = [] + r[net][iface].append(ip) return r @@ -4587,27 +4593,39 @@ def _list_ipv6_networks(ctx: CephadmContext): def _parse_ipv6_route(routes, ips): - r = {} # type: Dict[str,List[str]] + r = {} # type: Dict[str,Dict[str,List[str]]] route_p = re.compile(r'^(\S+) dev (\S+) proto (\S+) metric (\S+) .*pref (\S+)$') ip_p = re.compile(r'^\s+inet6 (\S+)/(.*)scope (.*)$') + iface_p = re.compile(r'^(\d+): (\S+): (.*)$') for line in routes.splitlines(): m = route_p.findall(line) if not m or m[0][0].lower() == 'default': continue net = m[0][0] + if '/' not in net: # only consider networks with a mask + continue + iface = m[0][1] if net not in r: - r[net] = [] + r[net] = {} + if iface not in r[net]: + r[net][iface] = [] + iface = None for line in ips.splitlines(): m = ip_p.findall(line) if not m: + m = iface_p.findall(line) + if m: + # drop @... suffix, if present + iface = m[0][1].split('@')[0] continue ip = m[0][0] # find the network it belongs to net = [n for n in r.keys() if ipaddress.ip_address(ip) in ipaddress.ip_network(n)] if net: - r[net[0]].append(ip) + assert(iface) + r[net[0]][iface].append(ip) return r diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py index 4689fa8789785..15b6efcc41d6b 100644 --- a/src/cephadm/tests/test_cephadm.py +++ b/src/cephadm/tests/test_cephadm.py @@ -191,11 +191,11 @@ default via 192.168.178.1 dev enxd89ef3f34260 proto dhcp metric 100 195.135.221.12 via 192.168.178.1 dev enxd89ef3f34260 proto static metric 100 """, { - '10.4.0.1': ['10.4.0.2'], - '172.17.0.0/16': ['172.17.0.1'], - '192.168.39.0/24': ['192.168.39.1'], - '192.168.122.0/24': ['192.168.122.1'], - '192.168.178.0/24': ['192.168.178.28'] + '10.4.0.1': {'tun0': ['10.4.0.2']}, + '172.17.0.0/16': {'docker0': ['172.17.0.1']}, + '192.168.39.0/24': {'virbr1': ['192.168.39.1']}, + '192.168.122.0/24': {'virbr0': ['192.168.122.1']}, + '192.168.178.0/24': {'enxd89ef3f34260': ['192.168.178.28']} } ), ( """ @@ -214,10 +214,10 @@ default via 10.3.64.1 dev eno1 proto static metric 100 192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1 linkdown """, { - '10.3.64.0/24': ['10.3.64.23', '10.3.64.27'], - '10.88.0.0/16': ['10.88.0.1'], - '172.21.3.1': ['172.21.3.2'], - '192.168.122.0/24': ['192.168.122.1']} + '10.3.64.0/24': {'eno1': ['10.3.64.23', '10.3.64.27']}, + '10.88.0.0/16': {'cni-podman0': ['10.88.0.1']}, + '172.21.3.1': {'tun0': ['172.21.3.2']}, + '192.168.122.0/24': {'virbr0': ['192.168.122.1']}} ), ]) def test_parse_ipv4_route(self, test_input, expected): @@ -227,61 +227,144 @@ default via 10.3.64.1 dev eno1 proto static metric 100 ( """ ::1 dev lo proto kernel metric 256 pref medium -fdbc:7574:21fe:9200::/64 dev wlp2s0 proto ra metric 600 pref medium -fdd8:591e:4969:6363::/64 dev wlp2s0 proto ra metric 600 pref medium -fde4:8dba:82e1::/64 dev eth1 proto kernel metric 256 expires 1844sec pref medium +fe80::/64 dev eno1 proto kernel metric 100 pref medium +fe80::/64 dev br-3d443496454c proto kernel metric 256 linkdown pref medium fe80::/64 dev tun0 proto kernel metric 256 pref medium -fe80::/64 dev wlp2s0 proto kernel metric 600 pref medium -default dev tun0 proto static metric 50 pref medium -default via fe80::2480:28ec:5097:3fe2 dev wlp2s0 proto ra metric 20600 pref medium +fe80::/64 dev br-4355f5dbb528 proto kernel metric 256 pref medium +fe80::/64 dev docker0 proto kernel metric 256 linkdown pref medium +fe80::/64 dev cni-podman0 proto kernel metric 256 linkdown pref medium +fe80::/64 dev veth88ba1e8 proto kernel metric 256 pref medium +fe80::/64 dev vethb6e5fc7 proto kernel metric 256 pref medium +fe80::/64 dev vethaddb245 proto kernel metric 256 pref medium +fe80::/64 dev vethbd14d6b proto kernel metric 256 pref medium +fe80::/64 dev veth13e8fd2 proto kernel metric 256 pref medium +fe80::/64 dev veth1d3aa9e proto kernel metric 256 pref medium +fe80::/64 dev vethe485ca9 proto kernel metric 256 pref medium """, """ 1: lo: mtu 65536 state UNKNOWN qlen 1000 - inet6 ::1/128 scope host + inet6 ::1/128 scope host + valid_lft forever preferred_lft forever +2: eno1: mtu 1500 state UP qlen 1000 + inet6 fe80::225:90ff:fee5:26e8/64 scope link noprefixroute + valid_lft forever preferred_lft forever +6: br-3d443496454c: mtu 1500 state DOWN + inet6 fe80::42:23ff:fe9d:ee4/64 scope link + valid_lft forever preferred_lft forever +7: br-4355f5dbb528: mtu 1500 state UP + inet6 fe80::42:6eff:fe35:41fe/64 scope link + valid_lft forever preferred_lft forever +8: docker0: mtu 1500 state DOWN + inet6 fe80::42:faff:fee6:40a0/64 scope link + valid_lft forever preferred_lft forever +11: tun0: mtu 1500 state UNKNOWN qlen 100 + inet6 fe80::98a6:733e:dafd:350/64 scope link stable-privacy + valid_lft forever preferred_lft forever +28: cni-podman0: mtu 1500 state DOWN qlen 1000 + inet6 fe80::3449:cbff:fe89:b87e/64 scope link + valid_lft forever preferred_lft forever +31: vethaddb245@if30: mtu 1500 state UP + inet6 fe80::90f7:3eff:feed:a6bb/64 scope link + valid_lft forever preferred_lft forever +33: veth88ba1e8@if32: mtu 1500 state UP + inet6 fe80::d:f5ff:fe73:8c82/64 scope link + valid_lft forever preferred_lft forever +35: vethbd14d6b@if34: mtu 1500 state UP + inet6 fe80::b44f:8ff:fe6f:813d/64 scope link + valid_lft forever preferred_lft forever +37: vethb6e5fc7@if36: mtu 1500 state UP + inet6 fe80::4869:c6ff:feaa:8afe/64 scope link valid_lft forever preferred_lft forever -2: eth0: mtu 1500 state UP qlen 1000 - inet6 fdd8:591e:4969:6363:4c52:cafe:8dd4:dc4/64 scope global temporary dynamic - valid_lft 86394sec preferred_lft 14394sec - inet6 fdbc:7574:21fe:9200:4c52:cafe:8dd4:dc4/64 scope global temporary dynamic - valid_lft 6745sec preferred_lft 3145sec - inet6 fdd8:591e:4969:6363:103a:abcd:af1f:57f3/64 scope global temporary deprecated dynamic - valid_lft 86394sec preferred_lft 0sec - inet6 fdbc:7574:21fe:9200:103a:abcd:af1f:57f3/64 scope global temporary deprecated dynamic - valid_lft 6745sec preferred_lft 0sec - inet6 fdd8:591e:4969:6363:a128:1234:2bdd:1b6f/64 scope global temporary deprecated dynamic - valid_lft 86394sec preferred_lft 0sec - inet6 fdbc:7574:21fe:9200:a128:1234:2bdd:1b6f/64 scope global temporary deprecated dynamic - valid_lft 6745sec preferred_lft 0sec - inet6 fdd8:591e:4969:6363:d581:4321:380b:3905/64 scope global temporary deprecated dynamic - valid_lft 86394sec preferred_lft 0sec - inet6 fdbc:7574:21fe:9200:d581:4321:380b:3905/64 scope global temporary deprecated dynamic - valid_lft 6745sec preferred_lft 0sec - inet6 fe80::1111:2222:3333:4444/64 scope link noprefixroute +39: veth13e8fd2@if38: mtu 1500 state UP + inet6 fe80::78f4:71ff:fefe:eb40/64 scope link + valid_lft forever preferred_lft forever +41: veth1d3aa9e@if40: mtu 1500 state UP + inet6 fe80::24bd:88ff:fe28:5b18/64 scope link + valid_lft forever preferred_lft forever +43: vethe485ca9@if42: mtu 1500 state UP + inet6 fe80::6425:87ff:fe42:b9f0/64 scope link + valid_lft forever preferred_lft forever +""", + { + "fe80::/64": { + "eno1": [ + "fe80::225:90ff:fee5:26e8" + ], + "br-3d443496454c": [ + "fe80::42:23ff:fe9d:ee4" + ], + "tun0": [ + "fe80::98a6:733e:dafd:350" + ], + "br-4355f5dbb528": [ + "fe80::42:6eff:fe35:41fe" + ], + "docker0": [ + "fe80::42:faff:fee6:40a0" + ], + "cni-podman0": [ + "fe80::3449:cbff:fe89:b87e" + ], + "veth88ba1e8": [ + "fe80::d:f5ff:fe73:8c82" + ], + "vethb6e5fc7": [ + "fe80::4869:c6ff:feaa:8afe" + ], + "vethaddb245": [ + "fe80::90f7:3eff:feed:a6bb" + ], + "vethbd14d6b": [ + "fe80::b44f:8ff:fe6f:813d" + ], + "veth13e8fd2": [ + "fe80::78f4:71ff:fefe:eb40" + ], + "veth1d3aa9e": [ + "fe80::24bd:88ff:fe28:5b18" + ], + "vethe485ca9": [ + "fe80::6425:87ff:fe42:b9f0" + ] + } + } + ), + ( +""" +::1 dev lo proto kernel metric 256 pref medium +2001:1458:301:eb::100:1a dev ens20f0 proto kernel metric 100 pref medium +2001:1458:301:eb::/64 dev ens20f0 proto ra metric 100 pref medium +fd01:1458:304:5e::/64 dev ens20f0 proto ra metric 100 pref medium +fe80::/64 dev ens20f0 proto kernel metric 100 pref medium +default proto ra metric 100 + nexthop via fe80::46ec:ce00:b8a0:d3c8 dev ens20f0 weight 1 + nexthop via fe80::46ec:ce00:b8a2:33c8 dev ens20f0 weight 1 pref medium +""", +""" +1: lo: mtu 65536 state UNKNOWN qlen 1000 + inet6 ::1/128 scope host valid_lft forever preferred_lft forever - inet6 fde4:8dba:82e1:0:ec4a:e402:e9df:b357/64 scope global temporary dynamic - valid_lft 1074sec preferred_lft 1074sec - inet6 fde4:8dba:82e1:0:5054:ff:fe72:61af/64 scope global dynamic mngtmpaddr - valid_lft 1074sec preferred_lft 1074sec -12: tun0: mtu 1500 state UNKNOWN qlen 100 - inet6 fe80::cafe:cafe:cafe:cafe/64 scope link stable-privacy +2: ens20f0: mtu 1500 state UP qlen 1000 + inet6 2001:1458:301:eb::100:1a/128 scope global dynamic noprefixroute + valid_lft 590879sec preferred_lft 590879sec + inet6 fe80::2e60:cff:fef8:da41/64 scope link noprefixroute valid_lft forever preferred_lft forever """, { - "::1": ["::1"], - "fdbc:7574:21fe:9200::/64": ["fdbc:7574:21fe:9200:4c52:cafe:8dd4:dc4", - "fdbc:7574:21fe:9200:103a:abcd:af1f:57f3", - "fdbc:7574:21fe:9200:a128:1234:2bdd:1b6f", - "fdbc:7574:21fe:9200:d581:4321:380b:3905"], - "fdd8:591e:4969:6363::/64": ["fdd8:591e:4969:6363:4c52:cafe:8dd4:dc4", - "fdd8:591e:4969:6363:103a:abcd:af1f:57f3", - "fdd8:591e:4969:6363:a128:1234:2bdd:1b6f", - "fdd8:591e:4969:6363:d581:4321:380b:3905"], - "fde4:8dba:82e1::/64": ["fde4:8dba:82e1:0:ec4a:e402:e9df:b357", - "fde4:8dba:82e1:0:5054:ff:fe72:61af"], - "fe80::/64": ["fe80::1111:2222:3333:4444", - "fe80::cafe:cafe:cafe:cafe"] + '2001:1458:301:eb::/64': { + 'ens20f0': [ + '2001:1458:301:eb::100:1a' + ], + }, + 'fe80::/64': { + 'ens20f0': ['fe80::2e60:cff:fef8:da41'], + }, + 'fd01:1458:304:5e::/64': { + 'ens20f0': [] + }, } - )]) + ), + ]) def test_parse_ipv6_route(self, test_routes, test_ips, expected): assert cd._parse_ipv6_route(test_routes, test_ips) == expected diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py index 7edef7282812c..11ed505b334d9 100644 --- a/src/pybind/mgr/cephadm/inventory.py +++ b/src/pybind/mgr/cephadm/inventory.py @@ -263,7 +263,7 @@ class HostCache(): self.last_facts_update = {} # type: Dict[str, datetime.datetime] self.osdspec_previews = {} # type: Dict[str, List[Dict[str, Any]]] self.osdspec_last_applied = {} # type: Dict[str, Dict[str, datetime.datetime]] - self.networks = {} # type: Dict[str, Dict[str, List[str]]] + self.networks = {} # type: Dict[str, Dict[str, Dict[str, List[str]]]] self.last_device_update = {} # type: Dict[str, datetime.datetime] self.last_device_change = {} # type: Dict[str, datetime.datetime] self.daemon_refresh_queue = [] # type: List[str] @@ -309,7 +309,7 @@ class HostCache(): orchestrator.DaemonDescription.from_json(d) for d in j.get('devices', []): self.devices[host].append(inventory.Device.from_json(d)) - self.networks[host] = j.get('networks', {}) + self.networks[host] = j.get('networks_and_interfaces', {}) self.osdspec_previews[host] = j.get('osdspec_previews', {}) for name, ts in j.get('osdspec_last_applied', {}).items(): self.osdspec_last_applied[host][name] = str_to_datetime(ts) @@ -358,8 +358,12 @@ class HostCache(): return True return False - def update_host_devices_networks(self, host, dls, nets): - # type: (str, List[inventory.Device], Dict[str,List[str]]) -> None + def update_host_devices_networks( + self, + host: str, + dls: List[inventory.Device], + nets: Dict[str, Dict[str, List[str]]] + ) -> None: if ( host not in self.devices or host not in self.last_device_change @@ -438,7 +442,7 @@ class HostCache(): for d in self.devices[host]: j['devices'].append(d.to_json()) if host in self.networks: - j['networks'] = self.networks[host] + j['networks_and_interfaces'] = self.networks[host] if host in self.daemon_config_deps: for name, depi in self.daemon_config_deps[host].items(): j['daemon_config_deps'][name] = { diff --git a/src/pybind/mgr/cephadm/schedule.py b/src/pybind/mgr/cephadm/schedule.py index ae14dc7f3e153..7583a2042fa64 100644 --- a/src/pybind/mgr/cephadm/schedule.py +++ b/src/pybind/mgr/cephadm/schedule.py @@ -60,7 +60,7 @@ class HostAssignment(object): spec, # type: ServiceSpec hosts: List[orchestrator.HostSpec], daemons: List[orchestrator.DaemonDescription], - networks: Dict[str, Dict[str, List[str]]] = {}, + networks: Dict[str, Dict[str, Dict[str, List[str]]]] = {}, filter_new_host=None, # type: Optional[Callable[[str],bool]] allow_colo: bool = False, ): @@ -204,7 +204,9 @@ class HostAssignment(object): def find_ip_on_host(self, hostname: str, subnets: List[str]) -> Optional[str]: for subnet in subnets: - ips = self.networks.get(hostname, {}).get(subnet, []) + ips: List[str] = [] + for iface, ips in self.networks.get(hostname, {}).get(subnet, {}).items(): + ips.extend(ips) if ips: return sorted(ips)[0] return None diff --git a/src/pybind/mgr/cephadm/tests/test_scheduling.py b/src/pybind/mgr/cephadm/tests/test_scheduling.py index 49cc2ff7102fd..bfcdba906f6c9 100644 --- a/src/pybind/mgr/cephadm/tests/test_scheduling.py +++ b/src/pybind/mgr/cephadm/tests/test_scheduling.py @@ -753,7 +753,7 @@ def test_node_assignment3(service_type, placement, hosts, class NodeAssignmentTest4(NamedTuple): spec: ServiceSpec - networks: Dict[str, Dict[str, List[str]]] + networks: Dict[str, Dict[str, Dict[str, List[str]]]] daemons: List[DaemonDescription] expected: List[str] expected_add: List[str] @@ -770,9 +770,9 @@ class NodeAssignmentTest4(NamedTuple): networks=['10.0.0.0/8'], ), { - 'host1': {'10.0.0.0/8': ['10.0.0.1']}, - 'host2': {'10.0.0.0/8': ['10.0.0.2']}, - 'host3': {'192.168.0.0/16': ['192.168.0.1']}, + 'host1': {'10.0.0.0/8': {'eth0': ['10.0.0.1']}}, + 'host2': {'10.0.0.0/8': {'eth0': ['10.0.0.2']}}, + 'host3': {'192.168.0.0/16': {'eth0': ['192.168.0.1']}}, }, [], ['host1(10.0.0.1:80)', 'host2(10.0.0.2:80)',