From 75f0ba5703200f4420a4b53d1c728167daf19909 Mon Sep 17 00:00:00 2001 From: Dan van der Ster Date: Tue, 11 Jun 2024 13:31:05 -0700 Subject: [PATCH] cephadm: disable ms_bind_ipv4 if we will enable ms_bind_ipv6 While bootstrapping an ipv6 cluster with an ipv6 initial mon, cephadm correctly enables ms_bind_ipv6=true. However it leaves ms_bind_ipv4 as it's default (true). As a result, daemons (osd, mds, ...) will attempt to bind to both ipv6 and ipv4. Usually this results in an osdmap and fsmap like the following: ``` osd.2 up in weight 1 up_from 26 up_thru 909 down_at 0 last_clean_interval [0,0) [v2:[xxxx:4f8:d0:4401:3::29]:6800/3680761436,v1:[xxxx:4f8:d0:4401:3::29]:6801/3680761436,v2:0.0.0.0:6802/3680761436,v1:0.0.0.0:6803/3680761436] [v2:[xxxx:4f8:d0:4401:3::29]:6804/3680761436,v1:[xxxx:4f8:d0:4401:3::29]:6805/3680761436,v2:0.0.0.0:6806/3680761436,v1:0.0.0.0:6807/3680761436] exists,up 0978a571-cd00-4eba-b00b-f863603a9a70 ``` ``` [mds.cephfs.ceph-test-3.isityv{-1:793154} state up:standby seq 1 join_fscid=1 addr [v2:[xxxx:4f8:d0:4401:3::29]:6832/2213688825,v1:[xxxx:4f8:d0:4401:3::29]:6833/2213688825,v2:0.0.0.0:6834/2213688825,v1:0.0.0.0:6835/2213688825] compat {c=[1],r=[1],i=[7ff]}] ``` Dual stack is not support by kernels (https://tracker.ceph.com/issues/49581) which leads to hard to debug issues for the end users. (corrupt map messages in dmesg). Fix by disabling ms_bind_ipv4 in the case ipv6 is desired. Fixes: https://tracker.ceph.com/issues/66436 Signed-off-by: Dan van der Ster Signed-off-by: Joshua Blanch --- src/cephadm/cephadm.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/cephadm/cephadm.py b/src/cephadm/cephadm.py index a3b0acdb64e8a..dbaec45c0ed47 100755 --- a/src/cephadm/cephadm.py +++ b/src/cephadm/cephadm.py @@ -2564,6 +2564,12 @@ def finish_bootstrap_config( if ipv6 or ipv6_cluster_network: logger.info('Enabling IPv6 (ms_bind_ipv6) binding') cli(['config', 'set', 'global', 'ms_bind_ipv6', 'true']) + # note: Ceph does not fully support dual stack. + # kernel clients: https://tracker.ceph.com/issues/49581 + # if we do not disable ipv4 binding, daemons will bind + # to 0.0.0.0 and clients will misbehave. + logger.info('Disabling IPv4 (ms_bind_ipv4) binding') + cli(['config', 'set', 'global', 'ms_bind_ipv4', 'false']) with open(ctx.output_config, 'w') as f: f.write(config) -- 2.39.5