From: Sage Weil Date: Mon, 12 Apr 2021 21:21:33 +0000 (-0400) Subject: mgr/cephadm: ingress: add optional virtual_interface_networks X-Git-Tag: v17.1.0~2265^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=bbf6a12752092b406abbec1e600533366ac59548;p=ceph.git mgr/cephadm: ingress: add optional virtual_interface_networks It may be that the virtual IP we want to use is not in the same network as any existing IPs on the host. In that case, allow the spec to specify a list of networks to match against existing IPs so that a match will identify an ethernet interface to use. Signed-off-by: Sage Weil --- diff --git a/doc/cephadm/rgw.rst b/doc/cephadm/rgw.rst index 31084ff7d0a1..1815776dfb69 100644 --- a/doc/cephadm/rgw.rst +++ b/doc/cephadm/rgw.rst @@ -112,10 +112,6 @@ elected as master, and the virtual IP will be moved to that node. The active haproxy acts like a load balancer, distributing all RGW requests between all the RGW daemons available. -.. note:: The virtual IP will be configured on an ethernet interface on the host - that has an existing IP in the same subnet. (If there are multiple such - interfaces, cephadm will choose the "first" one it sees.) - **Prerequisites:** * An existing RGW service, without SSL. (If you want SSL service, the certificate @@ -145,6 +141,7 @@ It is a yaml format file with the following properties: virtual_ip: / # ex: 192.168.20.1/24 frontend_port: # ex: 8080 monitor_port: # ex: 1967, used by haproxy for load balancer status + virtual_interface_networks: [ ... ] # optional: list of CIDR networks ssl_cert: | # optional: SSL certificate and key -----BEGIN CERTIFICATE----- ... @@ -166,12 +163,47 @@ where the properties of this service specification are: to match the nodes where RGW is deployed. * ``virtual_ip`` The virtual IP (and network) in CIDR format where the ingress service will be available. +* ``virtual_interface_networks`` + A list of networks to identify which ethernet interface to use for the virtual IP. * ``frontend_port`` The port used to access the ingress service. * ``ssl_cert``: SSL certificate, if SSL is to be enabled. This must contain the both the certificate and private key blocks in .pem format. +**Selecting ethernet interfaces for the virtual IP:** + +You cannot simply provide the name of the network interface on which +to configure the virtual IP because interface names tend to vary +across hosts (and/or reboots). Instead, cephadm will select +interfaces based on other existing IP addresses that are already +configured. + +Normally, the virtual IP will be configured on the first network +interface that has an existing IP in the same subnet. For example, if +the virtual IP is 192.168.0.80/24 and eth2 has the static IP +192.168.0.40/24, cephadm will use eth2. + +In some cases, the virtual IP may not belong to the same subnet as an existing static +IP. In such cases, you can provide a list of subnets to match against existing IPs, +and cephadm will put the virtual IP on the first network interface to match. For example, +if the virtual IP is 192.168.0.80/24 and we want it on the same interface as the machine's +static IP in 10.10.0.0/16, you can use a spec like:: + + service_type: ingress + service_id: rgw.something + spec: + virtual_ip: 192.168.0.80/24 + virtual_interface_networks: + - 10.10.0.0/16 + ... + +A consequence of this strategy is that you cannot currently configure the virtual IP +on an interface that has no existing IP address. In this situation, we suggest +configuring a "dummy" IP address is an unroutable network on the correct interface +and reference that dummy network in the networks list (see above). + + **Useful hints for ingress:** * Good to have at least 3 RGW daemons diff --git a/src/pybind/mgr/cephadm/services/ingress.py b/src/pybind/mgr/cephadm/services/ingress.py index d34bbb45f582..639bef5a57db 100644 --- a/src/pybind/mgr/cephadm/services/ingress.py +++ b/src/pybind/mgr/cephadm/services/ingress.py @@ -155,13 +155,27 @@ class IngressService(CephService): hosts = sorted(list(set([str(d.hostname) for d in daemons]))) # interface - interface = 'eth0' + bare_ip = str(spec.virtual_ip).split('/')[0] + interface = None for subnet, ifaces in self.mgr.cache.networks.get(host, {}).items(): - logger.info(f'subnet {subnet} ifaces {ifaces} virtual_ip {spec.virtual_ip}') - if ifaces and ipaddress.ip_address(spec.virtual_ip) in ipaddress.ip_network(subnet): - logger.info(f'{spec.virtual_ip} is in {subnet}') + if ifaces and ipaddress.ip_address(bare_ip) in ipaddress.ip_network(subnet): interface = list(ifaces.keys())[0] + logger.info( + f'{bare_ip} is in {subnet} on {host} interface {interface}' + ) break + if not interface and spec.networks: + # hmm, try spec.networks + for subnet, ifaces in self.mgr.cache.networks.get(host, {}).items(): + if subnet in spec.networks: + interface = list(ifaces.keys())[0] + logger.info( + f'{spec.virtual_ip} will be configured on {host} interface ' + f'{interface} (which has guiding subnet {subnet})' + ) + break + if not interface: + interface = 'eth0' # script to monitor health script = '/usr/bin/false' diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index 75d99a0f61cb..3f3deaf666b8 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -877,6 +877,7 @@ class IngressSpec(ServiceSpec): enable_stats: Optional[bool] = None, keepalived_password: Optional[str] = None, virtual_ip: Optional[str] = None, + virtual_interface_networks: Optional[List[str]] = [], haproxy_container_image: Optional[str] = None, keepalived_container_image: Optional[str] = None, ): @@ -897,6 +898,7 @@ class IngressSpec(ServiceSpec): self.monitor_password = monitor_password self.keepalived_password = keepalived_password self.virtual_ip = virtual_ip + self.virtual_interface_networks = virtual_interface_networks or [] self.haproxy_container_image = haproxy_container_image self.keepalived_container_image = keepalived_container_image