From 64a7ba000a40d2079f8907a6ba92b2c2b6e8dcd6 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 12 Apr 2021 17:21:33 -0400 Subject: [PATCH] mgr/cephadm: ingress: add optional virtual_interface_networks It may be that the virtual IP we want to use is not in the same network as any existing IPs on the host. In that case, allow the spec to specify a list of networks to match against existing IPs so that a match will identify an ethernet interface to use. Signed-off-by: Sage Weil (cherry picked from commit bbf6a12752092b406abbec1e600533366ac59548) --- doc/cephadm/rgw.rst | 40 +++++++++++++++++-- src/pybind/mgr/cephadm/services/ingress.py | 22 ++++++++-- .../ceph/deployment/service_spec.py | 2 + 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/doc/cephadm/rgw.rst b/doc/cephadm/rgw.rst index 31084ff7d0a12..1815776dfb69e 100644 --- a/doc/cephadm/rgw.rst +++ b/doc/cephadm/rgw.rst @@ -112,10 +112,6 @@ elected as master, and the virtual IP will be moved to that node. The active haproxy acts like a load balancer, distributing all RGW requests between all the RGW daemons available. -.. note:: The virtual IP will be configured on an ethernet interface on the host - that has an existing IP in the same subnet. (If there are multiple such - interfaces, cephadm will choose the "first" one it sees.) - **Prerequisites:** * An existing RGW service, without SSL. (If you want SSL service, the certificate @@ -145,6 +141,7 @@ It is a yaml format file with the following properties: virtual_ip: / # ex: 192.168.20.1/24 frontend_port: # ex: 8080 monitor_port: # ex: 1967, used by haproxy for load balancer status + virtual_interface_networks: [ ... ] # optional: list of CIDR networks ssl_cert: | # optional: SSL certificate and key -----BEGIN CERTIFICATE----- ... @@ -166,12 +163,47 @@ where the properties of this service specification are: to match the nodes where RGW is deployed. * ``virtual_ip`` The virtual IP (and network) in CIDR format where the ingress service will be available. +* ``virtual_interface_networks`` + A list of networks to identify which ethernet interface to use for the virtual IP. * ``frontend_port`` The port used to access the ingress service. * ``ssl_cert``: SSL certificate, if SSL is to be enabled. This must contain the both the certificate and private key blocks in .pem format. +**Selecting ethernet interfaces for the virtual IP:** + +You cannot simply provide the name of the network interface on which +to configure the virtual IP because interface names tend to vary +across hosts (and/or reboots). Instead, cephadm will select +interfaces based on other existing IP addresses that are already +configured. + +Normally, the virtual IP will be configured on the first network +interface that has an existing IP in the same subnet. For example, if +the virtual IP is 192.168.0.80/24 and eth2 has the static IP +192.168.0.40/24, cephadm will use eth2. + +In some cases, the virtual IP may not belong to the same subnet as an existing static +IP. In such cases, you can provide a list of subnets to match against existing IPs, +and cephadm will put the virtual IP on the first network interface to match. For example, +if the virtual IP is 192.168.0.80/24 and we want it on the same interface as the machine's +static IP in 10.10.0.0/16, you can use a spec like:: + + service_type: ingress + service_id: rgw.something + spec: + virtual_ip: 192.168.0.80/24 + virtual_interface_networks: + - 10.10.0.0/16 + ... + +A consequence of this strategy is that you cannot currently configure the virtual IP +on an interface that has no existing IP address. In this situation, we suggest +configuring a "dummy" IP address is an unroutable network on the correct interface +and reference that dummy network in the networks list (see above). + + **Useful hints for ingress:** * Good to have at least 3 RGW daemons diff --git a/src/pybind/mgr/cephadm/services/ingress.py b/src/pybind/mgr/cephadm/services/ingress.py index d34bbb45f5821..639bef5a57db5 100644 --- a/src/pybind/mgr/cephadm/services/ingress.py +++ b/src/pybind/mgr/cephadm/services/ingress.py @@ -155,13 +155,27 @@ class IngressService(CephService): hosts = sorted(list(set([str(d.hostname) for d in daemons]))) # interface - interface = 'eth0' + bare_ip = str(spec.virtual_ip).split('/')[0] + interface = None for subnet, ifaces in self.mgr.cache.networks.get(host, {}).items(): - logger.info(f'subnet {subnet} ifaces {ifaces} virtual_ip {spec.virtual_ip}') - if ifaces and ipaddress.ip_address(spec.virtual_ip) in ipaddress.ip_network(subnet): - logger.info(f'{spec.virtual_ip} is in {subnet}') + if ifaces and ipaddress.ip_address(bare_ip) in ipaddress.ip_network(subnet): interface = list(ifaces.keys())[0] + logger.info( + f'{bare_ip} is in {subnet} on {host} interface {interface}' + ) break + if not interface and spec.networks: + # hmm, try spec.networks + for subnet, ifaces in self.mgr.cache.networks.get(host, {}).items(): + if subnet in spec.networks: + interface = list(ifaces.keys())[0] + logger.info( + f'{spec.virtual_ip} will be configured on {host} interface ' + f'{interface} (which has guiding subnet {subnet})' + ) + break + if not interface: + interface = 'eth0' # script to monitor health script = '/usr/bin/false' diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index 92e1e20a67d39..aa3b203e0104f 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -873,6 +873,7 @@ class IngressSpec(ServiceSpec): enable_stats: Optional[bool] = None, keepalived_password: Optional[str] = None, virtual_ip: Optional[str] = None, + virtual_interface_networks: Optional[List[str]] = [], haproxy_container_image: Optional[str] = None, keepalived_container_image: Optional[str] = None, ): @@ -893,6 +894,7 @@ class IngressSpec(ServiceSpec): self.monitor_password = monitor_password self.keepalived_password = keepalived_password self.virtual_ip = virtual_ip + self.virtual_interface_networks = virtual_interface_networks or [] self.haproxy_container_image = haproxy_container_image self.keepalived_container_image = keepalived_container_image -- 2.39.5