From 0193a6f73659f7aa4ac1d000cf11c6544ad6ab6d Mon Sep 17 00:00:00 2001 From: Luis Domingues Date: Mon, 8 Aug 2022 18:27:57 +0200 Subject: [PATCH] mgr/cephadm: allow for multiple vip configuration on ingress service Signed-off-by: Luis Domingues --- src/pybind/mgr/cephadm/services/ingress.py | 40 ++++++++++++++++--- .../services/ingress/keepalived.conf.j2 | 12 +++--- .../ceph/deployment/service_spec.py | 8 +++- 3 files changed, 48 insertions(+), 12 deletions(-) diff --git a/src/pybind/mgr/cephadm/services/ingress.py b/src/pybind/mgr/cephadm/services/ingress.py index 02fd8141d5b52..719eb3c34f650 100644 --- a/src/pybind/mgr/cephadm/services/ingress.py +++ b/src/pybind/mgr/cephadm/services/ingress.py @@ -197,7 +197,10 @@ class IngressService(CephService): hosts = sorted(list(set([host] + [str(d.hostname) for d in daemons]))) # interface - bare_ip = str(spec.virtual_ip).split('/')[0] + if spec.virtual_ip: + bare_ip = str(spec.virtual_ip).split('/')[0] + elif spec.virtual_ips_list: + bare_ip = str(spec.virtual_ips_list[0]).split('/')[0] interface = None for subnet, ifaces in self.mgr.cache.networks.get(host, {}).items(): if ifaces and ipaddress.ip_address(bare_ip) in ipaddress.ip_network(subnet): @@ -231,10 +234,33 @@ class IngressService(CephService): script = f'/usr/bin/curl {build_url(scheme="http", host=d.ip or "localhost", port=port)}/health' assert script - # set state. first host in placement is master all others backups - state = 'BACKUP' - if hosts[0] == host: - state = 'MASTER' + states = [] + priorities = [] + virtual_ips = [] + + # Set state and priority. Have one master for each VIP. Or at least the first one as master if only one VIP. + if spec.virtual_ip: + virtual_ips.append(spec.virtual_ip) + if hosts[0] == host: + states.append('MASTER') + priorities.append(100) + else: + states.append('BACKUP') + priorities.append(90) + + elif spec.virtual_ips_list: + virtual_ips = spec.virtual_ips_list + if len(virtual_ips) > len(hosts): + raise OrchestratorError( + "Number of virtual IPs for ingress is greater than number of available hosts" + ) + for x in range(len(virtual_ips)): + if hosts[x] == host: + states.append('MASTER') + priorities.append(100) + else: + states.append('BACKUP') + priorities.append(90) # remove host, daemon is being deployed on from hosts list for # other_ips in conf file and converter to ips @@ -249,7 +275,9 @@ class IngressService(CephService): 'script': script, 'password': password, 'interface': interface, - 'state': state, + 'virtual_ips': virtual_ips, + 'states': states, + 'priorities': priorities, 'other_ips': other_ips, 'host_ip': resolve_ip(self.mgr.inventory.get_addr(host)), } diff --git a/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2 b/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2 index 4caeebf51f9e0..f560c9756654d 100644 --- a/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2 +++ b/src/pybind/mgr/cephadm/templates/services/ingress/keepalived.conf.j2 @@ -7,11 +7,12 @@ vrrp_script check_backend { fall 2 } -vrrp_instance VI_0 { - state {{ state }} - priority 100 +{% for x in range(virtual_ips|length) %} +vrrp_instance VI_{{ x }} { + state {{ states[x] }} + priority {{ priorities[x] }} interface {{ interface }} - virtual_router_id 51 + virtual_router_id {{ 50 + x }} advert_int 1 authentication { auth_type PASS @@ -24,9 +25,10 @@ vrrp_instance VI_0 { {% endfor %} } virtual_ipaddress { - {{ spec.virtual_ip }} dev {{ interface }} + {{ virtual_ips[x] }} dev {{ interface }} } track_script { check_backend } } +{% endfor %} diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index 13b75728f5370..62543e692d67a 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -1007,6 +1007,7 @@ class IngressSpec(ServiceSpec): enable_stats: Optional[bool] = None, keepalived_password: Optional[str] = None, virtual_ip: Optional[str] = None, + virtual_ips_list: Optional[List[str]] = None, virtual_interface_networks: Optional[List[str]] = [], unmanaged: bool = False, ssl: bool = False, @@ -1014,6 +1015,7 @@ class IngressSpec(ServiceSpec): custom_configs: Optional[List[CustomConfig]] = None, ): assert service_type == 'ingress' + super(IngressSpec, self).__init__( 'ingress', service_id=service_id, placement=placement, config=config, @@ -1033,6 +1035,7 @@ class IngressSpec(ServiceSpec): self.monitor_password = monitor_password self.keepalived_password = keepalived_password self.virtual_ip = virtual_ip + self.virtual_ips_list = virtual_ips_list self.virtual_interface_networks = virtual_interface_networks or [] self.unmanaged = unmanaged self.ssl = ssl @@ -1056,9 +1059,12 @@ class IngressSpec(ServiceSpec): if not self.monitor_port: raise SpecValidationError( 'Cannot add ingress: No monitor_port specified') - if not self.virtual_ip: + if not self.virtual_ip and not self.virtual_ips_list: raise SpecValidationError( 'Cannot add ingress: No virtual_ip provided') + if self.virtual_ip is not None and self.virtual_ips_list is not None: + raise SpecValidationError( + 'Cannot add ingress: Single and multiple virtual IPs specified') yaml.add_representer(IngressSpec, ServiceSpec.yaml_representer) -- 2.39.5