From: Sage Weil Date: Wed, 10 Mar 2021 19:25:23 +0000 (-0500) Subject: mgr/cephadm/schedule: dynamically assign ports for rgw X-Git-Tag: v17.1.0~2601^2~9 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=df28e236a66f04ecd95edb6198c45f91d70686f3;p=ceph.git mgr/cephadm/schedule: dynamically assign ports for rgw Dynamically number ports for RGW instances, with the start port being the one configured on the service (or the default of 80 or 443). Signed-off-by: Sage Weil --- diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 9702e7d52eb64..d2618b40d0744 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -2034,8 +2034,11 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, self.cephadm_services[service_type].config(spec, daemon_id) did_config = True + port = spec.get_port_start() daemon_spec = self.cephadm_services[service_type].make_daemon_spec( - host, daemon_id, network, spec) + host, daemon_id, network, spec, + # NOTE: this does not consider port conflicts! + ports=[port] if port else None) self.log.debug('Placing %s.%s on host %s' % ( daemon_type, daemon_id, host)) args.append(daemon_spec) diff --git a/src/pybind/mgr/cephadm/schedule.py b/src/pybind/mgr/cephadm/schedule.py index b2dad5b0e7cee..3a4c3cad29528 100644 --- a/src/pybind/mgr/cephadm/schedule.py +++ b/src/pybind/mgr/cephadm/schedule.py @@ -33,6 +33,15 @@ class DaemonPlacement(NamedTuple): res += '(' + ' '.join(other) + ')' return res + def renumber_port(self, n: int) -> 'DaemonPlacement': + return DaemonPlacement( + self.hostname, + self.network, + self.name, + self.ip, + (self.port + n) if self.port is not None else None + ) + def matches_daemon(self, dd: DaemonDescription) -> bool: if self.hostname != dd.hostname: return False @@ -62,6 +71,7 @@ class HostAssignment(object): self.service_name = spec.service_name() self.daemons = daemons self.allow_colo = allow_colo + self.port_start = spec.get_port_start() def hosts_by_label(self, label: str) -> List[orchestrator.HostSpec]: return [h for h in self.hosts if label in h.labels] @@ -123,16 +133,22 @@ class HostAssignment(object): # get candidate hosts based on [hosts, label, host_pattern] candidates = self.get_candidates() # type: List[DaemonPlacement] + def expand_candidates(ls: List[DaemonPlacement], num: int) -> List[DaemonPlacement]: + r = [] + for offset in range(num): + r.extend([dp.renumber_port(offset) for dp in ls]) + return r + # consider enough slots to fulfill target count-per-host or count if count is None: if self.spec.placement.count_per_host: per_host = self.spec.placement.count_per_host else: per_host = 1 - candidates = candidates * per_host + candidates = expand_candidates(candidates, per_host) elif self.allow_colo and candidates: per_host = 1 + ((count - 1) // len(candidates)) - candidates = candidates * per_host + candidates = expand_candidates(candidates, per_host) # consider active daemons first daemons = [ @@ -188,17 +204,18 @@ class HostAssignment(object): def get_candidates(self) -> List[DaemonPlacement]: if self.spec.placement.hosts: ls = [ - DaemonPlacement(hostname=h.hostname, network=h.network, name=h.name) + DaemonPlacement(hostname=h.hostname, network=h.network, name=h.name, + port=self.port_start) for h in self.spec.placement.hosts ] elif self.spec.placement.label: ls = [ - DaemonPlacement(hostname=x.hostname) + DaemonPlacement(hostname=x.hostname, port=self.port_start) for x in self.hosts_by_label(self.spec.placement.label) ] elif self.spec.placement.host_pattern: ls = [ - DaemonPlacement(hostname=x) + DaemonPlacement(hostname=x, port=self.port_start) for x in self.spec.placement.filter_matching_hostspecs(self.hosts) ] elif ( @@ -206,7 +223,7 @@ class HostAssignment(object): or self.spec.placement.count_per_host is not None ): ls = [ - DaemonPlacement(hostname=x.hostname) + DaemonPlacement(hostname=x.hostname, port=self.port_start) for x in self.hosts ] else: diff --git a/src/pybind/mgr/cephadm/services/cephadmservice.py b/src/pybind/mgr/cephadm/services/cephadmservice.py index a4d8d3eeb210b..6513cbf7a4971 100644 --- a/src/pybind/mgr/cephadm/services/cephadmservice.py +++ b/src/pybind/mgr/cephadm/services/cephadmservice.py @@ -681,6 +681,9 @@ class MdsService(CephService): class RgwService(CephService): TYPE = 'rgw' + def allow_colo(self) -> bool: + return True + def config(self, spec: RGWSpec, rgw_id: str) -> None: # type: ignore assert self.TYPE == spec.service_type @@ -745,11 +748,11 @@ class RgwService(CephService): # configure frontend args = [] if spec.ssl: - args.append(f"ssl_port={spec.get_port()}") + args.append(f"ssl_port={daemon_spec.ports[0]}") args.append(f"ssl_certificate=config://rgw/cert/{spec.rgw_realm}/{spec.rgw_zone}.crt") args.append(f"ssl_key=config://rgw/cert/{spec.rgw_realm}/{spec.rgw_zone}.key") else: - args.append(f"port={spec.get_port()}") + args.append(f"port={daemon_spec.ports[0]}") frontend = f'beast {" ".join(args)}' ret, out, err = self.mgr.check_mon_command({ diff --git a/src/pybind/mgr/cephadm/tests/test_scheduling.py b/src/pybind/mgr/cephadm/tests/test_scheduling.py index f98a593bdf7e9..5e10bc729facf 100644 --- a/src/pybind/mgr/cephadm/tests/test_scheduling.py +++ b/src/pybind/mgr/cephadm/tests/test_scheduling.py @@ -361,7 +361,7 @@ class NodeAssignmentTest(NamedTuple): # just hosts NodeAssignmentTest( 'mgr', - PlacementSpec(hosts=['smithi060:[v2:172.21.15.60:3301,v1:172.21.15.60:6790]=c']), + PlacementSpec(hosts=['smithi060']), ['smithi060'], [], ['smithi060'], ['smithi060'], [] @@ -528,6 +528,34 @@ class NodeAssignmentTest(NamedTuple): ['mdshost1', 'mdshost2', 'mdshost1', 'mdshost2', 'mdshost1', 'mdshost2'], [] ), + # label + count_per_host + ports + NodeAssignmentTest( + 'rgw', + PlacementSpec(count=6, label='foo'), + 'host1 host2 host3'.split(), + [], + ['host1(port=80)', 'host2(port=80)', 'host3(port=80)', + 'host1(port=81)', 'host2(port=81)', 'host3(port=81)'], + ['host1(port=80)', 'host2(port=80)', 'host3(port=80)', + 'host1(port=81)', 'host2(port=81)', 'host3(port=81)'], + [] + ), + # label + count_per_host + ports (+ xisting) + NodeAssignmentTest( + 'rgw', + PlacementSpec(count=6, label='foo'), + 'host1 host2 host3'.split(), + [ + DaemonDescription('rgw', 'a', 'host1', ports=[81]), + DaemonDescription('rgw', 'b', 'host2', ports=[80]), + DaemonDescription('rgw', 'c', 'host1', ports=[82]), + ], + ['host1(port=80)', 'host2(port=80)', 'host3(port=80)', + 'host1(port=81)', 'host2(port=81)', 'host3(port=81)'], + ['host1(port=80)', 'host3(port=80)', + 'host2(port=81)', 'host3(port=81)'], + ['rgw.c'] + ), ]) def test_node_assignment(service_type, placement, hosts, daemons, expected, expected_add, expected_remove): @@ -535,6 +563,7 @@ def test_node_assignment(service_type, placement, hosts, daemons, allow_colo = False if service_type == 'rgw': service_id = 'realm.zone' + allow_colo = True elif service_type == 'mds': service_id = 'myfs' allow_colo = True @@ -547,24 +576,26 @@ def test_node_assignment(service_type, placement, hosts, daemons, spec=spec, hosts=[HostSpec(h, labels=['foo']) for h in hosts], daemons=daemons, - allow_colo=allow_colo + allow_colo=allow_colo, ).place() - got = [hs.hostname for hs in all_slots] + got = [str(p) for p in all_slots] num_wildcard = 0 for i in expected: if i == '*': num_wildcard += 1 else: + assert i in got got.remove(i) assert num_wildcard == len(got) - got = [hs.hostname for hs in to_add] + got = [str(p) for p in to_add] num_wildcard = 0 for i in expected_add: if i == '*': num_wildcard += 1 else: + assert i in got got.remove(i) assert num_wildcard == len(got) diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index af07d254c330d..339dbe0a48340 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -573,6 +573,11 @@ class ServiceSpec(object): n += '.' + self.service_id return n + def get_port_start(self) -> Optional[int]: + # If defined, we will allocate and number ports starting at this + # point. + return None + def to_json(self): # type: () -> OrderedDict[str, Any] ret: OrderedDict[str, Any] = OrderedDict() @@ -727,6 +732,9 @@ class RGWSpec(ServiceSpec): self.rgw_frontend_ssl_key = rgw_frontend_ssl_key self.ssl = ssl + def get_port_start(self) -> Optional[int]: + return self.get_port() + def get_port(self) -> int: if self.rgw_frontend_port: return self.rgw_frontend_port