From 544629833949a4104e236f19054fc614e0077f96 Mon Sep 17 00:00:00 2001 From: Kiefer Chang Date: Fri, 29 May 2020 17:23:01 +0800 Subject: [PATCH] mgr/cephadm: refactor templating in services Using Jinja2 to render configs for services: - Grafana - Alertmanager - Prometheus - NFS - iSCSI Signed-off-by: Kiefer Chang --- src/pybind/mgr/cephadm/services/iscsi.py | 23 +-- src/pybind/mgr/cephadm/services/monitoring.py | 147 ++++-------------- src/pybind/mgr/cephadm/services/nfs.py | 47 +----- .../services/alertmanager/alertmanager.yml.j2 | 18 +++ .../services/grafana/ceph-dashboard.yml.j2 | 18 +++ .../templates/services/grafana/grafana.ini.j2 | 17 ++ .../services/iscsi/iscsi-gateway.cfg.j2 | 13 ++ .../templates/services/nfs/ganesha.conf.j2 | 36 +++++ .../services/prometheus/prometheus.yml.j2 | 32 ++++ 9 files changed, 173 insertions(+), 178 deletions(-) create mode 100644 src/pybind/mgr/cephadm/templates/services/alertmanager/alertmanager.yml.j2 create mode 100644 src/pybind/mgr/cephadm/templates/services/grafana/ceph-dashboard.yml.j2 create mode 100644 src/pybind/mgr/cephadm/templates/services/grafana/grafana.ini.j2 create mode 100644 src/pybind/mgr/cephadm/templates/services/iscsi/iscsi-gateway.cfg.j2 create mode 100644 src/pybind/mgr/cephadm/templates/services/nfs/ganesha.conf.j2 create mode 100644 src/pybind/mgr/cephadm/templates/services/prometheus/prometheus.yml.j2 diff --git a/src/pybind/mgr/cephadm/services/iscsi.py b/src/pybind/mgr/cephadm/services/iscsi.py index 0e5a3a3d118..9b10d0a1652 100644 --- a/src/pybind/mgr/cephadm/services/iscsi.py +++ b/src/pybind/mgr/cephadm/services/iscsi.py @@ -52,25 +52,14 @@ class IscsiService(CephadmService): 'val': key_data, }) - api_secure = 'false' if spec.api_secure is None else spec.api_secure - igw_conf = f""" - # generated by cephadm - [config] - cluster_client_name = {utils.name_to_config_section('iscsi')}.{igw_id} - pool = {spec.pool} - trusted_ip_list = {spec.trusted_ip_list or ''} - minimum_gateways = 1 - api_port = {spec.api_port or ''} - api_user = {spec.api_user or ''} - api_password = {spec.api_password or ''} - api_secure = {api_secure} - log_to_stderr = True - log_to_stderr_prefix = debug - log_to_file = False - """ + context = { + 'client_name': '{}.{}'.format(utils.name_to_config_section('iscsi'), igw_id), + 'spec': spec + } + igw_conf = self.mgr.template.render('services/iscsi/iscsi-gateway.cfg.j2', context) extra_config = {'iscsi-gateway.cfg': igw_conf} return self.mgr._create_daemon('iscsi', igw_id, host, keyring=keyring, - extra_config=extra_config) + extra_config=extra_config) def daemon_check_post(self, daemon_descrs: List[DaemonDescription]): try: diff --git a/src/pybind/mgr/cephadm/services/monitoring.py b/src/pybind/mgr/cephadm/services/monitoring.py index dc20ac9e0e4..b17a448755b 100644 --- a/src/pybind/mgr/cephadm/services/monitoring.py +++ b/src/pybind/mgr/cephadm/services/monitoring.py @@ -18,48 +18,13 @@ class GrafanaService(CephadmService): def generate_config(self): # type: () -> Tuple[Dict[str, Any], List[str]] deps = [] # type: List[str] - def generate_grafana_ds_config(hosts: List[str]) -> str: - config = '''# generated by cephadm -deleteDatasources: -{delete_data_sources} - -datasources: -{data_sources} -''' - delete_ds_template = ''' - - name: '{name}' - orgId: 1\n'''.lstrip('\n') - ds_template = ''' - - name: '{name}' - type: 'prometheus' - access: 'proxy' - orgId: 1 - url: 'http://{host}:9095' - basicAuth: false - isDefault: {is_default} - editable: false\n'''.lstrip('\n') - - delete_data_sources = '' - data_sources = '' - for i, host in enumerate(hosts): - name = "Dashboard %d" % (i + 1) - data_sources += ds_template.format( - name=name, - host=host, - is_default=str(i == 0).lower() - ) - delete_data_sources += delete_ds_template.format( - name=name - ) - return config.format( - delete_data_sources=delete_data_sources, - data_sources=data_sources, - ) prom_services = [] # type: List[str] for dd in self.mgr.cache.get_daemons_by_service('prometheus'): prom_services.append(dd.hostname) deps.append(dd.name()) + grafana_data_sources = self.mgr.template.render( + 'services/grafana/ceph-dashboard.yml.j2', {'hosts': prom_services}) cert = self.mgr.get_store('grafana_crt') pkey = self.mgr.get_store('grafana_key') @@ -78,29 +43,13 @@ datasources: 'value': 'false', }) - + grafana_ini = self.mgr.template.render( + 'services/grafana/grafana.ini.j2', {'http_port': self.DEFAULT_SERVICE_PORT}) config_file = { 'files': { - "grafana.ini": """# generated by cephadm -[users] - default_theme = light -[auth.anonymous] - enabled = true - org_name = 'Main Org.' - org_role = 'Viewer' -[server] - domain = 'bootstrap.storage.lab' - protocol = https - cert_file = /etc/grafana/certs/cert_file - cert_key = /etc/grafana/certs/cert_key - http_port = {} -[security] - admin_user = admin - admin_password = admin - allow_embedding = true -""".format(self.DEFAULT_SERVICE_PORT), - 'provisioning/datasources/ceph-dashboard.yml': generate_grafana_ds_config(prom_services), + "grafana.ini": grafana_ini, + 'provisioning/datasources/ceph-dashboard.yml': grafana_data_sources, 'certs/cert_file': '# generated by cephadm\n%s' % cert, 'certs/cert_key': '# generated by cephadm\n%s' % pkey, } @@ -155,29 +104,13 @@ class AlertmanagerService(CephadmService): continue addr = self.mgr.inventory.get_addr(dd.hostname) dashboard_urls.append('%s//%s:%s/' % (proto, addr.split(':')[0], - port)) - - yml = """# generated by cephadm -# See https://prometheus.io/docs/alerting/configuration/ for documentation. - -global: - resolve_timeout: 5m - -route: - group_by: ['alertname'] - group_wait: 10s - group_interval: 10s - repeat_interval: 1h - receiver: 'ceph-dashboard' -receivers: -- name: 'ceph-dashboard' - webhook_configs: -{urls} -""".format( - urls='\n'.join( - [" - url: '{}api/prometheus_receiver'".format(u) - for u in dashboard_urls] - )) + port)) + + context = { + 'dashboard_urls': dashboard_urls + } + yml = self.mgr.template.render('services/alertmanager/alertmanager.yml.j2', context) + peers = [] port = '9094' for dd in self.mgr.cache.get_daemons_by_service('alertmanager'): @@ -241,60 +174,34 @@ class PrometheusService(CephadmService): mgr_scrape_list.append(addr.split(':')[0] + ':' + port) # scrape node exporters - node_configs = '' + nodes = [] for dd in self.mgr.cache.get_daemons_by_service('node-exporter'): deps.append(dd.name()) addr = self.mgr.inventory.get_addr(dd.hostname) - if not node_configs: - node_configs = """ - - job_name: 'node' - static_configs: -""" - node_configs += """ - targets: {} - labels: - instance: '{}' -""".format([addr.split(':')[0] + ':9100'], - dd.hostname) + nodes.append({ + 'hostname': dd.hostname, + 'url': addr.split(':')[0] + ':9100' + }) # scrape alert managers - alertmgr_configs = "" alertmgr_targets = [] for dd in self.mgr.cache.get_daemons_by_service('alertmanager'): deps.append(dd.name()) addr = self.mgr.inventory.get_addr(dd.hostname) alertmgr_targets.append("'{}:9093'".format(addr.split(':')[0])) - if alertmgr_targets: - alertmgr_configs = """alerting: - alertmanagers: - - scheme: http - path_prefix: /alertmanager - static_configs: - - targets: [{}] -""".format(", ".join(alertmgr_targets)) # generate the prometheus configuration + context = { + 'alertmgr_targets': alertmgr_targets, + 'mgr_scrape_list': mgr_scrape_list, + 'nodes': nodes, + } r = { 'files': { - 'prometheus.yml': """# generated by cephadm -global: - scrape_interval: 10s - evaluation_interval: 10s -rule_files: - - /etc/prometheus/alerting/* -{alertmgr_configs} -scrape_configs: - - job_name: 'ceph' - static_configs: - - targets: {mgr_scrape_list} - labels: - instance: 'ceph_cluster' -{node_configs} -""".format( - mgr_scrape_list=str(mgr_scrape_list), - node_configs=str(node_configs), - alertmgr_configs=str(alertmgr_configs) - ), - }, + 'prometheus.yml': + self.mgr.template.render( + 'services/prometheus/prometheus.yml.j2', context) + } } # include alerts, if present in the container diff --git a/src/pybind/mgr/cephadm/services/nfs.py b/src/pybind/mgr/cephadm/services/nfs.py index cc786d33ed6..822920aacaf 100644 --- a/src/pybind/mgr/cephadm/services/nfs.py +++ b/src/pybind/mgr/cephadm/services/nfs.py @@ -156,47 +156,12 @@ class NFSGanesha(object): def get_ganesha_conf(self): # type: () -> str - return '''# generated by cephadm -NFS_CORE_PARAM {{ - Enable_NLM = false; - Enable_RQUOTA = false; - Protocols = 4; -}} - -MDCACHE {{ - Dir_Chunk = 0; - NParts = 1; - Cache_Size = 1; -}} - -EXPORT_DEFAULTS {{ - Attr_Expiration_Time = 0; -}} - -NFSv4 {{ - Delegations = false; - RecoveryBackend = 'rados_cluster'; - Minor_Versions = 1, 2; -}} - -RADOS_KV {{ - UserId = "{user}"; - nodeid = "{nodeid}"; - pool = "{pool}"; - namespace = "{namespace}"; -}} - -RADOS_URLS {{ - UserId = "{user}"; - watch_url = "{url}"; -}} - -%url {url} -'''.format(user=self.get_rados_user(), - nodeid=self.get_daemon_name(), - pool=self.spec.pool, - namespace=self.spec.namespace if self.spec.namespace else '', - url=self.spec.rados_config_location()) + context = dict(user=self.get_rados_user(), + nodeid=self.get_daemon_name(), + pool=self.spec.pool, + namespace=self.spec.namespace if self.spec.namespace else '', + url=self.spec.rados_config_location()) + return self.mgr.template.render('services/nfs/ganesha.conf.j2', context) def get_cephadm_config(self): # type: () -> Dict diff --git a/src/pybind/mgr/cephadm/templates/services/alertmanager/alertmanager.yml.j2 b/src/pybind/mgr/cephadm/templates/services/alertmanager/alertmanager.yml.j2 new file mode 100644 index 00000000000..69d5e73bbec --- /dev/null +++ b/src/pybind/mgr/cephadm/templates/services/alertmanager/alertmanager.yml.j2 @@ -0,0 +1,18 @@ +# {{ cephadm_managed }} +# See https://prometheus.io/docs/alerting/configuration/ for documentation. + +global: + resolve_timeout: 5m + +route: + group_by: ['alertname'] + group_wait: 10s + group_interval: 10s + repeat_interval: 1h + receiver: 'ceph-dashboard' +receivers: +- name: 'ceph-dashboard' + webhook_configs: +{% for url in dashboard_urls %} + - url: '{{ url }}api/prometheus_receiver' +{% endfor %} diff --git a/src/pybind/mgr/cephadm/templates/services/grafana/ceph-dashboard.yml.j2 b/src/pybind/mgr/cephadm/templates/services/grafana/ceph-dashboard.yml.j2 new file mode 100644 index 00000000000..c539cfc6f65 --- /dev/null +++ b/src/pybind/mgr/cephadm/templates/services/grafana/ceph-dashboard.yml.j2 @@ -0,0 +1,18 @@ +# {{ cephadm_managed }} +deleteDatasources: +{% for host in hosts %} + - name: 'Dashboard{{ loop.index }}' + orgId: 1 +{% endfor %} + +datasources: +{% for host in hosts %} + - name: 'Dashboard{{ loop.index }}' + type: 'prometheus' + access: 'proxy' + orgId: 1 + url: 'http://{{ host }}:9095' + basicAuth: false + isDefault: {{ 'true' if loop.first else 'false' }} + editable: false +{% endfor %} diff --git a/src/pybind/mgr/cephadm/templates/services/grafana/grafana.ini.j2 b/src/pybind/mgr/cephadm/templates/services/grafana/grafana.ini.j2 new file mode 100644 index 00000000000..51aff3f9fb5 --- /dev/null +++ b/src/pybind/mgr/cephadm/templates/services/grafana/grafana.ini.j2 @@ -0,0 +1,17 @@ +# {{ cephadm_managed }} +[users] + default_theme = light +[auth.anonymous] + enabled = true + org_name = 'Main Org.' + org_role = 'Viewer' +[server] + domain = 'bootstrap.storage.lab' + protocol = https + cert_file = /etc/grafana/certs/cert_file + cert_key = /etc/grafana/certs/cert_key + http_port = {{ http_port }} +[security] + admin_user = admin + admin_password = admin + allow_embedding = true diff --git a/src/pybind/mgr/cephadm/templates/services/iscsi/iscsi-gateway.cfg.j2 b/src/pybind/mgr/cephadm/templates/services/iscsi/iscsi-gateway.cfg.j2 new file mode 100644 index 00000000000..2bc13451d65 --- /dev/null +++ b/src/pybind/mgr/cephadm/templates/services/iscsi/iscsi-gateway.cfg.j2 @@ -0,0 +1,13 @@ +# {{ cephadm_managed }} +[config] +cluster_client_name = {{ client_name }} +pool = {{ spec.pool }} +trusted_ip_list = {{ spec.trusted_ip_list|default("''", true) }} +minimum_gateways = 1 +api_port = {{ spec.api_port|default("''", true) }} +api_user = {{ spec.api_user|default("''", true) }} +api_password = {{ spec.api_password|default("''", true) }} +api_secure = {{ spec.api_secure|default('False', true) }} +log_to_stderr = True +log_to_stderr_prefix = debug +log_to_file = False diff --git a/src/pybind/mgr/cephadm/templates/services/nfs/ganesha.conf.j2 b/src/pybind/mgr/cephadm/templates/services/nfs/ganesha.conf.j2 new file mode 100644 index 00000000000..381dbb87a14 --- /dev/null +++ b/src/pybind/mgr/cephadm/templates/services/nfs/ganesha.conf.j2 @@ -0,0 +1,36 @@ +# {{ cephadm_managed }} +NFS_CORE_PARAM { + Enable_NLM = false; + Enable_RQUOTA = false; + Protocols = 4; +} + +MDCACHE { + Dir_Chunk = 0; + NParts = 1; + Cache_Size = 1; +} + +EXPORT_DEFAULTS { + Attr_Expiration_Time = 0; +} + +NFSv4 { + Delegations = false; + RecoveryBackend = 'rados_cluster'; + Minor_Versions = 1, 2; +} + +RADOS_KV { + UserId = "{{ user }}"; + nodeid = "{{ nodeid}}"; + pool = "{{ pool }}"; + namespace = "{{ namespace }}"; +} + +RADOS_URLS { + UserId = "{{ user }}"; + watch_url = "{{ url }}"; +} + +%url {{ url }} diff --git a/src/pybind/mgr/cephadm/templates/services/prometheus/prometheus.yml.j2 b/src/pybind/mgr/cephadm/templates/services/prometheus/prometheus.yml.j2 new file mode 100644 index 00000000000..0ea8101df6d --- /dev/null +++ b/src/pybind/mgr/cephadm/templates/services/prometheus/prometheus.yml.j2 @@ -0,0 +1,32 @@ +# {{ cephadm_managed }} +global: + scrape_interval: 10s + evaluation_interval: 10s +rule_files: + - /etc/prometheus/alerting/* +{% if alertmgr_targets %} +alerting: + alertmanagers: + - scheme: http + path_prefix: /alertmanager + static_configs: + - targets: [{{ alertmgr_targets|join(', ') }}] +{% endif %} +scrape_configs: + - job_name: 'ceph' + static_configs: + - targets: +{% for mgr in mgr_scrape_list %} + - '{{ mgr }}' +{% endfor %} + labels: + instance: 'ceph_cluster' +{% if nodes %} + - job_name: 'node' + static_configs: +{% for node in nodes %} + - targets: ['{{ node.url }}'] + labels: + instance: '{{ node.hostname }}' +{% endfor %} +{% endif %} -- 2.39.5