From: Sébastien Han Date: Wed, 28 Feb 2018 16:08:07 +0000 (+0100) Subject: add support for installation checkpoint X-Git-Tag: v3.1.0beta4~27 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=f2e0ceed78b3d72ab2240a0c8982e83e24410276;p=ceph-ansible.git add support for installation checkpoint This was taken from the openshift ansible repository here: https://github.com/leseb/openshift-ansible/tree/master/roles/installer_checkpoint Rationale: A complete OpenShift cluster installation is comprised of many different components which can take 30 minutes to several hours to complete. If the installation should fail, it could be confusing to understand at which component the failure occurred. Additionally, it may be desired to re-run only the component which failed instead of starting over from the beginning. Components which came after the failed component would also need to be run individually. Ceph has a similar situation so we can benefit from that callback_plugin. Signed-off-by: Sébastien Han --- diff --git a/callback_plugins/installer_checkpoint.py b/callback_plugins/installer_checkpoint.py new file mode 100644 index 000000000..65e4967f3 --- /dev/null +++ b/callback_plugins/installer_checkpoint.py @@ -0,0 +1,139 @@ +"""Ansible callback plugin to print a summary completion status of installation +phases. +""" +from datetime import datetime +from ansible.plugins.callback import CallbackBase +from ansible import constants as C + + +class CallbackModule(CallbackBase): + """This callback summarizes installation phase status.""" + + CALLBACK_VERSION = 2.0 + CALLBACK_TYPE = 'aggregate' + CALLBACK_NAME = 'installer_checkpoint' + CALLBACK_NEEDS_WHITELIST = False + + def __init__(self): + super(CallbackModule, self).__init__() + + def v2_playbook_on_stats(self, stats): + + # Set the order of the installer phases + installer_phases = [ + 'installer_phase_ceph_mon', + 'installer_phase_ceph_mgr', + 'installer_phase_ceph_agent', + 'installer_phase_ceph_osd', + 'installer_phase_ceph_mds', + 'installer_phase_ceph_rgw', + 'installer_phase_ceph_nfs', + 'installer_phase_ceph_restapi', + 'installer_phase_ceph_rbdmirror', + 'installer_phase_ceph_client', + 'installer_phase_ceph_iscsi_gw', + ] + + # Define the attributes of the installer phases + phase_attributes = { + 'installer_phase_ceph_mon': { + 'title': 'Install Ceph Monitor', + 'playbook': 'roles/ceph-mon/tasks/main.yml' + }, + 'installer_phase_ceph_mgr': { + 'title': 'Install Ceph Manager', + 'playbook': 'roles/ceph-mgr/tasks/main.yml' + }, + 'installer_phase_ceph_agent': { + 'title': 'Install Ceph Agent', + 'playbook': 'roles/ceph-agent/tasks/main.yml' + }, + 'installer_phase_ceph_osd': { + 'title': 'Install Ceph OSD', + 'playbook': 'roles/ceph-osd/tasks/main.yml' + }, + 'installer_phase_ceph_mds': { + 'title': 'Install Ceph MDS', + 'playbook': 'roles/ceph-mds/tasks/main.yml' + }, + 'installer_phase_ceph_rgw': { + 'title': 'Install Ceph RGW', + 'playbook': 'roles/ceph-rgw/tasks/main.yml' + }, + 'installer_phase_ceph_nfs': { + 'title': 'Install Ceph NFS', + 'playbook': 'roles/ceph-nfs/tasks/main.yml' + }, + 'installer_phase_ceph_restapi': { + 'title': 'Install Ceph REST API', + 'playbook': 'roles/ceph-restapi/tasks/main.yml' + }, + 'installer_phase_ceph_rbdmirror': { + 'title': 'Install Ceph RBD Mirror', + 'playbook': 'roles/ceph-rbd-mirror/tasks/main.yml' + }, + 'installer_phase_ceph_client': { + 'title': 'Install Ceph Client', + 'playbook': 'roles/ceph-client/tasks/main.yml' + }, + 'installer_phase_ceph_iscsi_gw': { + 'title': 'Install Ceph iSCSI Gateway', + 'playbook': 'roles/ceph-iscsi-gw/tasks/main.yml' + }, + } + + # Find the longest phase title + max_column = 0 + for phase in phase_attributes: + max_column = max(max_column, len(phase_attributes[phase]['title'])) + + if '_run' in stats.custom: + self._display.banner('INSTALLER STATUS') + for phase in installer_phases: + phase_title = phase_attributes[phase]['title'] + padding = max_column - len(phase_title) + 2 + if phase in stats.custom['_run']: + phase_status = stats.custom['_run'][phase]['status'] + phase_time = phase_time_delta(stats.custom['_run'][phase]) + self._display.display( + '{}{}: {} ({})'.format(phase_title, ' ' * padding, phase_status, phase_time), + color=self.phase_color(phase_status)) + if phase_status == 'In Progress' and phase != 'installer_phase_initialize': + self._display.display( + '\tThis phase can be restarted by running: {}'.format( + phase_attributes[phase]['playbook'])) + + self._display.display("", screen_only=True) + + def phase_color(self, status): + """ Return color code for installer phase""" + valid_status = [ + 'In Progress', + 'Complete', + ] + + if status not in valid_status: + self._display.warning('Invalid phase status defined: {}'.format(status)) + + if status == 'Complete': + phase_color = C.COLOR_OK + elif status == 'In Progress': + phase_color = C.COLOR_ERROR + else: + phase_color = C.COLOR_WARN + + return phase_color + + +def phase_time_delta(phase): + """ Calculate the difference between phase start and end times """ + time_format = '%Y%m%d%H%M%SZ' + phase_start = datetime.strptime(phase['start'], time_format) + if 'end' not in phase: + # The phase failed so set the end time to now + phase_end = datetime.now() + else: + phase_end = datetime.strptime(phase['end'], time_format) + delta = str(phase_end - phase_start).split(".")[0] # Trim microseconds + + return delta diff --git a/site-docker.yml.sample b/site-docker.yml.sample index e0b0fc2f7..58098ad3c 100644 --- a/site-docker.yml.sample +++ b/site-docker.yml.sample @@ -63,6 +63,16 @@ - is_atomic - (ceph_docker_dev_image is undefined or not ceph_docker_dev_image) +- hosts: mons + tasks: + - name: set ceph monitor install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_mon: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" + - hosts: mons become: True gather_facts: false @@ -73,74 +83,212 @@ - ceph-mon serial: 1 # MUST be '1' WHEN DEPLOYING MONITORS ON DOCKER CONTAINERS +- hosts: mons + tasks: + - name: set ceph monitor install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_mon: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" + - hosts: mgrs become: True gather_facts: false + pre_tasks: + - name: set ceph manager install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_mgr: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" roles: - { role: ceph-defaults, tags: ['ceph_update_config'] } - ceph-docker-common - { role: ceph-config, tags: ['ceph_update_config'], when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" } - { role: ceph-mgr, when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" } + post_tasks: + - name: set ceph manager install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_mgr: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - hosts: osds become: True gather_facts: false + pre_tasks: + - name: set ceph osd install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_osd: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" roles: - { role: ceph-defaults, tags: ['ceph_update_config'] } - ceph-docker-common - { role: ceph-config, tags: ['ceph_update_config'] } - ceph-osd + post_tasks: + - name: set ceph osd install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_osd: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - hosts: mdss become: True gather_facts: false + pre_tasks: + - name: set ceph mds install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_mds: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" roles: - { role: ceph-defaults, tags: ['ceph_update_config'] } - ceph-docker-common - { role: ceph-config, tags: ['ceph_update_config'] } - ceph-mds + post_tasks: + - name: set ceph mds install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_mds: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - hosts: rgws become: True gather_facts: false + pre_tasks: + - name: set ceph rgw install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_rgw: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" roles: - { role: ceph-defaults, tags: ['ceph_update_config'] } - ceph-docker-common - { role: ceph-config, tags: ['ceph_update_config'] } - ceph-rgw + post_tasks: + - name: set ceph rgw install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_rgw: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - hosts: nfss become: True gather_facts: false + pre_tasks: + - name: set ceph nfs install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_nfs: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" roles: - { role: ceph-defaults, tags: ['ceph_update_config'] } - ceph-docker-common - { role: ceph-config, tags: ['ceph_update_config'], when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" } - { role: ceph-nfs, when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" } + post_tasks: + - name: set ceph nfs install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_nfs: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - hosts: rbdmirrors become: True gather_facts: false + pre_tasks: + - name: set ceph rbd mirror install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_rbdmirror: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" roles: - { role: ceph-defaults, tags: ['ceph_update_config'] } - ceph-docker-common - { role: ceph-config, tags: ['ceph_update_config'] } - ceph-rbd-mirror + post_tasks: + - name: set ceph rbd mirror install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_rbdmirror: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - hosts: restapis become: True gather_facts: false + pre_tasks: + - name: set ceph rest api install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_restapi: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" roles: - { role: ceph-defaults, tags: ['ceph_update_config'] } - ceph-docker-common - { role: ceph-config, tags: ['ceph_update_config'] } - ceph-restapi + post_tasks: + - name: set ceph rest api install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_restapi: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - hosts: clients become: True gather_facts: false + pre_tasks: + - name: set ceph client install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_client: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" roles: - { role: ceph-defaults, tags: ['ceph_update_config'] } - ceph-docker-common - { role: ceph-config, tags: ['ceph_update_config'] } - ceph-client + post_tasks: + - name: set ceph client install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_client: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" diff --git a/site.yml.sample b/site.yml.sample index 29f88f246..5684a502e 100644 --- a/site.yml.sample +++ b/site.yml.sample @@ -71,99 +71,274 @@ - hosts: mons gather_facts: false become: True + pre_tasks: + - name: set ceph monitor install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_mon: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" roles: - { role: ceph-defaults, tags: ['ceph_update_config'] } - ceph-common - { role: ceph-config, tags: ['ceph_update_config'] } - ceph-mon + post_tasks: + - name: set ceph monitor install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_mon: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - hosts: mgrs gather_facts: false become: True + pre_tasks: + - name: set ceph manager install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_mgr: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" roles: - { role: ceph-defaults, tags: ['ceph_update_config'] } - ceph-common - { role: ceph-config, tags: ['ceph_update_config'], when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" } - { role: ceph-mgr, when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" } + post_tasks: + - name: set ceph manager install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_mgr: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - hosts: agents gather_facts: false become: True + pre_tasks: + - name: set ceph agent install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_agent: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" roles: - { role: ceph-defaults, tags: ['ceph_update_config'] } - ceph-common - { role: ceph-config, tags: ['ceph_update_config'] } - ceph-agent + post_tasks: + - name: set ceph agent install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_agent: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - hosts: osds gather_facts: false become: True + pre_tasks: + - name: set ceph osd install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_osd: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" roles: - { role: ceph-defaults, tags: ['ceph_update_config'] } - ceph-common - { role: ceph-config, tags: ['ceph_update_config'] } - ceph-osd + post_tasks: + - name: set ceph osd install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_osd: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - hosts: mdss gather_facts: false become: True + pre_tasks: + - name: set ceph mds install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_mds: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" roles: - { role: ceph-defaults, tags: ['ceph_update_config'] } - ceph-common - { role: ceph-config, tags: ['ceph_update_config'] } - ceph-mds + post_tasks: + - name: set ceph mds install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_mds: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - hosts: rgws gather_facts: false become: True + pre_tasks: + - name: set ceph rgw install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_rgw: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" roles: - { role: ceph-defaults, tags: ['ceph_update_config'] } - ceph-common - { role: ceph-config, tags: ['ceph_update_config'] } - ceph-rgw + post_tasks: + - name: set ceph rgw install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_rgw: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - hosts: nfss gather_facts: false become: True + pre_tasks: + - name: set ceph nfs install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_nfs: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" roles: - { role: ceph-defaults, tags: ['ceph_update_config'] } - ceph-common - { role: ceph-config, tags: ['ceph_update_config'], when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" } - { role: ceph-nfs, when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" } + post_tasks: + - name: set ceph nfs install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_nfs: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - hosts: restapis gather_facts: false become: True + pre_tasks: + - name: set ceph rest api install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_restapi: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" roles: - { role: ceph-defaults, tags: ['ceph_update_config'] } - ceph-common - { role: ceph-config, tags: ['ceph_update_config'] } - ceph-restapi + post_tasks: + - name: set ceph rest api install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_restapi: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - hosts: rbdmirrors gather_facts: false become: True + pre_tasks: + - name: set ceph rbd mirror install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_rbdmirror: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" roles: - { role: ceph-defaults, tags: ['ceph_update_config'] } - ceph-common - { role: ceph-config, tags: ['ceph_update_config'] } - ceph-rbd-mirror + post_tasks: + - name: set ceph rbd mirror install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_rbdmirror: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - hosts: clients gather_facts: false become: True + pre_tasks: + - name: set ceph client install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_client: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" roles: - { role: ceph-defaults, tags: ['ceph_update_config'] } - ceph-common - { role: ceph-config, tags: ['ceph_update_config'] } - ceph-client + post_tasks: + - name: set ceph client install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_client: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - hosts: iscsi-gws gather_facts: false become: True + pre_tasks: + - name: set ceph iscsi gateway install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_iscsi_gw: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" roles: - { role: ceph-defaults, tags: ['ceph_update_config'] } - ceph-common - { role: ceph-config, tags: ['ceph_update_config'], when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" } - { role: ceph-iscsi-gw, when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" } - + post_tasks: + - name: set ceph iscsi gw install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_iscsi_gw: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"