add support for installation checkpoint

This was taken from the openshift ansible repository here:
https://github.com/leseb/openshift-ansible/tree/master/roles/installer_checkpoint

Rationale:

A complete OpenShift cluster installation is comprised of many different
components which can take 30 minutes to several hours to complete. If
the installation should fail, it could be confusing to understand at
which component the failure occurred. Additionally, it may be desired to
re-run only the component which failed instead of starting over from the
beginning. Components which came after the failed component would also
need to be run individually.

Ceph has a similar situation so we can benefit from that
callback_plugin.

Signed-off-by: Sébastien Han <seb@redhat.com>

diff --git a/callback_plugins/installer_checkpoint.py b/callback_plugins/installer_checkpoint.py
new file mode 100644 (file)
index 0000000..65e4967
--- /dev/null
+++ b/callback_plugins/installer_checkpoint.py
@@ -0,0 +1,139 @@
+"""Ansible callback plugin to print a summary completion status of installation
+phases.
+"""
+from datetime import datetime
+from ansible.plugins.callback import CallbackBase
+from ansible import constants as C
+
+
+class CallbackModule(CallbackBase):
+    """This callback summarizes installation phase status."""
+
+    CALLBACK_VERSION = 2.0
+    CALLBACK_TYPE = 'aggregate'
+    CALLBACK_NAME = 'installer_checkpoint'
+    CALLBACK_NEEDS_WHITELIST = False
+
+    def __init__(self):
+        super(CallbackModule, self).__init__()
+
+    def v2_playbook_on_stats(self, stats):
+
+        # Set the order of the installer phases
+        installer_phases = [
+            'installer_phase_ceph_mon',
+            'installer_phase_ceph_mgr',
+            'installer_phase_ceph_agent',
+            'installer_phase_ceph_osd',
+            'installer_phase_ceph_mds',
+            'installer_phase_ceph_rgw',
+            'installer_phase_ceph_nfs',
+            'installer_phase_ceph_restapi',
+            'installer_phase_ceph_rbdmirror',
+            'installer_phase_ceph_client',
+            'installer_phase_ceph_iscsi_gw',
+        ]
+
+        # Define the attributes of the installer phases
+        phase_attributes = {
+            'installer_phase_ceph_mon': {
+                'title': 'Install Ceph Monitor',
+                'playbook': 'roles/ceph-mon/tasks/main.yml'
+            },
+            'installer_phase_ceph_mgr': {
+                'title': 'Install Ceph Manager',
+                'playbook': 'roles/ceph-mgr/tasks/main.yml'
+            },
+            'installer_phase_ceph_agent': {
+                'title': 'Install Ceph Agent',
+                'playbook': 'roles/ceph-agent/tasks/main.yml'
+            },
+            'installer_phase_ceph_osd': {
+                'title': 'Install Ceph OSD',
+                'playbook': 'roles/ceph-osd/tasks/main.yml'
+            },
+            'installer_phase_ceph_mds': {
+                'title': 'Install Ceph MDS',
+                'playbook': 'roles/ceph-mds/tasks/main.yml'
+            },
+            'installer_phase_ceph_rgw': {
+                'title': 'Install Ceph RGW',
+                'playbook': 'roles/ceph-rgw/tasks/main.yml'
+            },
+            'installer_phase_ceph_nfs': {
+                'title': 'Install Ceph NFS',
+                'playbook': 'roles/ceph-nfs/tasks/main.yml'
+            },
+            'installer_phase_ceph_restapi': {
+                'title': 'Install Ceph REST API',
+                'playbook': 'roles/ceph-restapi/tasks/main.yml'
+            },
+            'installer_phase_ceph_rbdmirror': {
+                'title': 'Install Ceph RBD Mirror',
+                'playbook': 'roles/ceph-rbd-mirror/tasks/main.yml'
+            },
+            'installer_phase_ceph_client': {
+                'title': 'Install Ceph Client',
+                'playbook': 'roles/ceph-client/tasks/main.yml'
+            },
+            'installer_phase_ceph_iscsi_gw': {
+                'title': 'Install Ceph iSCSI Gateway',
+                'playbook': 'roles/ceph-iscsi-gw/tasks/main.yml'
+            },
+        }
+
+        # Find the longest phase title
+        max_column = 0
+        for phase in phase_attributes:
+            max_column = max(max_column, len(phase_attributes[phase]['title']))
+
+        if '_run' in stats.custom:
+            self._display.banner('INSTALLER STATUS')
+            for phase in installer_phases:
+                phase_title = phase_attributes[phase]['title']
+                padding = max_column - len(phase_title) + 2
+                if phase in stats.custom['_run']:
+                    phase_status = stats.custom['_run'][phase]['status']
+                    phase_time = phase_time_delta(stats.custom['_run'][phase])
+                    self._display.display(
+                        '{}{}: {} ({})'.format(phase_title, ' ' * padding, phase_status, phase_time),
+                        color=self.phase_color(phase_status))
+                    if phase_status == 'In Progress' and phase != 'installer_phase_initialize':
+                        self._display.display(
+                            '\tThis phase can be restarted by running: {}'.format(
+                                phase_attributes[phase]['playbook']))
+
+        self._display.display("", screen_only=True)
+
+    def phase_color(self, status):
+        """ Return color code for installer phase"""
+        valid_status = [
+            'In Progress',
+            'Complete',
+        ]
+
+        if status not in valid_status:
+            self._display.warning('Invalid phase status defined: {}'.format(status))
+
+        if status == 'Complete':
+            phase_color = C.COLOR_OK
+        elif status == 'In Progress':
+            phase_color = C.COLOR_ERROR
+        else:
+            phase_color = C.COLOR_WARN
+
+        return phase_color
+
+
+def phase_time_delta(phase):
+    """ Calculate the difference between phase start and end times """
+    time_format = '%Y%m%d%H%M%SZ'
+    phase_start = datetime.strptime(phase['start'], time_format)
+    if 'end' not in phase:
+        # The phase failed so set the end time to now
+        phase_end = datetime.now()
+    else:
+        phase_end = datetime.strptime(phase['end'], time_format)
+    delta = str(phase_end - phase_start).split(".")[0]  # Trim microseconds
+
+    return delta

diff --git a/site-docker.yml.sample b/site-docker.yml.sample
index e0b0fc2f74e747e154fa3882a3076f6ffa7ac90f..58098ad3c217bbd4fc265f9bd27f93a161d7573d 100644 (file)
--- a/site-docker.yml.sample
+++ b/site-docker.yml.sample
@@ -63,6 +63,16 @@
         - is_atomic
         - (ceph_docker_dev_image is undefined or not ceph_docker_dev_image)
 
+- hosts: mons
+  tasks:
+    - name: set ceph monitor install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_mon:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
+
 - hosts: mons
   become: True
   gather_facts: false
@@ -73,74 +83,212 @@
     - ceph-mon
   serial: 1 # MUST be '1' WHEN DEPLOYING MONITORS ON DOCKER CONTAINERS
 
+- hosts: mons
+  tasks:
+    - name: set ceph monitor install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_mon:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
+
 - hosts: mgrs
   become: True
   gather_facts: false
+  pre_tasks:
+    - name: set ceph manager install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_mgr:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
   roles:
     - { role: ceph-defaults, tags: ['ceph_update_config'] }
     - ceph-docker-common
     - { role: ceph-config, tags: ['ceph_update_config'], when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }
     - { role: ceph-mgr, when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }
+  post_tasks:
+    - name: set ceph manager install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_mgr:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - hosts: osds
   become: True
   gather_facts: false
+  pre_tasks:
+    - name: set ceph osd install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_osd:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
   roles:
     - { role: ceph-defaults, tags: ['ceph_update_config'] }
     - ceph-docker-common
     - { role: ceph-config, tags: ['ceph_update_config'] }
     - ceph-osd
+  post_tasks:
+    - name: set ceph osd install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_osd:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - hosts: mdss
   become: True
   gather_facts: false
+  pre_tasks:
+    - name: set ceph mds install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_mds:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
   roles:
     - { role: ceph-defaults, tags: ['ceph_update_config'] }
     - ceph-docker-common
     - { role: ceph-config, tags: ['ceph_update_config'] }
     - ceph-mds
+  post_tasks:
+    - name: set ceph mds install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_mds:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - hosts: rgws
   become: True
   gather_facts: false
+  pre_tasks:
+    - name: set ceph rgw install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_rgw:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
   roles:
     - { role: ceph-defaults, tags: ['ceph_update_config'] }
     - ceph-docker-common
     - { role: ceph-config, tags: ['ceph_update_config'] }
     - ceph-rgw
+  post_tasks:
+    - name: set ceph rgw install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_rgw:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - hosts: nfss
   become: True
   gather_facts: false
+  pre_tasks:
+    - name: set ceph nfs install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_nfs:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
   roles:
     - { role: ceph-defaults, tags: ['ceph_update_config'] }
     - ceph-docker-common
     - { role: ceph-config, tags: ['ceph_update_config'], when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }
     - { role: ceph-nfs, when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }
+  post_tasks:
+    - name: set ceph nfs install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_nfs:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - hosts: rbdmirrors
   become: True
   gather_facts: false
+  pre_tasks:
+    - name: set ceph rbd mirror install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_rbdmirror:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
   roles:
     - { role: ceph-defaults, tags: ['ceph_update_config'] }
     - ceph-docker-common
     - { role: ceph-config, tags: ['ceph_update_config'] }
     - ceph-rbd-mirror
+  post_tasks:
+    - name: set ceph rbd mirror install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_rbdmirror:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - hosts: restapis
   become: True
   gather_facts: false
+  pre_tasks:
+    - name: set ceph rest api install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_restapi:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
   roles:
     - { role: ceph-defaults, tags: ['ceph_update_config'] }
     - ceph-docker-common
     - { role: ceph-config, tags: ['ceph_update_config'] }
     - ceph-restapi
+  post_tasks:
+    - name: set ceph rest api install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_restapi:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - hosts: clients
   become: True
   gather_facts: false
+  pre_tasks:
+    - name: set ceph client install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_client:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
   roles:
     - { role: ceph-defaults, tags: ['ceph_update_config'] }
     - ceph-docker-common
     - { role: ceph-config, tags: ['ceph_update_config'] }
     - ceph-client
+  post_tasks:
+    - name: set ceph client install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_client:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"

diff --git a/site.yml.sample b/site.yml.sample
index 29f88f24638c81d90cfdc9d315c969ccae83aa2d..5684a502eaeb51593c44fdb8d6aa4e0b4d24902f 100644 (file)
--- a/site.yml.sample
+++ b/site.yml.sample
@@ -71,99 +71,274 @@
 - hosts: mons
   gather_facts: false
   become: True
+  pre_tasks:
+    - name: set ceph monitor install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_mon:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
   roles:
     - { role: ceph-defaults, tags: ['ceph_update_config'] }
     - ceph-common
     - { role: ceph-config, tags: ['ceph_update_config'] }
     - ceph-mon
+  post_tasks:
+    - name: set ceph monitor install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_mon:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - hosts: mgrs
   gather_facts: false
   become: True
+  pre_tasks:
+    - name: set ceph manager install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_mgr:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
   roles:
     - { role: ceph-defaults, tags: ['ceph_update_config'] }
     - ceph-common
     - { role: ceph-config, tags: ['ceph_update_config'], when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }
     - { role: ceph-mgr, when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }
+  post_tasks:
+    - name: set ceph manager install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_mgr:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - hosts: agents
   gather_facts: false
   become: True
+  pre_tasks:
+    - name: set ceph agent install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_agent:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
   roles:
     - { role: ceph-defaults, tags: ['ceph_update_config'] }
     - ceph-common
     - { role: ceph-config, tags: ['ceph_update_config'] }
     - ceph-agent
+  post_tasks:
+    - name: set ceph agent install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_agent:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - hosts: osds
   gather_facts: false
   become: True
+  pre_tasks:
+    - name: set ceph osd install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_osd:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
   roles:
     - { role: ceph-defaults, tags: ['ceph_update_config'] }
     - ceph-common
     - { role: ceph-config, tags: ['ceph_update_config'] }
     - ceph-osd
+  post_tasks:
+    - name: set ceph osd install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_osd:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - hosts: mdss
   gather_facts: false
   become: True
+  pre_tasks:
+    - name: set ceph mds install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_mds:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
   roles:
     - { role: ceph-defaults, tags: ['ceph_update_config'] }
     - ceph-common
     - { role: ceph-config, tags: ['ceph_update_config'] }
     - ceph-mds
+  post_tasks:
+    - name: set ceph mds install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_mds:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - hosts: rgws
   gather_facts: false
   become: True
+  pre_tasks:
+    - name: set ceph rgw install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_rgw:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
   roles:
     - { role: ceph-defaults, tags: ['ceph_update_config'] }
     - ceph-common
     - { role: ceph-config, tags: ['ceph_update_config'] }
     - ceph-rgw
+  post_tasks:
+    - name: set ceph rgw install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_rgw:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - hosts: nfss
   gather_facts: false
   become: True
+  pre_tasks:
+    - name: set ceph nfs install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_nfs:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
   roles:
     - { role: ceph-defaults, tags: ['ceph_update_config'] }
     - ceph-common
     - { role: ceph-config, tags: ['ceph_update_config'], when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }
     - { role: ceph-nfs, when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }
+  post_tasks:
+    - name: set ceph nfs install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_nfs:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - hosts: restapis
   gather_facts: false
   become: True
+  pre_tasks:
+    - name: set ceph rest api install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_restapi:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
   roles:
     - { role: ceph-defaults, tags: ['ceph_update_config'] }
     - ceph-common
     - { role: ceph-config, tags: ['ceph_update_config'] }
     - ceph-restapi
+  post_tasks:
+    - name: set ceph rest api install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_restapi:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - hosts: rbdmirrors
   gather_facts: false
   become: True
+  pre_tasks:
+    - name: set ceph rbd mirror install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_rbdmirror:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
   roles:
     - { role: ceph-defaults, tags: ['ceph_update_config'] }
     - ceph-common
     - { role: ceph-config, tags: ['ceph_update_config'] }
     - ceph-rbd-mirror
+  post_tasks:
+    - name: set ceph rbd mirror install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_rbdmirror:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - hosts: clients
   gather_facts: false
   become: True
+  pre_tasks:
+    - name: set ceph client install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_client:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
   roles:
     - { role: ceph-defaults, tags: ['ceph_update_config'] }
     - ceph-common
     - { role: ceph-config, tags: ['ceph_update_config'] }
     - ceph-client
+  post_tasks:
+    - name: set ceph client install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_client:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - hosts: iscsi-gws
   gather_facts: false
   become: True
+  pre_tasks:
+    - name: set ceph iscsi gateway install 'In Progress'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_iscsi_gw:
+            status: "In Progress"
+            start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
   roles:
     - { role: ceph-defaults, tags: ['ceph_update_config'] }
     - ceph-common
     - { role: ceph-config, tags: ['ceph_update_config'], when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }
     - { role: ceph-iscsi-gw, when: "ceph_release_num[ceph_release] >= ceph_release_num.luminous" }
-
+  post_tasks:
+    - name: set ceph iscsi gw install 'Complete'
+      run_once: true
+      set_stats:
+        data:
+          installer_phase_ceph_iscsi_gw:
+            status: "Complete"
+            end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"