From: Andrew Schoen <aschoen@redhat.com>
Date: Wed, 22 Aug 2018 18:40:41 +0000 (-0400)
Subject: ceph-volume: run tests without waiting on ceph repos
X-Git-Tag: v14.0.1~501^2~1
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=6d712ef959f7ea5385bc6bf9b9ade60662940369;p=ceph-ci.git

ceph-volume: run tests without waiting on ceph repos

This provides a new playbook that installs ceph on all nodes and
generates the ceph config. Then it will rsync ceph-volume from the
control node, or jenkins slave, to all testing vms before completing
the deployment. This means we can run tests on PRs without waiting
for repos to be built for the branch in the PR.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---

diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini b/src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini
index c36a70d0c93..37a9c1819ab 100644
--- a/src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini
+++ b/src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini
@@ -7,6 +7,7 @@ whitelist_externals =
     vagrant
     bash
     git
+    cp
 passenv=*
 setenv=
   ANSIBLE_SSH_ARGS = -F {changedir}/vagrant_ssh_config
@@ -36,8 +37,10 @@ commands=
   vagrant up {env:VAGRANT_UP_FLAGS:"--no-provision"} {posargs:--provider=virtualbox}
   bash {toxinidir}/../scripts/generate_ssh_config.sh {changedir}
 
+  cp {toxinidir}/../playbooks/deploy.yml {envdir}/tmp/ceph-ansible
+
   # use ceph-ansible to deploy a ceph cluster on the vms
-  ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/site.yml.sample --extra-vars "fetch_directory={changedir}/fetch ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest}"
+  ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/deploy.yml --extra-vars "fetch_directory={changedir}/fetch ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} toxinidir={toxinidir}"
 
   # prepare nodes for testing with testinfra
   ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/tests/functional/setup.yml
diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini b/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini
index 8171ffbaed6..97f5d51a39e 100644
--- a/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini
+++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini
@@ -7,6 +7,7 @@ whitelist_externals =
     vagrant
     bash
     git
+    cp
 passenv=*
 setenv=
   ANSIBLE_SSH_ARGS = -F {changedir}/vagrant_ssh_config
@@ -53,8 +54,10 @@ commands=
   # ad-hoc/local test setup for lvm
   ansible-playbook -vv -i {changedir}/hosts {changedir}/setup.yml
 
+  cp {toxinidir}/../playbooks/deploy.yml {envdir}/tmp/ceph-ansible
+
   # use ceph-ansible to deploy a ceph cluster on the vms
-  ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/site.yml.sample --extra-vars "fetch_directory={changedir}/fetch ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest}"
+  ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/deploy.yml --extra-vars "fetch_directory={changedir}/fetch ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} toxinidir={toxinidir}"
 
   # prepare nodes for testing with testinfra
   ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/tests/functional/setup.yml
diff --git a/src/ceph-volume/ceph_volume/tests/functional/playbooks/deploy.yml b/src/ceph-volume/ceph_volume/tests/functional/playbooks/deploy.yml
new file mode 100644
index 00000000000..6561e6e51d4
--- /dev/null
+++ b/src/ceph-volume/ceph_volume/tests/functional/playbooks/deploy.yml
@@ -0,0 +1,121 @@
+---
+# Defines deployment design and assigns role to server groups
+
+- hosts:
+  - mons
+  - osds
+  - mgrs
+
+  gather_facts: false
+  any_errors_fatal: true
+  become: true
+
+  tags:
+    - always
+
+  vars:
+    delegate_facts_host: True
+
+  pre_tasks:
+    # If we can't get python2 installed before any module is used we will fail
+    # so just try what we can to get it installed
+    - name: check for python2
+      stat:
+        path: /usr/bin/python
+      ignore_errors: yes
+      register: systempython2
+
+    - name: install python2 for debian based systems
+      raw: sudo apt-get -y install python-simplejson
+      ignore_errors: yes
+      when:
+        - systempython2.stat is undefined or systempython2.stat.exists == false
+
+    - name: install python2 for fedora
+      raw: sudo dnf -y install python creates=/usr/bin/python
+      ignore_errors: yes
+      when:
+        - systempython2.stat is undefined or systempython2.stat.exists == false
+
+    - name: install python2 for opensuse
+      raw: sudo zypper -n install python-base creates=/usr/bin/python2.7
+      ignore_errors: yes
+      when:
+        - systempython2.stat is undefined or systempython2.stat.exists == false
+
+    - name: gather facts
+      setup:
+      when:
+        - not delegate_facts_host | bool
+
+    - name: gather and delegate facts
+      setup:
+      delegate_to: "{{ item }}"
+      delegate_facts: True
+      with_items: "{{ groups['all'] }}"
+      run_once: true
+      when:
+        - delegate_facts_host | bool
+
+    - name: install required packages for fedora > 23
+      raw: sudo dnf -y install python2-dnf libselinux-python ntp
+      when:
+        - ansible_distribution == 'Fedora'
+        - ansible_distribution_major_version|int >= 23
+
+  roles:
+    - ceph-defaults
+    - ceph-validate
+
+- hosts:
+  - mons
+  - osds
+  - mgrs
+  gather_facts: false
+  become: True
+  roles:
+    - role: ceph-defaults
+      tags: ['ceph_update_config']
+    - role: ceph-common
+    - role: ceph-config
+      tags: ['ceph_update_config']
+
+- hosts: mons
+  gather_facts: false
+  become: True
+  roles:
+    - role: ceph-defaults
+    - role: ceph-common
+    - role: ceph-mon
+
+- hosts: mgrs
+  gather_facts: false
+  become: True
+  roles:
+    - role: ceph-defaults
+    - role: ceph-common
+    - role: ceph-mgr
+
+- hosts: osds
+  gather_facts: false
+  become: True
+  tasks:
+    - name: rsync ceph-volume to test nodes on centos
+      synchronize:
+        src: "{{ toxinidir}}/../../../../ceph_volume"
+        dest: "/usr/lib/python2.7/site-packages"
+      when: ansible_os_family == "RedHat"
+
+    - name: rsync ceph-volume to test nodes on ubuntu
+      synchronize:
+        src: "{{ toxinidir}}/../../../../ceph_volume"
+        dest: "/usr/lib/python2.7/dist-packages"
+      when: ansible_os_family == "Ubuntu"
+
+- hosts: osds
+  gather_facts: false
+  become: True
+  roles:
+    - role: ceph-defaults
+    - role: ceph-common
+    - role: ceph-osd
diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini b/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini
index ea732778d09..632cbb02f2f 100644
--- a/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini
+++ b/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini
@@ -8,6 +8,7 @@ whitelist_externals =
     bash
     git
     sleep
+    cp
 passenv=*
 setenv=
   ANSIBLE_SSH_ARGS = -F {changedir}/vagrant_ssh_config
@@ -44,8 +45,10 @@ commands=
   vagrant up {env:VAGRANT_UP_FLAGS:"--no-provision"} {posargs:--provider=virtualbox}
   bash {toxinidir}/../scripts/generate_ssh_config.sh {changedir}
 
+  cp {toxinidir}/../playbooks/deploy.yml {envdir}/tmp/ceph-ansible
+
   # use ceph-ansible to deploy a ceph cluster on the vms
-  ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/site.yml.sample --extra-vars "fetch_directory={changedir}/fetch ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest}"
+  ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/deploy.yml --extra-vars "fetch_directory={changedir}/fetch ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} toxinidir={toxinidir}"
 
   # prepare nodes for testing with testinfra
   ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/tests/functional/setup.yml