From dab1f1a649a51173b008c5a185395b6741899994 Mon Sep 17 00:00:00 2001 From: David Galloway Date: Thu, 21 May 2020 12:52:14 -0400 Subject: [PATCH] ansible: One playbook to rule them all I/we got into a bad habit of updating one playbook to support X new distro or X new package but wouldn't update the others. I think having all the tasks for all the types of slaves in one playbook will help keep things homogenous. And the playbooks are still idempotent (they can't be run while a job is running of course). Signed-off-by: David Galloway --- ansible/examples/slave.yml | 712 +++++++++++++++++++++++++++++++++++++ ansible/slave.yml | 1 + 2 files changed, 713 insertions(+) create mode 100644 ansible/examples/slave.yml create mode 120000 ansible/slave.yml diff --git a/ansible/examples/slave.yml b/ansible/examples/slave.yml new file mode 100644 index 00000000..67547432 --- /dev/null +++ b/ansible/examples/slave.yml @@ -0,0 +1,712 @@ +--- +## Instead of trying to keep 4 separate playbooks up to date, let's do it all here. +## The only difference from using multiple playbooks is we need to specify `-e libvirt=true` and/or `-e permanent=true` if the slave will be permanent/static. +## Tested on: CentOS 7, CentOS 8, Xenial, Bionic, Focal, Leap 15.1 using ansible 2.8.5 +## +## Example: +## ansible-playbook -vvv -M ./library/ slave.yml -e '{"labels": "x86_64 xenial etc", "token": "XXXXX", "jenkins_credentials_uuid": "jenkins-build", "api_uri": "https://jenkins.ceph.com"}' -e permanent=true -e ansible_ssh_user=ubuntu --limit braggi01* + +- hosts: all + become: true + user: ubuntu # This should be overridden on the CLI (e.g., -e user=centos). It doesn't matter on a mita/prado slave because the playbook is run locally by root. + vars: + - libvirt: false # Should vagrant be installed? + - permanent: false # Is this a permanent slave? Since the ephemeral (non-permanent) tasks get run more often, we'll default to false. + - jenkins_user: 'jenkins-build' + #- jenkins_key: This gets defined below now. + # jenkins API credentials: + - api_user: 'ceph-jenkins' + - token: '{{ token }}' + - api_uri: 'https://jenkins.ceph.com' + - jenkins_credentials_uuid: 'jenkins-build' + - nodename: '{{ nodename }}' + - labels: '{{ labels }}' + - grant_sudo: true + - osc_user: 'username' + - osc_pass: 'password' + + tasks: + ## DEFINE PACKAGE LISTS BELOW + # Universal DEBs + - set_fact: + universal_debs: + - git + - fakeroot + - fakeroot-ng + - debhelper + - reprepro + - devscripts + - pbuilder + - pkg-config + - libtool + - autotools-dev + - automake + - libssl-dev + - libffi-dev + - default-jdk + - default-jre + - debian-keyring + - debian-archive-keyring + # jenkins-job-builder job: + - libyaml-dev + - jq + # ceph-docs job: + - doxygen + - ditaa + - ant + when: ansible_os_family == "Debian" + + # Libvirt DEBs (Bionic and older) + - set_fact: + libvirt_debs: + - qemu-kvm + - libvirt-bin + - libvirt-dev + - vagrant + when: + - ansible_os_family == "Debian" + - ansible_distribution_major_version|int <= 18 + - libvirt|bool + + # Libvirt DEBs (Focal and newer) + - set_fact: + libvirt_debs: + - qemu-kvm + - libvirt-daemon-system + - libvirt-clients + - libvirt-dev + - vagrant + when: + - ansible_os_family == "Debian" + - ansible_distribution_major_version|int >= 20 + - libvirt|bool + + # python2 DEBs + - set_fact: + python2_debs: + - python + - python-dev + - python-pip + - python-virtualenv + when: + - ansible_os_family == "Debian" + - ansible_distribution_major_version|int <= 18 + + # python3 DEBs (We only install python2 *and* python3 on Bionic) + - set_fact: + python3_debs: + - python3 + - python3-dev + - python3-pip + - python3-virtualenv + when: + - ansible_os_family == "Debian" + - ansible_distribution_major_version|int >= 18 + + # chroot DEBs (Xenial and older) + - set_fact: + chroot_deb: dchroot + when: + - ansible_os_family == "Debian" + - ansible_distribution_major_version|int <= 16 + + # chroot DEBs (Bionic and newer) + - set_fact: + chroot_deb: schroot + when: + - ansible_os_family == "Debian" + - ansible_distribution_major_version|int >= 18 + + # Universal RPMs + - set_fact: + universal_rpms: + - createrepo + - java-1.8.0-openjdk + - git + - libtool + #- rpm-sign + - autoconf + - redhat-lsb-core + - automake + - cmake + - binutils + - bison + - flex + - gcc + - gcc-c++ + - gettext + - libtool + - make + - patch + - pkgconfig + - redhat-rpm-config + - rpm-build + - rpmdevtools + - openssl-devel + - libffi-devel + - mock + when: ansible_os_family == "RedHat" + + # Libvirt RPMs + - set_fact: + libvirt_rpms: + - qemu-kvm + - libvirt-devel + - libguestfs + - libvirt + - libguestfs-tools + - vagrant + when: + - ansible_os_family == "RedHat" + - libvirt|bool + + # EPEL7 RPMs + - set_fact: + epel_rpms: + - jq + - python-pip + - python-devel + - python-virtualenv + when: + - ansible_os_family == "RedHat" + - ansible_distribution_major_version|int <= 7 + + # EPEL8 RPMs + - set_fact: + epel_rpms: + - jq + - python3-pip + - python3-devel + - python3-virtualenv + when: + - ansible_os_family == "RedHat" + - ansible_distribution_major_version|int >= 8 + + # OpenSUSE RPMs + - set_fact: + zypper_rpms: + - autoconf + - automake + - binutils + - bison + - cmake + - ccache + - createrepo + - flex + - gcc + - gcc-c++ + - gettext-runtime + - git + - java-1_8_0-openjdk + - jq + - libffi-devel + - libopenssl-devel + - libtool + - lsb-release + - make + - patch + - pkg-config + - python2-pip + - python2-virtualenv + - python3-pip + - python3-virtualenv + - rpm-build + - rpmdevtools + - tig + - wget + # obs requirements + - osc + - build + when: ansible_os_family == "Suse" + + # OpenSUSE Libvirt RPMs (We've never tried this to date so more might be needed) + - set_fact: + zypper_libvirt_rpms: + - libvirt + - libvirt-devel + - qemu + - kvm + - vagrant + - ruby-devel + when: + - ansible_os_family == "Suse" + - libvirt|bool + + ## Let's make sure we don't accidentally set up a permanent slave from Sepia as ephemeral + - set_fact: + permanent: true + with_items: "{{ ansible_all_ipv4_addresses }}" + when: "item.startswith('172.21.')" + + ## EPHEMERAL SLAVE TASKS + # We would occasionally have issues with name resolution on the Ephemeral slaves + # so we force them to use Google's DNS servers. This has to be done before + # package-related tasks to avoid communication errors with various repos. + - name: Ephemeral Slave Tasks + block: + - name: Uninstall resolvconf on Ubuntu to manually manage resolv.conf + apt: + name: resolvconf + state: absent + when: ansible_os_family == "Debian" + + - name: Check for NetworkManager conf + stat: + path: /etc/NetworkManager/NetworkManager.conf + register: nm_conf + + - name: Tell NetworkManager to leave resolv.conf alone on CentOS + lineinfile: + dest: /etc/NetworkManager/NetworkManager.conf + regexp: '^dns=' + line: 'dns=none' + state: present + when: ansible_os_family == "RedHat" and nm_conf.stat.exists + + - name: Tell dhclient to leave resolv.conf alone on Ubuntu + lineinfile: + dest: /etc/dhcp/dhclient.conf + regexp: 'prepend domain-name-servers' + line: 'supersede domain-name-servers 8.8.8.8;' + state: present + when: ansible_os_family == "Debian" + + - name: Use Google DNS for name resolution + lineinfile: + dest: /etc/resolv.conf + regexp: '^nameserver' + line: 'nameserver 8.8.8.8' + state: present + + - name: Set Hostname with hostname command + hostname: + name: "ceph-builders" + when: ansible_os_family != "Suse" + + # https://github.com/ansible/ansible/issues/42726 + - name: Set Hostname on OpenSUSE Leap + command: 'hostname ceph-builders' + when: ansible_os_family == "Suse" + + - name: Ensure that 127.0.1.1 is present with an actual hostname + lineinfile: + backup: yes + dest: /etc/hosts + line: '127.0.1.1 ceph-builders' + + - name: Update etc cloud templates for debian /etc/hosts + lineinfile: + backup: yes + dest: /etc/cloud/templates/hosts.debian.tmpl + line: '127.0.1.1 ceph-builders' + when: ansible_os_family == "Debian" + + - name: Update /etc/cloud templates for Red Hat /etc/hosts + lineinfile: + backup: yes + dest: /etc/cloud/templates/hosts.redhat.tmpl + line: '127.0.1.1 ceph-builders' + failed_when: false + when: ansible_os_family == "RedHat" + + - name: Update /etc/cloud templates for Suse /etc/hosts + lineinfile: + backup: yes + dest: /etc/cloud/templates/hosts.suse.tmpl + line: '127.0.1.1 ceph-builders' + failed_when: false + when: ansible_os_family == "Suse" + when: not permanent|bool + + ## VAGRANT REPO TASKS (for libvirt slaves) + # vagrant doesn't have repositories, this chacra repo will be better to have + # around and can get updates as soon as a new vagrant version is published via + # chacractl + - name: Vagrant/Libvirt Repo Tasks + block: + - name: Add our vagrant DEB repository + apt_repository: + repo: "deb [trusted=yes] https://chacra.ceph.com/r/vagrant/latest/HEAD/ubuntu/{{ ansible_distribution_release }}/flavors/default/ {{ ansible_distribution_release }} main" + state: present + when: ansible_os_family == "Debian" + + - name: Add our vagrant RPM repository + yum_repository: + name: vagrant + description: self-hosted vagrant repo + # Although this is a 'CentOS7' repo, the vagrant RPM is OS-version agnostic + baseurl: "https://chacra.ceph.com/r/vagrant/latest/HEAD/centos/7/flavors/default/x86_64/" + enabled: yes + gpgcheck: no + when: ansible_os_family == "RedHat" + when: libvirt|bool + + ## PACKAGE INSTALLATION TASKS + # We do this in one big task to save time and avoid using `with` loops. If a variable isn't defined, it's fine because of the |defaults. + - name: Install DEBs + apt: + name: "{{ universal_debs + libvirt_debs|default([]) + python2_debs|default([]) + python3_debs|default([]) + [ chroot_deb|default('') ] }}" + state: latest + update_cache: yes + when: ansible_os_family == "Debian" + + - name: Install EPEL repo + yum: + name: epel-release + state: latest + when: ansible_os_family == "RedHat" + + - name: Install RPMs without EPEL + yum: + name: "{{ universal_rpms + libvirt_rpms|default([]) }}" + state: present + disablerepo: epel + when: ansible_os_family == "RedHat" + + - name: Install RPMs with EPEL + yum: + name: "{{ epel_rpms|default([]) }}" + state: latest + enablerepo: epel + when: ansible_os_family == "RedHat" + + - name: Install Suse RPMs + zypper: + name: "{{ zypper_rpms + zypper_libvirt_rpms|default([]) }}" + state: latest + update_cache: yes + when: ansible_os_family == "Suse" + + ## JENKINS USER TASKS + - set_fact: + jenkins_groups: + - "{{ jenkins_user }}" + - libvirtd + when: + - ansible_os_family == "Debian" + - ansible_distribution_version == '16.04' + - libvirt|bool + + # The group name changed to 'libvirt' in Ubuntu 16.10 and is already 'libvirt' everywhere else + - set_fact: + jenkins_groups: + - "{{ jenkins_user }}" + - libvirt + when: + - not (ansible_os_family == "Debian" and ansible_distribution_version == '16.04') + - libvirt|bool + + - name: "Create a {{ jenkins_user }} group" + group: + name: "{{ jenkins_user }}" + state: present + + - name: "Create a {{ jenkins_user }} user" + user: + name: "{{ jenkins_user }}" + # This will add to the jenkins_user and appropriate libvirt group if jenkins_groups is defined. + # Otherwise, default to just adding to {{ jenkins_user }} group. + groups: "{{ jenkins_groups|default(jenkins_user) }}" + state: present + comment: "Jenkins Build Slave User" + + - name: "Add {{ jenkins_user }} to mock group" + user: + name: "{{ jenkins_user }}" + groups: mock + append: yes + when: ansible_os_family == "RedHat" + + - name: "Create a {{ jenkins_user }} home directory" + file: + path: "/home/{{ jenkins_user }}/" + state: directory + owner: "{{ jenkins_user }}" + + - name: Create .ssh directory + file: + path: "/home/{{ jenkins_user }}/.ssh" + state: directory + owner: "{{ jenkins_user }}" + + # On a mita/prado provisioned slave, everything gets put into a 'playbook' dir. + # Otherwise it can be found in files/ssh/... + - set_fact: + jenkins_key: "{{ lookup('first_found', key_locations) }}" + vars: + key_locations: + - "playbook/files/ssh/keys/jenkins_build.pub" + - "files/ssh/keys/jenkins_build.pub" + + # And worst case scenario, we just pull the key from github. + - name: Set the authorized keys + authorized_key: + user: "{{ jenkins_user }}" + key: "{{ lookup('file', '{{ jenkins_key }}')|default('https://raw.githubusercontent.com/ceph/ceph-build/master/ansible/files/ssh/keys/jenkins_build.pub') }}" + + - name: "Ensure {{ jenkins_user }} can sudo without a prompt" + lineinfile: + dest: /etc/sudoers + regexp: '^{{ jenkins_user }} ALL' + line: '{{ jenkins_user }} ALL=(ALL:ALL) NOPASSWD:ALL' + validate: 'visudo -cf %s' + when: grant_sudo|bool + + - name: Set utf-8 for LC_ALL + lineinfile: + dest: "/home/{{ jenkins_user }}/.bashrc" + regexp: '^export LC_ALL=' + line: "export LC_ALL=en_US.UTF-8" + create: true + state: present + + - name: Set utf-8 for LANG + lineinfile: + dest: "/home/{{ jenkins_user }}/.bashrc" + regexp: '^export LANG=' + line: "export LANG=en_US.UTF-8" + + - name: Set utf-8 for LANGUAGE + lineinfile: + dest: "/home/{{ jenkins_user }}/.bashrc" + regexp: '^export LANGUAGE=' + line: "export LANGUAGE=en_US.UTF-8" + + - name: Ensure the build dir exists + file: + path: "/home/{{ jenkins_user }}/build" + state: directory + owner: "{{ jenkins_user }}" + + - name: Create .config/osc directory + file: + path: "/home/{{ jenkins_user }}/.config/osc" + state: directory + owner: "{{ jenkins_user }}" + when: ansible_os_family == "Suse" + + - name: Add oscrc file + blockinfile: + create: yes + block: | + [general] + apiurl = https://api.opensuse.org + #build-root = /var/tmp/build-root/%(repo)s-%{arch)s + [https://api.opensuse.org] + user = {{ osc_user }} + pass = {{ osc_pass }} + path: "/home/{{ jenkins_user }}/.config/osc/oscrc" + become_user: "{{ jenkins_user }}" + when: ansible_os_family == "Suse" + + - name: Ensure the home dir has the right owner permissions + file: + path: "/home/{{ jenkins_user }}" + state: directory + owner: "{{ jenkins_user }}" + group: "{{ jenkins_user }}" + recurse: yes + + ## DEBIAN GPG KEY TASKS + - name: Install Debian GPG Keys on Ubuntu + block: + - name: Add the Debian Jessie Key + apt_key: + id: 2B90D010 + url: https://ftp-master.debian.org/keys/archive-key-8.asc + keyring: /etc/apt/trusted.gpg + state: present + + - name: Add the Debian Security Jessie Key + apt_key: + id: C857C906 + url: https://ftp-master.debian.org/keys/archive-key-8-security.asc + keyring: /etc/apt/trusted.gpg + state: present + + - name: Add the Debian Jessie Stable Key + apt_key: + id: 518E17E1 + url: https://ftp-master.debian.org/keys/release-8.asc + keyring: /etc/apt/trusted.gpg + state: present + + - name: Add the Debian Buster Key + apt_key: + id: 3CBBABEE + url: https://ftp-master.debian.org/keys/archive-key-10.asc + keyring: /etc/apt/trusted.gpg + state: present + + - name: Add the Debian Security Buster Key + apt_key: + id: CAA96DFA + url: https://ftp-master.debian.org/keys/archive-key-10-security.asc + keyring: /etc/apt/trusted.gpg + state: present + + - name: Add the Debian Buster Stable Key + apt_key: + id: 77E11517 + url: https://ftp-master.debian.org/keys/release-10.asc + keyring: /etc/apt/trusted.gpg + state: present + when: ansible_os_family == "Debian" + + ## VAGRANT PLUGIN TASKS + - name: Install vagrant-libvirt plugin + block: + - name: Install the vagrant-libvirt plugin (without args) + shell: vagrant plugin install vagrant-libvirt + become_user: "{{ jenkins_user }}" + when: (ansible_os_family == "RedHat" and ansible_distribution_major_version|int <= 7) or + (ansible_os_family == "Debian" and ansible_distribution_major_version|int <= 18) + + - name: Install the vagrant-libvirt plugin (EL8/Suse) + command: vagrant plugin install vagrant-libvirt + become_user: "{{ jenkins_user }}" + environment: + CONFIGURE_ARGS: 'with-libvirt-include=/usr/include/libvirt with-libvirt-lib=/usr/lib64' + when: (ansible_os_family == "RedHat" and ansible_distribution_major_version|int >= 8) or + ansible_os_family == "Suse" + + - name: Install the vagrant-libvirt plugin (Focal) + command: vagrant plugin install vagrant-libvirt + become_user: "{{ jenkins_user }}" + environment: + CONFIGURE_ARGS: 'with-libvirt-include=/usr/include/libvirt with-libvirt-lib=/usr/lib' + when: ansible_os_family == "Debian" and ansible_distribution_major_version|int >= 20 + when: libvirt|bool + + ## RPMMACROS TASKS + - name: rpmmacros Tasks + block: + - name: Ensure the rpmmacros file exists to fix centos builds + file: + path: "/home/{{ jenkins_user }}/.rpmmacros" + owner: "{{ jenkins_user }}" + state: touch + + - name: Write the rpmmacros needed in centos + lineinfile: + dest: "/home/{{ jenkins_user }}/.rpmmacros" + regexp: '^%dist' + line: '%dist .el{{ ansible_distribution_major_version }}' + when: ansible_os_family == "RedHat" + + ## GITCONFIG TASKS + - name: Ensure the gitconfig file exists + shell: printf "[user]\name=Ceph CI\nemail=ceph-release-team@redhat.com\n" > /home/{{ jenkins_user }}/.gitconfig + tags: github + + - name: Ensure the gitconfig file has right permissions + file: + path: "/home/{{ jenkins_user }}/.gitconfig" + owner: "{{ jenkins_user }}" + tags: github + + # On a mita/prado provisioned slave, everything gets put into a 'playbook' dir. + # If all else fails, get it from github (using the |default) + - set_fact: + github_key: "{{ lookup('first_found', key_locations) }}" + vars: + key_locations: + # github.com.pub is the output of `ssh-keyscan github.com` + - "playbook/files/ssh/hostkeys/github.com.pub" + - "files/ssh/hostkeys/github.com.pub" + tags: github + + - name: Add github.com host key + known_hosts: + name: github.com + path: '/etc/ssh/ssh_known_hosts' + key: "{{ lookup('file', '{{ github_key }}')|default('https://raw.githubusercontent.com/ceph/ceph-build/master/ansible/files/ssh/hostkeys/github.com.pub') }}" + tags: github + + ## PIP TASKS + - set_fact: + pip_version: pip + ansible_python_interpreter: /usr/bin/python + when: (ansible_os_family == "RedHat" and ansible_distribution_major_version|int <= 7) or + (ansible_os_family == "Debian" and ansible_distribution_major_version|int <= 16) + + - set_fact: + pip_version: pip3 + ansible_python_interpreter: /usr/bin/python3 + when: (ansible_os_family == "RedHat" and ansible_distribution_major_version|int >= 8) or + (ansible_os_family == "Debian" and ansible_distribution_major_version|int >= 18) or + ansible_os_family == "Suse" + + - name: Install six, latest one + pip: + name: six + state: latest + executable: "{{ pip_version }}" + when: ansible_os_family != "Suse" + + - name: Install python-jenkins + # https://review.openstack.org/460363 + pip: + name: python-jenkins + version: 0.4.15 + executable: "{{ pip_version }}" + + ## JENKINS SLAVE AGENT TASKS + # We use jnlp on ephemeral non-permanent slaves because ??? + - name: Register the new slave to jenkins master with jnlp + jenkins_node: + username: "{{ api_user }}" + uri: "{{ api_uri }}" + password: "{{ token }}" + # relies on a convention to set a unique name that allows a reverse + # mapping from Jenkins back to whatever service created the current + # node + name: "{{ ansible_default_ipv4.address }}+{{ nodename }}" + labels: "{{ labels }}" + host: "{{ ansible_default_ipv4.address }}" + credentialsId: "{{ jenkins_credentials_uuid }}" + launcher: 'hudson.slaves.JNLPLauncher' + remoteFS: '/home/{{ jenkins_user }}/build' + executors: '{{ executors|default(1) }}' + exclusive: true + when: not permanent|bool + + - name: Register Permanent Slave + block: + - name: Register the new slave to jenkins master with ssh + jenkins_node: + username: "{{ api_user }}" + uri: "{{ api_uri }}" + password: "{{ token }}" + # relies on a convention to set a unique name that allows a reverse + # mapping from Jenkins back to whatever service created the current + # node + name: "{{ ansible_default_ipv4.address }}+{{ ansible_hostname }}" + labels: "{{ labels }}" + host: "{{ ansible_default_ipv4.address }}" + credentialsId: "{{ jenkins_credentials_uuid }}" + remoteFS: '/home/{{ jenkins_user }}/build' + executors: '{{ executors|default(1) }}' + exclusive: true + + - name: Download slave.jar + get_url: + url: "{{ api_uri }}/jnlpJars/slave.jar" + dest: "/home/{{ jenkins_user }}/slave.jar" + force: yes + + - name: Install the systemd unit file for jenkins + template: + src: "templates/systemd/jenkins.service.j2" + dest: "/etc/systemd/system/jenkins.service" + + - name: Reload systemd unit files (to pick up potential changes) + systemd: + daemon_reload: yes + + - name: Start jenkins service + service: + name: jenkins + state: started + enabled: yes + when: permanent|bool diff --git a/ansible/slave.yml b/ansible/slave.yml new file mode 120000 index 00000000..b9862807 --- /dev/null +++ b/ansible/slave.yml @@ -0,0 +1 @@ +examples/slave.yml \ No newline at end of file -- 2.39.5