From: Sébastien Han Date: Wed, 5 Nov 2014 16:57:28 +0000 (+0100) Subject: Merge branch 'master' of https://github.com/jjoos/ceph-ansible into jjoos-master X-Git-Tag: v1.0.0~316^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=74ede6e16639cd0246325f26889cef9023355264;p=ceph-ansible.git Merge branch 'master' of https://github.com/jjoos/ceph-ansible into jjoos-master Signed-off-by: Sébastien Han Conflicts: roles/ceph-common/defaults/main.yml roles/ceph-common/tasks/Debian.yml roles/ceph-osd/tasks/journal_collocation.yml roles/ceph-osd/tasks/osd_directory.yml roles/ceph-osd/tasks/raw_journal.yml roles/ceph-osd/tasks/raw_multi_journal.yml --- 74ede6e16639cd0246325f26889cef9023355264 diff --cc roles/ceph-common/handlers/main.yml index 7de099f7c,43787c461..ac61a1b9f --- a/roles/ceph-common/handlers/main.yml +++ b/roles/ceph-common/handlers/main.yml @@@ -1,12 -1,11 +1,11 @@@ --- + - name: update apt cache - action: apt update-cache=yes ++ apt: update-cache=yes - - name: "update apt cache" - action: apt update-cache=yes - - - name: "restart ceph ubuntu" - shell: service ceph restart ; service ceph-osd-all restart - when: socket.rc == 0 - - - name: "restart ceph debian redhat" + - name: restart ceph - shell: service ceph restart + command: service ceph restart when: socket.rc == 0 + + - name: restart ceph-osd-all on ubuntu - shell: service ceph-osd-all restart - when: socket.rc == 0 and ansible_distribution == 'Ubuntu' ++ shell: service ceph restart ; service ceph-osd-all restart ++ when: socket.rc == 0 and ansible_distribution == 'Ubuntu' diff --cc roles/ceph-common/tasks/install_on_debian.yml index 000000000,c9089d326..972210065 mode 000000,100644..100644 --- a/roles/ceph-common/tasks/install_on_debian.yml +++ b/roles/ceph-common/tasks/install_on_debian.yml @@@ -1,0 -1,47 +1,47 @@@ + --- + - name: Install dependencies - apt: > - pkg={{ item }} - state=present - update_cache=yes ++ apt: > ++ pkg={{ item }} ++ state=present ++ update_cache=yes + cache_valid_time=3600 + with_items: + - python-pycurl + - ntp + - hdparm + + - name: Install the Ceph repository stable key - apt_key: > ++ apt_key: > + data="{{ lookup('file', 'cephstable.asc') }}" + state=present + when: ceph_stable + + - name: Install the Ceph developement repository key - apt_key: > ++ apt_key: > + data="{{ lookup('file', 'cephdev.asc') }}" + state=present + when: ceph_dev + + - name: Add Ceph stable repository - apt_repository: > ++ apt_repository: > + repo="deb http://ceph.com/debian-{{ ceph_stable_release }}/ {{ ansible_lsb.codename }} main" + state=present + when: ceph_stable + + - name: Add Ceph development repository + apt_repository: > + repo="deb http://gitbuilder.ceph.com/ceph-deb-{{ ansible_lsb.codename }}-x86_64-basic/ref/{{ ceph_dev_branch }} {{ ansible_lsb.codename }} main" + state=present + when: ceph_dev + + - name: Install Ceph + apt: > + pkg={{ item }} + state=latest + with_items: + - ceph + - ceph-common #| + - ceph-fs-common #|--> yes, they are already all dependencies from 'ceph' + - ceph-fuse #|--> however while proceding to rolling upgrades and the 'ceph' package upgrade + - ceph-mds #|--> they don't get update so we need to force them + - libcephfs1 #| diff --cc roles/ceph-common/tasks/install_on_redhat.yml index 000000000,2903113c8..0463db46f mode 000000,100644..100644 --- a/roles/ceph-common/tasks/install_on_redhat.yml +++ b/roles/ceph-common/tasks/install_on_redhat.yml @@@ -1,0 -1,35 +1,35 @@@ + --- + - name: Install dependencies - yum: > - name={{ item }} ++ yum: > ++ name={{ item }} + state=present + with_items: + - python-pycurl + - ntp + - hdparm + + - name: Install the Ceph stable repository key + rpm_key: > - key={{ ceph_key }} ++ key={{ ceph_key }} + state=present + when: ceph_stable + + - name: Install the Ceph developement repository key + rpm_key: > - key={{ ceph_key }} ++ key={{ ceph_key }} + state=present + when: ceph_dev + + - name: Add Ceph stable repository + command: "rpm -U http://ceph.com/rpm-{{ ceph_stable_release }}/{{ redhat_distro }}/noarch/ceph-release-1-0.el6.noarch.rpm creates=/etc/yum.repos.d/ceph.repo" + when: ceph_stable + + - name: Add Ceph development repository + command: "rpm -U http://gitbuilder.ceph.com/ceph-rpm-{{ ceph_dev_redhat_distro }}-x86_64-basic/ref/{{ ceph_dev_branch }}/noarch/ceph-release-1-0.{{ ceph_stable_redhat_distro }}.noarch.rpm creates=/etc/yum.repos.d/ceph.repo" + when: ceph_dev + + - name: Install Ceph - yum: > - name=ceph ++ yum: > ++ name=ceph + state=latest + diff --cc roles/ceph-common/tasks/main.yml index 602b57c69,4129bea96..4d23e76a0 --- a/roles/ceph-common/tasks/main.yml +++ b/roles/ceph-common/tasks/main.yml @@@ -1,8 -1,36 +1,38 @@@ --- - ## Check OS family - # + - name: Fail on unsupported system + fail: "msg=System not supported {{ ansible_system }}" + when: "ansible_system not in ['Linux']" - - include: RedHat.yml + - name: Fail on unsupported architecture + fail: "msg=Architecture not supported {{ ansible_architecture }}" + when: "ansible_architecture not in ['x86_64']" + + - name: Fail on unsupported distribution + fail: "msg=Distribution not supported {{ ansible_os_family }}" + when: "ansible_os_family not in ['Debian', 'RedHat']" + + - include: install_on_redhat.yml when: ansible_os_family == 'RedHat' - - include: Debian.yml ++ + - include: install_on_debian.yml when: ansible_os_family == 'Debian' + + - name: Check for a Ceph socket + shell: "stat /var/run/ceph/*.asok > /dev/null 2>&1" + ignore_errors: true + register: socket + + - name: Generate Ceph configuration file + template: > - src=ceph.conf.j2 - dest=/etc/ceph/ceph.conf - owner=root - group=root ++ src=ceph.conf.j2 ++ dest=/etc/ceph/ceph.conf ++ owner=root ++ group=root + mode=0644 + notify: + - restart ceph + - restart ceph-osd-all on ubuntu + + - name: Disable OSD directory parsing by updatedb + command: updatedb -e /var/lib/ceph ++ ignore_errors: true diff --cc roles/ceph-common/vars/main.yml index d91813410,abe6f2c13..bc7de5048 --- a/roles/ceph-common/vars/main.yml +++ b/roles/ceph-common/vars/main.yml @@@ -1,3 -1,87 +1,89 @@@ --- - # You can override default vars defined in defaults/main.yml here, - # but I would advice to use host or group vars instead + # You can override vars by using host or group vars + + ## Setup options + # + + # ACTIVATE THE FSID VARIABLE FOR NON-VAGRANT DEPLOYMENT + #fsid: + + ## Packages branch + ceph_key: https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc + ceph_stable: true # use ceph stable branch + ceph_stable_release: firefly # ceph stable release + + # This option is needed for _both_ stable and dev version, so please always fill the right version + # # supported distros are el6, rhel6, f18, f19, opensuse12.2, sles11, centos7 (see http://ceph.com/rpm-firefly/) + ceph_stable_redhat_distro: el7 + + ceph_dev: false # use ceph developement branch + ceph_dev_branch: master # developement branch you would like to use e.g: master, wip-hack + # supported distros are centos6, centos7, fc17, fc18, fc19, fc20, fedora17, fedora18, + # fedora19, fedora20, opensuse12, sles0. (see http://gitbuilder.ceph.com/). + # For rhel, please pay attention to the versions: 'rhel6 3' or 'rhel 4', the fullname is _very_ important. + ceph_dev_redhat_distro: centos7 + + ## Ceph options + # + cephx: true + cephx_require_signatures: true + cephx_cluster_require_signatures: true + cephx_service_require_signatures: false + disable_in_memory_logs: true + + ## Monitor options + # + monitor_interface: eth1 + mon_osd_down_out_interval: 600 + mon_osd_min_down_reporters: 7 # number of OSDs per host + 1 + mon_clock_drift_allowed: .15 + mon_clock_drift_warn_backoff: 30 + mon_osd_full_ratio: .95 + mon_osd_nearfull_ratio: .85 + mon_osd_report_timeout: 300 + + ## OSD options + # + journal_size: 100 + pool_default_pg_num: 128 + pool_default_pgp_num: 128 + pool_default_size: 2 + pool_default_min_size: 1 + cluster_network: 192.168.42.0/24 + public_network: 192.168.42.0/24 + osd_mkfs_type: xfs + osd_mkfs_options_xfs: -f -i size=2048 + osd_mount_options_xfs: noatime + osd_mon_heartbeat_interval: 30 + # CRUSH + pool_default_crush_rule: 0 + osd_crush_update_on_start: "true" + # Object backend + osd_objectstore: filestore + # Performance tuning + filestore_merge_threshold: 40 + filestore_split_multiple: 8 + osd_op_threads: 8 + filestore_op_threads: 8 + filestore_max_sync_interval: 5 + osd_max_scrubs: 1 + # Recovery tuning + osd_recovery_max_active: 5 + osd_max_backfills: 2 + osd_recovery_op_priority: 2 ++osd_recovery_max_chunk: 1048576 ++osd_recovery_threads: 1 + + ## MDS options + # + mds: true # disable mds configuration in ceph.conf + + # Rados Gateway options + # + radosgw: true # referenced in monitor role too. + #radosgw_dns_name: your.subdomain.tld # subdomains used by radosgw. See http://ceph.com/docs/master/radosgw/config/#enabling-subdomain-s3-calls + + ## Testing mode + # enable this mode _only_ when you have a single node + # if you don't want it keep the option commented + #common_single_host_mode: true diff --cc roles/ceph-mds/tasks/main.yml index 1b08d8eb7,26cb5f279..35393ba42 --- a/roles/ceph-mds/tasks/main.yml +++ b/roles/ceph-mds/tasks/main.yml @@@ -1,13 -1,22 +1,22 @@@ --- ## Deploy Ceph metadata server(s) - # - name: Copy MDS bootstrap key - copy: src=fetch/{{ fsid }}/var/lib/ceph/bootstrap-mds/ceph.keyring dest=/var/lib/ceph/bootstrap-mds/ceph.keyring owner=root group=root mode=600 + copy: > - src=fetch/{{ fsid }}/var/lib/ceph/bootstrap-mds/ceph.keyring - dest=/var/lib/ceph/bootstrap-mds/ceph.keyring - owner=root - group=root ++ src=fetch/{{ fsid }}/var/lib/ceph/bootstrap-mds/ceph.keyring ++ dest=/var/lib/ceph/bootstrap-mds/ceph.keyring ++ owner=root ++ group=root + mode=600 when: cephx - name: Create MDS directory - action: file path=/var/lib/ceph/mds/ceph-{{ ansible_hostname }} state=directory owner=root group=root mode=0644 + file: > - path=/var/lib/ceph/mds/ceph-{{ ansible_hostname }} - state=directory - owner=root - group=root ++ path=/var/lib/ceph/mds/ceph-{{ ansible_hostname }} ++ state=directory ++ owner=root ++ group=root + mode=0644 when: cephx - name: Create MDS keyring @@@ -16,8 -25,16 +25,16 @@@ changed_when: False - name: Set MDS key permissions - file: path=/var/lib/ceph/mds/ceph-{{ ansible_hostname }}/keyring mode=0600 owner=root group=root + file: > - path=/var/lib/ceph/mds/ceph-{{ ansible_hostname }}/keyring - mode=0600 - owner=root ++ path=/var/lib/ceph/mds/ceph-{{ ansible_hostname }}/keyring ++ mode=0600 ++ owner=root + group=root when: cephx - name: Start and add that the MDS service to the init sequence - service: name=ceph state=started enabled=yes args=mds - service: > - name=ceph - state=started - enabled=yes ++ service: > ++ name=ceph ++ state=started ++ enabled=yes + args=mds diff --cc roles/ceph-mon/tasks/main.yml index b0b0c3e54,a57746b87..f64c1ef9d --- a/roles/ceph-mon/tasks/main.yml +++ b/roles/ceph-mon/tasks/main.yml @@@ -33,7 -15,10 +15,10 @@@ changed_when: False - name: Copy keys to the ansible server - fetch: src={{ item }} dest=fetch/{{ fsid }}/{{ item }} flat=yes + fetch: > - src={{ item }} - dest=fetch/{{ fsid }}/{{ item }} ++ src={{ item }} ++ dest=fetch/{{ fsid }}/{{ item }} + flat=yes when: cephx with_items: - /etc/ceph/ceph.client.admin.keyring # just in case another application needs it @@@ -42,5 -27,10 +27,10 @@@ - /etc/ceph/keyring.radosgw.gateway - name: Drop in a motd script to report status when logging in - copy: src=precise/92-ceph dest=/etc/update-motd.d/92-ceph owner=root group=root mode=0755 + copy: > - src=precise/92-ceph - dest=/etc/update-motd.d/92-ceph - owner=root - group=root ++ src=precise/92-ceph ++ dest=/etc/update-motd.d/92-ceph ++ owner=root ++ group=root + mode=0755 when: ansible_distribution_release == 'precise' diff --cc roles/ceph-osd/tasks/activate_osds.yml index 000000000,2d937a530..806b2b187 mode 000000,100644..100644 --- a/roles/ceph-osd/tasks/activate_osds.yml +++ b/roles/ceph-osd/tasks/activate_osds.yml @@@ -1,0 -1,38 +1,38 @@@ + --- + # Activate means: + # - mount the volume in a temp location + # - allocate an osd id (if needed) + # - remount in the correct location /var/lib/ceph/osd/$cluster-$id + # - start ceph-osd + # + + # This task is for disk devices only because of the explicit use of the first + # partition. + + - name: Activate OSD(s) when device is a disk + command: | + ceph-disk activate {{ item.2 | regex_replace('^(\/dev\/cciss\/c[0-9]{1}d[0-9]{1})$', '\\1p') }}1 + with_together: + - parted.results + - ispartition.results + - devices + when: item.0.rc == 0 and item.1.rc != 0 + ignore_errors: True + changed_when: False + + # This task is for partitions because we don't explicitly use a partition. + + - name: Activate OSD(s) when device is a partition + command: "ceph-disk activate {{ item.1 }}" + with_together: + - ispartition.results + - devices + when: item.0.rc == 0 + ignore_errors: True + changed_when: False + + - name: Start and add that the OSD service to the init sequence - service: > - name=ceph - state=started ++ service: > ++ name=ceph ++ state=started + enabled=yes diff --cc roles/ceph-osd/tasks/main.yml index 1ff2a2db2,f9288bf6d..e7bc3d413 --- a/roles/ceph-osd/tasks/main.yml +++ b/roles/ceph-osd/tasks/main.yml @@@ -1,6 -1,26 +1,26 @@@ --- ## Deploy Ceph Oject Storage Daemon(s) - # + + - name: Install dependencies + apt: > - pkg=parted ++ pkg=parted + state=present + when: ansible_os_family == 'Debian' + + - name: Install dependencies + yum: > - name=parted ++ name=parted + state=present + when: ansible_os_family == 'RedHat' + + - name: Copy OSD bootstrap key + copy: > - src=fetch/{{ fsid }}/var/lib/ceph/bootstrap-osd/ceph.keyring - dest=/var/lib/ceph/bootstrap-osd/ceph.keyring - owner=root - group=root ++ src=fetch/{{ fsid }}/var/lib/ceph/bootstrap-osd/ceph.keyring ++ dest=/var/lib/ceph/bootstrap-osd/ceph.keyring ++ owner=root ++ group=root + mode=600 + when: cephx - include: journal_collocation.yml when: journal_collocation diff --cc roles/ceph-osd/tasks/osd_directory.yml index 419da1476,3d477d3b0..d01a206f3 --- a/roles/ceph-osd/tasks/osd_directory.yml +++ b/roles/ceph-osd/tasks/osd_directory.yml @@@ -19,7 -6,11 +6,11 @@@ # Declaring more than one directory on the same filesystem will confuse Ceph. - name: Create OSD directories - file: path={{ item }} state=directory owner=root group=root - file: > - path={{ item }} - state=directory - owner=root ++ file: > ++ path={{ item }} ++ state=directory ++ owner=root + group=root with_items: osd_directories # Prepare means @@@ -53,4 -44,7 +44,7 @@@ changed_when: False - name: Start and add that the OSD service to the init sequence - service: name=ceph state=started enabled=yes args=osd + service: > - name=ceph - state=started ++ name=ceph ++ state=started + enabled=yes diff --cc roles/ceph-osd/vars/main.yml index d91813410,020470c69..bd81387d7 --- a/roles/ceph-osd/vars/main.yml +++ b/roles/ceph-osd/vars/main.yml @@@ -1,3 -1,76 +1,76 @@@ --- # You can override default vars defined in defaults/main.yml here, --# but I would advice to use host or group vars instead ++# but I would advice to use host or group vars instead + + ## Ceph options + # + + # ACTIVATE THE FSID VARIABLE FOR NON-VAGRANT DEPLOYMENT + # fsid: + cephx: true + + # Devices to be used as OSDs + # You can pre-provision disks that are not present yet. + # Ansible will just skip them. Newly added disk will be + # automatically configured during the next run. + # + + + # !! WARNING !! + # + # /!\ ENABLE ONLY ONE SCENARIO AT A TIME /!\ + # + # !! WARNING !! + + + # Declare devices + # All the scenarii inherit from the following device declaration + # -devices: ++devices: + - /dev/sdb + - /dev/sdc + - /dev/sdd + - /dev/sde + + + # I. First scenario: journal and osd_data on the same device + # Use 'true' to enable this scenario + # This will collocate both journal and data on the same disk + # creating a partition at the beginning of the device + + journal_collocation: true + + + # II. Second scenario: single journal device for N OSDs + # Use 'true' to enable this scenario + + # deprecated, please use scenario III with a single raw_journal_device + + + # III. Third scenario: N journal devices for N OSDs + # Use 'true' to enable this scenario + # + # In the following example: + # * sdd and sde will get sdb as a journal + # * sdf and sdg will get sdc as a journal + # While starting you have 2 options: + # 1. Pre-allocate all the devices + # 2. Progressively add new devices + + raw_multi_journal: false + raw_journal_devices: + - /dev/sdb + - /dev/sdb + - /dev/sdc + - /dev/sdc + + + # IV. Fourth scenario: use directory instead of disk for OSDs + # Use 'true' to enable this scenario + + osd_directory: false -osd_directories: - - /var/lib/ceph/osd/mydir1 - - /var/lib/ceph/osd/mydir2 - - /var/lib/ceph/osd/mydir3 - - /var/lib/ceph/osd/mydir4 ++osd_directories: ++ - /var/lib/ceph/osd/mydir1 ++ - /var/lib/ceph/osd/mydir2 ++ - /var/lib/ceph/osd/mydir3 ++ - /var/lib/ceph/osd/mydir4 diff --cc roles/ceph-radosgw/tasks/install_redhat.yml index 000000000,511af7b48..9da4ee4e1 mode 000000,100644..100644 --- a/roles/ceph-radosgw/tasks/install_redhat.yml +++ b/roles/ceph-radosgw/tasks/install_redhat.yml @@@ -1,0 -1,71 +1,71 @@@ + --- + ## Deploy RADOS Gateway + # + + - name: Add Ceph extra + template: > - src=ceph-extra.repo - dest=/etc/yum.repos.d - owner=root ++ src=ceph-extra.repo ++ dest=/etc/yum.repos.d ++ owner=root + group=root + + - name: Add special fastcgi repository key + rpm_key: key=http://dag.wieers.com/rpm/packages/RPM-GPG-KEY.dag.txt + + - name: Add special fastcgi repository + command: rpm -ivh http://pkgs.repoforge.org/rpmforge-release/rpmforge-release-0.5.3-1.el6.rf.x86_64.rpm + + - name: "Install Apache, fastcgi, and Rados Gateway" + yum: > + name={{ item }} + state=present + with_items: + - httpd + - mod_fastcgi + - mod_fcgid + - ceph-radosgw + + ## Prepare Apache + # + + - name: Install Rados Gateway vhost + template: > + src=rgw.conf + dest=/etc/httpd/conf.d/rgw.conf + owner=root + group=root + + ## Prepare RGW + # + + - name: Create RGW directory + file: > + path=/var/lib/ceph/radosgw/{{ ansible_fqdn }} + state=directory + owner=root + group=root + mode=0644 + + - name: Install s3gw.fcgi script + copy: > + src=s3gw.fcgi + dest=/var/www/s3gw.fcgi + mode=0555 + owner=root + group=root + + - name: Disable default site + shell: sed -i "s/^[^+#]/#/g" /etc/httpd/conf.d/welcome.conf + changed_when: False + notify: + - restart apache2 + + ## If we don't perform this check Ansible will start multiple instance of radosgw + - name: Check if RGW is started + command: /etc/init.d/ceph-radosgw status + register: rgwstatus + ignore_errors: True + + - name: Start RGW + command: /etc/init.d/ceph-radosgw start + when: rgwstatus.rc != 0 diff --cc roles/ceph-radosgw/tasks/main.yml index f23dc6097,9463a8250..3f993725d --- a/roles/ceph-radosgw/tasks/main.yml +++ b/roles/ceph-radosgw/tasks/main.yml @@@ -2,8 -10,8 +10,8 @@@ ## Check OS family # --- include: RedHat.yml ++- include: install_redhat.yml when: ansible_os_family == 'RedHat' --- include: Debian.yml ++- include: install_debian.yml when: ansible_os_family == 'Debian' diff --cc roles/haproxy/tasks/precise.yml index d1e2f0518,b9c409fb1..428b054e9 --- a/roles/haproxy/tasks/precise.yml +++ b/roles/haproxy/tasks/precise.yml @@@ -1,15 -1,32 +1,32 @@@ --- - - apt_repository: repo=ppa:vbernat/haproxy-1.5 state=present -- name: add repository ++- name: Add repository + apt_repository: > - repo=ppa:vbernat/haproxy-1.5 ++ repo=ppa:vbernat/haproxy-1.5 + state=present - - apt: name={{ item }} state=present -- name: install haproxy ++- name: Install haproxy + apt: > + name={{ item }} + state=present with_items: - haproxy - socat - - copy: src=precise/haproxy dest=/etc/default/haproxy -- name: copy default configuration ++- name: Copy default configuration + copy: > + src=precise/haproxy + dest=/etc/default/haproxy notify: restart haproxy - - template: src=precise/haproxy.cfg dest=/etc/haproxy/haproxy.cfg backup=yes -- name: create configuration ++- name: Create configuration + template: > + src=precise/haproxy.cfg + dest=/etc/haproxy/haproxy.cfg + backup=yes notify: restart haproxy - - service: name=haproxy state=started enabled=yes -- name: start and enable haproxy ++- name: Start and enable haproxy + service: > + name=haproxy + state=started + enabled=yes diff --cc rolling_update.yml index 56b63573e,798def97f..3c606271c --- a/rolling_update.yml +++ b/rolling_update.yml @@@ -47,4 -50,7 +50,7 @@@ - ceph-mds post_tasks: - name: restart metadata server(s) - service: name=ceph state=restarted args=mds + service: > - name=ceph - state=restarted ++ name=ceph ++ state=restarted + args=mds