From b2b18db9caef135c0ecd87de3c557d5495336edd Mon Sep 17 00:00:00 2001 From: Guilhem Lettron Date: Wed, 5 Feb 2014 23:36:31 +0100 Subject: [PATCH] Instead of doing some crappy things, we are using the yum cookbook. We do it this way. It avoids complexity and breaking tests. --- .kitchen.yml | 41 ++++++ .rubocop.yml | 16 +++ .travis.yml | 8 ++ Berksfile | 3 + Gemfile | 14 ++ README.md | 127 ++++++++++++++++++ Strainerfile | 5 + attributes/cephfs.rb | 1 + attributes/conf.rb | 2 + attributes/default.rb | 2 + attributes/mds.rb | 6 + attributes/mon.rb | 7 + attributes/osd.rb | 7 + attributes/radosgw.rb | 31 +++++ attributes/repo.rb | 47 +++++++ libraries/default.rb | 103 +++++++++++++++ metadata.rb | 12 ++ providers/client.rb | 103 +++++++++++++++ recipes/apt.rb | 46 +++++++ recipes/cephfs.rb | 44 +++++++ recipes/conf.rb | 20 +++ recipes/default.rb | 53 ++++++++ recipes/mds.rb | 72 +++++++++++ recipes/mon.rb | 118 +++++++++++++++++ recipes/osd.rb | 163 ++++++++++++++++++++++++ recipes/radosgw.rb | 76 +++++++++++ recipes/radosgw_apache2.rb | 99 ++++++++++++++ recipes/repo.rb | 8 ++ recipes/rpm.rb | 27 ++++ recipes/tgt.rb | 49 +++++++ resources/client.rb | 19 +++ roles/ceph-mds.json | 8 ++ roles/ceph-mds.rb | 6 + roles/ceph-mon.json | 8 ++ roles/ceph-mon.rb | 6 + roles/ceph-osd.json | 8 ++ roles/ceph-osd.rb | 6 + roles/ceph-radosgw.json | 8 ++ roles/ceph-radosgw.rb | 6 + roles/ceph-tgt.rb | 6 + templates/default/ceph.conf.erb | 50 ++++++++ templates/default/mods/fastcgi.conf.erb | 5 + templates/default/rgw.conf.erb | 39 ++++++ templates/default/s3gw.fcgi.erb | 2 + 44 files changed, 1487 insertions(+) create mode 100644 .kitchen.yml create mode 100644 .rubocop.yml create mode 100644 .travis.yml create mode 100644 Berksfile create mode 100644 Gemfile create mode 100644 README.md create mode 100644 Strainerfile create mode 100644 attributes/cephfs.rb create mode 100644 attributes/conf.rb create mode 100644 attributes/default.rb create mode 100644 attributes/mds.rb create mode 100644 attributes/mon.rb create mode 100644 attributes/osd.rb create mode 100644 attributes/radosgw.rb create mode 100644 attributes/repo.rb create mode 100644 libraries/default.rb create mode 100644 metadata.rb create mode 100644 providers/client.rb create mode 100644 recipes/apt.rb create mode 100644 recipes/cephfs.rb create mode 100644 recipes/conf.rb create mode 100644 recipes/default.rb create mode 100644 recipes/mds.rb create mode 100644 recipes/mon.rb create mode 100644 recipes/osd.rb create mode 100644 recipes/radosgw.rb create mode 100644 recipes/radosgw_apache2.rb create mode 100644 recipes/repo.rb create mode 100644 recipes/rpm.rb create mode 100644 recipes/tgt.rb create mode 100644 resources/client.rb create mode 100644 roles/ceph-mds.json create mode 100644 roles/ceph-mds.rb create mode 100644 roles/ceph-mon.json create mode 100644 roles/ceph-mon.rb create mode 100644 roles/ceph-osd.json create mode 100644 roles/ceph-osd.rb create mode 100644 roles/ceph-radosgw.json create mode 100644 roles/ceph-radosgw.rb create mode 100644 roles/ceph-tgt.rb create mode 100644 templates/default/ceph.conf.erb create mode 100644 templates/default/mods/fastcgi.conf.erb create mode 100644 templates/default/rgw.conf.erb create mode 100644 templates/default/s3gw.fcgi.erb diff --git a/.kitchen.yml b/.kitchen.yml new file mode 100644 index 0000000..5963e97 --- /dev/null +++ b/.kitchen.yml @@ -0,0 +1,41 @@ +--- +driver_plugin: vagrant +driver_config: + require_chef_omnibus: true + +platforms: +- name: ubuntu-10.04 +- name: ubuntu-12.04 +- name: centos-6.4 +- name: centos-5.9 + +provisioner: + name: chef_zero + +suites: +- name: default + run_list: + - "recipe[ceph::repo]" + - "recipe[ceph]" + attributes: &defaults + ceph: + config: + fsid: ae3f1d03-bacd-4a90-b869-1a4fabb107f2 + mon_initial_members: + - "127.0.0.1" +- name: osd + run_list: + - "role[ceph-osd]" + attributes: *defaults +- name: mon + run_list: + - "role[ceph-mon]" + attributes: *defaults +- name: mds + run_list: + - "role[ceph-mds]" + attributes: *defaults +- name: radosgw + run_list: + - "role[ceph-radosgw]" + attributes: *defaults diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 0000000..1ebbd9a --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,16 @@ +AllCops: + Excludes: + - vendor/** + +AlignParameters: + Enabled: false +Encoding: + Enabled: false +HashSyntax: + Enabled: false +StringLiterals: + Enabled: false +LineLength: + Enabled: false +MethodLength: + Max: 30 diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..67dc113 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,8 @@ +rvm: + - 1.9.3 + - 2.0.0 +before_script: + - bundle exec berks install +bundler_args: --without integration +script: + - bundle exec strainer test --except kitchen diff --git a/Berksfile b/Berksfile new file mode 100644 index 0000000..c4bb297 --- /dev/null +++ b/Berksfile @@ -0,0 +1,3 @@ +site :opscode + +metadata diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..1211303 --- /dev/null +++ b/Gemfile @@ -0,0 +1,14 @@ +source "https://rubygems.org" + +gem "chef", "~> 11.4.4" +gem "json", "<= 1.7.7" # chef 11 dependency +gem "berkshelf", "~> 2.0.10" +gem "chefspec", "~> 3.0.2" +gem "foodcritic", "~> 3.0.3" +gem "strainer" +gem "rubocop" + +group :integration do + gem "test-kitchen", "~> 1.1.1" + gem "kitchen-vagrant", "~> 0.14" +end diff --git a/README.md b/README.md new file mode 100644 index 0000000..361b18e --- /dev/null +++ b/README.md @@ -0,0 +1,127 @@ +Chef cookbook for deploying the Ceph storage system +=================================================== + +Note: "knife cookbook upload" needs this directory to be named "ceph". +Please clone the repository as + + git clone https://github.com/ceph/ceph-cookbooks.git ceph + +(we cannot name this repository ceph.git, as that is the main project +itself) + + +DESCRIPTION +=========== + +Installs and configures Ceph, a distributed network storage and filesystem designed to provide excellent performance, reliability, and scalability. + +The current version is focused towards deploying Monitors and OSD on Ubuntu. + +For documentation on how to use this cookbook, refer to the [USAGE](#USAGE) section. + +Work in progress: + +* RadosGW +* MDS +* Other Distro (Debian, RHEL/CentOS, FC) + +REQUIREMENTS +============ + +Platform +-------- + +Tested as working: + +* Ubuntu Precise (12.04) + +Cookbooks +--------- + +The ceph cookbook requires the following cookbooks from Opscode: + +https://github.com/opscode/cookbooks + +* apt +* apache2 + + +ATTRIBUTES +========== + +Ceph Rados Gateway +------------------ + +* node[:ceph][:radosgw][:api_fqdn] +* node[:ceph][:radosgw][:admin_email] +* node[:ceph][:radosgw][:rgw_addr] + +TEMPLATES +========= + + + +USAGE +===== + +Ceph cluster design is beyond the scope of this README, please turn to the +public wiki, mailing lists, visit our IRC channel, or contact Inktank: + +http://ceph.com/docs/master +http://ceph.com/resources/mailing-list-irc/ +http://www.inktank.com/ + + +Ceph Monitor +------------ + +Ceph monitor nodes should use the ceph-mon role. + +Includes: + +* ceph::default +* ceph::conf + +Ceph Metadata Server +-------------------- + +Ceph metadata server nodes should use the ceph-mds role. + +Includes: + +* ceph::default + +Ceph OSD +-------- + +Ceph OSD nodes should use the ceph-osd role + +Includes: + +* ceph::default +* ceph::conf + +Ceph Rados Gateway +------------------ + +Ceph Rados Gateway nodes should use the ceph-radosgw role + + +LICENSE AND AUTHORS +=================== + +* Author: Kyle Bader + +* Copyright 2013, DreamHost Web Hosting and Inktank Storage Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Strainerfile b/Strainerfile new file mode 100644 index 0000000..2d6d8c4 --- /dev/null +++ b/Strainerfile @@ -0,0 +1,5 @@ +# Strainerfile +rubocop: bundle exec rubocop $SANDBOX/$COOKBOOK +knife test: bundle exec knife cookbook test $COOKBOOK +foodcritic: bundle exec foodcritic -f any -t ~FC003 -t ~FC023 $SANDBOX/$COOKBOOK +#chefspec: bundle exec rspec $SANDBOX/$COOKBOOK/spec diff --git a/attributes/cephfs.rb b/attributes/cephfs.rb new file mode 100644 index 0000000..1147528 --- /dev/null +++ b/attributes/cephfs.rb @@ -0,0 +1 @@ +default["ceph"]["cephfs_mount"] = "/ceph" diff --git a/attributes/conf.rb b/attributes/conf.rb new file mode 100644 index 0000000..27bef9e --- /dev/null +++ b/attributes/conf.rb @@ -0,0 +1,2 @@ +default["ceph"]["config"] = {} +default["ceph"]["config-sections"] = {} diff --git a/attributes/default.rb b/attributes/default.rb new file mode 100644 index 0000000..8d9a48e --- /dev/null +++ b/attributes/default.rb @@ -0,0 +1,2 @@ +default['ceph']['install_debug'] = true +default['ceph']['encrypted_data_bags'] = false diff --git a/attributes/mds.rb b/attributes/mds.rb new file mode 100644 index 0000000..be0c008 --- /dev/null +++ b/attributes/mds.rb @@ -0,0 +1,6 @@ +case node['platform'] +when 'ubuntu' + default["ceph"]["mds"]["init_style"] = "upstart" +else + default["ceph"]["mds"]["init_style"] = "sysvinit" +end diff --git a/attributes/mon.rb b/attributes/mon.rb new file mode 100644 index 0000000..629ccbd --- /dev/null +++ b/attributes/mon.rb @@ -0,0 +1,7 @@ +case node['platform'] +when 'ubuntu' + default["ceph"]["mon"]["init_style"] = "upstart" +else + default["ceph"]["mon"]["init_style"] = "sysvinit" +end +default["ceph"]["mon"]["secret_file"] = "/etc/chef/secrets/ceph_mon" diff --git a/attributes/osd.rb b/attributes/osd.rb new file mode 100644 index 0000000..f331963 --- /dev/null +++ b/attributes/osd.rb @@ -0,0 +1,7 @@ +case node['platform'] +when 'ubuntu' + default["ceph"]["osd"]["init_style"] = "upstart" +else + default["ceph"]["osd"]["init_style"] = "sysvinit" +end +default["ceph"]["osd"]["secret_file"] = "/etc/chef/secrets/ceph_osd" diff --git a/attributes/radosgw.rb b/attributes/radosgw.rb new file mode 100644 index 0000000..5fb3171 --- /dev/null +++ b/attributes/radosgw.rb @@ -0,0 +1,31 @@ +# +# Cookbook Name:: ceph +# Attributes:: radosgw +# +# Copyright 2011, DreamHost Web Hosting +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +default["ceph"]["radosgw"]["api_fqdn"] = "localhost" +default["ceph"]["radosgw"]["admin_email"] = "admin@example.com" +default["ceph"]["radosgw"]["rgw_addr"] = "*:80" +default["ceph"]["radosgw"]["rgw_port"] = false +default["ceph"]["radosgw"]["webserver_companion"] = "apache2" # can be false +default['ceph']["radosgw"]['use_apache_fork'] = true +case node['platform'] +when 'ubuntu' + default["ceph"]["radosgw"]["init_style"] = "upstart" +else + default["ceph"]["radosgw"]["init_style"] = "sysvinit" +end diff --git a/attributes/repo.rb b/attributes/repo.rb new file mode 100644 index 0000000..b58fb12 --- /dev/null +++ b/attributes/repo.rb @@ -0,0 +1,47 @@ +default['ceph']['branch'] = "stable" # Can be stable, testing or dev. +# Major release version to install or gitbuilder branch +default['ceph']['version'] = "dumpling" +default['ceph']['el_add_epel'] = true +default['ceph']['repo_url'] = "http://ceph.com" +default['ceph']['extras_repo_url'] = "http://ceph.com/packages/ceph-extras" + +case node['platform_family'] +when "debian" + # Debian/Ubuntu default repositories + default['ceph']['debian']['stable']['repository'] = "#{node['ceph']['repo_url']}/debian-#{node['ceph']['version']}/" + default['ceph']['debian']['stable']['repository_key'] = "https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc" + default['ceph']['debian']['testing']['repository'] = "#{node['ceph']['repo_url']}/debian-testing/" + default['ceph']['debian']['testing']['repository_key'] = "https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc" + default['ceph']['debian']['dev']['repository'] = "http://gitbuilder.ceph.com/ceph-deb-#{node['lsb']['codename']}-x86_64-basic/ref/#{node['ceph']['version']}" + default['ceph']['debian']['dev']['repository_key'] = "https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/autobuild.asc" + default['ceph']['debian']['extras']['repository'] = "#{node['ceph']['extras_repo_url']}/debian/" + default['ceph']['debian']['extras']['repository_key'] = "https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc" +when "rhel" + # Redhat/CentOS default repositories + default['ceph']['rhel']['stable']['repository'] = "#{node['ceph']['repo_url']}/rpm-#{node['ceph']['version']}/el6/noarch/ceph-release-1-0.el6.noarch.rpm" + default['ceph']['rhel']['testing']['repository'] = "#{node['ceph']['repo_url']}/rpm-testing/el6/x86_64/ceph-release-1-0.el6.noarch.rpm" + default['ceph']['rhel']['dev']['repository'] = "http://gitbuilder.ceph.com/ceph-rpm-centos6-x86_64-basic/ref/#{node['ceph']['version']}/x86_64/" + default['ceph']['rhel']['dev']['repository_key'] = "https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/autobuild.asc" + default['ceph']['rhel']['extras']['repository'] = "#{node['ceph']['extras_repo_url']}/rpm/rhel6/x86_64/" + default['ceph']['rhel']['extras']['repository_key'] = "https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/autobuild.asc" +when "fedora" + # Fedora default repositories + default['ceph']['fedora']['stable']['repository'] = "#{node['ceph']['repo_url']}/rpm-#{node['ceph']['version']}/fc#{node['platform_version']}/x86_64/ceph-release-1-0.fc#{node['platform_version']}.noarch.rpm" + default['ceph']['fedora']['testing']['repository'] = "#{node['ceph']['repo_url']}/rpm-testing/fc#{node['platform_version']}/x86_64/ceph-release-1-0.fc#{node['platform_version']}.noarch.rpm" + default['ceph']['fedora']['dev']['repository'] = "http://gitbuilder.ceph.com/ceph-rpm-fc#{node['platform_version']}-x86_64-basic/ref/#{node['ceph']['version']}/RPMS/x86_64/" + default['ceph']['fedora']['dev']['repository_key'] = "https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/autobuild.asc" + default['ceph']['fedora']['extras']['repository'] = "#{node['ceph']['extras_repo_url']}/rpm/fedora#{node['platform_version']}/x86_64/" + default['ceph']['fedora']['extras']['repository_key'] = "https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/autobuild.asc" +when "suse" + # (Open)SuSE default repositories + # Chef doesn't make a difference between suse and opensuse + suse = %x[ head -n1 /etc/SuSE-release| awk '{print $1}' ].chomp.downcase # can be suse or opensuse + suse = "sles" if suse == "suse" + suse_version = suse << %x[ grep VERSION /etc/SuSE-release | awk -F'= ' '{print $2}' ].chomp + + default['ceph']['suse']['stable']['repository'] = "#{node['ceph']['repo_url']}/rpm-#{node['ceph']['version']}/#{suse_version}/x86_64/ceph-release-1-0.#{suse_version}.noarch.rpm" + default['ceph']['suse']['testing']['repository'] = "#{node['ceph']['repo_url']}/rpm-testing/#{suse_version}/x86_64/ceph-release-1-0.#{suse_version}.noarch.rpm" + default['ceph']['suse']['extras']['repository'] = "#{node['ceph']['extras_repo_url']}/rpm/#{suse_version}/x86_64/" +else + fail "#{node['platform_family']} is not supported" +end diff --git a/libraries/default.rb b/libraries/default.rb new file mode 100644 index 0000000..6312451 --- /dev/null +++ b/libraries/default.rb @@ -0,0 +1,103 @@ +require 'ipaddr' +require 'json' + +def crowbar? + !defined?(Chef::Recipe::Barclamp).nil? +end + +def get_mon_nodes(extra_search = nil) + if crowbar? + mon_roles = search(:role, 'name:crowbar-* AND run_list:role\[ceph-mon\]') + unless mon_roles.empty? + search_string = mon_roles.map { |role_object| "roles:" + role_object.name }.join(' OR ') + search_string = "(#{search_string}) AND ceph_config_environment:#{node['ceph']['config']['environment']}" + end + else + search_string = "ceph_is_mon:true AND chef_environment:#{node.chef_environment}" + end + + unless extra_search.nil? + search_string = "(#{search_string}) AND (#{extra_search})" + end + search(:node, search_string) +end + +# If public_network is specified +# we need to search for the monitor IP +# in the node environment. +# 1. We look if the network is IPv6 or IPv4 +# 2. We look for a route matching the network +# 3. We grab the IP and return it with the port +def find_node_ip_in_network(network, nodeish = nil) + nodeish = node unless nodeish + net = IPAddr.new(network) + nodeish["network"]["interfaces"].each do |iface, addrs| + addrs["addresses"].each do |ip, params| + if params['family'].eql?("inet6") && net.include?(ip) + return "[#{ip}]:6789" + elsif params['family'].eql?("inet") && net.include?(ip) + return "#{ip}:6789" + end + end + end + nil +end + +def mon_addresses + mon_ips = [] + + if File.exists?("/var/run/ceph/ceph-mon.#{node['hostname']}.asok") + mon_ips = quorum_members_ips + else + mons = [] + # make sure if this node runs ceph-mon, it's always included even if + # search is laggy; put it first in the hopes that clients will talk + # primarily to local node + mons << node if node['ceph']['is_mon'] + + mons += get_mon_nodes + if crowbar? + mon_ips = mons.map { |node| Chef::Recipe::Barclamp::Inventory.get_network_by_type(node, "admin").address } + else + if node['ceph']['config']['global'] && node['ceph']['config']['global']['public network'] + mon_ips = mons.map { |nodeish| find_node_ip_in_network(node['ceph']['config']['global']['public network'], nodeish) } + else + mon_ips = mons.map { |node| node['ipaddress'] + ":6789" } + end + end + end + mon_ips.reject { |m| m.nil? }.uniq +end + +def quorum_members_ips + mon_ips = [] + cmd = Mixlib::ShellOut.new("ceph --admin-daemon /var/run/ceph/ceph-mon.#{node['hostname']}.asok mon_status") + cmd.run_command + cmd.error! + + mons = JSON.parse(cmd.stdout)['monmap']['mons'] + mons.each do |k| + mon_ips.push(k['addr'][0..-3]) + end + mon_ips +end + +QUORUM_STATES = %w(leader, peon) +def have_quorum? + # "ceph auth get-or-create-key" would hang if the monitor wasn't + # in quorum yet, which is highly likely on the first run. This + # helper lets us delay the key generation into the next + # chef-client run, instead of hanging. + # + # Also, as the UNIX domain socket connection has no timeout logic + # in the ceph tool, this exits immediately if the ceph-mon is not + # running for any reason; trying to connect via TCP/IP would wait + # for a relatively long timeout. + + cmd = Mixlib::ShellOut.new("ceph --admin-daemon /var/run/ceph/ceph-mon.#{node['hostname']}.asok mon_status") + cmd.run_command + cmd.error! + + state = JSON.parse(cmd.stdout)['state'] + QUORUM_STATES.include?(state) +end diff --git a/metadata.rb b/metadata.rb new file mode 100644 index 0000000..b5ea8b2 --- /dev/null +++ b/metadata.rb @@ -0,0 +1,12 @@ +name "ceph" +maintainer "Kyle Bader" +maintainer_email "kyle.bader@dreamhost.com" +license "Apache 2.0" +description "Installs/Configures the Ceph distributed filesystem" +long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) +version "0.1.0" + +depends "apache2", ">= 1.1.12" +depends "apt" +depends "yum", ">= 3.0" +depends "yum-epel" diff --git a/providers/client.rb b/providers/client.rb new file mode 100644 index 0000000..d01292a --- /dev/null +++ b/providers/client.rb @@ -0,0 +1,103 @@ +def whyrun_supported? + true +end + +action :add do + filename = @current_resource.filename + keyname = @current_resource.keyname + caps = @new_resource.caps.map { |k, v| "#{k} '#{v}'" }.join(' ') + if @current_resource.exists + Chef::Log.info "#{ @new_resource} already exists - nothing to do" + else + if @current_resource.caps != @new_resource.caps + converge_by("create ceph auth key #{keyname}") do + auth_set_key(keyname, caps) unless @current_resource.exists + end + end + if @current_resource.as_keyring + get_new_content = method(:get_new_key_file) + else + get_new_content = method(:get_new_key) + end + if get_saved_key_file(@current_resource.filename) != get_new_content.call(keyname) + converge_by("save ceph auth key to #{filename}") do + file filename do + content lazy { get_new_content.call(keyname) } + owner "root" + group "root" + mode "640" + end + end + end + end +end + +def load_current_resource + @current_resource = Chef::Resource::CephClient.new(@new_resource.name) + @current_resource.name(@new_resource.name) + @current_resource.as_keyring(@new_resource.as_keyring) + @current_resource.keyname(@new_resource.keyname || "client.#{current_resource.name}.#{node['hostname']}") + @current_resource.caps(get_caps(@current_resource.keyname)) + if @current_resource.as_keyring + get_new_content = method(:get_new_key_file) + @current_resource.filename(@new_resource.filename || "/etc/ceph/ceph.client.#{current_resource.name}.#{node['hostname']}.keyring") + else + get_new_content = method(:get_new_key) + @current_resource.filename(@new_resource.filename || "/etc/ceph/ceph.client.#{current_resource.name}.#{node['hostname']}.secret") + end + if @current_resource.caps == @new_resource.caps && + get_saved_key_file(@current_resource.filename) == get_new_content.call(@current_resource.keyname) + @current_resource.exists = true + end +end + +def get_new_key(keyname) + cmd = "ceph auth print_key #{keyname}" + key = Mixlib::ShellOut.new(cmd).run_command.stdout + key +end + +def get_new_key_file(keyname) + cmd = "ceph auth print_key #{keyname}" + key = Mixlib::ShellOut.new(cmd).run_command.stdout + "[#{keyname}]\n\tkey = #{key}\n" +end + +def get_saved_key_file(filename) + ::IO.read(filename) rescue "" +end + +def get_caps(keyname) + caps = {} + cmd = "ceph auth get #{keyname}" + output = Mixlib::ShellOut.new(cmd).run_command.stdout + output.scan(/caps\s*(\S+)\s*=\s*"([^"]*)"/) {|k, v| + caps[k] = v + } + caps +end + +def auth_set_key(keyname, caps) + # find the monitor secret + mon_secret = "" + mons = get_mon_nodes + if !mons.empty? + mon_secret = mons[0]["ceph"]["monitor-secret"] + elsif mons.empty? && node["ceph"]["monitor-secret"] + mon_secret = node["ceph"]["monitor-secret"] + else + Chef::Log.warn("No monitor secret found") + end + # try to add the key + cmd = "ceph auth get-or-create #{keyname} #{caps} --name mon. --key='#{mon_secret}'" + get_or_create = Mixlib::ShellOut.new(cmd) + get_or_create.run_command + if get_or_create.stderr.scan(/EINVAL.*but cap.*does not match/) + Chef::Log.info("Deleting old key with incorrect caps") + # delete an old key if it exists and is wrong + Mixlib::ShellOut.new("ceph auth del #{keyname}").run_command + # try to create again + get_or_create.run_command + end + get_or_create.error! +end diff --git a/recipes/apt.rb b/recipes/apt.rb new file mode 100644 index 0000000..3704e9a --- /dev/null +++ b/recipes/apt.rb @@ -0,0 +1,46 @@ + +include_recipe "apt" + +branch = node['ceph']['branch'] + +apt_repository "ceph-#{branch}" do + repo_name "ceph" + uri node['ceph']['debian'][branch]['repository'] + distribution node['lsb']['codename'] == "jessie" ? "sid" : node['lsb']['codename'] + components ['main'] + key node['ceph']['debian'][branch]['repository_key'] +end + +if node['roles'].include?("ceph-tgt") + apt_repository "ceph-extras-#{branch}" do + repo_name "ceph-extras" + uri node['ceph']['debian']['extras']['repository'] + distribution node['lsb']['codename'] == "jessie" ? "sid" : node['lsb']['codename'] + components ['main'] + key node['ceph']['debian']['extras']['repository_key'] + end +end + +if node['roles'].include?("ceph-radosgw") \ + && node["ceph"]["radosgw"]["webserver_companion"] == "apache2" \ + && node["ceph"]["radosgw"]["use_apache_fork"] == true + case node['lsb']['codename'] + when "precise", "oneiric" + apt_repository "ceph-apache2" do + repo_name "ceph-apache2" + uri "http://gitbuilder.ceph.com/apache2-deb-#{node['lsb']['codename']}-x86_64-basic/ref/master" + distribution node['lsb']['codename'] + components ["main"] + key "https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/autobuild.asc" + end + apt_repository "ceph-modfastcgi" do + repo_name "ceph-modfastcgi" + uri "http://gitbuilder.ceph.com/libapache-mod-fastcgi-deb-#{node['lsb']['codename']}-x86_64-basic/ref/master" + distribution node['lsb']['codename'] + components ["main"] + key "https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/autobuild.asc" + end + else + Log.info("Ceph's Apache and Apache FastCGI forks not available for this distribution") + end +end diff --git a/recipes/cephfs.rb b/recipes/cephfs.rb new file mode 100644 index 0000000..6997521 --- /dev/null +++ b/recipes/cephfs.rb @@ -0,0 +1,44 @@ +# +# Author:: Kyle Bader +# Cookbook Name:: ceph +# Recipe:: cephfs +# +# Copyright 2011, DreamHost Web Hosting +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include_recipe "ceph::conf" + +name = "cephfs" +client_name = "cephfs.#{node['hostname']}" +filename = "/etc/ceph/ceph.client.#{client_name}.secret" + +ceph_client name do + filename filename + caps ({ "mon" => "allow r", "osd" => "allow rw", "mds" => "allow" }) + as_keyring false +end + +mons = mon_addresses.join(",") + ":/" + +directory node['ceph']['cephfs_mount'] + +mount node['ceph']['cephfs_mount'] do + fstype "ceph" + device mons + options "_netdev,name=#{client_name},secretfile=#{filename}" + dump 0 + pass 0 + action [:mount, :enable] + not_if { mons.empty? } +end diff --git a/recipes/conf.rb b/recipes/conf.rb new file mode 100644 index 0000000..b1c8f52 --- /dev/null +++ b/recipes/conf.rb @@ -0,0 +1,20 @@ +fail "fsid must be set in config" if node["ceph"]["config"]['fsid'].nil? +fail "mon_initial_members must be set in config" if node["ceph"]["config"]['mon_initial_members'].nil? + +is_rgw = node['roles'].include?('ceph-radosgw') + +directory "/etc/ceph" do + owner "root" + group "root" + mode "0755" + action :create +end + +template '/etc/ceph/ceph.conf' do + source 'ceph.conf.erb' + variables( + :mon_addresses => mon_addresses, + :is_rgw => is_rgw + ) + mode '0644' +end diff --git a/recipes/default.rb b/recipes/default.rb new file mode 100644 index 0000000..389de9a --- /dev/null +++ b/recipes/default.rb @@ -0,0 +1,53 @@ +# +# Author:: Kyle Bader +# Cookbook Name:: ceph +# Recipe:: default +# +# Copyright 2011, DreamHost Web Hosting +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +packages = [] + +case node['platform_family'] +when "debian" + packages = %w{ + ceph + ceph-common + } + + if node['ceph']['install_debug'] + packages_dbg = %w{ + ceph-dbg + ceph-common-dbg + } + packages += packages_dbg + end +when "rhel", "fedora" + packages = %w{ + ceph + } + + if node['ceph']['install_debug'] + packages_dbg = %w{ + ceph-debug + } + packages += packages_dbg + end +end + +packages.each do |pkg| + package pkg do + action :install + end +end diff --git a/recipes/mds.rb b/recipes/mds.rb new file mode 100644 index 0000000..9546dfd --- /dev/null +++ b/recipes/mds.rb @@ -0,0 +1,72 @@ +# +# Author:: Kyle Bader +# Cookbook Name:: ceph +# Recipe:: mds +# +# Copyright 2011, DreamHost Web Hosting +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include_recipe "ceph::default" +include_recipe "ceph::conf" + +cluster = 'ceph' + +directory "/var/lib/ceph/mds/#{cluster}-#{node["hostname"]}" do + owner "root" + group "root" + mode 00755 + recursive true + action :create +end + +ruby_block "create mds client key" do + block do + keyring = %x[ ceph auth get-or-create mds.#{node['hostname']} osd 'allow *' mon 'allow rwx' --name mon. --key='#{node["ceph"]["monitor-secret"]}' ] + keyfile = File.new("/var/lib/ceph/mds/#{cluster}-#{node['hostname']}/keyring", "w") + keyfile.puts(keyring) + keyfile.close + end +end + +file "/var/lib/ceph/mds/#{cluster}-#{node["hostname"]}/done" do + owner "root" + group "root" + mode 00644 +end + +service_type = node["ceph"]["osd"]["init_style"] + +case service_type +when "upstart" + filename = "upstart" +else + filename = "sysvinit" +end +file "/var/lib/ceph/mds/#{cluster}-#{node["hostname"]}/#{filename}" do + owner "root" + group "root" + mode 00644 +end + +service "ceph_mds" do + case service_type + when "upstart" + service_name "ceph-mds-all-starter" + provider Chef::Provider::Service::Upstart + else + service_name "ceph" + end + action [:enable, :start] + supports :restart => true +end diff --git a/recipes/mon.rb b/recipes/mon.rb new file mode 100644 index 0000000..4656002 --- /dev/null +++ b/recipes/mon.rb @@ -0,0 +1,118 @@ +# This recipe creates a monitor cluster +# +# You should never change the mon default path or +# the keyring path. +# Don't change the cluster name either +# Default path for mon data: /var/lib/ceph/mon/$cluster-$id/ +# which will be /var/lib/ceph/mon/ceph-`hostname`/ +# This path is used by upstart. If changed, upstart won't +# start the monitor +# The keyring files are created using the following pattern: +# /etc/ceph/$cluster.client.$name.keyring +# e.g. /etc/ceph/ceph.client.admin.keyring +# The bootstrap-osd and bootstrap-mds keyring are a bit +# different and are created in +# /var/lib/ceph/bootstrap-{osd,mds}/ceph.keyring + +include_recipe "ceph::default" +include_recipe "ceph::conf" + +service_type = node["ceph"]["mon"]["init_style"] + +node.default['ceph']['is_mon'] = true + +directory "/var/run/ceph" do + owner "root" + group "root" + mode 00755 + recursive true + action :create +end + +directory "/var/lib/ceph/mon/ceph-#{node["hostname"]}" do + owner "root" + group "root" + mode 00755 + recursive true + action :create +end + +# TODO: cluster name +cluster = 'ceph' + +unless File.exists?("/var/lib/ceph/mon/ceph-#{node["hostname"]}/done") + keyring = "#{Chef::Config[:file_cache_path]}/#{cluster}-#{node['hostname']}.mon.keyring" + + if node['ceph']['encrypted_data_bags'] + secret = Chef::EncryptedDataBagItem.load_secret(node["ceph"]["mon"]["secret_file"]) + monitor_secret = Chef::EncryptedDataBagItem.load("ceph", "mon", secret)["secret"] + else + monitor_secret = node["ceph"]["monitor-secret"] + end + + execute "format as keyring" do + command "ceph-authtool '#{keyring}' --create-keyring --name=mon. --add-key='#{monitor_secret}' --cap mon 'allow *'" + creates "#{Chef::Config[:file_cache_path]}/#{cluster}-#{node['hostname']}.mon.keyring" + end + + execute 'ceph-mon mkfs' do + command "ceph-mon --mkfs -i #{node['hostname']} --keyring '#{keyring}'" + end + + ruby_block "finalise" do + block do + ["done", service_type].each do |ack| + ::File.open("/var/lib/ceph/mon/ceph-#{node["hostname"]}/#{ack}", "w").close + end + end + end +end + +if service_type == "upstart" + service "ceph-mon" do + provider Chef::Provider::Service::Upstart + action :enable + end + service "ceph-mon-all" do + provider Chef::Provider::Service::Upstart + supports :status => true + action [:enable, :start] + end +end + +service "ceph_mon" do + case service_type + when "upstart" + service_name "ceph-mon-all-starter" + provider Chef::Provider::Service::Upstart + else + service_name "ceph" + end + supports :restart => true, :status => true + action [:enable, :start] +end + +mon_addresses.each do |addr| + execute "peer #{addr}" do + command "ceph --admin-daemon '/var/run/ceph/ceph-mon.#{node['hostname']}.asok' add_bootstrap_peer_hint #{addr}" + ignore_failure true + end +end + +# The key is going to be automatically +# created, +# We store it when it is created +unless node['ceph']['encrypted_data_bags'] + ruby_block "get osd-bootstrap keyring" do + block do + run_out = "" + while run_out.empty? + run_out = Mixlib::ShellOut.new("ceph auth get-key client.bootstrap-osd").run_command.stdout.strip + sleep 2 + end + node.override['ceph']['bootstrap_osd_key'] = run_out + node.save + end + not_if { node['ceph']['bootstrap_osd_key'] } + end +end diff --git a/recipes/osd.rb b/recipes/osd.rb new file mode 100644 index 0000000..1d82923 --- /dev/null +++ b/recipes/osd.rb @@ -0,0 +1,163 @@ +# +# Author:: Kyle Bader +# Cookbook Name:: ceph +# Recipe:: osd +# +# Copyright 2011, DreamHost Web Hosting +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this recipe allows bootstrapping new osds, with help from mon +# Sample environment: +# #knife node edit ceph1 +# "osd_devices": [ +# { +# "device": "/dev/sdc" +# }, +# { +# "device": "/dev/sdd", +# "dmcrypt": true, +# "journal": "/dev/sdd" +# } +# ] + +include_recipe "ceph::default" +include_recipe "ceph::conf" + +package 'gdisk' do + action :upgrade +end + +package 'cryptsetup' do + action :upgrade + only_if { node[:dmcrypt] } +end + +service_type = node["ceph"]["osd"]["init_style"] +mons = node['ceph']['encrypted_data_bags'] ? get_mon_nodes : get_mon_nodes("ceph_bootstrap_osd_key:*") + +if mons.empty? + puts "No ceph-mon found." +else + + directory "/var/lib/ceph/bootstrap-osd" do + owner "root" + group "root" + mode "0755" + end + + # TODO: cluster name + cluster = 'ceph' + + if node['ceph']['encrypted_data_bags'] + secret = Chef::EncryptedDataBagItem.load_secret(node["ceph"]["osd"]["secret_file"]) + osd_secret = Chef::EncryptedDataBagItem.load("ceph", "osd", secret)["secret"] + else + osd_secret = mons[0]["ceph"]["bootstrap_osd_key"] + end + + execute "format as keyring" do + command "ceph-authtool '/var/lib/ceph/bootstrap-osd/#{cluster}.keyring' --create-keyring --name=client.bootstrap-osd --add-key='#{osd_secret}'" + creates "/var/lib/ceph/bootstrap-osd/#{cluster}.keyring" + end + + if crowbar? + ruby_block "select new disks for ceph osd" do + block do + do_trigger = false + node["crowbar"]["disks"].each do |disk, data| + if node["crowbar"]["disks"][disk]["usage"] == "Storage" + puts "Disk: #{disk} should be used for ceph" + + system 'ceph-disk-prepare', \ + "/dev/#{disk}" + fail 'ceph-disk-prepare failed' unless $?.exitstatus == 0 + + do_trigger = true + + node.set["crowbar"]["disks"][disk]["usage"] = "ceph-osd" + node.save + end + end + + if do_trigger + system 'udevadm', \ + "trigger", \ + "--subsystem-match=block", \ + "--action=add" + fail 'udevadm trigger failed' unless $?.exitstatus == 0 + end + + end + end + else + # Calling ceph-disk-prepare is sufficient for deploying an OSD + # After ceph-disk-prepare finishes, the new device will be caught + # by udev which will run ceph-disk-activate on it (udev will map + # the devices if dm-crypt is used). + # IMPORTANT: + # - Always use the default path for OSD (i.e. /var/lib/ceph/ + # osd/$cluster-$id) + # - $cluster should always be ceph + # - The --dmcrypt option will be available starting w/ Cuttlefish + if !node["ceph"]["osd_devices"].nil? + node["ceph"]["osd_devices"].each_with_index do |osd_device, index| + unless osd_device["status"].nil? + Log.info("osd: osd_device #{osd_device} has already been setup.") + next + end + + dmcrypt = osd_device["encrypted"] == true ? "--dmcrypt" : "" + + create_cmd = "ceph-disk-prepare #{dmcrypt} #{osd_device['device']} #{osd_device['journal']}" + if osd_device["type"] == "directory" + directory osd_device["device"] do + owner "root" + group "root" + recursive true + end + create_cmd << " && ceph-disk-activate #{osd_device['device']}" + end + execute "Creating Ceph OSD on #{osd_device['device']}" do + command create_cmd + action :run + notifies :create, "ruby_block[save osd_device status #{index}]" + end + # we add this status to the node env + # so that we can implement recreate + # and/or delete functionalities in the + # future. + ruby_block "save osd_device status #{index}" do + block do + node.normal["ceph"]["osd_devices"][index]["status"] = "deployed" + node.save + end + action :nothing + end + end + service "ceph_osd" do + case service_type + when "upstart" + service_name "ceph-osd-all-starter" + provider Chef::Provider::Service::Upstart + else + service_name "ceph" + end + action [:enable, :start] + supports :restart => true + end + else + Log.info('node["ceph"]["osd_devices"] empty') + end + end +end diff --git a/recipes/radosgw.rb b/recipes/radosgw.rb new file mode 100644 index 0000000..0426dc4 --- /dev/null +++ b/recipes/radosgw.rb @@ -0,0 +1,76 @@ +# +# Author:: Kyle Bader +# Cookbook Name:: ceph +# Recipe:: radosgw +# +# Copyright 2011, DreamHost Web Hosting +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +case node['platform_family'] +when "debian" + packages = %w{ + radosgw + } + + if node['ceph']['install_debug'] + packages_dbg = %w{ + radosgw-dbg + } + packages += packages_dbg + end +when "rhel", "fedora", "suse" + packages = %w{ + ceph-radosgw + } +end + +packages.each do |pkg| + package pkg do + action :upgrade + end +end + +include_recipe "ceph::conf" + +if !::File.exists?("/var/lib/ceph/radosgw/ceph-radosgw.#{node['hostname']}/done") + if node["ceph"]["radosgw"]["webserver_companion"] + include_recipe "ceph::radosgw_#{node["ceph"]["radosgw"]["webserver_companion"]}" + end + + ceph_client "radosgw" do + caps ({ "mon" => "allow rw", "osd" => "allow rwx" }) + end + + file "/var/lib/ceph/radosgw/ceph-radosgw.#{node['hostname']}/done" do + action :create + end + + service "radosgw" do + case node["ceph"]["radosgw"]["init_style"] + when "upstart" + service_name "radosgw-all-starter" + provider Chef::Provider::Service::Upstart + else + if node['platform'] == "debian" + service_name "radosgw" + else + service_name "ceph-radosgw" + end + end + supports :restart => true + action [:enable, :start] + end +else + Log.info("Rados Gateway already deployed") +end diff --git a/recipes/radosgw_apache2.rb b/recipes/radosgw_apache2.rb new file mode 100644 index 0000000..53c1bcb --- /dev/null +++ b/recipes/radosgw_apache2.rb @@ -0,0 +1,99 @@ +# +# Author:: Kyle Bader +# Cookbook Name:: ceph +# Recipe:: radosgw_apache2 +# +# Copyright 2011, DreamHost Web Hosting +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +case node['platform_family'] +when "debian", "suse" + packages = %w{ + apache2 + libapache2-mod-fastcgi + } +when "rhel", "fedora" + packages = %w{ + httpd + mod_fastcgi + } +end + +packages.each do |pkg| + package pkg do + action :upgrade + end +end + +# For EL, delete the current fastcgi configuration +# and set the correct owners for dirs and logs +d_owner = d_group = "root" +if node['platform_family'] == "rhel" + file "#{node['apache']['dir']}/conf.d/fastcgi.conf" do + action :delete + backup false + end + d_owner = d_group = "apache" +end + +%W{ /var/run/ceph + /var/lib/ceph/radosgw/ceph-radosgw.#{node['hostname']} + /var/lib/apache2/ +}.each do |dir| + directory dir do + owner d_owner + group d_group + mode "0755" + recursive true + action :create + end +end + +file "/var/log/ceph/radosgw.log" do + owner d_owner + group d_group + mode "0644" + action :create +end + +include_recipe "apache2" + +apache_module "fastcgi" do + conf true +end + +apache_module "rewrite" do + conf false +end + +web_app "rgw" do + template "rgw.conf.erb" + server_name node['ceph']['radosgw']['api_fqdn'] + admin_email node['ceph']['radosgw']['admin_email'] + ceph_rgw_addr node['ceph']['radosgw']['rgw_addr'] +end + +service "apache2" do + action :restart +end + +template "/var/www/s3gw.fcgi" do + source "s3gw.fcgi.erb" + owner "root" + group "root" + mode "0755" + variables( + :ceph_rgw_client => "client.radosgw.#{node['hostname']}" + ) +end diff --git a/recipes/repo.rb b/recipes/repo.rb new file mode 100644 index 0000000..dbdf4fd --- /dev/null +++ b/recipes/repo.rb @@ -0,0 +1,8 @@ +case node['platform_family'] +when "debian" + include_recipe "ceph::apt" +when "rhel", "suse" + include_recipe "ceph::rpm" +else + fail "not supported" +end diff --git a/recipes/rpm.rb b/recipes/rpm.rb new file mode 100644 index 0000000..1d45679 --- /dev/null +++ b/recipes/rpm.rb @@ -0,0 +1,27 @@ +platform_family = node['platform_family'] + +case platform_family +when "rhel" + if node['ceph']['el_add_epel'] == true + include_recipe "yum_epel" + end +end + +branch = node['ceph']['branch'] +if branch == "dev" && platform_family != "centos" && platform_family != "fedora" + fail "Dev branch for #{platform_family} is not yet supported" +end + +repo = node['ceph'][platform_family][branch]['repository'] + +yum_repository "ceph" do + baseurl repo + gpgkey node['ceph'][platform_family]['dev']['repository_key'] if branch == "dev" +end + +if node['roles'].include?("ceph-tgt") + yum_repository "ceph-extra" do + baseurl node['ceph'][platform_family]['extras']['repository'] + gpgkey node['ceph'][platform_family]['extras']['repository_key'] + end +end diff --git a/recipes/tgt.rb b/recipes/tgt.rb new file mode 100644 index 0000000..0200404 --- /dev/null +++ b/recipes/tgt.rb @@ -0,0 +1,49 @@ +# +# Author:: Kyle Bader +# Cookbook Name:: ceph +# Recipe:: radosgw +# +# Copyright 2011, DreamHost Web Hosting +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +case node['platform_family'] +when "debian" + packages = %w{ + tgt + } +when "rhel", "fedora" + packages = %w{ + scsi-target-utils + } +end + +packages.each do |pkg| + package pkg do + action :upgrade + end +end + +include_recipe "ceph::conf" +# probably needs the key +service "tgt" do + if node['platform'] == "ubuntu" + # The ceph version of tgt does not provide an Upstart script + provider Chef::Provider::Service::Init::Debian + service_name "tgt" + else + service_name "tgt" + end + supports :restart => true + action [:enable, :start] +end diff --git a/resources/client.rb b/resources/client.rb new file mode 100644 index 0000000..0ca1436 --- /dev/null +++ b/resources/client.rb @@ -0,0 +1,19 @@ +actions :add +default_action :add + +attribute :name, :kind_of => String, :name_attribute => true +attribute :caps, :kind_of => Hash, :default => { "mon" => "allow r", "osd" => "allow r" } + +# Whether to store the secret in a keyring file or a plain secret file +attribute :as_keyring, :kind_of => [TrueClass, FalseClass], :default => true + +# what the key should be called in the ceph cluster +# defaults to client.#{name}.#{hostname} +attribute :keyname, :kind_of => String + +# where the key should be saved +# defaults to /etc/ceph/ceph.client.#{name}.#{hostname}.keyring if as_keyring +# defaults to /etc/ceph/ceph.client.#{name}.#{hostname}.secret if not as_keyring +attribute :filename, :kind_of => String + +attr_accessor :exists diff --git a/roles/ceph-mds.json b/roles/ceph-mds.json new file mode 100644 index 0000000..2fe9d3b --- /dev/null +++ b/roles/ceph-mds.json @@ -0,0 +1,8 @@ +{ + "name": "ceph-mds", + "description": "Ceph Metadata Server", + "run_list": [ + "recipe[ceph::repo]", + "recipe[ceph::mds]" + ] +} \ No newline at end of file diff --git a/roles/ceph-mds.rb b/roles/ceph-mds.rb new file mode 100644 index 0000000..d662fd5 --- /dev/null +++ b/roles/ceph-mds.rb @@ -0,0 +1,6 @@ +name "ceph-mds" +description "Ceph Metadata Server" +run_list( + 'recipe[ceph::repo]', + 'recipe[ceph::mds]' +) diff --git a/roles/ceph-mon.json b/roles/ceph-mon.json new file mode 100644 index 0000000..f44b3aa --- /dev/null +++ b/roles/ceph-mon.json @@ -0,0 +1,8 @@ +{ + "name": "ceph-mon", + "description": "Ceph Monitor", + "run_list": [ + "recipe[ceph::repo]", + "recipe[ceph::mon]" + ] +} \ No newline at end of file diff --git a/roles/ceph-mon.rb b/roles/ceph-mon.rb new file mode 100644 index 0000000..b27d453 --- /dev/null +++ b/roles/ceph-mon.rb @@ -0,0 +1,6 @@ +name "ceph-mon" +description "Ceph Monitor" +run_list( + 'recipe[ceph::repo]', + 'recipe[ceph::mon]' +) diff --git a/roles/ceph-osd.json b/roles/ceph-osd.json new file mode 100644 index 0000000..c37b66c --- /dev/null +++ b/roles/ceph-osd.json @@ -0,0 +1,8 @@ +{ + "name": "ceph-osd", + "description": "Ceph Object Storage Device", + "run_list": [ + "recipe[ceph::repo]", + "recipe[ceph::osd]" + ] +} \ No newline at end of file diff --git a/roles/ceph-osd.rb b/roles/ceph-osd.rb new file mode 100644 index 0000000..7532f24 --- /dev/null +++ b/roles/ceph-osd.rb @@ -0,0 +1,6 @@ +name "ceph-osd" +description "Ceph Object Storage Device" +run_list( + 'recipe[ceph::repo]', + 'recipe[ceph::osd]' +) diff --git a/roles/ceph-radosgw.json b/roles/ceph-radosgw.json new file mode 100644 index 0000000..722d26d --- /dev/null +++ b/roles/ceph-radosgw.json @@ -0,0 +1,8 @@ +{ + "name": "ceph-radosgw", + "description": "Ceph RADOS Gateway", + "run_list": [ + "recipe[ceph::repo]", + "recipe[ceph::radosgw]" + ] +} \ No newline at end of file diff --git a/roles/ceph-radosgw.rb b/roles/ceph-radosgw.rb new file mode 100644 index 0000000..baf0ddc --- /dev/null +++ b/roles/ceph-radosgw.rb @@ -0,0 +1,6 @@ +name "ceph-radosgw" +description "Ceph RADOS Gateway" +run_list( + 'recipe[ceph::repo]', + 'recipe[ceph::radosgw]' +) diff --git a/roles/ceph-tgt.rb b/roles/ceph-tgt.rb new file mode 100644 index 0000000..80f6b4a --- /dev/null +++ b/roles/ceph-tgt.rb @@ -0,0 +1,6 @@ +name "ceph-tgt" +description "Ceph iSCSI Target" +run_list( + 'recipe[ceph::repo]', + 'recipe[ceph::tgt]' +) diff --git a/templates/default/ceph.conf.erb b/templates/default/ceph.conf.erb new file mode 100644 index 0000000..cf63ce3 --- /dev/null +++ b/templates/default/ceph.conf.erb @@ -0,0 +1,50 @@ +[global] + fsid = <%= node["ceph"]["config"]["fsid"] %> + mon initial members = <%= node["ceph"]["config"]["mon_initial_members"] %> + mon host = <%= @mon_addresses.join(', ') %> +<% if (! node['ceph']['config']['global'].nil?) -%> + <% node['ceph']['config']['global'].each do |k, v| %> + <%= k %> = <%= v %> + <% end %> +<% end -%> + +<% if (! node['ceph']['config']['osd'].nil?) -%> +[osd] + <% node['ceph']['config']['osd'].each do |k, v| %> + <%= k %> = <%= v %> + <% end %> +<% end -%> + +<% if (! node['ceph']['config']['mon'].nil?) -%> +[mon] + <% node['ceph']['config']['mon'].each do |k, v| %> + <%= k %> = <%= v %> + <% end %> +<% end -%> + +<% if (! node['ceph']['config']['mds'].nil?) -%> +[mds] + <% node['ceph']['config']['mds'].each do |key, value| -%> + <%= key %> = <%= value %> + <% end -%> +<% end -%> + +<% if (@is_rgw) -%> +[client.radosgw.<%= node['hostname'] %>] + host = <%= node['hostname'] %> + rgw socket path = /var/run/ceph/radosgw.<%= node['hostname'] %> + keyring = /etc/ceph/ceph.client.radosgw.<%= node['hostname'] %>.keyring + log file = /var/log/ceph/radosgw.log +<% if (! node['ceph']['config']['rgw'].nil?) -%> + <% node['ceph']['config']['rgw'].each do |k, v| %> + <%= k %> = <%= v %> + <% end %> +<% end -%> +<% end -%> + +<% node['ceph']['config-sections'].each do |name, sect| %> +[<%= name %>] + <% sect.each do |k, v| %> + <%= k %> = <%= v %> + <% end %> +<% end %> diff --git a/templates/default/mods/fastcgi.conf.erb b/templates/default/mods/fastcgi.conf.erb new file mode 100644 index 0000000..a252609 --- /dev/null +++ b/templates/default/mods/fastcgi.conf.erb @@ -0,0 +1,5 @@ + + AddHandler fastcgi-script .fcgi + #FastCgiWrapper /usr/lib/apache2/suexec + FastCgiIpcDir /var/lib/apache2/fastcgi + diff --git a/templates/default/rgw.conf.erb b/templates/default/rgw.conf.erb new file mode 100644 index 0000000..322fb01 --- /dev/null +++ b/templates/default/rgw.conf.erb @@ -0,0 +1,39 @@ +<% if node['ceph']['radosgw']['rgw_port'] -%> +FastCgiExternalServer /var/www/s3gw.fcgi -host 127.0.0.1:<%= node['ceph']['radosgw']['rgw_port'] %> +<% else -%> +FastCgiExternalServer /var/www/s3gw.fcgi -socket /var/run/ceph/radosgw.<%= node['hostname'] %> +<% end -%> + +LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" \"%{Host}i\"" proxy_combined +LogFormat "%{X-Forwarded-For}i %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" \"%{Host}i\"" proxy_debug + +> + ServerName <%= @params[:server_name] %> +<% if node['ceph']['radosgw']['api_aliases'] -%> +<% node['ceph']['radosgw']['api_aliases'].each do |api_alias| -%> + ServerAlias <%= api_alias %> +<% end -%> +<% end -%> + ServerAdmin <%= node["ceph"]["radosgw"]["admin_email"] %> + DocumentRoot /var/www/ + + RewriteEngine On + RewriteRule ^/([a-zA-Z0-9-_.]*)([/]?.*) /s3gw.fcgi?page=$1¶ms=$2&%{QUERY_STRING} [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L] + + + + Options +ExecCGI + AllowOverride All + SetHandler fastcgi-script + Order allow,deny + Allow from all + AuthBasicAuthoritative Off + + + + AllowEncodedSlashes On + + ErrorLog /var/log/<%= node['apache']['package'] %>/error.log + CustomLog /var/log/<%= node['apache']['package'] %>/rgw-access.log proxy_combined + ServerSignature Off + diff --git a/templates/default/s3gw.fcgi.erb b/templates/default/s3gw.fcgi.erb new file mode 100644 index 0000000..c6b684f --- /dev/null +++ b/templates/default/s3gw.fcgi.erb @@ -0,0 +1,2 @@ +#!/bin/sh +exec /usr/bin/radosgw -c /etc/ceph/ceph.conf -n <%= @ceph_rgw_client %> -- 2.47.3