+DESCRIPTION
+===========
+
These are incomplete, use with caution. They have pulled from a working configuration using Debian. They will require work for other distributions. They also assume your package manager (apt-get, etc) are already configured for a ceph repository.
+
+Installs and configures Ceph, a distributed network storage and filesystem
+designed to provide excellent performance, reliability, and scalability.
+
+REQUIREMENTS
+============
+
+Platform
+--------
+
+Tested as working:
+ * Debian Squeeze (6.x)
+
+Cookbooks
+---------
+
+The ceph cookbook requires the following cookbooks from Opscode:
+
+https://github.com/opscode/cookbooks
+
+* apache2
+
+Also required are the following cookbooks New Dream Network (DreamHost.com):
+
+https://github.com/NewDreamNetwork/ceph-cookbooks
+
+
+ATTRIBUTES
+==========
+
+Ceph Rados Gateway
+------------------
+
+* node[:ceph][:radosgw][:api_fqdn]
+* node[:ceph][:radosgw][:admin_email]
+* node[:ceph][:radosgw][:rgw_addr]
+
+TEMPLATES
+=========
+
+
+
+USAGE
+=====
+
+Ceph cluster design is beyond the scope of this README, please turn to the
+public wiki, mailing lists, visit our IRC channel or Ceph Support page:
+
+http://ceph.newdream.net/wiki/
+http://ceph.newdream.net/mailing-lists-and-irc/
+http://www.cephsupport.com/
+
+This diagram helps visualize recipe inheritence of the ceph cookbook recipes:
+
+ <diagram url>
+
+Ceph Monitor
+------------
+
+Ceph monitor nodes should use the ceph::mon recipe.
+
+Includes:
+
+* ceph::default
+* ceph::rados-rest
+
+Ceph Metadata Server
+--------------------
+
+Ceph metadata server nodes should use the ceph::mds recipe.
+
+Includes:
+
+* ceph::default
+
+Ceph OSD
+--------
+
+Ceph OSD nodes should use the ceph::osd recipe
+
+Includes:
+
+* ceph::default
+
+Ceph Rados Gateway
+------------------
+
+Ceph Rados Gateway nodes should use the ceph::radosgw recipe
+
+Includes:
+
+* ceph::rados-rest
+
+LICENSE AND AUTHORS
+===================
+
+* Author: Kyle Bader <kyle.bader@dreamhost.com>
+
+* Copyright 2011, DreamHost Web Hosting
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
--- /dev/null
+default["ceph"]["config"] = {}
+default["ceph"]["config-sections"] = {}
--- /dev/null
+#
+# Cookbook Name:: ceph
+# Attributes:: radosgw
+#
+# Copyright 2011, DreamHost Web Hosting
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+default["ceph"]["radosgw"]["api_fqdn"] = "localhost"
+default["ceph"]["radosgw"]["admin-email"] = "admin@example.com"
+default["ceph"]["radosgw"]["rgw_addr"] = "*:80"
+
+++ /dev/null
-DESCRIPTION
-===========
-
-Installs and configures Ceph, a distributed network storage and filesystem
-designed to provide excellent performance, reliability, and scalability.
-
-REQUIREMENTS
-============
-
-Platform
---------
-
-Tested as working:
- * Debian Squeeze (6.x)
-
-Cookbooks
----------
-
-The ceph cookbook requires the following cookbooks from Opscode:
-
-https://github.com/opscode/cookbooks
-
-* apache2
-
-Also required are the following cookbooks New Dream Network (DreamHost.com):
-
-https://github.com/NewDreamNetwork/ceph-cookbooks
-
-
-ATTRIBUTES
-==========
-
-Ceph Rados Gateway
-------------------
-
-* node[:ceph][:radosgw][:api_fqdn]
-* node[:ceph][:radosgw][:admin_email]
-* node[:ceph][:radosgw][:rgw_addr]
-
-TEMPLATES
-=========
-
-
-
-USAGE
-=====
-
-Ceph cluster design is beyond the scope of this README, please turn to the
-public wiki, mailing lists, visit our IRC channel or Ceph Support page:
-
-http://ceph.newdream.net/wiki/
-http://ceph.newdream.net/mailing-lists-and-irc/
-http://www.cephsupport.com/
-
-This diagram helps visualize recipe inheritence of the ceph cookbook recipes:
-
- <diagram url>
-
-Ceph Monitor
-------------
-
-Ceph monitor nodes should use the ceph::mon recipe.
-
-Includes:
-
-* ceph::default
-* ceph::rados-rest
-
-Ceph Metadata Server
---------------------
-
-Ceph metadata server nodes should use the ceph::mds recipe.
-
-Includes:
-
-* ceph::default
-
-Ceph OSD
---------
-
-Ceph OSD nodes should use the ceph::osd recipe
-
-Includes:
-
-* ceph::default
-
-Ceph Rados Gateway
-------------------
-
-Ceph Rados Gateway nodes should use the ceph::radosgw recipe
-
-Includes:
-
-* ceph::rados-rest
-
-LICENSE AND AUTHORS
-===================
-
-* Author: Kyle Bader <kyle.bader@dreamhost.com>
-
-* Copyright 2011, DreamHost Web Hosting
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
+++ /dev/null
-default["ceph"]["config"] = {}
-default["ceph"]["config-sections"] = {}
+++ /dev/null
-#
-# Cookbook Name:: ceph
-# Attributes:: radosgw
-#
-# Copyright 2011, DreamHost Web Hosting
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-default["ceph"]["radosgw"]["api_fqdn"] = "localhost"
-default["ceph"]["radosgw"]["admin-email"] = "admin@example.com"
-default["ceph"]["radosgw"]["rgw_addr"] = "*:80"
-
+++ /dev/null
-#! /bin/sh
-### BEGIN INIT INFO
-# Provides: single
-# Required-Start: $remote_fs $named $network $time
-# Required-Stop: $remote_fs $named $network $time
-# Default-Start: 2 3 4 5
-# Default-Stop: 0 1 6
-# Short-Description: radosgw
-### END INIT INFO
-
-PATH=/sbin:/bin:/usr/bin
-
-. /lib/lsb/init-functions
-
-CLIENT_NAME=client.rgw-`hostname | sed 's/\..*//g'`
-USER=www-data
-
-
-do_start () {
- su $USER -c "radosgw -c /etc/ceph/ceph.conf -n $CLIENT_NAME >> /var/log/radosgw/radosgw.log 2>&1 &"
-}
-
-case "$1" in
- start)
- do_start
- ;;
- restart|reload|force-reload)
- echo "Error: argument '$1' not supported" >&2
- exit 3
- ;;
- stop)
- # No-op
- ;;
- *)
- echo "Usage: $0 start|stop" >&2
- exit 3
- ;;
-esac
+++ /dev/null
-def is_crowbar?()
- return defined?(Chef::Recipe::Barclamp) != nil
-end
-
-def get_mon_addresses()
- if is_crowbar?
- mon_addresses = search(:node, "role:ceph-mon AND ceph_config_environment:#{node['ceph']['config']['environment']}").map { |node| Chef::Recipe::Barclamp::Inventory.get_network_by_type(node, "admin").address + ":6789" }
- else
- mon_addresses = search(:node, "role:ceph-mon AND chef_environment:#{node.chef_environment}").map { |node| node["ipaddress"] + ":6789" }
- end
- return mon_addresses
-end
-
-QUORUM_STATES = ['leader', 'peon']
-
-def have_quorum?()
- # "ceph auth get-or-create-key" would hang if the monitor wasn't
- # in quorum yet, which is highly likely on the first run. This
- # helper lets us delay the key generation into the next
- # chef-client run, instead of hanging.
- #
- # Also, as the UNIX domain socket connection has no timeout logic
- # in the ceph tool, this exits immediately if the ceph-mon is not
- # running for any reason; trying to connect via TCP/IP would wait
- # for a relatively long timeout.
- mon_status = %x[ceph --admin-daemon /var/run/ceph/ceph-mon.#{node['hostname']}.asok mon_status]
- raise 'getting monitor state failed' unless $?.exitstatus == 0
- state = JSON.parse(mon_status)['state']
- return QUORUM_STATES.include?(state)
-end
+++ /dev/null
-maintainer "Kyle Bader"
-maintainer_email "kyle.bader@dreamhost.com"
-license "Apache 2.0"
-description "Installs/Configures the Ceph distributed filesystem"
-long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
-version "0.0.10"
-depends "apache2"
+++ /dev/null
-release_or_autobuild = node["ceph_branch"].nil? ? "release" : "autobuild"
-
-execute "add autobuild gpg key to apt" do
- command <<-EOH
-wget -q -O- https://raw.github.com/ceph/ceph/master/keys/#{release_or_autobuild}.asc \
-| sudo apt-key add -
- EOH
-end
-
-template '/etc/apt/sources.list.d/ceph.list' do
- owner 'root'
- group 'root'
- mode '0644'
- source 'apt-sources-list.release.erb'
- variables(
- :codename => node[:lsb][:codename],
- :branch => node["ceph_branch"]
- )
-end
-
-execute 'apt-get update'
+++ /dev/null
-# this recipe allows bootstrapping new osds, with help from mon
-
-include_recipe "ceph::osd"
-include_recipe "ceph::conf"
-
-if is_crowbar?
- mons = search(:node, "role:ceph-mon AND ceph_config_environment:#{node['ceph']['config']['environment']} AND ceph_bootstrap_osd_key:*")
-else
- mons = search(:node, "role:ceph-mon AND chef_environment:#{node.chef_environment} AND ceph_bootstrap_osd_key:*")
-end
-
-if mons.length < 1 then
- puts "No ceph-mon found."
-else
-
- directory "/var/lib/ceph/bootstrap-osd" do
- owner "root"
- group "root"
- mode "0755"
- end
-
- # TODO cluster name
- cluster = 'ceph'
-
- file "/var/lib/ceph/bootstrap-osd/#{cluster}.keyring.raw" do
- owner "root"
- group "root"
- mode "0440"
- content mons[0]["ceph_bootstrap_osd_key"]
- end
-
- execute "format as keyring" do
- command <<-EOH
- set -e
- # TODO don't put the key in "ps" output, stdout
- read KEY <'/var/lib/ceph/bootstrap-osd/#{cluster}.keyring.raw'
- ceph-authtool '/var/lib/ceph/bootstrap-osd/#{cluster}.keyring' --create-keyring --name=client.bootstrap-osd --add-key="$KEY"
- rm -f '/var/lib/ceph/bootstrap-osd/#{cluster}.keyring.raw'
- EOH
- end
-end
+++ /dev/null
-raise "fsid must be set in config" if node["ceph"]["config"]['fsid'].nil?
-raise "mon_initial_members must be set in config" if node["ceph"]["config"]['mon_initial_members'].nil?
-
-mon_addresses = get_mon_addresses()
-
-template '/etc/ceph/ceph.conf' do
- source 'ceph.conf.erb'
- variables(
- :mon_addresses => mon_addresses
- )
- mode '0644'
-end
+++ /dev/null
-#
-# Author:: Kyle Bader <kyle.bader@dreamhost.com>
-# Cookbook Name:: ceph
-# Recipe:: default
-#
-# Copyright 2011, DreamHost Web Hosting
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-packages = %w{
- ceph
- ceph-dbg
- ceph-common
- ceph-common-dbg
-}
-
-packages.each do |pkg|
- package pkg do
- action :upgrade
- end
-end
+++ /dev/null
-#
-# Author:: Kyle Bader <kyle.bader@dreamhost.com>
-# Cookbook Name:: ceph
-# Recipe:: mds
-#
-# Copyright 2011, DreamHost Web Hosting
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-include_recipe "ceph::default"
+++ /dev/null
-# this recipe creates a monitor cluster
-
-require 'json'
-
-include_recipe "ceph::default"
-include_recipe "ceph::conf"
-
-if is_crowbar?
- ipaddress = Chef::Recipe::Barclamp::Inventory.get_network_by_type(node, "admin").address
-else
- ipaddress = node['ipaddress']
-end
-
-service "ceph-mon-all-starter" do
- provider Chef::Provider::Service::Upstart
- action [:enable]
-end
-
-# TODO cluster name
-cluster = 'ceph'
-
-execute 'ceph-mon mkfs' do
- command <<-EOH
-set -e
-# TODO chef creates doesn't seem to suppressing re-runs, do it manually
-if [ -e '/var/lib/ceph/mon/ceph-#{node["hostname"]}/done' ]; then
- echo 'ceph-mon mkfs already done, skipping'
- exit 0
-fi
-KR='/var/lib/ceph/tmp/#{cluster}-#{node['hostname']}.mon.keyring'
-# TODO don't put the key in "ps" output, stdout
-ceph-authtool "$KR" --create-keyring --name=mon. --add-key='#{node["ceph"]["monitor-secret"]}' --cap mon 'allow *'
-
-ceph-mon --mkfs -i #{node['hostname']} --keyring "$KR"
-rm -f -- "$KR"
-touch /var/lib/ceph/mon/ceph-#{node['hostname']}/done
-EOH
- # TODO built-in done-ness flag for ceph-mon?
- creates '/var/lib/ceph/mon/ceph-#{node["hostname"]}/done'
- notifies :start, "service[ceph-mon-all-starter]", :immediately
-end
-
-ruby_block "tell ceph-mon about its peers" do
- block do
- mon_addresses = get_mon_addresses()
- mon_addresses.each do |addr|
- system 'ceph', \
- '--admin-daemon', "/var/run/ceph/ceph-mon.#{node['hostname']}.asok", \
- 'add_bootstrap_peer_hint', "#{addr}"
- # ignore errors
- end
- end
-end
-
-ruby_block "create client.admin keyring" do
- block do
- if not ::File.exists?('/etc/ceph/ceph.client.admin.keyring') then
- if not have_quorum? then
- puts 'ceph-mon is not in quorum, skipping bootstrap-osd key generation for this run'
- else
- # TODO --set-uid=0
- key = %x[
- ceph \
- --name mon. \
- --keyring '/var/lib/ceph/mon/#{cluster}-#{node['hostname']}/keyring' \
- auth get-or-create-key client.admin \
- mon 'allow *' \
- osd 'allow *' \
- mds allow
- ]
- raise 'adding or getting admin key failed' unless $?.exitstatus == 0
- # TODO don't put the key in "ps" output, stdout
- system 'ceph-authtool', \
- '/etc/ceph/ceph.client.admin.keyring', \
- '--create-keyring', \
- '--name=client.admin', \
- "--add-key=#{key}"
- raise 'creating admin keyring failed' unless $?.exitstatus == 0
- end
- end
- end
-end
-
-ruby_block "save osd bootstrap key in node attributes" do
- block do
- if node['ceph_bootstrap_osd_key'].nil? then
- if not have_quorum? then
- puts 'ceph-mon is not in quorum, skipping bootstrap-osd key generation for this run'
- else
- key = %x[
- ceph \
- --name mon. \
- --keyring '/var/lib/ceph/mon/#{cluster}-#{node['hostname']}/keyring' \
- auth get-or-create-key client.bootstrap-osd mon \
- "allow command osd create ...; \
- allow command osd crush set ...; \
- allow command auth add * osd allow\\ * mon allow\\ rwx; \
- allow command mon getmap"
- ]
- raise 'adding or getting bootstrap-osd key failed' unless $?.exitstatus == 0
- node.override['ceph_bootstrap_osd_key'] = key
- node.save
- end
- end
- end
-end
+++ /dev/null
-#
-# Author:: Kyle Bader <kyle.bader@dreamhost.com>
-# Cookbook Name:: ceph
-# Recipe:: osd
-#
-# Copyright 2011, DreamHost Web Hosting
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-include_recipe "ceph::default"
+++ /dev/null
-#
-# Author:: Kyle Bader <kyle.bader@dreamhost.com>
-# Cookbook Name:: ceph
-# Recipe:: rados-client
-#
-# Copyright 2011, DreamHost Web Hosting
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-packages = %w{
- radosgw
- radosgw-dbg
-}
-
-packages.each do |pkg|
- package pkg do
- action :upgrade
- end
-end
+++ /dev/null
-#
-# Author:: Kyle Bader <kyle.bader@dreamhost.com>
-# Cookbook Name:: ceph
-# Recipe:: radosgw
-#
-# Copyright 2011, DreamHost Web Hosting
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-include_recipe "ceph::rados-rest"
-
-packages = %w{
- libapache2-mod-fastcgi
-}
-
-include_recipe "apache2"
-
-packages.each do |pkg|
- package pkg do
- action :upgrade
- end
-end
-
-cookbook_file "/etc/init.d/radosgw" do
- source "radosgw"
- mode 0755
- owner "root"
- group "root"
-end
-
-service "radosgw" do
- service_name "radosgw"
- supports :restart => true
- action[:enable,:start]
-end
-
-apache_module "fastcgi" do
- conf true
-end
-
-apache_module "rewrite" do
- conf false
-end
-
-template "/etc/apache2/sites-available/rgw.conf" do
- source "rgw.conf.erb"
- mode 0400
- owner "root"
- group "root"
- variables(
- :ceph_api_fqdn => node['ceph']['radosgw']['api_fqdn'],
- :ceph_admin_email => node['ceph']['radosgw']['admin_email'],
- :ceph_rgw_addr => node['ceph']['radosgw']['rgw_addr']
- )
- if ::File.exists?("#{node['apache']['dir']}/sites-enabled/rgw.conf")
- notifies :restart, "service[apache2]"
- end
-end
-
-apache_site "rgw.conf" do
- enable enable_setting
-end
-
+++ /dev/null
-name "ceph-mds"
-description "Ceph Metadata Server"
-run_list(
- 'recipe[ceph::mds]'
-)
+++ /dev/null
-name "ceph-mon"
-description "Ceph Monitor"
-run_list(
- 'recipe[ceph::mon]'
-)
+++ /dev/null
-name "ceph-osd"
-description "Ceph Object Storage Device"
-run_list(
- 'recipe[ceph::bootstrap_osd]'
-)
+++ /dev/null
-name "ceph-radosgw"
-description "Ceph RADOS Gateway"
-run_list(
- 'recipe[ceph::radosgw]'
-)
+++ /dev/null
-<% if @branch.nil? %>
-deb http://ceph.newdream.net/debian/ <%= @codename %> main
-deb-src http://ceph.newdream.net/debian/ <%= @codename %> main
-<% else %>
-deb http://gitbuilder.ceph.com/ceph-deb-<%= @codename %>-x86_64-basic/ref/<%= @branch %>/ <%= @codename %> main
-deb-src http://gitbuilder.ceph.com/ceph-deb-<%= @codename %>-x86_64-basic/ref/<%= @branch %>/ <%= @codename %> main
-<% end %>
+++ /dev/null
-[global]
- auth supported = cephx
- keyring = /etc/ceph/$cluster.$name.keyring
-
- # this will be used by all clients to discover the mons to
- # talk to; this way we don't need to have separate [mon.foo]
- # sections, and come up with names for the mons just to be
- # able to list their addresses
- mon host = <%= @mon_addresses.join(', ') %>
-
- <% node['ceph']['config'].each do |k, v| %>
- <%= k %> = <%= v %>
- <% end %>
-
-[mon]
- keyring = /var/lib/ceph/mon/$cluster-$id/keyring
-
-[mds]
- keyring = /var/lib/ceph/mds/$cluster-$id/keyring
-
-[osd]
- keyring = /var/lib/ceph/osd/$cluster-$id/keyring
- osd journal size = 1000
- filestore_xattr_use_omap = true
-
-<% node['ceph']['config-sections'].each do |name, sect| %>
-
-[<%= name %>]
-<% sect.each do |k, v| %>
-<%= k %> = <%= v %>
-<% end %>
-<% end %>
+++ /dev/null
-<IfModule mod_fastcgi.c>
- AddHandler fastcgi-script .fcgi
- #FastCgiWrapper /usr/lib/apache2/suexec
- FastCgiIpcDir /var/lib/apache2/fastcgi
-</IfModule>
+++ /dev/null
-FastCgiExternalServer /var/www/dummyradosgw.fcgi -socket /var/run/ceph/radosgw.client.radosgw.<%= node[:hostname] %>
-
-LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" proxy_combined
-LogFormat "%{X-Forwarded-For}i %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" proxy_debug
-
-<VirtualHost <%= @ceph_rgw_addr %>>
- ServerName <%= @ceph_api_fqdn %>
- ServerAdmin <%= @ceph_admin_email %>
- DocumentRoot /var/www/
-
- RewriteEngine On
- RewriteRule ^/(.*) /dummyradosgw.fcgi?params=$1&%{QUERY_STRING} [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L]
-
- <IfModule mod_fastcgi.c>
- <Directory /var/www/>
- Options +ExecCGI
- AllowOverride All
- SetHandler fastcgi-script
- Order allow,deny
- Allow from all
- AuthBasicAuthoritative Off
- </Directory>
- </IfModule>
-
- AllowEncodedSlashes On
-
- ErrorLog /var/log/apache2/error.log
- CustomLog /var/log/apache2/sssrgw-access.log proxy_combined
- ServerSignature Off
-</VirtualHost>
--- /dev/null
+#! /bin/sh
+### BEGIN INIT INFO
+# Provides: single
+# Required-Start: $remote_fs $named $network $time
+# Required-Stop: $remote_fs $named $network $time
+# Default-Start: 2 3 4 5
+# Default-Stop: 0 1 6
+# Short-Description: radosgw
+### END INIT INFO
+
+PATH=/sbin:/bin:/usr/bin
+
+. /lib/lsb/init-functions
+
+CLIENT_NAME=client.rgw-`hostname | sed 's/\..*//g'`
+USER=www-data
+
+
+do_start () {
+ su $USER -c "radosgw -c /etc/ceph/ceph.conf -n $CLIENT_NAME >> /var/log/radosgw/radosgw.log 2>&1 &"
+}
+
+case "$1" in
+ start)
+ do_start
+ ;;
+ restart|reload|force-reload)
+ echo "Error: argument '$1' not supported" >&2
+ exit 3
+ ;;
+ stop)
+ # No-op
+ ;;
+ *)
+ echo "Usage: $0 start|stop" >&2
+ exit 3
+ ;;
+esac
--- /dev/null
+def is_crowbar?()
+ return defined?(Chef::Recipe::Barclamp) != nil
+end
+
+def get_mon_addresses()
+ if is_crowbar?
+ mon_addresses = search(:node, "role:ceph-mon AND ceph_config_environment:#{node['ceph']['config']['environment']}").map { |node| Chef::Recipe::Barclamp::Inventory.get_network_by_type(node, "admin").address + ":6789" }
+ else
+ mon_addresses = search(:node, "role:ceph-mon AND chef_environment:#{node.chef_environment}").map { |node| node["ipaddress"] + ":6789" }
+ end
+ return mon_addresses
+end
+
+QUORUM_STATES = ['leader', 'peon']
+
+def have_quorum?()
+ # "ceph auth get-or-create-key" would hang if the monitor wasn't
+ # in quorum yet, which is highly likely on the first run. This
+ # helper lets us delay the key generation into the next
+ # chef-client run, instead of hanging.
+ #
+ # Also, as the UNIX domain socket connection has no timeout logic
+ # in the ceph tool, this exits immediately if the ceph-mon is not
+ # running for any reason; trying to connect via TCP/IP would wait
+ # for a relatively long timeout.
+ mon_status = %x[ceph --admin-daemon /var/run/ceph/ceph-mon.#{node['hostname']}.asok mon_status]
+ raise 'getting monitor state failed' unless $?.exitstatus == 0
+ state = JSON.parse(mon_status)['state']
+ return QUORUM_STATES.include?(state)
+end
--- /dev/null
+maintainer "Kyle Bader"
+maintainer_email "kyle.bader@dreamhost.com"
+license "Apache 2.0"
+description "Installs/Configures the Ceph distributed filesystem"
+long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
+version "0.0.10"
+depends "apache2"
--- /dev/null
+release_or_autobuild = node["ceph_branch"].nil? ? "release" : "autobuild"
+
+execute "add autobuild gpg key to apt" do
+ command <<-EOH
+wget -q -O- https://raw.github.com/ceph/ceph/master/keys/#{release_or_autobuild}.asc \
+| sudo apt-key add -
+ EOH
+end
+
+template '/etc/apt/sources.list.d/ceph.list' do
+ owner 'root'
+ group 'root'
+ mode '0644'
+ source 'apt-sources-list.release.erb'
+ variables(
+ :codename => node[:lsb][:codename],
+ :branch => node["ceph_branch"]
+ )
+end
+
+execute 'apt-get update'
--- /dev/null
+# this recipe allows bootstrapping new osds, with help from mon
+
+include_recipe "ceph::osd"
+include_recipe "ceph::conf"
+
+if is_crowbar?
+ mons = search(:node, "role:ceph-mon AND ceph_config_environment:#{node['ceph']['config']['environment']} AND ceph_bootstrap_osd_key:*")
+else
+ mons = search(:node, "role:ceph-mon AND chef_environment:#{node.chef_environment} AND ceph_bootstrap_osd_key:*")
+end
+
+if mons.length < 1 then
+ puts "No ceph-mon found."
+else
+
+ directory "/var/lib/ceph/bootstrap-osd" do
+ owner "root"
+ group "root"
+ mode "0755"
+ end
+
+ # TODO cluster name
+ cluster = 'ceph'
+
+ file "/var/lib/ceph/bootstrap-osd/#{cluster}.keyring.raw" do
+ owner "root"
+ group "root"
+ mode "0440"
+ content mons[0]["ceph_bootstrap_osd_key"]
+ end
+
+ execute "format as keyring" do
+ command <<-EOH
+ set -e
+ # TODO don't put the key in "ps" output, stdout
+ read KEY <'/var/lib/ceph/bootstrap-osd/#{cluster}.keyring.raw'
+ ceph-authtool '/var/lib/ceph/bootstrap-osd/#{cluster}.keyring' --create-keyring --name=client.bootstrap-osd --add-key="$KEY"
+ rm -f '/var/lib/ceph/bootstrap-osd/#{cluster}.keyring.raw'
+ EOH
+ end
+end
--- /dev/null
+raise "fsid must be set in config" if node["ceph"]["config"]['fsid'].nil?
+raise "mon_initial_members must be set in config" if node["ceph"]["config"]['mon_initial_members'].nil?
+
+mon_addresses = get_mon_addresses()
+
+template '/etc/ceph/ceph.conf' do
+ source 'ceph.conf.erb'
+ variables(
+ :mon_addresses => mon_addresses
+ )
+ mode '0644'
+end
--- /dev/null
+#
+# Author:: Kyle Bader <kyle.bader@dreamhost.com>
+# Cookbook Name:: ceph
+# Recipe:: default
+#
+# Copyright 2011, DreamHost Web Hosting
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+packages = %w{
+ ceph
+ ceph-dbg
+ ceph-common
+ ceph-common-dbg
+}
+
+packages.each do |pkg|
+ package pkg do
+ action :upgrade
+ end
+end
--- /dev/null
+#
+# Author:: Kyle Bader <kyle.bader@dreamhost.com>
+# Cookbook Name:: ceph
+# Recipe:: mds
+#
+# Copyright 2011, DreamHost Web Hosting
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include_recipe "ceph::default"
--- /dev/null
+# this recipe creates a monitor cluster
+
+require 'json'
+
+include_recipe "ceph::default"
+include_recipe "ceph::conf"
+
+if is_crowbar?
+ ipaddress = Chef::Recipe::Barclamp::Inventory.get_network_by_type(node, "admin").address
+else
+ ipaddress = node['ipaddress']
+end
+
+service "ceph-mon-all-starter" do
+ provider Chef::Provider::Service::Upstart
+ action [:enable]
+end
+
+# TODO cluster name
+cluster = 'ceph'
+
+execute 'ceph-mon mkfs' do
+ command <<-EOH
+set -e
+# TODO chef creates doesn't seem to suppressing re-runs, do it manually
+if [ -e '/var/lib/ceph/mon/ceph-#{node["hostname"]}/done' ]; then
+ echo 'ceph-mon mkfs already done, skipping'
+ exit 0
+fi
+KR='/var/lib/ceph/tmp/#{cluster}-#{node['hostname']}.mon.keyring'
+# TODO don't put the key in "ps" output, stdout
+ceph-authtool "$KR" --create-keyring --name=mon. --add-key='#{node["ceph"]["monitor-secret"]}' --cap mon 'allow *'
+
+ceph-mon --mkfs -i #{node['hostname']} --keyring "$KR"
+rm -f -- "$KR"
+touch /var/lib/ceph/mon/ceph-#{node['hostname']}/done
+EOH
+ # TODO built-in done-ness flag for ceph-mon?
+ creates '/var/lib/ceph/mon/ceph-#{node["hostname"]}/done'
+ notifies :start, "service[ceph-mon-all-starter]", :immediately
+end
+
+ruby_block "tell ceph-mon about its peers" do
+ block do
+ mon_addresses = get_mon_addresses()
+ mon_addresses.each do |addr|
+ system 'ceph', \
+ '--admin-daemon', "/var/run/ceph/ceph-mon.#{node['hostname']}.asok", \
+ 'add_bootstrap_peer_hint', "#{addr}"
+ # ignore errors
+ end
+ end
+end
+
+ruby_block "create client.admin keyring" do
+ block do
+ if not ::File.exists?('/etc/ceph/ceph.client.admin.keyring') then
+ if not have_quorum? then
+ puts 'ceph-mon is not in quorum, skipping bootstrap-osd key generation for this run'
+ else
+ # TODO --set-uid=0
+ key = %x[
+ ceph \
+ --name mon. \
+ --keyring '/var/lib/ceph/mon/#{cluster}-#{node['hostname']}/keyring' \
+ auth get-or-create-key client.admin \
+ mon 'allow *' \
+ osd 'allow *' \
+ mds allow
+ ]
+ raise 'adding or getting admin key failed' unless $?.exitstatus == 0
+ # TODO don't put the key in "ps" output, stdout
+ system 'ceph-authtool', \
+ '/etc/ceph/ceph.client.admin.keyring', \
+ '--create-keyring', \
+ '--name=client.admin', \
+ "--add-key=#{key}"
+ raise 'creating admin keyring failed' unless $?.exitstatus == 0
+ end
+ end
+ end
+end
+
+ruby_block "save osd bootstrap key in node attributes" do
+ block do
+ if node['ceph_bootstrap_osd_key'].nil? then
+ if not have_quorum? then
+ puts 'ceph-mon is not in quorum, skipping bootstrap-osd key generation for this run'
+ else
+ key = %x[
+ ceph \
+ --name mon. \
+ --keyring '/var/lib/ceph/mon/#{cluster}-#{node['hostname']}/keyring' \
+ auth get-or-create-key client.bootstrap-osd mon \
+ "allow command osd create ...; \
+ allow command osd crush set ...; \
+ allow command auth add * osd allow\\ * mon allow\\ rwx; \
+ allow command mon getmap"
+ ]
+ raise 'adding or getting bootstrap-osd key failed' unless $?.exitstatus == 0
+ node.override['ceph_bootstrap_osd_key'] = key
+ node.save
+ end
+ end
+ end
+end
--- /dev/null
+#
+# Author:: Kyle Bader <kyle.bader@dreamhost.com>
+# Cookbook Name:: ceph
+# Recipe:: osd
+#
+# Copyright 2011, DreamHost Web Hosting
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include_recipe "ceph::default"
--- /dev/null
+#
+# Author:: Kyle Bader <kyle.bader@dreamhost.com>
+# Cookbook Name:: ceph
+# Recipe:: rados-client
+#
+# Copyright 2011, DreamHost Web Hosting
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+packages = %w{
+ radosgw
+ radosgw-dbg
+}
+
+packages.each do |pkg|
+ package pkg do
+ action :upgrade
+ end
+end
--- /dev/null
+#
+# Author:: Kyle Bader <kyle.bader@dreamhost.com>
+# Cookbook Name:: ceph
+# Recipe:: radosgw
+#
+# Copyright 2011, DreamHost Web Hosting
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include_recipe "ceph::rados-rest"
+
+packages = %w{
+ libapache2-mod-fastcgi
+}
+
+include_recipe "apache2"
+
+packages.each do |pkg|
+ package pkg do
+ action :upgrade
+ end
+end
+
+cookbook_file "/etc/init.d/radosgw" do
+ source "radosgw"
+ mode 0755
+ owner "root"
+ group "root"
+end
+
+service "radosgw" do
+ service_name "radosgw"
+ supports :restart => true
+ action[:enable,:start]
+end
+
+apache_module "fastcgi" do
+ conf true
+end
+
+apache_module "rewrite" do
+ conf false
+end
+
+template "/etc/apache2/sites-available/rgw.conf" do
+ source "rgw.conf.erb"
+ mode 0400
+ owner "root"
+ group "root"
+ variables(
+ :ceph_api_fqdn => node['ceph']['radosgw']['api_fqdn'],
+ :ceph_admin_email => node['ceph']['radosgw']['admin_email'],
+ :ceph_rgw_addr => node['ceph']['radosgw']['rgw_addr']
+ )
+ if ::File.exists?("#{node['apache']['dir']}/sites-enabled/rgw.conf")
+ notifies :restart, "service[apache2]"
+ end
+end
+
+apache_site "rgw.conf" do
+ enable enable_setting
+end
+
--- /dev/null
+name "ceph-mds"
+description "Ceph Metadata Server"
+run_list(
+ 'recipe[ceph::mds]'
+)
--- /dev/null
+name "ceph-mon"
+description "Ceph Monitor"
+run_list(
+ 'recipe[ceph::mon]'
+)
--- /dev/null
+name "ceph-osd"
+description "Ceph Object Storage Device"
+run_list(
+ 'recipe[ceph::bootstrap_osd]'
+)
--- /dev/null
+name "ceph-radosgw"
+description "Ceph RADOS Gateway"
+run_list(
+ 'recipe[ceph::radosgw]'
+)
--- /dev/null
+<% if @branch.nil? %>
+deb http://ceph.newdream.net/debian/ <%= @codename %> main
+deb-src http://ceph.newdream.net/debian/ <%= @codename %> main
+<% else %>
+deb http://gitbuilder.ceph.com/ceph-deb-<%= @codename %>-x86_64-basic/ref/<%= @branch %>/ <%= @codename %> main
+deb-src http://gitbuilder.ceph.com/ceph-deb-<%= @codename %>-x86_64-basic/ref/<%= @branch %>/ <%= @codename %> main
+<% end %>
--- /dev/null
+[global]
+ auth supported = cephx
+ keyring = /etc/ceph/$cluster.$name.keyring
+
+ # this will be used by all clients to discover the mons to
+ # talk to; this way we don't need to have separate [mon.foo]
+ # sections, and come up with names for the mons just to be
+ # able to list their addresses
+ mon host = <%= @mon_addresses.join(', ') %>
+
+ <% node['ceph']['config'].each do |k, v| %>
+ <%= k %> = <%= v %>
+ <% end %>
+
+[mon]
+ keyring = /var/lib/ceph/mon/$cluster-$id/keyring
+
+[mds]
+ keyring = /var/lib/ceph/mds/$cluster-$id/keyring
+
+[osd]
+ keyring = /var/lib/ceph/osd/$cluster-$id/keyring
+ osd journal size = 1000
+ filestore_xattr_use_omap = true
+
+<% node['ceph']['config-sections'].each do |name, sect| %>
+
+[<%= name %>]
+<% sect.each do |k, v| %>
+<%= k %> = <%= v %>
+<% end %>
+<% end %>
--- /dev/null
+<IfModule mod_fastcgi.c>
+ AddHandler fastcgi-script .fcgi
+ #FastCgiWrapper /usr/lib/apache2/suexec
+ FastCgiIpcDir /var/lib/apache2/fastcgi
+</IfModule>
--- /dev/null
+FastCgiExternalServer /var/www/dummyradosgw.fcgi -socket /var/run/ceph/radosgw.client.radosgw.<%= node[:hostname] %>
+
+LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" proxy_combined
+LogFormat "%{X-Forwarded-For}i %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" proxy_debug
+
+<VirtualHost <%= @ceph_rgw_addr %>>
+ ServerName <%= @ceph_api_fqdn %>
+ ServerAdmin <%= @ceph_admin_email %>
+ DocumentRoot /var/www/
+
+ RewriteEngine On
+ RewriteRule ^/(.*) /dummyradosgw.fcgi?params=$1&%{QUERY_STRING} [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L]
+
+ <IfModule mod_fastcgi.c>
+ <Directory /var/www/>
+ Options +ExecCGI
+ AllowOverride All
+ SetHandler fastcgi-script
+ Order allow,deny
+ Allow from all
+ AuthBasicAuthoritative Off
+ </Directory>
+ </IfModule>
+
+ AllowEncodedSlashes On
+
+ ErrorLog /var/log/apache2/error.log
+ CustomLog /var/log/apache2/sssrgw-access.log proxy_combined
+ ServerSignature Off
+</VirtualHost>
projects / ceph-cookbooks.git / commitdiff