--- /dev/null
+These are incomplete, use with caution. They have pulled from a working configuration using Debian. They will require work for other distributions. They also assume your package manager (apt-get, etc) are already configured for a ceph repository.
--- /dev/null
+DESCRIPTION
+===========
+
+Installs and configures Ceph, a distributed network storage and filesystem
+designed to provide excellent performance, reliability, and scalability.
+
+REQUIREMENTS
+============
+
+Platform
+--------
+
+Tested as working:
+ * Debian Squeeze (6.x)
+
+Cookbooks
+---------
+
+The ceph cookbook requires the following cookbooks from Opscode:
+
+https://github.com/opscode/cookbooks
+
+* apache2
+
+Also required are the following cookbooks New Dream Network (DreamHost.com):
+
+https://github.com/NewDreamNetwork/ceph-cookbooks
+
+
+ATTRIBUTES
+==========
+
+Ceph Rados Gateway
+------------------
+
+* node[:ceph][:radosgw][:api_fqdn]
+* node[:ceph][:radosgw][:admin_email]
+* node[:ceph][:radosgw][:rgw_addr]
+
+TEMPLATES
+=========
+
+
+
+USAGE
+=====
+
+Ceph cluster design is beyond the scope of this README, please turn to the
+public wiki, mailing lists, visit our IRC channel or Ceph Support page:
+
+http://ceph.newdream.net/wiki/
+http://ceph.newdream.net/mailing-lists-and-irc/
+http://www.cephsupport.com/
+
+This diagram helps visualize recipe inheritence of the ceph cookbook recipes:
+
+ <diagram url>
+
+Ceph Monitor
+------------
+
+Ceph monitor nodes should use the ceph::mon recipe.
+
+Includes:
+
+* ceph::default
+* ceph::rados-rest
+
+Ceph Metadata Server
+--------------------
+
+Ceph metadata server nodes should use the ceph::mds recipe.
+
+Includes:
+
+* ceph::default
+
+Ceph OSD
+--------
+
+Ceph OSD nodes should use the ceph::osd recipe
+
+Includes:
+
+* ceph::default
+
+Ceph Rados Gateway
+------------------
+
+Ceph Rados Gateway nodes should use the ceph::radosgw recipe
+
+Includes:
+
+* ceph::rados-rest
+
+LICENSE AND AUTHORS
+===================
+
+* Author: Kyle Bader <kyle.bader@dreamhost.com>
+
+* Copyright 2011, DreamHost Web Hosting
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
--- /dev/null
+#
+# Cookbook Name:: ceph
+# Attributes:: radosgw
+#
+# Copyright 2011, DreamHost Web Hosting
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+default["ceph"]["radosgw"]["api_fqdn"] = "127.0.0.1"
+default["ceph"]["radosgw"]["admin-email"] = "admin@example.com"
+default["ceph"]["radosgw"]["rgw_addr"] = "*:80"
+
--- /dev/null
+#! /bin/sh
+### BEGIN INIT INFO
+# Provides: single
+# Required-Start: $remote_fs $named $network $time
+# Required-Stop: $remote_fs $named $network $time
+# Default-Start: 2 3 4 5
+# Default-Stop: 0 1 6
+# Short-Description: radosgw
+### END INIT INFO
+
+PATH=/sbin:/bin:/usr/bin
+
+. /lib/lsb/init-functions
+
+CLIENT_NAME=client.rgw-`hostname | sed 's/\..*//g'`
+USER=www-data
+
+
+do_start () {
+ su $USER -c "radosgw -c /etc/ceph/ceph.conf -n $CLIENT_NAME >> /var/log/radosgw/radosgw.log 2>&1 &"
+}
+
+case "$1" in
+ start)
+ do_start
+ ;;
+ restart|reload|force-reload)
+ echo "Error: argument '$1' not supported" >&2
+ exit 3
+ ;;
+ stop)
+ # No-op
+ ;;
+ *)
+ echo "Usage: $0 start|stop" >&2
+ exit 3
+ ;;
+esac
--- /dev/null
+def is_crowbar?()
+ return defined?(Chef::Recipe::Barclamp) != nil
+end
+
+QUORUM_STATES = ['leader', 'peon']
+
+def have_quorum?()
+ # "ceph auth get-or-create-key" would hang if the monitor wasn't
+ # in quorum yet, which is highly likely on the first run. This
+ # helper lets us delay the key generation into the next
+ # chef-client run, instead of hanging.
+ #
+ # Also, as the UNIX domain socket connection has no timeout logic
+ # in the ceph tool, this exits immediately if the ceph-mon is not
+ # running for any reason; trying to connect via TCP/IP would wait
+ # for a relatively long timeout.
+ mon_status = %x[ceph --admin-daemon /var/run/ceph/ceph-mon.#{node['hostname']}.asok mon_status]
+ raise 'getting monitor state failed' unless $?.exitstatus == 0
+ state = JSON.parse(mon_status)['state']
+ return QUORUM_STATES.include?(state)
+end
--- /dev/null
+maintainer "Kyle Bader"
+maintainer_email "kyle.bader@dreamhost.com"
+license "Apache 2.0"
+description "Installs/Configures the Ceph distributed filesystem"
+long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
+version "0.0.10"
+depends "apache2"
--- /dev/null
+release_or_autobuild = node["ceph_branch"].nil? ? "release" : "autobuild"
+
+execute "add autobuild gpg key to apt" do
+ command <<-EOH
+wget -q -O- https://raw.github.com/ceph/ceph/master/keys/#{release_or_autobuild}.asc \
+| sudo apt-key add -
+ EOH
+end
+
+template '/etc/apt/sources.list.d/ceph.list' do
+ owner 'root'
+ group 'root'
+ mode '0644'
+ source 'apt-sources-list.release.erb'
+ variables(
+ :codename => node[:lsb][:codename],
+ :branch => node["ceph_branch"]
+ )
+end
+
+execute 'apt-get update'
--- /dev/null
+# this recipe allows bootstrapping new osds, with help from mon
+
+include_recipe "ceph::osd"
+include_recipe "ceph::conf"
+
+if is_crowbar?
+ mons = search(:node, "role:ceph-mon AND ceph_config_environment:#{node['ceph']['config']['environment']} AND ceph_bootstrap_osd_key:*")
+else
+ mons = search(:node, "role:ceph-mon AND chef_environment:#{node.chef_environment} AND ceph_bootstrap_osd_key:*")
+end
+
+if mons.length < 1 then
+ puts "No ceph-mon found."
+else
+
+ directory "/var/lib/ceph/bootstrap-osd" do
+ owner "root"
+ group "root"
+ mode "0755"
+ end
+
+ # TODO cluster name
+ cluster = 'ceph'
+
+ file "/var/lib/ceph/bootstrap-osd/#{cluster}.keyring.raw" do
+ owner "root"
+ group "root"
+ mode "0440"
+ content mons[0]["ceph_bootstrap_osd_key"]
+ end
+
+ execute "format as keyring" do
+ command <<-EOH
+ set -e
+ # TODO don't put the key in "ps" output, stdout
+ read KEY <'/var/lib/ceph/bootstrap-osd/#{cluster}.keyring.raw'
+ ceph-authtool '/var/lib/ceph/bootstrap-osd/#{cluster}.keyring' --create-keyring --name=client.bootstrap-osd --add-key="$KEY"
+ rm -f '/var/lib/ceph/bootstrap-osd/#{cluster}.keyring.raw'
+ EOH
+ end
+end
--- /dev/null
+if is_crowbar?
+ mon_addresses = search(:node, "role:ceph-mon AND ceph_config_environment:#{node['ceph']['config']['environment']}").map { |node| Chef::Recipe::Barclamp::Inventory.get_network_by_type(node, "admin").address + ":6789" }
+else
+ mon_addresses = search(:node, "role:ceph-mon AND chef_environment:#{node.chef_environment}").map { |node| node["ipaddress"] + ":6789" }
+end
+
+template '/etc/ceph/ceph.conf' do
+ source 'ceph.conf.erb'
+ variables(
+ :fsid => node["ceph"]["config"]["fsid"],
+ :mon_initial_members => node["ceph"]["config"]["mon_initial_members"],
+ :mon_addresses => mon_addresses
+ )
+ mode '0644'
+end
--- /dev/null
+#
+# Author:: Kyle Bader <kyle.bader@dreamhost.com>
+# Cookbook Name:: ceph
+# Recipe:: default
+#
+# Copyright 2011, DreamHost Web Hosting
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+packages = %w{
+ ceph
+ ceph-dbg
+ ceph-common
+ ceph-common-dbg
+}
+
+packages.each do |pkg|
+ package pkg do
+ action :upgrade
+ end
+end
--- /dev/null
+#
+# Author:: Kyle Bader <kyle.bader@dreamhost.com>
+# Cookbook Name:: ceph
+# Recipe:: mds
+#
+# Copyright 2011, DreamHost Web Hosting
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include_recipe "ceph::default"
--- /dev/null
+#
+# Author:: Kyle Bader <kyle.bader@dreamhost.com>
+# Cookbook Name:: ceph
+# Recipe:: mon
+#
+# Copyright 2011, DreamHost Web Hosting
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include_recipe "ceph::default"
+include_recipe "ceph::rados-rest"
--- /dev/null
+#
+# Author:: Kyle Bader <kyle.bader@dreamhost.com>
+# Cookbook Name:: ceph
+# Recipe:: osd
+#
+# Copyright 2011, DreamHost Web Hosting
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include_recipe "ceph::default"
--- /dev/null
+#
+# Author:: Kyle Bader <kyle.bader@dreamhost.com>
+# Cookbook Name:: ceph
+# Recipe:: oss
+#
+# Copyright 2011, DreamHost Web Hosting
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include_recipe "ceph::rados-rest"
+
+package "obsync" do
+ version = node['ceph']['version']
+ action :install
+end
--- /dev/null
+#
+# Author:: Kyle Bader <kyle.bader@dreamhost.com>
+# Cookbook Name:: ceph
+# Recipe:: rados-client
+#
+# Copyright 2011, DreamHost Web Hosting
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+packages = %w{
+ radosgw
+ radosgw-dbg
+}
+
+packages.each do |pkg|
+ package pkg do
+ action :upgrade
+ end
+end
--- /dev/null
+#
+# Author:: Kyle Bader <kyle.bader@dreamhost.com>
+# Cookbook Name:: ceph
+# Recipe:: radosgw
+#
+# Copyright 2011, DreamHost Web Hosting
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include_recipe "ceph::rados-rest"
+
+packages = %w{
+ apache2
+ apache2-mpm-worker
+ apache2-utils
+ apache2.2-bin
+ apache2.2-common
+ libapache2-mod-fastcgi
+}
+
+include_recipe "apache2"
+
+packages.each do |pkg|
+ package pkg do
+ action :upgrade
+ end
+end
+
+cookbook_file "/etc/init.d/radosgw" do
+ source "radosgw"
+ mode 0755
+ owner "root"
+ group "root"
+end
+
+service "radosgw" do
+ service_name "radosgw"
+ supports :restart => true
+ action[:enable,:start]
+end
+
+apache_module "fastcgi" do
+ conf true
+end
+
+apache_module "rewrite" do
+ conf false
+end
+
+template "/etc/apache2/sites-available/rgw.conf" do
+ source "rgw.conf.erb"
+ mode 0400
+ owner "root"
+ group "root"
+ variables(
+ :ceph_api_fqdn => node['ceph']['api_fqdn'],
+ :ceph_admin_email => node['ceph']['admin_email'],
+ :ceph_rgw_addr => node['ceph']['rgw_addr']
+ )
+ if ::File.exists?("#{node['apache']['dir']}/sites-enabled/rgw.conf")
+ notifies :restart, "service[apache2]"
+ end
+end
+
+apache_site "rgw.conf" do
+ enable enable_setting
+end
+
--- /dev/null
+# this recipe creates a monitor cluster
+
+require 'json'
+
+include_recipe "ceph::mon"
+include_recipe "ceph::conf"
+
+if is_crowbar?
+ ipaddress = Chef::Recipe::Barclamp::Inventory.get_network_by_type(node, "admin").address
+else
+ ipaddress = node['ipaddress']
+end
+
+service "ceph-mon-all-starter" do
+ provider Chef::Provider::Service::Upstart
+ action [:enable]
+end
+
+# TODO cluster name
+cluster = 'ceph'
+
+execute 'ceph-mon mkfs' do
+ command <<-EOH
+set -e
+# TODO chef creates doesn't seem to suppressing re-runs, do it manually
+if [ -e '/var/lib/ceph/mon/ceph-#{node["hostname"]}/done' ]; then
+ echo 'ceph-mon mkfs already done, skipping'
+ exit 0
+fi
+KR='/var/lib/ceph/tmp/#{cluster}-#{node['hostname']}.mon.keyring'
+# TODO don't put the key in "ps" output, stdout
+ceph-authtool "$KR" --create-keyring --name=mon. --add-key='#{node["ceph"]["monitor-secret"]}' --cap mon 'allow *'
+
+ceph-mon --mkfs -i #{node['hostname']} --keyring "$KR"
+rm -f -- "$KR"
+touch /var/lib/ceph/mon/ceph-#{node['hostname']}/done
+EOH
+ # TODO built-in done-ness flag for ceph-mon?
+ creates '/var/lib/ceph/mon/ceph-#{node["hostname"]}/done'
+ notifies :start, "service[ceph-mon-all-starter]", :immediately
+end
+
+ruby_block "create client.admin keyring" do
+ block do
+ if not ::File.exists?('/etc/ceph/ceph.client.admin.keyring') then
+ if not have_quorum? then
+ puts 'ceph-mon is not in quorum, skipping bootstrap-osd key generation for this run'
+ else
+ # TODO --set-uid=0
+ key = %x[
+ ceph \
+ --name mon. \
+ --keyring '/var/lib/ceph/mon/#{cluster}-#{node['hostname']}/keyring' \
+ auth get-or-create-key client.admin \
+ mon 'allow *' \
+ osd 'allow *' \
+ mds allow
+ ]
+ raise 'adding or getting admin key failed' unless $?.exitstatus == 0
+ # TODO don't put the key in "ps" output, stdout
+ system 'ceph-authtool', \
+ '/etc/ceph/ceph.client.admin.keyring', \
+ '--create-keyring', \
+ '--name=client.admin', \
+ "--add-key=#{key}"
+ raise 'creating admin keyring failed' unless $?.exitstatus == 0
+ end
+ end
+ end
+end
+
+ruby_block "save osd bootstrap key in node attributes" do
+ block do
+ if node['ceph_bootstrap_osd_key'].nil? then
+ if not have_quorum? then
+ puts 'ceph-mon is not in quorum, skipping bootstrap-osd key generation for this run'
+ else
+ key = %x[
+ ceph \
+ --name mon. \
+ --keyring '/var/lib/ceph/mon/#{cluster}-#{node['hostname']}/keyring' \
+ auth get-or-create-key client.bootstrap-osd mon \
+ "allow command osd create ...; \
+ allow command osd crush set ...; \
+ allow command auth add * osd allow\\ * mon allow\\ rwx; \
+ allow command mon getmap"
+ ]
+ raise 'adding or getting bootstrap-osd key failed' unless $?.exitstatus == 0
+ node.override['ceph_bootstrap_osd_key'] = key
+ node.save
+ end
+ end
+ end
+end
--- /dev/null
+# Make sure / is mounted with extended attributes. Useful for running
+# Ceph OSDs without any extra filesystems mounted. Assumes "osd data"
+# will point to a directory on the root file system.
+
+execute "add user_xattr to root mount options in fstab" do
+ # fugly but works! which is more than i can say for the "mount"
+ # resource, which doesn't seem to like a rootfs with an unknown UUID
+ # at all.
+ command <<-'EOH'
+ perl -pe 'if (m{^([^#]\S*\s+/\s+\S+\s+)(\S+)(\s+.*)$}) { $_="$1$2,user_xattr$3\n" unless $2=~m{(^|,)user_xattr(,|$)}; }' -i.bak /etc/fstab
+ EOH
+end
+
+execute "enable xattr for this boot" do
+ command "mount -o remount,user_xattr /"
+end
--- /dev/null
+<% if @branch.nil? %>
+deb http://ceph.newdream.net/debian/ <%= @codename %> main
+deb-src http://ceph.newdream.net/debian/ <%= @codename %> main
+<% else %>
+deb http://gitbuilder.ceph.com/ceph-deb-<%= @codename %>-x86_64-basic/ref/<%= @branch %>/ <%= @codename %> main
+deb-src http://gitbuilder.ceph.com/ceph-deb-<%= @codename %>-x86_64-basic/ref/<%= @branch %>/ <%= @codename %> main
+<% end %>
--- /dev/null
+[global]
+ fsid = <%= @fsid %>
+ auth supported = cephx
+ keyring = /etc/ceph/$cluster.$name.keyring
+
+ # this will be used by all clients to discover the mons to
+ # talk to; this way we don't need to have separate [mon.foo]
+ # sections, and come up with names for the mons just to be
+ # able to list their addresses
+ mon host = <%= @mon_addresses.join(', ') %>
+ mon initial members = <%= @mon_initial_members.join(', ') %>
+
+[mon]
+ keyring = /var/lib/ceph/mon/$cluster-$id/keyring
+
+[mds]
+ keyring = /var/lib/ceph/mds/$cluster-$id/keyring
+
+[osd]
+ keyring = /var/lib/ceph/osd/$cluster-$id/keyring
+ osd journal size = 1000
+ filestore_xattr_use_omap = true
--- /dev/null
+<IfModule mod_fastcgi.c>
+ AddHandler fastcgi-script .fcgi
+ #FastCgiWrapper /usr/lib/apache2/suexec
+ FastCgiIpcDir /var/lib/apache2/fastcgi
+</IfModule>
--- /dev/null
+FastCgiExternalServer /var/www/dummyradosgw.fcgi -socket /var/run/ceph/radosgw.client.radosgw.<%= node[:hostname] %>
+
+LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" proxy_combined
+LogFormat "%{X-Forwarded-For}i %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" proxy_debug
+
+<VirtualHost <%= @ceph_rgw_addr %>>
+ ServerName <%= @ceph_api_fqdn %>
+ ServerAdmin <%= @ceph_admin_email %>
+ DocumentRoot /var/www/
+
+ RewriteEngine On
+ RewriteRule ^/(.*) /dummyradosgw.fcgi?params=$1&%{QUERY_STRING} [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L]
+
+ <IfModule mod_fastcgi.c>
+ <Directory /var/www/>
+ Options +ExecCGI
+ AllowOverride All
+ SetHandler fastcgi-script
+ Order allow,deny
+ Allow from all
+ AuthBasicAuthoritative Off
+ </Directory>
+ </IfModule>
+
+ AllowEncodedSlashes On
+
+ ErrorLog /var/log/apache2/error.log
+ CustomLog /var/log/apache2/sssrgw-access.log proxy_combined
+ ServerSignature Off
+</VirtualHost>
projects / ceph-cookbooks.git / commitdiff