From 787a78e8b1d8e5d92ee447b970ec8b79baa46f0b Mon Sep 17 00:00:00 2001 From: Tommi Virtanen Date: Thu, 30 Aug 2012 10:16:52 -0400 Subject: [PATCH] upstart, ceph-create-keys: Make client.admin key generation automatic. This should help simplify Chef etc deployments. Now (when using the Upstart jobs), when a ceph-mon is started, ceph-create-admin-key is triggered. If /etc/ceph/$cluster.client.admin.keyring already exists, it does nothing; otherwise, it waits for ceph-mon to reach quorum, and then does a "ceph auth get-or-create" to create the key, and writes it atomically to disk. The equivalent code can be removed from the Chef cookbook once this is in. Signed-off-by: Tommi Virtanen --- ceph.spec.in | 1 + debian/ceph.install | 1 + src/Makefile.am | 6 +- src/ceph-create-keys | 145 ++++++++++++++++++++++++++++++ src/upstart/ceph-create-keys.conf | 7 ++ 5 files changed, 158 insertions(+), 2 deletions(-) create mode 100755 src/ceph-create-keys create mode 100644 src/upstart/ceph-create-keys.conf diff --git a/ceph.spec.in b/ceph.spec.in index 54b307e4224c2..770ac45247088 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -328,6 +328,7 @@ fi %{_libdir}/rados-classes/libcls_rgw.so* /sbin/ceph-disk-activate /sbin/ceph-disk-prepare +/sbin/ceph-create-keys ################################################################################# %files fuse diff --git a/debian/ceph.install b/debian/ceph.install index 9674dbc88d688..da097b24c8601 100644 --- a/debian/ceph.install +++ b/debian/ceph.install @@ -8,6 +8,7 @@ usr/bin/ceph-osd usr/bin/ceph-debugpack sbin/ceph-disk-prepare usr/sbin/ sbin/ceph-disk-activate usr/sbin/ +sbin/ceph-create-keys usr/sbin/ sbin/mkcephfs usr/lib/ceph/ceph_common.sh usr/lib/rados-classes/* diff --git a/src/Makefile.am b/src/Makefile.am index 91532b4c60bdd..b57648e2e3305 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -28,7 +28,8 @@ bin_DEBUGPROGRAMS = sbin_PROGRAMS = sbin_SCRIPTS = \ ceph-disk-prepare \ - ceph-disk-activate + ceph-disk-activate \ + ceph-create-keys bin_SCRIPTS = ceph-run $(srcdir)/ceph-clsinfo ceph-debugpack ceph-rbdnamer dist_bin_SCRIPTS = # C/C++ tests to build will be appended to this @@ -905,7 +906,8 @@ EXTRA_DIST += \ $(srcdir)/upstart/radosgw-all.conf \ $(srcdir)/upstart/radosgw-all-starter.conf \ ceph-disk-prepare \ - ceph-disk-activate + ceph-disk-activate \ + ceph-create-keys EXTRA_DIST += $(srcdir)/$(shell_scripts:%=%.in) diff --git a/src/ceph-create-keys b/src/ceph-create-keys new file mode 100755 index 0000000000000..7c56b0e8a52a1 --- /dev/null +++ b/src/ceph-create-keys @@ -0,0 +1,145 @@ +#!/usr/bin/python +import argparse +import errno +import json +import logging +import os +import subprocess +import sys +import time + + +log = logging.getLogger(os.path.basename(sys.argv[0])) + +QUORUM_STATES = ['leader', 'peon'] + +def wait_for_quorum(cluster, mon_id): + while True: + p = subprocess.Popen( + args=[ + 'ceph', + '--cluster={cluster}'.format(cluster=cluster), + '--admin-daemon=/var/run/ceph/{cluster}-mon.{mon_id}.asok'.format( + cluster=cluster, + mon_id=mon_id, + ), + 'mon_status', + ], + stdout=subprocess.PIPE, + ) + out = p.stdout.read() + returncode = p.wait() + if returncode != 0: + log.info('ceph-mon admin socket not ready yet.') + time.sleep(1) + continue + + data = json.loads(out) + state = data['state'] + if state not in QUORUM_STATES: + log.info('ceph-mon is not in quorum: %r', state) + time.sleep(1) + continue + + break + + +def get_key(cluster, mon_id): + path = '/etc/ceph/{cluster}.client.admin.keyring'.format( + cluster=cluster, + ) + if os.path.exists(path): + log.info('Key exists already: %s', path) + return + tmp = '{path}.{pid}.tmp'.format( + path=path, + pid=os.getpid(), + ) + wait_for_quorum(cluster=cluster, mon_id=mon_id) + while True: + try: + with file(tmp, 'w') as f: + os.fchmod(f.fileno(), 0600) + log.info('Talking to monitor...') + returncode = subprocess.call( + args=[ + 'ceph', + '--cluster={cluster}'.format(cluster=cluster), + '--name=mon.', + '--keyring=/var/lib/ceph/mon/{cluster}-{mon_id}/keyring'.format( + cluster=cluster, + mon_id=mon_id, + ), + 'auth', + 'get-or-create', + 'client.admin', + 'mon', 'allow *', + 'osd', 'allow *', + 'mds', 'allow', + ], + stdout=f, + ) + if returncode != 0: + log.info('Cannot get or create admin key') + time.sleep(1) + continue + + os.rename(tmp, path) + break + finally: + try: + os.unlink(tmp) + except OSError as e: + if e.errno == errno.ENOENT: + pass + else: + raise + + +def parse_args(): + parser = argparse.ArgumentParser( + description='Create Ceph client.admin key when ceph-mon is ready', + ) + parser.add_argument( + '-v', '--verbose', + action='store_true', default=None, + help='be more verbose', + ) + parser.add_argument( + '--cluster', + metavar='NAME', + help='name of the cluster', + ) + parser.add_argument( + '--id', '-i', + metavar='ID', + help='id of a ceph-mon that is coming up', + required=True, + ) + parser.set_defaults( + cluster='ceph', + ) + parser.set_defaults( + # we want to hold on to this, for later + prog=parser.prog, + ) + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + + loglevel = logging.INFO + if args.verbose: + loglevel = logging.DEBUG + + logging.basicConfig( + level=loglevel, + ) + + get_key(cluster=args.cluster, mon_id=args.id) + + +if __name__ == '__main__': + main() diff --git a/src/upstart/ceph-create-keys.conf b/src/upstart/ceph-create-keys.conf new file mode 100644 index 0000000000000..de215d98ff347 --- /dev/null +++ b/src/upstart/ceph-create-keys.conf @@ -0,0 +1,7 @@ +description "Create Ceph client.admin key when possible" + +start on started ceph-mon + +task + +exec /usr/sbin/ceph-create-keys --cluster="${cluster:-ceph}" -i "${id:-$(hostname)}" -- 2.39.5