from teuthology import misc as teuthology
from teuthology import contextutil
from ..orchestra import run
+import ceph_manager
log = logging.getLogger(__name__)
+# this was lifted from radosgw-admin-rest.
+def rgwadmin(ctx, remote, cmd):
+ log.info('radosgw-admin: %s' % cmd)
+ testdir = teuthology.get_testdir(ctx)
+ pre = [
+ '{tdir}/adjust-ulimits'.format(tdir=testdir),
+ 'ceph-coverage'.format(tdir=testdir),
+ '{tdir}/archive/coverage'.format(tdir=testdir),
+ 'radosgw-admin'.format(tdir=testdir),
+ '--log-to-stderr',
+ '--format', 'json',
+ ]
+ pre.extend(cmd)
+ proc = remote.run(
+ args=pre,
+ check_status=False,
+ stdout=StringIO(),
+ stderr=StringIO(),
+ )
+ r = proc.exitstatus
+ out = proc.stdout.getvalue()
+ j = None
+ if not r and out != '':
+ try:
+ j = json.loads(out)
+ log.info(' json result: %s' % j)
+ except ValueError:
+ j = out
+ log.info(' raw result: %s' % j)
+ return (r, j)
+
+# this was lifted from lost_unfound.py
+def rados(ctx, remote, cmd):
+ testdir = teuthology.get_testdir(ctx)
+ log.info("rados %s" % ' '.join(cmd))
+ pre = [
+ '{tdir}/adjust-ulimits'.format(tdir=testdir),
+ 'ceph-coverage',
+ '{tdir}/archive/coverage'.format(tdir=testdir),
+ 'rados',
+ ];
+ pre.extend(cmd)
+ proc = remote.run(
+ args=pre,
+ check_status=False
+ )
+
+ return proc.exitstatus
@contextlib.contextmanager
def create_dirs(ctx, config):
if client_config is None:
client_config = {}
log.info("rgw %s config is %s", client, client_config)
-
+ id_ = client.split('.', 1)[1]
+ log.info('client {client} is id {id}'.format(client=client, id=id_))
run_cmd=[
'sudo',
'{tdir}/adjust-ulimits'.format(tdir=testdir),
run_cmd_tail=[
'radosgw',
# authenticate as the client this is co-located with
- '-i', '{client}'.format(client=client),
- '-k', '/etc/ceph/ceph.keyring',
+ '--name', '{client}'.format(client=client),
+ # use the keyring for the client we're running as
+ '-k', '/etc/ceph/ceph.{client}.keyring'.format(client=client),
'--log-file', '/var/log/ceph/rgw.log',
'--rgw_ops_log_socket_path', '{tdir}/rgw.opslog.sock'.format(tdir=testdir),
'{tdir}/apache/apache.conf'.format(tdir=testdir),
run.wait(apaches.itervalues())
+# pulls the configured zone info out and also
+# inserts some sensible defaults.
+def extract_zone_info(zone_info):
+
+ # now send the zone settings across the wire
+ system_user = zone_info['user']
+ system_access_key = zone_info['access_key']
+ system_secret_key = zone_info['secret_key']
+ zone_suffix = zone_info['zone_suffix']
+
+ # new dict to hold the data
+ zone_dict = {}
+ zone_dict['domain_root'] = '.rgw.root' + zone_suffix
+ zone_dict['control_pool'] = '.rgw.control' + zone_suffix
+ zone_dict['gc_pool'] = '.rgw.gc' + zone_suffix
+ zone_dict['log_pool'] = '.log' + zone_suffix
+ zone_dict['intent_log_pool'] = '.intent-log' + zone_suffix
+ zone_dict['usage_log_pool'] = '.usage' + zone_suffix
+ zone_dict['user_keys_pool'] = '.users' + zone_suffix
+ zone_dict['user_email_pool'] = '.users.email' + zone_suffix
+ zone_dict['user_swift_pool'] = '.users.swift' + zone_suffix
+ zone_dict['user_uid_pool'] = '.users.id' + zone_suffix
+
+ system_user_dict = {}
+ system_user_dict['user'] = system_user
+ system_user_dict['access_key'] = system_access_key
+ system_user_dict['secret_key'] = system_secret_key
+
+ zone_dict['system_key'] = system_user_dict
+
+ return zone_dict
+
+# pulls the configured region info out and also
+# inserts some sensible defaults.
+def extract_region_info(region_info, host_name):
+ # create a new, empty dict to populate
+ region_dict = {}
+ # use specifed region name or default <client name>_'region'
+ if region_info.has_key('name'):
+ region_dict['name'] = region_info['name']
+ else:
+ region_dict['name'] = '{client}_region'.format(client=client)
+
+ # use specified api name or default to 'default'
+ if region_info.has_key('api_name'):
+ region_dict['api_name'] = region_info['api_name']
+ else:
+ region_dict['api_name'] = 'default'
+
+ # Going to assume this is specified for now
+ if region_info.has_key('is_master'):
+ region_dict['is_master'] = region_info['is_master']
+ else:
+ pass
+ #region_dict['is_master'] = 'false'
+
+ # use specified api name or default to 'default'
+ if region_info.has_key('api_name'):
+ region_dict['api_name'] = region_info['api_name']
+ else:
+ region_dict['api_name'] = 'default'
+
+ # add in sensible defaults for the endpoints field
+ if region_info.has_key('endpoints'):
+ region_dict['endpoints'] = region_info['endpoints']
+ else:
+ region_dict['endpoints'] = []
+ region_dict['endpoints'].append('http://{client}:80/'.format(client=host_name))
+
+ # use specified master zone or default to the first specified zone
+ # (if there's more than one)
+ if region_info.has_key('master_zone'):
+ region_dict['master_zone'] = region_info['master_zone']
+ else:
+ region_dict['master_zone'] = 'default'
+
+ region_dict['zones'] = []
+ zones = region_info['zones'].split(',')
+ for zone in zones:
+ # build up a zone
+ name, log_meta, log_data = zone.split(':')
+ log.info('zone: {zone} meta:{meta} data:{data}'.format(zone=name,
+ meta=log_meta, data=log_data))
+ new_zone_dict = {}
+ new_zone_dict['name'] = name
+ new_zone_dict['endpoints'] = []
+ new_zone_dict['endpoints'].append('http://{client}:80/'.format(client=host_name))
+ new_zone_dict['log_meta'] = log_meta
+ new_zone_dict['log_data'] = log_data
+ region_dict['zones'].append(new_zone_dict)
+
+ # just using defaults for now, revisit this later to allow
+ # the configs to specify placement_targets and default_placement policies
+ region_dict['placement_targets'] = []
+ default_placement_dict = {}
+ default_placement_dict['name'] = 'default-placement'
+ default_placement_dict['tags'] = []
+ region_dict['placement_targets'].append(default_placement_dict)
+
+ region_dict['default_placement'] = 'default-placement'
+
+ return region_dict
+
@contextlib.contextmanager
def configure_regions_and_zones(ctx, config):
log.info('Configuring regions and zones...')
- # track files to delete, just easier than reproducing host/filepaths
- files_to_delete = {}
+ client_list = []
+ region_files = []
+ zone_files = []
+
+ # iterate through all the clients, creating files with the region zone info and shipping those
+ # to all hosts. Then, pull out the zone info and stash it in client_list
for client in config.iterkeys():
(remote,) = ctx.cluster.only(client).remotes.iterkeys()
log.info("rgw %s config is %s dns is %s", client, client_config, host_name)
- files_to_delete[remote] = []
-
if 'region_info' in client_config:
- region_info = client_config['region_info']
- log.info('region info for {client}: {data}'.format(client=client, data=region_info))
-
- # create a new, empty dict to populate
- region_dict = {}
- # use specifed region name or default <client name>_'region'
- if region_info.has_key('name'):
- region_dict['name'] = region_info['name']
- else:
- region_dict['name'] = '{client}_region'.format(client=client)
-
- # use specified api name or default to 'default'
- if region_info.has_key('api_name'):
- region_dict['api_name'] = region_info['api_name']
- else:
- region_dict['api_name'] = 'default'
-
- # I don't think we should make assumptions here.
- # Going to assume this is specified for now
- if region_info.has_key('is_master'):
- region_dict['is_master'] = region_info['is_master']
- else:
- pass
- #region_dict['is_master'] = 'false'
-
- # use specified api name or default to 'default'
- if region_info.has_key('api_name'):
- region_dict['api_name'] = region_info['api_name']
- else:
- region_dict['api_name'] = 'default'
-
- # add in sensible defaults for the endpoints field
- if region_info.has_key('endpoints'):
- region_dict['endpoints'] = region_info['endpoints']
- else:
- region_dict['endpoints'] = []
- region_dict['endpoints'].append('http:\/\/{client}:80\/'.format(client=host_name))
-
- # use specified master zone or default to the first specified zone
- # (if there's more than one)
- if region_info.has_key('master_zone'):
- region_dict['master_zone'] = region_info['master_zone']
- else:
- region_dict['master_zone'] = 'default'
-
- region_dict['zones'] = []
- zones = region_info['zones'].split(',')
- for zone in zones:
- # build up a zone
- name, log_meta, log_data = zone.split(':')
- log.info('zone: {zone} meta:{meta} data:{data}'.format(zone=name, meta=log_meta, data=log_data))
- new_zone_dict = {}
- new_zone_dict['name'] = name
- new_zone_dict['endpoints'] = []
- #end_point_list = []
- new_zone_dict['endpoints'].append('http://{client}:80/'.format(client=host_name))
- new_zone_dict['log_meta'] = log_meta
- new_zone_dict['log_data'] = log_data
- region_dict['zones'].append(new_zone_dict)
-
- # just using defaults for now, revisit this later to allow
- # the configs to specify placement_targets and default_placement policies
- region_dict['placement_targets'] = []
- default_placement_dict = {}
- default_placement_dict['name'] = 'default-placement'
- default_placement_dict['tags'] = []
- region_dict['placement_targets'].append(default_placement_dict)
-
- region_dict['default_placement'] = 'default-placement'
+ region_dict = extract_region_info(client_config['region_info'], host_name)
+ region_name = region_dict['name']
log.info('constructed region info: {data}'.format(data=region_dict))
file_name = region_dict['name'] + '.input'
testdir = teuthology.get_testdir(ctx)
region_file_path = os.path.join(testdir, file_name)
- log.info('Shipping {file_out} to host {host}'.format(file_out=region_file_path, \
- host=host_name))
tmpFile = StringIO()
- tmpFile.write('{data}'.format(data=json.dumps(region_dict, sort_keys=True, \
+ # use json.dumps as a pretty printer so that the generated file
+ # is easier to read
+ tmpFile.write('{data}'.format(data=json.dumps(region_dict, sort_keys=True,
indent=4)))
- tmpFile.seek(0)
- #(remote,) = ctx.cluster.only(client).remotes.keys()
- teuthology.write_file(
- remote=remote,
- path=region_file_path,
- data=tmpFile,
- )
+ # ship this region file to all clients
+ for region_client in config.iterkeys():
+ (region_remote,) = ctx.cluster.only(region_client).remotes.iterkeys()
- # add this file to the dictionary of files to be deleted
- files_to_delete[remote].append(region_file_path)
+ log.info('sending region file {region_file} to {host}'.format( \
+ region_file=region_file_path,
+ host=str(region_remote)))
- else: # if 'region_info' in client_config:
- log.info('no region info found for client {client}'.format(client=client))
- # now work on the zone info
- if 'zone_info' in client_config:
- zone_info = client_config['zone_info']
- # now send the zone settings across the wire
- system_user = zone_info['user']
- system_access_key = zone_info['access_key']
- system_secret_key = zone_info['secret_key']
- zone_suffix = zone_info['zone_suffix']
- log.info('jb:\n\tuser:{user}\n\taccess:{access}\n\tsecret:{secret} \
- \n\tsuffix:{suffix}'.format(user=system_user, access=system_access_key, \
- secret=system_secret_key, suffix=zone_suffix))
-
- # new dict to hold the data
- zone_dict = {}
- zone_dict['domain_root'] = '.rgw.root' + zone_suffix
- zone_dict['control_pool'] = '.rgw.control' + zone_suffix
- zone_dict['gc_pool'] = '.rgw.gc' + zone_suffix
- zone_dict['log_pool'] = '.log' + zone_suffix
- zone_dict['intent_log_pool'] = '.intent-log' + zone_suffix
- zone_dict['usage_log_pool'] = '.usage' + zone_suffix
- zone_dict['user_keys_pool'] = '.users' + zone_suffix
- zone_dict['user_email_pool'] = '.users.email' + zone_suffix
- zone_dict['user_swift_pool'] = '.users.swift' + zone_suffix
-
- system_user_dict = {}
- system_user_dict['user'] = system_user
- system_user_dict['access_key'] = system_access_key
- system_user_dict['secret_key'] = system_secret_key
-
- zone_dict['system_key'] = system_user_dict
-
- log.info('constructed zone info: {data}'.format(data=zone_dict))
-
- file_name = 'zone' + zone_suffix + '.input'
- testdir = teuthology.get_testdir(ctx)
- zone_file_path = os.path.join(testdir, file_name)
- log.info('Shipping {file_out} to host {host}'.format(file_out=zone_file_path, \
- host=host_name))
+ tmpFile.seek(0)
- tmpFile = StringIO()
+ teuthology.write_file(
+ remote=region_remote,
+ path=region_file_path,
+ data=tmpFile,
+ )
- tmpFile.write('{data}'.format(data=zone_dict))
- tmpFile.seek(0)
- #(remote,) = ctx.cluster.only(client).remotes.keys()
- teuthology.write_file(
- remote=remote,
- path=zone_file_path,
- data=tmpFile,
- )
+ region_files.append(region_file_path)
+
+
+ # now work on the zone info
+ if 'zone_info' in client_config:
+ zone_info = client_config['zone_info']
+ zone_dict = extract_zone_info(zone_info)
+
+ zone_suffix = zone_info['zone_suffix']
+ zone_name = zone_info['name']
+
+ file_name = 'zone' + zone_suffix + '.input'
+ testdir = teuthology.get_testdir(ctx)
+ zone_file_path = os.path.join(testdir, file_name)
+ log.info('Sending zone file {file_out} to host {host}'.format(file_out=zone_file_path, \
+ host=host_name))
+
+ tmpFile = StringIO()
+
+ tmpFile.write('{data}'.format(data=json.dumps(zone_dict,
+ sort_keys=True, indent=4)))
+
+ # zone info only gets sent to the particular client
+ tmpFile.seek(0)
+ teuthology.write_file(
+ remote=remote,
+ path=zone_file_path,
+ data=tmpFile,
+ )
+
+ zone_files.append(zone_file_path)
+
+ # add an entry that is client, remote, region, zone, suffix, zone file path
+ # this data is used later
+ client_entry = []
+ client_entry.append(client)
+ client_entry.append(remote)
+ client_entry.append(region_name)
+ client_entry.append(zone_name)
+ client_entry.append(zone_suffix)
+ client_entry.append(zone_file_path)
+ client_list.append(client_entry)
+
+ else: # if 'zone_info' in client_config:
+ log.info('no zone info found for client {client}'.format(client=client))
+
+ else: # if 'region_info' in client_config:
+ log.info('no region info found for client {client}'.format(client=client))
- files_to_delete[remote].append(zone_file_path)
- else:
- log.info('no zone info found for client {client}'.format(client=client))
+ # now apply the region info on all clients
+ for region_file_path in region_files:
+ for client_entry in client_list:
+
+ client = client_entry[0]
+ remote = client_entry[1]
+ region_name = client_entry[2]
+ zone_name = client_entry[3]
+ suffix = client_entry[4]
+ zone_file_path = client_entry[5]
+
+ log.info('cl: {client_name} rn: {region} zn: {zone} rf: {rf} suf:{suffix} zfp:{zfp}'.format(\
+ client_name=str(remote), region=region_name, zone=zone_name,
+ rf=region_file_path, suffix=suffix, zfp=zone_file_path))
+
+ # now use this file to set the region info on the host
+ (err, out) = rgwadmin(ctx, remote, ['region', 'set', '--infile',
+ region_file_path, '--name',
+ '{client}'.format(client=client),
+ '--rgw-region',
+ '{region}'.format(region=region_name),
+ '--rgw-zone',
+ '{zone}'.format(zone=zone_name)])
+ assert not err
+
+ # go through the clients and set the zone info
+ for client_entry in client_list:
+ client = client_entry[0]
+ remote = client_entry[1]
+ region_name = client_entry[2]
+ zone_name = client_entry[3]
+ suffix = client_entry[4]
+ zone_file_path = client_entry[5]
+
+
+ (err, out) = rgwadmin(ctx, remote, ['zone', 'set',
+ '--rgw-zone={zone}'.format(zone=zone_name),
+ '--infile', zone_file_path,
+ '--name', '{client}'.format(client=client),
+ '--rgw-region',
+ '{region}'.format(region=region_name),
+ ])
+ assert not err
+
+ # set this to the default region
+ for client_entry in client_list:
+
+ client = client_entry[0]
+ remote = client_entry[1]
+ region_name = client_entry[2]
+ zone_name = client_entry[3]
+ suffix = client_entry[4]
+ zone_file_path = client_entry[5]
+ log.info('rf: {rf}'.format(rf=region_file_path))
+ (err, out) = rgwadmin(ctx, remote, ['region', 'default',
+ '--name', '{client}'.format(client=client),
+ '--rgw-region',
+ '{region}'.format(region=region_name),
+ #'--rgw-zone',
+ #'{zone}'.format(zone=zone_name)
+ ])
+ if out:
+ log.info('JB out: {data}'.format(data=out()))
+ if err:
+ log.info('err: {err} out: {out}'.format(err=err, out=out))
+
+ assert not err
+
+ # only do this if there's at least one zone specified
+ if len(client_list) > 0:
+ # need the mon to make rados calls
+ first_mon = teuthology.get_first_mon(ctx, config)
+ (mon,) = ctx.cluster.only(first_mon).remotes.iterkeys()
+
+ # delete the pools for the default
+ err = rados(ctx, mon, ['-p', '.rgw.root', 'rm', 'region_info.default'])
+ log.info('err is %d' % err)
+
+ err = rados(ctx, mon, ['-p', '.rgw.root', 'rm', 'zone_info.default'])
+ log.info('err is %d' % err)
+
+ # delete the pools for the suffixs
+ for client_entry in client_list:
+
+ client = client_entry[0]
+ remote = client_entry[1]
+ region_name = client_entry[2]
+ zone_name = client_entry[3]
+ suffix = client_entry[4]
+ zone_file_path = client_entry[5]
+
+ err = rados(ctx, mon, ['-p', '.rgw.root' + suffix,
+ 'rm', 'region_info.default'])
+ log.info('err is %d' % err)
+
+ err = rados(ctx, mon, ['-p', '.rgw.root' + suffix,
+ 'rm', 'zone_info.default'])
+ log.info('err is %d' % err)
+
+ # update the region map
+ for client_entry in client_list:
+
+ client = client_entry[0]
+ remote = client_entry[1]
+ region_name = client_entry[2]
+ zone_name = client_entry[3]
+ suffix = client_entry[4]
+ zone_file_path = client_entry[5]
+
+ (err, out) = rgwadmin(ctx, remote, ['region-map', 'update',
+ '--name',
+ '{client}'.format(client=client),
+ '--rgw-region',
+ '{region}'.format(region=region_name),
+ '--rgw-zone',
+ '{zone}'.format(zone=zone_name)])
+ assert not err
try:
yield
finally:
log.info('Cleaning up regions and zones....')
- for remote in files_to_delete.keys():
- per_host_files_to_delete = files_to_delete[remote]
- for to_delete in per_host_files_to_delete:
- log.info('deleting {file_name} from host {host}'.format(file_name=to_delete, \
- host=str(remote)))
- ctx.cluster.only(remote).run(
+
+ for client_entry in client_list:
+
+ client = client_entry[0]
+ remote = client_entry[1]
+ region_name = client_entry[2]
+ zone_name = client_entry[3]
+ suffix = client_entry[4]
+ zone_file_path = client_entry[5]
+
+ for region_file in region_files:
+ log.info('deleting {file_name} from host {host}'.format( \
+ file_name=region_file,
+ host=str(remote)))
+ remote.run(
args=[
'rm',
'-f',
- '{file_path}'.format(file_path=to_delete),
+ '{file_path}'.format(file_path=region_file),
],
)
+ for zone_file in zone_files:
+ log.info('deleting {file_name} from host {host}'.format( \
+ file_name=zone_file,
+ host=str(remote)))
+ remote.run(
+ args=[
+ 'rm',
+ '-f',
+ '{file_path}'.format(file_path=zone_file),
+ ],
+ )
@contextlib.contextmanager
def task(ctx, config):
projects / teuthology.git / commitdiff