config = {}
assert isinstance(config, dict), \
'divergent_priors task only accepts a dict for configuration'
- first_mon = teuthology.get_first_mon(ctx, config)
- (mon,) = ctx.cluster.only(first_mon).remotes.iterkeys()
- manager = ceph_manager.CephManager(
- mon,
- ctx=ctx,
- logger=log.getChild('ceph_manager'),
- )
- ctx.manager = manager
-
- while len(manager.get_osd_status()['up']) < 3:
+ while len(ctx.manager.get_osd_status()['up']) < 3:
time.sleep(10)
- manager.raw_cluster_cmd('tell', 'osd.0', 'flush_pg_stats')
- manager.raw_cluster_cmd('tell', 'osd.1', 'flush_pg_stats')
- manager.raw_cluster_cmd('tell', 'osd.2', 'flush_pg_stats')
- manager.raw_cluster_cmd('osd', 'set', 'noout')
- manager.raw_cluster_cmd('osd', 'set', 'noin')
- manager.raw_cluster_cmd('osd', 'set', 'nodown')
- manager.wait_for_clean()
+ ctx.manager.raw_cluster_cmd('tell', 'osd.0', 'flush_pg_stats')
+ ctx.manager.raw_cluster_cmd('tell', 'osd.1', 'flush_pg_stats')
+ ctx.manager.raw_cluster_cmd('tell', 'osd.2', 'flush_pg_stats')
+ ctx.manager.raw_cluster_cmd('osd', 'set', 'noout')
+ ctx.manager.raw_cluster_cmd('osd', 'set', 'noin')
+ ctx.manager.raw_cluster_cmd('osd', 'set', 'nodown')
+ ctx.manager.wait_for_clean()
# something that is always there
dummyfile = '/etc/fstab'
# create 1 pg pool
log.info('creating foo')
- manager.raw_cluster_cmd('osd', 'pool', 'create', 'foo', '1')
+ ctx.manager.raw_cluster_cmd('osd', 'pool', 'create', 'foo', '1')
osds = [0, 1, 2]
for i in osds:
- manager.set_config(i, osd_min_pg_log_entries=1)
+ ctx.manager.set_config(i, osd_min_pg_log_entries=1)
# determine primary
- divergent = manager.get_pg_primary('foo', 0)
+ divergent = ctx.manager.get_pg_primary('foo', 0)
log.info("primary and soon to be divergent is %d", divergent)
non_divergent = [0,1,2]
non_divergent.remove(divergent)
for i in range(1000):
rados(ctx, mon, ['-p', 'foo', 'put', 'existing_%d' % i, dummyfile])
- manager.wait_for_clean()
+ ctx.manager.wait_for_clean()
# blackhole non_divergent
log.info("blackholing osds %s", str(non_divergent))
for i in non_divergent:
- manager.set_config(i, filestore_blackhole='')
+ ctx.manager.set_config(i, filestore_blackhole='')
# write 1 (divergent) object
log.info('writing divergent object existing_0')
# kill all the osds
log.info('killing all the osds')
for i in osds:
- manager.kill_osd(i)
+ ctx.manager.kill_osd(i)
for i in osds:
- manager.mark_down_osd(i)
+ ctx.manager.mark_down_osd(i)
for i in osds:
- manager.mark_out_osd(i)
+ ctx.manager.mark_out_osd(i)
# bring up non-divergent
log.info("bringing up non_divergent %s", str(non_divergent))
for i in non_divergent:
- manager.revive_osd(i)
+ ctx.manager.revive_osd(i)
for i in non_divergent:
- manager.mark_in_osd(i)
+ ctx.manager.mark_in_osd(i)
log.info('making log long to prevent backfill')
for i in non_divergent:
- manager.set_config(i, osd_min_pg_log_entries=100000)
+ ctx.manager.set_config(i, osd_min_pg_log_entries=100000)
# write 1 non-divergent object (ensure that old divergent one is divergent)
log.info('writing non-divergent object existing_1')
rados(ctx, mon, ['-p', 'foo', 'put', 'existing_1', dummyfile2])
- manager.wait_for_recovery()
+ ctx.manager.wait_for_recovery()
# ensure no recovery
log.info('delay recovery')
for i in non_divergent:
- manager.set_config(i, osd_recovery_delay_start=100000)
+ ctx.manager.set_config(i, osd_recovery_delay_start=100000)
# bring in our divergent friend
log.info("revive divergent %d", divergent)
- manager.revive_osd(divergent)
+ ctx.manager.revive_osd(divergent)
- while len(manager.get_osd_status()['up']) < 3:
+ while len(ctx.manager.get_osd_status()['up']) < 3:
time.sleep(10)
log.info('delay recovery divergent')
- manager.set_config(divergent, osd_recovery_delay_start=100000)
+ ctx.manager.set_config(divergent, osd_recovery_delay_start=100000)
log.info('mark divergent in')
- manager.mark_in_osd(divergent)
+ ctx.manager.mark_in_osd(divergent)
log.info('wait for peering')
rados(ctx, mon, ['-p', 'foo', 'put', 'foo', dummyfile])
log.info("killing divergent %d", divergent)
- manager.kill_osd(divergent)
+ ctx.manager.kill_osd(divergent)
log.info("reviving divergent %d", divergent)
- manager.revive_osd(divergent)
+ ctx.manager.revive_osd(divergent)
log.info('allowing recovery')
for i in non_divergent:
- manager.set_config(i, osd_recovery_delay_start=0)
+ ctx.manager.set_config(i, osd_recovery_delay_start=0)
log.info('reading existing_0')
exit_status = rados(ctx, mon,
config = {}
assert isinstance(config, dict), \
'osd_failsafe_enospc task only accepts a dict for configuration'
- first_mon = teuthology.get_first_mon(ctx, config)
- (mon,) = ctx.cluster.only(first_mon).remotes.iterkeys()
-
- manager = ceph_manager.CephManager(
- mon,
- ctx=ctx,
- logger=log.getChild('ceph_manager'),
- )
- ctx.manager = manager
# Give 2 seconds for injectargs + osd_op_complaint_time (30) + 2 * osd_heartbeat_interval (6) + 6 padding
sleep_time = 50
dummyfile2 = '/etc/resolv.conf'
# create 1 pg pool with 1 rep which can only be on osd.0
- osds = manager.get_osd_dump()
+ osds = ctx.manager.get_osd_dump()
for osd in osds:
if osd['osd'] != 0:
- manager.mark_out_osd(osd['osd'])
+ ctx.manager.mark_out_osd(osd['osd'])
log.info('creating pool foo')
- manager.create_pool("foo")
- manager.raw_cluster_cmd('osd', 'pool', 'set', 'foo', 'size', '1')
+ ctx.manager.create_pool("foo")
+ ctx.manager.raw_cluster_cmd('osd', 'pool', 'set', 'foo', 'size', '1')
# State NONE -> NEAR
log.info('1. Verify warning messages when exceeding nearfull_ratio')
wait=False,
)
- manager.raw_cluster_cmd('tell', 'osd.0', 'injectargs', '--osd_failsafe_nearfull_ratio .00001')
+ ctx.manager.raw_cluster_cmd('tell', 'osd.0', 'injectargs', '--osd_failsafe_nearfull_ratio .00001')
time.sleep(sleep_time)
proc.stdin.close() # causes daemon-helper send SIGKILL to ceph -w
wait=False,
)
- manager.raw_cluster_cmd('tell', 'osd.0', 'injectargs', '--osd_failsafe_full_ratio .00001')
+ ctx.manager.raw_cluster_cmd('tell', 'osd.0', 'injectargs', '--osd_failsafe_full_ratio .00001')
time.sleep(sleep_time)
proc.stdin.close() # causes daemon-helper send SIGKILL to ceph -w
assert ret != 0, 'Expected write failure but it succeeded with exit status 0'
# Put back default
- manager.raw_cluster_cmd('tell', 'osd.0', 'injectargs', '--osd_failsafe_full_ratio .97')
+ ctx.manager.raw_cluster_cmd('tell', 'osd.0', 'injectargs', '--osd_failsafe_full_ratio .97')
time.sleep(10)
# State FULL -> NEAR
count = len(filter(lambda line: '[ERR] OSD full dropping all updates' in line, lines))
assert count == 0, 'Incorrect number of error messages expected 0 got %d' % count
- manager.raw_cluster_cmd('tell', 'osd.0', 'injectargs', '--osd_failsafe_nearfull_ratio .90')
+ ctx.manager.raw_cluster_cmd('tell', 'osd.0', 'injectargs', '--osd_failsafe_nearfull_ratio .90')
time.sleep(10)
# State NONE -> FULL
wait=False,
)
- manager.raw_cluster_cmd('tell', 'osd.0', 'injectargs', '--osd_failsafe_full_ratio .00001')
+ ctx.manager.raw_cluster_cmd('tell', 'osd.0', 'injectargs', '--osd_failsafe_full_ratio .00001')
time.sleep(sleep_time)
proc.stdin.close() # causes daemon-helper send SIGKILL to ceph -w
# State FULL -> NONE
log.info('7. Verify no messages settings back to default')
- manager.raw_cluster_cmd('tell', 'osd.0', 'injectargs', '--osd_failsafe_full_ratio .97')
+ ctx.manager.raw_cluster_cmd('tell', 'osd.0', 'injectargs', '--osd_failsafe_full_ratio .97')
time.sleep(10)
proc = mon.run(
log.info('Test Passed')
# Bring all OSDs back in
- manager.remove_pool("foo")
+ ctx.manager.remove_pool("foo")
for osd in osds:
if osd['osd'] != 0:
- manager.mark_in_osd(osd['osd'])
+ ctx.manager.mark_in_osd(osd['osd'])