--- /dev/null
+"""
+Special regression test for tracker #11184
+
+Synopsis: osd/SnapMapper.cc: 282: FAILED assert(check(oid))
+
+This is accomplished by moving a pg that wasn't part of split and still include
+divergent priors.
+"""
+import logging
+import time
+from cStringIO import StringIO
+
+from teuthology import misc as teuthology
+from util.rados import rados
+import os
+
+
+log = logging.getLogger(__name__)
+
+
+def task(ctx, config):
+ """
+ Test handling of divergent entries during export / import
+ to regression test tracker #11184
+
+ overrides:
+ ceph:
+ conf:
+ osd:
+ debug osd: 5
+
+ Requires 3 osds on a single test node.
+ """
+ if config is None:
+ config = {}
+ assert isinstance(config, dict), \
+ 'divergent_priors task only accepts a dict for configuration'
+
+ while len(ctx.manager.get_osd_status()['up']) < 3:
+ time.sleep(10)
+ ctx.manager.raw_cluster_cmd('tell', 'osd.0', 'flush_pg_stats')
+ ctx.manager.raw_cluster_cmd('tell', 'osd.1', 'flush_pg_stats')
+ ctx.manager.raw_cluster_cmd('tell', 'osd.2', 'flush_pg_stats')
+ ctx.manager.raw_cluster_cmd('osd', 'set', 'noout')
+ ctx.manager.raw_cluster_cmd('osd', 'set', 'noin')
+ ctx.manager.raw_cluster_cmd('osd', 'set', 'nodown')
+ ctx.manager.wait_for_clean()
+
+ # something that is always there
+ dummyfile = '/etc/fstab'
+ dummyfile2 = '/etc/resolv.conf'
+ testdir = teuthology.get_testdir(ctx)
+
+ # create 1 pg pool
+ log.info('creating foo')
+ ctx.manager.raw_cluster_cmd('osd', 'pool', 'create', 'foo', '1')
+
+ osds = [0, 1, 2]
+ for i in osds:
+ ctx.manager.set_config(i, osd_min_pg_log_entries=10)
+ ctx.manager.set_config(i, osd_max_pg_log_entries=10)
+ ctx.manager.set_config(i, osd_pg_log_trim_min=5)
+
+ # determine primary
+ divergent = ctx.manager.get_pg_primary('foo', 0)
+ log.info("primary and soon to be divergent is %d", divergent)
+ non_divergent = list(osds)
+ non_divergent.remove(divergent)
+
+ log.info('writing initial objects')
+ first_mon = teuthology.get_first_mon(ctx, config)
+ (mon,) = ctx.cluster.only(first_mon).remotes.iterkeys()
+ # write 100 objects
+ for i in range(100):
+ rados(ctx, mon, ['-p', 'foo', 'put', 'existing_%d' % i, dummyfile])
+
+ ctx.manager.wait_for_clean()
+
+ # blackhole non_divergent
+ log.info("blackholing osds %s", str(non_divergent))
+ for i in non_divergent:
+ ctx.manager.set_config(i, filestore_blackhole=1)
+
+ DIVERGENT_WRITE = 5
+ DIVERGENT_REMOVE = 5
+ # Write some soon to be divergent
+ log.info('writing divergent objects')
+ for i in range(DIVERGENT_WRITE):
+ rados(ctx, mon, ['-p', 'foo', 'put', 'existing_%d' % i,
+ dummyfile2], wait=False)
+ # Remove some soon to be divergent
+ log.info('remove divergent objects')
+ for i in range(DIVERGENT_REMOVE):
+ rados(ctx, mon, ['-p', 'foo', 'rm',
+ 'existing_%d' % (i + DIVERGENT_WRITE)], wait=False)
+ time.sleep(10)
+ mon.run(
+ args=['killall', '-9', 'rados'],
+ wait=True,
+ check_status=False)
+
+ # kill all the osds but leave divergent in
+ log.info('killing all the osds')
+ for i in osds:
+ ctx.manager.kill_osd(i)
+ for i in osds:
+ ctx.manager.mark_down_osd(i)
+ for i in non_divergent:
+ ctx.manager.mark_out_osd(i)
+
+ # bring up non-divergent
+ log.info("bringing up non_divergent %s", str(non_divergent))
+ for i in non_divergent:
+ ctx.manager.revive_osd(i)
+ for i in non_divergent:
+ ctx.manager.mark_in_osd(i)
+
+ # write 1 non-divergent object (ensure that old divergent one is divergent)
+ objname = "existing_%d" % (DIVERGENT_WRITE + DIVERGENT_REMOVE)
+ log.info('writing non-divergent object ' + objname)
+ rados(ctx, mon, ['-p', 'foo', 'put', objname, dummyfile2])
+
+ ctx.manager.wait_for_recovery()
+
+ # ensure no recovery of up osds first
+ log.info('delay recovery')
+ for i in non_divergent:
+ ctx.manager.wait_run_admin_socket(
+ 'osd', i, ['set_recovery_delay', '100000'])
+
+ # bring in our divergent friend
+ log.info("revive divergent %d", divergent)
+ ctx.manager.raw_cluster_cmd('osd', 'set', 'noup')
+ ctx.manager.revive_osd(divergent)
+
+ log.info('delay recovery divergent')
+ ctx.manager.wait_run_admin_socket(
+ 'osd', divergent, ['set_recovery_delay', '100000'])
+
+ ctx.manager.raw_cluster_cmd('osd', 'unset', 'noup')
+ while len(ctx.manager.get_osd_status()['up']) < 3:
+ time.sleep(10)
+
+ log.info('wait for peering')
+ rados(ctx, mon, ['-p', 'foo', 'put', 'foo', dummyfile])
+
+ # At this point the divergent_priors should have been detected
+
+ log.info("killing divergent %d", divergent)
+ ctx.manager.kill_osd(divergent)
+
+ # Split pgs for pool foo
+ ctx.manager.raw_cluster_cmd('osd', 'pool', 'set', 'foo', 'pg_num', '2')
+ time.sleep(5)
+
+ # Export a pg
+ (exp_remote,) = ctx.\
+ cluster.only('osd.{o}'.format(o=divergent)).remotes.iterkeys()
+ FSPATH = ctx.manager.get_filepath()
+ JPATH = os.path.join(FSPATH, "journal")
+ prefix = ("sudo adjust-ulimits ceph-objectstore-tool "
+ "--data-path {fpath} --journal-path {jpath} "
+ "--log-file="
+ "/var/log/ceph/objectstore_tool.$$.log ".
+ format(fpath=FSPATH, jpath=JPATH))
+ pid = os.getpid()
+ expfile = os.path.join(testdir, "exp.{pid}.out".format(pid=pid))
+ cmd = ((prefix + "--op export --pgid 1.0 --file {file}").
+ format(id=divergent, file=expfile))
+ proc = exp_remote.run(args=cmd, wait=True,
+ check_status=False, stdout=StringIO())
+ assert proc.exitstatus == 0
+
+ # Remove the same pg that was exported
+ cmd = ((prefix + "--op remove --pgid 1.0").
+ format(id=divergent, file=expfile))
+ proc = exp_remote.run(args=cmd, wait=True,
+ check_status=False, stdout=StringIO())
+ assert proc.exitstatus == 0
+
+ # Kill one of non-divergent OSDs
+ log.info('killing osd.%d' % non_divergent[1])
+ ctx.manager.kill_osd(non_divergent[1])
+ ctx.manager.mark_down_osd(non_divergent[1])
+ # ctx.manager.mark_out_osd(non_divergent[1])
+
+ cmd = ((prefix + "--op import --file {file}").
+ format(id=non_divergent[1], file=expfile))
+ proc = exp_remote.run(args=cmd, wait=True,
+ check_status=False, stdout=StringIO())
+ assert proc.exitstatus == 0
+
+ # bring in our divergent friend and other node
+ log.info("revive divergent %d", divergent)
+ ctx.manager.revive_osd(divergent)
+ ctx.manager.mark_in_osd(divergent)
+ log.info("revive %d", non_divergent[1])
+ ctx.manager.revive_osd(non_divergent[1])
+
+ while len(ctx.manager.get_osd_status()['up']) < 3:
+ time.sleep(10)
+
+ log.info('delay recovery divergent')
+ ctx.manager.set_config(divergent, osd_recovery_delay_start=100000)
+ log.info('mark divergent in')
+ ctx.manager.mark_in_osd(divergent)
+
+ log.info('wait for peering')
+ rados(ctx, mon, ['-p', 'foo', 'put', 'foo', dummyfile])
+
+ log.info("killing divergent %d", divergent)
+ ctx.manager.kill_osd(divergent)
+ log.info("reviving divergent %d", divergent)
+ ctx.manager.revive_osd(divergent)
+ time.sleep(3)
+
+ log.info('allowing recovery')
+ # Set osd_recovery_delay_start back to 0 and kick the queue
+ for i in osds:
+ ctx.manager.raw_cluster_cmd('tell', 'osd.%d' % i, 'debug',
+ 'kick_recovery_wq', ' 0')
+
+ log.info('reading divergent objects')
+ for i in range(DIVERGENT_WRITE + DIVERGENT_REMOVE):
+ exit_status = rados(ctx, mon, ['-p', 'foo', 'get', 'existing_%d' % i,
+ '-o', '/tmp/existing'])
+ assert exit_status is 0
+
+ (remote,) = ctx.\
+ cluster.only('osd.{o}'.format(o=divergent)).remotes.iterkeys()
+ msg = "dirty_divergent_priors: true, divergent_priors: %d" \
+ % (DIVERGENT_WRITE + DIVERGENT_REMOVE)
+ cmd = 'grep "{msg}" /var/log/ceph/ceph-osd.{osd}.log'\
+ .format(msg=msg, osd=divergent)
+ proc = remote.run(args=cmd, wait=True, check_status=False)
+ assert proc.exitstatus == 0
+
+ cmd = 'rm {file}'.format(file=expfile)
+ remote.run(args=cmd, wait=True)
+ log.info("success")
projects / ceph.git / commitdiff