roles:
-- [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3]
-- [mon.b, mon.c, mgr.x, mds.a-s, osd.4, osd.5, osd.6, osd.7]
+- [mon.a, mgr.y, mds.a, mds.x-s, osd.0, osd.1, osd.2, osd.3]
+- [mon.b, mon.c, mgr.x, mds.y-s, osd.4, osd.5, osd.6, osd.7]
- [client.0]
openstack:
- volumes: # attached to each instance
--- /dev/null
+roles:
+- [mon.a, mon.b, mon.c, mgr.x, mgr.y, mds.a, mds.b, mds.c, osd.0, osd.1, osd.2, osd.3]
+- [client.0]
+- [client.1]
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 10 # GB
roles:
-- [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3]
-- [mon.b, mon.c, mgr.x, mds.a-s, osd.4, osd.5, osd.6, osd.7]
+- [mon.a, mgr.y, mds.a, mds.x-s, osd.0, osd.1, osd.2, osd.3]
+- [mon.b, mon.c, mgr.x, mds.y-s, osd.4, osd.5, osd.6, osd.7]
- [client.0]
- [client.1]
openstack:
roles:
-- [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3]
-- [mon.b, mon.c, mgr.x, mds.a-s, osd.4, osd.5, osd.6, osd.7]
+- [mon.a, mgr.y, mds.a, mds.x-s, osd.0, osd.1, osd.2, osd.3]
+- [mon.b, mon.c, mgr.x, mds.y-s, osd.4, osd.5, osd.6, osd.7]
- [client.0]
- [client.1]
- [client.2]
--- /dev/null
+../../../../
\ No newline at end of file
--- /dev/null
+../.qa/
\ No newline at end of file
--- /dev/null
+../../../../../cephfs/objectstore-ec/bluestore.yaml
\ No newline at end of file
--- /dev/null
+../.qa/
\ No newline at end of file
--- /dev/null
+.qa/cephfs/clusters/1-mds-2-client-micro.yaml
\ No newline at end of file
--- /dev/null
+.qa/cephfs/conf/
\ No newline at end of file
--- /dev/null
+../.qa/
\ No newline at end of file
--- /dev/null
+.qa/cephfs/overrides/frag_enable.yaml
\ No newline at end of file
--- /dev/null
+overrides:
+ ceph:
+ max_mds: 1
--- /dev/null
+overrides:
+ ceph:
+ max_mds: 2
--- /dev/null
+.qa/cephfs/overrides/whitelist_health.yaml
\ No newline at end of file
--- /dev/null
+.qa/cephfs/overrides/whitelist_wrongly_marked_down.yaml
\ No newline at end of file
--- /dev/null
+../.qa/
\ No newline at end of file
--- /dev/null
+meta:
+- desc: |
+ install ceph/luminous latest
+tasks:
+- install:
+ branch: luminous
+- print: "**** done installing luminous"
+- ceph:
+ log-whitelist:
+ - overall HEALTH_
+ - \(FS_
+ - \(MDS_
+ - \(OSD_
+ - \(MON_DOWN\)
+ - \(CACHE_POOL_
+ - \(POOL_
+ - \(MGR_DOWN\)
+ - \(PG_
+ - \(SMALLER_PGP_NUM\)
+ - Monitor daemon marked osd
+ - Behind on trimming
+ - Manager daemon
+ conf:
+ global:
+ mon warn on pool no app: false
+- exec:
+ osd.0:
+ - ceph osd require-osd-release luminous
+ - ceph osd set-require-min-compat-client luminous
+- print: "**** done ceph"
--- /dev/null
+tasks:
+- ceph-fuse:
+- print: "**** done luminous client"
+- workunit:
+ clients:
+ all:
+ - suites/fsstress.sh
+- print: "**** done fsstress"
--- /dev/null
+tasks:
+- mds_pre_upgrade:
+- print: "**** done mds pre-upgrade sequence"
+- install.upgrade:
+ mon.a:
+ mon.b:
+- print: "**** done install.upgrade both hosts"
+- ceph.stop: [mds.*]
+- ceph.restart:
+ daemons: [mon.*, mgr.*, osd.*, mds.*]
+ mon-health-to-clog: false
+- print: "**** done ceph.restart"
--- /dev/null
+overrides:
+ ceph:
+ log-whitelist:
+ - missing required features
+tasks:
+- exec:
+ mon.a:
+ - ceph fs dump --format=json-pretty
+ - ceph fs set cephfs min_compat_client mimic
+- fs.clients_evicted:
--- /dev/null
+tasks:
+- workunit:
+ clients:
+ all:
+ - suites/fsstress.sh
+- print: "**** done fsstress"
--- /dev/null
+../.qa
\ No newline at end of file
--- /dev/null
+../../../../../cephfs/objectstore-ec/bluestore.yaml
\ No newline at end of file
--- /dev/null
+../.qa/
\ No newline at end of file
--- /dev/null
+.qa/cephfs/clusters/1-mds-2-client-micro.yaml
\ No newline at end of file
--- /dev/null
+.qa/cephfs/conf/
\ No newline at end of file
--- /dev/null
+../.qa/
\ No newline at end of file
--- /dev/null
+.qa/cephfs/overrides/frag_enable.yaml
\ No newline at end of file
--- /dev/null
+overrides:
+ ceph:
+ max_mds: 1
--- /dev/null
+overrides:
+ ceph:
+ max_mds: 2
--- /dev/null
+.qa/cephfs/overrides/whitelist_health.yaml
\ No newline at end of file
--- /dev/null
+.qa/cephfs/overrides/whitelist_wrongly_marked_down.yaml
\ No newline at end of file
--- /dev/null
+../.qa/
\ No newline at end of file
--- /dev/null
+meta:
+- desc: |
+ install ceph/luminous latest
+tasks:
+- install:
+ branch: luminous
+- print: "**** done installing luminous"
+- ceph:
+ log-whitelist:
+ - overall HEALTH_
+ - \(FS_
+ - \(MDS_
+ - \(OSD_
+ - \(MON_DOWN\)
+ - \(CACHE_POOL_
+ - \(POOL_
+ - \(MGR_DOWN\)
+ - \(PG_
+ - \(SMALLER_PGP_NUM\)
+ - Monitor daemon marked osd
+ - Behind on trimming
+ - Manager daemon
+ conf:
+ global:
+ mon warn on pool no app: false
+- exec:
+ osd.0:
+ - ceph osd require-osd-release luminous
+ - ceph osd set-require-min-compat-client luminous
+- print: "**** done ceph"
--- /dev/null
+nuke-on-error: false
+overrides:
+ nuke-on-error: false
+tasks:
+- ceph-fuse:
+- print: "**** done luminous client"
+#- workunit:
+# clients:
+# all:
+# - suites/fsstress.sh
+- print: "**** done fsstress"
--- /dev/null
+tasks:
+- mds_pre_upgrade:
+- print: "**** done mds pre-upgrade sequence"
+- install.upgrade:
+ mon.a:
+ mon.b:
+- print: "**** done install.upgrade both hosts"
+- ceph.stop: [mds.*]
+- ceph.restart:
+ daemons: [mon.*, mgr.*, osd.*, mds.*]
+ mon-health-to-clog: false
+- print: "**** done ceph.restart"
--- /dev/null
+tasks:
+- install.upgrade:
+ client.0:
+- print: "**** done install.upgrade on client.0"
+- ceph-fuse:
+ client.0:
+ mounted: false
+ client.1:
+ skip: true
+- ceph-fuse:
+ client.0:
+ client.1:
+ skip: true
+- print: "**** done remount client"
--- /dev/null
+overrides:
+ ceph:
+ log-whitelist:
+ - missing required features
+tasks:
+- exec:
+ mon.a:
+ - ceph fs dump --format=json-pretty
+ - ceph fs set cephfs min_compat_client mimic
+- fs.clients_evicted:
+ clients:
+ client.0: False
+ client.1: True
--- /dev/null
+tasks:
+- workunit:
+ clients:
+ client.0:
+ - suites/fsstress.sh
+- print: "**** done fsstress"
from teuthology import misc as teuthology
from cephfs.fuse_mount import FuseMount
-from tasks.cephfs.filesystem import MDSCluster
from tasks.cephfs.filesystem import Filesystem
log = logging.getLogger(__name__)
:param ctx: Context
:param config: Configuration
"""
- log.info('Mounting ceph-fuse clients...')
+ log.info('Running ceph_fuse task...')
testdir = teuthology.get_testdir(ctx)
+ log.info("config is {}".format(str(config)))
config = get_client_configs(ctx, config)
+ log.info("new config is {}".format(str(config)))
# List clients we will configure mounts for, default is all clients
clients = list(teuthology.get_clients(ctx=ctx, roles=filter(lambda x: 'client.' in x, config.keys())))
all_mounts = getattr(ctx, 'mounts', {})
mounted_by_me = {}
-
- log.info('Wait for MDS to reach steady state...')
- mds_cluster = MDSCluster(ctx)
- status = mds_cluster.status()
- for filesystem in status.get_filesystems():
- fs = Filesystem(ctx, fscid=filesystem['id'])
- fs.wait_for_daemons()
- log.info('Ready to start ceph-fuse...')
+ skipped = {}
# Construct any new FuseMount instances
for id_, remote in clients:
if client_config is None:
client_config = {}
+ skip = client_config.get("skip", False)
+ if skip:
+ skipped[id_] = skip
+ continue
+
if id_ not in all_mounts:
- fuse_mount = FuseMount(client_config, testdir, id_, remote)
+ fuse_mount = FuseMount(ctx, client_config, testdir, id_, remote)
all_mounts[id_] = fuse_mount
else:
# Catch bad configs where someone has e.g. tried to use ceph-fuse and kcephfs for the same client
ctx.mounts = all_mounts
# Mount any clients we have been asked to (default to mount all)
+ log.info('Mounting ceph-fuse clients...')
for mount in mounted_by_me.values():
mount.mount()
mount.wait_until_mounted()
# Umount any pre-existing clients that we have not been asked to mount
- for client_id in set(all_mounts.keys()) - set(mounted_by_me.keys()):
+ for client_id in set(all_mounts.keys()) - set(mounted_by_me.keys()) - set(skipped.keys()):
mount = all_mounts[client_id]
if mount.is_mounted():
mount.umount_wait()
return self.json_asok(command, 'mds', mds_id)
- def rank_asok(self, command, rank=0):
- info = self.get_rank(rank=rank)
+ def rank_asok(self, command, rank=0, status=None):
+ info = self.get_rank(rank=rank, status=status)
return self.json_asok(command, 'mds', info['name'])
def read_cache(self, path, depth=None):
from teuthology.orchestra import run
from teuthology.orchestra.run import CommandFailedError
from .mount import CephFSMount
+from tasks.cephfs.filesystem import Filesystem
log = logging.getLogger(__name__)
class FuseMount(CephFSMount):
- def __init__(self, client_config, test_dir, client_id, client_remote):
- super(FuseMount, self).__init__(test_dir, client_id, client_remote)
+ def __init__(self, ctx, client_config, test_dir, client_id, client_remote):
+ super(FuseMount, self).__init__(ctx, test_dir, client_id, client_remote)
self.client_config = client_config if client_config else {}
self.fuse_daemon = None
self._fuse_conn = None
+ self.id = None
+ self.inst = None
+ self.addr = None
def mount(self, mount_path=None, mount_fs_name=None):
+ self.setupfs(name=mount_fs_name)
+
try:
return self._mount(mount_path, mount_fs_name)
except RuntimeError:
else:
self._fuse_conn = new_conns[0]
+ status = self.admin_socket(['status'])
+ self.id = status['id']
+ try:
+ self.inst = status['inst_str']
+ self.addr = status['addr_str']
+ except KeyError as e:
+ sessions = self.fs.rank_asok(['session', 'ls'])
+ for s in sessions:
+ if s['id'] == self.id:
+ self.inst = s['inst']
+ self.addr = self.inst.split()[1]
+ if self.inst is None:
+ raise RuntimeError("cannot find client session")
+
def is_mounted(self):
proc = self.client_remote.run(
args=[
assert not self.is_mounted()
self._fuse_conn = None
+ self.id = None
+ self.inst = None
+ self.addr = None
def umount_wait(self, force=False, require_clean=False, timeout=900):
"""
"""
return self.admin_socket(['mds_sessions'])['id']
+ def get_global_inst(self):
+ """
+ Look up the CephFS client instance for this mount
+ """
+ return self.inst
+
+ def get_global_addr(self):
+ """
+ Look up the CephFS client addr for this mount
+ """
+ return self.addr
+
def get_client_pid(self):
"""
return pid of ceph-fuse process
class KernelMount(CephFSMount):
- def __init__(self, mons, test_dir, client_id, client_remote,
+ def __init__(self, ctx, mons, test_dir, client_id, client_remote,
ipmi_user, ipmi_password, ipmi_domain):
- super(KernelMount, self).__init__(test_dir, client_id, client_remote)
+ super(KernelMount, self).__init__(ctx, test_dir, client_id, client_remote)
self.mons = mons
self.mounted = False
)
def mount(self, mount_path=None, mount_fs_name=None):
+ self.setupfs(name=mount_fs_name)
+
log.info('Mounting kclient client.{id} at {remote} {mnt}...'.format(
id=self.client_id, remote=self.client_remote, mnt=self.mountpoint))
from StringIO import StringIO
from teuthology.orchestra import run
from teuthology.orchestra.run import CommandFailedError, ConnectionLostError
+from tasks.cephfs.filesystem import Filesystem
log = logging.getLogger(__name__)
class CephFSMount(object):
- def __init__(self, test_dir, client_id, client_remote):
+ def __init__(self, ctx, test_dir, client_id, client_remote):
"""
:param test_dir: Global teuthology test dir
:param client_id: Client ID, the 'foo' in client.foo
:param client_remote: Remote instance for the host where client will run
"""
+ self.ctx = ctx
self.test_dir = test_dir
self.client_id = client_id
self.client_remote = client_remote
self.mountpoint_dir_name = 'mnt.{id}'.format(id=self.client_id)
+ self.fs = None
self.test_files = ['a', 'b', 'c']
def is_mounted(self):
raise NotImplementedError()
+ def setupfs(self, name=None):
+ self.fs = Filesystem(self.ctx, name=name)
+ log.info('Wait for MDS to reach steady state...')
+ self.fs.wait_for_daemons()
+ log.info('Ready to start {}...'.format(type(self).__name__))
+
def mount(self, mount_path=None, mount_fs_name=None):
raise NotImplementedError()
finally:
self.umount_wait()
+ def is_blacklisted(self):
+ addr = self.get_global_addr()
+ blacklist = json.loads(self.fs.mon_manager.raw_cluster_cmd("osd", "blacklist", "ls", "--format=json"))
+ for b in blacklist:
+ if addr == b["addr"]:
+ return True
+ return False
+
def create_files(self):
assert(self.is_mounted())
def get_global_id(self):
raise NotImplementedError()
+ def get_global_inst(self):
+ raise NotImplementedError()
+
+ def get_global_addr(self):
+ raise NotImplementedError()
+
def get_osd_epoch(self):
raise NotImplementedError()
--- /dev/null
+"""
+CephFS sub-tasks.
+"""
+
+import contextlib
+import logging
+import re
+import time
+
+from tasks.cephfs.filesystem import Filesystem
+
+log = logging.getLogger(__name__)
+
+def clients_evicted(ctx, config):
+ """
+ Check clients are evicted, unmount (cleanup) if so.
+ """
+
+ if config is None:
+ config = {}
+ assert isinstance(config, dict), \
+ 'task only accepts a dict for configuration'
+
+ clients = config.get('clients')
+
+ if clients is None:
+ clients = {("client."+client_id): True for client_id in ctx.mounts}
+
+ log.info("clients is {}".format(str(clients)))
+
+ fs = Filesystem(ctx)
+ status = fs.status()
+
+ has_session = set()
+ mounts = {}
+ for client in clients:
+ client_id = re.match("^client.([0-9]+)$", client).groups(1)[0]
+ mounts[client] = ctx.mounts.get(client_id)
+
+ for rank in fs.get_ranks(status=status):
+ ls = fs.rank_asok(['session', 'ls'], rank=rank['rank'], status=status)
+ for session in ls:
+ for client, evicted in clients.viewitems():
+ mount = mounts.get(client)
+ if mount is not None:
+ global_id = mount.get_global_id()
+ if session['id'] == global_id:
+ if evicted:
+ raise RuntimeError("client still has session: {}".format(str(session)))
+ else:
+ log.info("client {} has a session with MDS {}.{}".format(client, fs.id, rank['rank']))
+ has_session.add(client)
+
+ no_session = set(clients) - has_session
+ should_assert = False
+ for client, evicted in clients.viewitems():
+ mount = mounts.get(client)
+ if mount is not None:
+ if evicted:
+ log.info("confirming client {} is blacklisted".format(client))
+ assert mount.is_blacklisted()
+ elif client in no_session:
+ log.info("client {} should not be evicted but has no session with an MDS".format(client))
+ mount.is_blacklisted() # for debugging
+ should_assert = True
+ if should_assert:
+ raise RuntimeError("some clients which should not be evicted have no session with an MDS?")
continue
kernel_mount = KernelMount(
+ ctx,
mons,
test_dir,
id_,
projects / ceph-ci.git / commitdiff