from six import StringIO
from textwrap import dedent
import os
+import re
+from IPy import IP
from teuthology.orchestra import run
from teuthology.orchestra.run import CommandFailedError, ConnectionLostError
from tasks.cephfs.filesystem import Filesystem
+import platform
log = logging.getLogger(__name__)
class CephFSMount(object):
- def __init__(self, ctx, test_dir, client_id, client_remote):
+ def __init__(self, ctx, test_dir, client_id, client_remote, brxnet):
"""
:param test_dir: Global teuthology test dir
:param client_id: Client ID, the 'foo' in client.foo
self.mountpoint_dir_name = 'mnt.{id}'.format(id=self.client_id)
self._mountpoint = None
self.fs = None
+ self._netns_name = None
+ self.nsid = -1
+ if brxnet is None:
+ self.ceph_brx_net = '192.168.0.0/16'
+ else:
+ self.ceph_brx_net = brxnet
self.test_files = ['a', 'b', 'c']
self.background_procs = []
+ # On Centos/Redhat 8 the 'brctl' has been deprecated. But the
+ # 'nmcli' in ubuntu 18.04 is buggy to setup the network bridge
+ # and there is no workaround, will continue to use the 'brctl'
+ args = ["bash", "-c",
+ "cat /etc/os-release"]
+ p = self.client_remote.run(args=args, stderr=BytesIO(),
+ stdout=BytesIO(), timeout=(5*60))
+ distro = re.findall(r'NAME="Ubuntu"', p.stdout.getvalue())
+ version = re.findall(r'VERSION_ID="18.04"', p.stdout.getvalue())
+ self.use_brctl = len(distro) is not 0 and len(version) is not 0
+
+ def _parse_netns_name(self):
+ self._netns_name = '-'.join(["ceph-ns",
+ re.sub(r'/+', "-", self.mountpoint)])
+
@property
def mountpoint(self):
if self._mountpoint == None:
if not isinstance(path, str):
raise RuntimeError('path should be of str type.')
self._mountpoint = path
+ self._parse_netns_name()
+
+ @property
+ def netns_name(self):
+ if self._netns_name == None:
+ self._parse_netns_name()
+ return self._netns_name
+
+ @netns_name.setter
+ def netns_name(self, name):
+ if not isinstance(path, str):
+ raise RuntimeError('path should be of str type.')
+ self._netns_name = name
def is_mounted(self):
raise NotImplementedError()
self.fs.wait_for_daemons()
log.info('Ready to start {}...'.format(type(self).__name__))
+ def _bringup_network_manager_service(self):
+ args = ["sudo", "bash", "-c",
+ "systemctl start NetworkManager"]
+ self.client_remote.run(args=args, timeout=(5*60))
+
+ def _setup_brx_and_nat(self):
+ # The ip for ceph-brx should be
+ ip = IP(self.ceph_brx_net)[-2]
+ mask = self.ceph_brx_net.split('/')[1]
+ brd = IP(self.ceph_brx_net).broadcast()
+
+ brx = self.client_remote.run(args=['ip', 'addr'], stderr=BytesIO(),
+ stdout=BytesIO(), timeout=(5*60))
+ brx = re.findall(r'inet .* ceph-brx', brx.stdout.getvalue())
+ if brx:
+ # If the 'ceph-brx' already exists, then check whether
+ # the new net is conflicting with it
+ _ip, _mask = brx[0].split()[1].split('/', 1)
+ if _ip != "{}".format(ip) or _mask != mask:
+ raise RuntimeError("Conflict with existing ceph-brx {0}, new {1}/{2}".format(brx[0].split()[1], ip, mask))
+ return
+
+ log.info("Setuping the 'ceph-brx' with {0}/{1}".format(ip, mask))
+
+ # Setup the ceph-brx and always use the last valid IP
+ if self.use_brctl == True:
+ args = ["sudo", "bash", "-c", "brctl addbr ceph-brx"]
+ self.client_remote.run(args=args, timeout=(5*60))
+ args = ["sudo", "bash", "-c", "ip link set ceph-brx up"]
+ self.client_remote.run(args=args, timeout=(5*60))
+ args = ["sudo", "bash", "-c",
+ "ip addr add {0}/{1} brd {2} dev ceph-brx".format(ip, mask, brd)]
+ self.client_remote.run(args=args, timeout=(5*60))
+ else:
+ self._bringup_network_manager_service()
+ args = ["sudo", "bash", "-c",
+ "nmcli connection add type bridge con-name ceph-brx ifname ceph-brx stp no"]
+ self.client_remote.run(args=args, timeout=(5*60))
+ args = ["sudo", "bash", "-c",
+ "nmcli connection modify ceph-brx ipv4.addresses {0}/{1} ipv4.method manual".format(ip, mask)]
+ self.client_remote.run(args=args, timeout=(5*60))
+ args = ["sudo", "bash", "-c", "nmcli connection up ceph-brx"]
+ self.client_remote.run(args=args, timeout=(5*60))
+
+ # Save the ip_forward
+ self.client_remote.run(args=['touch', '/tmp/python-ceph-brx'],
+ timeout=(5*60))
+ p = self.client_remote.run(args=['cat', '/proc/sys/net/ipv4/ip_forward'],
+ stderr=BytesIO(), stdout=BytesIO(),
+ timeout=(5*60))
+ val = p.stdout.getvalue().strip()
+ args = ["sudo", "bash", "-c",
+ "echo {} > /tmp/python-ceph-brx".format(val)]
+ self.client_remote.run(args=args, timeout=(5*60))
+ args = ["sudo", "bash", "-c",
+ "echo 1 > /proc/sys/net/ipv4/ip_forward"]
+ self.client_remote.run(args=args, timeout=(5*60))
+
+ # Setup the NAT
+ p = self.client_remote.run(args=['route'], stderr=BytesIO(),
+ stdout=BytesIO(), timeout=(5*60))
+ p = re.findall(r'default .*', p.stdout.getvalue())
+ if p == False:
+ raise RuntimeError("No default gw found")
+ gw = p[0].split()[7]
+ args = ["sudo", "bash", "-c",
+ "iptables -A FORWARD -o {0} -i ceph-brx -j ACCEPT".format(gw)]
+ self.client_remote.run(args=args, timeout=(5*60))
+ args = ["sudo", "bash", "-c",
+ "iptables -A FORWARD -i {0} -o ceph-brx -j ACCEPT".format(gw)]
+ self.client_remote.run(args=args, timeout=(5*60))
+ args = ["sudo", "bash", "-c",
+ "iptables -t nat -A POSTROUTING -s {0}/{1} -o {2} -j MASQUERADE".format(ip, mask, gw)]
+ self.client_remote.run(args=args, timeout=(5*60))
+
+ def _setup_netns(self):
+ p = self.client_remote.run(args=['ip', 'netns', 'list'],
+ stderr=BytesIO(), stdout=BytesIO(),
+ timeout=(5*60))
+ p = p.stdout.getvalue().strip()
+ if re.match(self.netns_name, p) is not None:
+ raise RuntimeError("the netns '{}' already exists!".format(self.netns_name))
+
+ # Get the netns name list
+ netns_list = re.findall(r'[^()\s][-.\w]+[^():\s]', p)
+
+ # Get an uniq netns id
+ nsid = 0
+ while True:
+ p = self.client_remote.run(args=['ip', 'netns', 'list-id'],
+ stderr=BytesIO(), stdout=BytesIO(),
+ timeout=(5*60))
+ p = re.search(r"nsid {} ".format(nsid), p.stdout.getvalue())
+ if p is None:
+ break
+
+ nsid += 1
+
+ self.nsid = nsid;
+
+ # Add one new netns and set it id
+ args = ["sudo", "bash", "-c",
+ "ip netns add {0}".format(self.netns_name)]
+ self.client_remote.run(args=args, timeout=(5*60))
+ args = ["sudo", "bash", "-c",
+ "ip netns set {0} {1}".format(self.netns_name, nsid)]
+ self.client_remote.run(args=args, timeout=(5*60))
+
+ # Get one ip address for netns
+ ips = IP(self.ceph_brx_net)
+ for ip in ips:
+ found = False
+ if ip == ips[0]:
+ continue
+ if ip == ips[-2]:
+ raise RuntimeError("we have ran out of the ip addresses")
+
+ for ns in netns_list:
+ ns_name = ns.split()[0]
+ args = ["sudo", "bash", "-c",
+ "ip netns exec {0} ip addr".format(ns_name)]
+ p = self.client_remote.run(args=args, stderr=BytesIO(),
+ stdout=BytesIO(), timeout=(5*60))
+ q = re.search("{0}".format(ip), p.stdout.getvalue())
+ if q is not None:
+ found = True
+ break
+
+ if found == False:
+ break
+
+ mask = self.ceph_brx_net.split('/')[1]
+ brd = IP(self.ceph_brx_net).broadcast()
+
+ log.info("Setuping the netns '{0}' with {1}/{2}".format(self.netns_name, ip, mask))
+
+ # Setup the veth interfaces
+ args = ["sudo", "bash", "-c",
+ "ip link add veth0 netns {0} type veth peer name brx.{1}".format(self.netns_name, nsid)]
+ self.client_remote.run(args=args, timeout=(5*60))
+ args = ["sudo", "bash", "-c",
+ "ip netns exec {0} ip addr add {1}/{2} brd {3} dev veth0".format(self.netns_name, ip, mask, brd)]
+ self.client_remote.run(args=args, timeout=(5*60))
+ args = ["sudo", "bash", "-c",
+ "ip netns exec {0} ip link set veth0 up".format(self.netns_name)]
+ self.client_remote.run(args=args, timeout=(5*60))
+ args = ["sudo", "bash", "-c",
+ "ip netns exec {0} ip link set lo up".format(self.netns_name)]
+ self.client_remote.run(args=args, timeout=(5*60))
+
+ brxip = IP(self.ceph_brx_net)[-2]
+ args = ["sudo", "bash", "-c",
+ "ip netns exec {0} ip route add default via {1}".format(self.netns_name, brxip)]
+ self.client_remote.run(args=args, timeout=(5*60))
+
+ # Bring up the brx interface and join it to 'ceph-brx'
+ if self.use_brctl == True:
+ args = ["sudo", "bash", "-c",
+ "ip link set brx.{0} up".format(nsid)]
+ self.client_remote.run(args=args, timeout=(5*60))
+ args = ["sudo", "bash", "-c",
+ "brctl addif ceph-brx brx.{0}".format(nsid)]
+ self.client_remote.run(args=args, timeout=(5*60))
+ else:
+ self._bringup_network_manager_service()
+ args = ["sudo", "bash", "-c",
+ "nmcli connection add type bridge-slave con-name brx.{0} ifname brx.{0} master ceph-brx".format(nsid)]
+ self.client_remote.run(args=args, timeout=(5*60))
+ args = ["sudo", "bash", "-c",
+ "nmcli connection up brx.{0}".format(self.nsid)]
+ self.client_remote.run(args=args, timeout=(5*60))
+
+ def _cleanup_netns(self):
+ if self.nsid == -1:
+ return
+ log.info("Removing the netns '{0}'".format(self.netns_name))
+
+ # Delete the netns and the peer veth interface
+ if self.use_brctl == True:
+ args = ["sudo", "bash", "-c",
+ "ip link set brx.{0} down".format(self.nsid)]
+ self.client_remote.run(args=args, timeout=(5*60))
+ args = ["sudo", "bash", "-c",
+ "brctl delif ceph-brx brx.{0}".format(self.nsid)]
+ self.client_remote.run(args=args, timeout=(5*60))
+ args = ["sudo", "bash", "-c",
+ "ip link delete brx.{0}".format(self.nsid)]
+ self.client_remote.run(args=args, timeout=(5*60))
+ else:
+ self._bringup_network_manager_service()
+ args = ["sudo", "bash", "-c",
+ "nmcli connection down brx.{0}".format(self.nsid)]
+ self.client_remote.run(args=args, timeout=(5*60))
+ args = ["sudo", "bash", "-c",
+ "nmcli connection delete brx.{0}".format(self.nsid)]
+ self.client_remote.run(args=args, timeout=(5*60))
+
+ args = ["sudo", "bash", "-c",
+ "ip netns delete {0}".format(self.netns_name)]
+ self.client_remote.run(args=args, timeout=(5*60))
+
+ self.nsid = -1
+
+ def _cleanup_brx_and_nat(self):
+ brx = self.client_remote.run(args=['ip', 'addr'], stderr=BytesIO(),
+ stdout=BytesIO(), timeout=(5*60))
+ brx = re.findall(r'inet .* ceph-brx', brx.stdout.getvalue())
+ if not brx:
+ return
+
+ # If we are the last netns, will delete the ceph-brx
+ if self.use_brctl == True:
+ p = self.client_remote.run(args=['brctl', 'show', 'ceph-brx'],
+ stderr=BytesIO(), stdout=BytesIO(),
+ timeout=(5*60))
+ else:
+ self._bringup_network_manager_service()
+ p = self.client_remote.run(args=['nmcli', 'connection', 'show'],
+ stderr=BytesIO(), stdout=BytesIO(),
+ timeout=(5*60))
+ _list = re.findall(r'brx\.', p.stdout.getvalue().strip())
+ if len(_list) != 0:
+ return
+
+ log.info("Removing the 'ceph-brx'")
+
+ if self.use_brctl == True:
+ args = ["sudo", "bash", "-c",
+ "ip link set ceph-brx down"]
+ self.client_remote.run(args=args, timeout=(5*60))
+ args = ["sudo", "bash", "-c",
+ "brctl delbr ceph-brx"]
+ self.client_remote.run(args=args, timeout=(5*60))
+ else:
+ args = ["sudo", "bash", "-c",
+ "nmcli connection down ceph-brx"]
+ self.client_remote.run(args=args, timeout=(5*60))
+ args = ["sudo", "bash", "-c",
+ "nmcli connection delete ceph-brx"]
+ self.client_remote.run(args=args, timeout=(5*60))
+
+ # Drop the iptables NAT rules
+ ip = IP(self.ceph_brx_net)[-2]
+ mask = self.ceph_brx_net.split('/')[1]
+
+ p = self.client_remote.run(args=['route'], stderr=BytesIO(),
+ stdout=BytesIO(), timeout=(5*60))
+ p = re.findall(r'default .*', p.stdout.getvalue())
+ if p == False:
+ raise RuntimeError("No default gw found")
+ gw = p[0].split()[7]
+ args = ["sudo", "bash", "-c",
+ "iptables -D FORWARD -o {0} -i ceph-brx -j ACCEPT".format(gw)]
+ self.client_remote.run(args=args, timeout=(5*60))
+ args = ["sudo", "bash", "-c",
+ "iptables -D FORWARD -i {0} -o ceph-brx -j ACCEPT".format(gw)]
+ self.client_remote.run(args=args, timeout=(5*60))
+ args = ["sudo", "bash", "-c",
+ "iptables -t nat -D POSTROUTING -s {0}/{1} -o {2} -j MASQUERADE".format(ip, mask, gw)]
+ self.client_remote.run(args=args, timeout=(5*60))
+
+ # Restore the ip_forward
+ p = self.client_remote.run(args=['cat', '/tmp/python-ceph-brx'],
+ stderr=BytesIO(), stdout=BytesIO(),
+ timeout=(5*60))
+ val = p.stdout.getvalue().strip()
+ args = ["sudo", "bash", "-c",
+ "echo {} > /proc/sys/net/ipv4/ip_forward".format(val)]
+ self.client_remote.run(args=args, timeout=(5*60))
+ self.client_remote.run(args=['rm', '-f', '/tmp/python-ceph-brx'],
+ timeout=(5*60))
+
+ def setup_netns(self):
+ """
+ Setup the netns for the mountpoint.
+ """
+ log.info("Setting the '{0}' netns for '{1}'".format(self._netns_name, self.mountpoint))
+ self._setup_brx_and_nat()
+ self._setup_netns()
+
+ def cleanup_netns(self):
+ """
+ Cleanup the netns for the mountpoint.
+ """
+ log.info("Cleaning the '{0}' netns for '{1}'".format(self._netns_name, self.mountpoint))
+ self._cleanup_netns()
+ self._cleanup_brx_and_nat()
+
+ def suspend_netns(self):
+ """
+ Suspend the netns veth interface.
+ """
+ if self.nsid == -1:
+ return
+
+ log.info("Suspending the '{0}' netns for '{1}'".format(self._netns_name, self.mountpoint))
+
+ args = ["sudo", "bash", "-c",
+ "ip link set brx.{0} down".format(self.nsid)]
+ self.client_remote.run(args=args, timeout=(5*60))
+
+ def resume_netns(self):
+ """
+ Resume the netns veth interface.
+ """
+ if self.nsid == -1:
+ return
+
+ log.info("Resuming the '{0}' netns for '{1}'".format(self._netns_name, self.mountpoint))
+
+ args = ["sudo", "bash", "-c",
+ "ip link set brx.{0} up".format(self.nsid)]
+ self.client_remote.run(args=args, timeout=(5*60))
+
def mount(self, mount_path=None, mount_fs_name=None, mountpoint=None, mount_options=[]):
raise NotImplementedError()
"""
raise NotImplementedError()
- def kill_cleanup(self):
- raise NotImplementedError()
-
def kill(self):
- raise NotImplementedError()
+ """
+ Suspend the netns veth interface to make the client disconnected
+ from the ceph cluster
+ """
+ log.info('Killing connection on {0}...'.format(self.client_remote.name))
+ self.suspend_netns()
+
+ def kill_cleanup(self):
+ """
+ Follow up ``kill`` to get to a clean unmounted state.
+ """
+ log.info('Cleaning up killed connection on {0}'.format(self.client_remote.name))
+ self.umount_wait(force=True)
+ self.cleanup()
def cleanup(self):
- raise NotImplementedError()
+ """
+ Remove the mount point.
+
+ Prerequisite: the client is not mounted.
+ """
+ stderr = BytesIO()
+ try:
+ self.client_remote.run(
+ args=[
+ 'rmdir',
+ '--',
+ self.mountpoint,
+ ],
+ cwd=self.test_dir,
+ stderr=stderr,
+ timeout=(60*5),
+ check_status=False,
+ )
+ except CommandFailedError:
+ if "No such file or directory" in stderr.getvalue():
+ pass
+ else:
+ raise
def wait_until_mounted(self):
raise NotImplementedError()
import time
import sys
import errno
+from IPy import IP
from unittest import suite, loader
import unittest
import platform
class LocalKernelMount(KernelMount):
- def __init__(self, ctx, test_dir, client_id):
- super(LocalKernelMount, self).__init__(ctx, test_dir, client_id, LocalRemote(), None, None, None)
+ def __init__(self, ctx, test_dir, client_id, brxnet):
+ super(LocalKernelMount, self).__init__(ctx, test_dir, client_id, LocalRemote(), brxnet)
@property
def config_path(self):
def mount(self, mount_path=None, mount_fs_name=None, mount_options=[]):
self.setupfs(name=mount_fs_name)
+ self.setup_netns()
log.info('Mounting kclient client.{id} at {remote} {mnt}...'.format(
id=self.client_id, remote=self.client_remote, mnt=self.mountpoint))
self.client_remote.run(
args=[
'sudo',
+ 'nsenter',
+ '--net=/var/run/netns/{0}'.format(self.netns_name),
'./bin/mount.ceph',
':{mount_path}'.format(mount_path=mount_path),
self.mountpoint,
wait=False)
class LocalFuseMount(FuseMount):
- def __init__(self, ctx, test_dir, client_id):
- super(LocalFuseMount, self).__init__(ctx, None, test_dir, client_id, LocalRemote())
+ def __init__(self, ctx, test_dir, client_id, brxnet):
+ super(LocalFuseMount, self).__init__(ctx, None, test_dir, client_id, LocalRemote(), brxnet)
@property
def config_path(self):
if mountpoint is not None:
self.mountpoint = mountpoint
self.setupfs(name=mount_fs_name)
+ self.setup_netns()
self.client_remote.run(args=['mkdir', '-p', self.mountpoint])
prefix += mount_options;
self.fuse_daemon = self.client_remote.run(args=
- prefix + [
+ ['nsenter',
+ '--net=/var/run/netns/{0}'.format(self.netns_name),
+ ] + prefix + [
"-f",
"--name",
"client.{0}".format(self.client_id),
def enumerate_methods(s):
log.info("e: {0}".format(s))
for t in s._tests:
+ print("t {0}, s._tests {1}".format(t, s._tests))
if isinstance(t, suite.BaseTestSuite):
for sub in enumerate_methods(t):
yield sub
max_required_mgr = 0
require_memstore = False
+ print("module = {}".format(overall_suite))
for suite_, case in enumerate_methods(overall_suite):
+ print("suite {0}".format(suite_))
+ print("case {0}".format(case))
max_required_mds = max(max_required_mds,
getattr(case, "MDSS_REQUIRED", 0))
max_required_clients = max(max_required_clients,
global opt_log_ps_output
opt_log_ps_output = False
use_kernel_client = False
+ opt_brxnet= None
args = sys.argv[1:]
flags = [a for a in args if a.startswith("-")]
clear_old_log()
elif f == "--kclient":
use_kernel_client = True
+ elif '--brxnet' in f:
+ if re.search(r'=[0-9./]+', f) is None:
+ log.error("--brxnet=<ip/mask> option needs one argument: '{0}'".format(f))
+ sys.exit(-1)
+ opt_brxnet=f.split('=')[1]
+ try:
+ IP(opt_brxnet)
+ if IP(opt_brxnet).iptype() is 'PUBLIC':
+ raise RuntimeError('is public')
+ except Exception as e:
+ log.error("Invalid ip '{0}' {1}".format(opt_brxnet, e))
+ sys.exit(-1)
else:
log.error("Unknown option '{0}'".format(f))
sys.exit(-1)
open("./keyring", "a").write(p.stdout.getvalue())
if use_kernel_client:
- mount = LocalKernelMount(ctx, test_dir, client_id)
+ mount = LocalKernelMount(ctx, test_dir, client_id, opt_brxnet)
else:
- mount = LocalFuseMount(ctx, test_dir, client_id)
+ mount = LocalFuseMount(ctx, test_dir, client_id, opt_brxnet)
mounts.append(mount)
if os.path.exists(mount.mountpoint):
projects / ceph.git / commitdiff