log = logging.getLogger(__name__)
-class Remote(object):
+class RemoteShell(object):
+ """
+ Contains methods to run miscellaneous shell commands on remote machines.
+
+ These methods were originally part of orchestra.remote.Remote. The reason
+ for moving these methods from Remote is that applications that use
+ teuthology for testing usually have programs that can run tests locally on
+ a single node machine for development work (for example, vstart_runner.py
+ in case of Ceph). These programs can import and reuse these methods
+ without having to deal SSH stuff. In short, this class serves a shared
+ interface.
+
+ To use these methods, inherit the class here and implement "run()" method in
+ the subclass.
+ """
+
+ def remove(self, path):
+ self.run(args=['rm', '-fr', path])
+
+ def mkdtemp(self, suffix=None, parentdir=None):
+ """
+ Create a temporary directory on remote machine and return it's path.
+ """
+ args = ['mktemp', '-d']
+
+ if suffix:
+ args.append('--suffix=%s' % suffix)
+ if parentdir:
+ args.append('--tmpdir=%s' % parentdir)
+
+ return self.sh(args).strip()
+
+ def mktemp(self, suffix=None, parentdir=None, data=None):
+ """
+ Make a remote temporary file.
+
+ :param suffix: suffix for the temporary file
+ :param parentdir: parent dir where temp file should be created
+ :param data: write data to the file if provided
+
+ Returns: the path of the temp file created.
+ """
+ args = ['mktemp']
+ if suffix:
+ args.append('--suffix=%s' % suffix)
+ if parentdir:
+ args.append('--tmpdir=%s' % parentdir)
+
+ path = self.sh(args).strip()
+
+ if data:
+ self.write_file(path=path, data=data)
+
+ return path
+
+ def sh(self, script, **kwargs):
+ """
+ Shortcut for run method.
+
+ Usage:
+ my_name = remote.sh('whoami')
+ remote_date = remote.sh('date')
+ """
+ if 'stdout' not in kwargs:
+ kwargs['stdout'] = BytesIO()
+ if 'args' not in kwargs:
+ kwargs['args'] = script
+ proc = self.run(**kwargs)
+ out = proc.stdout.getvalue()
+ if isinstance(out, bytes):
+ return out.decode()
+ else:
+ return out
+
+ def sh_file(self, script, label="script", sudo=False, **kwargs):
+ """
+ Run shell script after copying its contents to a remote file
+
+ :param script: string with script text, or file object
+ :param sudo: run command with sudo if True,
+ run as user name if string value (defaults to False)
+ :param label: string value which will be part of file name
+ Returns: stdout
+ """
+ ftempl = '/tmp/teuthology-remote-$(date +%Y%m%d%H%M%S)-{}-XXXX'\
+ .format(label)
+ script_file = self.sh("mktemp %s" % ftempl).strip()
+ self.sh("cat - | tee {script} ; chmod a+rx {script}"\
+ .format(script=script_file), stdin=script)
+ if sudo:
+ if isinstance(sudo, str):
+ command="sudo -u %s %s" % (sudo, script_file)
+ else:
+ command="sudo %s" % script_file
+ else:
+ command="%s" % script_file
+
+ return self.sh(command, **kwargs)
+
+ def chmod(self, file_path, permissions):
+ """
+ As super-user, set permissions on the remote file specified.
+ """
+ args = [
+ 'sudo',
+ 'chmod',
+ permissions,
+ file_path,
+ ]
+ self.run(
+ args=args,
+ )
+
+ def chcon(self, file_path, context):
+ """
+ Set the SELinux context of a given file.
+
+ VMs and non-RPM-based hosts will skip this operation because ours
+ currently have SELinux disabled.
+
+ :param file_path: The path to the file
+ :param context: The SELinux context to be used
+ """
+ if self.os.package_type != 'rpm' or \
+ self.os.name in ['opensuse', 'sle']:
+ return
+ if teuthology.lock.query.is_vm(self.shortname):
+ return
+ self.run(args="sudo chcon {con} {path}".format(
+ con=context, path=file_path))
+
+ def copy_file(self, src, dst, sudo=False, mode=None, owner=None,
+ mkdir=False, append=False):
+ """
+ Copy data to remote file
+
+ :param src: source file path on remote host
+ :param dst: destination file path on remote host
+ :param sudo: use sudo to write file, defaults False
+ :param mode: set file mode bits if provided
+ :param owner: set file owner if provided
+ :param mkdir: ensure the destination directory exists, defaults
+ False
+ :param append: append data to the file, defaults False
+ """
+ dd = 'sudo dd' if sudo else 'dd'
+ args = dd + ' if=' + src + ' of=' + dst
+ if append:
+ args += ' conv=notrunc oflag=append'
+ if mkdir:
+ mkdirp = 'sudo mkdir -p' if sudo else 'mkdir -p'
+ dirpath = os.path.dirname(dst)
+ if dirpath:
+ args = mkdirp + ' ' + dirpath + '\n' + args
+ if mode:
+ chmod = 'sudo chmod' if sudo else 'chmod'
+ args += '\n' + chmod + ' ' + mode + ' ' + dst
+ if owner:
+ chown = 'sudo chown' if sudo else 'chown'
+ args += '\n' + chown + ' ' + owner + ' ' + dst
+ args = 'set -ex' + '\n' + args
+ self.run(args=args)
+ def move_file(self, src, dst, sudo=False, mode=None, owner=None,
+ mkdir=False):
+ """
+ Move data to remote file
+
+ :param src: source file path on remote host
+ :param dst: destination file path on remote host
+ :param sudo: use sudo to write file, defaults False
+ :param mode: set file mode bits if provided
+ :param owner: set file owner if provided
+ :param mkdir: ensure the destination directory exists, defaults
+ False
+ """
+ mv = 'sudo mv' if sudo else 'mv'
+ args = mv + ' ' + src + ' ' + dst
+ if mkdir:
+ mkdirp = 'sudo mkdir -p' if sudo else 'mkdir -p'
+ dirpath = os.path.dirname(dst)
+ if dirpath:
+ args = mkdirp + ' ' + dirpath + '\n' + args
+ if mode:
+ chmod = 'sudo chmod' if sudo else 'chmod'
+ args += ' && ' + chmod + ' ' + mode + ' ' + dst
+ if owner:
+ chown = 'sudo chown' if sudo else 'chown'
+ args += ' && ' + chown + ' ' + owner + ' ' + dst
+ self.run(args=args)
+
+ def read_file(self, path, sudo=False, stdout=None,
+ offset=0, length=0):
+ """
+ Read data from remote file
+
+ :param path: file path on remote host
+ :param sudo: use sudo to read the file, defaults False
+ :param stdout: output object, defaults to io.BytesIO()
+ :param offset: number of bytes to skip from the file
+ :param length: number of bytes to read from the file
+
+ :raises: :class:`FileNotFoundError`: there is no such file by the path
+ :raises: :class:`RuntimeError`: unexpected error occurred
+
+ :returns: the file contents in bytes, if stdout is `io.BytesIO`, by
+ default
+ :returns: the file contents in str, if stdout is `io.StringIO`
+ """
+ dd = 'sudo dd' if sudo else 'dd'
+ args = dd + ' if=' + path + ' of=/dev/stdout'
+ iflags=[]
+ # we have to set defaults here instead of the method's signature,
+ # because python is reusing the object from call to call
+ stdout = stdout or BytesIO()
+ if offset:
+ args += ' skip=' + str(offset)
+ iflags += 'skip_bytes'
+ if length:
+ args += ' count=' + str(length)
+ iflags += 'count_bytes'
+ if iflags:
+ args += ' iflag=' + ','.join(iflags)
+ args = 'set -ex' + '\n' + args
+ proc = self.run(args=args, stdout=stdout, stderr=StringIO(),
+ check_status=False, quiet=True)
+ if proc.returncode:
+ if 'No such file or directory' in proc.stderr.getvalue():
+ raise FileNotFoundError(errno.ENOENT,
+ f"Cannot find file on the remote '{self.name}'", path)
+ else:
+ raise RuntimeError("Unexpected error occurred while trying to "
+ f"read '{path}' file on the remote '{self.name}'")
+
+ return proc.stdout.getvalue()
+
+
+ def write_file(self, path, data, sudo=False, mode=None, owner=None,
+ mkdir=False, append=False):
+ """
+ Write data to remote file
+
+ :param path: file path on remote host
+ :param data: str, binary or fileobj to be written
+ :param sudo: use sudo to write file, defaults False
+ :param mode: set file mode bits if provided
+ :param owner: set file owner if provided
+ :param mkdir: preliminary create the file directory, defaults False
+ :param append: append data to the file, defaults False
+ """
+ dd = 'sudo dd' if sudo else 'dd'
+ args = dd + ' of=' + path
+ if append:
+ args += ' conv=notrunc oflag=append'
+ if mkdir:
+ mkdirp = 'sudo mkdir -p' if sudo else 'mkdir -p'
+ dirpath = os.path.dirname(path)
+ if dirpath:
+ args = mkdirp + ' ' + dirpath + '\n' + args
+ if mode:
+ chmod = 'sudo chmod' if sudo else 'chmod'
+ args += '\n' + chmod + ' ' + mode + ' ' + path
+ if owner:
+ chown = 'sudo chown' if sudo else 'chown'
+ args += '\n' + chown + ' ' + owner + ' ' + path
+ args = 'set -ex' + '\n' + args
+ self.run(args=args, stdin=data, quiet=True)
+
+ def sudo_write_file(self, path, data, **kwargs):
+ """
+ Write data to remote file with sudo, for more info see `write_file()`.
+ """
+ self.write_file(path, data, sudo=True, **kwargs)
+
+ @property
+ def os(self):
+ if not hasattr(self, '_os'):
+ try:
+ os_release = self.sh('cat /etc/os-release').strip()
+ self._os = OS.from_os_release(os_release)
+ return self._os
+ except CommandFailedError:
+ pass
+
+ lsb_release = self.sh('lsb_release -a').strip()
+ self._os = OS.from_lsb_release(lsb_release)
+ return self._os
+
+ @property
+ def arch(self):
+ if not hasattr(self, '_arch'):
+ self._arch = self.sh('uname -m').strip()
+ return self._arch
+
+
+class Remote(RemoteShell):
"""
A connection to a remote host.
@property
def machine_type(self):
if not getattr(self, '_machine_type', None):
- remote_info = teuthology.lock.query.get_status(self.hostname)
- if not remote_info:
- return None
- self._machine_type = remote_info.get("machine_type", None)
- return self._machine_type
-
- @property
- def is_reimageable(self):
- return self.machine_type in self._reimage_types
-
- @property
- def shortname(self):
- if self._shortname is None:
- self._shortname = host_shortname(self.hostname)
- return self._shortname
-
- @property
- def is_online(self):
- if self.ssh is None:
- return False
- if self.ssh.get_transport() is None:
- return False
- try:
- self.run(args="true")
- except Exception:
- return False
- return self.ssh.get_transport().is_active()
-
- def ensure_online(self):
- if self.is_online:
- return
- self.connect()
- if not self.is_online:
- raise Exception('unable to connect')
-
- @property
- def system_type(self):
- """
- System type decorator
- """
- return misc.get_system_type(self)
-
- def __str__(self):
- return self.name
-
- def __repr__(self):
- return '{classname}(name={name!r})'.format(
- classname=self.__class__.__name__,
- name=self.name,
- )
-
- def run(self, **kwargs):
- """
- This calls `orchestra.run.run` with our SSH client.
-
- TODO refactor to move run.run here?
- """
- if not self.ssh or \
- not self.ssh.get_transport() or \
- not self.ssh.get_transport().is_active():
- self.reconnect()
- r = self._runner(client=self.ssh, name=self.shortname, **kwargs)
- r.remote = self
- return r
-
- def mkdtemp(self, suffix=None, parentdir=None):
- """
- Create a temporary directory on remote machine and return it's path.
- """
- args = ['mktemp', '-d']
-
- if suffix:
- args.append('--suffix=%s' % suffix)
- if parentdir:
- args.append('--tmpdir=%s' % parentdir)
-
- return self.sh(args).strip()
-
- def mktemp(self, suffix=None, parentdir=None, data=None):
- """
- Make a remote temporary file.
-
- :param suffix: suffix for the temporary file
- :param parentdir: parent dir where temp file should be created
- :param data: write data to the file if provided
-
- Returns: the path of the temp file created.
- """
- args = ['mktemp']
- if suffix:
- args.append('--suffix=%s' % suffix)
- if parentdir:
- args.append('--tmpdir=%s' % parentdir)
-
- path = self.sh(args).strip()
-
- if data:
- self.write_file(path=path, data=data)
-
- return path
-
- def sh(self, script, **kwargs):
- """
- Shortcut for run method.
-
- Usage:
- my_name = remote.sh('whoami')
- remote_date = remote.sh('date')
- """
- if 'stdout' not in kwargs:
- kwargs['stdout'] = BytesIO()
- if 'args' not in kwargs:
- kwargs['args'] = script
- proc = self.run(**kwargs)
- out = proc.stdout.getvalue()
- if isinstance(out, bytes):
- return out.decode()
- else:
- return out
+ remote_info = teuthology.lock.query.get_status(self.hostname)
+ if not remote_info:
+ return None
+ self._machine_type = remote_info.get("machine_type", None)
+ return self._machine_type
- def sh_file(self, script, label="script", sudo=False, **kwargs):
- """
- Run shell script after copying its contents to a remote file
+ @property
+ def is_reimageable(self):
+ return self.machine_type in self._reimage_types
- :param script: string with script text, or file object
- :param sudo: run command with sudo if True,
- run as user name if string value (defaults to False)
- :param label: string value which will be part of file name
- Returns: stdout
- """
- ftempl = '/tmp/teuthology-remote-$(date +%Y%m%d%H%M%S)-{}-XXXX'\
- .format(label)
- script_file = self.sh("mktemp %s" % ftempl).strip()
- self.sh("cat - | tee {script} ; chmod a+rx {script}"\
- .format(script=script_file), stdin=script)
- if sudo:
- if isinstance(sudo, str):
- command="sudo -u %s %s" % (sudo, script_file)
- else:
- command="sudo %s" % script_file
- else:
- command="%s" % script_file
+ @property
+ def shortname(self):
+ if self._shortname is None:
+ self._shortname = host_shortname(self.hostname)
+ return self._shortname
- return self.sh(command, **kwargs)
+ @property
+ def is_online(self):
+ if self.ssh is None:
+ return False
+ if self.ssh.get_transport() is None:
+ return False
+ try:
+ self.run(args="true")
+ except Exception:
+ return False
+ return self.ssh.get_transport().is_active()
- def chmod(self, file_path, permissions):
+ def ensure_online(self):
+ if self.is_online:
+ return
+ self.connect()
+ if not self.is_online:
+ raise Exception('unable to connect')
+
+ @property
+ def system_type(self):
"""
- As super-user, set permissions on the remote file specified.
+ System type decorator
"""
- args = [
- 'sudo',
- 'chmod',
- permissions,
- file_path,
- ]
- self.run(
- args=args,
+ return misc.get_system_type(self)
+
+ def __str__(self):
+ return self.name
+
+ def __repr__(self):
+ return '{classname}(name={name!r})'.format(
+ classname=self.__class__.__name__,
+ name=self.name,
)
- def chcon(self, file_path, context):
+ def run(self, **kwargs):
"""
- Set the SELinux context of a given file.
-
- VMs and non-RPM-based hosts will skip this operation because ours
- currently have SELinux disabled.
+ This calls `orchestra.run.run` with our SSH client.
- :param file_path: The path to the file
- :param context: The SELinux context to be used
+ TODO refactor to move run.run here?
"""
- if self.os.package_type != 'rpm' or \
- self.os.name in ['opensuse', 'sle']:
- return
- if teuthology.lock.query.is_vm(self.shortname):
- return
- self.run(args="sudo chcon {con} {path}".format(
- con=context, path=file_path))
+ if not self.ssh or \
+ not self.ssh.get_transport() or \
+ not self.ssh.get_transport().is_active():
+ self.reconnect()
+ r = self._runner(client=self.ssh, name=self.shortname, **kwargs)
+ r.remote = self
+ return r
def _sftp_put_file(self, local_path, remote_path):
"""
file_size = file_size / 1024.0
return "{:3.0f}{}".format(file_size, unit)
- def remove(self, path):
- self.run(args=['rm', '-fr', path])
-
def put_file(self, path, dest_path, sudo=False):
"""
Copy a local filename to a remote file
])
return self.run(args=args, wait=False, stdout=run.PIPE)
- def copy_file(self, src, dst, sudo=False, mode=None, owner=None,
- mkdir=False, append=False):
- """
- Copy data to remote file
-
- :param src: source file path on remote host
- :param dst: destination file path on remote host
- :param sudo: use sudo to write file, defaults False
- :param mode: set file mode bits if provided
- :param owner: set file owner if provided
- :param mkdir: ensure the destination directory exists, defaults False
- :param append: append data to the file, defaults False
- """
- dd = 'sudo dd' if sudo else 'dd'
- args = dd + ' if=' + src + ' of=' + dst
- if append:
- args += ' conv=notrunc oflag=append'
- if mkdir:
- mkdirp = 'sudo mkdir -p' if sudo else 'mkdir -p'
- dirpath = os.path.dirname(dst)
- if dirpath:
- args = mkdirp + ' ' + dirpath + '\n' + args
- if mode:
- chmod = 'sudo chmod' if sudo else 'chmod'
- args += '\n' + chmod + ' ' + mode + ' ' + dst
- if owner:
- chown = 'sudo chown' if sudo else 'chown'
- args += '\n' + chown + ' ' + owner + ' ' + dst
- args = 'set -ex' + '\n' + args
- self.run(args=args)
-
- def move_file(self, src, dst, sudo=False, mode=None, owner=None,
- mkdir=False):
- """
- Move data to remote file
-
- :param src: source file path on remote host
- :param dst: destination file path on remote host
- :param sudo: use sudo to write file, defaults False
- :param mode: set file mode bits if provided
- :param owner: set file owner if provided
- :param mkdir: ensure the destination directory exists, defaults False
- """
- mv = 'sudo mv' if sudo else 'mv'
- args = mv + ' ' + src + ' ' + dst
- if mkdir:
- mkdirp = 'sudo mkdir -p' if sudo else 'mkdir -p'
- dirpath = os.path.dirname(dst)
- if dirpath:
- args = mkdirp + ' ' + dirpath + '\n' + args
- if mode:
- chmod = 'sudo chmod' if sudo else 'chmod'
- args += ' && ' + chmod + ' ' + mode + ' ' + dst
- if owner:
- chown = 'sudo chown' if sudo else 'chown'
- args += ' && ' + chown + ' ' + owner + ' ' + dst
- self.run(args=args)
-
- def read_file(self, path, sudo=False, stdout=None,
- offset=0, length=0):
- """
- Read data from remote file
-
- :param path: file path on remote host
- :param sudo: use sudo to read the file, defaults False
- :param stdout: output object, defaults to io.BytesIO()
- :param offset: number of bytes to skip from the file
- :param length: number of bytes to read from the file
-
- :raises: :class:`FileNotFoundError`: there is no such file by the path
- :raises: :class:`RuntimeError`: unexpected error occurred
-
- :returns: the file contents in bytes, if stdout is `io.BytesIO`, by default
- :returns: the file contents in str, if stdout is `io.StringIO`
- """
- dd = 'sudo dd' if sudo else 'dd'
- args = dd + ' if=' + path + ' of=/dev/stdout'
- iflags=[]
- # we have to set defaults here instead of the method's signature,
- # because python is reusing the object from call to call
- stdout = stdout or BytesIO()
- if offset:
- args += ' skip=' + str(offset)
- iflags += 'skip_bytes'
- if length:
- args += ' count=' + str(length)
- iflags += 'count_bytes'
- if iflags:
- args += ' iflag=' + ','.join(iflags)
- args = 'set -ex' + '\n' + args
- proc = self.run(args=args, stdout=stdout, stderr=StringIO(), check_status=False, quiet=True)
- if proc.returncode:
- if 'No such file or directory' in proc.stderr.getvalue():
- raise FileNotFoundError(errno.ENOENT,
- f"Cannot find file on the remote '{self.name}'", path)
- else:
- raise RuntimeError("Unexpected error occurred while trying to "
- f"read '{path}' file on the remote '{self.name}'")
-
- return proc.stdout.getvalue()
-
-
- def write_file(self, path, data, sudo=False, mode=None, owner=None,
- mkdir=False, append=False):
- """
- Write data to remote file
-
- :param path: file path on remote host
- :param data: str, binary or fileobj to be written
- :param sudo: use sudo to write file, defaults False
- :param mode: set file mode bits if provided
- :param owner: set file owner if provided
- :param mkdir: preliminary create the file directory, defaults False
- :param append: append data to the file, defaults False
- """
- dd = 'sudo dd' if sudo else 'dd'
- args = dd + ' of=' + path
- if append:
- args += ' conv=notrunc oflag=append'
- if mkdir:
- mkdirp = 'sudo mkdir -p' if sudo else 'mkdir -p'
- dirpath = os.path.dirname(path)
- if dirpath:
- args = mkdirp + ' ' + dirpath + '\n' + args
- if mode:
- chmod = 'sudo chmod' if sudo else 'chmod'
- args += '\n' + chmod + ' ' + mode + ' ' + path
- if owner:
- chown = 'sudo chown' if sudo else 'chown'
- args += '\n' + chown + ' ' + owner + ' ' + path
- args = 'set -ex' + '\n' + args
- self.run(args=args, stdin=data, quiet=True)
-
- def sudo_write_file(self, path, data, **kwargs):
- """
- Write data to remote file with sudo, for more info see `write_file()`.
- """
- self.write_file(path, data, sudo=True, **kwargs)
-
- @property
- def os(self):
- if not hasattr(self, '_os'):
- try:
- os_release = self.sh('cat /etc/os-release').strip()
- self._os = OS.from_os_release(os_release)
- return self._os
- except CommandFailedError:
- pass
-
- lsb_release = self.sh('lsb_release -a').strip()
- self._os = OS.from_lsb_release(lsb_release)
- return self._os
-
- @property
- def arch(self):
- if not hasattr(self, '_arch'):
- self._arch = self.sh('uname -m').strip()
- return self._arch
-
@property
def host_key(self):
if not self._host_key:
projects / teuthology.git / commitdiff