From 11c30bf71082bed97207de6dc636cd35ee67698c Mon Sep 17 00:00:00 2001 From: Rishabh Dave Date: Thu, 4 Mar 2021 16:38:37 +0530 Subject: [PATCH] orchestra: move methods for shell commands from remote.Remote Move methods that issue commands via shell and that don't necessarily need to depend on SHH from class Remote to a different class. This enables applications like vstart_runner.py (in Ceph repo) to reuse these methods for running tests locally without necessarily depending on SSH and without duplicating them in vstart_runner.py. Signed-off-by: Rishabh Dave --- teuthology/orchestra/remote.py | 676 +++++++++++++++++---------------- 1 file changed, 348 insertions(+), 328 deletions(-) diff --git a/teuthology/orchestra/remote.py b/teuthology/orchestra/remote.py index bedfcc6f45..1b5cb6fda7 100644 --- a/teuthology/orchestra/remote.py +++ b/teuthology/orchestra/remote.py @@ -26,8 +26,302 @@ import netaddr log = logging.getLogger(__name__) -class Remote(object): +class RemoteShell(object): + """ + Contains methods to run miscellaneous shell commands on remote machines. + + These methods were originally part of orchestra.remote.Remote. The reason + for moving these methods from Remote is that applications that use + teuthology for testing usually have programs that can run tests locally on + a single node machine for development work (for example, vstart_runner.py + in case of Ceph). These programs can import and reuse these methods + without having to deal SSH stuff. In short, this class serves a shared + interface. + + To use these methods, inherit the class here and implement "run()" method in + the subclass. + """ + + def remove(self, path): + self.run(args=['rm', '-fr', path]) + + def mkdtemp(self, suffix=None, parentdir=None): + """ + Create a temporary directory on remote machine and return it's path. + """ + args = ['mktemp', '-d'] + + if suffix: + args.append('--suffix=%s' % suffix) + if parentdir: + args.append('--tmpdir=%s' % parentdir) + + return self.sh(args).strip() + + def mktemp(self, suffix=None, parentdir=None, data=None): + """ + Make a remote temporary file. + + :param suffix: suffix for the temporary file + :param parentdir: parent dir where temp file should be created + :param data: write data to the file if provided + + Returns: the path of the temp file created. + """ + args = ['mktemp'] + if suffix: + args.append('--suffix=%s' % suffix) + if parentdir: + args.append('--tmpdir=%s' % parentdir) + + path = self.sh(args).strip() + + if data: + self.write_file(path=path, data=data) + + return path + + def sh(self, script, **kwargs): + """ + Shortcut for run method. + + Usage: + my_name = remote.sh('whoami') + remote_date = remote.sh('date') + """ + if 'stdout' not in kwargs: + kwargs['stdout'] = BytesIO() + if 'args' not in kwargs: + kwargs['args'] = script + proc = self.run(**kwargs) + out = proc.stdout.getvalue() + if isinstance(out, bytes): + return out.decode() + else: + return out + + def sh_file(self, script, label="script", sudo=False, **kwargs): + """ + Run shell script after copying its contents to a remote file + + :param script: string with script text, or file object + :param sudo: run command with sudo if True, + run as user name if string value (defaults to False) + :param label: string value which will be part of file name + Returns: stdout + """ + ftempl = '/tmp/teuthology-remote-$(date +%Y%m%d%H%M%S)-{}-XXXX'\ + .format(label) + script_file = self.sh("mktemp %s" % ftempl).strip() + self.sh("cat - | tee {script} ; chmod a+rx {script}"\ + .format(script=script_file), stdin=script) + if sudo: + if isinstance(sudo, str): + command="sudo -u %s %s" % (sudo, script_file) + else: + command="sudo %s" % script_file + else: + command="%s" % script_file + + return self.sh(command, **kwargs) + + def chmod(self, file_path, permissions): + """ + As super-user, set permissions on the remote file specified. + """ + args = [ + 'sudo', + 'chmod', + permissions, + file_path, + ] + self.run( + args=args, + ) + + def chcon(self, file_path, context): + """ + Set the SELinux context of a given file. + + VMs and non-RPM-based hosts will skip this operation because ours + currently have SELinux disabled. + + :param file_path: The path to the file + :param context: The SELinux context to be used + """ + if self.os.package_type != 'rpm' or \ + self.os.name in ['opensuse', 'sle']: + return + if teuthology.lock.query.is_vm(self.shortname): + return + self.run(args="sudo chcon {con} {path}".format( + con=context, path=file_path)) + + def copy_file(self, src, dst, sudo=False, mode=None, owner=None, + mkdir=False, append=False): + """ + Copy data to remote file + + :param src: source file path on remote host + :param dst: destination file path on remote host + :param sudo: use sudo to write file, defaults False + :param mode: set file mode bits if provided + :param owner: set file owner if provided + :param mkdir: ensure the destination directory exists, defaults + False + :param append: append data to the file, defaults False + """ + dd = 'sudo dd' if sudo else 'dd' + args = dd + ' if=' + src + ' of=' + dst + if append: + args += ' conv=notrunc oflag=append' + if mkdir: + mkdirp = 'sudo mkdir -p' if sudo else 'mkdir -p' + dirpath = os.path.dirname(dst) + if dirpath: + args = mkdirp + ' ' + dirpath + '\n' + args + if mode: + chmod = 'sudo chmod' if sudo else 'chmod' + args += '\n' + chmod + ' ' + mode + ' ' + dst + if owner: + chown = 'sudo chown' if sudo else 'chown' + args += '\n' + chown + ' ' + owner + ' ' + dst + args = 'set -ex' + '\n' + args + self.run(args=args) + def move_file(self, src, dst, sudo=False, mode=None, owner=None, + mkdir=False): + """ + Move data to remote file + + :param src: source file path on remote host + :param dst: destination file path on remote host + :param sudo: use sudo to write file, defaults False + :param mode: set file mode bits if provided + :param owner: set file owner if provided + :param mkdir: ensure the destination directory exists, defaults + False + """ + mv = 'sudo mv' if sudo else 'mv' + args = mv + ' ' + src + ' ' + dst + if mkdir: + mkdirp = 'sudo mkdir -p' if sudo else 'mkdir -p' + dirpath = os.path.dirname(dst) + if dirpath: + args = mkdirp + ' ' + dirpath + '\n' + args + if mode: + chmod = 'sudo chmod' if sudo else 'chmod' + args += ' && ' + chmod + ' ' + mode + ' ' + dst + if owner: + chown = 'sudo chown' if sudo else 'chown' + args += ' && ' + chown + ' ' + owner + ' ' + dst + self.run(args=args) + + def read_file(self, path, sudo=False, stdout=None, + offset=0, length=0): + """ + Read data from remote file + + :param path: file path on remote host + :param sudo: use sudo to read the file, defaults False + :param stdout: output object, defaults to io.BytesIO() + :param offset: number of bytes to skip from the file + :param length: number of bytes to read from the file + + :raises: :class:`FileNotFoundError`: there is no such file by the path + :raises: :class:`RuntimeError`: unexpected error occurred + + :returns: the file contents in bytes, if stdout is `io.BytesIO`, by + default + :returns: the file contents in str, if stdout is `io.StringIO` + """ + dd = 'sudo dd' if sudo else 'dd' + args = dd + ' if=' + path + ' of=/dev/stdout' + iflags=[] + # we have to set defaults here instead of the method's signature, + # because python is reusing the object from call to call + stdout = stdout or BytesIO() + if offset: + args += ' skip=' + str(offset) + iflags += 'skip_bytes' + if length: + args += ' count=' + str(length) + iflags += 'count_bytes' + if iflags: + args += ' iflag=' + ','.join(iflags) + args = 'set -ex' + '\n' + args + proc = self.run(args=args, stdout=stdout, stderr=StringIO(), + check_status=False, quiet=True) + if proc.returncode: + if 'No such file or directory' in proc.stderr.getvalue(): + raise FileNotFoundError(errno.ENOENT, + f"Cannot find file on the remote '{self.name}'", path) + else: + raise RuntimeError("Unexpected error occurred while trying to " + f"read '{path}' file on the remote '{self.name}'") + + return proc.stdout.getvalue() + + + def write_file(self, path, data, sudo=False, mode=None, owner=None, + mkdir=False, append=False): + """ + Write data to remote file + + :param path: file path on remote host + :param data: str, binary or fileobj to be written + :param sudo: use sudo to write file, defaults False + :param mode: set file mode bits if provided + :param owner: set file owner if provided + :param mkdir: preliminary create the file directory, defaults False + :param append: append data to the file, defaults False + """ + dd = 'sudo dd' if sudo else 'dd' + args = dd + ' of=' + path + if append: + args += ' conv=notrunc oflag=append' + if mkdir: + mkdirp = 'sudo mkdir -p' if sudo else 'mkdir -p' + dirpath = os.path.dirname(path) + if dirpath: + args = mkdirp + ' ' + dirpath + '\n' + args + if mode: + chmod = 'sudo chmod' if sudo else 'chmod' + args += '\n' + chmod + ' ' + mode + ' ' + path + if owner: + chown = 'sudo chown' if sudo else 'chown' + args += '\n' + chown + ' ' + owner + ' ' + path + args = 'set -ex' + '\n' + args + self.run(args=args, stdin=data, quiet=True) + + def sudo_write_file(self, path, data, **kwargs): + """ + Write data to remote file with sudo, for more info see `write_file()`. + """ + self.write_file(path, data, sudo=True, **kwargs) + + @property + def os(self): + if not hasattr(self, '_os'): + try: + os_release = self.sh('cat /etc/os-release').strip() + self._os = OS.from_os_release(os_release) + return self._os + except CommandFailedError: + pass + + lsb_release = self.sh('lsb_release -a').strip() + self._os = OS.from_lsb_release(lsb_release) + return self._os + + @property + def arch(self): + if not hasattr(self, '_arch'): + self._arch = self.sh('uname -m').strip() + return self._arch + + +class Remote(RemoteShell): """ A connection to a remote host. @@ -151,182 +445,70 @@ class Remote(object): @property def machine_type(self): if not getattr(self, '_machine_type', None): - remote_info = teuthology.lock.query.get_status(self.hostname) - if not remote_info: - return None - self._machine_type = remote_info.get("machine_type", None) - return self._machine_type - - @property - def is_reimageable(self): - return self.machine_type in self._reimage_types - - @property - def shortname(self): - if self._shortname is None: - self._shortname = host_shortname(self.hostname) - return self._shortname - - @property - def is_online(self): - if self.ssh is None: - return False - if self.ssh.get_transport() is None: - return False - try: - self.run(args="true") - except Exception: - return False - return self.ssh.get_transport().is_active() - - def ensure_online(self): - if self.is_online: - return - self.connect() - if not self.is_online: - raise Exception('unable to connect') - - @property - def system_type(self): - """ - System type decorator - """ - return misc.get_system_type(self) - - def __str__(self): - return self.name - - def __repr__(self): - return '{classname}(name={name!r})'.format( - classname=self.__class__.__name__, - name=self.name, - ) - - def run(self, **kwargs): - """ - This calls `orchestra.run.run` with our SSH client. - - TODO refactor to move run.run here? - """ - if not self.ssh or \ - not self.ssh.get_transport() or \ - not self.ssh.get_transport().is_active(): - self.reconnect() - r = self._runner(client=self.ssh, name=self.shortname, **kwargs) - r.remote = self - return r - - def mkdtemp(self, suffix=None, parentdir=None): - """ - Create a temporary directory on remote machine and return it's path. - """ - args = ['mktemp', '-d'] - - if suffix: - args.append('--suffix=%s' % suffix) - if parentdir: - args.append('--tmpdir=%s' % parentdir) - - return self.sh(args).strip() - - def mktemp(self, suffix=None, parentdir=None, data=None): - """ - Make a remote temporary file. - - :param suffix: suffix for the temporary file - :param parentdir: parent dir where temp file should be created - :param data: write data to the file if provided - - Returns: the path of the temp file created. - """ - args = ['mktemp'] - if suffix: - args.append('--suffix=%s' % suffix) - if parentdir: - args.append('--tmpdir=%s' % parentdir) - - path = self.sh(args).strip() - - if data: - self.write_file(path=path, data=data) - - return path - - def sh(self, script, **kwargs): - """ - Shortcut for run method. - - Usage: - my_name = remote.sh('whoami') - remote_date = remote.sh('date') - """ - if 'stdout' not in kwargs: - kwargs['stdout'] = BytesIO() - if 'args' not in kwargs: - kwargs['args'] = script - proc = self.run(**kwargs) - out = proc.stdout.getvalue() - if isinstance(out, bytes): - return out.decode() - else: - return out + remote_info = teuthology.lock.query.get_status(self.hostname) + if not remote_info: + return None + self._machine_type = remote_info.get("machine_type", None) + return self._machine_type - def sh_file(self, script, label="script", sudo=False, **kwargs): - """ - Run shell script after copying its contents to a remote file + @property + def is_reimageable(self): + return self.machine_type in self._reimage_types - :param script: string with script text, or file object - :param sudo: run command with sudo if True, - run as user name if string value (defaults to False) - :param label: string value which will be part of file name - Returns: stdout - """ - ftempl = '/tmp/teuthology-remote-$(date +%Y%m%d%H%M%S)-{}-XXXX'\ - .format(label) - script_file = self.sh("mktemp %s" % ftempl).strip() - self.sh("cat - | tee {script} ; chmod a+rx {script}"\ - .format(script=script_file), stdin=script) - if sudo: - if isinstance(sudo, str): - command="sudo -u %s %s" % (sudo, script_file) - else: - command="sudo %s" % script_file - else: - command="%s" % script_file + @property + def shortname(self): + if self._shortname is None: + self._shortname = host_shortname(self.hostname) + return self._shortname - return self.sh(command, **kwargs) + @property + def is_online(self): + if self.ssh is None: + return False + if self.ssh.get_transport() is None: + return False + try: + self.run(args="true") + except Exception: + return False + return self.ssh.get_transport().is_active() - def chmod(self, file_path, permissions): + def ensure_online(self): + if self.is_online: + return + self.connect() + if not self.is_online: + raise Exception('unable to connect') + + @property + def system_type(self): """ - As super-user, set permissions on the remote file specified. + System type decorator """ - args = [ - 'sudo', - 'chmod', - permissions, - file_path, - ] - self.run( - args=args, + return misc.get_system_type(self) + + def __str__(self): + return self.name + + def __repr__(self): + return '{classname}(name={name!r})'.format( + classname=self.__class__.__name__, + name=self.name, ) - def chcon(self, file_path, context): + def run(self, **kwargs): """ - Set the SELinux context of a given file. - - VMs and non-RPM-based hosts will skip this operation because ours - currently have SELinux disabled. + This calls `orchestra.run.run` with our SSH client. - :param file_path: The path to the file - :param context: The SELinux context to be used + TODO refactor to move run.run here? """ - if self.os.package_type != 'rpm' or \ - self.os.name in ['opensuse', 'sle']: - return - if teuthology.lock.query.is_vm(self.shortname): - return - self.run(args="sudo chcon {con} {path}".format( - con=context, path=file_path)) + if not self.ssh or \ + not self.ssh.get_transport() or \ + not self.ssh.get_transport().is_active(): + self.reconnect() + r = self._runner(client=self.ssh, name=self.shortname, **kwargs) + r.remote = self + return r def _sftp_put_file(self, local_path, remote_path): """ @@ -374,9 +556,6 @@ class Remote(object): file_size = file_size / 1024.0 return "{:3.0f}{}".format(file_size, unit) - def remove(self, path): - self.run(args=['rm', '-fr', path]) - def put_file(self, path, dest_path, sudo=False): """ Copy a local filename to a remote file @@ -468,165 +647,6 @@ class Remote(object): ]) return self.run(args=args, wait=False, stdout=run.PIPE) - def copy_file(self, src, dst, sudo=False, mode=None, owner=None, - mkdir=False, append=False): - """ - Copy data to remote file - - :param src: source file path on remote host - :param dst: destination file path on remote host - :param sudo: use sudo to write file, defaults False - :param mode: set file mode bits if provided - :param owner: set file owner if provided - :param mkdir: ensure the destination directory exists, defaults False - :param append: append data to the file, defaults False - """ - dd = 'sudo dd' if sudo else 'dd' - args = dd + ' if=' + src + ' of=' + dst - if append: - args += ' conv=notrunc oflag=append' - if mkdir: - mkdirp = 'sudo mkdir -p' if sudo else 'mkdir -p' - dirpath = os.path.dirname(dst) - if dirpath: - args = mkdirp + ' ' + dirpath + '\n' + args - if mode: - chmod = 'sudo chmod' if sudo else 'chmod' - args += '\n' + chmod + ' ' + mode + ' ' + dst - if owner: - chown = 'sudo chown' if sudo else 'chown' - args += '\n' + chown + ' ' + owner + ' ' + dst - args = 'set -ex' + '\n' + args - self.run(args=args) - - def move_file(self, src, dst, sudo=False, mode=None, owner=None, - mkdir=False): - """ - Move data to remote file - - :param src: source file path on remote host - :param dst: destination file path on remote host - :param sudo: use sudo to write file, defaults False - :param mode: set file mode bits if provided - :param owner: set file owner if provided - :param mkdir: ensure the destination directory exists, defaults False - """ - mv = 'sudo mv' if sudo else 'mv' - args = mv + ' ' + src + ' ' + dst - if mkdir: - mkdirp = 'sudo mkdir -p' if sudo else 'mkdir -p' - dirpath = os.path.dirname(dst) - if dirpath: - args = mkdirp + ' ' + dirpath + '\n' + args - if mode: - chmod = 'sudo chmod' if sudo else 'chmod' - args += ' && ' + chmod + ' ' + mode + ' ' + dst - if owner: - chown = 'sudo chown' if sudo else 'chown' - args += ' && ' + chown + ' ' + owner + ' ' + dst - self.run(args=args) - - def read_file(self, path, sudo=False, stdout=None, - offset=0, length=0): - """ - Read data from remote file - - :param path: file path on remote host - :param sudo: use sudo to read the file, defaults False - :param stdout: output object, defaults to io.BytesIO() - :param offset: number of bytes to skip from the file - :param length: number of bytes to read from the file - - :raises: :class:`FileNotFoundError`: there is no such file by the path - :raises: :class:`RuntimeError`: unexpected error occurred - - :returns: the file contents in bytes, if stdout is `io.BytesIO`, by default - :returns: the file contents in str, if stdout is `io.StringIO` - """ - dd = 'sudo dd' if sudo else 'dd' - args = dd + ' if=' + path + ' of=/dev/stdout' - iflags=[] - # we have to set defaults here instead of the method's signature, - # because python is reusing the object from call to call - stdout = stdout or BytesIO() - if offset: - args += ' skip=' + str(offset) - iflags += 'skip_bytes' - if length: - args += ' count=' + str(length) - iflags += 'count_bytes' - if iflags: - args += ' iflag=' + ','.join(iflags) - args = 'set -ex' + '\n' + args - proc = self.run(args=args, stdout=stdout, stderr=StringIO(), check_status=False, quiet=True) - if proc.returncode: - if 'No such file or directory' in proc.stderr.getvalue(): - raise FileNotFoundError(errno.ENOENT, - f"Cannot find file on the remote '{self.name}'", path) - else: - raise RuntimeError("Unexpected error occurred while trying to " - f"read '{path}' file on the remote '{self.name}'") - - return proc.stdout.getvalue() - - - def write_file(self, path, data, sudo=False, mode=None, owner=None, - mkdir=False, append=False): - """ - Write data to remote file - - :param path: file path on remote host - :param data: str, binary or fileobj to be written - :param sudo: use sudo to write file, defaults False - :param mode: set file mode bits if provided - :param owner: set file owner if provided - :param mkdir: preliminary create the file directory, defaults False - :param append: append data to the file, defaults False - """ - dd = 'sudo dd' if sudo else 'dd' - args = dd + ' of=' + path - if append: - args += ' conv=notrunc oflag=append' - if mkdir: - mkdirp = 'sudo mkdir -p' if sudo else 'mkdir -p' - dirpath = os.path.dirname(path) - if dirpath: - args = mkdirp + ' ' + dirpath + '\n' + args - if mode: - chmod = 'sudo chmod' if sudo else 'chmod' - args += '\n' + chmod + ' ' + mode + ' ' + path - if owner: - chown = 'sudo chown' if sudo else 'chown' - args += '\n' + chown + ' ' + owner + ' ' + path - args = 'set -ex' + '\n' + args - self.run(args=args, stdin=data, quiet=True) - - def sudo_write_file(self, path, data, **kwargs): - """ - Write data to remote file with sudo, for more info see `write_file()`. - """ - self.write_file(path, data, sudo=True, **kwargs) - - @property - def os(self): - if not hasattr(self, '_os'): - try: - os_release = self.sh('cat /etc/os-release').strip() - self._os = OS.from_os_release(os_release) - return self._os - except CommandFailedError: - pass - - lsb_release = self.sh('lsb_release -a').strip() - self._os = OS.from_lsb_release(lsb_release) - return self._os - - @property - def arch(self): - if not hasattr(self, '_arch'): - self._arch = self.sh('uname -m').strip() - return self._arch - @property def host_key(self): if not self._host_key: -- 2.39.5