From 524385b671924131e04fd8b074b0942ad0a31564 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Mon, 25 Jul 2022 17:01:12 -0400 Subject: [PATCH] script: add a new cpatch.py that is like cpatch but in python This is a somewhat straightforward translation of the current cpatch bash script into python. The image is built in the same manner but some of the python components can be selected in a more granular manner. Why do this? I had my own private copy of the cpatch script that I have been modifying for my own needs, one that has the previously mentioned more granular selection of python components. However, I had additional plans for this script and found it difficult to manage and I felt that a single-source-file python program would allow the use of python's nicer (IMO) CLI parsing, data structures, and standard library modules. I also think that this provides a cleaner program structure that will allow for experimentation with different backends vs. the current approach of copying files and creating a temporary Dockerfile. For whatever reason the original cpatch was assuming that the python version was 3.8, however in the current ceph images using centos 8 stream as a base, python 3.6 is the available python version. Old versions of ceph kept cephadm as one large single source file in the source tree. New versions "compile" cephadm into a python zipapp. Try to automatically detect which case and "install" the correct form of cephadm. Signed-off-by: John Mulligan --- src/script/cpatch.py | 726 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 726 insertions(+) create mode 100755 src/script/cpatch.py diff --git a/src/script/cpatch.py b/src/script/cpatch.py new file mode 100755 index 0000000000000..cbca0587dde8b --- /dev/null +++ b/src/script/cpatch.py @@ -0,0 +1,726 @@ +#!/usr/bin/python +"""Patch Ceph Container images with development (work-in-progress) build +artifacts and sources. + +""" + +from functools import lru_cache +import argparse +import enum +import logging +import logging.config +import os +import pathlib +import shutil +import subprocess +import sys +import tarfile +import tempfile +import time + +log = logging.getLogger() + + +BASE_IMAGE = "quay.ceph.io/ceph-ci/ceph:main" +DEST_IMAGE = "ceph/ceph:wip" + +# Set CLEANUP to false to skip cleaning up temporary files for debugging. +CLEANUP = True +CATCH_ERRORS = True + + +class Component(enum.Enum): + """Component to be copied into the new container image.""" + + PY_MGR = ("mgr", "python", False) + PY_COMMON = ("common", "python", False) + CEPHADM = ("cephadm", "python", False) + PYBIND = ("bindings", "python", True) + DASHBOARD = ("dashboard", "dashboard", True) + CORE = ("core", "core", True) + CEPHFS = ("cephfs", "core", True) + RBD = ("rbd", "core", True) + RGW = ("rgw", "rgw", True) + + def __init__(self, name, group, needs_compilation=True): + self.component_name = name + self.component_group = group + self.needs_compilation = needs_compilation + + def __str__(self): + return f"{self.component_group}/{self.component_name}" + + +def _exclude_dirs(dirs, excludes): + """Exclude directories from a fs walk.""" + for exclude in excludes: + try: + del dirs[dirs.index(exclude)] + except ValueError: + pass + + +def _run(cmd, *args, **kwargs): + """Wrapper for subprocess.run with additional logging""" + log.debug("running command: %r", cmd) + result = subprocess.run(cmd, *args, **kwargs) + log.debug("command exited with returncode=%d", result.returncode) + return result + + +def check_build_dir(cli_ctx): + """Raise an error if the build dir is not populated by a build.""" + log.debug("checking if build dir is OK") + bdir = cli_ctx.build_dir + makefile = bdir / "Makefile" + ninja = bdir / "build.ninja" + if not (makefile.is_file() or ninja.is_file()): + log.error(f"neither {makefile} nor {ninja} found") + raise ValueError("invalid build dir") + return + + +def pull_base_image(cli_ctx): + """Pull the base image.""" + cmd = [cli_ctx.engine, "pull", cli_ctx.base_image] + if cli_ctx.root_build: + cmd.insert(0, "sudo") + _run(cmd).check_returncode() + + +def push_image(cli_ctx): + """Push the target image. Assumes login already performed.""" + cmd = [cli_ctx.engine, "push", cli_ctx.target] + if cli_ctx.root_build: + cmd.insert(0, "sudo") + _run(cmd).check_returncode() + + +class ChangeDir: + """Context manager for temporarily changing directory.""" + + def __init__(self, path): + self.path = path + self.orig = None + + def __enter__(self): + self.orig = pathlib.Path.cwd().absolute() + os.chdir(self.path) + return self + + def __exit__(self, exc_type, exc_value, traceback): + os.chdir(self.orig) + + +class AddComponentsAction(argparse.Action): + """Helper for adding a component to the CLI.""" + + def __init__( + self, + option_strings, + dest, + const=None, + default=False, + required=False, + help=None, + ): + super().__init__( + option_strings=option_strings, + dest=dest, + const=const, + nargs=0, + default=default, + required=required, + help=help, + ) + + def __call__(self, parser, namespace, values, option_string): + destination = getattr(namespace, self.dest, None) or [] + try: + comps = list(self.const) + except TypeError: + comps = [self.const] + for comp in comps: + if comp not in destination: + destination.append(comp) + setattr(namespace, self.dest, destination) + + @classmethod + def add_argument(cls, parser, long_opt, components, hint): + parser.add_argument( + long_opt, + dest="components", + action=cls, + const=components, + help=f"Includes: {hint}", + ) + + +class CLIContext: + """Manages state related to the CLI.""" + + def __init__(self, cli_ns): + self._cli = cli_ns + + @property + def needs_build_dir(self): + return any(c.needs_compilation for c in self.build_components()) + + @property + @lru_cache() + def engine(self): + for ctr_eng in ["podman", "docker"]: + if shutil.which(ctr_eng): + break + else: + raise ValueError("no container engine found") + log.debug("found container engine: %r", ctr_eng) + return pathlib.Path(ctr_eng) + + @property + def base_image(self): + return self._cli.base + + @property + def target(self): + return self._cli.target + + @property + def root_build(self): + return self._cli.root_build + + @property + def build_dir(self): + if self._cli.build_dir: + return pathlib.Path(self._cli.build_dir).absolute() + return pathlib.Path(".").absolute() + + @property + def source_dir(self): + if self._cli.source_dir: + return pathlib.Path(self._cli.source_dir).absolute() + return (self.build_dir / "..").absolute() + + @property + def push(self): + return self._cli.push + + @property + def pull(self): + return self._cli.pull + + @property + def strip_binaries(self): + return self._cli.strip + + @property + def components_selected(self): + return bool(self._cli.components) + + def build_components(self): + if self._cli.components: + return self._cli.components + # because everything is nothing. + return list(Component) + + def setup_logging(self): + logging.config.dictConfig( + { + "version": 1, + "disable_existing_loggers": True, + "formatters": { + "default": { + "class": "logging.Formatter", + "format": "%(asctime)s %(levelname)s: %(message)s", + }, + }, + "handlers": { + "stderr": { + "formatter": "default", + "level": "DEBUG", + "class": "logging.StreamHandler", + "stream": sys.stderr, + } + }, + "root": { + "level": self._cli.log_level, + "handlers": ["stderr"], + }, + } + ) + + @classmethod + def parse(cls): + parser = argparse.ArgumentParser() + parser.add_argument( + "--base", + default=BASE_IMAGE, + help=f"base container image [default: {BASE_IMAGE}]", + ) + parser.add_argument( + "--target", + default=DEST_IMAGE, + help=f"target image to create [default: {DEST_IMAGE}]", + ) + parser.add_argument( + "--push", action="store_true", help="push when done" + ) + parser.add_argument( + "--pull", + action="store_true", + help=( + "pull base image before building (useful to prevent building" + " with a stale base image)" + ), + ) + parser.add_argument( + "--build-dir", + "-b", + help="path to the build dir (defaults to current workding dir)", + ) + parser.add_argument( + "--source-dir", + help="path to the source dir (defaults to parent of build dir)", + ) + parser.add_argument( + "--strip", action="store_true", help="strip binaries" + ) + parser.add_argument( + "--root-build", action="store_true", help="build image as root" + ) + parser.add_argument("--container-engine") + parser.set_defaults(log_level=logging.INFO) + parser.add_argument( + "--debug", + action="store_const", + dest="log_level", + const=logging.DEBUG, + help="Enable debug level logging", + ) + parser.add_argument( + "--quiet", + "-q", + action="store_const", + dest="log_level", + const=logging.WARNING, + help="Only print errors and warnings", + ) + # selectors + component_selections = [ + # aggregated components: + ( + # aggregates all python components + "--py", + "python", + [ + Component.CEPHADM, + Component.PY_MGR, + Component.PY_COMMON, + Component.PYBIND, + ], + ), + ( + # aggregates python components that don't need to compile + "--pure-py", + "non-compiled python", + [ + Component.CEPHADM, + Component.PY_MGR, + Component.PY_COMMON, + ], + ), + # currently no single component selectors for rbd & cephfs + # this is the way it was in the shell script too. + ( + "--core", + "mon, mgr, osd, mds, bins and libraries", + [ + Component.CORE, + Component.RBD, + Component.CEPHFS, + ], + ), + # single components: + ( + "--python-common", + "common python libs", + [ + Component.PY_COMMON, + ], + ), + ( + "--python-bindings", + "compiled python bindings", + [ + Component.PYBIND, + ], + ), + ( + "--python-mgr-modules", + "mgr modules written in python", + [ + Component.PY_MGR, + ], + ), + ( + "--cephadm", + "cephadm command", + [ + Component.CEPHADM, + ], + ), + ( + "--dashboard", + "dashboard", + [ + Component.DASHBOARD, + ], + ), + ( + "--rgw", + "radosgw, radosgw-admin", + [ + Component.RGW, + ], + ), + ] + for long_opt, hint, comps in component_selections: + AddComponentsAction.add_argument( + parser, + long_opt, + components=comps, + hint=hint, + ) + + return cls(parser.parse_args()) + + +class Builder: + """Builds an image based on the components added.""" + + def __init__(self, cli_ctx): + self._ctx = cli_ctx + self._jobs = [] + self._include_dashboard = False + self._cached_py_site_packages = None + self._workdir = pathlib.Path( + tempfile.mkdtemp(suffix=".cpatch", dir=self._ctx.build_dir) + ) + log.debug("cpatch working dir: %s", self._workdir) + + def add(self, component): + """Request that the given component be added to the build.""" + log.info("Including: %s", component) + dispatch = { + Component.PY_COMMON: self._py_common_job, + Component.CEPHADM: self._cephadm_job, + Component.PY_MGR: self._py_mgr_job, + Component.DASHBOARD: self._py_mgr_job, + Component.PYBIND: self._pybind_job, + Component.CORE: self._core_job, + Component.CEPHFS: self._cephfs_job, + Component.RBD: self._rbd_job, + Component.RGW: self._rgw_job, + } + job = dispatch[component] + if component == Component.DASHBOARD: + self._include_dashboard = True + if job not in {j for _, j in self._jobs}: + self._jobs.append((component, job)) + + def build(self): + """Build the container image.""" + dlines = [f"FROM {self._ctx.base_image}"] + jcount = len(self._jobs) + for idx, (component, job) in enumerate(self._jobs): + num = idx + 1 + log.info(f"Executing job {num}/{jcount} for {component}") + dresult = job(component) + dlines.extend(dresult) + + with open(self._workdir / "Dockerfile", "w") as fout: + for line in dlines: + print(line, file=fout) + self._container_build() + + def _container_build(self): + log.info("Building container image") + cmd = [self._ctx.engine, "build", "--tag", self._ctx.target, "."] + if self._ctx.root_build: + cmd.insert(0, "sudo") + _run(cmd, cwd=self._workdir).check_returncode() + + def _build_tar( + self, + tar, + start_path=".", + exclude_dirs=None, + exclude_file_suffixes=None, + ): + + for cur, dirs, files in os.walk(start_path): + cur = pathlib.Path(cur) + # skip 'tests' directories in both tar and walk + if exclude_dirs: + _exclude_dirs(dirs, exclude_dirs) + for sdir in dirs: + ddir = cur / sdir + tar.add(ddir, recursive=False) + for sfile in files: + if exclude_file_suffixes and sfile.endswith( + exclude_file_suffixes + ): + continue + dfile = cur / sfile + tar.add(dfile, recursive=False) + + def _copy_binary(self, src_path, dst_path): + log.debug("binary: %s -> %s", src_path, dst_path) + if self._ctx.strip_binaries: + log.debug("copy and strip: %s", dst_path) + shutil.copy2(src_path, dst_path) + _run(["strip", str(dst_path)]).check_returncode() + return + log.debug("hard linking: %s", dst_path) + try: + os.unlink(dst_path) + except FileNotFoundError: + pass + os.link(src_path, dst_path) + + def _bins_and_libs(self, prefix, bin_patterns, lib_patterns): + out = [] + + bin_src = self._ctx.build_dir / "bin" + bin_dst = self._workdir / f"{prefix}_bin" + bin_dst.mkdir(parents=True, exist_ok=True) + for path in bin_src.iterdir(): + if any(path.match(m) for m in bin_patterns): + self._copy_binary(path, bin_dst / path.name) + out.append(f"ADD {prefix}_bin /usr/bin") + + lib_src = self._ctx.build_dir / "lib" + lib_dst = self._workdir / f"{prefix}_lib" + lib_dst.mkdir(parents=True, exist_ok=True) + for path in lib_src.iterdir(): + if any(path.match(m) for m in lib_patterns): + self._copy_binary(path, lib_dst / path.name) + out.append(f"ADD {prefix}_lib /usr/lib64") + + return out + + def _conditional_libs(self, src_dir, name, destination, lib_patterns): + lib_src = self._ctx.build_dir / src_dir + lib_dst = self._workdir / name + lib_dst.mkdir(parents=True, exist_ok=True) + try: + for path in lib_src.iterdir(): + if any(path.match(m) for m in lib_patterns): + self._copy_binary(path, lib_dst / path.name) + except FileNotFoundError as err: + log.warning("skipping lib %s: %s", name, err) + return f"ADD {name} {destination}" + + def _py_site_packages(self): + """Return the correct python site packages dir for the image.""" + if self._cached_py_site_packages is not None: + return self._cached_py_site_packages + # use the container image to probe for the correct python site-packages dir + valid_site_packages = [ + "/usr/lib/python3.8/site-packages", + "/usr/lib/python3.6/site-packages", + ] + cmd = [ + self._ctx.engine, + "run", + "--rm", + self._ctx.base_image, + "ls", + "-d", + ] + cmd += valid_site_packages + if self._ctx.root_build: + cmd.insert(0, "sudo") + result = _run(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + log.debug(f"container output: {result.stdout!r}") + for line in result.stdout.decode("utf8").splitlines(): + pdir = line.strip() + if line.strip() in valid_site_packages: + log.debug(f"found site packages dir: {pdir}") + self._cached_py_site_packages = pdir + return self._cached_py_site_packages + raise ValueError( + "no valid python site-packages dir found in container" + ) + + def _py_mgr_job(self, component): + name = "mgr_plugins.tar" + if self._include_dashboard: + log.debug("Including dashboard with mgr") + exclude_dirs = ("tests",) + else: + log.debug("Excluding dashboard from mgr") + exclude_dirs = ("tests", "node_modules") + exclude_file_suffixes = (".pyc", ".pyo", ".tmp", "~") + with tarfile.open(self._workdir / name, mode="w") as tar: + with ChangeDir(self._ctx.source_dir / "src/pybind/mgr"): + self._build_tar( + tar, + exclude_dirs=exclude_dirs, + exclude_file_suffixes=exclude_file_suffixes, + ) + return [f"ADD {name} /usr/share/ceph/mgr"] + + def _py_common_job(self, component): + name = "python_common.tar" + exclude_dirs = ("tests", "node_modules") + exclude_file_suffixes = (".pyc", ".pyo", ".tmp", "~") + with tarfile.open(self._workdir / name, mode="w") as tar: + with ChangeDir(self._ctx.source_dir / "src/python-common"): + self._build_tar( + tar, + exclude_dirs=exclude_dirs, + exclude_file_suffixes=exclude_file_suffixes, + ) + return [f"ADD {name} {self._py_site_packages()}"] + + def _cephadm_job(self, component): + src_path = self._ctx.source_dir / "src/cephadm/cephadm" + dst_path = self._workdir / "cephadm" + if src_path.is_file(): + # traditional, uncompiled cephadm + log.debug(f"found single-source file cephadm: {src_path}") + os.link(src_path, dst_path) + else: + build_cephadm_path = self._ctx.source_dir / "src/cephadm/build.py" + if not build_cephadm_path.is_file(): + raise ValueError("no cephadm build script found") + log.debug("found cephadm compilation script: compiling cephadm") + _run([build_cephadm_path, dst_path]).check_returncode() + return ["ADD cephadm /usr/sbin/cephadm"] + + def _pybind_job(self, component): + sodir = self._ctx.build_dir / "lib/cython_modules/lib.3" + dst = self._workdir / "cythonlib" + dst.mkdir(parents=True, exist_ok=True) + for pth in sodir.iterdir(): + if pth.match("*.cpython-3*.so"): + self._copy_binary(pth, dst / pth.name) + return [f"ADD cythonlib {self._py_site_packages()}"] + + def _core_job(self, component): + # [Quoth the original script]: + # binaries are annoying because the ceph version is embedded all over + # the place, so we have to include everything but the kitchen sink. + out = [] + + out.extend( + self._bins_and_libs( + prefix="core", + bin_patterns=["ceph-mgr", "ceph-mon", "ceph-osd", "rados"], + lib_patterns=["libceph-common.so*", "librados.so*"], + ) + ) + + out.append( + self._conditional_libs( + src_dir="lib", + name="eclib", + destination="/usr/lib64/ceph/erasure-code", + lib_patterns=["libec_*.so*"], + ) + ) + out.append( + self._conditional_libs( + src_dir="lib", + name="clslib", + destination="/usr/lib64/rados-classes", + lib_patterns=["libcls_*.so*"], + ) + ) + + # [Quoth the original script]: + # by default locally built binaries assume /usr/local + out.append( + "RUN rm -rf /usr/local/lib64 && ln -s /usr/lib64 /usr/local && ln -s /usr/share/ceph /usr/local/share" + ) + + return out + + def _rgw_job(self, component): + return self._bins_and_libs( + prefix="rgw", + bin_patterns=["radosgw", "radosgw-admin"], + lib_patterns=["libradosgw.so*"], + ) + return out + + def _cephfs_job(self, component): + return self._bins_and_libs( + prefix="cephfs", + bin_patterns=["ceph-mds"], + lib_patterns=["libcephfs.so*"], + ) + return out + + def _rbd_job(self, component): + return self._bins_and_libs( + prefix="rbd", + bin_patterns=["rbd", "rbd-mirror"], + lib_patterns=["librbd.so*"], + ) + return out + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + if CLEANUP: + shutil.rmtree(self._workdir) + return + + +def clean_errors(f): + def wrapped(*args, **kwargs): + try: + return f(*args, **kwargs) + except ValueError as err: + if not CATCH_ERRORS: + raise + print("ERROR: {}".format(err)) + sys.exit(1) + except subprocess.CalledProcessError: + if not CATCH_ERRORS: + raise + print("ERROR: command failed: {}".format(err)) + sys.exit(1) + + return wrapped + + +@clean_errors +def main(): + cli_ctx = CLIContext.parse() + cli_ctx.setup_logging() + if not cli_ctx.components_selected: + log.warning( + "consider --py, --core, and/or --rgw for an abbreviated (faster) build." + ) + time.sleep(2) + + if cli_ctx.needs_build_dir: + check_build_dir(cli_ctx) + + if cli_ctx.pull: + pull_base_image(cli_ctx) + with Builder(cli_ctx) as builder: + for component in cli_ctx.build_components(): + builder.add(component) + builder.build() + + if cli_ctx.push: + push_image(cli_ctx) + + +if __name__ == "__main__": + main() -- 2.47.3