From f2eb86ea0c4a96099fecc3951f5d880a2bc2271d Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Tue, 14 Oct 2025 21:04:42 +0800 Subject: [PATCH] cephadm/build: Add Debian package support for bundled dependencies Extends the cephadm build script to support bundling dependencies from Debian packages in addition to pip and RPM packages. This allows building cephadm on Debian-based distributions using system packages. Key changes: - Add 'deb' to DependencyMode enum to enable Debian package mode - Implement _setup_deb() to configure Debian dependency requirements - Add _install_deb_deps() to orchestrate Debian package installation - Add _gather_deb_package_dirs() to parse Debian package file listings and locate Python package directories (handles both site-packages and dist-packages directories used by Debian) - Add _deps_from_deb() to extract Python dependencies from installed Debian packages using dpkg/apt-cache tools - Fix variable reference bug in _install_deps() (deps.mode -> config.deps_mode) The Debian implementation follows a similar pattern to the existing RPM support, using dpkg-query and dpkg -L to locate installed packages and their files, with special handling for Debian naming conventions (e.g., PyYAML -> python3-yaml). Signed-off-by: Kefu Chai --- src/cephadm/build.py | 156 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 155 insertions(+), 1 deletion(-) diff --git a/src/cephadm/build.py b/src/cephadm/build.py index d86737dfd2a..0c5cc329b9e 100755 --- a/src/cephadm/build.py +++ b/src/cephadm/build.py @@ -154,6 +154,7 @@ class PipEnv(enum.Enum): class DependencyMode(enum.Enum): pip = enum.auto() rpm = enum.auto() + deb = enum.auto() none = enum.auto() @@ -169,6 +170,8 @@ class Config: self._setup_pip() elif self.deps_mode == DependencyMode.rpm: self._setup_rpm() + elif self.deps_mode == DependencyMode.deb: + self._setup_deb() def _setup_pip(self): if self._maj_min == (3, 6): @@ -180,6 +183,9 @@ class Config: def _setup_rpm(self): self.requirements = [InstallSpec(**v) for v in PY_REQUIREMENTS] + def _setup_deb(self): + self.requirements = [InstallSpec(**v) for v in PY_REQUIREMENTS] + class DependencyInfo: """Type for tracking bundled dependencies.""" @@ -336,7 +342,9 @@ def _install_deps(tempdir, config): return _install_pip_deps(tempdir, config) if config.deps_mode == DependencyMode.rpm: return _install_rpm_deps(tempdir, config) - raise ValueError(f'unexpected deps mode: {deps.mode}') + if config.deps_mode == DependencyMode.deb: + return _install_deb_deps(tempdir, config) + raise ValueError(f'unexpected deps mode: {config.deps_mode}') def _install_pip_deps(tempdir, config): @@ -437,6 +445,15 @@ def _install_rpm_deps(tempdir, config): return dinfo +def _install_deb_deps(tempdir, config): + log.info("Installing dependencies using Debian packages") + dinfo = DependencyInfo(config) + for pkg in config.requirements: + log.info(f"Looking for debian package for: {pkg.name!r}") + _deps_from_deb(tempdir, config, dinfo, pkg.name) + return dinfo + + def _gather_rpm_package_dirs(paths): # = The easy way = # the top_level.txt file can be used to determine where the python packages @@ -478,6 +495,52 @@ def _gather_rpm_package_dirs(paths): return dist_info, siblings +def _gather_deb_package_dirs(paths): + """Parse Debian package file listing to find Python package directories. + Similar to _gather_rpm_package_dirs but for Debian packages. + """ + # = The easy way = + # the top_level.txt file can be used to determine where the python packages + # actually are. We need all of those and the meta-data dir (parent of + # top_level.txt) to be included in our zipapp + top_level = None + for path in paths: + if path.endswith('top_level.txt'): + top_level = pathlib.Path(path) + if top_level: + meta_dir = top_level.parent + pkg_dirs = [ + top_level.parent.parent / p + for p in top_level.read_text().splitlines() + ] + return meta_dir, pkg_dirs + # = The hard way = + # loop through the directories to find the .dist-info dir (containing the + # mandatory METADATA file, according to the spec) and once we know the + # location of dist info we find the sibling paths from the deb listing + dist_info = None + ppaths = [] + for path in paths: + ppath = pathlib.Path(path) + ppaths.append(ppath) + if ppath.name == 'METADATA' and ppath.parent.name.endswith('.dist-info'): + dist_info = ppath.parent + break + if not dist_info: + raise ValueError('no .dist-info METADATA found') + # Debian uses 'dist-packages' for all system-managed Python packages + # per Debian Python Policy: https://www.debian.org/doc/packaging-manuals/python-policy/ + if dist_info.parent.name != 'dist-packages': + raise ValueError( + 'unexpected parent directory (not dist-packages):' + f' {dist_info.parent.name}' + ) + siblings = [ + p for p in ppaths if p.parent == dist_info.parent and p != dist_info + ] + return dist_info, siblings + + def _deps_from_rpm(tempdir, config, dinfo, pkg): # first, figure out what rpm provides a particular python lib dist = f'python3.{sys.version_info.minor}dist({pkg})'.lower() @@ -525,6 +588,97 @@ def _deps_from_rpm(tempdir, config, dinfo, pkg): shutil.copytree(pkg_dir, pkg_dest, ignore=_ignore_cephadmlib) +def _deps_from_deb(tempdir, config, dinfo, pkg): + """Extract Python dependencies from Debian packages. + + Args: + tempdir: Temporary directory to copy package files to + config: Build configuration + dinfo: DependencyInfo instance to track dependencies + pkg: Python package name (e.g., 'MarkupSafe', 'Jinja2', 'PyYAML') + """ + # Convert Python package name to Debian package name + # Python packages are typically named python3- + # Handle special cases: PyYAML -> python3-yaml, MarkupSafe -> python3-markupsafe + pkg_lower = pkg.lower() + if pkg_lower == 'pyyaml': + deb_pkg_name = 'python3-yaml' + else: + deb_pkg_name = f'python3-{pkg_lower}' + + # First, try to find the package using apt-cache + # This helps verify the package exists before trying to list its files + try: + res = subprocess.run( + ['apt-cache', 'show', deb_pkg_name], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + except subprocess.CalledProcessError: + # Package not found, try alternative naming + log.warning(f"Package {deb_pkg_name} not found via apt-cache, trying dpkg -S") + # Try to search for files that might belong to this package + # Search for the Python module in site-packages + search_pattern = f'/usr/lib/python3*/dist-packages/{pkg.lower()}*' + try: + res = subprocess.run( + ['dpkg', '-S', search_pattern], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + # dpkg -S output format: "package: /path/to/file" + deb_pkg_name = res.stdout.decode('utf8').split(':')[0].strip() + except subprocess.CalledProcessError as err: + log.error(f"Could not find Debian package for {pkg}") + log.error(f"Tried: {deb_pkg_name} and pattern search") + sys.exit(1) + + # Get version information using dpkg-query + try: + res = subprocess.run( + ['dpkg-query', '-W', '-f=${Version}\\n', deb_pkg_name], + check=True, + stdout=subprocess.PIPE, + ) + version = res.stdout.decode('utf8').strip() + except subprocess.CalledProcessError as err: + log.error(f"Could not query version for package {deb_pkg_name}: {err}") + sys.exit(1) + + log.info(f"Debian Package: {deb_pkg_name} (version: {version})") + dinfo.add( + pkg, + deb_name=deb_pkg_name, + version=version, + package_source='deb', + ) + + # Get the list of files provided by the Debian package + try: + res = subprocess.run( + ['dpkg', '-L', deb_pkg_name], + check=True, + stdout=subprocess.PIPE, + ) + except subprocess.CalledProcessError as err: + log.error(f"Could not list files for package {deb_pkg_name}: {err}") + sys.exit(1) + + paths = [l.decode('utf8') for l in res.stdout.splitlines()] + meta_dir, pkg_dirs = _gather_deb_package_dirs(paths) + meta_dest = tempdir / meta_dir.name + log.info(f"Copying {meta_dir} to {meta_dest}") + # copy the meta data directory + shutil.copytree(meta_dir, meta_dest, ignore=_ignore_cephadmlib) + # copy all the package directories + for pkg_dir in pkg_dirs: + pkg_dest = tempdir / pkg_dir.name + log.info(f"Copying {pkg_dir} to {pkg_dest}") + shutil.copytree(pkg_dir, pkg_dest, ignore=_ignore_cephadmlib) + + def generate_version_file(versioning_vars, dest): log.info("Generating version file") log.debug("versioning_vars=%r", versioning_vars) -- 2.39.5