From: John Agombar Date: Tue, 9 Jul 2024 09:19:49 +0000 (+0100) Subject: cpatch.py: search base image when patching files X-Git-Tag: v20.0.0~1501^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=051a304154ebf445d339b3ca5bd36cdfd2da0e5d;p=ceph.git cpatch.py: search base image when patching files Refactor the script so that for each component specified: binaries are copied to the /tmpbin directory libraries are copied to the /tmplib directory For the core component ensure that ALL locally built libraries are copied to the /tmplib directory rather than a subset of those that are used by the binaries If no component is specified then copy ALL locally built binaries are copied to the /tmpbin directory Add commands to the Dockerfile so that the build process does the following: For every file in the /tmpbin directory: find the file with the same name in the base container image (search /usr/bin amd /usr/sbin directories) move the file - replacing the version in the base image For every file in the /tmplib directory: find the file with the same name in the base container image (search /usr/lib64 directory) move the file - replacing the version in the base image Create some links to the libraries that were just copied because locally built binaries assume that libraries are in a different directory to those in the base image. (original script did this too). Signed-off-by: John Agombar --- diff --git a/src/script/cpatch.py b/src/script/cpatch.py index 200bda837ca4..5e33e9b2c13c 100755 --- a/src/script/cpatch.py +++ b/src/script/cpatch.py @@ -459,6 +459,22 @@ class Builder: dresult = job(component) dlines.extend(dresult) + if os.path.isdir(self._workdir / "tmp_bin"): + # For every binary file that was copied to the tmp_bin directory by the jobs above, search for the existing file in the container and replace it. + dlines.append("ADD tmp_bin /tmpbin") + dlines.append("RUN for i in tmpbin/*; do find /usr/bin /usr/sbin -name $(basename $i) -exec mv -f $i '{}' \;; echo $(basename $i); done && rm -rf tmpbin") + + if os.path.isdir(self._workdir / "tmp_lib"): + # For every library file that was copied to the tmp_lib directory by the jobs above, search for the existing file in the container and replace it. + dlines.append("ADD tmp_lib /tmplib") + dlines.append("RUN for i in tmplib/*; do find /usr/lib64 -name $(basename $i) -exec mv -f $i '{}' \;; echo $(basename $i); done && rm -rf tmplib") + + if os.path.isdir(self._workdir / "tmp_bin"): + # by default locally built binaries assume /usr/local + dlines.append("RUN rm -rf /usr/local/lib64 && ln -sf /usr/lib64 /usr/local && ln -sf /usr/share/ceph /usr/local/share") + # locally built binaries assume libceph-common.so.2 is in /usr/lib64 - create link to library that was just copied + dlines.append("RUN ln -sf /usr/lib64/ceph/libceph-common.so.2 /usr/lib64/libceph-common.so.2") + with open(self._workdir / "Dockerfile", "w") as fout: for line in dlines: print(line, file=fout) @@ -513,7 +529,7 @@ class Builder: pass os.link(src_path, dst_path) - def _bins_and_libs(self, prefix, bin_patterns, lib_patterns): + def _bins_and_libs(self, prefix, bin_patterns, lib_patterns, exclude_prefixes=[]): out = [] bin_src = self._ctx.build_dir / "bin" @@ -521,31 +537,21 @@ class Builder: bin_dst.mkdir(parents=True, exist_ok=True) for path in bin_src.iterdir(): if any(path.match(m) for m in bin_patterns): + if any(path.match(f"{m}*") for m in exclude_prefixes): + continue self._copy_binary(path, bin_dst / path.name) - out.append(f"ADD {prefix}_bin /usr/bin") lib_src = self._ctx.build_dir / "lib" lib_dst = self._workdir / f"{prefix}_lib" lib_dst.mkdir(parents=True, exist_ok=True) for path in lib_src.iterdir(): if any(path.match(m) for m in lib_patterns): + if any(path.match(f"{m}*") for m in exclude_prefixes): + continue self._copy_binary(path, lib_dst / path.name) - out.append(f"ADD {prefix}_lib /usr/lib64") return out - def _conditional_libs(self, src_dir, name, destination, lib_patterns): - lib_src = self._ctx.build_dir / src_dir - lib_dst = self._workdir / name - lib_dst.mkdir(parents=True, exist_ok=True) - try: - for path in lib_src.iterdir(): - if any(path.match(m) for m in lib_patterns): - self._copy_binary(path, lib_dst / path.name) - except FileNotFoundError as err: - log.warning("skipping lib %s: %s", name, err) - return f"ADD {name} {destination}" - def _py_site_packages(self): """Return the correct python site packages dir for the image.""" if self._cached_py_site_packages is not None: @@ -639,44 +645,23 @@ class Builder: # [Quoth the original script]: # binaries are annoying because the ceph version is embedded all over # the place, so we have to include everything but the kitchen sink. - out = [] - - out.extend( - self._bins_and_libs( - prefix="core", - bin_patterns=["ceph-mgr", "ceph-mon", "ceph-osd", "rados"], - lib_patterns=["libceph-common.so*", "librados.so*"], - ) - ) - - out.append( - self._conditional_libs( - src_dir="lib", - name="eclib", - destination="/usr/lib64/ceph/erasure-code", - lib_patterns=["libec_*.so*"], - ) - ) - out.append( - self._conditional_libs( - src_dir="lib", - name="clslib", - destination="/usr/lib64/rados-classes", - lib_patterns=["libcls_*.so*"], - ) - ) + if not self._ctx.components_selected: + log.warning("Copying ALL locally built binaries over those that already exist in the image.") + bins=['*'] + else: + bins=["ceph-mgr", "ceph-mon", "ceph-osd", "rados"] - # [Quoth the original script]: - # by default locally built binaries assume /usr/local - out.append( - "RUN rm -rf /usr/local/lib64 && ln -s /usr/lib64 /usr/local && ln -s /usr/share/ceph /usr/local/share" + return self._bins_and_libs( + prefix="tmp", + bin_patterns=bins, + lib_patterns=["*.so","*.so.*"], + exclude_prefixes=["ceph_test","test_","unittest_"], ) - return out def _rgw_job(self, component): return self._bins_and_libs( - prefix="rgw", + prefix="tmp", bin_patterns=["radosgw", "radosgw-admin"], lib_patterns=["librados.so*", "libceph-common.so*"], ) @@ -684,7 +669,7 @@ class Builder: def _cephfs_job(self, component): return self._bins_and_libs( - prefix="cephfs", + prefix="tmp", bin_patterns=["ceph-mds"], lib_patterns=["libcephfs.so*"], ) @@ -692,7 +677,7 @@ class Builder: def _rbd_job(self, component): return self._bins_and_libs( - prefix="rbd", + prefix="tmp", bin_patterns=["rbd", "rbd-mirror"], lib_patterns=["librbd.so*"], )