cpatch.py: search base image when patching files

Refactor the script so that for each component specified:
      binaries are copied to the /tmpbin directory
      libraries are  copied to the /tmplib directory

For the core component ensure that ALL locally built libraries are copied to the /tmplib directory rather than a subset of those that are used by the binaries
If no component is specified then copy ALL locally built binaries are copied to the /tmpbin directory

Add commands to the Dockerfile so that the build process does the following:
  For every file in the /tmpbin directory:
     find the file with the same name in the base container image (search /usr/bin amd /usr/sbin directories)
     move the file - replacing the version in the base image
  For every file in the /tmplib directory:
     find the file with the same name in the base container image (search /usr/lib64 directory)
     move the file - replacing the version in the base image
  Create some links to the libraries that were just copied because locally built binaries assume that libraries are in
  a different directory to those in the base image. (original script did this too).

   Signed-off-by: John Agombar <agombar@uk.ibm.com>

diff --git a/src/script/cpatch.py b/src/script/cpatch.py
index 200bda837ca446b22d1053ea479d0801a9550f79..5e33e9b2c13cf7d7a7e68c4265eb35ac446c98d7 100755 (executable)
--- a/src/script/cpatch.py
+++ b/src/script/cpatch.py
@@ -459,6 +459,22 @@ class Builder:
             dresult = job(component)
             dlines.extend(dresult)
 
+        if os.path.isdir(self._workdir / "tmp_bin"):
+            # For every binary file that was copied to the tmp_bin directory by the jobs above, search for the existing file in the container and replace it.
+            dlines.append("ADD tmp_bin /tmpbin")
+            dlines.append("RUN for i in tmpbin/*; do find /usr/bin /usr/sbin -name $(basename $i) -exec mv -f $i '{}' \;; echo $(basename $i); done && rm -rf tmpbin")
+
+        if os.path.isdir(self._workdir / "tmp_lib"):
+            # For every library file that was copied to the tmp_lib directory by the jobs above, search for the existing file in the container and replace it.
+            dlines.append("ADD tmp_lib /tmplib")
+            dlines.append("RUN for i in tmplib/*; do find /usr/lib64 -name $(basename $i) -exec mv -f $i '{}' \;; echo $(basename $i); done && rm -rf tmplib")
+
+        if os.path.isdir(self._workdir / "tmp_bin"):
+            # by default locally built binaries assume /usr/local
+            dlines.append("RUN rm -rf /usr/local/lib64 && ln -sf /usr/lib64 /usr/local && ln -sf /usr/share/ceph /usr/local/share")
+            # locally built binaries assume libceph-common.so.2 is in /usr/lib64 - create link to library that was just copied
+            dlines.append("RUN ln -sf /usr/lib64/ceph/libceph-common.so.2 /usr/lib64/libceph-common.so.2")
+
         with open(self._workdir / "Dockerfile", "w") as fout:
             for line in dlines:
                 print(line, file=fout)
@@ -513,7 +529,7 @@ class Builder:
             pass
         os.link(src_path, dst_path)
 
-    def _bins_and_libs(self, prefix, bin_patterns, lib_patterns):
+    def _bins_and_libs(self, prefix, bin_patterns, lib_patterns, exclude_prefixes=[]):
         out = []
 
         bin_src = self._ctx.build_dir / "bin"
@@ -521,31 +537,21 @@ class Builder:
         bin_dst.mkdir(parents=True, exist_ok=True)
         for path in bin_src.iterdir():
             if any(path.match(m) for m in bin_patterns):
+                if any(path.match(f"{m}*") for m in exclude_prefixes):
+                    continue
                 self._copy_binary(path, bin_dst / path.name)
-        out.append(f"ADD {prefix}_bin /usr/bin")
 
         lib_src = self._ctx.build_dir / "lib"
         lib_dst = self._workdir / f"{prefix}_lib"
         lib_dst.mkdir(parents=True, exist_ok=True)
         for path in lib_src.iterdir():
             if any(path.match(m) for m in lib_patterns):
+                if any(path.match(f"{m}*") for m in exclude_prefixes):
+                    continue
                 self._copy_binary(path, lib_dst / path.name)
-        out.append(f"ADD {prefix}_lib /usr/lib64")
 
         return out
 
-    def _conditional_libs(self, src_dir, name, destination, lib_patterns):
-        lib_src = self._ctx.build_dir / src_dir
-        lib_dst = self._workdir / name
-        lib_dst.mkdir(parents=True, exist_ok=True)
-        try:
-            for path in lib_src.iterdir():
-                if any(path.match(m) for m in lib_patterns):
-                    self._copy_binary(path, lib_dst / path.name)
-        except FileNotFoundError as err:
-            log.warning("skipping lib %s: %s", name, err)
-        return f"ADD {name} {destination}"
-
     def _py_site_packages(self):
         """Return the correct python site packages dir for the image."""
         if self._cached_py_site_packages is not None:
@@ -639,44 +645,23 @@ class Builder:
         # [Quoth the original script]:
         # binaries are annoying because the ceph version is embedded all over
         # the place, so we have to include everything but the kitchen sink.
-        out = []
-
-        out.extend(
-            self._bins_and_libs(
-                prefix="core",
-                bin_patterns=["ceph-mgr", "ceph-mon", "ceph-osd", "rados"],
-                lib_patterns=["libceph-common.so*", "librados.so*"],
-            )
-        )
-
-        out.append(
-            self._conditional_libs(
-                src_dir="lib",
-                name="eclib",
-                destination="/usr/lib64/ceph/erasure-code",
-                lib_patterns=["libec_*.so*"],
-            )
-        )
-        out.append(
-            self._conditional_libs(
-                src_dir="lib",
-                name="clslib",
-                destination="/usr/lib64/rados-classes",
-                lib_patterns=["libcls_*.so*"],
-            )
-        )
+        if not self._ctx.components_selected:
+            log.warning("Copying ALL locally built binaries over those that already exist in the image.")
+            bins=['*']
+        else:
+            bins=["ceph-mgr", "ceph-mon", "ceph-osd", "rados"]           
 
-        # [Quoth the original script]:
-        # by default locally built binaries assume /usr/local
-        out.append(
-            "RUN rm -rf /usr/local/lib64 && ln -s /usr/lib64 /usr/local && ln -s /usr/share/ceph /usr/local/share"
+        return self._bins_and_libs(
+            prefix="tmp",
+            bin_patterns=bins,
+            lib_patterns=["*.so","*.so.*"],
+            exclude_prefixes=["ceph_test","test_","unittest_"],
         )
-
         return out
 
     def _rgw_job(self, component):
         return self._bins_and_libs(
-            prefix="rgw",
+            prefix="tmp",
             bin_patterns=["radosgw", "radosgw-admin"],
             lib_patterns=["librados.so*", "libceph-common.so*"],
         )
@@ -684,7 +669,7 @@ class Builder:
 
     def _cephfs_job(self, component):
         return self._bins_and_libs(
-            prefix="cephfs",
+            prefix="tmp",
             bin_patterns=["ceph-mds"],
             lib_patterns=["libcephfs.so*"],
         )
@@ -692,7 +677,7 @@ class Builder:
 
     def _rbd_job(self, component):
         return self._bins_and_libs(
-            prefix="rbd",
+            prefix="tmp",
             bin_patterns=["rbd", "rbd-mirror"],
             lib_patterns=["librbd.so*"],
         )