pybind/mgr/dashboard/frontend: add NPM_CACHEDIR envvar, use in bwc

Add an optional NPM_CACHEDIR environment variable to serve as the
cache parameter for npm in the dashboard frontend build.  The idea
is to allow it to persist across builds so that we decrease the load
on registry.npmjs.org, which has been throttling our requests when
using build-with-container.py, and also hopefully improve the time
of the frontend npm operations.

build-with-container.py also grows a --npm-cache-path option to allow
setting it for container builds and passing the envvar to the build.

Fixes: https://tracker.ceph.com/issues/72298
Signed-off-by: Dan Mick <dan.mick@redhat.com>
(cherry picked from commit ad7e6117a9e99061a3ad7e03709dd31e34832966)

diff --git a/src/pybind/mgr/dashboard/frontend/CMakeLists.txt b/src/pybind/mgr/dashboard/frontend/CMakeLists.txt
index 6c85e50716a7a858a51d2db7279d52f2c55cd905..a07864d957508777de81a92bdd7373a9e15c9133 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/CMakeLists.txt
+++ b/src/pybind/mgr/dashboard/frontend/CMakeLists.txt
@@ -60,6 +60,11 @@ else(WITH_SYSTEM_NPM)
   if(DEFINED ENV{NODE_MIRROR})
     set(node_mirror_opt "--mirror=$ENV{NODE_MIRROR}")
   endif()
+  if(DEFINED ENV{NPM_CACHEDIR})
+    set(npm-cache-dir "$ENV{NPM_CACHEDIR}")
+  else()
+    set(npm-cache-dir "${mgr-dashboard-nodeenv-dir}/.npm")
+  endif()
   add_custom_command(
     OUTPUT "${mgr-dashboard-nodeenv-dir}/bin/npm"
     COMMAND ${CMAKE_SOURCE_DIR}/src/tools/setup-virtualenv.sh --python=${MGR_PYTHON_EXECUTABLE} ${mgr-dashboard-nodeenv-dir}
@@ -70,7 +75,7 @@ else(WITH_SYSTEM_NPM)
     # uid 1000 and running the unpack in a id-mapped namespace (container)
     # that lets tar set the uid to a "bad" uid outside the namespace
     COMMAND bash -c 'chown -R `id -u`:`id -g` ${mgr-dashboard-nodeenv-dir}/src'
-    COMMAND mkdir ${mgr-dashboard-nodeenv-dir}/.npm
+    COMMAND mkdir -p ${npm-cache-dir}
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
     COMMENT "dashboard nodeenv is being installed")
   if(DEFINED ENV{NPM_REGISTRY})
@@ -79,7 +84,7 @@ else(WITH_SYSTEM_NPM)
   add_npm_options(
     NODEENV_DIR ${mgr-dashboard-nodeenv-dir}
     TARGET mgr-dashboard-nodeenv
-    OPTION cache=${mgr-dashboard-nodeenv-dir}/.npm
+    OPTION cache=${npm-cache-dir}
     ${npm_registry_opts})
   add_custom_target(mgr-dashboard-frontend-deps
     DEPENDS node_modules mgr-dashboard-nodeenv

diff --git a/src/script/build-with-container.py b/src/script/build-with-container.py
index 4dd35517f3e75b1eb99346dbacac4134f3ca8ddc..ef9e4cfa9245285b7e17fda42c9aeb5f5bae5e15 100755 (executable)
--- a/src/script/build-with-container.py
+++ b/src/script/build-with-container.py
@@ -243,6 +243,12 @@ def _container_cmd(ctx, args, *, workdir=None, interactive=False):
         cmd.append(f"-eCCACHE_BASEDIR={ctx.cli.homedir}")
     for extra_arg in ctx.cli.extra or []:
         cmd.append(extra_arg)
+    if ctx.npm_cache_dir:
+        # use :z so that other builds can use the cache
+        cmd.extend([
+            f'--volume={ctx.npm_cache_dir}:/npmcache:z',
+            '--env=NPM_CACHEDIR=/npmcache'
+        ])
     cmd.append(ctx.image_name)
     cmd.extend(args)
     return cmd
@@ -292,6 +298,7 @@ class Steps(StrEnum):
     BUILD_CONTAINER = "build-container"
     CONTAINER = "container"
     CONFIGURE = "configure"
+    NPM_CACHE = "npmcache"
     BUILD = "build"
     BUILD_TESTS = "buildtests"
     TESTS = "tests"
@@ -395,6 +402,14 @@ class Context:
             return path.resolve()
         return None
 
+    @property
+    def npm_cache_dir(self):
+        if self.cli.npm_cache_path:
+            path = pathlib.Path(self.cli.npm_cache_path)
+            path = path.expanduser()
+            return path.resolve()
+        return None
+
     @property
     def map_user(self):
         # TODO: detect if uid mapping is needed
@@ -528,6 +543,14 @@ def dnf_cache_dir(ctx):
     (cache_dir / ".DNF_CACHE").touch(exist_ok=True)
 
 
+@Builder.set(Steps.NPM_CACHE)
+def npm_cache_dir(ctx):
+    """Set up an NPM cache directory for reuse across container builds."""
+    if not ctx.cli.npm_cache_path:
+        return
+    ctx.npm_cache_dir.mkdir(parents=True, exist_ok=True)
+
+
 @Builder.set(Steps.BUILD_CONTAINER)
 def build_container(ctx):
     """Generate a build environment container image."""
@@ -545,7 +568,7 @@ def build_container(ctx):
         cmd.append(f"--build-arg=DISTRO={ctx.from_image}")
     if ctx.dnf_cache_dir and "docker" in ctx.container_engine:
         log.warning(
-            "The --volume option is not supported by docker. Skipping dnf cache dir mounts"
+            "The --volume option is not supported by docker build/buildx. Skipping dnf cache dir mounts"
         )
     elif ctx.dnf_cache_dir:
         cmd += [
@@ -638,6 +661,7 @@ def bc_configure(ctx):
 @Builder.set(Steps.BUILD)
 def bc_build(ctx):
     """Execute a standard build."""
+    ctx.build.wants(Steps.NPM_CACHE, ctx)
     ctx.build.wants(Steps.CONFIGURE, ctx)
     cmd = _container_cmd(
         ctx,
@@ -654,6 +678,7 @@ def bc_build(ctx):
 @Builder.set(Steps.BUILD_TESTS)
 def bc_build_tests(ctx):
     """Build the tests."""
+    ctx.build.wants(Steps.NPM_CACHE, ctx)
     ctx.build.wants(Steps.CONFIGURE, ctx)
     cmd = _container_cmd(
         ctx,
@@ -670,6 +695,7 @@ def bc_build_tests(ctx):
 @Builder.set(Steps.TESTS)
 def bc_run_tests(ctx):
     """Execute the tests."""
+    ctx.build.wants(Steps.NPM_CACHE, ctx)
     ctx.build.wants(Steps.BUILD_TESTS, ctx)
     cmd = _container_cmd(
         ctx,
@@ -685,7 +711,8 @@ def bc_run_tests(ctx):
 
 @Builder.set(Steps.SOURCE_RPM)
 def bc_make_source_rpm(ctx):
-    """Build SPRMs."""
+    """Build SRPMs."""
+    ctx.build.wants(Steps.NPM_CACHE, ctx)
     ctx.build.wants(Steps.CONTAINER, ctx)
     make_srpm_cmd = f"cd {ctx.cli.homedir} && ./make-srpm.sh"
     if ctx.cli.ceph_version:
@@ -927,7 +954,11 @@ def parse_cli(build_step_names):
     )
     parser.add_argument(
         "--dnf-cache-path",
-        help="DNF caching using provided base dir",
+        help="DNF caching using provided base dir (during build-container build)",
+    )
+    parser.add_argument(
+        "--npm-cache-path",
+        help="NPM caching using provided base dir (during build)",
     )
     parser.add_argument(
         "--build-dir",