test/encoding: use setarch -R for ASAN builds in readable.sh

When running readable.sh with a WITH_ASAN=ON build of ceph-dencoder,
ASAN processes need to find a contiguous 16+ TB shadow memory region
(1/8 of the 128 TB x86-64 user VA space). High ASLR entropy can
fragment the VA space, preventing ASAN from finding a suitable region.

Instead of requiring system-wide vm.mmap_rnd_bits=28 (which weakens
ASLR security for the entire host), wrap ceph-dencoder with 'setarch
$(uname -m) -R' when ASAN is detected. This disables ASLR only for the
specific ceph-dencoder processes, with no system-wide security impact.

Also simplify parallelism logic: extract NPROC calculation into a shared
variable and use it consistently across FreeBSD, Darwin, and Linux.

Reference: https://clang.llvm.org/docs/AddressSanitizer.html

Signed-off-by: Kefu Chai <k.chai@proxmox.com>

diff --git a/src/test/encoding/readable.sh b/src/test/encoding/readable.sh
index b05b3e55c9357de100139dd7dbac04f1c9316292..115d6d7b6f15edec986627466673a7c5f6cad125 100755 (executable)
--- a/src/test/encoding/readable.sh
+++ b/src/test/encoding/readable.sh
@@ -22,6 +22,13 @@ else
   CEPH_DENCODER=ceph-dencoder
 fi
 
+# ASAN builds need setarch -R to disable ASLR so each process can find a
+# contiguous 16+ TB shadow memory region. See https://clang.llvm.org/docs/AddressSanitizer.html
+if ldd $(command -v $CEPH_DENCODER) 2>/dev/null | grep -q libasan; then
+  echo "ASAN build detected: wrapping ceph-dencoder with 'setarch \$(uname -m) -R'"
+  CEPH_DENCODER="setarch $(uname -m) -R $CEPH_DENCODER"
+fi
+
 myversion=$($CEPH_DENCODER version)
 echo "Using ceph-dencoder version $myversion"
 if [ -z "$myversion" ]; then
@@ -279,14 +286,14 @@ do_join() {
         running_jobs=0
 }
 
-# Using $MAX_PARALLEL_JOBS jobs if defined, unless the number of logical
-# processors
+# Determine the number of parallel jobs to run.  Default to the number of
+# logical processors, or $MAX_PARALLEL_JOBS if set.
 if [ $(uname) == FreeBSD -o $(uname) == Darwin ]; then
   NPROC=$(sysctl -n hw.ncpu)
-  max_parallel_jobs=${MAX_PARALLEL_JOBS:-${NPROC}}
 else
-  max_parallel_jobs=${MAX_PARALLEL_JOBS:-$(nproc)}
+  NPROC=$(nproc)
 fi
+max_parallel_jobs=${MAX_PARALLEL_JOBS:-${NPROC}}
 
 output_file=$(mktemp /tmp/output_file-XXXXXXXXX)
 running_jobs=0