Include rest of dependencies in dependencies.sh

Summary:
This diff

  - Include the rest of the dependencies (kernel-headers, binutils, valgrind) in dependencies.sh
  - updtade zst to 0.4.7
  - It also fix a problem in clang scan build

Test Plan:
make check
USE_CLANG=1 make check
USE_CLANG=1 make analyze
coverage_test.sh

Reviewers: sdong, yhchiang, rven, andrewkr, kradhakrishnan

Reviewed By: kradhakrishnan

Subscribers: dhruba

Differential Revision: https://reviews.facebook.net/D53301

(cherry picked from commit f7ef1a613276641af93f02f74f17cf66174d8126)

diff --git a/build_tools/dependencies.sh b/build_tools/dependencies.sh
index a7e59090499b292e55d49c736c9d2c14248e485c..a7d9a65316e06aa9fa9a1f6fcde0bf3b859def0a 100644 (file)
--- a/build_tools/dependencies.sh
+++ b/build_tools/dependencies.sh
@@ -1,13 +1,16 @@
-GCC_BASE=/mnt/vol/engshare/fbcode/third-party2/gcc/4.9.x/centos6-native/*
-CLANG_BASE=/mnt/vol/engshare/fbcode/third-party2/clang/3.7.1
+GCC_BASE=/mnt/vol/engshare/fbcode/third-party2/gcc/4.9.x/centos6-native/1317bc4/
+CLANG_BASE=/mnt/vol/engshare/fbcode/third-party2/clang/3.7.1/centos6-native/9d9ecb9/
 LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/5710d6a0fb0d12820aac0bffcd7fcb8646e7fff7/4.9.x/gcc-4.9-glibc-2.20/024dbc3
 GLIBC_BASE=/mnt/gvfs/third-party2/glibc/0600c95b31226b5e535614c590677d87c62d8016/2.20/gcc-4.9-glibc-2.20/500e281
 SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/cbf6f1f209e5bd160bdc5d971744e039f36b1566/1.1.3/gcc-4.9-glibc-2.20/e9936bf
 ZLIB_BASE=/mnt/gvfs/third-party2/zlib/6d39cb54708049f527e713ad19f2aadb9d3667e8/1.2.8/gcc-4.9-glibc-2.20/e9936bf
 BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/2ddd45f0853bfc8bb1c27f0f447236a1a26c338a/1.0.6/gcc-4.9-glibc-2.20/e9936bf
 LZ4_BASE=/mnt/gvfs/third-party2/lz4/6858fac689e0f92e584224d91bdb0e39f6c8320d/r131/gcc-4.9-glibc-2.20/e9936bf
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/d4ac2c5f9be76d57a6cbd3eb1011e97574a56cde/0.4.5/gcc-4.9-glibc-2.20/e9936bf
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/69d56740ffb89d8bc81ded8ec428c01a813ea948/0.4.7/gcc-4.9-glibc-2.20/e9936bf
 GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/c7275a4ceae0aca0929e56964a31dafc53c1ee96/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
 JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/5f0be745ddc0f86f22c8c8bb64b6b1163c93df91/4.0.3/gcc-4.9-glibc-2.20/a6c5e1e
 NUMA_BASE=/mnt/gvfs/third-party2/numa/ae54a5ed22cdabb1c6446dce4e8ffae5b4446d73/2.0.8/gcc-4.9-glibc-2.20/e9936bf
 LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/121f1a75c4414683aea8c70b761bfaf187f7c1a3/trunk/gcc-4.9-glibc-2.20/12266b1
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/1a48835975c66d30e47770ec419758ed3b9ba010/3.10.62-62_fbk17_03959_ge29cc63/gcc-4.9-glibc-2.20/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/75670d0d8ef4891fd1ec2a7513ef01cd002c823b/2.25/centos6-native/da39a3e
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/af85c56f424cd5edfc2c97588299b44ecdec96bb/3.10.0/gcc-4.9-glibc-2.20/e9936bf

diff --git a/build_tools/fbcode_config.sh b/build_tools/fbcode_config.sh
index 8cc0b53ffbcdb22af3613b08d749a0ab3c2ebe2e..9e1c613ecdd053418f7121abc80c267ae24e4222 100644 (file)
--- a/build_tools/fbcode_config.sh
+++ b/build_tools/fbcode_config.sh
@@ -7,9 +7,8 @@
 # PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
 
 
-source "$PWD/build_tools/dependencies.sh"
-
-TP2_LATEST="/mnt/vol/engshare/fbcode/third-party2"
+BASEDIR=`dirname $BASH_SOURCE`
+source "$BASEDIR/dependencies.sh"
 
 CFLAGS=""
 
@@ -76,16 +75,19 @@ fi
 # use Intel SSE support for checksum calculations
 export USE_SSE=1
 
-BINUTILS="$TP2_LATEST/binutils/2.25/centos6-native/*/bin"
+BINUTILS="$BINUTILS_BASE/bin"
 AR="$BINUTILS/ar"
 
 DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE"
 
 STDLIBS="-L $GCC_BASE/lib64"
 
-CLANG_BIN="$CLANG_BASE/centos6-native/*/bin"
+CLANG_BIN="$CLANG_BASE/bin"
+CLANG_LIB="$CLANG_BASE/lib"
+CLANG_SRC="$CLANG_BASE/../../src"
+
 CLANG_ANALYZER="$CLANG_BIN/clang++"
-CLANG_SCAN_BUILD="$CLANG_BASE/src/clang/tools/scan-build/scan-build"
+CLANG_SCAN_BUILD="$CLANG_SRC/clang/tools/scan-build/scan-build"
 
 if [ -z "$USE_CLANG" ]; then
   # gcc
@@ -97,11 +99,11 @@ if [ -z "$USE_CLANG" ]; then
   CFLAGS+=" -isystem $LIBGCC_INCLUDE"
 else
   # clang
-  CLANG_INCLUDE="$CLANG_BASE/centos6-native/*/lib/clang/*/include"
+  CLANG_INCLUDE="$CLANG_LIB/clang/*/include"
   CC="$CLANG_BIN/clang"
   CXX="$CLANG_BIN/clang++"
 
-  KERNEL_HEADERS_INCLUDE="$TP2_LATEST/kernel-headers/3.2.18_70_fbk11_00129_gc8882d0/gcc-4.9-glibc-2.20/*/include"
+  KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
 
   CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
   CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x "
@@ -127,6 +129,6 @@ PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
 
 EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS"
 
-VALGRIND_VER="$TP2_LATEST/valgrind/3.10.0/gcc-4.9-glibc-2.20/*/bin/"
+VALGRIND_VER="$VALGRIND_BASE/bin/"
 
 export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD

diff --git a/build_tools/update_dependencies.sh b/build_tools/update_dependencies.sh
index 6a9c662a23d3eac195b5a2c2f95f5ba5e215c5a6..63074c372d04d6ef3676c71f99fa534003e3d012 100755 (executable)
--- a/build_tools/update_dependencies.sh
+++ b/build_tools/update_dependencies.sh
@@ -3,10 +3,7 @@
 # Update dependencies.sh file with the latest avaliable versions
 
 BASEDIR=$(dirname $0)
-OUTPUT="$BASEDIR/dependencies.sh"
-
-rm -f "$OUTPUT"
-touch "$OUTPUT"
+OUTPUT=""
 
 function log_variable()
 {
@@ -29,7 +26,7 @@ function get_lib_base()
   local result="$TP2_LATEST/$lib_name/"
   
   # Lib Version
-  if [ -z "$lib_version" ]; then
+  if [ -z "$lib_version" ] || [ "$lib_version" = "LATEST" ]; then
     # version is not provided, use latest
     result=`ls -dr1v $result/*/ | head -n1`
   else
@@ -44,36 +41,46 @@ function get_lib_base()
     result="$result/$lib_platform/"
   fi
   
-  result="$result/*/"
+  result=`ls -1d $result/*/ | head -n1`
   
   # lib_name => LIB_NAME_BASE
   local __res_var=${lib_name^^}"_BASE"
+  __res_var=`echo $__res_var | tr - _`
   # LIB_NAME_BASE=$result
   eval $__res_var=`readlink -f $result`
   
   log_variable $__res_var
 }
 
+OUTPUT="$BASEDIR/dependencies.sh"
+
+rm -f "$OUTPUT"
+touch "$OUTPUT"
+
 echo "Writing dependencies to $OUTPUT"
 
 # Compilers locations
-GCC_BASE="$TP2_LATEST/gcc/4.9.x/centos6-native/*"
-CLANG_BASE="$TP2_LATEST/clang/3.7.1"
+GCC_BASE=`ls -d1 $TP2_LATEST/gcc/4.9.x/centos6-native/*/ | head -n1`
+CLANG_BASE=`ls -d1 $TP2_LATEST/clang/3.7.1/centos6-native/*/ | head -n1`
 
 log_variable GCC_BASE
 log_variable CLANG_BASE
 
 # Libraries locations
-get_lib_base libgcc
-get_lib_base glibc 2.20 gcc-4.9-glibc-2.20
-get_lib_base snappy
-get_lib_base zlib
-get_lib_base bzip2
-get_lib_base lz4
-get_lib_base zstd
-get_lib_base gflags
-get_lib_base jemalloc
-get_lib_base numa
-get_lib_base libunwind
+get_lib_base libgcc     4.9.x
+get_lib_base glibc      2.20 
+get_lib_base snappy     LATEST
+get_lib_base zlib       LATEST
+get_lib_base bzip2      LATEST
+get_lib_base lz4        LATEST
+get_lib_base zstd       LATEST
+get_lib_base gflags     LATEST
+get_lib_base jemalloc   LATEST
+get_lib_base numa       LATEST
+get_lib_base libunwind  LATEST
+
+get_lib_base kernel-headers LATEST 
+get_lib_base binutils   LATEST centos6-native 
+get_lib_base valgrind   LATEST
 
 git diff $OUTPUT