From 614c6d453eaef8a0b5a12b0c8febeac77d971931 Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Thu, 30 Jan 2020 11:00:08 -0800 Subject: [PATCH] Don't download from (unreliable) maven.org (#6348) Summary: I set up a mirror of our Java deps on github so we can download them through github URLs rather than maven.org, which is proving terribly unreliable from Travis builds. Also sanitized calls to curl, so they are easier to read and appropriately fail on download failure. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6348 Test Plan: CI Differential Revision: D19633621 Pulled By: pdillinger fbshipit-source-id: 7eb3f730953db2ead758dc94039c040f406790f3 --- .travis.yml | 5 +- Makefile | 10 ++-- appveyor.yml | 70 +++++++++++++++++++++++++-- build_tools/regression_build_test.sh | 2 +- build_tools/rocksdb-lego-determinator | 2 +- java/CMakeLists.txt | 22 +++++---- java/Makefile | 17 ++++--- 7 files changed, 98 insertions(+), 30 deletions(-) diff --git a/.travis.yml b/.travis.yml index e759a642..d32b1044 100644 --- a/.travis.yml +++ b/.travis.yml @@ -71,7 +71,10 @@ install: CC=gcc-8 && CXX=g++-8; fi - if [[ "${JOB_NAME}" == cmake* ]] && [ "${TRAVIS_OS_NAME}" == linux ]; then - mkdir cmake-dist && curl -sfSL https://cmake.org/files/v3.8/cmake-3.8.1-Linux-x86_64.tar.gz | tar --strip-components=1 -C cmake-dist -xz && export PATH=$PWD/cmake-dist/bin:$PATH; + mkdir cmake-dist && curl --silent --fail --show-error --location https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.tar.gz | tar --strip-components=1 -C cmake-dist -xz && export PATH=$PWD/cmake-dist/bin:$PATH; + fi + - if [[ "${JOB_NAME}" == java_test ]]; then + java -version && echo "JAVA_HOME=${JAVA_HOME}"; fi before_script: diff --git a/Makefile b/Makefile index 9e4da2e6..c30f556e 100644 --- a/Makefile +++ b/Makefile @@ -1714,7 +1714,7 @@ endif libz.a: -rm -rf zlib-$(ZLIB_VER) ifeq (,$(wildcard ./zlib-$(ZLIB_VER).tar.gz)) - curl --output zlib-$(ZLIB_VER).tar.gz -L ${ZLIB_DOWNLOAD_BASE}/zlib-$(ZLIB_VER).tar.gz + curl --fail --output zlib-$(ZLIB_VER).tar.gz --location ${ZLIB_DOWNLOAD_BASE}/zlib-$(ZLIB_VER).tar.gz endif ZLIB_SHA256_ACTUAL=`$(SHA256_CMD) zlib-$(ZLIB_VER).tar.gz | cut -d ' ' -f 1`; \ if [ "$(ZLIB_SHA256)" != "$$ZLIB_SHA256_ACTUAL" ]; then \ @@ -1728,7 +1728,7 @@ endif libbz2.a: -rm -rf bzip2-$(BZIP2_VER) ifeq (,$(wildcard ./bzip2-$(BZIP2_VER).tar.gz)) - curl --output bzip2-$(BZIP2_VER).tar.gz -L ${CURL_SSL_OPTS} ${BZIP2_DOWNLOAD_BASE}/bzip2-$(BZIP2_VER).tar.gz + curl --fail --output bzip2-$(BZIP2_VER).tar.gz --location ${CURL_SSL_OPTS} ${BZIP2_DOWNLOAD_BASE}/bzip2-$(BZIP2_VER).tar.gz endif BZIP2_SHA256_ACTUAL=`$(SHA256_CMD) bzip2-$(BZIP2_VER).tar.gz | cut -d ' ' -f 1`; \ if [ "$(BZIP2_SHA256)" != "$$BZIP2_SHA256_ACTUAL" ]; then \ @@ -1742,7 +1742,7 @@ endif libsnappy.a: -rm -rf snappy-$(SNAPPY_VER) ifeq (,$(wildcard ./snappy-$(SNAPPY_VER).tar.gz)) - curl --output snappy-$(SNAPPY_VER).tar.gz -L ${CURL_SSL_OPTS} ${SNAPPY_DOWNLOAD_BASE}/$(SNAPPY_VER).tar.gz + curl --fail --output snappy-$(SNAPPY_VER).tar.gz --location ${CURL_SSL_OPTS} ${SNAPPY_DOWNLOAD_BASE}/$(SNAPPY_VER).tar.gz endif SNAPPY_SHA256_ACTUAL=`$(SHA256_CMD) snappy-$(SNAPPY_VER).tar.gz | cut -d ' ' -f 1`; \ if [ "$(SNAPPY_SHA256)" != "$$SNAPPY_SHA256_ACTUAL" ]; then \ @@ -1757,7 +1757,7 @@ endif liblz4.a: -rm -rf lz4-$(LZ4_VER) ifeq (,$(wildcard ./lz4-$(LZ4_VER).tar.gz)) - curl --output lz4-$(LZ4_VER).tar.gz -L ${CURL_SSL_OPTS} ${LZ4_DOWNLOAD_BASE}/v$(LZ4_VER).tar.gz + curl --fail --output lz4-$(LZ4_VER).tar.gz --location ${CURL_SSL_OPTS} ${LZ4_DOWNLOAD_BASE}/v$(LZ4_VER).tar.gz endif LZ4_SHA256_ACTUAL=`$(SHA256_CMD) lz4-$(LZ4_VER).tar.gz | cut -d ' ' -f 1`; \ if [ "$(LZ4_SHA256)" != "$$LZ4_SHA256_ACTUAL" ]; then \ @@ -1771,7 +1771,7 @@ endif libzstd.a: -rm -rf zstd-$(ZSTD_VER) ifeq (,$(wildcard ./zstd-$(ZSTD_VER).tar.gz)) - curl --output zstd-$(ZSTD_VER).tar.gz -L ${CURL_SSL_OPTS} ${ZSTD_DOWNLOAD_BASE}/v$(ZSTD_VER).tar.gz + curl --fail --output zstd-$(ZSTD_VER).tar.gz --location ${CURL_SSL_OPTS} ${ZSTD_DOWNLOAD_BASE}/v$(ZSTD_VER).tar.gz endif ZSTD_SHA256_ACTUAL=`$(SHA256_CMD) zstd-$(ZSTD_VER).tar.gz | cut -d ' ' -f 1`; \ if [ "$(ZSTD_SHA256)" != "$$ZSTD_SHA256_ACTUAL" ]; then \ diff --git a/appveyor.yml b/appveyor.yml index 576042cd..416b6550 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,15 +1,75 @@ version: 1.0.{build} + image: Visual Studio 2017 + +environment: + JAVA_HOME: C:\Program Files\Java\jdk1.8.0 + THIRDPARTY_HOME: $(APPVEYOR_BUILD_FOLDER)\thirdparty + SNAPPY_HOME: $(THIRDPARTY_HOME)\snappy-1.1.7 + SNAPPY_INCLUDE: $(SNAPPY_HOME);$(SNAPPY_HOME)\build + SNAPPY_LIB_DEBUG: $(SNAPPY_HOME)\build\Debug\snappy.lib + SNAPPY_LIB_RELEASE: $(SNAPPY_HOME)\build\Release\snappy.lib + LZ4_HOME: $(THIRDPARTY_HOME)\lz4-1.8.3 + LZ4_INCLUDE: $(LZ4_HOME)\lib + LZ4_LIB_DEBUG: $(LZ4_HOME)\visual\VS2010\bin\x64_Debug\liblz4_static.lib + LZ4_LIB_RELEASE: $(LZ4_HOME)\visual\VS2010\bin\x64_Release\liblz4_static.lib + ZSTD_HOME: $(THIRDPARTY_HOME)\zstd-1.4.0 + ZSTD_INCLUDE: $(ZSTD_HOME)\lib;$(ZSTD_HOME)\lib\dictBuilder + ZSTD_LIB_DEBUG: $(ZSTD_HOME)\build\VS2010\bin\x64_Debug\libzstd_static.lib + ZSTD_LIB_RELEASE: $(ZSTD_HOME)\build\VS2010\bin\x64_Release\libzstd_static.lib + matrix: + - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 + CMAKE_GENERATOR: Visual Studio 14 Win64 + DEV_ENV: C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\IDE\devenv.com + - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + CMAKE_GENERATOR: Visual Studio 15 Win64 + DEV_ENV: C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\IDE\devenv.com + +install: + - md %THIRDPARTY_HOME% + - echo "Building Snappy dependency..." + - cd %THIRDPARTY_HOME% + - curl --fail --silent --show-error --output snappy-1.1.7.zip --location https://github.com/google/snappy/archive/1.1.7.zip + - unzip snappy-1.1.7.zip + - cd snappy-1.1.7 + - mkdir build + - cd build + - cmake -G "%CMAKE_GENERATOR%" .. + - msbuild Snappy.sln /p:Configuration=Debug /p:Platform=x64 + - msbuild Snappy.sln /p:Configuration=Release /p:Platform=x64 + - echo "Building LZ4 dependency..." + - cd %THIRDPARTY_HOME% + - curl --fail --silent --show-error --output lz4-1.8.3.zip --location https://github.com/lz4/lz4/archive/v1.8.3.zip + - unzip lz4-1.8.3.zip + - cd lz4-1.8.3\visual\VS2010 + - ps: $CMD="$Env:DEV_ENV"; & $CMD lz4.sln /upgrade + - msbuild lz4.sln /p:Configuration=Debug /p:Platform=x64 + - msbuild lz4.sln /p:Configuration=Release /p:Platform=x64 + - echo "Building ZStd dependency..." + - cd %THIRDPARTY_HOME% + - curl --fail --silent --show-error --output zstd-1.4.0.zip --location https://github.com/facebook/zstd/archive/v1.4.0.zip + - unzip zstd-1.4.0.zip + - cd zstd-1.4.0\build\VS2010 + - ps: $CMD="$Env:DEV_ENV"; & $CMD zstd.sln /upgrade + - msbuild zstd.sln /p:Configuration=Debug /p:Platform=x64 + - msbuild zstd.sln /p:Configuration=Release /p:Platform=x64 + before_build: -- md %APPVEYOR_BUILD_FOLDER%\build -- cd %APPVEYOR_BUILD_FOLDER%\build -- cmake -G "Visual Studio 15 Win64" -DOPTDBG=1 -DWITH_XPRESS=1 -DPORTABLE=1 .. -- cd .. + - md %APPVEYOR_BUILD_FOLDER%\build + - cd %APPVEYOR_BUILD_FOLDER%\build + - cmake -G "%CMAKE_GENERATOR%" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DLZ4=1 -DZSTD=1 -DXPRESS=1 -DJNI=1 .. + - cd .. + build: project: build\rocksdb.sln parallel: true verbosity: normal + test: + test_script: -- ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test2,db_test,env_basic_test,env_test -Concurrency 8 + - ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test2,db_test,env_basic_test,env_test,db_merge_operand_test -Concurrency 8 + +on_failure: + - cmd: 7z a build-failed.zip %APPVEYOR_BUILD_FOLDER%\build\ && appveyor PushArtifact build-failed.zip diff --git a/build_tools/regression_build_test.sh b/build_tools/regression_build_test.sh index 69806332..99a5d2ae 100755 --- a/build_tools/regression_build_test.sh +++ b/build_tools/regression_build_test.sh @@ -377,7 +377,7 @@ function send_to_ods { echo >&2 "ERROR: Key $key doesn't have a value." return fi - curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build$git_br&key=$key&value=$value" \ + curl --silent "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build$git_br&key=$key&value=$value" \ --connect-timeout 60 } diff --git a/build_tools/rocksdb-lego-determinator b/build_tools/rocksdb-lego-determinator index 96933238..dd0be51a 100755 --- a/build_tools/rocksdb-lego-determinator +++ b/build_tools/rocksdb-lego-determinator @@ -646,7 +646,7 @@ run_regression() # parameters: $1 -- key, $2 -- value function send_size_to_ods { - curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \ + curl --silent "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \ --connect-timeout 60 } diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index 8f4ec9a5..3b58c9b6 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -313,17 +313,19 @@ if(NOT EXISTS ${JAVA_TEST_LIBDIR}) file(MAKE_DIRECTORY mkdir ${JAVA_TEST_LIBDIR}) endif() -if (DEFINED CUSTOM_REPO_URL) - set(SEARCH_REPO_URL ${CUSTOM_REPO_URL}/) - set(CENTRAL_REPO_URL ${CUSTOM_REPO_URL}/) +if (DEFINED CUSTOM_DEPS_URL) + set(DEPS_URL ${CUSTOM_DEPS_URL}/) else () - set(SEARCH_REPO_URL "http://search.maven.org/remotecontent?filepath=") - set(CENTRAL_REPO_URL "http://central.maven.org/maven2/") + # This is a URL for artifacts from a "fake" release on pdillinger's fork, + # so as not to put binaries in git (ew). We should move to hosting these + # under the facebook account on github, or something else more reliable + # than maven.org, which has been failing frequently from Travis. + set(DEPS_URL "https://github.com/pdillinger/rocksdb/releases/download/v6.6.x-java-deps") endif() if(NOT EXISTS ${JAVA_JUNIT_JAR}) message("Downloading ${JAVA_JUNIT_JAR}") - file(DOWNLOAD ${SEARCH_REPO_URL}junit/junit/4.12/junit-4.12.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${DEPS_URL}/junit-4.12.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) if(NOT error_code EQUAL 0) message(FATAL_ERROR "Failed downloading ${JAVA_JUNIT_JAR}") @@ -332,7 +334,7 @@ if(NOT EXISTS ${JAVA_JUNIT_JAR}) endif() if(NOT EXISTS ${JAVA_HAMCR_JAR}) message("Downloading ${JAVA_HAMCR_JAR}") - file(DOWNLOAD ${SEARCH_REPO_URL}org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${DEPS_URL}/hamcrest-core-1.3.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) if(NOT error_code EQUAL 0) message(FATAL_ERROR "Failed downloading ${JAVA_HAMCR_JAR}") @@ -341,7 +343,7 @@ if(NOT EXISTS ${JAVA_HAMCR_JAR}) endif() if(NOT EXISTS ${JAVA_MOCKITO_JAR}) message("Downloading ${JAVA_MOCKITO_JAR}") - file(DOWNLOAD ${SEARCH_REPO_URL}org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${DEPS_URL}/mockito-all-1.10.19.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) if(NOT error_code EQUAL 0) message(FATAL_ERROR "Failed downloading ${JAVA_MOCKITO_JAR}") @@ -350,7 +352,7 @@ if(NOT EXISTS ${JAVA_MOCKITO_JAR}) endif() if(NOT EXISTS ${JAVA_CGLIB_JAR}) message("Downloading ${JAVA_CGLIB_JAR}") - file(DOWNLOAD ${SEARCH_REPO_URL}cglib/cglib/2.2.2/cglib-2.2.2.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${DEPS_URL}/cglib-2.2.2.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) if(NOT error_code EQUAL 0) message(FATAL_ERROR "Failed downloading ${JAVA_CGLIB_JAR}") @@ -359,7 +361,7 @@ if(NOT EXISTS ${JAVA_CGLIB_JAR}) endif() if(NOT EXISTS ${JAVA_ASSERTJ_JAR}) message("Downloading ${JAVA_ASSERTJ_JAR}") - file(DOWNLOAD ${CENTRAL_REPO_URL}org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${DEPS_URL}/assertj-core-1.7.1.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) if(NOT error_code EQUAL 0) message(FATAL_ERROR "Failed downloading ${JAVA_ASSERTJ_JAR}") diff --git a/java/Makefile b/java/Makefile index b3b89eb8..e0527e64 100644 --- a/java/Makefile +++ b/java/Makefile @@ -192,8 +192,11 @@ ifneq ($(DEBUG_LEVEL),0) JAVAC_ARGS = -Xlint:deprecation -Xlint:unchecked endif -SEARCH_REPO_URL?=http://search.maven.org/remotecontent?filepath= -CENTRAL_REPO_URL?=http://central.maven.org/maven2/ +# This is a URL for artifacts from a "fake" release on pdillinger's fork, +# so as not to put binaries in git (ew). We should move to hosting these +# under the facebook account on github, or something else more reliable +# than maven.org, which has been failing frequently from Travis. +DEPS_URL?=https://github.com/pdillinger/rocksdb/releases/download/v6.6.x-java-deps clean: $(AM_V_at)rm -rf include/* @@ -250,11 +253,11 @@ optimistic_transaction_sample: java resolve_test_deps: test -d "$(JAVA_TEST_LIBDIR)" || mkdir -p "$(JAVA_TEST_LIBDIR)" - test -s "$(JAVA_JUNIT_JAR)" || cp $(MVN_LOCAL)/junit/junit/4.12/junit-4.12.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o $(JAVA_JUNIT_JAR) $(SEARCH_REPO_URL)junit/junit/4.12/junit-4.12.jar - test -s "$(JAVA_HAMCR_JAR)" || cp $(MVN_LOCAL)/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o $(JAVA_HAMCR_JAR) $(SEARCH_REPO_URL)org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar - test -s "$(JAVA_MOCKITO_JAR)" || cp $(MVN_LOCAL)/org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_MOCKITO_JAR)" $(SEARCH_REPO_URL)org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar - test -s "$(JAVA_CGLIB_JAR)" || cp $(MVN_LOCAL)/cglib/cglib/2.2.2/cglib-2.2.2.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_CGLIB_JAR)" $(SEARCH_REPO_URL)cglib/cglib/2.2.2/cglib-2.2.2.jar - test -s "$(JAVA_ASSERTJ_JAR)" || cp $(MVN_LOCAL)/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_ASSERTJ_JAR)" $(CENTRAL_REPO_URL)org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar + test -s "$(JAVA_JUNIT_JAR)" || cp $(MVN_LOCAL)/junit/junit/4.12/junit-4.12.jar $(JAVA_TEST_LIBDIR) || curl --fail --insecure --output $(JAVA_JUNIT_JAR) --location $(DEPS_URL)/junit-4.12.jar + test -s "$(JAVA_HAMCR_JAR)" || cp $(MVN_LOCAL)/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar $(JAVA_TEST_LIBDIR) || curl --fail --insecure --output $(JAVA_HAMCR_JAR) --location $(DEPS_URL)/hamcrest-core-1.3.jar + test -s "$(JAVA_MOCKITO_JAR)" || cp $(MVN_LOCAL)/org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar $(JAVA_TEST_LIBDIR) || curl --fail --insecure --output "$(JAVA_MOCKITO_JAR)" --location $(DEPS_URL)/mockito-all-1.10.19.jar + test -s "$(JAVA_CGLIB_JAR)" || cp $(MVN_LOCAL)/cglib/cglib/2.2.2/cglib-2.2.2.jar $(JAVA_TEST_LIBDIR) || curl --fail --insecure --output "$(JAVA_CGLIB_JAR)" --location $(DEPS_URL)/cglib-2.2.2.jar + test -s "$(JAVA_ASSERTJ_JAR)" || cp $(MVN_LOCAL)/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar $(JAVA_TEST_LIBDIR) || curl --fail --insecure --output "$(JAVA_ASSERTJ_JAR)" --location $(DEPS_URL)/assertj-core-1.7.1.jar java_test: java resolve_test_deps $(AM_V_GEN)mkdir -p $(TEST_CLASSES) -- 2.47.3