From: Peter Dillinger Date: Thu, 30 Jan 2020 19:00:08 +0000 (-0800) Subject: Don't download from (unreliable) maven.org (#6348) X-Git-Tag: v6.7.3~7 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=944346d0fa288e24469b76f9a0f37450b5bbce07;p=rocksdb.git Don't download from (unreliable) maven.org (#6348) Summary: I set up a mirror of our Java deps on github so we can download them through github URLs rather than maven.org, which is proving terribly unreliable from Travis builds. Also sanitized calls to curl, so they are easier to read and appropriately fail on download failure. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6348 Test Plan: CI Differential Revision: D19633621 Pulled By: pdillinger fbshipit-source-id: 7eb3f730953db2ead758dc94039c040f406790f3 --- diff --git a/.travis.yml b/.travis.yml index 5d91a4e7..6f8f77ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -81,7 +81,7 @@ install: sudo apt-get install -y mingw-w64 ; fi - if [[ "${JOB_NAME}" == cmake* ]] && [ "${TRAVIS_OS_NAME}" == linux ]; then - mkdir cmake-dist && curl -sfSL https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.tar.gz | tar --strip-components=1 -C cmake-dist -xz && export PATH=$PWD/cmake-dist/bin:$PATH; + mkdir cmake-dist && curl --silent --fail --show-error --location https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.tar.gz | tar --strip-components=1 -C cmake-dist -xz && export PATH=$PWD/cmake-dist/bin:$PATH; fi - if [[ "${JOB_NAME}" == java_test ]]; then java -version && echo "JAVA_HOME=${JAVA_HOME}"; diff --git a/Makefile b/Makefile index 8322b325..bad05c1b 100644 --- a/Makefile +++ b/Makefile @@ -1839,7 +1839,7 @@ endif libz.a: -rm -rf zlib-$(ZLIB_VER) ifeq (,$(wildcard ./zlib-$(ZLIB_VER).tar.gz)) - curl --output zlib-$(ZLIB_VER).tar.gz -L ${ZLIB_DOWNLOAD_BASE}/zlib-$(ZLIB_VER).tar.gz + curl --fail --output zlib-$(ZLIB_VER).tar.gz --location ${ZLIB_DOWNLOAD_BASE}/zlib-$(ZLIB_VER).tar.gz endif ZLIB_SHA256_ACTUAL=`$(SHA256_CMD) zlib-$(ZLIB_VER).tar.gz | cut -d ' ' -f 1`; \ if [ "$(ZLIB_SHA256)" != "$$ZLIB_SHA256_ACTUAL" ]; then \ @@ -1853,7 +1853,7 @@ endif libbz2.a: -rm -rf bzip2-$(BZIP2_VER) ifeq (,$(wildcard ./bzip2-$(BZIP2_VER).tar.gz)) - curl --output bzip2-$(BZIP2_VER).tar.gz -L ${CURL_SSL_OPTS} ${BZIP2_DOWNLOAD_BASE}/bzip2-$(BZIP2_VER).tar.gz + curl --fail --output bzip2-$(BZIP2_VER).tar.gz --location ${CURL_SSL_OPTS} ${BZIP2_DOWNLOAD_BASE}/bzip2-$(BZIP2_VER).tar.gz endif BZIP2_SHA256_ACTUAL=`$(SHA256_CMD) bzip2-$(BZIP2_VER).tar.gz | cut -d ' ' -f 1`; \ if [ "$(BZIP2_SHA256)" != "$$BZIP2_SHA256_ACTUAL" ]; then \ @@ -1867,7 +1867,7 @@ endif libsnappy.a: -rm -rf snappy-$(SNAPPY_VER) ifeq (,$(wildcard ./snappy-$(SNAPPY_VER).tar.gz)) - curl --output snappy-$(SNAPPY_VER).tar.gz -L ${CURL_SSL_OPTS} ${SNAPPY_DOWNLOAD_BASE}/$(SNAPPY_VER).tar.gz + curl --fail --output snappy-$(SNAPPY_VER).tar.gz --location ${CURL_SSL_OPTS} ${SNAPPY_DOWNLOAD_BASE}/$(SNAPPY_VER).tar.gz endif SNAPPY_SHA256_ACTUAL=`$(SHA256_CMD) snappy-$(SNAPPY_VER).tar.gz | cut -d ' ' -f 1`; \ if [ "$(SNAPPY_SHA256)" != "$$SNAPPY_SHA256_ACTUAL" ]; then \ @@ -1882,7 +1882,7 @@ endif liblz4.a: -rm -rf lz4-$(LZ4_VER) ifeq (,$(wildcard ./lz4-$(LZ4_VER).tar.gz)) - curl --output lz4-$(LZ4_VER).tar.gz -L ${CURL_SSL_OPTS} ${LZ4_DOWNLOAD_BASE}/v$(LZ4_VER).tar.gz + curl --fail --output lz4-$(LZ4_VER).tar.gz --location ${CURL_SSL_OPTS} ${LZ4_DOWNLOAD_BASE}/v$(LZ4_VER).tar.gz endif LZ4_SHA256_ACTUAL=`$(SHA256_CMD) lz4-$(LZ4_VER).tar.gz | cut -d ' ' -f 1`; \ if [ "$(LZ4_SHA256)" != "$$LZ4_SHA256_ACTUAL" ]; then \ @@ -1896,7 +1896,7 @@ endif libzstd.a: -rm -rf zstd-$(ZSTD_VER) ifeq (,$(wildcard ./zstd-$(ZSTD_VER).tar.gz)) - curl --output zstd-$(ZSTD_VER).tar.gz -L ${CURL_SSL_OPTS} ${ZSTD_DOWNLOAD_BASE}/v$(ZSTD_VER).tar.gz + curl --fail --output zstd-$(ZSTD_VER).tar.gz --location ${CURL_SSL_OPTS} ${ZSTD_DOWNLOAD_BASE}/v$(ZSTD_VER).tar.gz endif ZSTD_SHA256_ACTUAL=`$(SHA256_CMD) zstd-$(ZSTD_VER).tar.gz | cut -d ' ' -f 1`; \ if [ "$(ZSTD_SHA256)" != "$$ZSTD_SHA256_ACTUAL" ]; then \ diff --git a/appveyor.yml b/appveyor.yml index bae8374f..416b6550 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -29,7 +29,7 @@ install: - md %THIRDPARTY_HOME% - echo "Building Snappy dependency..." - cd %THIRDPARTY_HOME% - - curl -fsSL -o snappy-1.1.7.zip https://github.com/google/snappy/archive/1.1.7.zip + - curl --fail --silent --show-error --output snappy-1.1.7.zip --location https://github.com/google/snappy/archive/1.1.7.zip - unzip snappy-1.1.7.zip - cd snappy-1.1.7 - mkdir build @@ -39,7 +39,7 @@ install: - msbuild Snappy.sln /p:Configuration=Release /p:Platform=x64 - echo "Building LZ4 dependency..." - cd %THIRDPARTY_HOME% - - curl -fsSL -o lz4-1.8.3.zip https://github.com/lz4/lz4/archive/v1.8.3.zip + - curl --fail --silent --show-error --output lz4-1.8.3.zip --location https://github.com/lz4/lz4/archive/v1.8.3.zip - unzip lz4-1.8.3.zip - cd lz4-1.8.3\visual\VS2010 - ps: $CMD="$Env:DEV_ENV"; & $CMD lz4.sln /upgrade @@ -47,7 +47,7 @@ install: - msbuild lz4.sln /p:Configuration=Release /p:Platform=x64 - echo "Building ZStd dependency..." - cd %THIRDPARTY_HOME% - - curl -fsSL -o zstd-1.4.0.zip https://github.com/facebook/zstd/archive/v1.4.0.zip + - curl --fail --silent --show-error --output zstd-1.4.0.zip --location https://github.com/facebook/zstd/archive/v1.4.0.zip - unzip zstd-1.4.0.zip - cd zstd-1.4.0\build\VS2010 - ps: $CMD="$Env:DEV_ENV"; & $CMD zstd.sln /upgrade diff --git a/build_tools/regression_build_test.sh b/build_tools/regression_build_test.sh index bcf8907c..31fb90eb 100755 --- a/build_tools/regression_build_test.sh +++ b/build_tools/regression_build_test.sh @@ -378,7 +378,7 @@ function send_to_ods { echo >&2 "ERROR: Key $key doesn't have a value." return fi - curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build$git_br&key=$key&value=$value" \ + curl --silent "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build$git_br&key=$key&value=$value" \ --connect-timeout 60 } diff --git a/build_tools/rocksdb-lego-determinator b/build_tools/rocksdb-lego-determinator index 21ca5ea5..32f558b5 100755 --- a/build_tools/rocksdb-lego-determinator +++ b/build_tools/rocksdb-lego-determinator @@ -880,7 +880,7 @@ run_regression() # parameters: $1 -- key, $2 -- value function send_size_to_ods { - curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \ + curl --silent "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \ --connect-timeout 60 } diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index 8be8a741..750195c3 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -313,17 +313,19 @@ if(NOT EXISTS ${JAVA_TEST_LIBDIR}) file(MAKE_DIRECTORY mkdir ${JAVA_TEST_LIBDIR}) endif() -if (DEFINED CUSTOM_REPO_URL) - set(SEARCH_REPO_URL ${CUSTOM_REPO_URL}/) - set(CENTRAL_REPO_URL ${CUSTOM_REPO_URL}/) +if (DEFINED CUSTOM_DEPS_URL) + set(DEPS_URL ${CUSTOM_DEPS_URL}/) else () - set(SEARCH_REPO_URL "http://search.maven.org/remotecontent?filepath=") - set(CENTRAL_REPO_URL "https://repo1.maven.org/maven2/") + # This is a URL for artifacts from a "fake" release on pdillinger's fork, + # so as not to put binaries in git (ew). We should move to hosting these + # under the facebook account on github, or something else more reliable + # than maven.org, which has been failing frequently from Travis. + set(DEPS_URL "https://github.com/pdillinger/rocksdb/releases/download/v6.6.x-java-deps") endif() if(NOT EXISTS ${JAVA_JUNIT_JAR}) message("Downloading ${JAVA_JUNIT_JAR}") - file(DOWNLOAD ${SEARCH_REPO_URL}junit/junit/4.12/junit-4.12.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${DEPS_URL}/junit-4.12.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) if(NOT error_code EQUAL 0) message(FATAL_ERROR "Failed downloading ${JAVA_JUNIT_JAR}") @@ -332,7 +334,7 @@ if(NOT EXISTS ${JAVA_JUNIT_JAR}) endif() if(NOT EXISTS ${JAVA_HAMCR_JAR}) message("Downloading ${JAVA_HAMCR_JAR}") - file(DOWNLOAD ${SEARCH_REPO_URL}org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${DEPS_URL}/hamcrest-core-1.3.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) if(NOT error_code EQUAL 0) message(FATAL_ERROR "Failed downloading ${JAVA_HAMCR_JAR}") @@ -341,7 +343,7 @@ if(NOT EXISTS ${JAVA_HAMCR_JAR}) endif() if(NOT EXISTS ${JAVA_MOCKITO_JAR}) message("Downloading ${JAVA_MOCKITO_JAR}") - file(DOWNLOAD ${SEARCH_REPO_URL}org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${DEPS_URL}/mockito-all-1.10.19.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) if(NOT error_code EQUAL 0) message(FATAL_ERROR "Failed downloading ${JAVA_MOCKITO_JAR}") @@ -350,7 +352,7 @@ if(NOT EXISTS ${JAVA_MOCKITO_JAR}) endif() if(NOT EXISTS ${JAVA_CGLIB_JAR}) message("Downloading ${JAVA_CGLIB_JAR}") - file(DOWNLOAD ${SEARCH_REPO_URL}cglib/cglib/2.2.2/cglib-2.2.2.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${DEPS_URL}/cglib-2.2.2.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) if(NOT error_code EQUAL 0) message(FATAL_ERROR "Failed downloading ${JAVA_CGLIB_JAR}") @@ -359,7 +361,7 @@ if(NOT EXISTS ${JAVA_CGLIB_JAR}) endif() if(NOT EXISTS ${JAVA_ASSERTJ_JAR}) message("Downloading ${JAVA_ASSERTJ_JAR}") - file(DOWNLOAD ${CENTRAL_REPO_URL}org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${DEPS_URL}/assertj-core-1.7.1.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) if(NOT error_code EQUAL 0) message(FATAL_ERROR "Failed downloading ${JAVA_ASSERTJ_JAR}") diff --git a/java/Makefile b/java/Makefile index 96d41a4a..378d21b0 100644 --- a/java/Makefile +++ b/java/Makefile @@ -213,8 +213,11 @@ ifneq ($(DEBUG_LEVEL),0) JAVAC_ARGS = -Xlint:deprecation -Xlint:unchecked endif -SEARCH_REPO_URL?=http://search.maven.org/remotecontent?filepath= -CENTRAL_REPO_URL?=https://repo1.maven.org/maven2/ +# This is a URL for artifacts from a "fake" release on pdillinger's fork, +# so as not to put binaries in git (ew). We should move to hosting these +# under the facebook account on github, or something else more reliable +# than maven.org, which has been failing frequently from Travis. +DEPS_URL?=https://github.com/pdillinger/rocksdb/releases/download/v6.6.x-java-deps clean: clean-not-downloaded clean-downloaded @@ -283,11 +286,11 @@ optimistic_transaction_sample: java resolve_test_deps: test -d "$(JAVA_TEST_LIBDIR)" || mkdir -p "$(JAVA_TEST_LIBDIR)" - test -s "$(JAVA_JUNIT_JAR)" || cp $(MVN_LOCAL)/junit/junit/4.12/junit-4.12.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o $(JAVA_JUNIT_JAR) $(SEARCH_REPO_URL)junit/junit/4.12/junit-4.12.jar - test -s "$(JAVA_HAMCR_JAR)" || cp $(MVN_LOCAL)/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o $(JAVA_HAMCR_JAR) $(SEARCH_REPO_URL)org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar - test -s "$(JAVA_MOCKITO_JAR)" || cp $(MVN_LOCAL)/org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_MOCKITO_JAR)" $(SEARCH_REPO_URL)org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar - test -s "$(JAVA_CGLIB_JAR)" || cp $(MVN_LOCAL)/cglib/cglib/2.2.2/cglib-2.2.2.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_CGLIB_JAR)" $(SEARCH_REPO_URL)cglib/cglib/2.2.2/cglib-2.2.2.jar - test -s "$(JAVA_ASSERTJ_JAR)" || cp $(MVN_LOCAL)/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_ASSERTJ_JAR)" $(CENTRAL_REPO_URL)org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar + test -s "$(JAVA_JUNIT_JAR)" || cp $(MVN_LOCAL)/junit/junit/4.12/junit-4.12.jar $(JAVA_TEST_LIBDIR) || curl --fail --insecure --output $(JAVA_JUNIT_JAR) --location $(DEPS_URL)/junit-4.12.jar + test -s "$(JAVA_HAMCR_JAR)" || cp $(MVN_LOCAL)/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar $(JAVA_TEST_LIBDIR) || curl --fail --insecure --output $(JAVA_HAMCR_JAR) --location $(DEPS_URL)/hamcrest-core-1.3.jar + test -s "$(JAVA_MOCKITO_JAR)" || cp $(MVN_LOCAL)/org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar $(JAVA_TEST_LIBDIR) || curl --fail --insecure --output "$(JAVA_MOCKITO_JAR)" --location $(DEPS_URL)/mockito-all-1.10.19.jar + test -s "$(JAVA_CGLIB_JAR)" || cp $(MVN_LOCAL)/cglib/cglib/2.2.2/cglib-2.2.2.jar $(JAVA_TEST_LIBDIR) || curl --fail --insecure --output "$(JAVA_CGLIB_JAR)" --location $(DEPS_URL)/cglib-2.2.2.jar + test -s "$(JAVA_ASSERTJ_JAR)" || cp $(MVN_LOCAL)/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar $(JAVA_TEST_LIBDIR) || curl --fail --insecure --output "$(JAVA_ASSERTJ_JAR)" --location $(DEPS_URL)/assertj-core-1.7.1.jar java_test: java resolve_test_deps $(AM_V_GEN)mkdir -p $(TEST_CLASSES)