qa/rgw: pull Apache artifacts from mirror instead of archive.apache.org

Currently maven and kafka are pulled from archive.apache.org. This
uses Apache's "closer" calculator to find a mirror to use instead.

Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>
(cherry picked from commit 3aae66611dd7f05612056a757cb7a87dfcf95de0)

diff --git a/qa/suites/rgw/notifications/tasks/test_kafka.yaml b/qa/suites/rgw/notifications/tasks/test_kafka.yaml
index ae647df386532a0d405f403644d35095d0724f19..4407cd3eaccc3858de213c749cd3b7579c644935 100644 (file)
--- a/qa/suites/rgw/notifications/tasks/test_kafka.yaml
+++ b/qa/suites/rgw/notifications/tasks/test_kafka.yaml
@@ -1,7 +1,7 @@
 tasks:
 - kafka:
     client.0:
-      kafka_version: 2.6.0
+      kafka_version: 3.8.1
 - notification-tests:
     client.0:
       extra_attr: ["kafka_test"]

diff --git a/qa/tasks/kafka.py b/qa/tasks/kafka.py
index 5e6c208ca30e7e868745b5f58fcf8886252f7d36..833f03babf6907f678a14124de4ff369bf41847a 100644 (file)
--- a/qa/tasks/kafka.py
+++ b/qa/tasks/kafka.py
@@ -4,6 +4,7 @@ Deploy and configure Kafka for Teuthology
 import contextlib
 import logging
 import time
+import os
 
 from teuthology import misc as teuthology
 from teuthology import contextutil
@@ -33,6 +34,13 @@ def install_kafka(ctx, config):
     assert isinstance(config, dict)
     log.info('Installing Kafka...')
 
+    # programmatically find a nearby mirror so as not to hammer archive.apache.org
+    apache_mirror_cmd="curl 'https://www.apache.org/dyn/closer.cgi' 2>/dev/null | " \
+        "grep -o '<strong>[^<]*</strong>' | sed 's/<[^>]*>//g' | head -n 1"
+    log.info("determining apache mirror by running: " + apache_mirror_cmd)
+    apache_mirror_url_front = os.popen(apache_mirror_cmd).read().rstrip() # note: includes trailing slash (/)
+    log.info("chosen apache mirror is " + apache_mirror_url_front)
+
     for (client, _) in config.items():
         (remote,) = ctx.cluster.only(client).remotes.keys()
         test_dir=teuthology.get_testdir(ctx)
@@ -40,7 +48,8 @@ def install_kafka(ctx, config):
 
         kafka_file =  kafka_prefix + current_version + '.tgz'
 
-        link1 = 'https://archive.apache.org/dist/kafka/' + current_version + '/' + kafka_file
+        link1 = '{apache_mirror_url_front}/kafka/'.format(apache_mirror_url_front=apache_mirror_url_front) + \
+            current_version + '/' + kafka_file
         ctx.cluster.only(client).run(
             args=['cd', '{tdir}'.format(tdir=test_dir), run.Raw('&&'), 'wget', link1],
         )

diff --git a/qa/tasks/s3a_hadoop.py b/qa/tasks/s3a_hadoop.py
index 7b77359fcf2bb8a3c86dc9e9e0a2297a827f95b3..717d9858e60ceb4e270ed38ee2f0afb4100c1e01 100644 (file)
--- a/qa/tasks/s3a_hadoop.py
+++ b/qa/tasks/s3a_hadoop.py
@@ -1,5 +1,6 @@
 import contextlib
 import logging
+import os
 from teuthology import misc
 from teuthology.orchestra import run
 
@@ -40,7 +41,7 @@ def task(ctx, config):
 
     # get versions
     maven_major = config.get('maven-major', 'maven-3')
-    maven_version = config.get('maven-version', '3.6.3')
+    maven_version = config.get('maven-version', '3.9.9')
     hadoop_ver = config.get('hadoop-version', '2.9.2')
     bucket_name = config.get('bucket-name', 's3atest')
     access_key = config.get('access-key', 'EGAQRD2ULOIFKFSKCT4F')
@@ -48,11 +49,19 @@ def task(ctx, config):
         'secret-key',
         'zi816w1vZKfaSM85Cl0BxXTwSLyN7zB4RbTswrGb')
 
+    # programmatically find a nearby mirror so as not to hammer archive.apache.org
+    apache_mirror_cmd="curl 'https://www.apache.org/dyn/closer.cgi' 2>/dev/null | " \
+        "grep -o '<strong>[^<]*</strong>' | sed 's/<[^>]*>//g' | head -n 1"
+    log.info("determining apache mirror by running: " + apache_mirror_cmd)
+    apache_mirror_url_front = os.popen(apache_mirror_cmd).read().rstrip() # note: includes trailing slash (/)
+    log.info("chosen apache mirror is " + apache_mirror_url_front)
+
     # set versions for cloning the repo
     apache_maven = 'apache-maven-{maven_version}-bin.tar.gz'.format(
         maven_version=maven_version)
-    maven_link = 'http://archive.apache.org/dist/maven/' + \
-        '{maven_major}/{maven_version}/binaries/'.format(maven_major=maven_major, maven_version=maven_version) + apache_maven
+    maven_link = '{apache_mirror_url_front}/maven/'.format(apache_mirror_url_front=apache_mirror_url_front) + \
+        '{maven_major}/{maven_version}/binaries/'.format(maven_major=maven_major, maven_version=maven_version) + \
+        apache_maven
     hadoop_git = 'https://github.com/apache/hadoop'
     hadoop_rel = 'hadoop-{ver} rel/release-{ver}'.format(ver=hadoop_ver)
     if hadoop_ver == 'trunk':