mgr/dashboard: run-backend-api-tests.sh CI improvements

As there is now a jenkins job to run this script
(see https://github.com/ceph/ceph-build/pull/1351),
this refactoring adapt the script to be run in a jenkins job as well as locally.

Signed-off-by: Alfonso Martínez <almartin@redhat.com>
(cherry picked from commit 7563f8b6d661886b7b3c7a259b70433a8962f55a)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index bcd25ac680651bcb56a4ff0f4b2d92109640f5ef..69a755f04b5061385b4f783dd6eadf6b189fd59c 100644 (file)
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -30,8 +30,23 @@ This was just a quick overview.  More information for contributors is available
 https://raw.githubusercontent.com/ceph/ceph/master/SubmittingPatches.rst
 
 -->
-
+## Checklist
 - [ ] References tracker ticket
 - [ ] Updates documentation if necessary
 - [ ] Includes tests for new functionality or reproducer for bug
 
+---
+
+<details>
+<summary>Show available Jenkins commands</summary>
+
+- `jenkins retest this please`
+- `jenkins test make check`
+- `jenkins test make check arm64`
+- `jenkins test submodules`
+- `jenkins test dashboard`
+- `jenkins test dashboard backend`
+- `jenkins test docs`
+- `jenkins render docs`
+
+</details>

diff --git a/qa/tasks/vstart_runner.py b/qa/tasks/vstart_runner.py
index bdfc04fc3a0f732f9828c0348ecd7cff7e1b4fd1..74b26d23df466ea6cd37aac62672004f87f753fc 100644 (file)
--- a/qa/tasks/vstart_runner.py
+++ b/qa/tasks/vstart_runner.py
@@ -109,7 +109,7 @@ if os.path.exists("./CMakeCache.txt") and os.path.exists("./bin"):
                 python_paths.append(g_exp)
 
     ld_path = os.path.join(os.getcwd(), "lib/")
-    print "Using guessed paths {0} {1}".format(ld_path, python_paths)
+    print("Using guessed paths {0} {1}".format(ld_path, python_paths))
     respawn_in_path(ld_path, python_paths)
 
 
@@ -884,6 +884,7 @@ def exec_test():
     interactive_on_error = False
     create_cluster = False
     create_cluster_only = False
+    ignore_missing_binaries = False
 
     args = sys.argv[1:]
     flags = [a for a in args if a.startswith("-")]
@@ -895,6 +896,8 @@ def exec_test():
             create_cluster = True
         elif f == "--create-cluster-only":
             create_cluster_only = True
+        elif f == "--ignore-missing-binaries":
+            ignore_missing_binaries = True
         else:
             log.error("Unknown option '{0}'".format(f))
             sys.exit(-1)
@@ -904,7 +907,7 @@ def exec_test():
     require_binaries = ["ceph-dencoder", "cephfs-journal-tool", "cephfs-data-scan",
                         "cephfs-table-tool", "ceph-fuse", "rados"]
     missing_binaries = [b for b in require_binaries if not os.path.exists(os.path.join(BIN_PREFIX, b))]
-    if missing_binaries:
+    if missing_binaries and not ignore_missing_binaries:
         log.error("Some ceph binaries missing, please build them: {0}".format(" ".join(missing_binaries)))
         sys.exit(-1)
 

diff --git a/src/pybind/mgr/dashboard/run-backend-api-tests.sh b/src/pybind/mgr/dashboard/run-backend-api-tests.sh
index 2cc17867617cbfe92c0a26fe836aec16682c76de..1e11b612e5c4ad225f372c556bb14fcd6343b75b 100755 (executable)
--- a/src/pybind/mgr/dashboard/run-backend-api-tests.sh
+++ b/src/pybind/mgr/dashboard/run-backend-api-tests.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -eo pipefail
+
 if [[ "$1" = "-h" || "$1" = "--help" ]]; then
        echo "Usage (run from ./):"
        echo -e "\t./run-backend-api-tests.sh"
@@ -17,66 +19,14 @@ if [[ "$1" = "-h" || "$1" = "--help" ]]; then
        exit 0
 fi
 
-# creating temp directory to store virtualenv and teuthology
-
 get_cmake_variable() {
     local variable=$1
     grep "$variable" CMakeCache.txt | cut -d "=" -f 2
 }
 
-setup_teuthology() {
-    TEMP_DIR=`mktemp -d`
-
+get_build_py_version() {
     CURR_DIR=`pwd`
     BUILD_DIR="$CURR_DIR/../../../../build"
-
-    read -r -d '' TEUTHOLOGY_PY_REQS <<EOF
-apache-libcloud==2.2.1 \
-asn1crypto==0.22.0 \
-backports.ssl-match-hostname==3.5.0.1 \
-bcrypt==3.1.4 \
-certifi==2018.1.18 \
-cffi==1.10.0 \
-chardet==3.0.4 \
-configobj==5.0.6 \
-cryptography==2.1.4 \
-enum34==1.1.6 \
-gevent==1.2.2 \
-greenlet==0.4.13 \
-idna==2.5 \
-ipaddress==1.0.18 \
-Jinja2==2.9.6 \
-manhole==1.5.0 \
-MarkupSafe==1.0 \
-netaddr==0.7.19 \
-packaging==16.8 \
-paramiko==2.4.0 \
-pexpect==4.4.0 \
-psutil==5.4.3 \
-ptyprocess==0.5.2 \
-pyasn1==0.2.3 \
-pycparser==2.17 \
-PyNaCl==1.2.1 \
-pyparsing==2.2.0 \
-python-dateutil==2.6.1 \
-PyYAML==3.12 \
-requests==2.18.4 \
-six==1.10.0 \
-urllib3==1.22
-EOF
-
-
-
-    cd $TEMP_DIR
-    virtualenv --python=${TEUTHOLOGY_PYTHON_BIN:-/usr/bin/python} venv
-    source venv/bin/activate
-    pip install 'setuptools >= 12'
-    eval pip install $TEUTHOLOGY_PY_REQS
-    pip install -r $CURR_DIR/requirements.txt
-    deactivate
-
-    git clone --depth 1 https://github.com/ceph/teuthology.git
-
     cd $BUILD_DIR
 
     CEPH_MGR_PY_VERSION_MAJOR=$(get_cmake_variable MGR_PYTHON_VERSION | cut -d '.' -f1)
@@ -89,18 +39,86 @@ EOF
             CEPH_PY_VERSION_MAJOR=3
         fi
     fi
+}
 
+setup_teuthology() {
+    TEMP_DIR=`mktemp -d`
+    TEUTHOLOGY_PY_REQS="
+apache-libcloud==2.2.1
+asn1crypto==0.22.0
+backports.ssl-match-hostname==3.5.0.1
+bcrypt==3.1.4
+certifi==2018.1.18
+cffi==1.10.0
+chardet==3.0.4
+configobj==5.0.6
+cryptography==2.1.4
+enum34==1.1.6
+gevent==1.2.2
+greenlet==0.4.13
+idna==2.5
+ipaddress==1.0.18
+Jinja2==2.9.6
+manhole==1.5.0
+MarkupSafe==1.0
+netaddr==0.7.19
+packaging==16.8
+paramiko==2.4.0
+pexpect==4.4.0
+psutil==5.4.3
+ptyprocess==0.5.2
+pyasn1==0.2.3
+pycparser==2.17
+PyNaCl==1.2.1
+pyparsing==2.2.0
+python-dateutil==2.6.1
+PyYAML==3.12
+requests==2.18.4
+six==1.10.0
+urllib3==1.22
+"
+
+    cd $TEMP_DIR
+    virtualenv --python=${TEUTHOLOGY_PYTHON_BIN:-/usr/bin/python} venv
+    source venv/bin/activate
+    pip install 'setuptools >= 12'
+    eval pip install $TEUTHOLOGY_PY_REQS
+    pip install -r $CURR_DIR/requirements.txt
+
+    git clone --depth 1 https://github.com/ceph/teuthology.git
+
+    deactivate
+}
+
+setup_coverage() {
+    # In CI environment we cannot install coverage in system, so we install it in a dedicated venv
+    # so only coverage is available when adding this path.
+    cd $TEMP_DIR
+    virtualenv --python=${TEUTHOLOGY_PYTHON_BIN:-/usr/bin/python} coverage-venv
+    source coverage-venv/bin/activate
     cd $CURR_DIR
+    pip install coverage==4.5.2
+    COVERAGE_PATH=$(python -c "import sysconfig; print(sysconfig.get_paths()['purelib'])")
+    deactivate
+}
 
-    COVERAGE_VERSION=$(cat requirements.txt | grep coverage)
-    if [[ "$CEPH_MGR_PY_VERSION_MAJOR" == '3' ]]; then
-        pip3 install "$COVERAGE_VERSION"
-    else
-        pip install "$COVERAGE_VERSION"
+on_tests_error() {
+    if [[ -n "$JENKINS_HOME" ]]; then
+        CEPH_OUT_DIR=${CEPH_OUT_DIR:-"$BUILD_DIR"/out}
+        MGR_LOG_FILES=$(find "$CEPH_OUT_DIR" -iname "mgr.*.log" | tr '\n' ' ')
+        MGR_LOG_FILE_LAST_LINES=60000
+        for mgr_log_file in ${MGR_LOG_FILES[@]}; do
+            printf "\n\nDisplaying last ${MGR_LOG_FILE_LAST_LINES} lines of: $mgr_log_file\n\n"
+            tail -n ${MGR_LOG_FILE_LAST_LINES} $mgr_log_file
+            printf "\n\nEnd of: $mgr_log_file\n\n"
+        done
+        printf "\n\nTEST FAILED.\n\n"
     fi
 }
 
 run_teuthology_tests() {
+    trap on_tests_error ERR
+
     cd "$BUILD_DIR"
     find ../src/pybind/mgr/dashboard/ -name '*.pyc' -exec rm -f {} \;
 
@@ -108,7 +126,8 @@ run_teuthology_tests() {
     TEST_CASES=''
     if [[ "$@" == '' || "$@" == '--create-cluster-only' ]]; then
       TEST_CASES=`for i in \`ls $BUILD_DIR/../qa/tasks/mgr/dashboard/test_*\`; do F=$(basename $i); M="${F%.*}"; echo -n " tasks.mgr.dashboard.$M"; done`
-      TEST_CASES="tasks.mgr.test_module_selftest tasks.mgr.test_dashboard $TEST_CASES"
+      # Mgr selftest module tests have to be run at the end as they stress the mgr daemon.
+      TEST_CASES="tasks.mgr.test_dashboard $TEST_CASES tasks.mgr.test_module_selftest"
       if [[ "$@" == '--create-cluster-only' ]]; then
         OPTIONS="$@"
       fi
@@ -121,14 +140,22 @@ run_teuthology_tests() {
     export PATH=$BUILD_DIR/bin:$PATH
     source $TEMP_DIR/venv/bin/activate # Run after setting PATH as it does the last PATH export.
     export LD_LIBRARY_PATH=$BUILD_DIR/lib/cython_modules/lib.${CEPH_PY_VERSION_MAJOR}/:$BUILD_DIR/lib
-    export PYTHONPATH=$TEMP_DIR/teuthology:$BUILD_DIR/../qa:$BUILD_DIR/lib/cython_modules/lib.${CEPH_PY_VERSION_MAJOR}/:$BUILD_DIR/../src/pybind
+    local source_dir=$(dirname "$BUILD_DIR")
+    local pybind_dir=$source_dir/src/pybind
+    local python_common_dir=$source_dir/src/python-common
+    # In CI environment we set python paths inside build (where you find the required frontend build: "dist" dir).
+    if [[ -n "$JENKINS_HOME" ]]; then
+        export PYBIND=$BUILD_DIR/src/pybind
+        pybind_dir=$PYBIND
+    fi
+    export PYTHONPATH=$TEMP_DIR/teuthology:$source_dir/qa:$BUILD_DIR/lib/cython_modules/lib.${CEPH_PY_VERSION_MAJOR}/:$pybind_dir:$python_common_dir:${COVERAGE_PATH}
     export RGW=${RGW:-1}
 
     export COVERAGE_ENABLED=true
     export COVERAGE_FILE=.coverage.mgr.dashboard
     find . -iname "*${COVERAGE_FILE}*" -type f -delete
 
-    eval python ../qa/tasks/vstart_runner.py $OPTIONS $TEST_CASES
+    python ../qa/tasks/vstart_runner.py --ignore-missing-binaries $OPTIONS $TEST_CASES
 
     deactivate
     cd $CURR_DIR
@@ -139,7 +166,7 @@ cleanup_teuthology() {
     killall ceph-mgr
     sleep 10
     if [[ "$COVERAGE_ENABLED" == 'true' ]]; then
-        source $TEMP_DIR/venv/bin/activate
+        source $TEMP_DIR/coverage-venv/bin/activate
         (coverage combine && coverage report) || true
         deactivate
     fi
@@ -152,16 +179,23 @@ cleanup_teuthology() {
     unset TEMP_DIR
     unset CURR_DIR
     unset BUILD_DIR
+    unset COVERAGE_PATH
+    unset get_build_py_version
     unset setup_teuthology
+    unset setup_coverage
+    unset on_tests_error
     unset run_teuthology_tests
     unset cleanup_teuthology
 }
 
+get_build_py_version
 setup_teuthology
+setup_coverage
 run_teuthology_tests --create-cluster-only
 
-# End sourced section
-return 2> /dev/null
+# End sourced section. Do not exit shell when the script has been sourced.
+set +e
+return 2>/dev/null || set -eo pipefail
 
 run_teuthology_tests "$@"
 cleanup_teuthology