From 1e07237d92da3bb5eba82b49f6276d17696e6985 Mon Sep 17 00:00:00 2001 From: Ernesto Puerta Date: Fri, 19 Jul 2019 18:10:49 +0200 Subject: [PATCH] mgr/dashboard: clean-up tox and unit tests Refactor CMake add_tox_test to automatically add py27 and/or py3 to provided toxenvs. Refactor tox.ini: - Remove requirements-{py27,py3}.txt, as python release dependant packages can be handled with PEP 508 syntax. - Remove develepment dependencies from requirements. - Move pycodestyle settings to separate section. - Add flake8 check and other checkers (rst, naming, etc). Some of them are commented out for future clean-ups (Ceph trackers have been opened) - Pycodestyle removed, as flake8 is a wrapper for pycodestyle. - Add instafail plugin to report failures immediately - Add timeout plugin to limit max run time (sometimes test_tasks hangs) - Remove unused dependencies (lru_cache, pluggy) Test and code linting fixes: - Unused imports - Fixes to HACKING.rst Doc: - Update HACKING.rst Add conftest.py to mock imported modules (rados, rbd, cephfs), and mock also rados Error and OSError Exceptions. Fixes: https://tracker.ceph.com/issues/40487 Fixes: https://tracker.ceph.com/issues/41152 Signed-off-by: Ernesto Puerta --- cmake/modules/AddCephTest.cmake | 2 +- install-deps.sh | 17 ++-- src/pybind/mgr/dashboard/.pylintrc | 4 +- src/pybind/mgr/dashboard/CMakeLists.txt | 9 +- src/pybind/mgr/dashboard/HACKING.rst | 58 ++++++------ src/pybind/mgr/dashboard/__init__.py | 9 +- src/pybind/mgr/dashboard/conftest.py | 26 ++++++ src/pybind/mgr/dashboard/constraints.txt | 10 +++ .../mgr/dashboard/controllers/__init__.py | 10 +-- src/pybind/mgr/dashboard/controllers/osd.py | 2 +- src/pybind/mgr/dashboard/controllers/pool.py | 2 +- .../dashboard/controllers/rbd_mirroring.py | 2 +- src/pybind/mgr/dashboard/controllers/saml2.py | 5 +- src/pybind/mgr/dashboard/plugins/__init__.py | 2 +- .../mgr/dashboard/plugins/feature_toggles.py | 2 +- .../mgr/dashboard/requirements-lint.txt | 9 ++ .../mgr/dashboard/requirements-py27.txt | 3 - src/pybind/mgr/dashboard/requirements-py3.txt | 3 - .../mgr/dashboard/requirements-test.txt | 4 + src/pybind/mgr/dashboard/requirements.txt | 45 +++------- src/pybind/mgr/dashboard/rest_client.py | 1 - .../mgr/dashboard/services/exception.py | 6 +- .../mgr/dashboard/services/iscsi_client.py | 2 +- src/pybind/mgr/dashboard/services/sso.py | 14 ++- src/pybind/mgr/dashboard/tests/__init__.py | 3 +- .../mgr/dashboard/tests/test_api_auditing.py | 5 +- .../dashboard/tests/test_feature_toggles.py | 5 +- .../mgr/dashboard/tests/test_ganesha.py | 5 +- src/pybind/mgr/dashboard/tests/test_iscsi.py | 6 +- src/pybind/mgr/dashboard/tests/test_osd.py | 5 +- src/pybind/mgr/dashboard/tests/test_pool.py | 5 +- .../mgr/dashboard/tests/test_prometheus.py | 5 +- .../mgr/dashboard/tests/test_rbd_mirroring.py | 5 +- .../mgr/dashboard/tests/test_rest_client.py | 6 +- .../mgr/dashboard/tests/test_rest_tasks.py | 5 +- src/pybind/mgr/dashboard/tests/test_rgw.py | 5 +- .../mgr/dashboard/tests/test_rgw_client.py | 6 +- src/pybind/mgr/dashboard/tests/test_tools.py | 5 +- src/pybind/mgr/dashboard/tools.py | 2 +- src/pybind/mgr/dashboard/tox.ini | 90 ++++++++++++++----- src/tools/setup-virtualenv.sh | 13 ++- 41 files changed, 265 insertions(+), 158 deletions(-) create mode 100644 src/pybind/mgr/dashboard/conftest.py create mode 100644 src/pybind/mgr/dashboard/constraints.txt create mode 100644 src/pybind/mgr/dashboard/requirements-lint.txt delete mode 100644 src/pybind/mgr/dashboard/requirements-py27.txt delete mode 100644 src/pybind/mgr/dashboard/requirements-py3.txt create mode 100644 src/pybind/mgr/dashboard/requirements-test.txt diff --git a/cmake/modules/AddCephTest.cmake b/cmake/modules/AddCephTest.cmake index bb5936cf8ba61..b21ae660166ce 100644 --- a/cmake/modules/AddCephTest.cmake +++ b/cmake/modules/AddCephTest.cmake @@ -69,7 +69,7 @@ function(add_tox_test name) list(APPEND tox_envs py3) endif() if(DEFINED TOXTEST_TOX_ENVS) - set(tox_envs ${TOXTEST_TOX_ENVS}) + list(APPEND tox_envs ${TOXTEST_TOX_ENVS}) endif() string(REPLACE ";" "," tox_envs "${tox_envs}") add_custom_command( diff --git a/install-deps.sh b/install-deps.sh index 177d89229b017..f116c213b64c9 100755 --- a/install-deps.sh +++ b/install-deps.sh @@ -439,9 +439,11 @@ function populate_wheelhouse() { # although pip comes with virtualenv, having a recent version # of pip matters when it comes to using wheel packages - pip --timeout 300 $install 'setuptools >= 0.8' 'pip >= 7.0' 'wheel >= 0.24' || return 1 + PIP_OPTS="--timeout 300 --exists-action i" + pip $PIP_OPTS $install \ + 'setuptools >= 0.8' 'pip >= 7.0' 'wheel >= 0.24' 'tox >= 2.9.1' || return 1 if test $# != 0 ; then - pip --timeout 300 $install $@ || return 1 + pip $PIP_OPTS $install $@ || return 1 fi } @@ -485,10 +487,13 @@ wip_wheelhouse=wheelhouse-wip find . -name tox.ini | while read ini ; do ( cd $(dirname $ini) - require=$(ls *requirements.txt 2>/dev/null | sed -e 's/^/-r /') + require_files=$(ls *requirements*.txt 2>/dev/null) || true + constraint_files=$(ls *constraints*.txt 2>/dev/null) || true + require=$(echo -n "$require_files" | sed -e 's/^/-r /') + constraint=$(echo -n "$constraint_files" | sed -e 's/^/-c /') md5=wheelhouse/md5 if test "$require"; then - if ! test -f $md5 || ! md5sum -c $md5 ; then + if ! test -f $md5 || ! md5sum -c $md5 > /dev/null; then rm -rf wheelhouse fi fi @@ -496,10 +501,10 @@ find . -name tox.ini | while read ini ; do for interpreter in python2.7 python3 ; do type $interpreter > /dev/null 2>&1 || continue activate_virtualenv $top_srcdir $interpreter || exit 1 - populate_wheelhouse "wheel -w $wip_wheelhouse" $require || exit 1 + populate_wheelhouse "download -d $wip_wheelhouse" $require $constraint || exit 1 done mv $wip_wheelhouse wheelhouse - md5sum *requirements.txt > $md5 + md5sum $require_files $constraint_files > $md5 fi ) done diff --git a/src/pybind/mgr/dashboard/.pylintrc b/src/pybind/mgr/dashboard/.pylintrc index caa510fef7f26..8efd540f49e53 100644 --- a/src/pybind/mgr/dashboard/.pylintrc +++ b/src/pybind/mgr/dashboard/.pylintrc @@ -393,7 +393,7 @@ ignored-classes=optparse.Values,thread._local,_thread._local # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis. It # supports qualified module names, as well as Unix pattern matching. -ignored-modules=cherrypy,distutils +ignored-modules=cherrypy,distutils,rados,rbd,cephfs # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. @@ -432,7 +432,7 @@ ignore-comments=yes ignore-docstrings=yes # Ignore imports when computing similarities. -ignore-imports=no +ignore-imports=yes # Minimum lines number of a similarity. min-similarity-lines=4 diff --git a/src/pybind/mgr/dashboard/CMakeLists.txt b/src/pybind/mgr/dashboard/CMakeLists.txt index 946e730eb02ed..03c4b4b571480 100644 --- a/src/pybind/mgr/dashboard/CMakeLists.txt +++ b/src/pybind/mgr/dashboard/CMakeLists.txt @@ -98,12 +98,5 @@ endif(WITH_MGR_DASHBOARD_FRONTEND AND NOT CMAKE_SYSTEM_PROCESSOR MATCHES "aarch6 if(WITH_TESTS) include(AddCephTest) - if(WITH_PYTHON2) - list(APPEND dashboard_tox_envs py27-cov py27-lint py27-check) - endif() - if(WITH_PYTHON3) - list(APPEND dashboard_tox_envs py3-cov py3-lint py3-check) - endif() - add_tox_test(mgr-dashboard - TOX_ENVS ${dashboard_tox_envs}) + add_tox_test(mgr-dashboard TOX_ENVS lint check) endif() diff --git a/src/pybind/mgr/dashboard/HACKING.rst b/src/pybind/mgr/dashboard/HACKING.rst index 639434eba0076..d21dccb41c0eb 100644 --- a/src/pybind/mgr/dashboard/HACKING.rst +++ b/src/pybind/mgr/dashboard/HACKING.rst @@ -129,7 +129,7 @@ Run ``npm run test`` to execute the unit tests via `Jest `_. If you get errors on all tests, it could be because `Jest -`_ or something else was updated. +`__ or something else was updated. There are a few ways how you can try to resolve this: - Remove all modules with ``rm -rf dist node_modules`` and run ``npm install`` @@ -314,7 +314,7 @@ This components are declared on the components module: `src/pybind/mgr/dashboard/frontend/src/app/shared/components`. Helper -...... +~~~~~~ This component should be used to provide additional information to the user. @@ -453,9 +453,9 @@ To do that, check the settings in the i18n config file ``src/pybind/mgr/dashboard/frontend/i18n.config.json``:: and make sure that the organization is *ceph*, the project is *ceph-dashboard* and the resource is the one you want to pull from and push to e.g. *Master:master*. To find a list -of avaiable resources visit ``https://www.transifex.com/ceph/ceph-dashboard/content/``:: +of avaiable resources visit ``_. -After you checked the config go to the directory ``src/pybind/mgr/dashboard/frontend``:: and run +After you checked the config go to the directory ``src/pybind/mgr/dashboard/frontend`` and run:: $ npm run i18n @@ -467,7 +467,7 @@ The tool will ask you for an api token, unless you added it by running: $ npm run i18n:token -To create a transifex api token visit ``https://www.transifex.com/user/settings/api/``:: +To create a transifex api token visit ``_. After the command ran successfully, build the UI and check if everything is working as expected. You also might want to run the frontend tests. @@ -477,7 +477,7 @@ Suggestions Strings need to start and end in the same line as the element: -.. code-block:: xml +.. code-block:: html @@ -500,7 +500,7 @@ Strings need to start and end in the same line as the element: Isolated interpolations should not be translated: -.. code-block:: xml +.. code-block:: html {{ foo }} @@ -510,14 +510,14 @@ Isolated interpolations should not be translated: Interpolations used in a sentence should be kept in the translation: -.. code-block:: xml +.. code-block:: html There are {{ x }} OSDs. Remove elements that are outside the context of the translation: -.. code-block:: xml +.. code-block:: html