From eac1398c68361f2bf7056b68e90a1b55873dae21 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Thu, 14 Apr 2022 15:04:17 -0600 Subject: [PATCH] Revert "Merge pull request #1740 from ceph/wip-revert4333" This reverts commit 0e6dbbf154347a8424d198a9354985167ff64a43, reversing changes made to 43327234bc2c8066c173e4c6d602c8a0a05db642. --- MANIFEST.in | 2 + bootstrap | 66 +++------ docs/docker-compose/teuthology/Dockerfile | 2 +- pyproject.toml | 10 ++ requirements.in | 44 ------ requirements.txt | 119 +++++---------- scripts/dispatcher.py | 3 +- setup.cfg | 133 +++++++++++++++++ setup.py | 138 ------------------ teuthology/__init__.py | 40 ++--- teuthology/dispatcher/__init__.py | 5 +- teuthology/dispatcher/supervisor.py | 10 +- .../provision/cloud/test/test_openstack.py | 4 +- teuthology/run_tasks.py | 12 ++ teuthology/task/tests/__init__.py | 8 +- teuthology/test/test_worker.py | 11 +- teuthology/worker.py | 22 +-- tox.ini | 15 +- update-requirements.sh | 3 +- 19 files changed, 259 insertions(+), 388 deletions(-) create mode 100644 pyproject.toml delete mode 100644 requirements.in create mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/MANIFEST.in b/MANIFEST.in index bb73d9626..2683cd654 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,4 @@ include *.rst include requirements.txt +include tox.ini +include pytest.ini diff --git a/bootstrap b/bootstrap index 3d7f4a625..394dbb0b9 100755 --- a/bootstrap +++ b/bootstrap @@ -11,19 +11,19 @@ else fi fi +if [[ "$PYTHON" =~ "python2" ]]; then + echo "python2 is not supported." >&2 + exit 1 +fi + PYTHON=${PYTHON:-"python3"} -VENV=${VENV:-"virtualenv"} +VENV=${VENV:-"./virtualenv"} case "$(uname -s)" in Linux) case "$(lsb_release --id --short)" in Ubuntu|Debian|LinuxMint) - # Ensure setuptools is installed - if [[ "$PYTHON" =~ "python2" ]]; then - deps=(qemu-utils python-dev libssl-dev python-pip python-virtualenv libev-dev libvirt-dev libffi-dev libyaml-dev) - else - deps=(qemu-utils python3-dev libssl-dev python3-pip python3-virtualenv libev-dev libvirt-dev libffi-dev libyaml-dev) - fi + deps=(qemu-utils python3-dev libssl-dev python3-pip python3-venv libev-dev libvirt-dev libffi-dev libyaml-dev) for package in ${deps[@]}; do if [ "$(dpkg --status -- $package|sed -n 's/^Status: //p')" != "install ok installed" ]; then # add a space after old values @@ -44,11 +44,7 @@ Linux) fi ;; RedHatEnterpriseWorkstation|RedHatEnterpriseServer|RedHatEnterprise|CentOS) - if [[ "$PYTHON" =~ "python2" ]]; then - deps=(python-pip python-devel python-virtualenv mariadb-devel libev-devel libvirt-devel libffi-devel) - else - deps=(python3-pip python3-devel python3-virtualenv mariadb-devel libev-devel libvirt-devel libffi-devel) - fi + deps=(python3-pip python3-devel mariadb-devel libev-devel libvirt-devel libffi-devel) for package in ${deps[@]}; do if [ "$(rpm -q $package)" == "package $package is not installed" ]; then missing="${missing:+$missing }$package" @@ -68,11 +64,7 @@ Linux) fi ;; Fedora) - if [[ "$PYTHON" =~ "python2" ]]; then - deps=(python-pip python-devel python-virtualenv libev-devel libvirt-devel libffi-devel) - else - deps=(python3-pip python3-devel python3-virtualenv libev-devel libvirt-devel libffi-devel) - fi + deps=(python3-pip python3-devel libev-devel libvirt-devel libffi-devel) for package in ${deps[@]}; do if [ "$(rpm -q $package)" == "package $package is not installed" ]; then missing="${missing:+$missing }$package" @@ -97,12 +89,7 @@ Linux) fi ;; "openSUSE project"|"SUSE LINUX"|"openSUSE") - - if [[ "$PYTHON" =~ "python2" ]]; then - deps=(python-pip python-devel python-virtualenv libev-devel libvirt-devel libffi-devel) - else - deps=(python3-pip python3-devel python3-virtualenv libev-devel libvirt-devel libffi-devel) - fi + deps=(python3-pip python3-devel python3 libev-devel libvirt-devel libffi-devel) for package in ${deps[@]}; do if [ "$(rpm -q $package)" == "package $package is not installed" ]; then if [ "$(rpm -q --whatprovides $package)" == "no package provides $package" ]; then @@ -148,35 +135,22 @@ Darwin) ;; esac -# Forcibly remove old virtualenvs which used system site-packages -if [ -e ./$VENV ] && [ ! -e ./$VENV/lib/python*/no-global-site-packages.txt ]; then - echo "Removing old virtualenv because it uses system site-packages" - rm -rf ./$VENV +# If the venv was set to use system site-packages, fix that +if [ -d "$VENV" ]; then + sed -i'' -e 's/\(include-system-site-packages\s*=\s*\)true/\1false/g' $VENV/pyvenv.cfg fi export LC_ALL=en_US.UTF-8 -if [ -z "$NO_CLOBBER" ] || [ ! -e ./$VENV ]; then - if ! virtualenv --version &>/dev/null; then - pip install virtualenv - fi - virtualenv --python=$PYTHON $VENV +if [ -z "$NO_CLOBBER" ] || [ ! -e $VENV ]; then + python3 -m venv $VENV fi -# Upgrade pip first - then we have a new pip version with the --use-feature flag -./$VENV/bin/pip install --upgrade pip +# First, upgrade pip +$VENV/bin/pip install --upgrade pip -# Ensure setuptools is installed -./$VENV/bin/pip install --use-feature=2020-resolver setuptools --upgrade - -# Install all requirements -./$VENV/bin/pip install --use-feature=2020-resolver --upgrade -r requirements.txt +# By default, install teuthology in editable mode +$VENV/bin/pip install ${PIP_INSTALL_FLAGS:---editable '.[test]'} # Check to make sure requirements are met -./$VENV/bin/pip check - -# Remove leftover .pyc files -find teuthology -name '*.pyc' -exec rm {} \; - -# Install teuthology -./$VENV/bin/python setup.py develop +$VENV/bin/pip check diff --git a/docs/docker-compose/teuthology/Dockerfile b/docs/docker-compose/teuthology/Dockerfile index 5c9285f5f..149e8aca5 100644 --- a/docs/docker-compose/teuthology/Dockerfile +++ b/docs/docker-compose/teuthology/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update && \ libssl-dev \ ipmitool \ python3-pip \ - python3-virtualenv \ + python3-venv \ vim \ libev-dev \ libvirt-dev \ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..ece6fe5f5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +build-backend = "setuptools.build_meta" +requires = [ + "setuptools>=45", + "wheel", + "setuptools_scm>=6.2", +] + +[tool.setuptools_scm] +version_scheme = "python-simplified-semver" \ No newline at end of file diff --git a/requirements.in b/requirements.in deleted file mode 100644 index dd5bce9e3..000000000 --- a/requirements.in +++ /dev/null @@ -1,44 +0,0 @@ -# install requires -apache-libcloud -gevent -PyYAML -argparse>=1.2.1 -configobj -six>=1.9 -pexpect -docopt -netaddr -paramiko -pynacl >= 1.5.0 # for universal2 wheel -psutil >= 2.1.0 -configparser -ansible>=2.8,<2.10 -prettytable -manhole -humanfriendly -# orchestra requires -backports.ssl-match-hostname -beanstalkc3>=0.4.0 -httplib2 -ndg-httpsclient -pyasn1 -pyopenssl>=0.13 -python-dateutil -python-novaclient -python-openstackclient -requests!=2.13.0 -sentry-sdk -# test requires -boto>=2.0b4 -boto3 -cryptography>=2.7 -nose -pip-tools -pytest -mock -requests -tox -xmltodict -PyJWT -ipy -toml diff --git a/requirements.txt b/requirements.txt index 8276a9ceb..1b175b701 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,32 +2,28 @@ # This file is autogenerated by pip-compile with python 3.9 # To update, run: # -# pip-compile requirements.in +# pip-compile pyproject.toml # ansible==2.9.27 - # via -r requirements.in + # via teuthology (pyproject.toml) apache-libcloud==3.3.1 - # via -r requirements.in + # via teuthology (pyproject.toml) appdirs==1.4.4 # via openstacksdk argparse==1.4.0 - # via -r requirements.in + # via teuthology (pyproject.toml) attrs==21.2.0 - # via - # cmd2 - # pytest -backports-entry-points-selectable==1.1.0 - # via virtualenv + # via cmd2 backports-ssl-match-hostname==3.7.0.1 - # via -r requirements.in + # via teuthology (pyproject.toml) bcrypt==3.2.0 # via paramiko beanstalkc3==0.4.0 - # via -r requirements.in + # via teuthology (pyproject.toml) boto==2.49.0 - # via -r requirements.in + # via teuthology (pyproject.toml) boto3==1.21.20 - # via -r requirements.in + # via teuthology (pyproject.toml) botocore==1.24.20 # via # boto3 @@ -54,16 +50,16 @@ cmd2==2.1.2 colorama==0.4.4 # via cmd2 configobj==5.0.6 - # via -r requirements.in + # via teuthology (pyproject.toml) configparser==5.0.2 - # via -r requirements.in + # via teuthology (pyproject.toml) cryptography==3.4.7 # via - # -r requirements.in # ansible # openstacksdk # paramiko # pyopenssl + # teuthology (pyproject.toml) debtcollector==2.2.0 # via # oslo-config @@ -73,30 +69,22 @@ decorator==5.0.9 # via # dogpile-cache # openstacksdk -distlib==0.3.2 - # via virtualenv docopt==0.6.2 - # via -r requirements.in + # via teuthology (pyproject.toml) dogpile-cache==1.1.3 # via openstacksdk -filelock==3.0.12 - # via - # tox - # virtualenv gevent==21.8.0 - # via -r requirements.in + # via teuthology (pyproject.toml) greenlet==1.1.0 # via gevent httplib2==0.19.1 - # via -r requirements.in + # via teuthology (pyproject.toml) humanfriendly==9.2 - # via -r requirements.in + # via teuthology (pyproject.toml) idna==3.2 # via requests -iniconfig==1.1.1 - # via pytest ipy==1.1 - # via -r requirements.in + # via teuthology (pyproject.toml) iso8601==0.1.16 # via # keystoneauth1 @@ -122,29 +110,23 @@ keystoneauth1==4.3.1 # python-cinderclient # python-keystoneclient # python-novaclient -manhole==1.8.0 - # via -r requirements.in markupsafe==2.0.1 # via jinja2 -mock==4.0.3 - # via -r requirements.in msgpack==1.0.2 # via oslo-serialization munch==2.5.0 # via openstacksdk ndg-httpsclient==0.5.1 - # via -r requirements.in + # via teuthology (pyproject.toml) netaddr==0.8.0 # via - # -r requirements.in # oslo-config # oslo-utils + # teuthology (pyproject.toml) netifaces==0.11.0 # via # openstacksdk # oslo-utils -nose==1.3.7 - # via -r requirements.in openstacksdk==0.58.0 # via # osc-lib @@ -179,12 +161,9 @@ oslo-utils==4.9.2 # python-novaclient # python-openstackclient packaging==21.0 - # via - # oslo-utils - # pytest - # tox + # via oslo-utils paramiko==2.10.1 - # via -r requirements.in + # via teuthology (pyproject.toml) pbr==5.6.0 # via # cliff @@ -204,45 +183,33 @@ pbr==5.6.0 pep517==0.11.0 # via pip-tools pexpect==4.8.0 - # via -r requirements.in + # via teuthology (pyproject.toml) pip-tools==6.2.0 - # via -r requirements.in -platformdirs==2.2.0 - # via virtualenv -pluggy==0.13.1 - # via - # pytest - # tox + # via teuthology (pyproject.toml) prettytable==2.1.0 # via - # -r requirements.in # cliff # python-cinderclient # python-novaclient + # teuthology (pyproject.toml) psutil==5.8.0 - # via -r requirements.in + # via teuthology (pyproject.toml) ptyprocess==0.7.0 # via pexpect -py==1.10.0 - # via - # pytest - # tox pyasn1==0.4.8 # via - # -r requirements.in # ndg-httpsclient + # teuthology (pyproject.toml) pycparser==2.20 # via cffi -pyjwt==2.1.0 - # via -r requirements.in pynacl==1.5.0 # via - # -r requirements.in # paramiko + # teuthology (pyproject.toml) pyopenssl==20.0.1 # via - # -r requirements.in # ndg-httpsclient + # teuthology (pyproject.toml) pyparsing==2.4.7 # via # cliff @@ -251,41 +218,39 @@ pyparsing==2.4.7 # packaging pyperclip==1.8.2 # via cmd2 -pytest==6.2.5 - # via -r requirements.in python-cinderclient==8.0.0 # via python-openstackclient python-dateutil==2.8.2 # via - # -r requirements.in # botocore + # teuthology (pyproject.toml) python-keystoneclient==4.2.0 # via python-openstackclient python-novaclient==17.5.0 # via - # -r requirements.in # python-openstackclient + # teuthology (pyproject.toml) python-openstackclient==5.5.0 - # via -r requirements.in + # via teuthology (pyproject.toml) pytz==2021.1 # via # oslo-serialization # oslo-utils pyyaml==5.4.1 # via - # -r requirements.in # ansible # cliff # openstacksdk # oslo-config + # teuthology (pyproject.toml) requests==2.27.1 # via - # -r requirements.in # apache-libcloud # keystoneauth1 # oslo-config # python-cinderclient # python-keystoneclient + # teuthology (pyproject.toml) requestsexceptions==1.4.0 # via openstacksdk rfc3986==1.5.0 @@ -293,26 +258,23 @@ rfc3986==1.5.0 s3transfer==0.5.2 # via boto3 sentry-sdk==1.3.1 - # via -r requirements.in + # via teuthology (pyproject.toml) simplejson==3.17.3 # via # osc-lib # python-cinderclient six==1.16.0 # via - # -r requirements.in # bcrypt # configobj # debtcollector # keystoneauth1 # munch # oslo-i18n - # paramiko # pyopenssl # python-dateutil # python-keystoneclient - # tox - # virtualenv + # teuthology (pyproject.toml) stevedore==3.3.0 # via # cliff @@ -325,21 +287,14 @@ stevedore==3.3.0 # python-novaclient # python-openstackclient toml==0.10.2 - # via - # -r requirements.in - # pytest - # tox + # via teuthology (pyproject.toml) tomli==1.2.1 # via pep517 -tox==3.24.1 - # via -r requirements.in urllib3==1.26.6 # via # botocore # requests # sentry-sdk -virtualenv==20.7.0 - # via tox wcwidth==0.2.5 # via # cmd2 @@ -349,7 +304,7 @@ wheel==0.36.2 wrapt==1.12.1 # via debtcollector xmltodict==0.12.0 - # via -r requirements.in + # via teuthology (pyproject.toml) zope-event==4.5.0 # via gevent zope-interface==5.4.0 diff --git a/scripts/dispatcher.py b/scripts/dispatcher.py index 4cb1abdea..15df103fd 100644 --- a/scripts/dispatcher.py +++ b/scripts/dispatcher.py @@ -1,6 +1,6 @@ """ usage: teuthology-dispatcher --help - teuthology-dispatcher --supervisor [-v] --bin-path BIN_PATH --job-config COFNFIG --archive-dir DIR + teuthology-dispatcher --supervisor [-v] --job-config COFNFIG --archive-dir DIR teuthology-dispatcher [-v] [--archive-dir DIR] [--exit-on-empty-queue] --log-dir LOG_DIR --tube TUBE Start a dispatcher for the specified tube. Grab jobs from a beanstalk @@ -19,7 +19,6 @@ standard arguments: -l, --log-dir LOG_DIR path in which to store logs -a DIR, --archive-dir DIR path to archive results in --supervisor run dispatcher in job supervisor mode - --bin-path BIN_PATH teuthology bin path --job-config CONFIG file descriptor of job's config file --exit-on-empty-queue if the queue is empty, exit """ diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..4bfdfab5d --- /dev/null +++ b/setup.cfg @@ -0,0 +1,133 @@ +[metadata] +name = teuthology +long_description = file: README.rst +long_description_content_type = text/x-rst +url = https://github.com/ceph/teuthology +author = Red Hat, Inc. +license = MIT +license_file = LICENSE +classifiers = + Intended Audience :: Developers + License :: OSI Approved :: MIT License + Natural Language :: English + Operating System :: POSIX :: Linux + Programming Language :: Python :: 3 + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: Implementation :: CPython + Topic :: Software Development :: Quality Assurance + Topic :: Software Development :: Testing + Topic :: System :: Distributed Computing + Topic :: System :: Filesystems +description_content_type = text/x-rst; charset=UTF-8 +description_file = README.rst +keywords = teuthology, test, ceph, cluster +summary = Ceph test framework + +[options] +packages = find: +install_requires = + PyYAML + ansible>=2.8,<2.10 + apache-libcloud + argparse>=1.2.1 + backports.ssl-match-hostname + beanstalkc3>=0.4.0 + boto>=2.0b4 + boto3 + configobj + configparser + cryptography>=2.7 + docopt + gevent + httplib2 + humanfriendly + ipy + ndg-httpsclient + netaddr + paramiko + pexpect + pip-tools + prettytable + psutil>=2.1.0 + pyasn1 + pynacl>=1.5.0 + pyopenssl>=0.13 + python-dateutil + python-novaclient + python-openstackclient + requests>2.13.0 + sentry-sdk + six>=1.9 + toml + xmltodict +python_requires = >=3.6 + +[options.entry_points] +console_scripts = + teuthology = scripts.run:main + teuthology-openstack = scripts.openstack:main + teuthology-nuke = scripts.nuke:main + teuthology-suite = scripts.suite:main + teuthology-ls = scripts.ls:main + teuthology-worker = scripts.worker:main + teuthology-lock = scripts.lock:main + teuthology-schedule = scripts.schedule:main + teuthology-updatekeys = scripts.updatekeys:main + teuthology-update-inventory = scripts.update_inventory:main + teuthology-results = scripts.results:main + teuthology-report = scripts.report:main + teuthology-kill = scripts.kill:main + teuthology-queue = scripts.queue:main + teuthology-prune-logs = scripts.prune_logs:main + teuthology-describe = scripts.describe:main + teuthology-reimage = scripts.reimage:main + teuthology-dispatcher = scripts.dispatcher:main + teuthology-wait = scripts.wait:main + +[options.extras_require] +manhole = + manhole +rocketchat = + rocket-python>=1.2.15 +sentry = + sentry-sdk +test = + PyJWT + boto>=2.0b4 + boto3 + cryptography>=2.7 + ipy + mock + pip-tools + pytest + toml + tox + xmltodict + +[options.package_data] +teuthology.openstack = + archive-key + archive-key.pub + openstack-centos-6.5-user-data.txt + openstack-centos-7.0-user-data.txt + openstack-centos-7.1-user-data.txt + openstack-centos-7.2-user-data.txt + openstack-debian-8.0-user-data.txt + openstack-opensuse-42.1-user-data.txt + openstack-teuthology.cron + openstack-teuthology.init + openstack-ubuntu-12.04-user-data.txt + openstack-ubuntu-14.04-user-data.txt + openstack-user-data.txt + openstack.yaml + setup-openstack.sh +teuthology.task.install = + adjust-ulimits + daemon-helper +teuthology.task.internal = + edit_sudoers.sh diff --git a/setup.py b/setup.py deleted file mode 100644 index 87c573a4e..000000000 --- a/setup.py +++ /dev/null @@ -1,138 +0,0 @@ -from setuptools import setup, find_packages -import re - -module_file = open("teuthology/__init__.py").read() -metadata = dict(re.findall(r"__([a-z]+)__\s*=\s*['\"]([^'\"]*)['\"]", module_file)) -long_description = open('README.rst').read() - -setup( - name='teuthology', - version=metadata['version'], - packages=find_packages(), - package_data={ - 'teuthology.task': ['adjust-ulimits', 'edit_sudoers.sh', 'daemon-helper'], - 'teuthology.task': ['adjust-ulimits', 'edit_sudoers.sh', 'daemon-helper'], - 'teuthology.openstack': [ - 'archive-key', - 'archive-key.pub', - 'openstack-centos-6.5-user-data.txt', - 'openstack-centos-7.0-user-data.txt', - 'openstack-centos-7.1-user-data.txt', - 'openstack-centos-7.2-user-data.txt', - 'openstack-debian-8.0-user-data.txt', - 'openstack-opensuse-42.1-user-data.txt', - 'openstack-teuthology.cron', - 'openstack-teuthology.init', - 'openstack-ubuntu-12.04-user-data.txt', - 'openstack-ubuntu-14.04-user-data.txt', - 'openstack-user-data.txt', - 'openstack.yaml', - 'setup-openstack.sh' - ], - }, - author='Inktank Storage, Inc.', - author_email='ceph-qa@ceph.com', - description='Ceph test framework', - license='MIT', - keywords='teuthology test ceph cluster', - url='https://github.com/ceph/teuthology', - long_description=long_description, - classifiers=[ - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Natural Language :: English', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python :: 3.6', - 'Topic :: Software Development :: Quality Assurance', - 'Topic :: Software Development :: Testing', - 'Topic :: System :: Distributed Computing', - 'Topic :: System :: Filesystems', - ], - install_requires=['apache-libcloud', - 'gevent', - 'PyYAML', - 'argparse >= 1.2.1', - 'configobj', - 'six >= 1.9', # python-openstackclient won't work properly with less - 'pexpect', - 'docopt', - 'netaddr', # teuthology/misc.py - # only used by orchestra, but we monkey-patch it in - # teuthology/__init__.py - 'paramiko', - 'psutil >= 2.1.0', - 'configparser', - 'ansible>=2.0', - 'prettytable', - 'rocket-python >= 1.2.15', - 'manhole', - 'humanfriendly', - ], - extras_require = { - 'orchestra': [ - # For apache-libcloud when using python < 2.7.9 - 'backports.ssl_match_hostname', - 'beanstalkc3 >= 0.4.0', - 'httplib2', - 'ndg-httpsclient', # for requests, urllib3 - 'pyasn1', # for requests, urllib3 - 'pyopenssl>=0.13', # for requests, urllib3 - 'python-dateutil', - # python-novaclient is specified here, even though it is - # redundant, because python-openstackclient requires - # Babel, and installs 2.3.3, which is forbidden by - # python-novaclient 4.0.0 - 'python-novaclient', - 'python-openstackclient', - # with openstacklient >= 2.1.0, neutronclient no longer is - # a dependency but we need it anyway. - 'python-neutronclient', - 'requests != 2.13.0', - 'sentry-sdk', - ], - 'test': [ - 'boto >= 2.0b4', # for qa/tasks/radosgw_*.py - 'cryptography >= 2.7', # for qa/tasks/mgr/dashboard/test_rgw.py - 'nose', # for qa/tasks/rgw_multisite_tests.py', - 'pip-tools', - 'pytest', # for tox.ini - 'requests', # for qa/tasks/mgr/dashboard/helper.py - 'tox', - # For bucket notification testing in multisite - 'xmltodict', - 'boto3', - 'PyJWT', # for qa/tasks/mgr/dashboard/test_auth.py - 'ipy', # for qa/tasks/cephfs/mount.py - 'toml', # for qa/tasks/cephadm.py - ] - }, - - - # to find the code associated with entry point - # A.B:foo first cd into directory A, open file B - # and find sub foo - entry_points={ - 'console_scripts': [ - 'teuthology = scripts.run:main', - 'teuthology-openstack = scripts.openstack:main', - 'teuthology-nuke = scripts.nuke:main', - 'teuthology-suite = scripts.suite:main', - 'teuthology-ls = scripts.ls:main', - 'teuthology-worker = scripts.worker:main', - 'teuthology-lock = scripts.lock:main', - 'teuthology-schedule = scripts.schedule:main', - 'teuthology-updatekeys = scripts.updatekeys:main', - 'teuthology-update-inventory = scripts.update_inventory:main', - 'teuthology-results = scripts.results:main', - 'teuthology-report = scripts.report:main', - 'teuthology-kill = scripts.kill:main', - 'teuthology-queue = scripts.queue:main', - 'teuthology-prune-logs = scripts.prune_logs:main', - 'teuthology-describe = scripts.describe:main', - 'teuthology-reimage = scripts.reimage:main', - 'teuthology-dispatcher = scripts.dispatcher:main', - 'teuthology-wait = scripts.wait:main', - ], - }, - - ) diff --git a/teuthology/__init__.py b/teuthology/__init__.py index de15f251a..4781f59bc 100644 --- a/teuthology/__init__.py +++ b/teuthology/__init__.py @@ -1,5 +1,11 @@ from __future__ import print_function import os +try: + import importlib.metadata as importlib_metadata +except ImportError: + import importlib_metadata + +__version__ = importlib_metadata.version("teuthology") # Tell gevent not to patch os.waitpid() since it is susceptible to race # conditions. See: @@ -8,12 +14,15 @@ os.environ['GEVENT_NOWAITPID'] = 'true' # Use manhole to give us a way to debug hung processes # https://pypi.python.org/pypi/manhole -import manhole -manhole.install( - verbose=False, - # Listen for SIGUSR1 - oneshot_on="USR1" -) +try: + import manhole + manhole.install( + verbose=False, + # Listen for SIGUSR1 + oneshot_on="USR1" + ) +except ImportError: + pass from gevent import monkey monkey.patch_all( dns=False, @@ -30,25 +39,6 @@ from teuthology.orchestra import monkey monkey.patch_all() import logging -import subprocess - -__version__ = '1.1.0' - -# do our best, but if it fails, continue with above - -try: - teuthology_dir = os.path.dirname(os.path.realpath(__file__)) - site_dir = os.path.dirname(teuthology_dir) - git_dir = os.path.join(site_dir, '.git') - # make sure we use git repo otherwise it is a released version - if os.path.exists(git_dir): - __version__ += '-' + str(subprocess.check_output( - 'git rev-parse --short HEAD'.split(), - cwd=site_dir - ).decode()).strip() -except Exception as e: - # before logging; should be unusual - print("Can't get version from git rev-parse %s" % e, file=sys.stderr) # If we are running inside a virtualenv, ensure we have its 'bin' directory in # our PATH. This doesn't happen automatically if scripts are called without diff --git a/teuthology/dispatcher/__init__.py b/teuthology/dispatcher/__init__.py index 9f7fa257e..9e513a72e 100644 --- a/teuthology/dispatcher/__init__.py +++ b/teuthology/dispatcher/__init__.py @@ -124,7 +124,7 @@ def main(args): keep_running = False try: - job_config, teuth_bin_path = prep_job( + job_config = prep_job( job_config, log_file_path, archive_dir, @@ -137,10 +137,9 @@ def main(args): job_config = lock_machines(job_config) run_args = [ - os.path.join(teuth_bin_path, 'teuthology-dispatcher'), + 'teuthology-dispatcher', '--supervisor', '-v', - '--bin-path', teuth_bin_path, '--archive-dir', archive_dir, ] diff --git a/teuthology/dispatcher/supervisor.py b/teuthology/dispatcher/supervisor.py index f52e37938..fb7cdf2ff 100644 --- a/teuthology/dispatcher/supervisor.py +++ b/teuthology/dispatcher/supervisor.py @@ -31,7 +31,6 @@ def main(args): verbose = args["--verbose"] archive_dir = args["--archive-dir"] - teuth_bin_path = args["--bin-path"] config_file_path = args["--job-config"] with open(config_file_path, 'r') as config_file: @@ -56,7 +55,6 @@ def main(args): try: return run_job( job_config, - teuth_bin_path, archive_dir, verbose ) @@ -64,12 +62,12 @@ def main(args): return 0 -def run_job(job_config, teuth_bin_path, archive_dir, verbose): +def run_job(job_config, archive_dir, verbose): safe_archive = safepath.munge(job_config['name']) if job_config.get('first_in_suite') or job_config.get('last_in_suite'): job_archive = os.path.join(archive_dir, safe_archive) args = [ - os.path.join(teuth_bin_path, 'teuthology-results'), + 'teuthology-results', '--archive-dir', job_archive, '--name', job_config['name'], ] @@ -100,9 +98,7 @@ def run_job(job_config, teuth_bin_path, archive_dir, verbose): log.info('Running job %s', job_config['job_id']) - arg = [ - os.path.join(teuth_bin_path, 'teuthology'), - ] + arg = ['teuthology'] # The following is for compatibility with older schedulers, from before we # started merging the contents of job_config['config'] into job_config # itself. diff --git a/teuthology/provision/cloud/test/test_openstack.py b/teuthology/provision/cloud/test/test_openstack.py index 08f5b9c08..108532ed5 100644 --- a/teuthology/provision/cloud/test/test_openstack.py +++ b/teuthology/provision/cloud/test/test_openstack.py @@ -76,8 +76,8 @@ def get_fake_obj(mock_args=None, attributes=None): class TestOpenStackBase(object): - def setup(self, conf=dict(), test_config=test_config): - config.load(conf or dict(libcloud=deepcopy(test_config))) + def setup(self): + config.load(dict(libcloud=deepcopy(test_config))) self.start_patchers() def start_patchers(self): diff --git a/teuthology/run_tasks.py b/teuthology/run_tasks.py index 602559f7f..598947c80 100644 --- a/teuthology/run_tasks.py +++ b/teuthology/run_tasks.py @@ -1,3 +1,4 @@ +import importlib import jinja2 import logging import os @@ -60,6 +61,17 @@ def _import(from_package, module_name, task_name, fail_on_import_error=False): if fail_on_import_error: raise else: + if ( + importlib.util.find_spec(from_package) is not None and + importlib.util.find_spec(full_module_name) is not None + ): + # If we get here, it means we could _find_ both the module and + # the package that contains it, but still got an ImportError. + # Typically that means the module failed to import because it + # could not find one of its dependencies; if we don't raise + # here it will look like we just could't find the module, + # making the dependency issue difficult to discover. + raise return None return module diff --git a/teuthology/task/tests/__init__.py b/teuthology/task/tests/__init__.py index c066f6660..43c6c1169 100644 --- a/teuthology/task/tests/__init__.py +++ b/teuthology/task/tests/__init__.py @@ -44,10 +44,12 @@ class TeuthologyContextPlugin(object): metafunc.parametrize(["ctx", "config"], [(self.ctx, self.config),]) # log the outcome of each test - def pytest_runtest_makereport(self, __multicall__, item, call): - report = __multicall__.execute() + @pytest.hookimpl(hookwrapper=True) + def pytest_runtest_makereport(self, item, call): + outcome = yield + report = outcome.get_result() - # after the test has been called, get it's report and log it + # after the test has been called, get its report and log it if call.when == 'call': # item.location[0] is a slash delimeted path to the test file # being ran. We only want the portion after teuthology.task.tests diff --git a/teuthology/test/test_worker.py b/teuthology/test/test_worker.py index ae92e2441..e32cdf6b8 100644 --- a/teuthology/test/test_worker.py +++ b/teuthology/test/test_worker.py @@ -85,10 +85,10 @@ class TestWorker(object): m_p.returncode = 0 m_popen.return_value = m_p m_t_config.results_server = True - worker.run_job(config, "teuth/bin/path", "archive/dir", verbose=False) + worker.run_job(config, "archive/dir", verbose=False) m_run_watchdog.assert_called_with(m_p, config) expected_args = [ - 'teuth/bin/path/teuthology', + 'teuthology', '-v', '--lock', '--block', @@ -135,7 +135,7 @@ class TestWorker(object): m_p.returncode = 1 m_popen.return_value = m_p m_t_config.results_server = False - worker.run_job(config, "teuth/bin/path", "archive/dir", verbose=False) + worker.run_job(config, "archive/dir", verbose=False) m_symlink_log.assert_called_with(config["worker_log"], config["archive_path"]) @patch("teuthology.worker.report.try_push_job_info") @@ -195,7 +195,7 @@ class TestWorker(object): m_fetch_qa_suite.return_value = '/suite/path' m_isdir.return_value = True m_teuth_config.teuthology_path = None - got_config, teuth_bin_path = worker.prep_job( + got_config = worker.prep_job( config, log_file_path, archive_dir, @@ -208,7 +208,6 @@ class TestWorker(object): ) assert got_config['teuthology_branch'] == 'master' assert m_fetch_teuthology.called_once_with_args(branch='master') - assert teuth_bin_path == '/teuth/path/virtualenv/bin' assert m_fetch_qa_suite.called_once_with_args(branch='master') assert got_config['suite_path'] == '/suite/path' @@ -255,7 +254,7 @@ class TestWorker(object): ) m_connection.reserve.side_effect = jobs m_connect.return_value = m_connection - m_prep_job.return_value = (dict(), '/bin/path') + m_prep_job.return_value = dict() worker.main(self.ctx) # There should be one reserve call per item in the jobs list expected_reserve_calls = [ diff --git a/teuthology/worker.py b/teuthology/worker.py index be5d6c650..70cb72f62 100644 --- a/teuthology/worker.py +++ b/teuthology/worker.py @@ -114,14 +114,13 @@ def main(ctx): keep_running = False try: - job_config, teuth_bin_path = prep_job( + job_config = prep_job( job_config, log_file_path, ctx.archive_dir, ) run_job( job_config, - teuth_bin_path, ctx.archive_dir, ctx.verbose, ) @@ -163,11 +162,6 @@ def prep_job(job_config, log_file_path, archive_dir): log.info('Using teuthology sha1 %s', teuthology_sha1) try: - if teuth_config.teuthology_path is not None: - teuth_path = teuth_config.teuthology_path - else: - teuth_path = fetch_teuthology(branch=teuthology_branch, - commit=teuthology_sha1) # For the teuthology tasks, we look for suite_branch, and if we # don't get that, we look for branch, and fall back to 'master'. # last-in-suite jobs don't have suite_branch or branch set. @@ -196,14 +190,10 @@ def prep_job(job_config, log_file_path, archive_dir): ) raise SkipJob() - teuth_bin_path = os.path.join(teuth_path, 'virtualenv', 'bin') - if not os.path.isdir(teuth_bin_path): - raise RuntimeError("teuthology branch %s at %s not bootstrapped!" % - (teuthology_branch, teuth_bin_path)) - return job_config, teuth_bin_path + return job_config -def run_job(job_config, teuth_bin_path, archive_dir, verbose): +def run_job(job_config, archive_dir, verbose): safe_archive = safepath.munge(job_config['name']) if job_config.get('first_in_suite') or job_config.get('last_in_suite'): if teuth_config.results_server: @@ -215,7 +205,7 @@ def run_job(job_config, teuth_bin_path, archive_dir, verbose): suite_archive_dir = os.path.join(archive_dir, safe_archive) safepath.makedirs('/', suite_archive_dir) args = [ - os.path.join(teuth_bin_path, 'teuthology-results'), + 'teuthology-results', '--archive-dir', suite_archive_dir, '--name', job_config['name'], ] @@ -244,9 +234,7 @@ def run_job(job_config, teuth_bin_path, archive_dir, verbose): log.info('Running job %s', job_config['job_id']) suite_path = job_config['suite_path'] - arg = [ - os.path.join(teuth_bin_path, 'teuthology'), - ] + arg = ['teuthology'] # The following is for compatibility with older schedulers, from before we # started merging the contents of job_config['config'] into job_config # itself. diff --git a/tox.ini b/tox.ini index 3e77fd401..9193865d0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,6 @@ [tox] envlist = docs, py3, flake8, openstack +isolated_build = True [testenv] setenv = @@ -7,26 +8,25 @@ setenv = LANG=en_US [testenv:py3] -basepython=python3 install_command = pip install --upgrade {opts} {packages} passenv = HOME deps= -r{toxinidir}/requirements.txt pytest-cov==2.8.1 coverage==4.5.4 + mock==4.0.3 +extras = test log_format = %(asctime)s %(levelname)s %(message)s commands= python -m pytest --cov=teuthology --cov-report=term -v {posargs:teuthology scripts} [testenv:flake8] -basepython=python3 install_command = pip install --upgrade {opts} {packages} deps= flake8 commands=flake8 --select=F,E9 {posargs:teuthology scripts} [testenv:docs] -basepython=python3 install_command = pip install --upgrade {opts} {packages} changedir=docs deps= @@ -40,27 +40,22 @@ commands= [testenv:openstack] install_command = pip install --upgrade {opts} {packages} passenv = HOME OS_REGION_NAME OS_AUTH_URL OS_TENANT_ID OS_TENANT_NAME OS_PASSWORD OS_USERNAME -sitepackages=True deps= -r{toxinidir}/requirements.txt - +extras = test commands=py.test -v {posargs:teuthology/openstack/test/test_openstack.py} -basepython=python3 [testenv:openstack-integration] passenv = HOME OS_REGION_NAME OS_AUTH_URL OS_TENANT_ID OS_TENANT_NAME OS_PASSWORD OS_USERNAME -basepython=python3 deps= -r{toxinidir}/requirements.txt - +extras = test commands= py.test -v {posargs} teuthology/openstack/test/openstack-integration.py [testenv:openstack-delegate] passenv = HOME OS_REGION_NAME OS_AUTH_URL OS_TENANT_ID OS_TENANT_NAME OS_PASSWORD OS_USERNAME -basepython=python3 sitepackages=True deps= -r{toxinidir}/requirements.txt - commands={toxinidir}/openstack-delegate.sh diff --git a/update-requirements.sh b/update-requirements.sh index 133453844..8040c0d1e 100755 --- a/update-requirements.sh +++ b/update-requirements.sh @@ -1,4 +1,3 @@ #!/bin/bash -pip-compile $@ requirements.in -sed -i'' -e '/^-e / d' -e 's/-r requirements.in/teuthology/g' requirements.txt \ No newline at end of file +pip-compile $@ pyproject.toml -- 2.47.3