From 61a6eba59bf17f00a58d8a8ab79c0b4b26f7ed4a Mon Sep 17 00:00:00 2001 From: Guillaume Abrioux Date: Wed, 4 Sep 2024 17:24:17 +0200 Subject: [PATCH] ceph-volume: functional testing refactor See corresponding ceph/ceph PR: ceph/ceph#59606 for details. Signed-off-by: Guillaume Abrioux --- ceph-volume-ansible-prs/build/build | 52 ------ ceph-volume-cephadm-prs/build/build | 30 ++++ .../build/teardown | 0 .../config/definitions/ceph-volume-pr.yml | 63 ++----- .../build/build | 0 .../build/teardown | 0 .../config/definitions/ceph-volume-pr.yml | 8 +- .../definitions/ceph-volume-nightly.yml | 62 ++----- ceph-volume-scenario/build/build | 4 +- .../definitions/ceph-volume-scenario.yml | 15 +- .../config/definitions/ceph-volume-test.yml | 154 ++++-------------- 11 files changed, 103 insertions(+), 285 deletions(-) delete mode 100644 ceph-volume-ansible-prs/build/build create mode 100644 ceph-volume-cephadm-prs/build/build rename {ceph-volume-ansible-prs => ceph-volume-cephadm-prs}/build/teardown (100%) rename {ceph-volume-ansible-prs => ceph-volume-cephadm-prs}/config/definitions/ceph-volume-pr.yml (64%) rename {ceph-volume-pr => ceph-volume-linter}/build/build (100%) rename {ceph-volume-pr => ceph-volume-linter}/build/teardown (100%) rename {ceph-volume-pr => ceph-volume-linter}/config/definitions/ceph-volume-pr.yml (93%) diff --git a/ceph-volume-ansible-prs/build/build b/ceph-volume-ansible-prs/build/build deleted file mode 100644 index c4f71ce9..00000000 --- a/ceph-volume-ansible-prs/build/build +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -set -ex -env -WORKDIR=$(mktemp -td tox.XXXXXXXXXX) - -# set up variables needed for -# githubstatus to report back to the github PR -# if this project was started manually -github_status_setup - -# the following two methods exist in scripts/build_utils.sh -pkgs=( "tox" "github-status>0.0.3" ) -TEMPVENV=$(create_venv_dir) -VENV=${TEMPVENV}/bin -set_centos_python3_version "python3.9" -install_python_packages $TEMPVENV "pkgs[@]" "pip==22.0.4" - -GITHUB_STATUS_STATE="pending" $VENV/github-status create - -prune_stale_vagrant_vms $WORKSPACE/../**/tests -delete_libvirt_vms -clear_libvirt_networks -restart_libvirt_services -update_vagrant_boxes - -cd src/ceph-volume/ceph_volume/tests/functional/$SUBCOMMAND - -if [[ "$ghprbTargetBranch" == "luminous" ]]; then - CEPH_ANSIBLE_BRANCH="stable-3.2" -elif [[ "$ghprbTargetBranch" == "mimic" ]]; then - CEPH_ANSIBLE_BRANCH="stable-3.2" -elif [[ "$ghprbTargetBranch" == "nautilus" ]]; then - CEPH_ANSIBLE_BRANCH="stable-4.0" -elif [[ "$ghprbTargetBranch" == "octopus" ]]; then - CEPH_ANSIBLE_BRANCH="stable-5.0" -elif [[ "$ghprbTargetBranch" == "pacific" ]]; then - CEPH_ANSIBLE_BRANCH="stable-6.0" -elif [[ "$ghprbTargetBranch" == "quincy" ]]; then - CEPH_ANSIBLE_BRANCH="stable-7.0" -elif [[ "$ghprbTargetBranch" == "reef" ]]; then - CEPH_ANSIBLE_BRANCH="stable-8.0" -else - CEPH_ANSIBLE_BRANCH="main" -fi - -if [[ "$ghprbTargetBranch" == "nautilus" ]]; then - DISTRO="centos7" -fi - -CEPH_ANSIBLE_BRANCH=$CEPH_ANSIBLE_BRANCH CEPH_DEV_BRANCH=$ghprbTargetBranch $VENV/tox --workdir=$WORKDIR -vre $DISTRO-$OBJECTSTORE-$SCENARIO -- --provider=libvirt - -GITHUB_STATUS_STATE="success" $VENV/github-status create diff --git a/ceph-volume-cephadm-prs/build/build b/ceph-volume-cephadm-prs/build/build new file mode 100644 index 00000000..820c8ed1 --- /dev/null +++ b/ceph-volume-cephadm-prs/build/build @@ -0,0 +1,30 @@ +#!/bin/bash +set -ex +env +WORKDIR=$(mktemp -td tox.XXXXXXXXXX) + +# set up variables needed for +# githubstatus to report back to the github PR +# if this project was started manually +github_status_setup + +# the following two methods exist in scripts/build_utils.sh +pkgs=( "tox" "github-status>0.0.3" ) +TEMPVENV=$(create_venv_dir) +VENV=${TEMPVENV}/bin +set_centos_python3_version "python3.9" +install_python_packages $TEMPVENV "pkgs[@]" "pip==22.0.4" + +GITHUB_STATUS_STATE="pending" $VENV/github-status create + +prune_stale_vagrant_vms $WORKSPACE/../**/tests +delete_libvirt_vms +clear_libvirt_networks +restart_libvirt_services +update_vagrant_boxes + +cd src/ceph-volume/ceph_volume/tests/functional/${DISTRO}/${OBJECTSTORE}/${METHOD}/${SCENARIO} + +CEPH_DEV_BRANCH=$ghprbTargetBranch $VENV/tox --workdir=$WORKDIR -vre ${DISTRO}-${OBJECTSTORE}-${METHOD}-${SCENARIO} -- --provider=libvirt + +GITHUB_STATUS_STATE="success" $VENV/github-status create diff --git a/ceph-volume-ansible-prs/build/teardown b/ceph-volume-cephadm-prs/build/teardown similarity index 100% rename from ceph-volume-ansible-prs/build/teardown rename to ceph-volume-cephadm-prs/build/teardown diff --git a/ceph-volume-ansible-prs/config/definitions/ceph-volume-pr.yml b/ceph-volume-cephadm-prs/config/definitions/ceph-volume-pr.yml similarity index 64% rename from ceph-volume-ansible-prs/config/definitions/ceph-volume-pr.yml rename to ceph-volume-cephadm-prs/config/definitions/ceph-volume-pr.yml index 02daae02..96070d72 100644 --- a/ceph-volume-ansible-prs/config/definitions/ceph-volume-pr.yml +++ b/ceph-volume-cephadm-prs/config/definitions/ceph-volume-pr.yml @@ -1,54 +1,23 @@ - project: - name: ceph-volume-ansible-prs-lvm + name: ceph-volume-cephadm-prs distro: - centos objectstore: - bluestore - scenario: - - create - - dmcrypt - subcommand: + method: - lvm - - jobs: - - 'ceph-volume-prs-{subcommand}-{distro}-{objectstore}-{scenario}' - -- project: - name: ceph-volume-ansible-prs-batch - distro: - - centos - objectstore: - - bluestore - scenario: - - single_type - - single_type_dmcrypt - subcommand: - - batch - - jobs: - - 'ceph-volume-prs-{subcommand}-{distro}-{objectstore}-{scenario}' - -- project: - name: ceph-volume-ansible-prs-batch-mixed - distro: - - centos - objectstore: - - bluestore + - raw scenario: - - mixed_type - - mixed_type_dmcrypt - - mixed_type_explicit - - mixed_type_dmcrypt_explicit - subcommand: - - batch + - unencrypted + - dmcrypt jobs: - - 'ceph-volume-prs-{subcommand}-{distro}-{objectstore}-{scenario}' + - 'ceph-volume-prs-{distro}-{objectstore}-{method}-{scenario}' - job-template: - name: 'ceph-volume-prs-{subcommand}-{distro}-{objectstore}-{scenario}' - display-name: 'ceph-volume {subcommand}: Pull Request [{distro}-{objectstore}-{scenario}]' + name: 'ceph-volume-prs-{distro}-{objectstore}-{method}-{scenario}' + display-name: 'ceph-volume {method}: Pull Request [{distro}-{objectstore}-{scenario}]' node: vagrant&&libvirt&¢os9 concurrent: true project-type: freestyle @@ -91,14 +60,14 @@ org-list: - ceph only-trigger-phrase: true - trigger-phrase: '^jenkins test ceph-volume {subcommand} {distro}-{objectstore}-{scenario}|jenkins test ceph-volume all.*|jenkins test ceph-volume {subcommand} all.*|jenkins test ceph-volume {subcommand} {scenario}.*' + trigger-phrase: '^jenkins test ceph-volume {distro} {objectstore}-{method}-{scenario}|jenkins test ceph-volume all.*|jenkins test ceph-volume {distro} all.*' github-hooks: true permit-all: true auto-close-on-fail: false - status-context: "ceph-volume {subcommand} testing {distro}-{objectstore}-{scenario}" - started-status: "ceph-volume {subcommand} running {distro}-{objectstore}-{scenario}" - success-status: "ceph-volume {subcommand} {distro}-{objectstore}-{scenario} OK" - failure-status: "ceph-volume {subcommand} {distro}-{objectstore}-{scenario} failed" + status-context: "ceph-volume {method} testing {distro}-{objectstore}-{scenario}" + started-status: "ceph-volume {method} running {distro}-{objectstore}-{scenario}" + success-status: "ceph-volume {method} {distro}-{objectstore}-{scenario} OK" + failure-status: "ceph-volume {method} {distro}-{objectstore}-{scenario} failed" scm: - git: @@ -114,12 +83,12 @@ builders: - inject: properties-content: | - SCENARIO={scenario} DISTRO={distro} OBJECTSTORE={objectstore} - SUBCOMMAND={subcommand} + METHOD={method} + SCENARIO={scenario} GITHUB_REPOSITORY="ceph/ceph" - GITHUB_STATUS_CONTEXT="ceph-volume {subcommand} testing {distro}-{objectstore}-{scenario}" + GITHUB_STATUS_CONTEXT="ceph-volume testing {distro}-{objectstore}-{method}-{scenario}" GITHUB_STATUS_STARTED="running" GITHUB_STATUS_SUCCESS="OK" GITHUB_STATUS_FAILURE="failed" diff --git a/ceph-volume-pr/build/build b/ceph-volume-linter/build/build similarity index 100% rename from ceph-volume-pr/build/build rename to ceph-volume-linter/build/build diff --git a/ceph-volume-pr/build/teardown b/ceph-volume-linter/build/teardown similarity index 100% rename from ceph-volume-pr/build/teardown rename to ceph-volume-linter/build/teardown diff --git a/ceph-volume-pr/config/definitions/ceph-volume-pr.yml b/ceph-volume-linter/config/definitions/ceph-volume-pr.yml similarity index 93% rename from ceph-volume-pr/config/definitions/ceph-volume-pr.yml rename to ceph-volume-linter/config/definitions/ceph-volume-pr.yml index 84c80f46..4924fa83 100644 --- a/ceph-volume-pr/config/definitions/ceph-volume-pr.yml +++ b/ceph-volume-linter/config/definitions/ceph-volume-pr.yml @@ -1,6 +1,6 @@ - job: - name: ceph-volume-pr - display-name: 'ceph-volume: Pull Request tox tests' + name: ceph-volume-linter + display-name: 'ceph-volume: Pull Request linter tests' node: small && centos9 project-type: freestyle defaults: global @@ -38,7 +38,7 @@ - github-pull-request: cancel-builds-on-update: true only-trigger-phrase: true - trigger-phrase: 'jenkins test ceph-volume tox' + trigger-phrase: 'jenkins test ceph-volume linter' github-hooks: true permit-all: true auto-close-on-fail: false @@ -62,7 +62,7 @@ - inject: properties-content: | GITHUB_REPOSITORY="ceph/ceph" - GITHUB_STATUS_CONTEXT="ceph-volume tox tests" + GITHUB_STATUS_CONTEXT="ceph-volume linter tests" GITHUB_STATUS_STARTED="running" GITHUB_STATUS_SUCCESS="OK" GITHUB_STATUS_FAILURE="failed" diff --git a/ceph-volume-nightly/config/definitions/ceph-volume-nightly.yml b/ceph-volume-nightly/config/definitions/ceph-volume-nightly.yml index 215f13e2..b092c48f 100644 --- a/ceph-volume-nightly/config/definitions/ceph-volume-nightly.yml +++ b/ceph-volume-nightly/config/definitions/ceph-volume-nightly.yml @@ -4,63 +4,23 @@ - centos objectstore: - bluestore - scenario: - - create - - dmcrypt - subcommand: + method: - lvm - ceph_branch: - - main - - reef - - quincy - - jobs: - - 'ceph-volume-nightly-{ceph_branch}-{subcommand}-{distro}-{objectstore}-{scenario}' - -- project: - name: ceph-volume-nightly-batch - distro: - - centos - objectstore: - - bluestore + - raw scenario: - - single_type - - single_type_dmcrypt - subcommand: - - batch - ceph_branch: - - main - - reef - - quincy - - jobs: - - 'ceph-volume-nightly-{ceph_branch}-{subcommand}-{distro}-{objectstore}-{scenario}' - -- project: - name: ceph-volume-nightly-batch-mixed - distro: - - centos - objectstore: - - bluestore - scenario: - - mixed_type - - mixed_type_dmcrypt - - mixed_type_explicit - - mixed_type_dmcrypt_explicit - subcommand: - - batch + - unencrypted + - dmcrypt ceph_branch: - main + - squid - reef - - quincy jobs: - - 'ceph-volume-nightly-{ceph_branch}-{subcommand}-{distro}-{objectstore}-{scenario}' - + - 'ceph-volume-nightly-{ceph_branch}-{distro}-{objectstore}-{method}-{scenario}' - job-template: - name: 'ceph-volume-nightly-{ceph_branch}-{subcommand}-{distro}-{objectstore}-{scenario}' - display-name: 'ceph-volume {ceph_branch} {subcommand}: [{distro}-{objectstore}-{scenario}]' + name: 'ceph-volume-nightly-{ceph_branch}-{distro}-{objectstore}-{method}-{scenario}' + display-name: 'ceph-volume {ceph_branch}: [{distro}-{objectstore}-{method}-{scenario}]' node: vagrant&&libvirt&¢os9 concurrent: true project-type: freestyle @@ -91,10 +51,10 @@ builders: - inject: properties-content: | - SCENARIO={scenario} DISTRO={distro} OBJECTSTORE={objectstore} - SUBCOMMAND={subcommand} + METHOD={method} + SCENARIO={scenario} CEPH_BRANCH={ceph_branch} - shell: !include-raw-escape: @@ -125,4 +85,4 @@ latest-only: false - email: - recipients: aschoen@redhat.com gabrioux@redhat.com + recipients: gabrioux@ibm.com diff --git a/ceph-volume-scenario/build/build b/ceph-volume-scenario/build/build index 18e464c1..cf43bed1 100644 --- a/ceph-volume-scenario/build/build +++ b/ceph-volume-scenario/build/build @@ -14,6 +14,6 @@ clear_libvirt_networks restart_libvirt_services update_vagrant_boxes -cd src/ceph-volume/ceph_volume/tests/functional/$SUBCOMMAND +cd src/ceph-volume/ceph_volume/tests/functional/${DISTRO}/${OBJECTSTORE}/${METHOD}/${SCENARIO} -CEPH_DEV_BRANCH=$CEPH_BRANCH CEPH_DEV_SHA1=$CEPH_SHA1 CEPH_ANSIBLE_BRANCH=$CEPH_ANSIBLE_BRANCH $VENV/tox --workdir=$WORKDIR -vre $SCENARIO -- --provider=libvirt +CEPH_DEV_BRANCH=$CEPH_BRANCH CEPH_DEV_SHA1=$CEPH_SHA1 $VENV/tox --workdir=$WORKDIR -vre ${DISTRO}-${OBJECTSTORE}-${METHOD}-${OBJECTSTORE}-${SCENARIO} -- --provider=libvirt diff --git a/ceph-volume-scenario/config/definitions/ceph-volume-scenario.yml b/ceph-volume-scenario/config/definitions/ceph-volume-scenario.yml index e88bb9c3..1b610bc0 100644 --- a/ceph-volume-scenario/config/definitions/ceph-volume-scenario.yml +++ b/ceph-volume-scenario/config/definitions/ceph-volume-scenario.yml @@ -18,12 +18,17 @@ parameters: - string: - name: SUBCOMMAND - description: "The subcommand in ceph-volume we are testing. (lvm or simple)" + name: DISTRO + description: "The host OS to use." + default: "centos" + - string: + name: METHOD + description: "The subcommand in ceph-volume we are testing. (lvm or raw)" default: "lvm" - string: name: SCENARIO - description: "A full scenario name for ceph-volume testing, like xenial-bluestore-create" + description: "unencrypted or dmcrypt OSDs" + default: "unencrypted" - string: name: CEPH_BRANCH description: "The ceph branch to test against" @@ -32,10 +37,6 @@ name: CEPH_SHA1 description: "The ceph sha1 to test against" default: "latest" - - string: - name: CEPH_ANSIBLE_BRANCH - description: "The ceph-ansible branch to test against" - default: "main" - string: name: CEPH_REPO_URL description: "The full https url to clone from" diff --git a/ceph-volume-test/config/definitions/ceph-volume-test.yml b/ceph-volume-test/config/definitions/ceph-volume-test.yml index ded366c7..c6973392 100644 --- a/ceph-volume-test/config/definitions/ceph-volume-test.yml +++ b/ceph-volume-test/config/definitions/ceph-volume-test.yml @@ -23,131 +23,41 @@ name: CEPH_SHA1 description: "The ceph sha1 to test against" default: "latest" - - string: - name: CEPH_ANSIBLE_BRANCH - description: "The ceph-ansible branch to test against" - default: "main" - string: name: CEPH_REPO_URL description: "The full https url to clone from" default: "https://github.com/ceph/ceph.git" builders: - # centos8 based jobs for pacific & quincy - - conditional-step: - condition-kind: shell - condition-command: | - #!/bin/bash - set -x - if [[ ! "$CEPH_BRANCH" =~ pacific|quincy ]]; then - exit 1 - fi - on-evaluation-failure: dont-run - steps: - - multijob: - name: 'testing ceph-volume lvm - filestore' - condition: SUCCESSFUL - projects: - - name: ceph-volume-scenario - current-parameters: true - predefined-parameters: | - SCENARIO=centos8-filestore-create - SUBCOMMAND=lvm - - name: ceph-volume-scenario - current-parameters: true - predefined-parameters: | - SCENARIO=centos8-filestore-dmcrypt - SUBCOMMAND=lvm - - multijob: - name: 'testing ceph-volume batch - filestore' - condition: SUCCESSFUL - projects: - - name: ceph-volume-scenario - current-parameters: true - predefined-parameters: | - SCENARIO=centos8-filestore-single_type - SUBCOMMAND=batch - - name: ceph-volume-scenario - current-parameters: true - predefined-parameters: | - SCENARIO=centos8-filestore-single_type_dmcrypt - SUBCOMMAND=batch - - name: ceph-volume-scenario - current-parameters: true - predefined-parameters: | - SCENARIO=centos8-filestore-mixed_type_dmcrypt - SUBCOMMAND=batch - - name: ceph-volume-scenario - current-parameters: true - predefined-parameters: | - SCENARIO=centos8-filestore-mixed_type - SUBCOMMAND=batch - - name: ceph-volume-scenario - current-parameters: true - predefined-parameters: | - SCENARIO=centos8-filestore-mixed_type_dmcrypt_explicit - SUBCOMMAND=batch - - name: ceph-volume-scenario - current-parameters: true - predefined-parameters: | - SCENARIO=centos8-filestore-mixed_type_explicit - SUBCOMMAND=batch - # centos8 based jobs for pacific onward - - conditional-step: - condition-kind: shell - condition-command: | - #!/bin/bash - set -x - if [[ ! "$CEPH_BRANCH" =~ main|squid|reef|quincy|pacific ]]; then - exit 1 - fi - on-evaluation-failure: dont-run - steps: - - multijob: - name: 'testing ceph-volume lvm - bluestore' - condition: SUCCESSFUL - projects: - - name: ceph-volume-scenario - current-parameters: true - predefined-parameters: | - SCENARIO=centos-bluestore-create - SUBCOMMAND=lvm - - name: ceph-volume-scenario - current-parameters: true - predefined-parameters: | - SCENARIO=centos-bluestore-dmcrypt - SUBCOMMAND=lvm - - multijob: - name: 'testing ceph-volume batch - bluestore' - condition: SUCCESSFUL - projects: - - name: ceph-volume-scenario - current-parameters: true - predefined-parameters: | - SCENARIO=centos-bluestore-single_type - SUBCOMMAND=batch - - name: ceph-volume-scenario - current-parameters: true - predefined-parameters: | - SCENARIO=centos-bluestore-single_type_dmcrypt - SUBCOMMAND=batch - - name: ceph-volume-scenario - current-parameters: true - predefined-parameters: | - SCENARIO=centos-bluestore-mixed_type - SUBCOMMAND=batch - - name: ceph-volume-scenario - current-parameters: true - predefined-parameters: | - SCENARIO=centos-bluestore-mixed_type_dmcrypt - SUBCOMMAND=batch - - name: ceph-volume-scenario - current-parameters: true - predefined-parameters: | - SCENARIO=centos-bluestore-mixed_type_explicit - SUBCOMMAND=batch - - name: ceph-volume-scenario - current-parameters: true - predefined-parameters: | - SCENARIO=centos-bluestore-mixed_type_dmcrypt_explicit - SUBCOMMAND=batch + - multijob: + name: 'testing ceph-volume' + condition: SUCCESSFUL + projects: + - name: ceph-volume-scenario + current-parameters: true + predefined-parameters: | + DISTRO=centos + OBJECTSTORE=bluestore + METHOD=lvm + SCENARIO=unencrypted + - name: ceph-volume-scenario + current-parameters: true + predefined-parameters: | + DISTRO=centos + OBJECTSTORE=bluestore + METHOD=lvm + SCENARIO=dmcrypt + - name: ceph-volume-scenario + current-parameters: true + predefined-parameters: | + DISTRO=centos + OBJECTSTORE=bluestore + METHOD=raw + SCENARIO=unencrypted + - name: ceph-volume-scenario + current-parameters: true + predefined-parameters: | + DISTRO=centos + OBJECTSTORE=bluestore + METHOD=raw + SCENARIO=dmcrypt -- 2.39.5