From: Sage Weil Date: Fri, 21 Jul 2017 02:54:48 +0000 (-0400) Subject: qa: move ceph-helpers and misc src/test/*.sh tests to qa/standalone X-Git-Tag: v12.1.2~126^2~6 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=71ea1716043843dd191830f0bcbcc4a88059a9c2;p=ceph.git qa: move ceph-helpers and misc src/test/*.sh tests to qa/standalone - stop running via make check - add teuthology yamls to run them - disable ceph_objecstore_tool.py for now (too slow for make check, and we can't use vstart in teuthology via a package install) - drop cephtool tests since those are already covered by other teuthology tests - leave a handful of (fast!) ceph-helpers tests for make check for minimal integration tests. Signed-off-by: Sage Weil --- diff --git a/qa/standalone/ceph-helpers.sh b/qa/standalone/ceph-helpers.sh new file mode 100755 index 00000000000..3c623681e0d --- /dev/null +++ b/qa/standalone/ceph-helpers.sh @@ -0,0 +1,1912 @@ +#!/bin/bash +# +# Copyright (C) 2013,2014 Cloudwatt +# Copyright (C) 2014,2015 Red Hat +# Copyright (C) 2014 Federico Gimenez +# +# Author: Loic Dachary +# Author: Federico Gimenez +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# +TIMEOUT=300 +PG_NUM=4 +: ${CEPH_BUILD_VIRTUALENV:=/tmp} + +if type xmlstarlet > /dev/null 2>&1; then + XMLSTARLET=xmlstarlet +elif type xml > /dev/null 2>&1; then + XMLSTARLET=xml +else + echo "Missing xmlstarlet binary!" + exit 1 +fi + +if [ `uname` = FreeBSD ]; then + SED=gsed + DIFFCOLOPTS="" +else + SED=sed + termwidth=$(stty -a | head -1 | sed -e 's/.*columns \([0-9]*\).*/\1/') + if [ -n "$termwidth" -a "$termwidth" != "0" ]; then + termwidth="-W ${termwidth}" + fi + DIFFCOLOPTS="-y $termwidth" +fi + +EXTRA_OPTS="" +if [ -n "$CEPH_LIB" ]; then + EXTRA_OPTS+=" --erasure-code-dir $CEPH_LIB" + EXTRA_OPTS+=" --plugin-dir $CEPH_LIB" + EXTRA_OPTS+=" --osd-class-dir $CEPH_LIB" +fi + +#! @file ceph-helpers.sh +# @brief Toolbox to manage Ceph cluster dedicated to testing +# +# Example use case: +# +# ~~~~~~~~~~~~~~~~{.sh} +# source ceph-helpers.sh +# +# function mytest() { +# # cleanup leftovers and reset mydir +# setup mydir +# # create a cluster with one monitor and three osds +# run_mon mydir a +# run_osd mydir 0 +# run_osd mydir 2 +# run_osd mydir 3 +# # put and get an object +# rados --pool rbd put GROUP /etc/group +# rados --pool rbd get GROUP /tmp/GROUP +# # stop the cluster and cleanup the directory +# teardown mydir +# } +# ~~~~~~~~~~~~~~~~ +# +# The focus is on simplicity and efficiency, in the context of +# functional tests. The output is intentionally very verbose +# and functions return as soon as an error is found. The caller +# is also expected to abort on the first error so that debugging +# can be done by looking at the end of the output. +# +# Each function is documented, implemented and tested independently. +# When modifying a helper, the test and the documentation are +# expected to be updated and it is easier of they are collocated. A +# test for a given function can be run with +# +# ~~~~~~~~~~~~~~~~{.sh} +# ceph-helpers.sh TESTS test_get_osds +# ~~~~~~~~~~~~~~~~ +# +# and all the tests (i.e. all functions matching test_*) are run +# with: +# +# ~~~~~~~~~~~~~~~~{.sh} +# ceph-helpers.sh TESTS +# ~~~~~~~~~~~~~~~~ +# +# A test function takes a single argument : the directory dedicated +# to the tests. It is expected to not create any file outside of this +# directory and remove it entirely when it completes successfully. +# + + +function get_asok_dir() { + if [ -n "$CEPH_ASOK_DIR" ]; then + echo "$CEPH_ASOK_DIR" + else + echo ${TMPDIR:-/tmp}/ceph-asok.$$ + fi +} + +function get_asok_path() { + local name=$1 + if [ -n "$name" ]; then + echo $(get_asok_dir)/ceph-$name.asok + else + echo $(get_asok_dir)/\$cluster-\$name.asok + fi +} +## +# Cleanup any leftovers found in **dir** via **teardown** +# and reset **dir** as an empty environment. +# +# @param dir path name of the environment +# @return 0 on success, 1 on error +# +function setup() { + local dir=$1 + teardown $dir || return 1 + mkdir -p $dir + mkdir -p $(get_asok_dir) +} + +function test_setup() { + local dir=$dir + setup $dir || return 1 + test -d $dir || return 1 + setup $dir || return 1 + test -d $dir || return 1 + teardown $dir +} + +####################################################################### + +## +# Kill all daemons for which a .pid file exists in **dir** and remove +# **dir**. If the file system in which **dir** is btrfs, delete all +# subvolumes that relate to it. +# +# @param dir path name of the environment +# @return 0 on success, 1 on error +# +function teardown() { + local dir=$1 + kill_daemons $dir KILL + if [ `uname` != FreeBSD ] \ + && [ $(stat -f -c '%T' .) == "btrfs" ]; then + __teardown_btrfs $dir + fi + rm -fr $dir + rm -rf $(get_asok_dir) +} + +function __teardown_btrfs() { + local btrfs_base_dir=$1 + local btrfs_root=$(df -P . | tail -1 | awk '{print $NF}') + local btrfs_dirs=$(cd $btrfs_base_dir; sudo btrfs subvolume list . -t | awk '/^[0-9]/ {print $4}' | grep "$btrfs_base_dir/$btrfs_dir") + for subvolume in $btrfs_dirs; do + sudo btrfs subvolume delete $btrfs_root/$subvolume + done +} + +function test_teardown() { + local dir=$dir + setup $dir || return 1 + teardown $dir || return 1 + ! test -d $dir || return 1 +} + +####################################################################### + +## +# Sends a signal to a single daemon. +# This is a helper function for kill_daemons +# +# After the daemon is sent **signal**, its actual termination +# will be verified by sending it signal 0. If the daemon is +# still alive, kill_daemon will pause for a few seconds and +# try again. This will repeat for a fixed number of times +# before kill_daemon returns on failure. The list of +# sleep intervals can be specified as **delays** and defaults +# to: +# +# 0.1 0.2 1 1 1 2 3 5 5 5 10 10 20 60 60 60 120 +# +# This sequence is designed to run first a very short sleep time (0.1) +# if the machine is fast enough and the daemon terminates in a fraction of a +# second. The increasing sleep numbers should give plenty of time for +# the daemon to die even on the slowest running machine. If a daemon +# takes more than a few minutes to stop (the sum of all sleep times), +# there probably is no point in waiting more and a number of things +# are likely to go wrong anyway: better give up and return on error. +# +# @param pid the process id to send a signal +# @param send_signal the signal to send +# @param delays sequence of sleep times before failure +# +function kill_daemon() { + local pid=$(cat $1) + local send_signal=$2 + local delays=${3:-0.1 0.2 1 1 1 2 3 5 5 5 10 10 20 60 60 60 120} + local exit_code=1 + for try in $delays ; do + if kill -$send_signal $pid 2> /dev/null ; then + exit_code=1 + else + exit_code=0 + break + fi + send_signal=0 + sleep $try + done; + return $exit_code +} + +function test_kill_daemon() { + local dir=$1 + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + + name_prefix=osd + for pidfile in $(find $dir 2>/dev/null | grep $name_prefix'[^/]*\.pid') ; do + # + # sending signal 0 won't kill the daemon + # waiting just for one second instead of the default schedule + # allows us to quickly verify what happens when kill fails + # to stop the daemon (i.e. it must return false) + # + ! kill_daemon $pidfile 0 1 || return 1 + # + # killing just the osd and verify the mon still is responsive + # + kill_daemon $pidfile TERM || return 1 + done + + ceph osd dump | grep "osd.0 down" || return 1 + + name_prefix=mgr + for pidfile in $(find $dir 2>/dev/null | grep $name_prefix'[^/]*\.pid') ; do + # + # kill the mgr + # + kill_daemon $pidfile TERM || return 1 + done + + name_prefix=mon + for pidfile in $(find $dir 2>/dev/null | grep $name_prefix'[^/]*\.pid') ; do + # + # kill the mon and verify it cannot be reached + # + kill_daemon $pidfile TERM || return 1 + ! timeout 5 ceph status || return 1 + done + + teardown $dir || return 1 +} + +## +# Kill all daemons for which a .pid file exists in **dir**. Each +# daemon is sent a **signal** and kill_daemons waits for it to exit +# during a few minutes. By default all daemons are killed. If a +# **name_prefix** is provided, only the daemons for which a pid +# file is found matching the prefix are killed. See run_osd and +# run_mon for more information about the name conventions for +# the pid files. +# +# Send TERM to all daemons : kill_daemons $dir +# Send KILL to all daemons : kill_daemons $dir KILL +# Send KILL to all osds : kill_daemons $dir KILL osd +# Send KILL to osd 1 : kill_daemons $dir KILL osd.1 +# +# If a daemon is sent the TERM signal and does not terminate +# within a few minutes, it will still be running even after +# kill_daemons returns. +# +# If all daemons are kill successfully the function returns 0 +# if at least one daemon remains, this is treated as an +# error and the function return 1. +# +# @param dir path name of the environment +# @param signal name of the first signal (defaults to TERM) +# @param name_prefix only kill match daemons (defaults to all) +# @param delays sequence of sleep times before failure +# @return 0 on success, 1 on error +# +function kill_daemons() { + local trace=$(shopt -q -o xtrace && echo true || echo false) + $trace && shopt -u -o xtrace + local dir=$1 + local signal=${2:-TERM} + local name_prefix=$3 # optional, osd, mon, osd.1 + local delays=$4 #optional timing + local status=0 + local pids="" + + for pidfile in $(find $dir 2>/dev/null | grep $name_prefix'[^/]*\.pid') ; do + run_in_background pids kill_daemon $pidfile $signal $delays + done + + wait_background pids + status=$? + + $trace && shopt -s -o xtrace + return $status +} + +function test_kill_daemons() { + local dir=$1 + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + # + # sending signal 0 won't kill the daemon + # waiting just for one second instead of the default schedule + # allows us to quickly verify what happens when kill fails + # to stop the daemon (i.e. it must return false) + # + ! kill_daemons $dir 0 osd 1 || return 1 + # + # killing just the osd and verify the mon still is responsive + # + kill_daemons $dir TERM osd || return 1 + ceph osd dump | grep "osd.0 down" || return 1 + # + # kill the mgr + # + kill_daemons $dir TERM mgr || return 1 + # + # kill the mon and verify it cannot be reached + # + kill_daemons $dir TERM || return 1 + ! timeout 5 ceph status || return 1 + teardown $dir || return 1 +} + +####################################################################### + +## +# Run a monitor by the name mon.**id** with data in **dir**/**id**. +# The logs can be found in **dir**/mon.**id**.log and the pid file +# is **dir**/mon.**id**.pid and the admin socket is +# **dir**/**id**/ceph-mon.**id**.asok. +# +# The remaining arguments are passed verbatim to ceph-mon --mkfs +# and the ceph-mon daemon. +# +# Two mandatory arguments must be provided: --fsid and --mon-host +# Instead of adding them to every call to run_mon, they can be +# set in the CEPH_ARGS environment variable to be read implicitly +# by every ceph command. +# +# The CEPH_CONF variable is expected to be set to /dev/null to +# only rely on arguments for configuration. +# +# Examples: +# +# CEPH_ARGS="--fsid=$(uuidgen) " +# CEPH_ARGS+="--mon-host=127.0.0.1:7018 " +# run_mon $dir a # spawn a mon and bind port 7018 +# run_mon $dir a --debug-filestore=20 # spawn with filestore debugging +# +# If mon_initial_members is not set, the default rbd pool is deleted +# and replaced with a replicated pool with less placement groups to +# speed up initialization. If mon_initial_members is set, no attempt +# is made to recreate the rbd pool because it would hang forever, +# waiting for other mons to join. +# +# A **dir**/ceph.conf file is created but not meant to be used by any +# function. It is convenient for debugging a failure with: +# +# ceph --conf **dir**/ceph.conf -s +# +# @param dir path name of the environment +# @param id mon identifier +# @param ... can be any option valid for ceph-mon +# @return 0 on success, 1 on error +# +function run_mon_no_pool() { + local dir=$1 + shift + local id=$1 + shift + local data=$dir/$id + + ceph-mon \ + --id $id \ + --mkfs \ + --mon-data=$data \ + --run-dir=$dir \ + "$@" || return 1 + + ceph-mon \ + --id $id \ + --mon-osd-full-ratio=.99 \ + --mon-data-avail-crit=1 \ + --paxos-propose-interval=0.1 \ + --osd-crush-chooseleaf-type=0 \ + $EXTRA_OPTS \ + --debug-mon 20 \ + --debug-ms 20 \ + --debug-paxos 20 \ + --chdir= \ + --mon-data=$data \ + --log-file=$dir/\$name.log \ + --admin-socket=$(get_asok_path) \ + --mon-cluster-log-file=$dir/log \ + --run-dir=$dir \ + --pid-file=$dir/\$name.pid \ + --mon-allow-pool-delete \ + "$@" || return 1 + + cat > $dir/ceph.conf </dev/null | \ + jq '.acting | .[]') + # get rid of the trailing space + echo $osds +} + +function test_get_osds() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=2 || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + run_osd $dir 1 || return 1 + wait_for_clean || return 1 + get_osds rbd GROUP | grep --quiet '^[0-1] [0-1]$' || return 1 + teardown $dir || return 1 +} + +####################################################################### + +## +# Wait for the monitor to form quorum (optionally, of size N) +# +# @param timeout duration (lower-bound) to wait for quorum to be formed +# @param quorumsize size of quorum to wait for +# @return 0 on success, 1 on error +# +function wait_for_quorum() { + local timeout=$1 + local quorumsize=$2 + + if [[ -z "$timeout" ]]; then + timeout=300 + fi + + if [[ -z "$quorumsize" ]]; then + timeout $timeout ceph mon_status --format=json >&/dev/null || return 1 + return 0 + fi + + no_quorum=1 + wait_until=$((`date +%s` + $timeout)) + while [[ $(date +%s) -lt $wait_until ]]; do + jqfilter='.quorum | length == '$quorumsize + jqinput="$(timeout $timeout ceph mon_status --format=json 2>/dev/null)" + res=$(echo $jqinput | jq "$jqfilter") + if [[ "$res" == "true" ]]; then + no_quorum=0 + break + fi + done + return $no_quorum +} + +####################################################################### + +## +# Return the PG of supporting the **objectname** stored in +# **poolname**, as reported by ceph osd map. +# +# @param poolname an existing pool +# @param objectname an objectname (may or may not exist) +# @param STDOUT a PG +# @return 0 on success, 1 on error +# +function get_pg() { + local poolname=$1 + local objectname=$2 + + ceph --format json osd map $poolname $objectname 2>/dev/null | jq -r '.pgid' +} + +function test_get_pg() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + wait_for_clean || return 1 + get_pg rbd GROUP | grep --quiet '^[0-9]\.[0-9a-f][0-9a-f]*$' || return 1 + teardown $dir || return 1 +} + +####################################################################### + +## +# Return the value of the **config**, obtained via the config get command +# of the admin socket of **daemon**.**id**. +# +# @param daemon mon or osd +# @param id mon or osd ID +# @param config the configuration variable name as found in config_opts.h +# @param STDOUT the config value +# @return 0 on success, 1 on error +# +function get_config() { + local daemon=$1 + local id=$2 + local config=$3 + + CEPH_ARGS='' \ + ceph --format json daemon $(get_asok_path $daemon.$id) \ + config get $config 2> /dev/null | \ + jq -r ".$config" +} + +function test_get_config() { + local dir=$1 + + # override the default config using command line arg and check it + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 || return 1 + test $(get_config mon a osd_pool_default_size) = 1 || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 --osd_max_scrubs=3 || return 1 + test $(get_config osd 0 osd_max_scrubs) = 3 || return 1 + teardown $dir || return 1 +} + +####################################################################### + +## +# Set the **config** to specified **value**, via the config set command +# of the admin socket of **daemon**.**id** +# +# @param daemon mon or osd +# @param id mon or osd ID +# @param config the configuration variable name as found in config_opts.h +# @param value the config value +# @return 0 on success, 1 on error +# +function set_config() { + local daemon=$1 + local id=$2 + local config=$3 + local value=$4 + + test $(env CEPH_ARGS='' ceph --format json daemon $(get_asok_path $daemon.$id) \ + config set $config $value 2> /dev/null | \ + jq 'has("success")') == true +} + +function test_set_config() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 || return 1 + test $(get_config mon a ms_crc_header) = true || return 1 + set_config mon a ms_crc_header false || return 1 + test $(get_config mon a ms_crc_header) = false || return 1 + set_config mon a ms_crc_header true || return 1 + test $(get_config mon a ms_crc_header) = true || return 1 + teardown $dir || return 1 +} + +####################################################################### + +## +# Return the OSD id of the primary OSD supporting the **objectname** +# stored in **poolname**, as reported by ceph osd map. +# +# @param poolname an existing pool +# @param objectname an objectname (may or may not exist) +# @param STDOUT the primary OSD id +# @return 0 on success, 1 on error +# +function get_primary() { + local poolname=$1 + local objectname=$2 + + ceph --format json osd map $poolname $objectname 2>/dev/null | \ + jq '.acting_primary' +} + +function test_get_primary() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 || return 1 + local osd=0 + run_mgr $dir x || return 1 + run_osd $dir $osd || return 1 + wait_for_clean || return 1 + test $(get_primary rbd GROUP) = $osd || return 1 + teardown $dir || return 1 +} + +####################################################################### + +## +# Return the id of any OSD supporting the **objectname** stored in +# **poolname**, as reported by ceph osd map, except the primary. +# +# @param poolname an existing pool +# @param objectname an objectname (may or may not exist) +# @param STDOUT the OSD id +# @return 0 on success, 1 on error +# +function get_not_primary() { + local poolname=$1 + local objectname=$2 + + local primary=$(get_primary $poolname $objectname) + ceph --format json osd map $poolname $objectname 2>/dev/null | \ + jq ".acting | map(select (. != $primary)) | .[0]" +} + +function test_get_not_primary() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=2 || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + run_osd $dir 1 || return 1 + wait_for_clean || return 1 + local primary=$(get_primary rbd GROUP) + local not_primary=$(get_not_primary rbd GROUP) + test $not_primary != $primary || return 1 + test $not_primary = 0 -o $not_primary = 1 || return 1 + teardown $dir || return 1 +} + +####################################################################### + +## +# Run ceph-objectstore-tool against the OSD **id** using the data path +# **dir**. The OSD is killed with TERM prior to running +# ceph-objectstore-tool because access to the data path is +# exclusive. The OSD is restarted after the command completes. The +# objectstore_tool returns after all PG are active+clean again. +# +# @param dir the data path of the OSD +# @param id the OSD id +# @param ... arguments to ceph-objectstore-tool +# @param STDIN the input of ceph-objectstore-tool +# @param STDOUT the output of ceph-objectstore-tool +# @return 0 on success, 1 on error +# +# The value of $ceph_osd_args will be passed to restarted osds +# +function objectstore_tool() { + local dir=$1 + shift + local id=$1 + shift + local osd_data=$dir/$id + + local osd_type=$(cat $osd_data/type) + + kill_daemons $dir TERM osd.$id >&2 < /dev/null || return 1 + + local journal_args + if [ "$objectstore_type" == "filestore" ]; then + journal_args=" --journal-path $osd_data/journal" + fi + ceph-objectstore-tool \ + --data-path $osd_data \ + $journal_args \ + "$@" || return 1 + activate_osd $dir $id $ceph_osd_args >&2 || return 1 + wait_for_clean >&2 +} + +function test_objectstore_tool() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 || return 1 + local osd=0 + run_mgr $dir x || return 1 + run_osd $dir $osd || return 1 + wait_for_clean || return 1 + rados --pool rbd put GROUP /etc/group || return 1 + objectstore_tool $dir $osd GROUP get-bytes | \ + diff - /etc/group + ! objectstore_tool $dir $osd NOTEXISTS get-bytes || return 1 + teardown $dir || return 1 +} + +####################################################################### + +## +# Predicate checking if there is an ongoing recovery in the +# cluster. If any of the recovering_{keys,bytes,objects}_per_sec +# counters are reported by ceph status, it means recovery is in +# progress. +# +# @return 0 if recovery in progress, 1 otherwise +# +function get_is_making_recovery_progress() { + local recovery_progress + recovery_progress+=".recovering_keys_per_sec + " + recovery_progress+=".recovering_bytes_per_sec + " + recovery_progress+=".recovering_objects_per_sec" + local progress=$(ceph --format json status 2>/dev/null | \ + jq -r ".pgmap | $recovery_progress") + test "$progress" != null +} + +function test_get_is_making_recovery_progress() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + ! get_is_making_recovery_progress || return 1 + teardown $dir || return 1 +} + +####################################################################### + +## +# Return the number of active PGs in the cluster. A PG is active if +# ceph pg dump pgs reports it both **active** and **clean** and that +# not **stale**. +# +# @param STDOUT the number of active PGs +# @return 0 on success, 1 on error +# +function get_num_active_clean() { + local expression + expression+="select(contains(\"active\") and contains(\"clean\")) | " + expression+="select(contains(\"stale\") | not)" + ceph --format json pg dump pgs 2>/dev/null | \ + jq "[.[] | .state | $expression] | length" +} + +function test_get_num_active_clean() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + wait_for_clean || return 1 + local num_active_clean=$(get_num_active_clean) + test "$num_active_clean" = $PG_NUM || return 1 + teardown $dir || return 1 +} + +####################################################################### + +## +# Return the number of PGs in the cluster, according to +# ceph pg dump pgs. +# +# @param STDOUT the number of PGs +# @return 0 on success, 1 on error +# +function get_num_pgs() { + ceph --format json status 2>/dev/null | jq '.pgmap.num_pgs' +} + +function test_get_num_pgs() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + wait_for_clean || return 1 + local num_pgs=$(get_num_pgs) + test "$num_pgs" -gt 0 || return 1 + teardown $dir || return 1 +} + +####################################################################### + +## +# Return the OSD ids in use by at least one PG in the cluster (either +# in the up or the acting set), according to ceph pg dump pgs. Every +# OSD id shows as many times as they are used in up and acting sets. +# If an OSD id is in both the up and acting set of a given PG, it will +# show twice. +# +# @param STDOUT a sorted list of OSD ids +# @return 0 on success, 1 on error +# +function get_osd_id_used_by_pgs() { + ceph --format json pg dump pgs 2>/dev/null | jq '.[] | .up[], .acting[]' | sort +} + +function test_get_osd_id_used_by_pgs() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + wait_for_clean || return 1 + local osd_ids=$(get_osd_id_used_by_pgs | uniq) + test "$osd_ids" = "0" || return 1 + teardown $dir || return 1 +} + +####################################################################### + +## +# Wait until the OSD **id** shows **count** times in the +# PGs (see get_osd_id_used_by_pgs for more information about +# how OSD ids are counted). +# +# @param id the OSD id +# @param count the number of time it must show in the PGs +# @return 0 on success, 1 on error +# +function wait_osd_id_used_by_pgs() { + local id=$1 + local count=$2 + + status=1 + for ((i=0; i < $TIMEOUT / 5; i++)); do + echo $i + if ! test $(get_osd_id_used_by_pgs | grep -c $id) = $count ; then + sleep 5 + else + status=0 + break + fi + done + return $status +} + +function test_wait_osd_id_used_by_pgs() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + wait_for_clean || return 1 + wait_osd_id_used_by_pgs 0 8 || return 1 + ! TIMEOUT=1 wait_osd_id_used_by_pgs 123 5 || return 1 + teardown $dir || return 1 +} + +####################################################################### + +## +# Return the date and time of the last completed scrub for **pgid**, +# as reported by ceph pg dump pgs. Note that a repair also sets this +# date. +# +# @param pgid the id of the PG +# @param STDOUT the date and time of the last scrub +# @return 0 on success, 1 on error +# +function get_last_scrub_stamp() { + local pgid=$1 + local sname=${2:-last_scrub_stamp} + ceph --format json pg dump pgs 2>/dev/null | \ + jq -r ".[] | select(.pgid==\"$pgid\") | .$sname" +} + +function test_get_last_scrub_stamp() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + wait_for_clean || return 1 + stamp=$(get_last_scrub_stamp 2.0) + test -n "$stamp" || return 1 + teardown $dir || return 1 +} + +####################################################################### + +## +# Predicate checking if the cluster is clean, i.e. all of its PGs are +# in a clean state (see get_num_active_clean for a definition). +# +# @return 0 if the cluster is clean, 1 otherwise +# +function is_clean() { + num_pgs=$(get_num_pgs) + test $num_pgs != 0 || return 1 + test $(get_num_active_clean) = $num_pgs || return 1 +} + +function test_is_clean() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + wait_for_clean || return 1 + is_clean || return 1 + teardown $dir || return 1 +} + +####################################################################### + +## +# Return a list of numbers that are increasingly larger and whose +# total is **timeout** seconds. It can be used to have short sleep +# delay while waiting for an event on a fast machine. But if running +# very slowly the larger delays avoid stressing the machine even +# further or spamming the logs. +# +# @param timeout sum of all delays, in seconds +# @return a list of sleep delays +# +function get_timeout_delays() { + local trace=$(shopt -q -o xtrace && echo true || echo false) + $trace && shopt -u -o xtrace + local timeout=$1 + local first_step=${2:-1} + + local i + local total="0" + i=$first_step + while test "$(echo $total + $i \<= $timeout | bc -l)" = "1"; do + echo -n "$i " + total=$(echo $total + $i | bc -l) + i=$(echo $i \* 2 | bc -l) + done + if test "$(echo $total \< $timeout | bc -l)" = "1"; then + echo -n $(echo $timeout - $total | bc -l) + fi + $trace && shopt -s -o xtrace +} + +function test_get_timeout_delays() { + test "$(get_timeout_delays 1)" = "1 " || return 1 + test "$(get_timeout_delays 5)" = "1 2 2" || return 1 + test "$(get_timeout_delays 6)" = "1 2 3" || return 1 + test "$(get_timeout_delays 7)" = "1 2 4 " || return 1 + test "$(get_timeout_delays 8)" = "1 2 4 1" || return 1 + test "$(get_timeout_delays 1 .1)" = ".1 .2 .4 .3" || return 1 + test "$(get_timeout_delays 1.5 .1)" = ".1 .2 .4 .8 " || return 1 + test "$(get_timeout_delays 5 .1)" = ".1 .2 .4 .8 1.6 1.9" || return 1 + test "$(get_timeout_delays 6 .1)" = ".1 .2 .4 .8 1.6 2.9" || return 1 + test "$(get_timeout_delays 6.3 .1)" = ".1 .2 .4 .8 1.6 3.2 " || return 1 + test "$(get_timeout_delays 20 .1)" = ".1 .2 .4 .8 1.6 3.2 6.4 7.3" || return 1 +} + +####################################################################### + +## +# Wait until the cluster becomes clean or if it does not make progress +# for $TIMEOUT seconds. +# Progress is measured either via the **get_is_making_recovery_progress** +# predicate or if the number of clean PGs changes (as returned by get_num_active_clean) +# +# @return 0 if the cluster is clean, 1 otherwise +# +function wait_for_clean() { + local num_active_clean=-1 + local cur_active_clean + local -a delays=($(get_timeout_delays $TIMEOUT .1)) + local -i loop=0 + + while test $(get_num_pgs) == 0 ; do + sleep 1 + done + + while true ; do + # Comparing get_num_active_clean & get_num_pgs is used to determine + # if the cluster is clean. That's almost an inline of is_clean() to + # get more performance by avoiding multiple calls of get_num_active_clean. + cur_active_clean=$(get_num_active_clean) + test $cur_active_clean = $(get_num_pgs) && break + if test $cur_active_clean != $num_active_clean ; then + loop=0 + num_active_clean=$cur_active_clean + elif get_is_making_recovery_progress ; then + loop=0 + elif (( $loop >= ${#delays[*]} )) ; then + ceph report + return 1 + fi + sleep ${delays[$loop]} + loop+=1 + done + return 0 +} + +function test_wait_for_clean() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 + ! TIMEOUT=1 wait_for_clean || return 1 + run_osd $dir 0 || return 1 + wait_for_clean || return 1 + teardown $dir || return 1 +} + +####################################################################### + +## +# Wait until the cluster becomes HEALTH_OK again or if it does not make progress +# for $TIMEOUT seconds. +# +# @return 0 if the cluster is HEALTHY, 1 otherwise +# +function wait_for_health() { + local grepstr=$1 + local -a delays=($(get_timeout_delays $TIMEOUT .1)) + local -i loop=0 + + while ! ceph health detail | grep "$grepstr" ; do + if (( $loop >= ${#delays[*]} )) ; then + ceph health detail + return 1 + fi + sleep ${delays[$loop]} + loop+=1 + done +} + +function wait_for_health_ok() { + wait_for_health "HEALTH_OK" || return 1 +} + +function test_wait_for_health_ok() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 --osd_failsafe_full_ratio=.99 --mon_pg_warn_min_per_osd=0 || return 1 + run_mgr $dir x --mon_pg_warn_min_per_osd=0 || return 1 + run_osd $dir 0 || return 1 + kill_daemons $dir TERM osd || return 1 + ! TIMEOUT=1 wait_for_health_ok || return 1 + activate_osd $dir 0 || return 1 + wait_for_health_ok || return 1 + teardown $dir || return 1 +} + + +####################################################################### + +## +# Run repair on **pgid** and wait until it completes. The repair +# function will fail if repair does not complete within $TIMEOUT +# seconds. +# +# @param pgid the id of the PG +# @return 0 on success, 1 on error +# +function repair() { + local pgid=$1 + local last_scrub=$(get_last_scrub_stamp $pgid) + ceph pg repair $pgid + wait_for_scrub $pgid "$last_scrub" +} + +function test_repair() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + wait_for_clean || return 1 + repair 2.0 || return 1 + kill_daemons $dir KILL osd || return 1 + ! TIMEOUT=1 repair 2.0 || return 1 + teardown $dir || return 1 +} +####################################################################### + +## +# Run scrub on **pgid** and wait until it completes. The pg_scrub +# function will fail if repair does not complete within $TIMEOUT +# seconds. The pg_scrub is complete whenever the +# **get_last_scrub_stamp** function reports a timestamp different from +# the one stored before starting the scrub. +# +# @param pgid the id of the PG +# @return 0 on success, 1 on error +# +function pg_scrub() { + local pgid=$1 + local last_scrub=$(get_last_scrub_stamp $pgid) + ceph pg scrub $pgid + wait_for_scrub $pgid "$last_scrub" +} + +function pg_deep_scrub() { + local pgid=$1 + local last_scrub=$(get_last_scrub_stamp $pgid last_deep_scrub_stamp) + ceph pg deep-scrub $pgid + wait_for_scrub $pgid "$last_scrub" last_deep_scrub_stamp +} + +function test_pg_scrub() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + wait_for_clean || return 1 + pg_scrub 2.0 || return 1 + kill_daemons $dir KILL osd || return 1 + ! TIMEOUT=1 pg_scrub 2.0 || return 1 + teardown $dir || return 1 +} + +####################################################################### + +## +# Run the *command* and expect it to fail (i.e. return a non zero status). +# The output (stderr and stdout) is stored in a temporary file in *dir* +# and is expected to contain the string *expected*. +# +# Return 0 if the command failed and the string was found. Otherwise +# return 1 and cat the full output of the command on stderr for debug. +# +# @param dir temporary directory to store the output +# @param expected string to look for in the output +# @param command ... the command and its arguments +# @return 0 on success, 1 on error +# + +function expect_failure() { + local dir=$1 + shift + local expected="$1" + shift + local success + + if "$@" > $dir/out 2>&1 ; then + success=true + else + success=false + fi + + if $success || ! grep --quiet "$expected" $dir/out ; then + cat $dir/out >&2 + return 1 + else + return 0 + fi +} + +function test_expect_failure() { + local dir=$1 + + setup $dir || return 1 + expect_failure $dir FAIL bash -c 'echo FAIL ; exit 1' || return 1 + # the command did not fail + ! expect_failure $dir FAIL bash -c 'echo FAIL ; exit 0' > $dir/out || return 1 + grep --quiet FAIL $dir/out || return 1 + # the command failed but the output does not contain the expected string + ! expect_failure $dir FAIL bash -c 'echo UNEXPECTED ; exit 1' > $dir/out || return 1 + ! grep --quiet FAIL $dir/out || return 1 + teardown $dir || return 1 +} + +####################################################################### + +## +# Given the *last_scrub*, wait for scrub to happen on **pgid**. It +# will fail if scrub does not complete within $TIMEOUT seconds. The +# repair is complete whenever the **get_last_scrub_stamp** function +# reports a timestamp different from the one given in argument. +# +# @param pgid the id of the PG +# @param last_scrub timestamp of the last scrub for *pgid* +# @return 0 on success, 1 on error +# +function wait_for_scrub() { + local pgid=$1 + local last_scrub="$2" + local sname=${3:-last_scrub_stamp} + + for ((i=0; i < $TIMEOUT; i++)); do + if test "$last_scrub" != "$(get_last_scrub_stamp $pgid $sname)" ; then + return 0 + fi + sleep 1 + done + return 1 +} + +function test_wait_for_scrub() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + wait_for_clean || return 1 + local pgid=2.0 + ceph pg repair $pgid + local last_scrub=$(get_last_scrub_stamp $pgid) + wait_for_scrub $pgid "$last_scrub" || return 1 + kill_daemons $dir KILL osd || return 1 + last_scrub=$(get_last_scrub_stamp $pgid) + ! TIMEOUT=1 wait_for_scrub $pgid "$last_scrub" || return 1 + teardown $dir || return 1 +} + +####################################################################### + +## +# Return 0 if the erasure code *plugin* is available, 1 otherwise. +# +# @param plugin erasure code plugin +# @return 0 on success, 1 on error +# + +function erasure_code_plugin_exists() { + local plugin=$1 + local status + local grepstr + local s + case `uname` in + FreeBSD) grepstr="Cannot open.*$plugin" ;; + *) grepstr="$plugin.*No such file" ;; + esac + + s=$(ceph osd erasure-code-profile set TESTPROFILE plugin=$plugin 2>&1) + local status=$? + if [ $status -eq 0 ]; then + ceph osd erasure-code-profile rm TESTPROFILE + elif ! echo $s | grep --quiet "$grepstr" ; then + status=1 + # display why the string was rejected. + echo $s + fi + return $status +} + +function test_erasure_code_plugin_exists() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + erasure_code_plugin_exists jerasure || return 1 + ! erasure_code_plugin_exists FAKE || return 1 + teardown $dir || return 1 +} + +####################################################################### + +## +# Display all log files from **dir** on stdout. +# +# @param dir directory in which all data is stored +# + +function display_logs() { + local dir=$1 + + find $dir -maxdepth 1 -name '*.log' | \ + while read file ; do + echo "======================= $file" + cat $file + done +} + +function test_display_logs() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a || return 1 + kill_daemons $dir || return 1 + display_logs $dir > $dir/log.out + grep --quiet mon.a.log $dir/log.out || return 1 + teardown $dir || return 1 +} + +####################################################################### +## +# Spawn a command in background and save the pid in the variable name +# passed in argument. To make the output reading easier, the output is +# prepend with the process id. +# +# Example: +# pids1="" +# run_in_background pids1 bash -c 'sleep 1; exit 1' +# +# @param pid_variable the variable name (not value) where the pids will be stored +# @param ... the command to execute +# @return only the pid_variable output should be considered and used with **wait_background** +# +function run_in_background() { + local pid_variable=$1 + shift; + # Execute the command and prepend the output with its pid + # We enforce to return the exit status of the command and not the awk one. + ("$@" |& awk '{ a[i++] = $0 }END{for (i = 0; i in a; ++i) { print "'$$': " a[i]} }'; return ${PIPESTATUS[0]}) >&2 & + eval "$pid_variable+=\" $!\"" +} + +function test_run_in_background() { + local pids + run_in_background pids sleep 1 + run_in_background pids sleep 1 + test $(echo $pids | wc -w) = 2 || return 1 + wait $pids || return 1 +} + +####################################################################### +## +# Wait for pids running in background to complete. +# This function is usually used after a **run_in_background** call +# Example: +# pids1="" +# run_in_background pids1 bash -c 'sleep 1; exit 1' +# wait_background pids1 +# +# @param pids The variable name that contains the active PIDS. Set as empty at then end of the function. +# @return returns 1 if at least one process exits in error unless returns 0 +# +function wait_background() { + # We extract the PIDS from the variable name + pids=${!1} + + return_code=0 + for pid in $pids; do + if ! wait $pid; then + # If one process failed then return 1 + return_code=1 + fi + done + + # We empty the variable reporting that all process ended + eval "$1=''" + + return $return_code +} + + +function test_wait_background() { + local pids="" + run_in_background pids bash -c "sleep 1; exit 1" + run_in_background pids bash -c "sleep 2; exit 0" + wait_background pids + if [ $? -ne 1 ]; then return 1; fi + + run_in_background pids bash -c "sleep 1; exit 0" + run_in_background pids bash -c "sleep 2; exit 0" + wait_background pids + if [ $? -ne 0 ]; then return 1; fi + + if [ ! -z "$pids" ]; then return 1; fi +} + +function flush_pg_stats() +{ + local timeout=${1:-$TIMEOUT} + + ids=`ceph osd ls` + seqs='' + for osd in $ids; do + seq=`ceph tell osd.$osd flush_pg_stats` + seqs="$seqs $osd-$seq" + done + + for s in $seqs; do + osd=`echo $s | cut -d - -f 1` + seq=`echo $s | cut -d - -f 2` + echo "waiting osd.$osd seq $seq" + while test $(ceph osd last-stat-seq $osd) -lt $seq; do + sleep 1 + if [ $((timeout--)) -eq 0 ]; then + return 1 + fi + done + done +} + +function test_flush_pg_stats() +{ + local dir=$1 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + rados -p rbd put obj /etc/group + flush_pg_stats + local jq_filter='.pools | .[] | select(.name == "rbd") | .stats' + raw_bytes_used=`ceph df detail --format=json | jq "$jq_filter.raw_bytes_used"` + bytes_used=`ceph df detail --format=json | jq "$jq_filter.bytes_used"` + test $raw_bytes_used > 0 || return 1 + test $raw_bytes_used == $bytes_used || return 1 +} + +####################################################################### + +## +# Call the **run** function (which must be defined by the caller) with +# the **dir** argument followed by the caller argument list. +# +# If the **run** function returns on error, all logs found in **dir** +# are displayed for diagnostic purposes. +# +# **teardown** function is called when the **run** function returns +# (on success or on error), to cleanup leftovers. The CEPH_CONF is set +# to /dev/null and CEPH_ARGS is unset so that the tests are protected from +# external interferences. +# +# It is the responsibility of the **run** function to call the +# **setup** function to prepare the test environment (create a temporary +# directory etc.). +# +# The shell is required (via PS4) to display the function and line +# number whenever a statement is executed to help debugging. +# +# @param dir directory in which all data is stored +# @param ... arguments passed transparently to **run** +# @return 0 on success, 1 on error +# +function main() { + local dir=td/$1 + shift + + shopt -s -o xtrace + PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: ' + + export PATH=${CEPH_BUILD_VIRTUALENV}/ceph-disk-virtualenv/bin:${CEPH_BUILD_VIRTUALENV}/ceph-detect-init-virtualenv/bin:.:$PATH # make sure program from sources are preferred + #export PATH=$CEPH_ROOT/src/ceph-disk/virtualenv/bin:$CEPH_ROOT/src/ceph-detect-init/virtualenv/bin:.:$PATH # make sure program from sources are preferred + + export CEPH_CONF=/dev/null + unset CEPH_ARGS + + local code + if run $dir "$@" ; then + code=0 + else + display_logs $dir + code=1 + fi + teardown $dir || return 1 + return $code +} + +####################################################################### + +function run_tests() { + shopt -s -o xtrace + PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: ' + + export PATH=${CEPH_BUILD_VIRTUALENV}/ceph-disk-virtualenv/bin:${CEPH_BUILD_VIRTUALENV}/ceph-detect-init-virtualenv/bin:.:$PATH # make sure program from sources are preferred + #export PATH=$CEPH_ROOT/src/ceph-disk/virtualenv/bin:$CEPH_ROOT/src/ceph-detect-init/virtualenv/bin:.:$PATH # make sure program from sources are preferred + + export CEPH_MON="127.0.0.1:7109" # git grep '\<7109\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + export CEPH_CONF=/dev/null + + local funcs=${@:-$(set | sed -n -e 's/^\(test_[0-9a-z_]*\) .*/\1/p')} + local dir=td/ceph-helpers + + for func in $funcs ; do + $func $dir || return 1 + done +} + +if test "$1" = TESTS ; then + shift + run_tests "$@" +fi + +# NOTE: +# jq only support --exit-status|-e from version 1.4 forwards, which makes +# returning on error waaaay prettier and straightforward. +# However, the current automated upstream build is running with v1.3, +# which has no idea what -e is. Hence the convoluted error checking we +# need. Sad. +# The next time someone changes this code, please check if v1.4 is now +# a thing, and, if so, please change these to use -e. Thanks. + +# jq '.all.supported | select([.[] == "foo"] | any)' +function jq_success() { + input="$1" + filter="$2" + expects="\"$3\"" + + in_escaped=$(printf %s "$input" | sed "s/'/'\\\\''/g") + filter_escaped=$(printf %s "$filter" | sed "s/'/'\\\\''/g") + + ret=$(echo "$in_escaped" | jq "$filter_escaped") + if [[ "$ret" == "true" ]]; then + return 0 + elif [[ -n "$expects" ]]; then + if [[ "$ret" == "$expects" ]]; then + return 0 + fi + fi + return 1 + input=$1 + filter=$2 + expects="$3" + + ret="$(echo $input | jq \"$filter\")" + if [[ "$ret" == "true" ]]; then + return 0 + elif [[ -n "$expects" && "$ret" == "$expects" ]]; then + return 0 + fi + return 1 +} + +# Local Variables: +# compile-command: "cd ../../src ; make -j4 && ../qa/standalone/ceph-helpers.sh TESTS # test_get_config" +# End: diff --git a/qa/standalone/crush/crush-choose-args.sh b/qa/standalone/crush/crush-choose-args.sh new file mode 100755 index 00000000000..6e03a999c64 --- /dev/null +++ b/qa/standalone/crush/crush-choose-args.sh @@ -0,0 +1,161 @@ +#!/bin/bash +# +# Copyright (C) 2017 Red Hat +# +# Author: Loic Dachary +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7131" # git grep '\<7131\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + CEPH_ARGS+="--crush-location=root=default,host=HOST " + CEPH_ARGS+="--osd-crush-initial-weight=3 " + # + # Disable device auto class feature for now. + # The device class is non-deterministic and will + # crash the crushmap comparison below. + # + CEPH_ARGS+="--osd-class-update-on-start=false " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + +function TEST_choose_args_update() { + # + # adding a weighted OSD updates the weight up to the top + # + local dir=$1 + + run_mon $dir a || return 1 + run_osd $dir 0 || return 1 + + ceph osd set-require-min-compat-client luminous + ceph osd getcrushmap > $dir/map || return 1 + crushtool -d $dir/map -o $dir/map.txt || return 1 + sed -i -e '/end crush map/d' $dir/map.txt + cat >> $dir/map.txt < $dir/map-one-more || return 1 + crushtool -d $dir/map-one-more -o $dir/map-one-more.txt || return 1 + cat $dir/map-one-more.txt + diff -u $dir/map-one-more.txt $CEPH_ROOT/src/test/crush/crush-choose-args-expected-one-more-3.txt || return 1 + + destroy_osd $dir 1 || return 1 + ceph osd getcrushmap > $dir/map-one-less || return 1 + crushtool -d $dir/map-one-less -o $dir/map-one-less.txt || return 1 + diff -u $dir/map-one-less.txt $dir/map.txt || return 1 +} + +function TEST_no_update_weight_set() { + # + # adding a zero weight OSD does not update the weight set at all + # + local dir=$1 + + ORIG_CEPH_ARGS="$CEPH_ARGS" + CEPH_ARGS+="--osd-crush-update-weight-set=false " + + run_mon $dir a || return 1 + run_osd $dir 0 || return 1 + + ceph osd set-require-min-compat-client luminous + ceph osd crush tree + ceph osd getcrushmap > $dir/map || return 1 + crushtool -d $dir/map -o $dir/map.txt || return 1 + sed -i -e '/end crush map/d' $dir/map.txt + cat >> $dir/map.txt < $dir/map-one-more || return 1 + crushtool -d $dir/map-one-more -o $dir/map-one-more.txt || return 1 + cat $dir/map-one-more.txt + diff -u $dir/map-one-more.txt $CEPH_ROOT/src/test/crush/crush-choose-args-expected-one-more-0.txt || return 1 + + destroy_osd $dir 1 || return 1 + ceph osd crush tree + ceph osd getcrushmap > $dir/map-one-less || return 1 + crushtool -d $dir/map-one-less -o $dir/map-one-less.txt || return 1 + diff -u $dir/map-one-less.txt $dir/map.txt || return 1 + + CEPH_ARGS="$ORIG_CEPH_ARGS" +} + +main crush-choose-args "$@" + +# Local Variables: +# compile-command: "cd ../../../build ; ln -sf ../src/ceph-disk/ceph_disk/main.py bin/ceph-disk && make -j4 && ../src/test/crush/crush-choose-args.sh" +# End: diff --git a/qa/standalone/crush/crush-classes.sh b/qa/standalone/crush/crush-classes.sh new file mode 100755 index 00000000000..b674acfe4f0 --- /dev/null +++ b/qa/standalone/crush/crush-classes.sh @@ -0,0 +1,156 @@ +#!/bin/bash +# +# Copyright (C) 2017 Red Hat +# +# Author: Loic Dachary +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7130" # git grep '\<7130\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + # + # Disable auto-class, so we can inject device class manually below + # + CEPH_ARGS+="--osd-class-update-on-start=false " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + +function add_something() { + local dir=$1 + local obj=${2:-SOMETHING} + + local payload=ABCDEF + echo $payload > $dir/ORIGINAL + rados --pool rbd put $obj $dir/ORIGINAL || return 1 +} + +function get_osds_up() { + local poolname=$1 + local objectname=$2 + + local osds=$(ceph --format xml osd map $poolname $objectname 2>/dev/null | \ + $XMLSTARLET sel -t -m "//up/osd" -v . -o ' ') + # get rid of the trailing space + echo $osds +} + +function TEST_classes() { + local dir=$1 + + run_mon $dir a || return 1 + run_osd $dir 0 || return 1 + run_osd $dir 1 || return 1 + run_osd $dir 2 || return 1 + + test "$(get_osds_up rbd SOMETHING)" == "1 2 0" || return 1 + add_something $dir SOMETHING || return 1 + + # + # osd.0 has class ssd and the rule is modified + # to only take ssd devices. + # + ceph osd getcrushmap > $dir/map || return 1 + crushtool -d $dir/map -o $dir/map.txt || return 1 + ${SED} -i \ + -e '/device 0 osd.0/s/$/ class ssd/' \ + -e '/step take default/s/$/ class ssd/' \ + $dir/map.txt || return 1 + crushtool -c $dir/map.txt -o $dir/map-new || return 1 + ceph osd setcrushmap -i $dir/map-new || return 1 + + # + # There can only be one mapping since there only is + # one device with ssd class. + # + ok=false + for delay in 2 4 8 16 32 64 128 256 ; do + if test "$(get_osds_up rbd SOMETHING_ELSE)" == "0" ; then + ok=true + break + fi + sleep $delay + ceph osd dump # for debugging purposes + ceph pg dump # for debugging purposes + done + $ok || return 1 + # + # Writing keeps working because the pool is min_size 1 by + # default. + # + add_something $dir SOMETHING_ELSE || return 1 + + # + # Sanity check that the rule indeed has ssd + # generated bucket with a name including ~ssd. + # + ceph osd crush dump | grep -q '~ssd' || return 1 +} + +function TEST_set_device_class() { + local dir=$1 + + TEST_classes $dir || return 1 + + ceph osd crush set-device-class ssd osd.0 || return 1 + ceph osd crush class ls-osd ssd | grep 0 || return 1 + ceph osd crush set-device-class ssd osd.1 || return 1 + ceph osd crush class ls-osd ssd | grep 1 || return 1 + ceph osd crush set-device-class ssd 0 1 || return 1 # should be idempotent + + ok=false + for delay in 2 4 8 16 32 64 128 256 ; do + if test "$(get_osds_up rbd SOMETHING_ELSE)" == "0 1" ; then + ok=true + break + fi + sleep $delay + ceph osd crush dump + ceph osd dump # for debugging purposes + ceph pg dump # for debugging purposes + done + $ok || return 1 +} + +function TEST_mon_classes() { + local dir=$1 + + run_mon $dir a || return 1 + ceph osd crush class create CLASS || return 1 + ceph osd crush class create CLASS || return 1 # idempotent + ceph osd crush class ls | grep CLASS || return 1 + ceph osd crush class rename CLASS TEMP || return 1 + ceph osd crush class ls | grep TEMP || return 1 + ceph osd crush class rename TEMP CLASS || return 1 + ceph osd crush class ls | grep CLASS || return 1 + ceph osd crush class rm CLASS || return 1 + expect_failure $dir ENOENT ceph osd crush class rm CLASS || return 1 +} + +main crush-classes "$@" + +# Local Variables: +# compile-command: "cd ../../../build ; ln -sf ../src/ceph-disk/ceph_disk/main.py bin/ceph-disk && make -j4 && ../src/test/crush/crush-classes.sh" +# End: diff --git a/qa/standalone/erasure-code/test-erasure-code-plugins.sh b/qa/standalone/erasure-code/test-erasure-code-plugins.sh new file mode 100755 index 00000000000..2f7ef3afa8a --- /dev/null +++ b/qa/standalone/erasure-code/test-erasure-code-plugins.sh @@ -0,0 +1,116 @@ +#!/bin/bash -x + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +arch=$(uname -m) + +case $arch in + i[[3456]]86*|x86_64*|amd64*) + legacy_jerasure_plugins=(jerasure_generic jerasure_sse3 jerasure_sse4) + legacy_shec_plugins=(shec_generic shec_sse3 shec_sse4) + plugins=(jerasure shec lrc isa) + ;; + aarch64*|arm*) + legacy_jerasure_plugins=(jerasure_generic jerasure_neon) + legacy_shec_plugins=(shec_generic shec_neon) + plugins=(jerasure shec lrc) + ;; + *) + echo "unsupported platform ${arch}." + return 1 + ;; +esac + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:17110" # git grep '\<17110\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + $func $dir || return 1 + done +} + +function TEST_preload_warning() { + local dir=$1 + + for plugin in ${legacy_jerasure_plugins[*]} ${legacy_shec_plugins[*]}; do + setup $dir || return 1 + run_mon $dir a --osd_erasure_code_plugins="${plugin}" || return 1 + run_mgr $dir x || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1 + run_osd $dir 0 --osd_erasure_code_plugins="${plugin}" || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1 + grep "WARNING: osd_erasure_code_plugins contains plugin ${plugin}" $dir/mon.a.log || return 1 + grep "WARNING: osd_erasure_code_plugins contains plugin ${plugin}" $dir/osd.0.log || return 1 + teardown $dir || return 1 + done + return 0 +} + +function TEST_preload_no_warning() { + local dir=$1 + + for plugin in ${plugins[*]}; do + setup $dir || return 1 + run_mon $dir a --osd_erasure_code_plugins="${plugin}" || return 1 + run_mgr $dir x || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1 + run_osd $dir 0 --osd_erasure_code_plugins="${plugin}" || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1 + ! grep "WARNING: osd_erasure_code_plugins contains plugin" $dir/mon.a.log || return 1 + ! grep "WARNING: osd_erasure_code_plugins contains plugin" $dir/osd.0.log || return 1 + teardown $dir || return 1 + done + + return 0 +} + +function TEST_preload_no_warning_default() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1 + ! grep "WARNING: osd_erasure_code_plugins" $dir/mon.a.log || return 1 + ! grep "WARNING: osd_erasure_code_plugins" $dir/osd.0.log || return 1 + teardown $dir || return 1 + + return 0 +} + +function TEST_ec_profile_warning() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + for id in $(seq 0 2) ; do + run_osd $dir $id || return 1 + done + wait_for_clean || return 1 + + for plugin in ${legacy_jerasure_plugins[*]}; do + ceph osd erasure-code-profile set prof-${plugin} crush-failure-domain=osd technique=reed_sol_van plugin=${plugin} || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1 + grep "WARNING: erasure coding profile prof-${plugin} uses plugin ${plugin}" $dir/mon.a.log || return 1 + done + + for plugin in ${legacy_shec_plugins[*]}; do + ceph osd erasure-code-profile set prof-${plugin} crush-failure-domain=osd plugin=${plugin} || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1 + grep "WARNING: erasure coding profile prof-${plugin} uses plugin ${plugin}" $dir/mon.a.log || return 1 + done + + teardown $dir || return 1 +} + +main test-erasure-code-plugins "$@" diff --git a/qa/standalone/erasure-code/test-erasure-code.sh b/qa/standalone/erasure-code/test-erasure-code.sh new file mode 100755 index 00000000000..c97762e4628 --- /dev/null +++ b/qa/standalone/erasure-code/test-erasure-code.sh @@ -0,0 +1,338 @@ +#!/bin/bash +# +# Copyright (C) 2014 Cloudwatt +# Copyright (C) 2014, 2015 Red Hat +# +# Author: Loic Dachary +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7101" # git grep '\<7101\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON --mon-osd-prime-pg-temp=false" + + setup $dir || return 1 + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + # check that erasure code plugins are preloaded + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1 + grep 'load: jerasure.*lrc' $dir/mon.a.log || return 1 + for id in $(seq 0 10) ; do + run_osd $dir $id || return 1 + done + wait_for_clean || return 1 + # check that erasure code plugins are preloaded + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1 + grep 'load: jerasure.*lrc' $dir/osd.0.log || return 1 + create_erasure_coded_pool ecpool || return 1 + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + $func $dir || return 1 + done + + delete_pool ecpool || return 1 + teardown $dir || return 1 +} + +function create_erasure_coded_pool() { + local poolname=$1 + + ceph osd erasure-code-profile set myprofile \ + crush-failure-domain=osd || return 1 + ceph osd pool create $poolname 12 12 erasure myprofile \ + || return 1 + wait_for_clean || return 1 +} + +function delete_pool() { + local poolname=$1 + + ceph osd pool delete $poolname $poolname --yes-i-really-really-mean-it +} + +function rados_put_get() { + local dir=$1 + local poolname=$2 + local objname=${3:-SOMETHING} + + + for marker in AAA BBB CCCC DDDD ; do + printf "%*s" 1024 $marker + done > $dir/ORIGINAL + + # + # get and put an object, compare they are equal + # + rados --pool $poolname put $objname $dir/ORIGINAL || return 1 + rados --pool $poolname get $objname $dir/COPY || return 1 + diff $dir/ORIGINAL $dir/COPY || return 1 + rm $dir/COPY + + # + # take out an OSD used to store the object and + # check the object can still be retrieved, which implies + # recovery + # + local -a initial_osds=($(get_osds $poolname $objname)) + local last=$((${#initial_osds[@]} - 1)) + ceph osd out ${initial_osds[$last]} || return 1 + ! get_osds $poolname $objname | grep '\<'${initial_osds[$last]}'\>' || return 1 + rados --pool $poolname get $objname $dir/COPY || return 1 + diff $dir/ORIGINAL $dir/COPY || return 1 + ceph osd in ${initial_osds[$last]} || return 1 + + rm $dir/ORIGINAL +} + +function rados_osds_out_in() { + local dir=$1 + local poolname=$2 + local objname=${3:-SOMETHING} + + + for marker in FFFF GGGG HHHH IIII ; do + printf "%*s" 1024 $marker + done > $dir/ORIGINAL + + # + # get and put an object, compare they are equal + # + rados --pool $poolname put $objname $dir/ORIGINAL || return 1 + rados --pool $poolname get $objname $dir/COPY || return 1 + diff $dir/ORIGINAL $dir/COPY || return 1 + rm $dir/COPY + + # + # take out two OSDs used to store the object, wait for the cluster + # to be clean (i.e. all PG are clean and active) again which + # implies the PG have been moved to use the remaining OSDs. Check + # the object can still be retrieved. + # + wait_for_clean || return 1 + local osds_list=$(get_osds $poolname $objname) + local -a osds=($osds_list) + for osd in 0 1 ; do + ceph osd out ${osds[$osd]} || return 1 + done + wait_for_clean || return 1 + # + # verify the object is no longer mapped to the osds that are out + # + for osd in 0 1 ; do + ! get_osds $poolname $objname | grep '\<'${osds[$osd]}'\>' || return 1 + done + rados --pool $poolname get $objname $dir/COPY || return 1 + diff $dir/ORIGINAL $dir/COPY || return 1 + # + # bring the osds back in, , wait for the cluster + # to be clean (i.e. all PG are clean and active) again which + # implies the PG go back to using the same osds as before + # + for osd in 0 1 ; do + ceph osd in ${osds[$osd]} || return 1 + done + wait_for_clean || return 1 + test "$osds_list" = "$(get_osds $poolname $objname)" || return 1 + rm $dir/ORIGINAL +} + +function TEST_rados_put_get_lrc_advanced() { + local dir=$1 + local poolname=pool-lrc-a + local profile=profile-lrc-a + + ceph osd erasure-code-profile set $profile \ + plugin=lrc \ + mapping=DD_ \ + crush-steps='[ [ "chooseleaf", "osd", 0 ] ]' \ + layers='[ [ "DDc", "" ] ]' || return 1 + ceph osd pool create $poolname 12 12 erasure $profile \ + || return 1 + + rados_put_get $dir $poolname || return 1 + + delete_pool $poolname + ceph osd erasure-code-profile rm $profile +} + +function TEST_rados_put_get_lrc_kml() { + local dir=$1 + local poolname=pool-lrc + local profile=profile-lrc + + ceph osd erasure-code-profile set $profile \ + plugin=lrc \ + k=4 m=2 l=3 \ + crush-failure-domain=osd || return 1 + ceph osd pool create $poolname 12 12 erasure $profile \ + || return 1 + + rados_put_get $dir $poolname || return 1 + + delete_pool $poolname + ceph osd erasure-code-profile rm $profile +} + +function TEST_rados_put_get_isa() { + if ! erasure_code_plugin_exists isa ; then + echo "SKIP because plugin isa has not been built" + return 0 + fi + local dir=$1 + local poolname=pool-isa + + ceph osd erasure-code-profile set profile-isa \ + plugin=isa \ + crush-failure-domain=osd || return 1 + ceph osd pool create $poolname 1 1 erasure profile-isa \ + || return 1 + + rados_put_get $dir $poolname || return 1 + + delete_pool $poolname +} + +function TEST_rados_put_get_jerasure() { + local dir=$1 + + rados_put_get $dir ecpool || return 1 + + local poolname=pool-jerasure + local profile=profile-jerasure + + ceph osd erasure-code-profile set $profile \ + plugin=jerasure \ + k=4 m=2 \ + crush-failure-domain=osd || return 1 + ceph osd pool create $poolname 12 12 erasure $profile \ + || return 1 + + rados_put_get $dir $poolname || return 1 + rados_osds_out_in $dir $poolname || return 1 + + delete_pool $poolname + ceph osd erasure-code-profile rm $profile +} + +function TEST_rados_put_get_shec() { + local dir=$1 + + local poolname=pool-shec + local profile=profile-shec + + ceph osd erasure-code-profile set $profile \ + plugin=shec \ + k=2 m=1 c=1 \ + crush-failure-domain=osd || return 1 + ceph osd pool create $poolname 12 12 erasure $profile \ + || return 1 + + rados_put_get $dir $poolname || return 1 + + delete_pool $poolname + ceph osd erasure-code-profile rm $profile +} + +function TEST_alignment_constraints() { + local payload=ABC + echo "$payload" > $dir/ORIGINAL + # + # Verify that the rados command enforces alignment constraints + # imposed by the stripe width + # See http://tracker.ceph.com/issues/8622 + # + local stripe_unit=$(ceph-conf --show-config-value osd_pool_erasure_code_stripe_unit) + eval local $(ceph osd erasure-code-profile get myprofile | grep k=) + local block_size=$((stripe_unit * k - 1)) + dd if=/dev/zero of=$dir/ORIGINAL bs=$block_size count=2 + rados --block-size=$block_size \ + --pool ecpool put UNALIGNED $dir/ORIGINAL || return 1 + rm $dir/ORIGINAL +} + +function chunk_size() { + echo $(ceph-conf --show-config-value osd_pool_erasure_code_stripe_unit) +} + +# +# By default an object will be split in two (k=2) with the first part +# of the object in the first OSD of the up set and the second part in +# the next OSD in the up set. This layout is defined by the mapping +# parameter and this function helps verify that the first and second +# part of the object are located in the OSD where they should be. +# +function verify_chunk_mapping() { + local dir=$1 + local poolname=$2 + local first=$3 + local second=$4 + + local payload=$(printf '%*s' $(chunk_size) FIRST$poolname ; printf '%*s' $(chunk_size) SECOND$poolname) + echo -n "$payload" > $dir/ORIGINAL + + rados --pool $poolname put SOMETHING$poolname $dir/ORIGINAL || return 1 + rados --pool $poolname get SOMETHING$poolname $dir/COPY || return 1 + local -a osds=($(get_osds $poolname SOMETHING$poolname)) + for (( i = 0; i < ${#osds[@]}; i++ )) ; do + ceph daemon osd.${osds[$i]} flush_journal + done + diff $dir/ORIGINAL $dir/COPY || return 1 + rm $dir/COPY + + local -a osds=($(get_osds $poolname SOMETHING$poolname)) + grep --quiet --recursive --text FIRST$poolname $dir/${osds[$first]} || return 1 + grep --quiet --recursive --text SECOND$poolname $dir/${osds[$second]} || return 1 +} + +function TEST_chunk_mapping() { + local dir=$1 + + # + # mapping=DD_ is the default: + # first OSD (i.e. 0) in the up set has the first part of the object + # second OSD (i.e. 1) in the up set has the second part of the object + # + verify_chunk_mapping $dir ecpool 0 1 || return 1 + + ceph osd erasure-code-profile set remap-profile \ + plugin=lrc \ + layers='[ [ "_DD", "" ] ]' \ + mapping='_DD' \ + crush-steps='[ [ "choose", "osd", 0 ] ]' || return 1 + ceph osd erasure-code-profile get remap-profile + ceph osd pool create remap-pool 12 12 erasure remap-profile \ + || return 1 + + # + # mapping=_DD + # second OSD (i.e. 1) in the up set has the first part of the object + # third OSD (i.e. 2) in the up set has the second part of the object + # + verify_chunk_mapping $dir remap-pool 1 2 || return 1 + + delete_pool remap-pool + ceph osd erasure-code-profile rm remap-profile +} + +main test-erasure-code "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && test/erasure-code/test-erasure-code.sh" +# End: diff --git a/qa/standalone/erasure-code/test-erasure-eio.sh b/qa/standalone/erasure-code/test-erasure-eio.sh new file mode 100755 index 00000000000..60069ddab74 --- /dev/null +++ b/qa/standalone/erasure-code/test-erasure-eio.sh @@ -0,0 +1,337 @@ +#!/bin/bash +# +# Copyright (C) 2015 Red Hat +# +# +# Author: Kefu Chai +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7112" # git grep '\<7112\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + # check that erasure code plugins are preloaded + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1 + grep 'load: jerasure.*lrc' $dir/mon.a.log || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + +function setup_osds() { + for id in $(seq 0 3) ; do + run_osd $dir $id || return 1 + done + wait_for_clean || return 1 + + # check that erasure code plugins are preloaded + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1 + grep 'load: jerasure.*lrc' $dir/osd.0.log || return 1 +} + +function create_erasure_coded_pool() { + local poolname=$1 + + ceph osd erasure-code-profile set myprofile \ + plugin=jerasure \ + k=2 m=1 \ + crush-failure-domain=osd || return 1 + ceph osd pool create $poolname 1 1 erasure myprofile \ + || return 1 + wait_for_clean || return 1 +} + +function delete_pool() { + local poolname=$1 + + ceph osd pool delete $poolname $poolname --yes-i-really-really-mean-it + ceph osd erasure-code-profile rm myprofile +} + +function rados_put() { + local dir=$1 + local poolname=$2 + local objname=${3:-SOMETHING} + + for marker in AAA BBB CCCC DDDD ; do + printf "%*s" 1024 $marker + done > $dir/ORIGINAL + # + # get and put an object, compare they are equal + # + rados --pool $poolname put $objname $dir/ORIGINAL || return 1 +} + +function rados_get() { + local dir=$1 + local poolname=$2 + local objname=${3:-SOMETHING} + local expect=${4:-ok} + + # + # Expect a failure to get object + # + if [ $expect = "fail" ]; + then + ! rados --pool $poolname get $objname $dir/COPY + return + fi + # + # get an object, compare with $dir/ORIGINAL + # + rados --pool $poolname get $objname $dir/COPY || return 1 + diff $dir/ORIGINAL $dir/COPY || return 1 + rm $dir/COPY +} + +function rados_put_get() { + local dir=$1 + local poolname=$2 + local objname=${3:-SOMETHING} + local recovery=$4 + + # + # get and put an object, compare they are equal + # + rados_put $dir $poolname $objname || return 1 + # We can read even though caller injected read error on one of the shards + rados_get $dir $poolname $objname || return 1 + + if [ -n "$recovery" ]; + then + # + # take out the last OSD used to store the object, + # bring it back, and check for clean PGs which means + # recovery didn't crash the primary. + # + local -a initial_osds=($(get_osds $poolname $objname)) + local last=$((${#initial_osds[@]} - 1)) + # Kill OSD + kill_daemons $dir TERM osd.${initial_osds[$last]} >&2 < /dev/null || return 1 + ceph osd out ${initial_osds[$last]} || return 1 + ! get_osds $poolname $objname | grep '\<'${initial_osds[$last]}'\>' || return 1 + ceph osd in ${initial_osds[$last]} || return 1 + run_osd $dir ${initial_osds[$last]} || return 1 + wait_for_clean || return 1 + fi + + rm $dir/ORIGINAL +} + +function inject_eio() { + local objname=$1 + shift + local dir=$1 + shift + local shard_id=$1 + shift + + local poolname=pool-jerasure + local -a initial_osds=($(get_osds $poolname $objname)) + local osd_id=${initial_osds[$shard_id]} + set_config osd $osd_id filestore_debug_inject_read_err true || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.$osd_id) \ + injectdataerr $poolname $objname $shard_id || return 1 +} + +function rados_get_data_eio() { + local dir=$1 + shift + local shard_id=$1 + shift + local recovery=$1 + shift + + # inject eio to speificied shard + # + local poolname=pool-jerasure + local objname=obj-eio-$$-$shard_id + inject_eio $objname $dir $shard_id || return 1 + rados_put_get $dir $poolname $objname $recovery || return 1 + + shard_id=$(expr $shard_id + 1) + inject_eio $objname $dir $shard_id || return 1 + # Now 2 out of 3 shards get EIO, so should fail + rados_get $dir $poolname $objname fail || return 1 +} + +# Change the size of speificied shard +# +function set_size() { + local objname=$1 + shift + local dir=$1 + shift + local shard_id=$1 + shift + local bytes=$1 + shift + local mode=${1} + + local poolname=pool-jerasure + local -a initial_osds=($(get_osds $poolname $objname)) + local osd_id=${initial_osds[$shard_id]} + ceph osd set noout + if [ "$mode" = "add" ]; + then + objectstore_tool $dir $osd_id $objname get-bytes $dir/CORRUPT || return 1 + dd if=/dev/urandom bs=$bytes count=1 >> $dir/CORRUPT + elif [ "$bytes" = "0" ]; + then + touch $dir/CORRUPT + else + dd if=/dev/urandom bs=$bytes count=1 of=$dir/CORRUPT + fi + objectstore_tool $dir $osd_id $objname set-bytes $dir/CORRUPT || return 1 + rm -f $dir/CORRUPT + ceph osd unset noout +} + +function rados_get_data_bad_size() { + local dir=$1 + shift + local shard_id=$1 + shift + local bytes=$1 + shift + local mode=${1:-set} + + local poolname=pool-jerasure + local objname=obj-size-$$-$shard_id-$bytes + rados_put $dir $poolname $objname || return 1 + + # Change the size of speificied shard + # + set_size $objname $dir $shard_id $bytes $mode || return 1 + + rados_get $dir $poolname $objname || return 1 + + # Leave objname and modify another shard + shard_id=$(expr $shard_id + 1) + set_size $objname $dir $shard_id $bytes $mode || return 1 + rados_get $dir $poolname $objname fail || return 1 +} + +# +# These two test cases try to validate the following behavior: +# For object on EC pool, if there is one shard having read error ( +# either primary or replica), client can still read object. +# +# If 2 shards have read errors the client will get an error. +# +function TEST_rados_get_subread_eio_shard_0() { + local dir=$1 + setup_osds || return 1 + + local poolname=pool-jerasure + create_erasure_coded_pool $poolname || return 1 + # inject eio on primary OSD (0) and replica OSD (1) + local shard_id=0 + rados_get_data_eio $dir $shard_id || return 1 + delete_pool $poolname +} + +function TEST_rados_get_subread_eio_shard_1() { + local dir=$1 + setup_osds || return 1 + + local poolname=pool-jerasure + create_erasure_coded_pool $poolname || return 1 + # inject eio into replicas OSD (1) and OSD (2) + local shard_id=1 + rados_get_data_eio $dir $shard_id || return 1 + delete_pool $poolname +} + +# +# These two test cases try to validate that following behavior: +# For object on EC pool, if there is one shard which an incorrect +# size this will cause an internal read error, client can still read object. +# +# If 2 shards have incorrect size the client will get an error. +# +function TEST_rados_get_bad_size_shard_0() { + local dir=$1 + setup_osds || return 1 + + local poolname=pool-jerasure + create_erasure_coded_pool $poolname || return 1 + # Set incorrect size into primary OSD (0) and replica OSD (1) + local shard_id=0 + rados_get_data_bad_size $dir $shard_id 10 || return 1 + rados_get_data_bad_size $dir $shard_id 0 || return 1 + rados_get_data_bad_size $dir $shard_id 256 add || return 1 + delete_pool $poolname +} + +function TEST_rados_get_bad_size_shard_1() { + local dir=$1 + setup_osds || return 1 + + local poolname=pool-jerasure + create_erasure_coded_pool $poolname || return 1 + # Set incorrect size into replicas OSD (1) and OSD (2) + local shard_id=1 + rados_get_data_bad_size $dir $shard_id 10 || return 1 + rados_get_data_bad_size $dir $shard_id 0 || return 1 + rados_get_data_bad_size $dir $shard_id 256 add || return 1 + delete_pool $poolname +} + +function TEST_rados_get_with_subreadall_eio_shard_0() { + local dir=$1 + local shard_id=0 + + setup_osds || return 1 + + local poolname=pool-jerasure + create_erasure_coded_pool $poolname || return 1 + # inject eio on primary OSD (0) + local shard_id=0 + rados_get_data_eio $dir $shard_id recovery || return 1 + + delete_pool $poolname +} + +function TEST_rados_get_with_subreadall_eio_shard_1() { + local dir=$1 + local shard_id=0 + + setup_osds || return 1 + + local poolname=pool-jerasure + create_erasure_coded_pool $poolname || return 1 + # inject eio on replica OSD (1) + local shard_id=1 + rados_get_data_eio $dir $shard_id recovery || return 1 + + delete_pool $poolname +} + +main test-erasure-eio "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && test/erasure-code/test-erasure-eio.sh" +# End: diff --git a/qa/standalone/misc/rados-striper.sh b/qa/standalone/misc/rados-striper.sh new file mode 100755 index 00000000000..2844a5930c3 --- /dev/null +++ b/qa/standalone/misc/rados-striper.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# +# Copyright (C) 2014 Red Hat +# +# Author: Sebastien Ponce +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7116" # git grep '\<7116\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + # setup + setup $dir || return 1 + + # create a cluster with one monitor and three osds + run_mon $dir a || return 1 + run_osd $dir 0 || return 1 + run_osd $dir 1 || return 1 + run_osd $dir 2 || return 1 + + # create toyfile + dd if=/dev/urandom of=$dir/toyfile bs=1234 count=1 + + # put a striped object + rados --pool rbd --striper put toyfile $dir/toyfile || return 1 + + # stat it, with and without striping + rados --pool rbd --striper stat toyfile | cut -d ',' -f 2 > $dir/stripedStat || return 1 + rados --pool rbd stat toyfile.0000000000000000 | cut -d ',' -f 2 > $dir/stat || return 1 + echo ' size 1234' > $dir/refstat + diff -w $dir/stripedStat $dir/refstat || return 1 + diff -w $dir/stat $dir/refstat || return 1 + rados --pool rbd stat toyfile >& $dir/staterror + grep -q 'No such file or directory' $dir/staterror || return 1 + + # get the file back with and without striping + rados --pool rbd --striper get toyfile $dir/stripedGroup || return 1 + diff -w $dir/toyfile $dir/stripedGroup || return 1 + rados --pool rbd get toyfile.0000000000000000 $dir/nonSTripedGroup || return 1 + diff -w $dir/toyfile $dir/nonSTripedGroup || return 1 + + # test truncate + rados --pool rbd --striper truncate toyfile 12 + rados --pool rbd --striper stat toyfile | cut -d ',' -f 2 > $dir/stripedStat || return 1 + rados --pool rbd stat toyfile.0000000000000000 | cut -d ',' -f 2 > $dir/stat || return 1 + echo ' size 12' > $dir/reftrunc + diff -w $dir/stripedStat $dir/reftrunc || return 1 + diff -w $dir/stat $dir/reftrunc || return 1 + + # test xattrs + + rados --pool rbd --striper setxattr toyfile somexattr somevalue || return 1 + rados --pool rbd --striper getxattr toyfile somexattr > $dir/xattrvalue || return 1 + rados --pool rbd getxattr toyfile.0000000000000000 somexattr > $dir/xattrvalue2 || return 1 + echo 'somevalue' > $dir/refvalue + diff -w $dir/xattrvalue $dir/refvalue || return 1 + diff -w $dir/xattrvalue2 $dir/refvalue || return 1 + rados --pool rbd --striper listxattr toyfile > $dir/xattrlist || return 1 + echo 'somexattr' > $dir/reflist + diff -w $dir/xattrlist $dir/reflist || return 1 + rados --pool rbd listxattr toyfile.0000000000000000 | grep -v striper > $dir/xattrlist2 || return 1 + diff -w $dir/xattrlist2 $dir/reflist || return 1 + rados --pool rbd --striper rmxattr toyfile somexattr || return 1 + + local attr_not_found_str="No data available" + [ `uname` = FreeBSD ] && \ + attr_not_found_str="Attribute not found" + expect_failure $dir "$attr_not_found_str" \ + rados --pool rbd --striper getxattr toyfile somexattr || return 1 + expect_failure $dir "$attr_not_found_str" \ + rados --pool rbd getxattr toyfile.0000000000000000 somexattr || return 1 + + # test rm + rados --pool rbd --striper rm toyfile || return 1 + expect_failure $dir 'No such file or directory' \ + rados --pool rbd --striper stat toyfile || return 1 + expect_failure $dir 'No such file or directory' \ + rados --pool rbd stat toyfile.0000000000000000 || return 1 + + # cleanup + teardown $dir || return 1 +} + +main rados-striper "$@" diff --git a/qa/standalone/misc/test-ceph-helpers.sh b/qa/standalone/misc/test-ceph-helpers.sh new file mode 100755 index 00000000000..3a6788ecaa3 --- /dev/null +++ b/qa/standalone/misc/test-ceph-helpers.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Copyright (C) 2013,2014 Cloudwatt +# Copyright (C) 2014 Red Hat +# Copyright (C) 2014 Federico Gimenez +# +# Author: Loic Dachary +# Author: Federico Gimenez +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# + +$CEPH_ROOT/qa/standalone/ceph-helpers.sh TESTS diff --git a/qa/standalone/mon/misc.sh b/qa/standalone/mon/misc.sh new file mode 100755 index 00000000000..bd0aa1c3650 --- /dev/null +++ b/qa/standalone/mon/misc.sh @@ -0,0 +1,237 @@ +#!/bin/bash +# +# Copyright (C) 2014 Cloudwatt +# Copyright (C) 2014, 2015 Red Hat +# +# Author: Loic Dachary +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7102" # git grep '\<7102\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + $func $dir || return 1 + done +} + +TEST_POOL=rbd + +function TEST_osd_pool_get_set() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a || return 1 + ceph osd pool create $TEST_POOL 8 + + local flag + for flag in nodelete nopgchange nosizechange write_fadvise_dontneed noscrub nodeep-scrub; do + ceph osd pool set $TEST_POOL $flag 0 || return 1 + ! ceph osd dump | grep 'pool ' | grep $flag || return 1 + ceph osd pool set $TEST_POOL $flag 1 || return 1 + ceph osd dump | grep 'pool ' | grep $flag || return 1 + ceph osd pool set $TEST_POOL $flag false || return 1 + ! ceph osd dump | grep 'pool ' | grep $flag || return 1 + ceph osd pool set $TEST_POOL $flag false || return 1 + # check that setting false twice does not toggle to true (bug) + ! ceph osd dump | grep 'pool ' | grep $flag || return 1 + ceph osd pool set $TEST_POOL $flag true || return 1 + ceph osd dump | grep 'pool ' | grep $flag || return 1 + # cleanup + ceph osd pool set $TEST_POOL $flag 0 || return 1 + done + + local size=$(ceph osd pool get $TEST_POOL size|awk '{print $2}') + local min_size=$(ceph osd pool get $TEST_POOL min_size|awk '{print $2}') + + ceph osd pool set $TEST_POOL scrub_min_interval 123456 || return 1 + ceph osd dump | grep 'pool ' | grep 'scrub_min_interval 123456' || return 1 + ceph osd pool set $TEST_POOL scrub_min_interval 0 || return 1 + ceph osd dump | grep 'pool ' | grep 'scrub_min_interval' && return 1 + ceph osd pool set $TEST_POOL scrub_max_interval 123456 || return 1 + ceph osd dump | grep 'pool ' | grep 'scrub_max_interval 123456' || return 1 + ceph osd pool set $TEST_POOL scrub_max_interval 0 || return 1 + ceph osd dump | grep 'pool ' | grep 'scrub_max_interval' && return 1 + ceph osd pool set $TEST_POOL deep_scrub_interval 123456 || return 1 + ceph osd dump | grep 'pool ' | grep 'deep_scrub_interval 123456' || return 1 + ceph osd pool set $TEST_POOL deep_scrub_interval 0 || return 1 + ceph osd dump | grep 'pool ' | grep 'deep_scrub_interval' && return 1 + + #replicated pool size restrict in 1 and 10 + ! ceph osd pool set $TEST_POOL 11 || return 1 + #replicated pool min_size must be between in 1 and size + ! ceph osd pool set $TEST_POOL min_size $(expr $size + 1) || return 1 + ! ceph osd pool set $TEST_POOL min_size 0 || return 1 + + local ecpool=erasepool + ceph osd pool create $ecpool 12 12 erasure default || return 1 + #erasue pool size=k+m, min_size=k + local size=$(ceph osd pool get $ecpool size|awk '{print $2}') + local min_size=$(ceph osd pool get $ecpool min_size|awk '{print $2}') + local k=$(expr $min_size - 1) # default min_size=k+1 + #erasure pool size can't change + ! ceph osd pool set $ecpool size $(expr $size + 1) || return 1 + #erasure pool min_size must be between in k and size + ceph osd pool set $ecpool min_size $(expr $k + 1) || return 1 + ! ceph osd pool set $ecpool min_size $(expr $k - 1) || return 1 + ! ceph osd pool set $ecpool min_size $(expr $size + 1) || return 1 + + teardown $dir || return 1 +} + +function TEST_mon_add_to_single_mon() { + local dir=$1 + + fsid=$(uuidgen) + MONA=127.0.0.1:7117 # git grep '\<7117\>' : there must be only one + MONB=127.0.0.1:7118 # git grep '\<7118\>' : there must be only one + CEPH_ARGS_orig=$CEPH_ARGS + CEPH_ARGS="--fsid=$fsid --auth-supported=none " + CEPH_ARGS+="--mon-initial-members=a " + CEPH_ARGS+="--mon-host=$MONA " + + setup $dir || return 1 + run_mon $dir a --public-addr $MONA || return 1 + # wait for the quorum + timeout 120 ceph -s > /dev/null || return 1 + run_mon $dir b --public-addr $MONB || return 1 + teardown $dir || return 1 + + setup $dir || return 1 + run_mon $dir a --public-addr $MONA || return 1 + # without the fix of #5454, mon.a will assert failure at seeing the MMonJoin + # from mon.b + run_mon $dir b --public-addr $MONB || return 1 + # wait for the quorum + timeout 120 ceph -s > /dev/null || return 1 + local num_mons + num_mons=$(ceph mon dump --format=json 2>/dev/null | jq ".mons | length") || return 1 + [ $num_mons == 2 ] || return 1 + # no reason to take more than 120 secs to get this submitted + timeout 120 ceph mon add b $MONB || return 1 + teardown $dir || return 1 +} + +function TEST_no_segfault_for_bad_keyring() { + local dir=$1 + setup $dir || return 1 + # create a client.admin key and add it to ceph.mon.keyring + ceph-authtool --create-keyring $dir/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *' + ceph-authtool --create-keyring $dir/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' + ceph-authtool $dir/ceph.mon.keyring --import-keyring $dir/ceph.client.admin.keyring + CEPH_ARGS_TMP="--fsid=$(uuidgen) --mon-host=127.0.0.1:7102 --auth-supported=cephx " + CEPH_ARGS_orig=$CEPH_ARGS + CEPH_ARGS="$CEPH_ARGS_TMP --keyring=$dir/ceph.mon.keyring " + run_mon $dir a + # create a bad keyring and make sure no segfault occurs when using the bad keyring + echo -e "[client.admin]\nkey = BQAUlgtWoFePIxAAQ9YLzJSVgJX5V1lh5gyctg==" > $dir/bad.keyring + CEPH_ARGS="$CEPH_ARGS_TMP --keyring=$dir/bad.keyring" + ceph osd dump 2> /dev/null + # 139(11|128) means segfault and core dumped + [ $? -eq 139 ] && return 1 + CEPH_ARGS=$CEPH_ARGS_orig + teardown $dir || return 1 +} + +function TEST_mon_features() { + local dir=$1 + setup $dir || return 1 + + fsid=$(uuidgen) + MONA=127.0.0.1:7127 # git grep '\<7127\>' ; there must be only one + MONB=127.0.0.1:7128 # git grep '\<7128\>' ; there must be only one + MONC=127.0.0.1:7129 # git grep '\<7129\>' ; there must be only one + CEPH_ARGS_orig=$CEPH_ARGS + CEPH_ARGS="--fsid=$fsid --auth-supported=none " + CEPH_ARGS+="--mon-initial-members=a,b,c " + CEPH_ARGS+="--mon-host=$MONA,$MONB,$MONC " + CEPH_ARGS+="--mon-debug-no-initial-persistent-features " + CEPH_ARGS+="--mon-debug-no-require-luminous " + + run_mon $dir a --public-addr $MONA || return 1 + run_mon $dir b --public-addr $MONB || return 1 + timeout 120 ceph -s > /dev/null || return 1 + + # expect monmap to contain 3 monitors (a, b, and c) + jqinput="$(ceph mon_status --format=json 2>/dev/null)" + jq_success "$jqinput" '.monmap.mons | length == 3' || return 1 + # quorum contains two monitors + jq_success "$jqinput" '.quorum | length == 2' || return 1 + # quorum's monitor features contain kraken and luminous + jqfilter='.features.quorum_mon[]|select(. == "kraken")' + jq_success "$jqinput" "$jqfilter" "kraken" || return 1 + jqfilter='.features.quorum_mon[]|select(. == "luminous")' + jq_success "$jqinput" "$jqfilter" "luminous" || return 1 + + # monmap must have no persistent features set, because we + # don't currently have a quorum made out of all the monitors + # in the monmap. + jqfilter='.monmap.features.persistent | length == 0' + jq_success "$jqinput" "$jqfilter" || return 1 + + # nor do we have any optional features, for that matter. + jqfilter='.monmap.features.optional | length == 0' + jq_success "$jqinput" "$jqfilter" || return 1 + + # validate 'mon feature ls' + + jqinput="$(ceph mon feature ls --format=json 2>/dev/null)" + # 'kraken' and 'luminous' are supported + jqfilter='.all.supported[] | select(. == "kraken")' + jq_success "$jqinput" "$jqfilter" "kraken" || return 1 + jqfilter='.all.supported[] | select(. == "luminous")' + jq_success "$jqinput" "$jqfilter" "luminous" || return 1 + + # start third monitor + run_mon $dir c --public-addr $MONC || return 1 + + wait_for_quorum 300 3 || return 1 + + timeout 300 ceph -s > /dev/null || return 1 + + jqinput="$(ceph mon_status --format=json 2>/dev/null)" + # expect quorum to have all three monitors + jqfilter='.quorum | length == 3' + jq_success "$jqinput" "$jqfilter" || return 1 + # quorum's monitor features contain kraken and luminous + jqfilter='.features.quorum_mon[]|select(. == "kraken")' + jq_success "$jqinput" "$jqfilter" "kraken" || return 1 + jqfilter='.features.quorum_mon[]|select(. == "luminous")' + jq_success "$jqinput" "$jqfilter" "luminous" || return 1 + + # monmap must have no both 'kraken' and 'luminous' persistent + # features set. + jqfilter='.monmap.features.persistent | length == 2' + jq_success "$jqinput" "$jqfilter" || return 1 + jqfilter='.monmap.features.persistent[]|select(. == "kraken")' + jq_success "$jqinput" "$jqfilter" "kraken" || return 1 + jqfilter='.monmap.features.persistent[]|select(. == "luminous")' + jq_success "$jqinput" "$jqfilter" "luminous" || return 1 + + CEPH_ARGS=$CEPH_ARGS_orig + # that's all folks. thank you for tuning in. + teardown $dir || return 1 +} + +main misc "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && test/mon/misc.sh" +# End: diff --git a/qa/standalone/mon/mkfs.sh b/qa/standalone/mon/mkfs.sh new file mode 100755 index 00000000000..6b8e58d8a37 --- /dev/null +++ b/qa/standalone/mon/mkfs.sh @@ -0,0 +1,198 @@ +#!/bin/bash +# +# Copyright (C) 2013 Cloudwatt +# Copyright (C) 2014 Red Hat +# +# Author: Loic Dachary +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# +set -xe +PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: ' + + +DIR=mkfs +export CEPH_CONF=/dev/null +unset CEPH_ARGS +MON_ID=a +MON_DIR=$DIR/$MON_ID +CEPH_MON=127.0.0.1:7110 # git grep '\<7110\>' : there must be only one +TIMEOUT=360 + +EXTRAOPTS="" +if [ -n "$CEPH_LIB" ]; then + EXTRAOPTS+=" --erasure-code-dir $CEPH_LIB" + EXTRAOPTS+=" --plugin-dir $CEPH_LIB" + EXTRAOPTS+=" --osd-class-dir $CEPH_LIB" +fi + +function setup() { + teardown + mkdir $DIR +} + +function teardown() { + kill_daemons + rm -fr $DIR +} + +function mon_mkfs() { + local fsid=$(uuidgen) + + ceph-mon \ + --id $MON_ID \ + --fsid $fsid \ + $EXTRAOPTS \ + --mkfs \ + --mon-data=$MON_DIR \ + --mon-initial-members=$MON_ID \ + --mon-host=$CEPH_MON \ + "$@" +} + +function mon_run() { + ceph-mon \ + --id $MON_ID \ + --chdir= \ + --mon-osd-full-ratio=.99 \ + --mon-data-avail-crit=1 \ + $EXTRAOPTS \ + --mon-data=$MON_DIR \ + --log-file=$MON_DIR/log \ + --mon-cluster-log-file=$MON_DIR/log \ + --run-dir=$MON_DIR \ + --pid-file=$MON_DIR/pidfile \ + --public-addr $CEPH_MON \ + "$@" +} + +function kill_daemons() { + for pidfile in $(find $DIR -name pidfile) ; do + pid=$(cat $pidfile) + for try in 0 1 1 1 2 3 ; do + kill $pid || break + sleep $try + done + done +} + +function auth_none() { + mon_mkfs --auth-supported=none + + ceph-mon \ + --id $MON_ID \ + --mon-osd-full-ratio=.99 \ + --mon-data-avail-crit=1 \ + $EXTRAOPTS \ + --mon-data=$MON_DIR \ + --extract-monmap $MON_DIR/monmap + + [ -f $MON_DIR/monmap ] || return 1 + + [ ! -f $MON_DIR/keyring ] || return 1 + + mon_run --auth-supported=none + + timeout $TIMEOUT ceph --mon-host $CEPH_MON mon stat || return 1 +} + +function auth_cephx_keyring() { + cat > $DIR/keyring <&1 | tee $DIR/makedir.log + grep 'toodeep.*No such file' $DIR/makedir.log > /dev/null + rm $DIR/makedir.log + + # an empty directory does not mean the mon exists + mkdir $MON_DIR + mon_mkfs --auth-supported=none 2>&1 | tee $DIR/makedir.log + ! grep "$MON_DIR already exists" $DIR/makedir.log || return 1 +} + +function idempotent() { + mon_mkfs --auth-supported=none + mon_mkfs --auth-supported=none 2>&1 | tee $DIR/makedir.log + grep "'$MON_DIR' already exists" $DIR/makedir.log > /dev/null || return 1 +} + +function run() { + local actions + actions+="makedir " + actions+="idempotent " + actions+="auth_cephx_key " + actions+="auth_cephx_keyring " + actions+="auth_none " + for action in $actions ; do + setup + $action || return 1 + teardown + done +} + +run + +# Local Variables: +# compile-command: "cd ../.. ; make TESTS=test/mon/mkfs.sh check" +# End: diff --git a/qa/standalone/mon/mon-bind.sh b/qa/standalone/mon/mon-bind.sh new file mode 100755 index 00000000000..c0e0e0142fe --- /dev/null +++ b/qa/standalone/mon/mon-bind.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# +# Copyright (C) 2017 Quantum Corp. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +SOCAT_PIDS=() + +function port_forward() { + local source_port=$1 + local target_port=$2 + + socat TCP-LISTEN:${source_port},fork,reuseaddr TCP:localhost:${target_port} & + SOCAT_PIDS+=( $! ) +} + +function cleanup() { + for p in "${SOCAT_PIDS[@]}"; do + kill $p + done + SOCAT_PIDS=() +} + +trap cleanup SIGTERM SIGKILL SIGQUIT SIGINT + +function run() { + local dir=$1 + shift + + export MON_IP=127.0.0.1 + export MONA_PUBLIC=7132 # git grep '\<7132\>' ; there must be only one + export MONB_PUBLIC=7133 # git grep '\<7133\>' ; there must be only one + export MONC_PUBLIC=7134 # git grep '\<7134\>' ; there must be only one + export MONA_BIND=7135 # git grep '\<7135\>' ; there must be only one + export MONB_BIND=7136 # git grep '\<7136\>' ; there must be only one + export MONC_BIND=7137 # git grep '\<7137\>' ; there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir && cleanup || { cleanup; return 1; } + teardown $dir + done +} + +function TEST_mon_client_connect_fails() { + local dir=$1 + + # start the mon with a public-bind-addr that is different + # from the public-addr. + CEPH_ARGS+="--mon-initial-members=a " + CEPH_ARGS+="--mon-host=${MON_IP}:${MONA_PUBLIC} " + run_mon_no_pool $dir a --mon-host=${MON_IP}:${MONA_PUBLIC} --public-bind-addr=${MON_IP}:${MONA_BIND} || return 1 + + # now attempt to ping it that should fail. + timeout 3 ceph ping mon.a || return 0 + return 1 +} + +function TEST_mon_client_connect() { + local dir=$1 + + # start the mon with a public-bind-addr that is different + # from the public-addr. + CEPH_ARGS+="--mon-initial-members=a " + CEPH_ARGS+="--mon-host=${MON_IP}:${MONA_PUBLIC} " + run_mon_no_pool $dir a --mon-host=${MON_IP}:${MONA_PUBLIC} --public-bind-addr=${MON_IP}:${MONA_BIND} || return 1 + + # now forward the public port to the bind port. + port_forward ${MONA_PUBLIC} ${MONA_BIND} + + # attempt to connect. we expect that to work + ceph ping mon.a || return 1 +} + +function TEST_mon_quorum() { + local dir=$1 + + # start the mon with a public-bind-addr that is different + # from the public-addr. + CEPH_ARGS+="--mon-initial-members=a,b,c " + CEPH_ARGS+="--mon-host=${MON_IP}:${MONA_PUBLIC},${MON_IP}:${MONB_PUBLIC},${MON_IP}:${MONC_PUBLIC} " + run_mon_no_pool $dir a --public-addr=${MON_IP}:${MONA_PUBLIC} --public-bind-addr=${MON_IP}:${MONA_BIND} || return 1 + run_mon_no_pool $dir b --public-addr=${MON_IP}:${MONB_PUBLIC} --public-bind-addr=${MON_IP}:${MONB_BIND} || return 1 + run_mon_no_pool $dir c --public-addr=${MON_IP}:${MONC_PUBLIC} --public-bind-addr=${MON_IP}:${MONC_BIND} || return 1 + + # now forward the public port to the bind port. + port_forward ${MONA_PUBLIC} ${MONA_BIND} + port_forward ${MONB_PUBLIC} ${MONB_BIND} + port_forward ${MONC_PUBLIC} ${MONC_BIND} + + # expect monmap to contain 3 monitors (a, b, and c) + jqinput="$(ceph mon_status --format=json 2>/dev/null)" + jq_success "$jqinput" '.monmap.mons | length == 3' || return 1 + + # quorum should form + wait_for_quorum 300 3 || return 1 + # expect quorum to have all three monitors + jqfilter='.quorum | length == 3' + jq_success "$jqinput" "$jqfilter" || return 1 +} + +function TEST_put_get() { + local dir=$1 + + # start the mon with a public-bind-addr that is different + # from the public-addr. + CEPH_ARGS+="--mon-initial-members=a,b,c " + CEPH_ARGS+="--mon-host=${MON_IP}:${MONA_PUBLIC},${MON_IP}:${MONB_PUBLIC},${MON_IP}:${MONC_PUBLIC} " + run_mon_no_pool $dir a --public-addr=${MON_IP}:${MONA_PUBLIC} --public-bind-addr=${MON_IP}:${MONA_BIND} || return 1 + run_mon_no_pool $dir b --public-addr=${MON_IP}:${MONB_PUBLIC} --public-bind-addr=${MON_IP}:${MONB_BIND} || return 1 + run_mon_no_pool $dir c --public-addr=${MON_IP}:${MONC_PUBLIC} --public-bind-addr=${MON_IP}:${MONC_BIND} || return 1 + + # now forward the public port to the bind port. + port_forward ${MONA_PUBLIC} ${MONA_BIND} + port_forward ${MONB_PUBLIC} ${MONB_BIND} + port_forward ${MONC_PUBLIC} ${MONC_BIND} + + # quorum should form + wait_for_quorum 300 3 || return 1 + + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + run_osd $dir 1 || return 1 + run_osd $dir 2 || return 1 + + ceph osd pool create hello 8 || return 1 + + echo "hello world" > $dir/hello + rados --pool hello put foo $dir/hello || return 1 + rados --pool hello get foo $dir/hello2 || return 1 + diff $dir/hello $dir/hello2 || return 1 +} + +main mon-bind "$@" diff --git a/qa/standalone/mon/mon-created-time.sh b/qa/standalone/mon/mon-created-time.sh new file mode 100755 index 00000000000..0955803ae74 --- /dev/null +++ b/qa/standalone/mon/mon-created-time.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# +# Copyright (C) 2015 SUSE LINUX GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7125" # git grep '\<7125\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + +function TEST_mon_created_time() { + local dir=$1 + + run_mon $dir a || return 1 + + ceph mon dump || return 1 + + if test "$(ceph mon dump 2>/dev/null | sed -n '/created/p' | awk '{print $NF}')"x = ""x ; then + return 1 + fi + + if test "$(ceph mon dump 2>/dev/null | sed -n '/created/p' | awk '{print $NF}')"x = "0.000000"x ; then + return 1 + fi +} + +main mon-created-time "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && test/mon/mon-created-time.sh" +# End: diff --git a/qa/standalone/mon/mon-handle-forward.sh b/qa/standalone/mon/mon-handle-forward.sh new file mode 100755 index 00000000000..e3b539bf7b7 --- /dev/null +++ b/qa/standalone/mon/mon-handle-forward.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Copyright (C) 2013 Cloudwatt +# Copyright (C) 2014,2015 Red Hat +# +# Author: Loic Dachary +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + + setup $dir || return 1 + + MONA=127.0.0.1:7300 + MONB=127.0.0.1:7301 + ( + FSID=$(uuidgen) + export CEPH_ARGS + CEPH_ARGS+="--fsid=$FSID --auth-supported=none " + CEPH_ARGS+="--mon-initial-members=a,b --mon-host=$MONA,$MONB " + run_mon $dir a --public-addr $MONA || return 1 + run_mon $dir b --public-addr $MONB || return 1 + ) + + timeout 360 ceph --mon-host $MONA mon stat || return 1 + # check that MONB is indeed a peon + ceph --admin-daemon $(get_asok_path mon.b) mon_status | + grep '"peon"' || return 1 + # when the leader ( MONA ) is used, there is no message forwarding + ceph --mon-host $MONA osd pool create POOL1 12 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1 + grep 'mon_command(.*"POOL1"' $dir/a/mon.a.log + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.b) log flush || return 1 + grep 'mon_command(.*"POOL1"' $dir/mon.b.log && return 1 + # when the peon ( MONB ) is used, the message is forwarded to the leader + ceph --mon-host $MONB osd pool create POOL2 12 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.b) log flush || return 1 + grep 'forward_request.*mon_command(.*"POOL2"' $dir/mon.b.log + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1 + grep ' forward(mon_command(.*"POOL2"' $dir/mon.a.log + # forwarded messages must retain features from the original connection + features=$(sed -n -e 's|.*127.0.0.1:0.*accept features \([0-9][0-9]*\)|\1|p' < \ + $dir/mon.b.log) + grep ' forward(mon_command(.*"POOL2".*con_features '$features $dir/mon.a.log + + teardown $dir || return 1 +} + +main mon-handle-forward "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 TESTS=test/mon/mon-handle-forward.sh check" +# End: diff --git a/qa/standalone/mon/mon-ping.sh b/qa/standalone/mon/mon-ping.sh new file mode 100755 index 00000000000..9574f5f0f9b --- /dev/null +++ b/qa/standalone/mon/mon-ping.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# +# Copyright (C) 2015 SUSE LINUX GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7119" # git grep '\<7119\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + +function TEST_mon_ping() { + local dir=$1 + + run_mon $dir a || return 1 + + ceph ping mon.a || return 1 +} + +main mon-ping "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && test/mon/mon-ping.sh" +# End: diff --git a/qa/standalone/mon/mon-scrub.sh b/qa/standalone/mon/mon-scrub.sh new file mode 100755 index 00000000000..b40a6bc6740 --- /dev/null +++ b/qa/standalone/mon/mon-scrub.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# +# Copyright (C) 2014 Cloudwatt +# Copyright (C) 2014, 2015 Red Hat +# +# Author: Loic Dachary +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7120" # git grep '\<7120\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + +function TEST_mon_scrub() { + local dir=$1 + + run_mon $dir a || return 1 + + ceph mon scrub || return 1 +} + +main mon-scrub "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && test/mon/mon-scrub.sh" +# End: diff --git a/qa/standalone/mon/osd-crush.sh b/qa/standalone/mon/osd-crush.sh new file mode 100755 index 00000000000..cc754af2cf1 --- /dev/null +++ b/qa/standalone/mon/osd-crush.sh @@ -0,0 +1,228 @@ +#!/bin/bash +# +# Copyright (C) 2014 Cloudwatt +# Copyright (C) 2014, 2015 Red Hat +# +# Author: Loic Dachary +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7104" # git grep '\<7104\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | ${SED} -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + +function TEST_crush_rule_create_simple() { + local dir=$1 + + run_mon $dir a || return 1 + + ceph --format xml osd crush rule dump replicated_rule | \ + egrep 'take[^<]+default' | \ + grep 'choose_firstn0osd' || return 1 + local ruleset=ruleset0 + local root=host1 + ceph osd crush add-bucket $root host + local failure_domain=osd + ceph osd crush rule create-simple $ruleset $root $failure_domain || return 1 + ceph osd crush rule create-simple $ruleset $root $failure_domain 2>&1 | \ + grep "$ruleset already exists" || return 1 + ceph --format xml osd crush rule dump $ruleset | \ + egrep 'take[^<]+'$root'' | \ + grep 'choose_firstn0'$failure_domain'' || return 1 + ceph osd crush rule rm $ruleset || return 1 +} + +function TEST_crush_rule_dump() { + local dir=$1 + + run_mon $dir a || return 1 + + local ruleset=ruleset1 + ceph osd crush rule create-erasure $ruleset || return 1 + test $(ceph --format json osd crush rule dump $ruleset | \ + jq ".rule_name == \"$ruleset\"") == true || return 1 + test $(ceph --format json osd crush rule dump | \ + jq "map(select(.rule_name == \"$ruleset\")) | length == 1") == true || return 1 + ! ceph osd crush rule dump non_existent_ruleset || return 1 + ceph osd crush rule rm $ruleset || return 1 +} + +function TEST_crush_rule_rm() { + local ruleset=erasure2 + + run_mon $dir a || return 1 + + ceph osd crush rule create-erasure $ruleset default || return 1 + ceph osd crush rule ls | grep $ruleset || return 1 + ceph osd crush rule rm $ruleset || return 1 + ! ceph osd crush rule ls | grep $ruleset || return 1 +} + +function TEST_crush_rule_create_erasure() { + local dir=$1 + + run_mon $dir a || return 1 + # should have at least one OSD + run_osd $dir 0 || return 1 + + local ruleset=ruleset3 + # + # create a new ruleset with the default profile, implicitly + # + ceph osd crush rule create-erasure $ruleset || return 1 + ceph osd crush rule create-erasure $ruleset 2>&1 | \ + grep "$ruleset already exists" || return 1 + ceph --format xml osd crush rule dump $ruleset | \ + egrep 'take[^<]+default' | \ + grep 'chooseleaf_indep0host' || return 1 + ceph osd crush rule rm $ruleset || return 1 + ! ceph osd crush rule ls | grep $ruleset || return 1 + # + # create a new ruleset with the default profile, explicitly + # + ceph osd crush rule create-erasure $ruleset default || return 1 + ceph osd crush rule ls | grep $ruleset || return 1 + ceph osd crush rule rm $ruleset || return 1 + ! ceph osd crush rule ls | grep $ruleset || return 1 + # + # create a new ruleset and the default profile, implicitly + # + ceph osd erasure-code-profile rm default || return 1 + ! ceph osd erasure-code-profile ls | grep default || return 1 + ceph osd crush rule create-erasure $ruleset || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1 + grep 'profile set default' $dir/mon.a.log || return 1 + ceph osd erasure-code-profile ls | grep default || return 1 + ceph osd crush rule rm $ruleset || return 1 + ! ceph osd crush rule ls | grep $ruleset || return 1 +} + +function check_ruleset_id_match_rule_id() { + local rule_name=$1 + rule_id=`ceph osd crush rule dump $rule_name | grep "\"rule_id\":" | awk -F ":|," '{print int($2)}'` + ruleset_id=`ceph osd crush rule dump $rule_name | grep "\"ruleset\":"| awk -F ":|," '{print int($2)}'` + test $ruleset_id = $rule_id || return 1 +} + +function generate_manipulated_rules() { + local dir=$1 + ceph osd crush add-bucket $root host + ceph osd crush rule create-simple test_rule1 $root osd firstn || return 1 + ceph osd crush rule create-simple test_rule2 $root osd firstn || return 1 + ceph osd getcrushmap -o $dir/original_map + crushtool -d $dir/original_map -o $dir/decoded_original_map + #manipulate the rulesets , to make the rule_id != ruleset_id + ${SED} -i 's/ruleset 0/ruleset 3/' $dir/decoded_original_map + ${SED} -i 's/ruleset 2/ruleset 0/' $dir/decoded_original_map + ${SED} -i 's/ruleset 1/ruleset 2/' $dir/decoded_original_map + + crushtool -c $dir/decoded_original_map -o $dir/new_map + ceph osd setcrushmap -i $dir/new_map + + ceph osd crush rule dump +} + +function TEST_crush_ruleset_match_rule_when_creating() { + local dir=$1 + + run_mon $dir a || return 1 + + local root=host1 + + generate_manipulated_rules $dir + + ceph osd crush rule create-simple special_rule_simple $root osd firstn || return 1 + + ceph osd crush rule dump + #show special_rule_simple has same rule_id and ruleset_id + check_ruleset_id_match_rule_id special_rule_simple || return 1 +} + +function TEST_add_ruleset_failed() { + local dir=$1 + + run_mon $dir a || return 1 + + local root=host1 + + ceph osd crush add-bucket $root host + ceph osd crush rule create-simple test_rule1 $root osd firstn || return 1 + ceph osd crush rule create-simple test_rule2 $root osd firstn || return 1 + ceph osd getcrushmap > $dir/crushmap || return 1 + crushtool --decompile $dir/crushmap > $dir/crushmap.txt || return 1 + for i in $(seq 3 255) + do + cat <> $dir/crushmap.txt + crushtool --compile $dir/crushmap.txt -o $dir/crushmap || return 1 + ceph osd setcrushmap -i $dir/crushmap || return 1 + ceph osd crush rule create-simple test_rule_nospace $root osd firstn 2>&1 | grep "Error ENOSPC" || return 1 + +} + +function TEST_crush_rename_bucket() { + local dir=$1 + + run_mon $dir a || return 1 + + ceph osd crush add-bucket host1 host + ceph osd tree + ! ceph osd tree | grep host2 || return 1 + ceph osd crush rename-bucket host1 host2 || return 1 + ceph osd tree + ceph osd tree | grep host2 || return 1 + ceph osd crush rename-bucket host1 host2 || return 1 # idempotency + ceph osd crush rename-bucket nonexistent something 2>&1 | grep "Error ENOENT" || return 1 +} + +function TEST_crush_reject_empty() { + local dir=$1 + run_mon $dir a || return 1 + # should have at least one OSD + run_osd $dir 0 || return 1 + + local empty_map=$dir/empty_map + :> $empty_map.txt + crushtool -c $empty_map.txt -o $empty_map.map || return 1 + expect_failure $dir "Error EINVAL" \ + ceph osd setcrushmap -i $empty_map.map || return 1 +} + +main osd-crush "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && test/mon/osd-crush.sh" +# End: diff --git a/qa/standalone/mon/osd-erasure-code-profile.sh b/qa/standalone/mon/osd-erasure-code-profile.sh new file mode 100755 index 00000000000..6bfc2587f8b --- /dev/null +++ b/qa/standalone/mon/osd-erasure-code-profile.sh @@ -0,0 +1,229 @@ +#!/bin/bash +# +# Copyright (C) 2014 Cloudwatt +# Copyright (C) 2014, 2015 Red Hat +# +# Author: Loic Dachary +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7220" # git grep '\<7220\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + +function TEST_set() { + local dir=$1 + local id=$2 + + run_mon $dir a || return 1 + + local profile=myprofile + # + # no key=value pairs : use the default configuration + # + ceph osd erasure-code-profile set $profile 2>&1 || return 1 + ceph osd erasure-code-profile get $profile | \ + grep plugin=jerasure || return 1 + ceph osd erasure-code-profile rm $profile + # + # key=value pairs override the default + # + ceph osd erasure-code-profile set $profile \ + key=value plugin=isa || return 1 + ceph osd erasure-code-profile get $profile | \ + grep -e key=value -e plugin=isa || return 1 + # + # --force is required to override an existing profile + # + ! ceph osd erasure-code-profile set $profile > $dir/out 2>&1 || return 1 + grep 'will not override' $dir/out || return 1 + ceph osd erasure-code-profile set $profile key=other --force || return 1 + ceph osd erasure-code-profile get $profile | \ + grep key=other || return 1 + + ceph osd erasure-code-profile rm $profile # cleanup +} + +function TEST_ls() { + local dir=$1 + local id=$2 + + run_mon $dir a || return 1 + + local profile=myprofile + ! ceph osd erasure-code-profile ls | grep $profile || return 1 + ceph osd erasure-code-profile set $profile 2>&1 || return 1 + ceph osd erasure-code-profile ls | grep $profile || return 1 + ceph --format xml osd erasure-code-profile ls | \ + grep "$profile" || return 1 + + ceph osd erasure-code-profile rm $profile # cleanup +} + +function TEST_rm() { + local dir=$1 + local id=$2 + + run_mon $dir a || return 1 + + local profile=myprofile + ceph osd erasure-code-profile set $profile 2>&1 || return 1 + ceph osd erasure-code-profile ls | grep $profile || return 1 + ceph osd erasure-code-profile rm $profile || return 1 + ! ceph osd erasure-code-profile ls | grep $profile || return 1 + ceph osd erasure-code-profile rm WRONG 2>&1 | \ + grep "WRONG does not exist" || return 1 + + ceph osd erasure-code-profile set $profile || return 1 + ceph osd pool create poolname 12 12 erasure $profile || return 1 + ! ceph osd erasure-code-profile rm $profile > $dir/out 2>&1 || return 1 + grep "poolname.*using.*$profile" $dir/out || return 1 + ceph osd pool delete poolname poolname --yes-i-really-really-mean-it || return 1 + ceph osd erasure-code-profile rm $profile || return 1 + + ceph osd erasure-code-profile rm $profile # cleanup +} + +function TEST_get() { + local dir=$1 + local id=$2 + + run_mon $dir a || return 1 + + local default_profile=default + ceph osd erasure-code-profile get $default_profile | \ + grep plugin=jerasure || return 1 + ceph --format xml osd erasure-code-profile get $default_profile | \ + grep 'jerasure' || return 1 + ! ceph osd erasure-code-profile get WRONG > $dir/out 2>&1 || return 1 + grep -q "unknown erasure code profile 'WRONG'" $dir/out || return 1 +} + +function TEST_set_idempotent() { + local dir=$1 + local id=$2 + + run_mon $dir a || return 1 + # + # The default profile is set using a code path different from + # ceph osd erasure-code-profile set: verify that it is idempotent, + # as if it was using the same code path. + # + ceph osd erasure-code-profile set default k=2 m=1 2>&1 || return 1 + local profile + # + # Because plugin=jerasure is the default, it uses a slightly + # different code path where defaults (m=1 for instance) are added + # implicitly. + # + profile=profileidempotent1 + ! ceph osd erasure-code-profile ls | grep $profile || return 1 + ceph osd erasure-code-profile set $profile k=2 crush-failure-domain=osd 2>&1 || return 1 + ceph osd erasure-code-profile ls | grep $profile || return 1 + ceph osd erasure-code-profile set $profile k=2 crush-failure-domain=osd 2>&1 || return 1 + ceph osd erasure-code-profile rm $profile # cleanup + + # + # In the general case the profile is exactly what is on + # + profile=profileidempotent2 + ! ceph osd erasure-code-profile ls | grep $profile || return 1 + ceph osd erasure-code-profile set $profile plugin=lrc k=4 m=2 l=3 crush-failure-domain=osd 2>&1 || return 1 + ceph osd erasure-code-profile ls | grep $profile || return 1 + ceph osd erasure-code-profile set $profile plugin=lrc k=4 m=2 l=3 crush-failure-domain=osd 2>&1 || return 1 + ceph osd erasure-code-profile rm $profile # cleanup +} + +function TEST_format_invalid() { + local dir=$1 + + local profile=profile + # osd_pool_default_erasure-code-profile is + # valid JSON but not of the expected type + run_mon $dir a \ + --osd_pool_default_erasure-code-profile 1 || return 1 + ! ceph osd erasure-code-profile set $profile > $dir/out 2>&1 || return 1 + cat $dir/out + grep 'must be a JSON object' $dir/out || return 1 +} + +function TEST_format_json() { + local dir=$1 + + # osd_pool_default_erasure-code-profile is JSON + expected='"plugin":"isa"' + run_mon $dir a \ + --osd_pool_default_erasure-code-profile "{$expected}" || return 1 + ceph --format json osd erasure-code-profile get default | \ + grep "$expected" || return 1 +} + +function TEST_format_plain() { + local dir=$1 + + # osd_pool_default_erasure-code-profile is plain text + expected='"plugin":"isa"' + run_mon $dir a \ + --osd_pool_default_erasure-code-profile "plugin=isa" || return 1 + ceph --format json osd erasure-code-profile get default | \ + grep "$expected" || return 1 +} + +function TEST_profile_k_sanity() { + local dir=$1 + local profile=profile-sanity + + run_mon $dir a || return 1 + + expect_failure $dir 'k must be a multiple of (k + m) / l' \ + ceph osd erasure-code-profile set $profile \ + plugin=lrc \ + l=1 \ + k=1 \ + m=1 || return 1 + + if erasure_code_plugin_exists isa ; then + expect_failure $dir 'k=1 must be >= 2' \ + ceph osd erasure-code-profile set $profile \ + plugin=isa \ + k=1 \ + m=1 || return 1 + else + echo "SKIP because plugin isa has not been built" + fi + + expect_failure $dir 'k=1 must be >= 2' \ + ceph osd erasure-code-profile set $profile \ + plugin=jerasure \ + k=1 \ + m=1 || return 1 +} + +main osd-erasure-code-profile "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && test/mon/osd-erasure-code-profile.sh" +# End: diff --git a/qa/standalone/mon/osd-pool-create.sh b/qa/standalone/mon/osd-pool-create.sh new file mode 100755 index 00000000000..9562209fc51 --- /dev/null +++ b/qa/standalone/mon/osd-pool-create.sh @@ -0,0 +1,220 @@ +#!/bin/bash +# +# Copyright (C) 2013, 2014 Cloudwatt +# Copyright (C) 2014, 2015 Red Hat +# +# Author: Loic Dachary +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7105" # git grep '\<7105\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + +# Before http://tracker.ceph.com/issues/8307 the invalid profile was created +function TEST_erasure_invalid_profile() { + local dir=$1 + run_mon $dir a || return 1 + local poolname=pool_erasure + local notaprofile=not-a-valid-erasure-code-profile + ! ceph osd pool create $poolname 12 12 erasure $notaprofile || return 1 + ! ceph osd erasure-code-profile ls | grep $notaprofile || return 1 +} + +function TEST_erasure_crush_rule() { + local dir=$1 + run_mon $dir a || return 1 + # + # choose the crush ruleset used with an erasure coded pool + # + local crush_ruleset=myruleset + ! ceph osd crush rule ls | grep $crush_ruleset || return 1 + ceph osd crush rule create-erasure $crush_ruleset + ceph osd crush rule ls | grep $crush_ruleset + local poolname + poolname=pool_erasure1 + ! ceph --format json osd dump | grep '"crush_rule":1' || return 1 + ceph osd pool create $poolname 12 12 erasure default $crush_ruleset + ceph --format json osd dump | grep '"crush_rule":1' || return 1 + # + # a crush ruleset by the name of the pool is implicitly created + # + poolname=pool_erasure2 + ceph osd erasure-code-profile set myprofile + ceph osd pool create $poolname 12 12 erasure myprofile + ceph osd crush rule ls | grep $poolname || return 1 + # + # a non existent crush ruleset given in argument is an error + # http://tracker.ceph.com/issues/9304 + # + poolname=pool_erasure3 + ! ceph osd pool create $poolname 12 12 erasure myprofile INVALIDRULESET || return 1 +} + +function TEST_erasure_code_profile_default() { + local dir=$1 + run_mon $dir a || return 1 + ceph osd erasure-code-profile rm default || return 1 + ! ceph osd erasure-code-profile ls | grep default || return 1 + ceph osd pool create $poolname 12 12 erasure default + ceph osd erasure-code-profile ls | grep default || return 1 +} + +function TEST_erasure_crush_stripe_unit() { + local dir=$1 + # the default stripe unit is used to initialize the pool + run_mon $dir a --public-addr $CEPH_MON + stripe_unit=$(ceph-conf --show-config-value osd_pool_erasure_code_stripe_unit) + eval local $(ceph osd erasure-code-profile get myprofile | grep k=) + stripe_width = $((stripe_unit * k)) + ceph osd pool create pool_erasure 12 12 erasure + ceph --format json osd dump | tee $dir/osd.json + grep '"stripe_width":'$stripe_width $dir/osd.json > /dev/null || return 1 +} + +function TEST_erasure_crush_stripe_unit_padded() { + local dir=$1 + # setting osd_pool_erasure_code_stripe_unit modifies the stripe_width + # and it is padded as required by the default plugin + profile+=" plugin=jerasure" + profile+=" technique=reed_sol_van" + k=4 + profile+=" k=$k" + profile+=" m=2" + actual_stripe_unit=2048 + desired_stripe_unit=$((actual_stripe_unit - 1)) + actual_stripe_width=$((actual_stripe_unit * k)) + run_mon $dir a \ + --osd_pool_erasure_code_stripe_unit $desired_stripe_unit \ + --osd_pool_default_erasure_code_profile "$profile" || return 1 + ceph osd pool create pool_erasure 12 12 erasure + ceph osd dump | tee $dir/osd.json + grep "stripe_width $actual_stripe_width" $dir/osd.json > /dev/null || return 1 +} + +function TEST_erasure_code_pool() { + local dir=$1 + run_mon $dir a || return 1 + ceph --format json osd dump > $dir/osd.json + local expected='"erasure_code_profile":"default"' + ! grep "$expected" $dir/osd.json || return 1 + ceph osd pool create erasurecodes 12 12 erasure + ceph --format json osd dump | tee $dir/osd.json + grep "$expected" $dir/osd.json > /dev/null || return 1 + + ceph osd pool create erasurecodes 12 12 erasure 2>&1 | \ + grep 'already exists' || return 1 + ceph osd pool create erasurecodes 12 12 2>&1 | \ + grep 'cannot change to type replicated' || return 1 +} + +function TEST_replicated_pool_with_ruleset() { + local dir=$1 + run_mon $dir a + local ruleset=ruleset0 + local root=host1 + ceph osd crush add-bucket $root host + local failure_domain=osd + local poolname=mypool + ceph osd crush rule create-simple $ruleset $root $failure_domain || return 1 + ceph osd crush rule ls | grep $ruleset + ceph osd pool create $poolname 12 12 replicated $ruleset 2>&1 | \ + grep "pool 'mypool' created" || return 1 + rule_id=`ceph osd crush rule dump $ruleset | grep "rule_id" | awk -F[' ':,] '{print $4}'` + ceph osd pool get $poolname crush_rule 2>&1 | \ + grep "crush_rule: $rule_id" || return 1 + #non-existent crush ruleset + ceph osd pool create newpool 12 12 replicated non-existent 2>&1 | \ + grep "doesn't exist" || return 1 +} + +function TEST_erasure_code_pool_lrc() { + local dir=$1 + run_mon $dir a || return 1 + + ceph osd erasure-code-profile set LRCprofile \ + plugin=lrc \ + mapping=DD_ \ + layers='[ [ "DDc", "" ] ]' || return 1 + + ceph --format json osd dump > $dir/osd.json + local expected='"erasure_code_profile":"LRCprofile"' + local poolname=erasurecodes + ! grep "$expected" $dir/osd.json || return 1 + ceph osd pool create $poolname 12 12 erasure LRCprofile + ceph --format json osd dump | tee $dir/osd.json + grep "$expected" $dir/osd.json > /dev/null || return 1 + ceph osd crush rule ls | grep $poolname || return 1 +} + +function TEST_replicated_pool() { + local dir=$1 + run_mon $dir a || return 1 + ceph osd pool create replicated 12 12 replicated replicated_rule 2>&1 | \ + grep "pool 'replicated' created" || return 1 + ceph osd pool create replicated 12 12 replicated replicated_rule 2>&1 | \ + grep 'already exists' || return 1 + # default is replicated + ceph osd pool create replicated1 12 12 2>&1 | \ + grep "pool 'replicated1' created" || return 1 + # default is replicated, pgp_num = pg_num + ceph osd pool create replicated2 12 2>&1 | \ + grep "pool 'replicated2' created" || return 1 + ceph osd pool create replicated 12 12 erasure 2>&1 | \ + grep 'cannot change to type erasure' || return 1 +} + +function TEST_no_pool_delete() { + local dir=$1 + run_mon $dir a || return 1 + ceph osd pool create foo 1 || return 1 + ceph tell mon.a injectargs -- --no-mon-allow-pool-delete || return 1 + ! ceph osd pool delete foo foo --yes-i-really-really-mean-it || return 1 + ceph tell mon.a injectargs -- --mon-allow-pool-delete || return 1 + ceph osd pool delete foo foo --yes-i-really-really-mean-it || return 1 +} + +function TEST_utf8_cli() { + local dir=$1 + run_mon $dir a || return 1 + # Hopefully it's safe to include literal UTF-8 characters to test + # the fix for http://tracker.ceph.com/issues/7387. If it turns out + # to not be OK (when is the default encoding *not* UTF-8?), maybe + # the character '黄' can be replaced with the escape $'\xe9\xbb\x84' + ceph osd pool create 黄 1024 2>&1 | \ + grep "pool '黄' created" || return 1 + ceph osd lspools 2>&1 | \ + grep "黄" || return 1 + ceph -f json-pretty osd dump | \ + python -c "import json; import sys; json.load(sys.stdin)" || return 1 + ceph osd pool delete 黄 黄 --yes-i-really-really-mean-it +} + +main osd-pool-create "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && test/mon/osd-pool-create.sh" +# End: diff --git a/qa/standalone/mon/test_pool_quota.sh b/qa/standalone/mon/test_pool_quota.sh new file mode 100755 index 00000000000..dcf89fd26e2 --- /dev/null +++ b/qa/standalone/mon/test_pool_quota.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# +# Generic pool quota test +# + +# Includes + + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:17108" # git grep '\<17108\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + $func $dir || return 1 + done +} + +function TEST_pool_quota() { + local dir=$1 + setup $dir || return 1 + + run_mon $dir a || return 1 + run_osd $dir 0 || return 1 + run_osd $dir 1 || return 1 + run_osd $dir 2 || return 1 + + local poolname=testquoa + ceph osd pool create $poolname 20 + local objects=`ceph df detail | grep -w $poolname|awk '{print $3}'` + local bytes=`ceph df detail | grep -w $poolname|awk '{print $4}'` + + echo $objects + echo $bytes + if [ $objects != 'N/A' ] || [ $bytes != 'N/A' ] ; + then + return 1 + fi + + ceph osd pool set-quota $poolname max_objects 1000 + ceph osd pool set-quota $poolname max_bytes 1024 + + objects=`ceph df detail | grep -w $poolname|awk '{print $3}'` + bytes=`ceph df detail | grep -w $poolname|awk '{print $4}'` + + if [ $objects != '1000' ] || [ $bytes != '1024' ] ; + then + return 1 + fi + + ceph osd pool delete $poolname $poolname --yes-i-really-really-mean-it + teardown $dir || return 1 +} + +main testpoolquota diff --git a/qa/standalone/osd/osd-bench.sh b/qa/standalone/osd/osd-bench.sh new file mode 100755 index 00000000000..59a6f8d8d7e --- /dev/null +++ b/qa/standalone/osd/osd-bench.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# +# Copyright (C) 2014 Cloudwatt +# Copyright (C) 2014, 2015 Red Hat +# +# Author: Loic Dachary +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7106" # git grep '\<7106\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + +function TEST_bench() { + local dir=$1 + + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + + local osd_bench_small_size_max_iops=$(CEPH_ARGS='' ceph-conf \ + --show-config-value osd_bench_small_size_max_iops) + local osd_bench_large_size_max_throughput=$(CEPH_ARGS='' ceph-conf \ + --show-config-value osd_bench_large_size_max_throughput) + local osd_bench_max_block_size=$(CEPH_ARGS='' ceph-conf \ + --show-config-value osd_bench_max_block_size) + local osd_bench_duration=$(CEPH_ARGS='' ceph-conf \ + --show-config-value osd_bench_duration) + + # + # block size too high + # + expect_failure $dir osd_bench_max_block_size \ + ceph tell osd.0 bench 1024 $((osd_bench_max_block_size + 1)) || return 1 + + # + # count too high for small (< 1MB) block sizes + # + local bsize=1024 + local max_count=$(($bsize * $osd_bench_duration * $osd_bench_small_size_max_iops)) + expect_failure $dir bench_small_size_max_iops \ + ceph tell osd.0 bench $(($max_count + 1)) $bsize || return 1 + + # + # count too high for large (>= 1MB) block sizes + # + local bsize=$((1024 * 1024 + 1)) + local max_count=$(($osd_bench_large_size_max_throughput * $osd_bench_duration)) + expect_failure $dir osd_bench_large_size_max_throughput \ + ceph tell osd.0 bench $(($max_count + 1)) $bsize || return 1 + + # + # default values should work + # + ceph tell osd.0 bench || return 1 + + # + # test object_size < block_size + ceph tell osd.0 bench 10 14456 4444 3 + # + + # + # test object_size < block_size & object_size = 0(default value) + # + ceph tell osd.0 bench 1 14456 +} + +main osd-bench "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && test/osd/osd-bench.sh" +# End: diff --git a/qa/standalone/osd/osd-config.sh b/qa/standalone/osd/osd-config.sh new file mode 100755 index 00000000000..d2dfe992848 --- /dev/null +++ b/qa/standalone/osd/osd-config.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# +# Copyright (C) 2014 Cloudwatt +# Copyright (C) 2014, 2015 Red Hat +# +# Author: Loic Dachary +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7100" # git grep '\<7100\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + +function TEST_config_init() { + local dir=$1 + + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + local advance=1000 + local stale=1000 + local cache=500 + run_osd $dir 0 \ + --osd-map-max-advance $advance \ + --osd-map-cache-size $cache \ + --osd-pg-epoch-persisted-max-stale $stale \ + || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1 + grep 'is not > osd_map_max_advance' $dir/osd.0.log || return 1 + grep 'is not > osd_pg_epoch_persisted_max_stale' $dir/osd.0.log || return 1 +} + +function TEST_config_track() { + local dir=$1 + + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + + local osd_map_cache_size=$(CEPH_ARGS='' ceph-conf \ + --show-config-value osd_map_cache_size) + local osd_map_max_advance=$(CEPH_ARGS='' ceph-conf \ + --show-config-value osd_map_max_advance) + local osd_pg_epoch_persisted_max_stale=$(CEPH_ARGS='' ceph-conf \ + --show-config-value osd_pg_epoch_persisted_max_stale) + # + # lower cache_size under max_advance to trigger the warning + # + ! grep 'is not > osd_map_max_advance' $dir/osd.0.log || return 1 + local cache=$(($osd_map_max_advance / 2)) + ceph tell osd.0 injectargs "--osd-map-cache-size $cache" || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1 + grep 'is not > osd_map_max_advance' $dir/osd.0.log || return 1 + rm $dir/osd.0.log + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log reopen || return 1 + + # + # reset cache_size to the default and assert that it does not trigger the warning + # + ! grep 'is not > osd_map_max_advance' $dir/osd.0.log || return 1 + local cache=$osd_map_cache_size + ceph tell osd.0 injectargs "--osd-map-cache-size $cache" || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1 + ! grep 'is not > osd_map_max_advance' $dir/osd.0.log || return 1 + rm $dir/osd.0.log + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log reopen || return 1 + + # + # increase the osd_map_max_advance above the default cache_size + # + ! grep 'is not > osd_map_max_advance' $dir/osd.0.log || return 1 + local advance=$(($osd_map_cache_size * 2)) + ceph tell osd.0 injectargs "--osd-map-max-advance $advance" || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1 + grep 'is not > osd_map_max_advance' $dir/osd.0.log || return 1 + rm $dir/osd.0.log + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log reopen || return 1 + + # + # increase the osd_pg_epoch_persisted_max_stale above the default cache_size + # + ! grep 'is not > osd_pg_epoch_persisted_max_stale' $dir/osd.0.log || return 1 + local stale=$(($osd_map_cache_size * 2)) + ceph tell osd.0 injectargs "--osd-pg-epoch-persisted-max-stale $stale" || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1 + grep 'is not > osd_pg_epoch_persisted_max_stale' $dir/osd.0.log || return 1 + rm $dir/osd.0.log + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log reopen || return 1 +} + +main osd-config "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && test/osd/osd-config.sh" +# End: diff --git a/qa/standalone/osd/osd-copy-from.sh b/qa/standalone/osd/osd-copy-from.sh new file mode 100755 index 00000000000..37851fab9a6 --- /dev/null +++ b/qa/standalone/osd/osd-copy-from.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Copyright (C) 2014 Cloudwatt +# Copyright (C) 2014, 2015 Red Hat +# +# Author: Loic Dachary +# Author: Sage Weil +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7111" # git grep '\<7111\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + +function TEST_copy_from() { + local dir=$1 + + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + run_osd $dir 1 || return 1 + + # success + rados -p rbd put foo $(which rados) + rados -p rbd cp foo foo2 + rados -p rbd stat foo2 + + # failure + ceph tell osd.\* injectargs -- --osd-debug-inject-copyfrom-error + ! rados -p rbd cp foo foo3 + ! rados -p rbd stat foo3 + + # success again + ceph tell osd.\* injectargs -- --no-osd-debug-inject-copyfrom-error + ! rados -p rbd cp foo foo3 + rados -p rbd stat foo3 +} + +main osd-copy-from "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && test/osd/osd-bench.sh" +# End: diff --git a/qa/standalone/osd/osd-dup.sh b/qa/standalone/osd/osd-dup.sh new file mode 100755 index 00000000000..a1bd1af5d62 --- /dev/null +++ b/qa/standalone/osd/osd-dup.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7146" # git grep '\<7146\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + # avoid running out of fds in rados bench + CEPH_ARGS+="--filestore_wbthrottle_xfs_ios_hard_limit=900 " + CEPH_ARGS+="--filestore_wbthrottle_btrfs_ios_hard_limit=900 " + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + +function TEST_filestore_to_bluestore() { + local dir=$1 + + local flimit=$(ulimit -n) + if [ $flimit -lt 1536 ]; then + echo "Low open file limit ($flimit), test may fail. Increase to 1536 or higher and retry if that happens." + fi + + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + osd_pid=$(cat $dir/osd.0.pid) + run_osd $dir 1 || return 1 + run_osd $dir 2 || return 1 + + sleep 5 + + ceph osd pool create foo 16 + + # write some objects + rados bench -p foo 10 write -b 4096 --no-cleanup || return 1 + + # kill + while kill $osd_pid; do sleep 1 ; done + ceph osd down 0 + + mv $dir/0 $dir/0.old || return 1 + mkdir $dir/0 || return 1 + ofsid=$(cat $dir/0.old/fsid) + echo "osd fsid $ofsid" + O=$CEPH_ARGS + CEPH_ARGS+="--log-file $dir/cot.log --log-max-recent 0 " + ceph-objectstore-tool --type bluestore --data-path $dir/0 --fsid $ofsid \ + --op mkfs || return 1 + ceph-objectstore-tool --data-path $dir/0.old --target-data-path $dir/0 \ + --op dup || return 1 + CEPH_ARGS=$O + + run_osd_bluestore $dir 0 || return 1 + + while ! ceph osd stat | grep '3 up' ; do sleep 1 ; done + ceph osd metadata 0 | grep bluestore || return 1 + + ceph osd scrub 0 + + # give it some time + sleep 15 + # and make sure mon is sync'ed + flush_pg_stats + + wait_for_clean || return 1 +} + +main osd-dup "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && test/osd/osd-dup.sh" +# End: diff --git a/qa/standalone/osd/osd-fast-mark-down.sh b/qa/standalone/osd/osd-fast-mark-down.sh new file mode 100755 index 00000000000..3d27930e6e6 --- /dev/null +++ b/qa/standalone/osd/osd-fast-mark-down.sh @@ -0,0 +1,133 @@ +#!/bin/bash +# +# Copyright (C) 2016 Piotr Dałek +# Copyright (C) 2014, 2015 Red Hat +# +# Author: Piotr Dałek +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh +MAX_PROPAGATION_TIME=30 + +function run() { + local dir=$1 + shift + rm -f $dir/*.pid + export CEPH_MON="127.0.0.1:7126" # git grep '\<7126\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + # + # Disable device auto class feature for this testing, + # as it will automatically make root clones based on new class types + # and hence affect the down osd counting. + # E.g., + # + # ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY + # -4 3.00000 root default~hdd + # -3 3.00000 host gitbuilder-ceph-rpm-centos7-amd64-basic~hdd + # 0 1.00000 osd.0 down 1.00000 1.00000 + # 1 1.00000 osd.1 up 1.00000 1.00000 + # 2 1.00000 osd.2 up 1.00000 1.00000 + # -1 3.00000 root default + # -2 3.00000 host gitbuilder-ceph-rpm-centos7-amd64-basic + # 0 1.00000 osd.0 down 1.00000 1.00000 + # 1 1.00000 osd.1 up 1.00000 1.00000 + # 2 1.00000 osd.2 up 1.00000 1.00000 + # + CEPH_ARGS+="--osd-class-update-on-start=false " + + OLD_ARGS=$CEPH_ARGS + CEPH_ARGS+="--osd-fast-fail-on-connection-refused=false " + echo "Ensuring old behavior is there..." + test_fast_kill $dir && (echo "OSDs died too early! Old behavior doesn't work." ; return 1) + + CEPH_ARGS=$OLD_ARGS"--osd-fast-fail-on-connection-refused=true " + OLD_ARGS=$CEPH_ARGS + + CEPH_ARGS+="--ms_type=simple" + echo "Testing simple msgr..." + test_fast_kill $dir || return 1 + + CEPH_ARGS=$OLD_ARGS"--ms_type=async" + echo "Testing async msgr..." + test_fast_kill $dir || return 1 + + return 0 + +} + +function test_fast_kill() { + # create cluster with 3 osds + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=3 || return 1 + run_mgr $dir x || return 1 + for oi in {0..2}; do + run_osd $dir $oi || return 1 + pids[$oi]=$(cat $dir/osd.$oi.pid) + done + + # make some objects so osds to ensure connectivity between osds + rados -p rbd bench 10 write -b 4096 --max-objects 128 --no-cleanup + sleep 1 + + killid=0 + previd=0 + + # kill random osd and see if after max MAX_PROPAGATION_TIME, the osd count decreased. + for i in {1..2}; do + while [ $killid -eq $previd ]; do + killid=${pids[$RANDOM%${#pids[@]}]} + done + previd=$killid + + kill -9 $killid + time_left=$MAX_PROPAGATION_TIME + down_osds=0 + + while [ $time_left -gt 0 ]; do + sleep 1 + time_left=$[$time_left - 1]; + + grep -m 1 -c -F "ms_handle_refused" $dir/osd.*.log > /dev/null + if [ $? -ne 0 ]; then + continue + fi + + down_osds=$(ceph osd tree | grep -c down) + if [ $down_osds -lt $i ]; then + # osds not marked down yet, try again in a second + continue + elif [ $down_osds -gt $i ]; then + echo Too many \($down_osds\) osds died! + return 1 + else + break + fi + done + + if [ $down_osds -lt $i ]; then + echo Killed the OSD, yet it is not marked down + ceph osd tree + return 1 + fi + done + pkill -SIGTERM rados + teardown $dir || return 1 +} + +main osd-fast-mark-down "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && test/osd/osd-fast-mark-down.sh" +# End: diff --git a/qa/standalone/osd/osd-markdown.sh b/qa/standalone/osd/osd-markdown.sh new file mode 100755 index 00000000000..b3c800c081d --- /dev/null +++ b/qa/standalone/osd/osd-markdown.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# +# Copyright (C) 2015 Intel +# Copyright (C) 2014, 2015 Red Hat +# +# Author: Xiaoxi Chen +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7108" # git grep '\<7108\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + +function markdown_N_impl() { + markdown_times=$1 + total_time=$2 + sleeptime=$3 + for i in `seq 1 $markdown_times` + do + # check the OSD is UP + ceph osd tree + ceph osd tree | grep osd.0 |grep up || return 1 + # mark the OSD down. + ceph osd down 0 + sleep $sleeptime + done +} + + +function TEST_markdown_exceed_maxdown_count() { + local dir=$1 + + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + run_osd $dir 1 || return 1 + run_osd $dir 2 || return 1 + # 3+1 times within 300s, osd should stay dead on the 4th time + local count=3 + local sleeptime=10 + local period=300 + ceph tell osd.0 injectargs '--osd_max_markdown_count '$count'' || return 1 + ceph tell osd.0 injectargs '--osd_max_markdown_period '$period'' || return 1 + + markdown_N_impl $(($count+1)) $period $sleeptime + # down N+1 times ,the osd.0 shoud die + ceph osd tree | grep down | grep osd.0 || return 1 +} + +function TEST_markdown_boot() { + local dir=$1 + + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + run_osd $dir 1 || return 1 + run_osd $dir 2 || return 1 + + # 3 times within 120s, should stay up + local count=3 + local sleeptime=10 + local period=120 + ceph tell osd.0 injectargs '--osd_max_markdown_count '$count'' || return 1 + ceph tell osd.0 injectargs '--osd_max_markdown_period '$period'' || return 1 + + markdown_N_impl $count $period $sleeptime + #down N times, osd.0 should be up + sleep 15 # give osd plenty of time to notice and come back up + ceph osd tree | grep up | grep osd.0 || return 1 +} + +function TEST_markdown_boot_exceed_time() { + local dir=$1 + + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + run_osd $dir 1 || return 1 + run_osd $dir 2 || return 1 + + + # 3+1 times, but over 40s, > 20s, so should stay up + local count=3 + local period=20 + local sleeptime=10 + ceph tell osd.0 injectargs '--osd_max_markdown_count '$count'' || return 1 + ceph tell osd.0 injectargs '--osd_max_markdown_period '$period'' || return 1 + + markdown_N_impl $(($count+1)) $period $sleeptime + sleep 15 # give osd plenty of time to notice and come back up + ceph osd tree | grep up | grep osd.0 || return 1 +} + +main osd-markdown "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && test/osd/osd-bench.sh" +# End: diff --git a/qa/standalone/osd/osd-reactivate.sh b/qa/standalone/osd/osd-reactivate.sh new file mode 100755 index 00000000000..ddeee951a9e --- /dev/null +++ b/qa/standalone/osd/osd-reactivate.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# +# Author: Vicente Cheng +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7122" # git grep '\<7122\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + +function TEST_reactivate() { + local dir=$1 + + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + + kill_daemons $dir TERM osd || return 1 + + ready_path=$dir"/0/ready" + activate_path=$dir"/0/active" + # trigger mkfs again + rm -rf $ready_path $activate_path + activate_osd $dir 0 || return 1 + +} + +main osd-reactivate "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && test/osd/osd-reactivate.sh" +# End: diff --git a/qa/standalone/osd/osd-reuse-id.sh b/qa/standalone/osd/osd-reuse-id.sh new file mode 100755 index 00000000000..04666e1b634 --- /dev/null +++ b/qa/standalone/osd/osd-reuse-id.sh @@ -0,0 +1,51 @@ +#! /bin/bash +# +# Copyright (C) 2015 Red Hat +# +# Author: Loic Dachary +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7123" # git grep '\<7123\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + $func $dir || return 1 + done +} + +function TEST_reuse_id() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + run_osd $dir 1 || return 1 + wait_for_clean || return 1 + destroy_osd $dir 1 || return 1 + run_osd $dir 1 || return 1 +} + +main osd-reuse-id "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && test/osd/osd-reuse-id.sh" +# End: diff --git a/qa/standalone/osd/osd-scrub-repair.sh b/qa/standalone/osd/osd-scrub-repair.sh new file mode 100755 index 00000000000..9b3083b05ab --- /dev/null +++ b/qa/standalone/osd/osd-scrub-repair.sh @@ -0,0 +1,2618 @@ +#!/bin/bash -x +# +# Copyright (C) 2014 Red Hat +# +# Author: Loic Dachary +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +if [ `uname` = FreeBSD ]; then + # erasure coding overwrites are only tested on Bluestore + # erasure coding on filestore is unsafe + # http://docs.ceph.com/docs/master/rados/operations/erasure-code/#erasure-coding-with-overwrites + use_ec_overwrite=false +else + use_ec_overwrite=true +fi + +# Test development and debugging +# Set to "yes" in order to ignore diff errors and save results to update test +getjson="no" + +# Ignore the epoch and filter out the attr '_' value because it has date information and won't match +jqfilter='.inconsistents | (.[].shards[].attrs[]? | select(.name == "_") | .value) |= "----Stripped-by-test----"' +sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)' + +# Remove items are not consistent across runs, the pg interval and client +sedfilter='s/\([ ]*\"\(selected_\)*object_info\":.*head[(]\)[^[:space:]]* [^[:space:]]* \(.*\)/\1\3/' + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7107" # git grep '\<7107\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + $func $dir || return 1 + done +} + +function add_something() { + local dir=$1 + local poolname=$2 + local obj=${3:-SOMETHING} + local scrub=${4:-noscrub} + + if [ "$scrub" = "noscrub" ]; + then + ceph osd set noscrub || return 1 + ceph osd set nodeep-scrub || return 1 + else + ceph osd unset noscrub || return 1 + ceph osd unset nodeep-scrub || return 1 + fi + + local payload=ABCDEF + echo $payload > $dir/ORIGINAL + rados --pool $poolname put $obj $dir/ORIGINAL || return 1 +} + +# +# Corrupt one copy of a replicated pool +# +function TEST_corrupt_and_repair_replicated() { + local dir=$1 + local poolname=rbd + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=2 || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + run_osd $dir 1 || return 1 + wait_for_clean || return 1 + + add_something $dir $poolname || return 1 + corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1 + # Reproduces http://tracker.ceph.com/issues/8914 + corrupt_and_repair_one $dir $poolname $(get_primary $poolname SOMETHING) || return 1 + + teardown $dir || return 1 +} + +function corrupt_and_repair_two() { + local dir=$1 + local poolname=$2 + local first=$3 + local second=$4 + + # + # 1) remove the corresponding file from the OSDs + # + pids="" + run_in_background pids objectstore_tool $dir $first SOMETHING remove + run_in_background pids objectstore_tool $dir $second SOMETHING remove + wait_background pids + return_code=$? + if [ $return_code -ne 0 ]; then return $return_code; fi + + # + # 2) repair the PG + # + local pg=$(get_pg $poolname SOMETHING) + repair $pg + # + # 3) The files must be back + # + pids="" + run_in_background pids objectstore_tool $dir $first SOMETHING list-attrs + run_in_background pids objectstore_tool $dir $second SOMETHING list-attrs + wait_background pids + return_code=$? + if [ $return_code -ne 0 ]; then return $return_code; fi + + rados --pool $poolname get SOMETHING $dir/COPY || return 1 + diff $dir/ORIGINAL $dir/COPY || return 1 +} + +# +# 1) add an object +# 2) remove the corresponding file from a designated OSD +# 3) repair the PG +# 4) check that the file has been restored in the designated OSD +# +function corrupt_and_repair_one() { + local dir=$1 + local poolname=$2 + local osd=$3 + + # + # 1) remove the corresponding file from the OSD + # + objectstore_tool $dir $osd SOMETHING remove || return 1 + # + # 2) repair the PG + # + local pg=$(get_pg $poolname SOMETHING) + repair $pg + # + # 3) The file must be back + # + objectstore_tool $dir $osd SOMETHING list-attrs || return 1 + rados --pool $poolname get SOMETHING $dir/COPY || return 1 + diff $dir/ORIGINAL $dir/COPY || return 1 +} + +function corrupt_and_repair_erasure_coded() { + local dir=$1 + local poolname=$2 + + add_something $dir $poolname || return 1 + + local primary=$(get_primary $poolname SOMETHING) + local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//")) + local not_primary_first=${osds[0]} + local not_primary_second=${osds[1]} + + # Reproduces http://tracker.ceph.com/issues/10017 + corrupt_and_repair_one $dir $poolname $primary || return 1 + # Reproduces http://tracker.ceph.com/issues/10409 + corrupt_and_repair_one $dir $poolname $not_primary_first || return 1 + corrupt_and_repair_two $dir $poolname $not_primary_first $not_primary_second || return 1 + corrupt_and_repair_two $dir $poolname $primary $not_primary_first || return 1 + +} + +function create_ec_pool() { + local pool_name=$1 + local allow_overwrites=$2 + + ceph osd erasure-code-profile set myprofile crush-failure-domain=osd $3 $4 $5 $6 $7 || return 1 + + ceph osd pool create "$poolname" 1 1 erasure myprofile || return 1 + + if [ "$allow_overwrites" = "true" ]; then + ceph osd pool set "$poolname" allow_ec_overwrites true || return 1 + fi + + wait_for_clean || return 1 + return 0 +} + +function auto_repair_erasure_coded() { + local dir=$1 + local allow_overwrites=$2 + local poolname=ecpool + + # Launch a cluster with 5 seconds scrub interval + setup $dir || return 1 + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + local ceph_osd_args="--osd-scrub-auto-repair=true \ + --osd-deep-scrub-interval=5 \ + --osd-scrub-max-interval=5 \ + --osd-scrub-min-interval=5 \ + --osd-scrub-interval-randomize-ratio=0" + for id in $(seq 0 2) ; do + if [ "$allow_overwrites" = "true" ]; then + run_osd_bluestore $dir $id $ceph_osd_args || return 1 + else + run_osd $dir $id $ceph_osd_args || return 1 + fi + done + wait_for_clean || return 1 + + # Create an EC pool + create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1 + + # Put an object + local payload=ABCDEF + echo $payload > $dir/ORIGINAL + rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1 + + # Remove the object from one shard physically + # Restarted osd get $ceph_osd_args passed + objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1 + # Wait for auto repair + local pgid=$(get_pg $poolname SOMETHING) + wait_for_scrub $pgid "$(get_last_scrub_stamp $pgid)" + wait_for_clean || return 1 + # Verify - the file should be back + # Restarted osd get $ceph_osd_args passed + objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1 + rados --pool $poolname get SOMETHING $dir/COPY || return 1 + diff $dir/ORIGINAL $dir/COPY || return 1 + + # Tear down + teardown $dir || return 1 +} + +function TEST_auto_repair_erasure_coded_appends() { + auto_repair_erasure_coded $1 false +} + +function TEST_auto_repair_erasure_coded_overwrites() { + if [ "$use_ec_overwrite" = "true" ]; then + auto_repair_erasure_coded $1 true + fi +} + +function corrupt_and_repair_jerasure() { + local dir=$1 + local allow_overwrites=$2 + local poolname=ecpool + + setup $dir || return 1 + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + for id in $(seq 0 3) ; do + if [ "$allow_overwrites" = "true" ]; then + run_osd_bluestore $dir $id || return 1 + else + run_osd $dir $id || return 1 + fi + done + wait_for_clean || return 1 + + create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1 + corrupt_and_repair_erasure_coded $dir $poolname || return 1 + + teardown $dir || return 1 +} + +function TEST_corrupt_and_repair_jerasure_appends() { + corrupt_and_repair_jerasure $1 +} + +function TEST_corrupt_and_repair_jerasure_overwrites() { + if [ "$use_ec_overwrite" = "true" ]; then + corrupt_and_repair_jerasure $1 true + fi +} + +function corrupt_and_repair_lrc() { + local dir=$1 + local allow_overwrites=$2 + local poolname=ecpool + + setup $dir || return 1 + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + for id in $(seq 0 9) ; do + if [ "$allow_overwrites" = "true" ]; then + run_osd_bluestore $dir $id || return 1 + else + run_osd $dir $id || return 1 + fi + done + wait_for_clean || return 1 + + create_ec_pool $poolname $allow_overwrites k=4 m=2 l=3 plugin=lrc || return 1 + corrupt_and_repair_erasure_coded $dir $poolname || return 1 + + teardown $dir || return 1 +} + +function TEST_corrupt_and_repair_lrc_appends() { + corrupt_and_repair_jerasure $1 +} + +function TEST_corrupt_and_repair_lrc_overwrites() { + if [ "$use_ec_overwrite" = "true" ]; then + corrupt_and_repair_jerasure $1 true + fi +} + +function unfound_erasure_coded() { + local dir=$1 + local allow_overwrites=$2 + local poolname=ecpool + local payload=ABCDEF + + setup $dir || return 1 + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + for id in $(seq 0 3) ; do + if [ "$allow_overwrites" = "true" ]; then + run_osd_bluestore $dir $id || return 1 + else + run_osd $dir $id || return 1 + fi + done + wait_for_clean || return 1 + + create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1 + + add_something $dir $poolname || return 1 + + local primary=$(get_primary $poolname SOMETHING) + local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//")) + local not_primary_first=${osds[0]} + local not_primary_second=${osds[1]} + local not_primary_third=${osds[2]} + + # + # 1) remove the corresponding file from the OSDs + # + pids="" + run_in_background pids objectstore_tool $dir $not_primary_first SOMETHING remove + run_in_background pids objectstore_tool $dir $not_primary_second SOMETHING remove + run_in_background pids objectstore_tool $dir $not_primary_third SOMETHING remove + wait_background pids + return_code=$? + if [ $return_code -ne 0 ]; then return $return_code; fi + + # + # 2) repair the PG + # + local pg=$(get_pg $poolname SOMETHING) + repair $pg + # + # 3) check pg state + # + # it may take a bit to appear due to mon/mgr asynchrony + for f in `seq 1 60`; do + ceph -s | grep "1/1 unfound" && break + sleep 1 + done + ceph -s|grep "4 osds: 4 up, 4 in" || return 1 + ceph -s|grep "1/1 unfound" || return 1 + + teardown $dir || return 1 +} + +function TEST_unfound_erasure_coded_appends() { + unfound_erasure_coded $1 +} + +function TEST_unfound_erasure_coded_overwrites() { + if [ "$use_ec_overwrite" = "true" ]; then + unfound_erasure_coded $1 true + fi +} + +# +# list_missing for EC pool +# +function list_missing_erasure_coded() { + local dir=$1 + local allow_overwrites=$2 + local poolname=ecpool + + setup $dir || return 1 + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + for id in $(seq 0 2) ; do + if [ "$allow_overwrites" = "true" ]; then + run_osd_bluestore $dir $id || return 1 + else + run_osd $dir $id || return 1 + fi + done + wait_for_clean || return 1 + + create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1 + + # Put an object and remove the two shards (including primary) + add_something $dir $poolname MOBJ0 || return 1 + local -a osds0=($(get_osds $poolname MOBJ0)) + + # Put another object and remove two shards (excluding primary) + add_something $dir $poolname MOBJ1 || return 1 + local -a osds1=($(get_osds $poolname MOBJ1)) + + # Stop all osd daemons + for id in $(seq 0 2) ; do + kill_daemons $dir TERM osd.$id >&2 < /dev/null || return 1 + done + + id=${osds0[0]} + ceph-objectstore-tool --data-path $dir/$id \ + MOBJ0 remove || return 1 + id=${osds0[1]} + ceph-objectstore-tool --data-path $dir/$id \ + MOBJ0 remove || return 1 + + id=${osds1[1]} + ceph-objectstore-tool --data-path $dir/$id \ + MOBJ1 remove || return 1 + id=${osds1[2]} + ceph-objectstore-tool --data-path $dir/$id \ + MOBJ1 remove || return 1 + + for id in $(seq 0 2) ; do + activate_osd $dir $id >&2 || return 1 + done + wait_for_clean || return 1 + + # Get get - both objects should in the same PG + local pg=$(get_pg $poolname MOBJ0) + + # Repair the PG, which triggers the recovering, + # and should mark the object as unfound + repair $pg + + for i in $(seq 0 120) ; do + [ $i -lt 60 ] || return 1 + matches=$(ceph pg $pg list_missing | egrep "MOBJ0|MOBJ1" | wc -l) + [ $matches -eq 2 ] && break + done + + teardown $dir || return 1 +} + +function TEST_list_missing_erasure_coded_appends() { + list_missing_erasure_coded $1 false +} + +function TEST_list_missing_erasure_coded_overwrites() { + if [ "$use_ec_overwrite" = "true" ]; then + list_missing_erasure_coded $1 true + fi +} + +# +# Corrupt one copy of a replicated pool +# +function TEST_corrupt_scrub_replicated() { + local dir=$1 + local poolname=csr_pool + local total_objs=15 + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=2 || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + run_osd $dir 1 || return 1 + wait_for_clean || return 1 + + ceph osd pool create $poolname 1 1 || return 1 + wait_for_clean || return 1 + + for i in $(seq 1 $total_objs) ; do + objname=ROBJ${i} + add_something $dir $poolname $objname || return 1 + + rados --pool $poolname setomapheader $objname hdr-$objname || return 1 + rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1 + done + + local pg=$(get_pg $poolname ROBJ0) + + # Compute an old omap digest and save oi + CEPH_ARGS='' ceph daemon $(get_asok_path osd.0) \ + config set osd_deep_scrub_update_digest_min_age 0 + CEPH_ARGS='' ceph daemon $(get_asok_path osd.1) \ + config set osd_deep_scrub_update_digest_min_age 0 + pg_deep_scrub $pg + + for i in $(seq 1 $total_objs) ; do + objname=ROBJ${i} + + # Alternate corruption between osd.0 and osd.1 + local osd=$(expr $i % 2) + + case $i in + 1) + # Size (deep scrub data_digest too) + local payload=UVWXYZZZ + echo $payload > $dir/CORRUPT + objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 + ;; + + 2) + # digest (deep scrub only) + local payload=UVWXYZ + echo $payload > $dir/CORRUPT + objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 + ;; + + 3) + # missing + objectstore_tool $dir $osd $objname remove || return 1 + ;; + + 4) + # Modify omap value (deep scrub only) + objectstore_tool $dir $osd $objname set-omap key-$objname $dir/CORRUPT || return 1 + ;; + + 5) + # Delete omap key (deep scrub only) + objectstore_tool $dir $osd $objname rm-omap key-$objname || return 1 + ;; + + 6) + # Add extra omap key (deep scrub only) + echo extra > $dir/extra-val + objectstore_tool $dir $osd $objname set-omap key2-$objname $dir/extra-val || return 1 + rm $dir/extra-val + ;; + + 7) + # Modify omap header (deep scrub only) + echo -n newheader > $dir/hdr + objectstore_tool $dir $osd $objname set-omaphdr $dir/hdr || return 1 + rm $dir/hdr + ;; + + 8) + rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1 + rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1 + + # Break xattrs + echo -n bad-val > $dir/bad-val + objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1 + objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1 + echo -n val3-$objname > $dir/newval + objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1 + rm $dir/bad-val $dir/newval + ;; + + 9) + objectstore_tool $dir $osd $objname get-attr _ > $dir/robj9-oi + echo -n D > $dir/change + rados --pool $poolname put $objname $dir/change + objectstore_tool $dir $osd $objname set-attr _ $dir/robj9-oi + rm $dir/oi $dir/change + ;; + + # ROBJ10 must be handled after digests are re-computed by a deep scrub below + # ROBJ11 must be handled with config change before deep scrub + # ROBJ12 must be handled with config change before scrubs + # ROBJ13 must be handled before scrubs + + 14) + echo -n bad-val > $dir/bad-val + objectstore_tool $dir 0 $objname set-attr _ $dir/bad-val || return 1 + objectstore_tool $dir 1 $objname rm-attr _ || return 1 + rm $dir/bad-val + ;; + + 15) + objectstore_tool $dir $osd $objname rm-attr _ || return 1 + + esac + done + + local pg=$(get_pg $poolname ROBJ0) + + set_config osd 0 filestore_debug_inject_read_err true || return 1 + set_config osd 1 filestore_debug_inject_read_err true || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.1) \ + injectdataerr $poolname ROBJ11 || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) \ + injectmdataerr $poolname ROBJ12 || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) \ + injectmdataerr $poolname ROBJ13 || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.1) \ + injectdataerr $poolname ROBJ13 || return 1 + + pg_scrub $pg + + rados list-inconsistent-pg $poolname > $dir/json || return 1 + # Check pg count + test $(jq '. | length' $dir/json) = "1" || return 1 + # Check pgid + test $(jq -r '.[0]' $dir/json) = $pg || return 1 + + rados list-inconsistent-obj $pg > $dir/json || return 1 + # Get epoch for repair-get requests + epoch=$(jq .epoch $dir/json) + + jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson +{ + "inconsistents": [ + { + "shards": [ + { + "size": 7, + "errors": [], + "osd": 0 + }, + { + "size": 9, + "errors": [ + "size_mismatch_oi" + ], + "osd": 1 + } + ], + "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])", + "union_shard_errors": [ + "size_mismatch_oi" + ], + "errors": [ + "size_mismatch" + ], + "object": { + "version": 3, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ1" + } + }, + { + "shards": [ + { + "errors": [ + "stat_error" + ], + "osd": 0 + }, + { + "size": 7, + "errors": [], + "osd": 1 + } + ], + "selected_object_info": "3:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])", + "union_shard_errors": [ + "stat_error" + ], + "errors": [], + "object": { + "version": 36, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ12" + } + }, + { + "shards": [ + { + "errors": [ + "stat_error" + ], + "osd": 0 + }, + { + "size": 7, + "errors": [], + "osd": 1 + } + ], + "selected_object_info": "3:d60617f9:::ROBJ13:head(47'55 osd.0.0:54 dirty|omap|data_digest|omap_digest s 7 uv 39 dd 2ddbf8f5 od 6441854d alloc_hint [0 0 0])", + "union_shard_errors": [ + "stat_error" + ], + "errors": [], + "object": { + "version": 39, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ13" + } + }, + { + "shards": [ + { + "size": 7, + "errors": [ + "oi_attr_corrupted" + ], + "osd": 0 + }, + { + "size": 7, + "errors": [ + "oi_attr_missing" + ], + "osd": 1 + } + ], + "union_shard_errors": [ + "oi_attr_missing", + "oi_attr_corrupted" + ], + "errors": [], + "object": { + "version": 0, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ14" + } + }, + { + "shards": [ + { + "attrs": [ + { + "Base64": true, + "value": "", + "name": "_" + }, + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ], + "size": 7, + "errors": [], + "osd": 0 + }, + { + "attrs": [ + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ], + "size": 7, + "errors": [ + "oi_attr_missing" + ], + "osd": 1 + } + ], + "selected_object_info": "3:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])", + "union_shard_errors": [ + "oi_attr_missing" + ], + "errors": [ + "attr_name_mismatch" + ], + "object": { + "version": 45, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ15" + } + }, + { + "shards": [ + { + "size": 7, + "errors": [], + "osd": 0 + }, + { + "errors": [ + "missing" + ], + "osd": 1 + } + ], + "selected_object_info": "3:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])", + "union_shard_errors": [ + "missing" + ], + "errors": [], + "object": { + "version": 9, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ3" + } + }, + { + "shards": [ + { + "attrs": [ + { + "Base64": true, + "value": "", + "name": "_" + }, + { + "Base64": false, + "value": "bad-val", + "name": "_key1-ROBJ8" + }, + { + "Base64": false, + "value": "val3-ROBJ8", + "name": "_key3-ROBJ8" + }, + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ], + "size": 7, + "errors": [], + "osd": 0 + }, + { + "attrs": [ + { + "Base64": true, + "value": "", + "name": "_" + }, + { + "Base64": false, + "value": "val1-ROBJ8", + "name": "_key1-ROBJ8" + }, + { + "Base64": false, + "value": "val2-ROBJ8", + "name": "_key2-ROBJ8" + }, + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ], + "size": 7, + "errors": [], + "osd": 1 + } + ], + "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 62 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])", + "union_shard_errors": [], + "errors": [ + "attr_value_mismatch", + "attr_name_mismatch" + ], + "object": { + "version": 62, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ8" + } + }, + { + "shards": [ + { + "attrs": [ + { + "Base64": true, + "value": "", + "name": "_" + }, + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ], + "object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 63 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])", + "size": 1, + "errors": [], + "osd": 0 + }, + { + "attrs": [ + { + "Base64": true, + "value": "", + "name": "_" + }, + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ], + "object_info": "3:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])", + "size": 1, + "errors": [], + "osd": 1 + } + ], + "selected_object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 63 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])", + "union_shard_errors": [], + "errors": [ + "object_info_inconsistency", + "attr_value_mismatch" + ], + "object": { + "version": 63, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ9" + } + } + ], + "epoch": 0 +} +EOF + + jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson + diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 + if test $getjson = "yes" + then + jq '.' $dir/json > save1.json + fi + + if which jsonschema > /dev/null; + then + jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 + fi + + objname=ROBJ9 + # Change data and size again because digest was recomputed + echo -n ZZZ > $dir/change + rados --pool $poolname put $objname $dir/change + # Set one to an even older value + objectstore_tool $dir 0 $objname set-attr _ $dir/robj9-oi + rm $dir/oi $dir/change + + objname=ROBJ10 + objectstore_tool $dir 1 $objname get-attr _ > $dir/oi + rados --pool $poolname setomapval $objname key2-$objname val2-$objname + objectstore_tool $dir 0 $objname set-attr _ $dir/oi + objectstore_tool $dir 1 $objname set-attr _ $dir/oi + rm $dir/oi + + set_config osd 0 filestore_debug_inject_read_err true || return 1 + set_config osd 1 filestore_debug_inject_read_err true || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.1) \ + injectdataerr $poolname ROBJ11 || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) \ + injectmdataerr $poolname ROBJ12 || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) \ + injectmdataerr $poolname ROBJ13 || return 1 + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.1) \ + injectdataerr $poolname ROBJ13 || return 1 + pg_deep_scrub $pg + + rados list-inconsistent-pg $poolname > $dir/json || return 1 + # Check pg count + test $(jq '. | length' $dir/json) = "1" || return 1 + # Check pgid + test $(jq -r '.[0]' $dir/json) = $pg || return 1 + + rados list-inconsistent-obj $pg > $dir/json || return 1 + # Get epoch for repair-get requests + epoch=$(jq .epoch $dir/json) + + jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson +{ + "inconsistents": [ + { + "shards": [ + { + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xf5fba2c6", + "size": 7, + "errors": [], + "osd": 0 + }, + { + "data_digest": "0x2d4a11c2", + "omap_digest": "0xf5fba2c6", + "size": 9, + "errors": [ + "data_digest_mismatch_oi", + "size_mismatch_oi" + ], + "osd": 1 + } + ], + "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])", + "union_shard_errors": [ + "data_digest_mismatch_oi", + "size_mismatch_oi" + ], + "errors": [ + "data_digest_mismatch", + "size_mismatch" + ], + "object": { + "version": 3, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ1" + } + }, + { + "shards": [ + { + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xa8dd5adc", + "size": 7, + "errors": [ + "omap_digest_mismatch_oi" + ], + "osd": 0 + }, + { + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xa8dd5adc", + "size": 7, + "errors": [ + "omap_digest_mismatch_oi" + ], + "osd": 1 + } + ], + "selected_object_info": "3:b1f19cbd:::ROBJ10:head(47'51 osd.0.0:50 dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [0 0 0])", + "union_shard_errors": [ + "omap_digest_mismatch_oi" + ], + "errors": [], + "object": { + "version": 30, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ10" + } + }, + { + "shards": [ + { + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xa03cef03", + "size": 7, + "errors": [], + "osd": 0 + }, + { + "size": 7, + "errors": [ + "read_error" + ], + "osd": 1 + } + ], + "selected_object_info": "3:87abbf36:::ROBJ11:head(47'48 osd.0.0:47 dirty|omap|data_digest|omap_digest s 7 uv 33 dd 2ddbf8f5 od a03cef03 alloc_hint [0 0 0])", + "union_shard_errors": [ + "read_error" + ], + "errors": [], + "object": { + "version": 33, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ11" + } + }, + { + "shards": [ + { + "errors": [ + "stat_error" + ], + "osd": 0 + }, + { + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x067f306a", + "size": 7, + "errors": [], + "osd": 1 + } + ], + "selected_object_info": "3:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])", + "union_shard_errors": [ + "stat_error" + ], + "errors": [], + "object": { + "version": 36, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ12" + } + }, + { + "shards": [ + { + "errors": [ + "stat_error" + ], + "osd": 0 + }, + { + "size": 7, + "errors": [ + "read_error" + ], + "osd": 1 + } + ], + "union_shard_errors": [ + "stat_error", + "read_error" + ], + "errors": [], + "object": { + "version": 0, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ13" + } + }, + { + "shards": [ + { + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x4f14f849", + "size": 7, + "errors": [ + "oi_attr_corrupted" + ], + "osd": 0 + }, + { + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x4f14f849", + "size": 7, + "errors": [ + "oi_attr_missing" + ], + "osd": 1 + } + ], + "union_shard_errors": [ + "oi_attr_missing", + "oi_attr_corrupted" + ], + "errors": [], + "object": { + "version": 0, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ14" + } + }, + { + "shards": [ + { + "attrs": [ + { + "Base64": true, + "value": "", + "name": "_" + }, + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ], + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x2d2a4d6e", + "size": 7, + "errors": [], + "osd": 0 + }, + { + "attrs": [ + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ], + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x2d2a4d6e", + "size": 7, + "errors": [ + "oi_attr_missing" + ], + "osd": 1 + } + ], + "selected_object_info": "3:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])", + "union_shard_errors": [ + "oi_attr_missing" + ], + "errors": [ + "attr_name_mismatch" + ], + "object": { + "version": 45, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ15" + } + }, + { + "shards": [ + { + "data_digest": "0x578a4830", + "omap_digest": "0xf8e11918", + "size": 7, + "errors": [ + "data_digest_mismatch_oi" + ], + "osd": 0 + }, + { + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xf8e11918", + "size": 7, + "errors": [], + "osd": 1 + } + ], + "selected_object_info": "3:e97ce31e:::ROBJ2:head(47'56 osd.0.0:55 dirty|omap|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od f8e11918 alloc_hint [0 0 0])", + "union_shard_errors": [ + "data_digest_mismatch_oi" + ], + "errors": [ + "data_digest_mismatch" + ], + "object": { + "version": 6, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ2" + } + }, + { + "shards": [ + { + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x00b35dfd", + "size": 7, + "errors": [], + "osd": 0 + }, + { + "errors": [ + "missing" + ], + "osd": 1 + } + ], + "selected_object_info": "3:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])", + "union_shard_errors": [ + "missing" + ], + "errors": [], + "object": { + "version": 9, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ3" + } + }, + { + "shards": [ + { + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xd7178dfe", + "size": 7, + "errors": [ + "omap_digest_mismatch_oi" + ], + "osd": 0 + }, + { + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xe2d46ea4", + "size": 7, + "errors": [], + "osd": 1 + } + ], + "selected_object_info": "3:f4981d31:::ROBJ4:head(47'58 osd.0.0:57 dirty|omap|data_digest|omap_digest s 7 uv 12 dd 2ddbf8f5 od e2d46ea4 alloc_hint [0 0 0])", + "union_shard_errors": [ + "omap_digest_mismatch_oi" + ], + "errors": [ + "omap_digest_mismatch" + ], + "object": { + "version": 12, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ4" + } + }, + { + "shards": [ + { + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x1a862a41", + "size": 7, + "errors": [], + "osd": 0 + }, + { + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x06cac8f6", + "size": 7, + "errors": [ + "omap_digest_mismatch_oi" + ], + "osd": 1 + } + ], + "selected_object_info": "3:f4bfd4d1:::ROBJ5:head(47'59 osd.0.0:58 dirty|omap|data_digest|omap_digest s 7 uv 15 dd 2ddbf8f5 od 1a862a41 alloc_hint [0 0 0])", + "union_shard_errors": [ + "omap_digest_mismatch_oi" + ], + "errors": [ + "omap_digest_mismatch" + ], + "object": { + "version": 15, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ5" + } + }, + { + "shards": [ + { + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x689ee887", + "size": 7, + "errors": [ + "omap_digest_mismatch_oi" + ], + "osd": 0 + }, + { + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x179c919f", + "size": 7, + "errors": [], + "osd": 1 + } + ], + "selected_object_info": "3:a53c12e8:::ROBJ6:head(47'50 osd.0.0:49 dirty|omap|data_digest|omap_digest s 7 uv 18 dd 2ddbf8f5 od 179c919f alloc_hint [0 0 0])", + "union_shard_errors": [ + "omap_digest_mismatch_oi" + ], + "errors": [ + "omap_digest_mismatch" + ], + "object": { + "version": 18, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ6" + } + }, + { + "shards": [ + { + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xefced57a", + "size": 7, + "errors": [], + "osd": 0 + }, + { + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x6a73cc07", + "size": 7, + "errors": [ + "omap_digest_mismatch_oi" + ], + "osd": 1 + } + ], + "selected_object_info": "3:8b55fa4b:::ROBJ7:head(47'49 osd.0.0:48 dirty|omap|data_digest|omap_digest s 7 uv 21 dd 2ddbf8f5 od efced57a alloc_hint [0 0 0])", + "union_shard_errors": [ + "omap_digest_mismatch_oi" + ], + "errors": [ + "omap_digest_mismatch" + ], + "object": { + "version": 21, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ7" + } + }, + { + "shards": [ + { + "attrs": [ + { + "Base64": true, + "value": "", + "name": "_" + }, + { + "Base64": false, + "value": "bad-val", + "name": "_key1-ROBJ8" + }, + { + "Base64": false, + "value": "val3-ROBJ8", + "name": "_key3-ROBJ8" + }, + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ], + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xd6be81dc", + "size": 7, + "errors": [], + "osd": 0 + }, + { + "attrs": [ + { + "Base64": true, + "value": "", + "name": "_" + }, + { + "Base64": false, + "value": "val1-ROBJ8", + "name": "_key1-ROBJ8" + }, + { + "Base64": false, + "value": "val2-ROBJ8", + "name": "_key2-ROBJ8" + }, + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ], + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xd6be81dc", + "size": 7, + "errors": [], + "osd": 1 + } + ], + "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 62 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])", + "union_shard_errors": [], + "errors": [ + "attr_value_mismatch", + "attr_name_mismatch" + ], + "object": { + "version": 62, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ8" + } + }, + { + "shards": [ + { + "attrs": [ + { + "Base64": true, + "value": "", + "name": "_" + }, + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ], + "object_info": "3:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])", + "data_digest": "0x1f26fb26", + "omap_digest": "0x2eecc539", + "size": 3, + "errors": [], + "osd": 0 + }, + { + "attrs": [ + { + "Base64": true, + "value": "", + "name": "_" + }, + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ], + "object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 64 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])", + "data_digest": "0x1f26fb26", + "omap_digest": "0x2eecc539", + "size": 3, + "errors": [], + "osd": 1 + } + ], + "selected_object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 64 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])", + "union_shard_errors": [], + "errors": [ + "object_info_inconsistency", + "attr_value_mismatch" + ], + "object": { + "version": 64, + "snap": "head", + "locator": "", + "nspace": "", + "name": "ROBJ9" + } + } + ], + "epoch": 0 +} +EOF + + jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson + diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 + if test $getjson = "yes" + then + jq '.' $dir/json > save2.json + fi + + if which jsonschema > /dev/null; + then + jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 + fi + + rados rmpool $poolname $poolname --yes-i-really-really-mean-it + teardown $dir || return 1 +} + + +# +# Test scrub errors for an erasure coded pool +# +function corrupt_scrub_erasure() { + local dir=$1 + local allow_overwrites=$2 + local poolname=ecpool + local total_objs=5 + + setup $dir || return 1 + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + for id in $(seq 0 2) ; do + if [ "$allow_overwrites" = "true" ]; then + run_osd_bluestore $dir $id || return 1 + else + run_osd $dir $id || return 1 + fi + done + wait_for_clean || return 1 + + create_ec_pool $poolname $allow_overwrites k=2 m=1 stripe_unit=2K --force || return 1 + + for i in $(seq 1 $total_objs) ; do + objname=EOBJ${i} + add_something $dir $poolname $objname || return 1 + + local osd=$(expr $i % 2) + + case $i in + 1) + # Size (deep scrub data_digest too) + local payload=UVWXYZZZ + echo $payload > $dir/CORRUPT + objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 + ;; + + 2) + # Corrupt EC shard + dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=1 + objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 + ;; + + 3) + # missing + objectstore_tool $dir $osd $objname remove || return 1 + ;; + + 4) + rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1 + rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1 + + # Break xattrs + echo -n bad-val > $dir/bad-val + objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1 + objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1 + echo -n val3-$objname > $dir/newval + objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1 + rm $dir/bad-val $dir/newval + ;; + + 5) + # Corrupt EC shard + dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=2 + objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 + ;; + + esac + done + + local pg=$(get_pg $poolname EOBJ0) + + pg_scrub $pg + + rados list-inconsistent-pg $poolname > $dir/json || return 1 + # Check pg count + test $(jq '. | length' $dir/json) = "1" || return 1 + # Check pgid + test $(jq -r '.[0]' $dir/json) = $pg || return 1 + + rados list-inconsistent-obj $pg > $dir/json || return 1 + # Get epoch for repair-get requests + epoch=$(jq .epoch $dir/json) + + jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson +{ + "inconsistents": [ + { + "shards": [ + { + "size": 2048, + "errors": [], + "shard": 2, + "osd": 0 + }, + { + "size": 9, + "shard": 0, + "errors": [ + "size_mismatch_oi" + ], + "osd": 1 + }, + { + "size": 2048, + "shard": 1, + "errors": [], + "osd": 2 + } + ], + "selected_object_info": "3:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "union_shard_errors": [ + "size_mismatch_oi" + ], + "errors": [ + "size_mismatch" + ], + "object": { + "version": 1, + "snap": "head", + "locator": "", + "nspace": "", + "name": "EOBJ1" + } + }, + { + "shards": [ + { + "size": 2048, + "errors": [], + "shard": 2, + "osd": 0 + }, + { + "shard": 0, + "errors": [ + "missing" + ], + "osd": 1 + }, + { + "size": 2048, + "shard": 1, + "errors": [], + "osd": 2 + } + ], + "selected_object_info": "3:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "union_shard_errors": [ + "missing" + ], + "errors": [], + "object": { + "version": 3, + "snap": "head", + "locator": "", + "nspace": "", + "name": "EOBJ3" + } + }, + { + "shards": [ + { + "attrs": [ + { + "Base64": true, + "value": "", + "name": "_" + }, + { + "Base64": false, + "value": "bad-val", + "name": "_key1-EOBJ4" + }, + { + "Base64": false, + "value": "val3-EOBJ4", + "name": "_key3-EOBJ4" + }, + { + "Base64": true, + "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", + "name": "hinfo_key" + }, + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ], + "size": 2048, + "errors": [], + "shard": 2, + "osd": 0 + }, + { + "osd": 1, + "shard": 0, + "errors": [], + "size": 2048, + "attrs": [ + { + "Base64": true, + "value": "", + "name": "_" + }, + { + "Base64": false, + "value": "val1-EOBJ4", + "name": "_key1-EOBJ4" + }, + { + "Base64": false, + "value": "val2-EOBJ4", + "name": "_key2-EOBJ4" + }, + { + "Base64": true, + "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", + "name": "hinfo_key" + }, + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ] + }, + { + "osd": 2, + "shard": 1, + "errors": [], + "size": 2048, + "attrs": [ + { + "Base64": true, + "value": "", + "name": "_" + }, + { + "Base64": false, + "value": "val1-EOBJ4", + "name": "_key1-EOBJ4" + }, + { + "Base64": false, + "value": "val2-EOBJ4", + "name": "_key2-EOBJ4" + }, + { + "Base64": true, + "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", + "name": "hinfo_key" + }, + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ] + } + ], + "selected_object_info": "3:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "union_shard_errors": [], + "errors": [ + "attr_value_mismatch", + "attr_name_mismatch" + ], + "object": { + "version": 6, + "snap": "head", + "locator": "", + "nspace": "", + "name": "EOBJ4" + } + }, + { + "shards": [ + { + "size": 2048, + "errors": [], + "shard": 2, + "osd": 0 + }, + { + "size": 4096, + "shard": 0, + "errors": [ + "size_mismatch_oi" + ], + "osd": 1 + }, + { + "size": 2048, + "shard": 1, + "errors": [], + "osd": 2 + } + ], + "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "union_shard_errors": [ + "size_mismatch_oi" + ], + "errors": [ + "size_mismatch" + ], + "object": { + "version": 7, + "snap": "head", + "locator": "", + "nspace": "", + "name": "EOBJ5" + } + } + ], + "epoch": 0 +} +EOF + + jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson + diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 + if test $getjson = "yes" + then + jq '.' $dir/json > save3.json + fi + + if which jsonschema > /dev/null; + then + jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 + fi + + pg_deep_scrub $pg + + rados list-inconsistent-pg $poolname > $dir/json || return 1 + # Check pg count + test $(jq '. | length' $dir/json) = "1" || return 1 + # Check pgid + test $(jq -r '.[0]' $dir/json) = $pg || return 1 + + rados list-inconsistent-obj $pg > $dir/json || return 1 + # Get epoch for repair-get requests + epoch=$(jq .epoch $dir/json) + + if [ "$allow_overwrites" = "true" ] + then + jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson +{ + "inconsistents": [ + { + "shards": [ + { + "data_digest": "0x00000000", + "omap_digest": "0xffffffff", + "size": 2048, + "errors": [], + "shard": 2, + "osd": 0 + }, + { + "size": 9, + "shard": 0, + "errors": [ + "read_error", + "size_mismatch_oi" + ], + "osd": 1 + }, + { + "data_digest": "0x00000000", + "omap_digest": "0xffffffff", + "size": 2048, + "shard": 1, + "errors": [], + "osd": 2 + } + ], + "selected_object_info": "3:9175b684:::EOBJ1:head(27'1 client.4155.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "union_shard_errors": [ + "read_error", + "size_mismatch_oi" + ], + "errors": [ + "size_mismatch" + ], + "object": { + "version": 1, + "snap": "head", + "locator": "", + "nspace": "", + "name": "EOBJ1" + } + }, + { + "shards": [ + { + "data_digest": "0x00000000", + "omap_digest": "0xffffffff", + "size": 2048, + "errors": [], + "shard": 2, + "osd": 0 + }, + { + "shard": 0, + "errors": [ + "missing" + ], + "osd": 1 + }, + { + "data_digest": "0x00000000", + "omap_digest": "0xffffffff", + "size": 2048, + "shard": 1, + "errors": [], + "osd": 2 + } + ], + "selected_object_info": "3:b197b25d:::EOBJ3:head(41'3 client.4199.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "union_shard_errors": [ + "missing" + ], + "errors": [], + "object": { + "version": 3, + "snap": "head", + "locator": "", + "nspace": "", + "name": "EOBJ3" + } + }, + { + "shards": [ + { + "attrs": [ + { + "Base64": true, + "value": "", + "name": "_" + }, + { + "Base64": false, + "value": "bad-val", + "name": "_key1-EOBJ4" + }, + { + "Base64": false, + "value": "val3-EOBJ4", + "name": "_key3-EOBJ4" + }, + { + "Base64": true, + "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", + "name": "hinfo_key" + }, + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ], + "data_digest": "0x00000000", + "omap_digest": "0xffffffff", + "size": 2048, + "errors": [], + "shard": 2, + "osd": 0 + }, + { + "attrs": [ + { + "Base64": true, + "value": "", + "name": "_" + }, + { + "Base64": false, + "value": "val1-EOBJ4", + "name": "_key1-EOBJ4" + }, + { + "Base64": false, + "value": "val2-EOBJ4", + "name": "_key2-EOBJ4" + }, + { + "Base64": true, + "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", + "name": "hinfo_key" + }, + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ], + "data_digest": "0x00000000", + "omap_digest": "0xffffffff", + "size": 2048, + "errors": [], + "shard": 0, + "osd": 1 + }, + { + "attrs": [ + { + "Base64": true, + "value": "", + "name": "_" + }, + { + "Base64": false, + "value": "val1-EOBJ4", + "name": "_key1-EOBJ4" + }, + { + "Base64": false, + "value": "val2-EOBJ4", + "name": "_key2-EOBJ4" + }, + { + "Base64": true, + "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", + "name": "hinfo_key" + }, + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ], + "data_digest": "0x00000000", + "omap_digest": "0xffffffff", + "size": 2048, + "errors": [], + "shard": 1, + "osd": 2 + } + ], + "selected_object_info": "3:5e723e06:::EOBJ4:head(48'6 client.4223.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "union_shard_errors": [], + "errors": [ + "attr_value_mismatch", + "attr_name_mismatch" + ], + "object": { + "version": 6, + "snap": "head", + "locator": "", + "nspace": "", + "name": "EOBJ4" + } + }, + { + "shards": [ + { + "data_digest": "0x00000000", + "omap_digest": "0xffffffff", + "size": 2048, + "errors": [], + "shard": 2, + "osd": 0 + }, + { + "data_digest": "0x00000000", + "omap_digest": "0xffffffff", + "size": 4096, + "errors": [ + "size_mismatch_oi" + ], + "shard": 0, + "osd": 1 + }, + { + "data_digest": "0x00000000", + "omap_digest": "0xffffffff", + "size": 2048, + "errors": [], + "shard": 1, + "osd": 2 + } + ], + "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4288.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "union_shard_errors": [ + "size_mismatch_oi" + ], + "errors": [ + "size_mismatch" + ], + "object": { + "version": 7, + "snap": "head", + "locator": "", + "nspace": "", + "name": "EOBJ5" + } + } + ], + "epoch": 0 +} +EOF + + else + + jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson +{ + "inconsistents": [ + { + "shards": [ + { + "data_digest": "0x04cfa72f", + "omap_digest": "0xffffffff", + "size": 2048, + "errors": [], + "shard": 2, + "osd": 0 + }, + { + "size": 9, + "shard": 0, + "errors": [ + "read_error", + "size_mismatch_oi" + ], + "osd": 1 + }, + { + "data_digest": "0x04cfa72f", + "omap_digest": "0xffffffff", + "size": 2048, + "shard": 1, + "errors": [], + "osd": 2 + } + ], + "selected_object_info": "3:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "union_shard_errors": [ + "read_error", + "size_mismatch_oi" + ], + "errors": [ + "size_mismatch" + ], + "object": { + "version": 1, + "snap": "head", + "locator": "", + "nspace": "", + "name": "EOBJ1" + } + }, + { + "shards": [ + { + "size": 2048, + "errors": [ + "ec_hash_error" + ], + "shard": 2, + "osd": 0 + }, + { + "data_digest": "0x04cfa72f", + "omap_digest": "0xffffffff", + "size": 2048, + "errors": [], + "shard": 0, + "osd": 1 + }, + { + "data_digest": "0x04cfa72f", + "omap_digest": "0xffffffff", + "size": 2048, + "errors": [], + "shard": 1, + "osd": 2 + } + ], + "selected_object_info": "3:9babd184:::EOBJ2:head(29'2 client.4217.0:1 dirty|data_digest|omap_digest s 7 uv 2 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "union_shard_errors": [ + "ec_hash_error" + ], + "errors": [], + "object": { + "version": 2, + "snap": "head", + "locator": "", + "nspace": "", + "name": "EOBJ2" + } + }, + { + "shards": [ + { + "data_digest": "0x04cfa72f", + "omap_digest": "0xffffffff", + "size": 2048, + "errors": [], + "shard": 2, + "osd": 0 + }, + { + "osd": 1, + "shard": 0, + "errors": [ + "missing" + ] + }, + { + "data_digest": "0x04cfa72f", + "omap_digest": "0xffffffff", + "size": 2048, + "shard": 1, + "errors": [], + "osd": 2 + } + ], + "selected_object_info": "3:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "union_shard_errors": [ + "missing" + ], + "errors": [], + "object": { + "version": 3, + "snap": "head", + "locator": "", + "nspace": "", + "name": "EOBJ3" + } + }, + { + "shards": [ + { + "attrs": [ + { + "Base64": true, + "value": "", + "name": "_" + }, + { + "Base64": false, + "value": "bad-val", + "name": "_key1-EOBJ4" + }, + { + "Base64": false, + "value": "val3-EOBJ4", + "name": "_key3-EOBJ4" + }, + { + "Base64": true, + "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", + "name": "hinfo_key" + }, + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ], + "data_digest": "0x04cfa72f", + "omap_digest": "0xffffffff", + "size": 2048, + "errors": [], + "shard": 2, + "osd": 0 + }, + { + "osd": 1, + "shard": 0, + "errors": [], + "size": 2048, + "omap_digest": "0xffffffff", + "data_digest": "0x04cfa72f", + "attrs": [ + { + "Base64": true, + "value": "", + "name": "_" + }, + { + "Base64": false, + "value": "val1-EOBJ4", + "name": "_key1-EOBJ4" + }, + { + "Base64": false, + "value": "val2-EOBJ4", + "name": "_key2-EOBJ4" + }, + { + "Base64": true, + "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", + "name": "hinfo_key" + }, + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ] + }, + { + "osd": 2, + "shard": 1, + "errors": [], + "size": 2048, + "omap_digest": "0xffffffff", + "data_digest": "0x04cfa72f", + "attrs": [ + { + "Base64": true, + "value": "", + "name": "_" + }, + { + "Base64": false, + "value": "val1-EOBJ4", + "name": "_key1-EOBJ4" + }, + { + "Base64": false, + "value": "val2-EOBJ4", + "name": "_key2-EOBJ4" + }, + { + "Base64": true, + "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", + "name": "hinfo_key" + }, + { + "Base64": true, + "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "name": "snapset" + } + ] + } + ], + "selected_object_info": "3:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "union_shard_errors": [], + "errors": [ + "attr_value_mismatch", + "attr_name_mismatch" + ], + "object": { + "version": 6, + "snap": "head", + "locator": "", + "nspace": "", + "name": "EOBJ4" + } + }, + { + "shards": [ + { + "data_digest": "0x04cfa72f", + "omap_digest": "0xffffffff", + "size": 2048, + "errors": [], + "shard": 2, + "osd": 0 + }, + { + "size": 4096, + "shard": 0, + "errors": [ + "size_mismatch_oi", + "ec_size_error" + ], + "osd": 1 + }, + { + "data_digest": "0x04cfa72f", + "omap_digest": "0xffffffff", + "size": 2048, + "shard": 1, + "errors": [], + "osd": 2 + } + ], + "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "union_shard_errors": [ + "size_mismatch_oi", + "ec_size_error" + ], + "errors": [ + "size_mismatch" + ], + "object": { + "version": 7, + "snap": "head", + "locator": "", + "nspace": "", + "name": "EOBJ5" + } + } + ], + "epoch": 0 +} +EOF + + fi + + jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson + diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 + if test $getjson = "yes" + then + if [ "$allow_overwrites" = "true" ] + then + num=4 + else + num=5 + fi + jq '.' $dir/json > save${num}.json + fi + + if which jsonschema > /dev/null; + then + jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 + fi + + rados rmpool $poolname $poolname --yes-i-really-really-mean-it + teardown $dir || return 1 +} + +function TEST_corrupt_scrub_erasure_appends() { + corrupt_scrub_erasure $1 false +} + +function TEST_corrupt_scrub_erasure_overwrites() { + if [ "$use_ec_overwrite" = "true" ]; then + corrupt_scrub_erasure $1 true + fi +} + +# +# Test to make sure that a periodic scrub won't cause deep-scrub info to be lost +# +function TEST_periodic_scrub_replicated() { + local dir=$1 + local poolname=psr_pool + local objname=POBJ + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=2 || return 1 + run_mgr $dir x || return 1 + local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0" + run_osd $dir 0 $ceph_osd_args || return 1 + run_osd $dir 1 $ceph_osd_args || return 1 + wait_for_clean || return 1 + + ceph osd pool create $poolname 1 1 || return 1 + wait_for_clean || return 1 + + local osd=0 + add_something $dir $poolname $objname scrub || return 1 + local primary=$(get_primary $poolname $objname) + local pg=$(get_pg $poolname $objname) + + # Add deep-scrub only error + local payload=UVWXYZ + echo $payload > $dir/CORRUPT + # Uses $ceph_osd_args for osd restart + objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 + + # No scrub information available, so expect failure + set -o pipefail + ! rados list-inconsistent-obj $pg | jq '.' || return 1 + set +o pipefail + + pg_deep_scrub $pg || return 1 + + # Make sure bad object found + rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1 + + local last_scrub=$(get_last_scrub_stamp $pg) + # Fake a schedule scrub + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \ + trigger_scrub $pg || return 1 + # Wait for schedule regular scrub + wait_for_scrub $pg "$last_scrub" + + # It needed to be upgraded + grep -q "Deep scrub errors, upgrading scrub to deep-scrub" $dir/osd.${primary}.log || return 1 + + # Bad object still known + rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1 + + # Can't upgrade with this set + ceph osd set nodeep-scrub + # Let map change propagate to OSDs + sleep 2 + + # Fake a schedule scrub + local last_scrub=$(get_last_scrub_stamp $pg) + CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \ + trigger_scrub $pg || return 1 + # Wait for schedule regular scrub + # to notice scrub and skip it + local found=false + for i in $(seq 14 -1 0) + do + sleep 1 + ! grep -q "Regular scrub skipped due to deep-scrub errors and nodeep-scrub set" $dir/osd.${primary}.log || { found=true ; break; } + echo Time left: $i seconds + done + test $found = "true" || return 1 + + # Bad object still known + rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1 + + # Request a regular scrub and it will be done + local scrub_backoff_ratio=$(get_config osd ${primary} osd_scrub_backoff_ratio) + set_config osd ${primary} osd_scrub_backoff_ratio 0 + pg_scrub $pg + sleep 1 + set_config osd ${primary} osd_scrub_backoff_ratio $scrub_backoff_ratio + grep -q "Regular scrub request, losing deep-scrub details" $dir/osd.${primary}.log || return 1 + + # deep-scrub error is no longer present + rados list-inconsistent-obj $pg | jq '.' | grep -qv $objname || return 1 +} + + +main osd-scrub-repair "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && \ +# test/osd/osd-scrub-repair.sh # TEST_corrupt_and_repair_replicated" +# End: diff --git a/qa/standalone/osd/osd-scrub-snaps.sh b/qa/standalone/osd/osd-scrub-snaps.sh new file mode 100755 index 00000000000..4ddd08eb68d --- /dev/null +++ b/qa/standalone/osd/osd-scrub-snaps.sh @@ -0,0 +1,479 @@ +#! /bin/bash +# +# Copyright (C) 2015 Red Hat +# +# Author: David Zafman +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7121" # git grep '\<7121\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + $func $dir || return 1 + done +} + +function TEST_scrub_snaps() { + local dir=$1 + local poolname=test + + TESTDATA="testdata.$$" + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + + wait_for_clean || return 1 + + # Create a pool with a single pg + ceph osd pool create $poolname 1 1 + poolid=$(ceph osd dump | grep "^pool.*[']test[']" | awk '{ print $2 }') + + dd if=/dev/urandom of=$TESTDATA bs=1032 count=1 + for i in `seq 1 15` + do + rados -p $poolname put obj${i} $TESTDATA + done + + SNAP=1 + rados -p $poolname mksnap snap${SNAP} + dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP} + rados -p $poolname put obj1 $TESTDATA + rados -p $poolname put obj5 $TESTDATA + rados -p $poolname put obj3 $TESTDATA + for i in `seq 6 14` + do rados -p $poolname put obj${i} $TESTDATA + done + + SNAP=2 + rados -p $poolname mksnap snap${SNAP} + dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP} + rados -p $poolname put obj5 $TESTDATA + + SNAP=3 + rados -p $poolname mksnap snap${SNAP} + dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP} + rados -p $poolname put obj3 $TESTDATA + + SNAP=4 + rados -p $poolname mksnap snap${SNAP} + dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP} + rados -p $poolname put obj5 $TESTDATA + rados -p $poolname put obj2 $TESTDATA + + SNAP=5 + rados -p $poolname mksnap snap${SNAP} + SNAP=6 + rados -p $poolname mksnap snap${SNAP} + dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP} + rados -p $poolname put obj5 $TESTDATA + + SNAP=7 + rados -p $poolname mksnap snap${SNAP} + + rados -p $poolname rm obj4 + rados -p $poolname rm obj2 + + kill_daemons $dir TERM osd || return 1 + + # Don't need to ceph_objectstore_tool function because osd stopped + + JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj1)" + ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" --force remove + + JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --op list obj5 | grep \"snapid\":2)" + ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" remove + + JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --op list obj5 | grep \"snapid\":1)" + OBJ5SAVE="$JSON" + ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" remove + + JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --op list obj5 | grep \"snapid\":4)" + dd if=/dev/urandom of=$TESTDATA bs=256 count=18 + ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" set-bytes $TESTDATA + + JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj3)" + dd if=/dev/urandom of=$TESTDATA bs=256 count=15 + ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" set-bytes $TESTDATA + + JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --op list obj4 | grep \"snapid\":7)" + ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" remove + + JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj2)" + ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" rm-attr snapset + + # Create a clone which isn't in snapset and doesn't have object info + JSON="$(echo "$OBJ5SAVE" | sed s/snapid\":1/snapid\":7/)" + dd if=/dev/urandom of=$TESTDATA bs=256 count=7 + ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" set-bytes $TESTDATA + + rm -f $TESTDATA + + JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj6)" + ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset + JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj7)" + ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset corrupt + JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj8)" + ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset seq + JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj9)" + ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset clone_size + JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj10)" + ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset clone_overlap + JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj11)" + ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset clones + JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj12)" + ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset head + JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj13)" + ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset snaps + JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj14)" + ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset size + + echo "garbage" > $dir/bad + JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj15)" + ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" set-attr snapset $dir/bad + rm -f $dir/bad + + run_osd $dir 0 || return 1 + wait_for_clean || return 1 + + local pgid="${poolid}.0" + if ! pg_scrub "$pgid" ; then + cat $dir/osd.0.log + return 1 + fi + grep 'log_channel' $dir/osd.0.log + + rados list-inconsistent-pg $poolname > $dir/json || return 1 + # Check pg count + test $(jq '. | length' $dir/json) = "1" || return 1 + # Check pgid + test $(jq -r '.[0]' $dir/json) = $pgid || return 1 + + rados list-inconsistent-snapset $pgid > $dir/json || return 1 + test $(jq '.inconsistents | length' $dir/json) = "21" || return 1 + + local jqfilter='.inconsistents' + local sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)' + + jq "$jqfilter" << EOF | python -c "$sortkeys" > $dir/checkcsjson +{ + "inconsistents": [ + { + "errors": [ + "headless" + ], + "snap": 1, + "locator": "", + "nspace": "", + "name": "obj1" + }, + { + "errors": [ + "size_mismatch" + ], + "snap": 1, + "locator": "", + "nspace": "", + "name": "obj10" + }, + { + "errors": [ + "headless" + ], + "snap": 1, + "locator": "", + "nspace": "", + "name": "obj11" + }, + { + "errors": [ + "size_mismatch" + ], + "snap": 1, + "locator": "", + "nspace": "", + "name": "obj14" + }, + { + "errors": [ + "headless" + ], + "snap": 1, + "locator": "", + "nspace": "", + "name": "obj6" + }, + { + "errors": [ + "headless" + ], + "snap": 1, + "locator": "", + "nspace": "", + "name": "obj7" + }, + { + "errors": [ + "size_mismatch" + ], + "snap": 1, + "locator": "", + "nspace": "", + "name": "obj9" + }, + { + "errors": [ + "headless" + ], + "snap": 4, + "locator": "", + "nspace": "", + "name": "obj2" + }, + { + "errors": [ + "size_mismatch" + ], + "snap": 4, + "locator": "", + "nspace": "", + "name": "obj5" + }, + { + "errors": [ + "headless" + ], + "snap": 7, + "locator": "", + "nspace": "", + "name": "obj2" + }, + { + "errors": [ + "oi_attr_missing", + "headless" + ], + "snap": 7, + "locator": "", + "nspace": "", + "name": "obj5" + }, + { + "extra clones": [ + 1 + ], + "errors": [ + "extra_clones" + ], + "snap": "head", + "locator": "", + "nspace": "", + "name": "obj11" + }, + { + "errors": [ + "head_mismatch" + ], + "snap": "head", + "locator": "", + "nspace": "", + "name": "obj12" + }, + { + "errors": [ + "ss_attr_corrupted" + ], + "snap": "head", + "locator": "", + "nspace": "", + "name": "obj15" + }, + { + "extra clones": [ + 7, + 4 + ], + "errors": [ + "ss_attr_missing", + "extra_clones" + ], + "snap": "head", + "locator": "", + "nspace": "", + "name": "obj2" + }, + { + "errors": [ + "size_mismatch" + ], + "snap": "head", + "locator": "", + "nspace": "", + "name": "obj3" + }, + { + "missing": [ + 7 + ], + "errors": [ + "clone_missing" + ], + "snap": "head", + "locator": "", + "nspace": "", + "name": "obj4" + }, + { + "missing": [ + 2, + 1 + ], + "extra clones": [ + 7 + ], + "errors": [ + "extra_clones", + "clone_missing" + ], + "snap": "head", + "locator": "", + "nspace": "", + "name": "obj5" + }, + { + "extra clones": [ + 1 + ], + "errors": [ + "extra_clones" + ], + "snap": "head", + "locator": "", + "nspace": "", + "name": "obj6" + }, + { + "extra clones": [ + 1 + ], + "errors": [ + "head_mismatch", + "extra_clones" + ], + "snap": "head", + "locator": "", + "nspace": "", + "name": "obj7" + }, + { + "errors": [ + "snapset_mismatch" + ], + "snap": "head", + "locator": "", + "nspace": "", + "name": "obj8" + } + ], + "epoch": 20 +} +EOF + + jq "$jqfilter" $dir/json | python -c "$sortkeys" > $dir/csjson + diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || return 1 + + if which jsonschema > /dev/null; + then + jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-snap.json || return 1 + fi + + for i in `seq 1 7` + do + rados -p $poolname rmsnap snap$i + done + + ERRORS=0 + + pidfile=$(find $dir 2>/dev/null | grep $name_prefix'[^/]*\.pid') + pid=$(cat $pidfile) + if ! kill -0 $pid + then + echo "OSD crash occurred" + tail -100 $dir/osd.0.log + ERRORS=$(expr $ERRORS + 1) + fi + + kill_daemons $dir || return 1 + + declare -a err_strings + err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj10:.* is missing in clone_overlap" + err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj5:7 no '_' attr" + err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj5:7 is an unexpected clone" + err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj5:4 on disk size [(]4608[)] does not match object info size [(]512[)] adjusted for ondisk to [(]512[)]" + err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj5:head expected clone .*:::obj5:2" + err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj5:head expected clone .*:::obj5:1" + err_strings[6]="log_channel[(]cluster[)] log [[]INF[]] : scrub [0-9]*[.]0 .*:::obj5:head 2 missing clone[(]s[)]" + err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj12:head snapset.head_exists=false, but head exists" + err_strings[8]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj8:head snaps.seq not set" + err_strings[9]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj7:head snapset.head_exists=false, but head exists" + err_strings[10]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj7:1 is an unexpected clone" + err_strings[11]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj3:head on disk size [(]3840[)] does not match object info size [(]768[)] adjusted for ondisk to [(]768[)]" + err_strings[12]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj6:1 is an unexpected clone" + err_strings[13]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj2:head no 'snapset' attr" + err_strings[14]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj2:7 clone ignored due to missing snapset" + err_strings[15]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj2:4 clone ignored due to missing snapset" + err_strings[16]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj4:head expected clone .*:::obj4:7" + err_strings[17]="log_channel[(]cluster[)] log [[]INF[]] : scrub [0-9]*[.]0 .*:::obj4:head 1 missing clone[(]s[)]" + err_strings[18]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj1:1 is an unexpected clone" + err_strings[19]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj9:1 is missing in clone_size" + err_strings[20]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj11:1 is an unexpected clone" + err_strings[21]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj14:1 size 1032 != clone_size 1033" + err_strings[22]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 23 errors" + err_strings[23]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj15:head can't decode 'snapset' attr buffer" + err_strings[24]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj12:1 has no oi or legacy_snaps; cannot convert 1=[[]1[]]:[[]1[]].stray_clone_snaps=[{]1=[[]1[]][}]" + + for i in `seq 0 ${#err_strings[@]}` + do + if ! grep "${err_strings[$i]}" $dir/osd.0.log > /dev/null; + then + echo "Missing log message '${err_strings[$i]}'" + ERRORS=$(expr $ERRORS + 1) + fi + done + + teardown $dir || return 1 + + if [ $ERRORS != "0" ]; + then + echo "TEST FAILED WITH $ERRORS ERRORS" + return 1 + fi + + echo "TEST PASSED" + return 0 +} + +main osd-scrub-snaps "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && \ +# test/osd/osd-scrub-snaps.sh" diff --git a/qa/suites/rados/singleton/all/erasure-code-nonregression.yaml b/qa/suites/rados/singleton/all/erasure-code-nonregression.yaml new file mode 100644 index 00000000000..3e8fa37616f --- /dev/null +++ b/qa/suites/rados/singleton/all/erasure-code-nonregression.yaml @@ -0,0 +1,17 @@ +roles: +- - mon.a + - mgr.x + - osd.0 + - osd.1 + - osd.2 + - client.0 +openstack: + - volumes: # attached to each instance + count: 3 + size: 10 # GB +tasks: +- install: +- workunit: + client.0: + all: + - erasure-code/encode-decode-non-regression.sh diff --git a/qa/suites/rados/standalone/crush.yaml b/qa/suites/rados/standalone/crush.yaml new file mode 100644 index 00000000000..a62a0dd8184 --- /dev/null +++ b/qa/suites/rados/standalone/crush.yaml @@ -0,0 +1,18 @@ +roles: +- - mon.a + - mgr.x + - osd.0 + - osd.1 + - osd.2 + - client.0 +openstack: + - volumes: # attached to each instance + count: 3 + size: 10 # GB +tasks: +- install: +- workunit: + basedir: qa/standalone + clients: + all: + - crush diff --git a/qa/suites/rados/standalone/erasure-code.yaml b/qa/suites/rados/standalone/erasure-code.yaml new file mode 100644 index 00000000000..7d79753ce73 --- /dev/null +++ b/qa/suites/rados/standalone/erasure-code.yaml @@ -0,0 +1,18 @@ +roles: +- - mon.a + - mgr.x + - osd.0 + - osd.1 + - osd.2 + - client.0 +openstack: + - volumes: # attached to each instance + count: 3 + size: 10 # GB +tasks: +- install: +- workunit: + basedir: qa/standalone + clients: + all: + - erasure-code diff --git a/qa/suites/rados/standalone/misc.yaml b/qa/suites/rados/standalone/misc.yaml new file mode 100644 index 00000000000..4aa9ee27ebc --- /dev/null +++ b/qa/suites/rados/standalone/misc.yaml @@ -0,0 +1,18 @@ +roles: +- - mon.a + - mgr.x + - osd.0 + - osd.1 + - osd.2 + - client.0 +openstack: + - volumes: # attached to each instance + count: 3 + size: 10 # GB +tasks: +- install: +- workunit: + basedir: qa/standalone + clients: + all: + - misc diff --git a/qa/suites/rados/standalone/mon.yaml b/qa/suites/rados/standalone/mon.yaml new file mode 100644 index 00000000000..c19606f4243 --- /dev/null +++ b/qa/suites/rados/standalone/mon.yaml @@ -0,0 +1,18 @@ +roles: +- - mon.a + - mgr.x + - osd.0 + - osd.1 + - osd.2 + - client.0 +openstack: + - volumes: # attached to each instance + count: 3 + size: 10 # GB +tasks: +- install: +- workunit: + basedir: qa/standalone + clients: + all: + - mon diff --git a/qa/suites/rados/standalone/osd.yaml b/qa/suites/rados/standalone/osd.yaml new file mode 100644 index 00000000000..e28b52210d4 --- /dev/null +++ b/qa/suites/rados/standalone/osd.yaml @@ -0,0 +1,18 @@ +roles: +- - mon.a + - mgr.x + - osd.0 + - osd.1 + - osd.2 + - client.0 +openstack: + - volumes: # attached to each instance + count: 3 + size: 10 # GB +tasks: +- install: +- workunit: + basedir: qa/standalone + clients: + all: + - osd diff --git a/qa/workunits/ceph-helpers.sh b/qa/workunits/ceph-helpers.sh deleted file mode 100755 index d1fc2c0fef7..00000000000 --- a/qa/workunits/ceph-helpers.sh +++ /dev/null @@ -1,1909 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2013,2014 Cloudwatt -# Copyright (C) 2014,2015 Red Hat -# Copyright (C) 2014 Federico Gimenez -# -# Author: Loic Dachary -# Author: Federico Gimenez -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# -TIMEOUT=300 -PG_NUM=4 -: ${CEPH_BUILD_VIRTUALENV:=/tmp} - -if type xmlstarlet > /dev/null 2>&1; then - XMLSTARLET=xmlstarlet -elif type xml > /dev/null 2>&1; then - XMLSTARLET=xml -else - echo "Missing xmlstarlet binary!" - exit 1 -fi - -if [ `uname` = FreeBSD ]; then - SED=gsed - DIFFCOLOPTS="" -else - SED=sed - termwidth=$(stty -a | head -1 | sed -e 's/.*columns \([0-9]*\).*/\1/') - if [ -n "$termwidth" -a "$termwidth" != "0" ]; then - termwidth="-W ${termwidth}" - fi - DIFFCOLOPTS="-y $termwidth" -fi - -#! @file ceph-helpers.sh -# @brief Toolbox to manage Ceph cluster dedicated to testing -# -# Example use case: -# -# ~~~~~~~~~~~~~~~~{.sh} -# source ceph-helpers.sh -# -# function mytest() { -# # cleanup leftovers and reset mydir -# setup mydir -# # create a cluster with one monitor and three osds -# run_mon mydir a -# run_osd mydir 0 -# run_osd mydir 2 -# run_osd mydir 3 -# # put and get an object -# rados --pool rbd put GROUP /etc/group -# rados --pool rbd get GROUP /tmp/GROUP -# # stop the cluster and cleanup the directory -# teardown mydir -# } -# ~~~~~~~~~~~~~~~~ -# -# The focus is on simplicity and efficiency, in the context of -# functional tests. The output is intentionally very verbose -# and functions return as soon as an error is found. The caller -# is also expected to abort on the first error so that debugging -# can be done by looking at the end of the output. -# -# Each function is documented, implemented and tested independently. -# When modifying a helper, the test and the documentation are -# expected to be updated and it is easier of they are collocated. A -# test for a given function can be run with -# -# ~~~~~~~~~~~~~~~~{.sh} -# ceph-helpers.sh TESTS test_get_osds -# ~~~~~~~~~~~~~~~~ -# -# and all the tests (i.e. all functions matching test_*) are run -# with: -# -# ~~~~~~~~~~~~~~~~{.sh} -# ceph-helpers.sh TESTS -# ~~~~~~~~~~~~~~~~ -# -# A test function takes a single argument : the directory dedicated -# to the tests. It is expected to not create any file outside of this -# directory and remove it entirely when it completes successfully. -# - - -function get_asok_dir() { - if [ -n "$CEPH_ASOK_DIR" ]; then - echo "$CEPH_ASOK_DIR" - else - echo ${TMPDIR:-/tmp}/ceph-asok.$$ - fi -} - -function get_asok_path() { - local name=$1 - if [ -n "$name" ]; then - echo $(get_asok_dir)/ceph-$name.asok - else - echo $(get_asok_dir)/\$cluster-\$name.asok - fi -} -## -# Cleanup any leftovers found in **dir** via **teardown** -# and reset **dir** as an empty environment. -# -# @param dir path name of the environment -# @return 0 on success, 1 on error -# -function setup() { - local dir=$1 - teardown $dir || return 1 - mkdir -p $dir - mkdir -p $(get_asok_dir) -} - -function test_setup() { - local dir=$dir - setup $dir || return 1 - test -d $dir || return 1 - setup $dir || return 1 - test -d $dir || return 1 - teardown $dir -} - -####################################################################### - -## -# Kill all daemons for which a .pid file exists in **dir** and remove -# **dir**. If the file system in which **dir** is btrfs, delete all -# subvolumes that relate to it. -# -# @param dir path name of the environment -# @return 0 on success, 1 on error -# -function teardown() { - local dir=$1 - kill_daemons $dir KILL - if [ `uname` != FreeBSD ] \ - && [ $(stat -f -c '%T' .) == "btrfs" ]; then - __teardown_btrfs $dir - fi - rm -fr $dir - rm -rf $(get_asok_dir) -} - -function __teardown_btrfs() { - local btrfs_base_dir=$1 - local btrfs_root=$(df -P . | tail -1 | awk '{print $NF}') - local btrfs_dirs=$(cd $btrfs_base_dir; sudo btrfs subvolume list . -t | awk '/^[0-9]/ {print $4}' | grep "$btrfs_base_dir/$btrfs_dir") - for subvolume in $btrfs_dirs; do - sudo btrfs subvolume delete $btrfs_root/$subvolume - done -} - -function test_teardown() { - local dir=$dir - setup $dir || return 1 - teardown $dir || return 1 - ! test -d $dir || return 1 -} - -####################################################################### - -## -# Sends a signal to a single daemon. -# This is a helper function for kill_daemons -# -# After the daemon is sent **signal**, its actual termination -# will be verified by sending it signal 0. If the daemon is -# still alive, kill_daemon will pause for a few seconds and -# try again. This will repeat for a fixed number of times -# before kill_daemon returns on failure. The list of -# sleep intervals can be specified as **delays** and defaults -# to: -# -# 0.1 0.2 1 1 1 2 3 5 5 5 10 10 20 60 60 60 120 -# -# This sequence is designed to run first a very short sleep time (0.1) -# if the machine is fast enough and the daemon terminates in a fraction of a -# second. The increasing sleep numbers should give plenty of time for -# the daemon to die even on the slowest running machine. If a daemon -# takes more than a few minutes to stop (the sum of all sleep times), -# there probably is no point in waiting more and a number of things -# are likely to go wrong anyway: better give up and return on error. -# -# @param pid the process id to send a signal -# @param send_signal the signal to send -# @param delays sequence of sleep times before failure -# -function kill_daemon() { - local pid=$(cat $1) - local send_signal=$2 - local delays=${3:-0.1 0.2 1 1 1 2 3 5 5 5 10 10 20 60 60 60 120} - local exit_code=1 - for try in $delays ; do - if kill -$send_signal $pid 2> /dev/null ; then - exit_code=1 - else - exit_code=0 - break - fi - send_signal=0 - sleep $try - done; - return $exit_code -} - -function test_kill_daemon() { - local dir=$1 - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - - name_prefix=osd - for pidfile in $(find $dir 2>/dev/null | grep $name_prefix'[^/]*\.pid') ; do - # - # sending signal 0 won't kill the daemon - # waiting just for one second instead of the default schedule - # allows us to quickly verify what happens when kill fails - # to stop the daemon (i.e. it must return false) - # - ! kill_daemon $pidfile 0 1 || return 1 - # - # killing just the osd and verify the mon still is responsive - # - kill_daemon $pidfile TERM || return 1 - done - - ceph osd dump | grep "osd.0 down" || return 1 - - name_prefix=mgr - for pidfile in $(find $dir 2>/dev/null | grep $name_prefix'[^/]*\.pid') ; do - # - # kill the mgr - # - kill_daemon $pidfile TERM || return 1 - done - - name_prefix=mon - for pidfile in $(find $dir 2>/dev/null | grep $name_prefix'[^/]*\.pid') ; do - # - # kill the mon and verify it cannot be reached - # - kill_daemon $pidfile TERM || return 1 - ! timeout 5 ceph status || return 1 - done - - teardown $dir || return 1 -} - -## -# Kill all daemons for which a .pid file exists in **dir**. Each -# daemon is sent a **signal** and kill_daemons waits for it to exit -# during a few minutes. By default all daemons are killed. If a -# **name_prefix** is provided, only the daemons for which a pid -# file is found matching the prefix are killed. See run_osd and -# run_mon for more information about the name conventions for -# the pid files. -# -# Send TERM to all daemons : kill_daemons $dir -# Send KILL to all daemons : kill_daemons $dir KILL -# Send KILL to all osds : kill_daemons $dir KILL osd -# Send KILL to osd 1 : kill_daemons $dir KILL osd.1 -# -# If a daemon is sent the TERM signal and does not terminate -# within a few minutes, it will still be running even after -# kill_daemons returns. -# -# If all daemons are kill successfully the function returns 0 -# if at least one daemon remains, this is treated as an -# error and the function return 1. -# -# @param dir path name of the environment -# @param signal name of the first signal (defaults to TERM) -# @param name_prefix only kill match daemons (defaults to all) -# @param delays sequence of sleep times before failure -# @return 0 on success, 1 on error -# -function kill_daemons() { - local trace=$(shopt -q -o xtrace && echo true || echo false) - $trace && shopt -u -o xtrace - local dir=$1 - local signal=${2:-TERM} - local name_prefix=$3 # optional, osd, mon, osd.1 - local delays=$4 #optional timing - local status=0 - local pids="" - - for pidfile in $(find $dir 2>/dev/null | grep $name_prefix'[^/]*\.pid') ; do - run_in_background pids kill_daemon $pidfile $signal $delays - done - - wait_background pids - status=$? - - $trace && shopt -s -o xtrace - return $status -} - -function test_kill_daemons() { - local dir=$1 - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - # - # sending signal 0 won't kill the daemon - # waiting just for one second instead of the default schedule - # allows us to quickly verify what happens when kill fails - # to stop the daemon (i.e. it must return false) - # - ! kill_daemons $dir 0 osd 1 || return 1 - # - # killing just the osd and verify the mon still is responsive - # - kill_daemons $dir TERM osd || return 1 - ceph osd dump | grep "osd.0 down" || return 1 - # - # kill the mgr - # - kill_daemons $dir TERM mgr || return 1 - # - # kill the mon and verify it cannot be reached - # - kill_daemons $dir TERM || return 1 - ! timeout 5 ceph status || return 1 - teardown $dir || return 1 -} - -####################################################################### - -## -# Run a monitor by the name mon.**id** with data in **dir**/**id**. -# The logs can be found in **dir**/mon.**id**.log and the pid file -# is **dir**/mon.**id**.pid and the admin socket is -# **dir**/**id**/ceph-mon.**id**.asok. -# -# The remaining arguments are passed verbatim to ceph-mon --mkfs -# and the ceph-mon daemon. -# -# Two mandatory arguments must be provided: --fsid and --mon-host -# Instead of adding them to every call to run_mon, they can be -# set in the CEPH_ARGS environment variable to be read implicitly -# by every ceph command. -# -# The CEPH_CONF variable is expected to be set to /dev/null to -# only rely on arguments for configuration. -# -# Examples: -# -# CEPH_ARGS="--fsid=$(uuidgen) " -# CEPH_ARGS+="--mon-host=127.0.0.1:7018 " -# run_mon $dir a # spawn a mon and bind port 7018 -# run_mon $dir a --debug-filestore=20 # spawn with filestore debugging -# -# If mon_initial_members is not set, the default rbd pool is deleted -# and replaced with a replicated pool with less placement groups to -# speed up initialization. If mon_initial_members is set, no attempt -# is made to recreate the rbd pool because it would hang forever, -# waiting for other mons to join. -# -# A **dir**/ceph.conf file is created but not meant to be used by any -# function. It is convenient for debugging a failure with: -# -# ceph --conf **dir**/ceph.conf -s -# -# @param dir path name of the environment -# @param id mon identifier -# @param ... can be any option valid for ceph-mon -# @return 0 on success, 1 on error -# -function run_mon_no_pool() { - local dir=$1 - shift - local id=$1 - shift - local data=$dir/$id - - ceph-mon \ - --id $id \ - --mkfs \ - --mon-data=$data \ - --run-dir=$dir \ - "$@" || return 1 - - ceph-mon \ - --id $id \ - --mon-osd-full-ratio=.99 \ - --mon-data-avail-crit=1 \ - --paxos-propose-interval=0.1 \ - --osd-crush-chooseleaf-type=0 \ - --erasure-code-dir=$CEPH_LIB \ - --plugin-dir=$CEPH_LIB \ - --debug-mon 20 \ - --debug-ms 20 \ - --debug-paxos 20 \ - --chdir= \ - --mon-data=$data \ - --log-file=$dir/\$name.log \ - --admin-socket=$(get_asok_path) \ - --mon-cluster-log-file=$dir/log \ - --run-dir=$dir \ - --pid-file=$dir/\$name.pid \ - --mon-allow-pool-delete \ - "$@" || return 1 - - cat > $dir/ceph.conf </dev/null | \ - jq '.acting | .[]') - # get rid of the trailing space - echo $osds -} - -function test_get_osds() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=2 || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - run_osd $dir 1 || return 1 - wait_for_clean || return 1 - get_osds rbd GROUP | grep --quiet '^[0-1] [0-1]$' || return 1 - teardown $dir || return 1 -} - -####################################################################### - -## -# Wait for the monitor to form quorum (optionally, of size N) -# -# @param timeout duration (lower-bound) to wait for quorum to be formed -# @param quorumsize size of quorum to wait for -# @return 0 on success, 1 on error -# -function wait_for_quorum() { - local timeout=$1 - local quorumsize=$2 - - if [[ -z "$timeout" ]]; then - timeout=300 - fi - - if [[ -z "$quorumsize" ]]; then - timeout $timeout ceph mon_status --format=json >&/dev/null || return 1 - return 0 - fi - - no_quorum=1 - wait_until=$((`date +%s` + $timeout)) - while [[ $(date +%s) -lt $wait_until ]]; do - jqfilter='.quorum | length == '$quorumsize - jqinput="$(timeout $timeout ceph mon_status --format=json 2>/dev/null)" - res=$(echo $jqinput | jq "$jqfilter") - if [[ "$res" == "true" ]]; then - no_quorum=0 - break - fi - done - return $no_quorum -} - -####################################################################### - -## -# Return the PG of supporting the **objectname** stored in -# **poolname**, as reported by ceph osd map. -# -# @param poolname an existing pool -# @param objectname an objectname (may or may not exist) -# @param STDOUT a PG -# @return 0 on success, 1 on error -# -function get_pg() { - local poolname=$1 - local objectname=$2 - - ceph --format json osd map $poolname $objectname 2>/dev/null | jq -r '.pgid' -} - -function test_get_pg() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - wait_for_clean || return 1 - get_pg rbd GROUP | grep --quiet '^[0-9]\.[0-9a-f][0-9a-f]*$' || return 1 - teardown $dir || return 1 -} - -####################################################################### - -## -# Return the value of the **config**, obtained via the config get command -# of the admin socket of **daemon**.**id**. -# -# @param daemon mon or osd -# @param id mon or osd ID -# @param config the configuration variable name as found in config_opts.h -# @param STDOUT the config value -# @return 0 on success, 1 on error -# -function get_config() { - local daemon=$1 - local id=$2 - local config=$3 - - CEPH_ARGS='' \ - ceph --format json daemon $(get_asok_path $daemon.$id) \ - config get $config 2> /dev/null | \ - jq -r ".$config" -} - -function test_get_config() { - local dir=$1 - - # override the default config using command line arg and check it - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 - test $(get_config mon a osd_pool_default_size) = 1 || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 --osd_max_scrubs=3 || return 1 - test $(get_config osd 0 osd_max_scrubs) = 3 || return 1 - teardown $dir || return 1 -} - -####################################################################### - -## -# Set the **config** to specified **value**, via the config set command -# of the admin socket of **daemon**.**id** -# -# @param daemon mon or osd -# @param id mon or osd ID -# @param config the configuration variable name as found in config_opts.h -# @param value the config value -# @return 0 on success, 1 on error -# -function set_config() { - local daemon=$1 - local id=$2 - local config=$3 - local value=$4 - - test $(env CEPH_ARGS='' ceph --format json daemon $(get_asok_path $daemon.$id) \ - config set $config $value 2> /dev/null | \ - jq 'has("success")') == true -} - -function test_set_config() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 - test $(get_config mon a ms_crc_header) = true || return 1 - set_config mon a ms_crc_header false || return 1 - test $(get_config mon a ms_crc_header) = false || return 1 - set_config mon a ms_crc_header true || return 1 - test $(get_config mon a ms_crc_header) = true || return 1 - teardown $dir || return 1 -} - -####################################################################### - -## -# Return the OSD id of the primary OSD supporting the **objectname** -# stored in **poolname**, as reported by ceph osd map. -# -# @param poolname an existing pool -# @param objectname an objectname (may or may not exist) -# @param STDOUT the primary OSD id -# @return 0 on success, 1 on error -# -function get_primary() { - local poolname=$1 - local objectname=$2 - - ceph --format json osd map $poolname $objectname 2>/dev/null | \ - jq '.acting_primary' -} - -function test_get_primary() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 - local osd=0 - run_mgr $dir x || return 1 - run_osd $dir $osd || return 1 - wait_for_clean || return 1 - test $(get_primary rbd GROUP) = $osd || return 1 - teardown $dir || return 1 -} - -####################################################################### - -## -# Return the id of any OSD supporting the **objectname** stored in -# **poolname**, as reported by ceph osd map, except the primary. -# -# @param poolname an existing pool -# @param objectname an objectname (may or may not exist) -# @param STDOUT the OSD id -# @return 0 on success, 1 on error -# -function get_not_primary() { - local poolname=$1 - local objectname=$2 - - local primary=$(get_primary $poolname $objectname) - ceph --format json osd map $poolname $objectname 2>/dev/null | \ - jq ".acting | map(select (. != $primary)) | .[0]" -} - -function test_get_not_primary() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=2 || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - run_osd $dir 1 || return 1 - wait_for_clean || return 1 - local primary=$(get_primary rbd GROUP) - local not_primary=$(get_not_primary rbd GROUP) - test $not_primary != $primary || return 1 - test $not_primary = 0 -o $not_primary = 1 || return 1 - teardown $dir || return 1 -} - -####################################################################### - -## -# Run ceph-objectstore-tool against the OSD **id** using the data path -# **dir**. The OSD is killed with TERM prior to running -# ceph-objectstore-tool because access to the data path is -# exclusive. The OSD is restarted after the command completes. The -# objectstore_tool returns after all PG are active+clean again. -# -# @param dir the data path of the OSD -# @param id the OSD id -# @param ... arguments to ceph-objectstore-tool -# @param STDIN the input of ceph-objectstore-tool -# @param STDOUT the output of ceph-objectstore-tool -# @return 0 on success, 1 on error -# -# The value of $ceph_osd_args will be passed to restarted osds -# -function objectstore_tool() { - local dir=$1 - shift - local id=$1 - shift - local osd_data=$dir/$id - - local osd_type=$(cat $osd_data/type) - - kill_daemons $dir TERM osd.$id >&2 < /dev/null || return 1 - - local journal_args - if [ "$objectstore_type" == "filestore" ]; then - journal_args=" --journal-path $osd_data/journal" - fi - ceph-objectstore-tool \ - --data-path $osd_data \ - $journal_args \ - "$@" || return 1 - activate_osd $dir $id $ceph_osd_args >&2 || return 1 - wait_for_clean >&2 -} - -function test_objectstore_tool() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 - local osd=0 - run_mgr $dir x || return 1 - run_osd $dir $osd || return 1 - wait_for_clean || return 1 - rados --pool rbd put GROUP /etc/group || return 1 - objectstore_tool $dir $osd GROUP get-bytes | \ - diff - /etc/group - ! objectstore_tool $dir $osd NOTEXISTS get-bytes || return 1 - teardown $dir || return 1 -} - -####################################################################### - -## -# Predicate checking if there is an ongoing recovery in the -# cluster. If any of the recovering_{keys,bytes,objects}_per_sec -# counters are reported by ceph status, it means recovery is in -# progress. -# -# @return 0 if recovery in progress, 1 otherwise -# -function get_is_making_recovery_progress() { - local recovery_progress - recovery_progress+=".recovering_keys_per_sec + " - recovery_progress+=".recovering_bytes_per_sec + " - recovery_progress+=".recovering_objects_per_sec" - local progress=$(ceph --format json status 2>/dev/null | \ - jq -r ".pgmap | $recovery_progress") - test "$progress" != null -} - -function test_get_is_making_recovery_progress() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - ! get_is_making_recovery_progress || return 1 - teardown $dir || return 1 -} - -####################################################################### - -## -# Return the number of active PGs in the cluster. A PG is active if -# ceph pg dump pgs reports it both **active** and **clean** and that -# not **stale**. -# -# @param STDOUT the number of active PGs -# @return 0 on success, 1 on error -# -function get_num_active_clean() { - local expression - expression+="select(contains(\"active\") and contains(\"clean\")) | " - expression+="select(contains(\"stale\") | not)" - ceph --format json pg dump pgs 2>/dev/null | \ - jq "[.[] | .state | $expression] | length" -} - -function test_get_num_active_clean() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - wait_for_clean || return 1 - local num_active_clean=$(get_num_active_clean) - test "$num_active_clean" = $PG_NUM || return 1 - teardown $dir || return 1 -} - -####################################################################### - -## -# Return the number of PGs in the cluster, according to -# ceph pg dump pgs. -# -# @param STDOUT the number of PGs -# @return 0 on success, 1 on error -# -function get_num_pgs() { - ceph --format json status 2>/dev/null | jq '.pgmap.num_pgs' -} - -function test_get_num_pgs() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - wait_for_clean || return 1 - local num_pgs=$(get_num_pgs) - test "$num_pgs" -gt 0 || return 1 - teardown $dir || return 1 -} - -####################################################################### - -## -# Return the OSD ids in use by at least one PG in the cluster (either -# in the up or the acting set), according to ceph pg dump pgs. Every -# OSD id shows as many times as they are used in up and acting sets. -# If an OSD id is in both the up and acting set of a given PG, it will -# show twice. -# -# @param STDOUT a sorted list of OSD ids -# @return 0 on success, 1 on error -# -function get_osd_id_used_by_pgs() { - ceph --format json pg dump pgs 2>/dev/null | jq '.[] | .up[], .acting[]' | sort -} - -function test_get_osd_id_used_by_pgs() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - wait_for_clean || return 1 - local osd_ids=$(get_osd_id_used_by_pgs | uniq) - test "$osd_ids" = "0" || return 1 - teardown $dir || return 1 -} - -####################################################################### - -## -# Wait until the OSD **id** shows **count** times in the -# PGs (see get_osd_id_used_by_pgs for more information about -# how OSD ids are counted). -# -# @param id the OSD id -# @param count the number of time it must show in the PGs -# @return 0 on success, 1 on error -# -function wait_osd_id_used_by_pgs() { - local id=$1 - local count=$2 - - status=1 - for ((i=0; i < $TIMEOUT / 5; i++)); do - echo $i - if ! test $(get_osd_id_used_by_pgs | grep -c $id) = $count ; then - sleep 5 - else - status=0 - break - fi - done - return $status -} - -function test_wait_osd_id_used_by_pgs() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - wait_for_clean || return 1 - wait_osd_id_used_by_pgs 0 8 || return 1 - ! TIMEOUT=1 wait_osd_id_used_by_pgs 123 5 || return 1 - teardown $dir || return 1 -} - -####################################################################### - -## -# Return the date and time of the last completed scrub for **pgid**, -# as reported by ceph pg dump pgs. Note that a repair also sets this -# date. -# -# @param pgid the id of the PG -# @param STDOUT the date and time of the last scrub -# @return 0 on success, 1 on error -# -function get_last_scrub_stamp() { - local pgid=$1 - local sname=${2:-last_scrub_stamp} - ceph --format json pg dump pgs 2>/dev/null | \ - jq -r ".[] | select(.pgid==\"$pgid\") | .$sname" -} - -function test_get_last_scrub_stamp() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - wait_for_clean || return 1 - stamp=$(get_last_scrub_stamp 2.0) - test -n "$stamp" || return 1 - teardown $dir || return 1 -} - -####################################################################### - -## -# Predicate checking if the cluster is clean, i.e. all of its PGs are -# in a clean state (see get_num_active_clean for a definition). -# -# @return 0 if the cluster is clean, 1 otherwise -# -function is_clean() { - num_pgs=$(get_num_pgs) - test $num_pgs != 0 || return 1 - test $(get_num_active_clean) = $num_pgs || return 1 -} - -function test_is_clean() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - wait_for_clean || return 1 - is_clean || return 1 - teardown $dir || return 1 -} - -####################################################################### - -## -# Return a list of numbers that are increasingly larger and whose -# total is **timeout** seconds. It can be used to have short sleep -# delay while waiting for an event on a fast machine. But if running -# very slowly the larger delays avoid stressing the machine even -# further or spamming the logs. -# -# @param timeout sum of all delays, in seconds -# @return a list of sleep delays -# -function get_timeout_delays() { - local trace=$(shopt -q -o xtrace && echo true || echo false) - $trace && shopt -u -o xtrace - local timeout=$1 - local first_step=${2:-1} - - local i - local total="0" - i=$first_step - while test "$(echo $total + $i \<= $timeout | bc -l)" = "1"; do - echo -n "$i " - total=$(echo $total + $i | bc -l) - i=$(echo $i \* 2 | bc -l) - done - if test "$(echo $total \< $timeout | bc -l)" = "1"; then - echo -n $(echo $timeout - $total | bc -l) - fi - $trace && shopt -s -o xtrace -} - -function test_get_timeout_delays() { - test "$(get_timeout_delays 1)" = "1 " || return 1 - test "$(get_timeout_delays 5)" = "1 2 2" || return 1 - test "$(get_timeout_delays 6)" = "1 2 3" || return 1 - test "$(get_timeout_delays 7)" = "1 2 4 " || return 1 - test "$(get_timeout_delays 8)" = "1 2 4 1" || return 1 - test "$(get_timeout_delays 1 .1)" = ".1 .2 .4 .3" || return 1 - test "$(get_timeout_delays 1.5 .1)" = ".1 .2 .4 .8 " || return 1 - test "$(get_timeout_delays 5 .1)" = ".1 .2 .4 .8 1.6 1.9" || return 1 - test "$(get_timeout_delays 6 .1)" = ".1 .2 .4 .8 1.6 2.9" || return 1 - test "$(get_timeout_delays 6.3 .1)" = ".1 .2 .4 .8 1.6 3.2 " || return 1 - test "$(get_timeout_delays 20 .1)" = ".1 .2 .4 .8 1.6 3.2 6.4 7.3" || return 1 -} - -####################################################################### - -## -# Wait until the cluster becomes clean or if it does not make progress -# for $TIMEOUT seconds. -# Progress is measured either via the **get_is_making_recovery_progress** -# predicate or if the number of clean PGs changes (as returned by get_num_active_clean) -# -# @return 0 if the cluster is clean, 1 otherwise -# -function wait_for_clean() { - local num_active_clean=-1 - local cur_active_clean - local -a delays=($(get_timeout_delays $TIMEOUT .1)) - local -i loop=0 - - while test $(get_num_pgs) == 0 ; do - sleep 1 - done - - while true ; do - # Comparing get_num_active_clean & get_num_pgs is used to determine - # if the cluster is clean. That's almost an inline of is_clean() to - # get more performance by avoiding multiple calls of get_num_active_clean. - cur_active_clean=$(get_num_active_clean) - test $cur_active_clean = $(get_num_pgs) && break - if test $cur_active_clean != $num_active_clean ; then - loop=0 - num_active_clean=$cur_active_clean - elif get_is_making_recovery_progress ; then - loop=0 - elif (( $loop >= ${#delays[*]} )) ; then - ceph report - return 1 - fi - sleep ${delays[$loop]} - loop+=1 - done - return 0 -} - -function test_wait_for_clean() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 - run_mgr $dir x || return 1 - ! TIMEOUT=1 wait_for_clean || return 1 - run_osd $dir 0 || return 1 - wait_for_clean || return 1 - teardown $dir || return 1 -} - -####################################################################### - -## -# Wait until the cluster becomes HEALTH_OK again or if it does not make progress -# for $TIMEOUT seconds. -# -# @return 0 if the cluster is HEALTHY, 1 otherwise -# -function wait_for_health() { - local grepstr=$1 - local -a delays=($(get_timeout_delays $TIMEOUT .1)) - local -i loop=0 - - while ! ceph health detail | grep "$grepstr" ; do - if (( $loop >= ${#delays[*]} )) ; then - ceph health detail - return 1 - fi - sleep ${delays[$loop]} - loop+=1 - done -} - -function wait_for_health_ok() { - wait_for_health "HEALTH_OK" || return 1 -} - -function test_wait_for_health_ok() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 --osd_failsafe_full_ratio=.99 --mon_pg_warn_min_per_osd=0 || return 1 - run_mgr $dir x --mon_pg_warn_min_per_osd=0 || return 1 - run_osd $dir 0 || return 1 - kill_daemons $dir TERM osd || return 1 - ! TIMEOUT=1 wait_for_health_ok || return 1 - activate_osd $dir 0 || return 1 - wait_for_health_ok || return 1 - teardown $dir || return 1 -} - - -####################################################################### - -## -# Run repair on **pgid** and wait until it completes. The repair -# function will fail if repair does not complete within $TIMEOUT -# seconds. -# -# @param pgid the id of the PG -# @return 0 on success, 1 on error -# -function repair() { - local pgid=$1 - local last_scrub=$(get_last_scrub_stamp $pgid) - ceph pg repair $pgid - wait_for_scrub $pgid "$last_scrub" -} - -function test_repair() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - wait_for_clean || return 1 - repair 2.0 || return 1 - kill_daemons $dir KILL osd || return 1 - ! TIMEOUT=1 repair 2.0 || return 1 - teardown $dir || return 1 -} -####################################################################### - -## -# Run scrub on **pgid** and wait until it completes. The pg_scrub -# function will fail if repair does not complete within $TIMEOUT -# seconds. The pg_scrub is complete whenever the -# **get_last_scrub_stamp** function reports a timestamp different from -# the one stored before starting the scrub. -# -# @param pgid the id of the PG -# @return 0 on success, 1 on error -# -function pg_scrub() { - local pgid=$1 - local last_scrub=$(get_last_scrub_stamp $pgid) - ceph pg scrub $pgid - wait_for_scrub $pgid "$last_scrub" -} - -function pg_deep_scrub() { - local pgid=$1 - local last_scrub=$(get_last_scrub_stamp $pgid last_deep_scrub_stamp) - ceph pg deep-scrub $pgid - wait_for_scrub $pgid "$last_scrub" last_deep_scrub_stamp -} - -function test_pg_scrub() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - wait_for_clean || return 1 - pg_scrub 2.0 || return 1 - kill_daemons $dir KILL osd || return 1 - ! TIMEOUT=1 pg_scrub 2.0 || return 1 - teardown $dir || return 1 -} - -####################################################################### - -## -# Run the *command* and expect it to fail (i.e. return a non zero status). -# The output (stderr and stdout) is stored in a temporary file in *dir* -# and is expected to contain the string *expected*. -# -# Return 0 if the command failed and the string was found. Otherwise -# return 1 and cat the full output of the command on stderr for debug. -# -# @param dir temporary directory to store the output -# @param expected string to look for in the output -# @param command ... the command and its arguments -# @return 0 on success, 1 on error -# - -function expect_failure() { - local dir=$1 - shift - local expected="$1" - shift - local success - - if "$@" > $dir/out 2>&1 ; then - success=true - else - success=false - fi - - if $success || ! grep --quiet "$expected" $dir/out ; then - cat $dir/out >&2 - return 1 - else - return 0 - fi -} - -function test_expect_failure() { - local dir=$1 - - setup $dir || return 1 - expect_failure $dir FAIL bash -c 'echo FAIL ; exit 1' || return 1 - # the command did not fail - ! expect_failure $dir FAIL bash -c 'echo FAIL ; exit 0' > $dir/out || return 1 - grep --quiet FAIL $dir/out || return 1 - # the command failed but the output does not contain the expected string - ! expect_failure $dir FAIL bash -c 'echo UNEXPECTED ; exit 1' > $dir/out || return 1 - ! grep --quiet FAIL $dir/out || return 1 - teardown $dir || return 1 -} - -####################################################################### - -## -# Given the *last_scrub*, wait for scrub to happen on **pgid**. It -# will fail if scrub does not complete within $TIMEOUT seconds. The -# repair is complete whenever the **get_last_scrub_stamp** function -# reports a timestamp different from the one given in argument. -# -# @param pgid the id of the PG -# @param last_scrub timestamp of the last scrub for *pgid* -# @return 0 on success, 1 on error -# -function wait_for_scrub() { - local pgid=$1 - local last_scrub="$2" - local sname=${3:-last_scrub_stamp} - - for ((i=0; i < $TIMEOUT; i++)); do - if test "$last_scrub" != "$(get_last_scrub_stamp $pgid $sname)" ; then - return 0 - fi - sleep 1 - done - return 1 -} - -function test_wait_for_scrub() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - wait_for_clean || return 1 - local pgid=2.0 - ceph pg repair $pgid - local last_scrub=$(get_last_scrub_stamp $pgid) - wait_for_scrub $pgid "$last_scrub" || return 1 - kill_daemons $dir KILL osd || return 1 - last_scrub=$(get_last_scrub_stamp $pgid) - ! TIMEOUT=1 wait_for_scrub $pgid "$last_scrub" || return 1 - teardown $dir || return 1 -} - -####################################################################### - -## -# Return 0 if the erasure code *plugin* is available, 1 otherwise. -# -# @param plugin erasure code plugin -# @return 0 on success, 1 on error -# - -function erasure_code_plugin_exists() { - local plugin=$1 - local status - local grepstr - local s - case `uname` in - FreeBSD) grepstr="Cannot open.*$plugin" ;; - *) grepstr="$plugin.*No such file" ;; - esac - - s=$(ceph osd erasure-code-profile set TESTPROFILE plugin=$plugin 2>&1) - local status=$? - if [ $status -eq 0 ]; then - ceph osd erasure-code-profile rm TESTPROFILE - elif ! echo $s | grep --quiet "$grepstr" ; then - status=1 - # display why the string was rejected. - echo $s - fi - return $status -} - -function test_erasure_code_plugin_exists() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - erasure_code_plugin_exists jerasure || return 1 - ! erasure_code_plugin_exists FAKE || return 1 - teardown $dir || return 1 -} - -####################################################################### - -## -# Display all log files from **dir** on stdout. -# -# @param dir directory in which all data is stored -# - -function display_logs() { - local dir=$1 - - find $dir -maxdepth 1 -name '*.log' | \ - while read file ; do - echo "======================= $file" - cat $file - done -} - -function test_display_logs() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a || return 1 - kill_daemons $dir || return 1 - display_logs $dir > $dir/log.out - grep --quiet mon.a.log $dir/log.out || return 1 - teardown $dir || return 1 -} - -####################################################################### -## -# Spawn a command in background and save the pid in the variable name -# passed in argument. To make the output reading easier, the output is -# prepend with the process id. -# -# Example: -# pids1="" -# run_in_background pids1 bash -c 'sleep 1; exit 1' -# -# @param pid_variable the variable name (not value) where the pids will be stored -# @param ... the command to execute -# @return only the pid_variable output should be considered and used with **wait_background** -# -function run_in_background() { - local pid_variable=$1 - shift; - # Execute the command and prepend the output with its pid - # We enforce to return the exit status of the command and not the awk one. - ("$@" |& awk '{ a[i++] = $0 }END{for (i = 0; i in a; ++i) { print "'$$': " a[i]} }'; return ${PIPESTATUS[0]}) >&2 & - eval "$pid_variable+=\" $!\"" -} - -function test_run_in_background() { - local pids - run_in_background pids sleep 1 - run_in_background pids sleep 1 - test $(echo $pids | wc -w) = 2 || return 1 - wait $pids || return 1 -} - -####################################################################### -## -# Wait for pids running in background to complete. -# This function is usually used after a **run_in_background** call -# Example: -# pids1="" -# run_in_background pids1 bash -c 'sleep 1; exit 1' -# wait_background pids1 -# -# @param pids The variable name that contains the active PIDS. Set as empty at then end of the function. -# @return returns 1 if at least one process exits in error unless returns 0 -# -function wait_background() { - # We extract the PIDS from the variable name - pids=${!1} - - return_code=0 - for pid in $pids; do - if ! wait $pid; then - # If one process failed then return 1 - return_code=1 - fi - done - - # We empty the variable reporting that all process ended - eval "$1=''" - - return $return_code -} - - -function test_wait_background() { - local pids="" - run_in_background pids bash -c "sleep 1; exit 1" - run_in_background pids bash -c "sleep 2; exit 0" - wait_background pids - if [ $? -ne 1 ]; then return 1; fi - - run_in_background pids bash -c "sleep 1; exit 0" - run_in_background pids bash -c "sleep 2; exit 0" - wait_background pids - if [ $? -ne 0 ]; then return 1; fi - - if [ ! -z "$pids" ]; then return 1; fi -} - -function flush_pg_stats() -{ - local timeout=${1:-$TIMEOUT} - - ids=`ceph osd ls` - seqs='' - for osd in $ids; do - seq=`ceph tell osd.$osd flush_pg_stats` - seqs="$seqs $osd-$seq" - done - - for s in $seqs; do - osd=`echo $s | cut -d - -f 1` - seq=`echo $s | cut -d - -f 2` - echo "waiting osd.$osd seq $seq" - while test $(ceph osd last-stat-seq $osd) -lt $seq; do - sleep 1 - if [ $((timeout--)) -eq 0 ]; then - return 1 - fi - done - done -} - -function test_flush_pg_stats() -{ - local dir=$1 - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - rados -p rbd put obj /etc/group - flush_pg_stats - local jq_filter='.pools | .[] | select(.name == "rbd") | .stats' - raw_bytes_used=`ceph df detail --format=json | jq "$jq_filter.raw_bytes_used"` - bytes_used=`ceph df detail --format=json | jq "$jq_filter.bytes_used"` - test $raw_bytes_used > 0 || return 1 - test $raw_bytes_used == $bytes_used || return 1 -} - -####################################################################### - -## -# Call the **run** function (which must be defined by the caller) with -# the **dir** argument followed by the caller argument list. -# -# If the **run** function returns on error, all logs found in **dir** -# are displayed for diagnostic purposes. -# -# **teardown** function is called when the **run** function returns -# (on success or on error), to cleanup leftovers. The CEPH_CONF is set -# to /dev/null and CEPH_ARGS is unset so that the tests are protected from -# external interferences. -# -# It is the responsibility of the **run** function to call the -# **setup** function to prepare the test environment (create a temporary -# directory etc.). -# -# The shell is required (via PS4) to display the function and line -# number whenever a statement is executed to help debugging. -# -# @param dir directory in which all data is stored -# @param ... arguments passed transparently to **run** -# @return 0 on success, 1 on error -# -function main() { - local dir=td/$1 - shift - - shopt -s -o xtrace - PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: ' - - export PATH=${CEPH_BUILD_VIRTUALENV}/ceph-disk-virtualenv/bin:${CEPH_BUILD_VIRTUALENV}/ceph-detect-init-virtualenv/bin:.:$PATH # make sure program from sources are preferred - #export PATH=$CEPH_ROOT/src/ceph-disk/virtualenv/bin:$CEPH_ROOT/src/ceph-detect-init/virtualenv/bin:.:$PATH # make sure program from sources are preferred - - export CEPH_CONF=/dev/null - unset CEPH_ARGS - - local code - if run $dir "$@" ; then - code=0 - else - display_logs $dir - code=1 - fi - teardown $dir || return 1 - return $code -} - -####################################################################### - -function run_tests() { - shopt -s -o xtrace - PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: ' - - export PATH=${CEPH_BUILD_VIRTUALENV}/ceph-disk-virtualenv/bin:${CEPH_BUILD_VIRTUALENV}/ceph-detect-init-virtualenv/bin:.:$PATH # make sure program from sources are preferred - #export PATH=$CEPH_ROOT/src/ceph-disk/virtualenv/bin:$CEPH_ROOT/src/ceph-detect-init/virtualenv/bin:.:$PATH # make sure program from sources are preferred - - export CEPH_MON="127.0.0.1:7109" # git grep '\<7109\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - export CEPH_CONF=/dev/null - - local funcs=${@:-$(set | sed -n -e 's/^\(test_[0-9a-z_]*\) .*/\1/p')} - local dir=td/ceph-helpers - - for func in $funcs ; do - $func $dir || return 1 - done -} - -if test "$1" = TESTS ; then - shift - run_tests "$@" -fi - -# NOTE: -# jq only support --exit-status|-e from version 1.4 forwards, which makes -# returning on error waaaay prettier and straightforward. -# However, the current automated upstream build is running with v1.3, -# which has no idea what -e is. Hence the convoluted error checking we -# need. Sad. -# The next time someone changes this code, please check if v1.4 is now -# a thing, and, if so, please change these to use -e. Thanks. - -# jq '.all.supported | select([.[] == "foo"] | any)' -function jq_success() { - input="$1" - filter="$2" - expects="\"$3\"" - - in_escaped=$(printf %s "$input" | sed "s/'/'\\\\''/g") - filter_escaped=$(printf %s "$filter" | sed "s/'/'\\\\''/g") - - ret=$(echo "$in_escaped" | jq "$filter_escaped") - if [[ "$ret" == "true" ]]; then - return 0 - elif [[ -n "$expects" ]]; then - if [[ "$ret" == "$expects" ]]; then - return 0 - fi - fi - return 1 - input=$1 - filter=$2 - expects="$3" - - ret="$(echo $input | jq \"$filter\")" - if [[ "$ret" == "true" ]]; then - return 0 - elif [[ -n "$expects" && "$ret" == "$expects" ]]; then - return 0 - fi - return 1 -} - -# Local Variables: -# compile-command: "cd ../../src ; make -j4 && ../qa/workunits/ceph-helpers.sh TESTS # test_get_config" -# End: diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh index 41a036e8514..8c1d41aed3f 100755 --- a/qa/workunits/cephtool/test.sh +++ b/qa/workunits/cephtool/test.sh @@ -2,7 +2,7 @@ # -*- mode:shell-script; tab-width:8; sh-basic-offset:2; indent-tabs-mode:t -*- # vim: ts=8 sw=8 ft=bash smarttab -source $(dirname $0)/../ceph-helpers.sh +source $(dirname $0)/../../standalone/ceph-helpers.sh set -e set -o functrace diff --git a/src/ceph-disk/tests/ceph-disk.sh b/src/ceph-disk/tests/ceph-disk.sh index eaa1c45af15..de349f95d27 100755 --- a/src/ceph-disk/tests/ceph-disk.sh +++ b/src/ceph-disk/tests/ceph-disk.sh @@ -25,7 +25,7 @@ if [ -z "$CEPH_ROOT" ] || [ -z "$CEPH_BIN" ] || [ -z "$CEPH_LIB" ]; then CEPH_BIN=$CEPH_ROOT CEPH_LIB=$CEPH_ROOT/.libs fi -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh set -x diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index cf0f028cbad..50c860dea8a 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -556,16 +556,6 @@ if(NOT FREEBSD) add_dependencies(tests ceph-detect-init) endif(NOT FREEBSD) -add_ceph_test(test-ceph-helpers.sh ${CMAKE_CURRENT_SOURCE_DIR}/test-ceph-helpers.sh) -add_ceph_test(erasure-decode-non-regression.sh ${CMAKE_SOURCE_DIR}/qa/workunits/erasure-code/encode-decode-non-regression.sh) - -add_ceph_test(ceph_objectstore_tool.py ${CMAKE_CURRENT_SOURCE_DIR}/ceph_objectstore_tool.py) -if(WITH_LIBCEPHFS) - add_ceph_test(cephtool-test-mds.sh ${CMAKE_CURRENT_SOURCE_DIR}/cephtool-test-mds.sh) -endif(WITH_LIBCEPHFS) -add_ceph_test(cephtool-test-mon.sh ${CMAKE_CURRENT_SOURCE_DIR}/cephtool-test-mon.sh) -add_ceph_test(cephtool-test-osd.sh ${CMAKE_CURRENT_SOURCE_DIR}/cephtool-test-osd.sh) -add_ceph_test(cephtool-test-rados.sh ${CMAKE_CURRENT_SOURCE_DIR}/cephtool-test-rados.sh) if(WITH_RBD) add_ceph_test(run-rbd-unit-tests.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh) endif(WITH_RBD) diff --git a/src/test/cephtool-test-mds.sh b/src/test/cephtool-test-mds.sh deleted file mode 100755 index 13bba3188e5..00000000000 --- a/src/test/cephtool-test-mds.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2014, 2015 Red Hat -# Copyright (C) 2013 Cloudwatt -# -# Author: Loic Dachary -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# -source $(dirname $0)/detect-build-env-vars.sh - -CEPH_CLI_TEST_DUP_COMMAND=1 \ -MDS=1 MON=1 OSD=3 MGR=1 CEPH_PORT=7200 CEPH_OBJECTSTORE="bluestore" $CEPH_ROOT/src/test/vstart_wrapper.sh \ - $CEPH_ROOT/qa/workunits/cephtool/test.sh \ - --test-mds \ - --asok-does-not-need-root diff --git a/src/test/cephtool-test-mon.sh b/src/test/cephtool-test-mon.sh deleted file mode 100755 index 856e73f8908..00000000000 --- a/src/test/cephtool-test-mon.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2014, 2015 Red Hat -# Copyright (C) 2013 Cloudwatt -# -# Author: Loic Dachary -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# -source $(dirname $0)/detect-build-env-vars.sh - -CEPH_CLI_TEST_DUP_COMMAND=1 \ -# uses CEPH_PORT going from 7202 7203 and 7204 because -# it starts at 7202 and runs 3 mons (see vstart.sh) -MON=3 OSD=4 MDS=0 MGR=1 CEPH_PORT=7202 $CEPH_ROOT/src/test/vstart_wrapper.sh \ - $CEPH_ROOT/qa/workunits/cephtool/test.sh \ - --test-mon \ - --asok-does-not-need-root diff --git a/src/test/cephtool-test-osd.sh b/src/test/cephtool-test-osd.sh deleted file mode 100755 index 0dbb5d4d825..00000000000 --- a/src/test/cephtool-test-osd.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2014, 2015 Red Hat -# Copyright (C) 2013 Cloudwatt -# -# Author: Loic Dachary -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# -source $(dirname $0)/detect-build-env-vars.sh - -CEPH_CLI_TEST_DUP_COMMAND=1 \ -MON=1 OSD=3 MDS=0 MGR=1 CEPH_PORT=7201 $CEPH_ROOT/src/test/vstart_wrapper.sh \ - $CEPH_ROOT/qa/workunits/cephtool/test.sh \ - --test-osd \ - --asok-does-not-need-root diff --git a/src/test/cephtool-test-rados.sh b/src/test/cephtool-test-rados.sh deleted file mode 100755 index d3c90a11f94..00000000000 --- a/src/test/cephtool-test-rados.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2015 Red Hat -# -# Author: David Zafman -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# -source $(dirname $0)/detect-build-env-vars.sh - -CEPH_CLI_TEST_DUP_COMMAND=1 \ -MON=1 OSD=3 MDS=0 MGR=1 CEPH_PORT=7205 $CEPH_ROOT/src/test/vstart_wrapper.sh \ - $CEPH_ROOT/src/test/test_rados_tool.sh diff --git a/src/test/crush/CMakeLists.txt b/src/test/crush/CMakeLists.txt index 81f587fec26..e527daf817a 100644 --- a/src/test/crush/CMakeLists.txt +++ b/src/test/crush/CMakeLists.txt @@ -13,5 +13,3 @@ add_ceph_unittest(unittest_crush ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/unittest_crus target_link_libraries(unittest_crush global m ${BLKID_LIBRARIES}) add_ceph_test(crush_weights.sh ${CMAKE_CURRENT_SOURCE_DIR}/crush_weights.sh) -add_ceph_test(crush-classes.sh ${CMAKE_CURRENT_SOURCE_DIR}/crush-classes.sh) -add_ceph_test(crush-choose-args.sh ${CMAKE_CURRENT_SOURCE_DIR}/crush-choose-args.sh) diff --git a/src/test/crush/crush-choose-args.sh b/src/test/crush/crush-choose-args.sh deleted file mode 100755 index 493d5d40f2d..00000000000 --- a/src/test/crush/crush-choose-args.sh +++ /dev/null @@ -1,162 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2017 Red Hat -# -# Author: Loic Dachary -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# - -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7131" # git grep '\<7131\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - CEPH_ARGS+="--crush-location=root=default,host=HOST " - CEPH_ARGS+="--osd-crush-initial-weight=3 " - # - # Disable device auto class feature for now. - # The device class is non-deterministic and will - # crash the crushmap comparison below. - # - CEPH_ARGS+="--osd-class-update-on-start=false " - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - setup $dir || return 1 - $func $dir || return 1 - teardown $dir || return 1 - done -} - -function TEST_choose_args_update() { - # - # adding a weighted OSD updates the weight up to the top - # - local dir=$1 - - run_mon $dir a || return 1 - run_osd $dir 0 || return 1 - - ceph osd set-require-min-compat-client luminous - ceph osd getcrushmap > $dir/map || return 1 - crushtool -d $dir/map -o $dir/map.txt || return 1 - sed -i -e '/end crush map/d' $dir/map.txt - cat >> $dir/map.txt < $dir/map-one-more || return 1 - crushtool -d $dir/map-one-more -o $dir/map-one-more.txt || return 1 - cat $dir/map-one-more.txt - diff -u $dir/map-one-more.txt $CEPH_ROOT/src/test/crush/crush-choose-args-expected-one-more-3.txt || return 1 - - destroy_osd $dir 1 || return 1 - ceph osd getcrushmap > $dir/map-one-less || return 1 - crushtool -d $dir/map-one-less -o $dir/map-one-less.txt || return 1 - diff -u $dir/map-one-less.txt $dir/map.txt || return 1 -} - -function TEST_no_update_weight_set() { - # - # adding a zero weight OSD does not update the weight set at all - # - local dir=$1 - - ORIG_CEPH_ARGS="$CEPH_ARGS" - CEPH_ARGS+="--osd-crush-update-weight-set=false " - - run_mon $dir a || return 1 - run_osd $dir 0 || return 1 - - ceph osd set-require-min-compat-client luminous - ceph osd crush tree - ceph osd getcrushmap > $dir/map || return 1 - crushtool -d $dir/map -o $dir/map.txt || return 1 - sed -i -e '/end crush map/d' $dir/map.txt - cat >> $dir/map.txt < $dir/map-one-more || return 1 - crushtool -d $dir/map-one-more -o $dir/map-one-more.txt || return 1 - cat $dir/map-one-more.txt - diff -u $dir/map-one-more.txt $CEPH_ROOT/src/test/crush/crush-choose-args-expected-one-more-0.txt || return 1 - - destroy_osd $dir 1 || return 1 - ceph osd crush tree - ceph osd getcrushmap > $dir/map-one-less || return 1 - crushtool -d $dir/map-one-less -o $dir/map-one-less.txt || return 1 - diff -u $dir/map-one-less.txt $dir/map.txt || return 1 - - CEPH_ARGS="$ORIG_CEPH_ARGS" -} - -main crush-choose-args "$@" - -# Local Variables: -# compile-command: "cd ../../../build ; ln -sf ../src/ceph-disk/ceph_disk/main.py bin/ceph-disk && make -j4 && ../src/test/crush/crush-choose-args.sh" -# End: diff --git a/src/test/crush/crush-classes.sh b/src/test/crush/crush-classes.sh deleted file mode 100755 index b26cc8691ad..00000000000 --- a/src/test/crush/crush-classes.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2017 Red Hat -# -# Author: Loic Dachary -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# - -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7130" # git grep '\<7130\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - # - # Disable auto-class, so we can inject device class manually below - # - CEPH_ARGS+="--osd-class-update-on-start=false " - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - setup $dir || return 1 - $func $dir || return 1 - teardown $dir || return 1 - done -} - -function add_something() { - local dir=$1 - local obj=${2:-SOMETHING} - - local payload=ABCDEF - echo $payload > $dir/ORIGINAL - rados --pool rbd put $obj $dir/ORIGINAL || return 1 -} - -function get_osds_up() { - local poolname=$1 - local objectname=$2 - - local osds=$(ceph --format xml osd map $poolname $objectname 2>/dev/null | \ - $XMLSTARLET sel -t -m "//up/osd" -v . -o ' ') - # get rid of the trailing space - echo $osds -} - -function TEST_classes() { - local dir=$1 - - run_mon $dir a || return 1 - run_osd $dir 0 || return 1 - run_osd $dir 1 || return 1 - run_osd $dir 2 || return 1 - - test "$(get_osds_up rbd SOMETHING)" == "1 2 0" || return 1 - add_something $dir SOMETHING || return 1 - - # - # osd.0 has class ssd and the rule is modified - # to only take ssd devices. - # - ceph osd getcrushmap > $dir/map || return 1 - crushtool -d $dir/map -o $dir/map.txt || return 1 - ${SED} -i \ - -e '/device 0 osd.0/s/$/ class ssd/' \ - -e '/step take default/s/$/ class ssd/' \ - $dir/map.txt || return 1 - crushtool -c $dir/map.txt -o $dir/map-new || return 1 - ceph osd setcrushmap -i $dir/map-new || return 1 - - # - # There can only be one mapping since there only is - # one device with ssd class. - # - ok=false - for delay in 2 4 8 16 32 64 128 256 ; do - if test "$(get_osds_up rbd SOMETHING_ELSE)" == "0" ; then - ok=true - break - fi - sleep $delay - ceph osd dump # for debugging purposes - ceph pg dump # for debugging purposes - done - $ok || return 1 - # - # Writing keeps working because the pool is min_size 1 by - # default. - # - add_something $dir SOMETHING_ELSE || return 1 - - # - # Sanity check that the rule indeed has ssd - # generated bucket with a name including ~ssd. - # - ceph osd crush dump | grep -q '~ssd' || return 1 -} - -function TEST_set_device_class() { - local dir=$1 - - TEST_classes $dir || return 1 - - ceph osd crush set-device-class ssd osd.0 || return 1 - ceph osd crush class ls-osd ssd | grep 0 || return 1 - ceph osd crush set-device-class ssd osd.1 || return 1 - ceph osd crush class ls-osd ssd | grep 1 || return 1 - ceph osd crush set-device-class ssd 0 1 || return 1 # should be idempotent - - ok=false - for delay in 2 4 8 16 32 64 128 256 ; do - if test "$(get_osds_up rbd SOMETHING_ELSE)" == "0 1" ; then - ok=true - break - fi - sleep $delay - ceph osd crush dump - ceph osd dump # for debugging purposes - ceph pg dump # for debugging purposes - done - $ok || return 1 -} - -function TEST_mon_classes() { - local dir=$1 - - run_mon $dir a || return 1 - ceph osd crush class create CLASS || return 1 - ceph osd crush class create CLASS || return 1 # idempotent - ceph osd crush class ls | grep CLASS || return 1 - ceph osd crush class rename CLASS TEMP || return 1 - ceph osd crush class ls | grep TEMP || return 1 - ceph osd crush class rename TEMP CLASS || return 1 - ceph osd crush class ls | grep CLASS || return 1 - ceph osd crush class rm CLASS || return 1 - expect_failure $dir ENOENT ceph osd crush class rm CLASS || return 1 -} - -main crush-classes "$@" - -# Local Variables: -# compile-command: "cd ../../../build ; ln -sf ../src/ceph-disk/ceph_disk/main.py bin/ceph-disk && make -j4 && ../src/test/crush/crush-classes.sh" -# End: diff --git a/src/test/crush/crush_weights.sh b/src/test/crush/crush_weights.sh index 52c0f20db32..6ae1900612d 100755 --- a/src/test/crush/crush_weights.sh +++ b/src/test/crush/crush_weights.sh @@ -1,7 +1,6 @@ #!/bin/bash source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh read -r -d '' cm <<'EOF' # devices diff --git a/src/test/encoding/check-generated.sh b/src/test/encoding/check-generated.sh index 1783cdbafcf..f94c1fd64da 100755 --- a/src/test/encoding/check-generated.sh +++ b/src/test/encoding/check-generated.sh @@ -1,7 +1,7 @@ #!/bin/bash -e source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh dir=$1 diff --git a/src/test/erasure-code/CMakeLists.txt b/src/test/erasure-code/CMakeLists.txt index 218e6908f95..dc4e0865d0a 100644 --- a/src/test/erasure-code/CMakeLists.txt +++ b/src/test/erasure-code/CMakeLists.txt @@ -1,7 +1,3 @@ -add_ceph_test(test-erasure-code-plugins.sh ${CMAKE_CURRENT_SOURCE_DIR}/test-erasure-code-plugins.sh) - -add_ceph_test(test-erasure-code.sh ${CMAKE_CURRENT_SOURCE_DIR}/test-erasure-code.sh) -add_ceph_test(test-erasure-eio.sh ${CMAKE_CURRENT_SOURCE_DIR}/test-erasure-eio.sh) add_executable(ceph_erasure_code_benchmark ${CMAKE_SOURCE_DIR}/src/erasure-code/ErasureCode.cc diff --git a/src/test/erasure-code/test-erasure-code-plugins.sh b/src/test/erasure-code/test-erasure-code-plugins.sh deleted file mode 100755 index b148f3a9bb3..00000000000 --- a/src/test/erasure-code/test-erasure-code-plugins.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/bash -x - -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -arch=$(uname -m) - -case $arch in - i[[3456]]86*|x86_64*|amd64*) - legacy_jerasure_plugins=(jerasure_generic jerasure_sse3 jerasure_sse4) - legacy_shec_plugins=(shec_generic shec_sse3 shec_sse4) - plugins=(jerasure shec lrc isa) - ;; - aarch64*|arm*) - legacy_jerasure_plugins=(jerasure_generic jerasure_neon) - legacy_shec_plugins=(shec_generic shec_neon) - plugins=(jerasure shec lrc) - ;; - *) - echo "unsupported platform ${arch}." - return 1 - ;; -esac - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:17110" # git grep '\<17110\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - $func $dir || return 1 - done -} - -function TEST_preload_warning() { - local dir=$1 - - for plugin in ${legacy_jerasure_plugins[*]} ${legacy_shec_plugins[*]}; do - setup $dir || return 1 - run_mon $dir a --osd_erasure_code_plugins="${plugin}" || return 1 - run_mgr $dir x || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1 - run_osd $dir 0 --osd_erasure_code_plugins="${plugin}" || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1 - grep "WARNING: osd_erasure_code_plugins contains plugin ${plugin}" $dir/mon.a.log || return 1 - grep "WARNING: osd_erasure_code_plugins contains plugin ${plugin}" $dir/osd.0.log || return 1 - teardown $dir || return 1 - done - return 0 -} - -function TEST_preload_no_warning() { - local dir=$1 - - for plugin in ${plugins[*]}; do - setup $dir || return 1 - run_mon $dir a --osd_erasure_code_plugins="${plugin}" || return 1 - run_mgr $dir x || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1 - run_osd $dir 0 --osd_erasure_code_plugins="${plugin}" || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1 - ! grep "WARNING: osd_erasure_code_plugins contains plugin" $dir/mon.a.log || return 1 - ! grep "WARNING: osd_erasure_code_plugins contains plugin" $dir/osd.0.log || return 1 - teardown $dir || return 1 - done - - return 0 -} - -function TEST_preload_no_warning_default() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1 - ! grep "WARNING: osd_erasure_code_plugins" $dir/mon.a.log || return 1 - ! grep "WARNING: osd_erasure_code_plugins" $dir/osd.0.log || return 1 - teardown $dir || return 1 - - return 0 -} - -function TEST_ec_profile_warning() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - for id in $(seq 0 2) ; do - run_osd $dir $id || return 1 - done - wait_for_clean || return 1 - - for plugin in ${legacy_jerasure_plugins[*]}; do - ceph osd erasure-code-profile set prof-${plugin} crush-failure-domain=osd technique=reed_sol_van plugin=${plugin} || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1 - grep "WARNING: erasure coding profile prof-${plugin} uses plugin ${plugin}" $dir/mon.a.log || return 1 - done - - for plugin in ${legacy_shec_plugins[*]}; do - ceph osd erasure-code-profile set prof-${plugin} crush-failure-domain=osd plugin=${plugin} || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1 - grep "WARNING: erasure coding profile prof-${plugin} uses plugin ${plugin}" $dir/mon.a.log || return 1 - done - - teardown $dir || return 1 -} - -main test-erasure-code-plugins "$@" diff --git a/src/test/erasure-code/test-erasure-code.sh b/src/test/erasure-code/test-erasure-code.sh deleted file mode 100755 index 91fd32062b2..00000000000 --- a/src/test/erasure-code/test-erasure-code.sh +++ /dev/null @@ -1,339 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2014 Cloudwatt -# Copyright (C) 2014, 2015 Red Hat -# -# Author: Loic Dachary -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# - -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7101" # git grep '\<7101\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON --mon-osd-prime-pg-temp=false" - - setup $dir || return 1 - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - # check that erasure code plugins are preloaded - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1 - grep 'load: jerasure.*lrc' $dir/mon.a.log || return 1 - for id in $(seq 0 10) ; do - run_osd $dir $id || return 1 - done - wait_for_clean || return 1 - # check that erasure code plugins are preloaded - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1 - grep 'load: jerasure.*lrc' $dir/osd.0.log || return 1 - create_erasure_coded_pool ecpool || return 1 - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - $func $dir || return 1 - done - - delete_pool ecpool || return 1 - teardown $dir || return 1 -} - -function create_erasure_coded_pool() { - local poolname=$1 - - ceph osd erasure-code-profile set myprofile \ - crush-failure-domain=osd || return 1 - ceph osd pool create $poolname 12 12 erasure myprofile \ - || return 1 - wait_for_clean || return 1 -} - -function delete_pool() { - local poolname=$1 - - ceph osd pool delete $poolname $poolname --yes-i-really-really-mean-it -} - -function rados_put_get() { - local dir=$1 - local poolname=$2 - local objname=${3:-SOMETHING} - - - for marker in AAA BBB CCCC DDDD ; do - printf "%*s" 1024 $marker - done > $dir/ORIGINAL - - # - # get and put an object, compare they are equal - # - rados --pool $poolname put $objname $dir/ORIGINAL || return 1 - rados --pool $poolname get $objname $dir/COPY || return 1 - diff $dir/ORIGINAL $dir/COPY || return 1 - rm $dir/COPY - - # - # take out an OSD used to store the object and - # check the object can still be retrieved, which implies - # recovery - # - local -a initial_osds=($(get_osds $poolname $objname)) - local last=$((${#initial_osds[@]} - 1)) - ceph osd out ${initial_osds[$last]} || return 1 - ! get_osds $poolname $objname | grep '\<'${initial_osds[$last]}'\>' || return 1 - rados --pool $poolname get $objname $dir/COPY || return 1 - diff $dir/ORIGINAL $dir/COPY || return 1 - ceph osd in ${initial_osds[$last]} || return 1 - - rm $dir/ORIGINAL -} - -function rados_osds_out_in() { - local dir=$1 - local poolname=$2 - local objname=${3:-SOMETHING} - - - for marker in FFFF GGGG HHHH IIII ; do - printf "%*s" 1024 $marker - done > $dir/ORIGINAL - - # - # get and put an object, compare they are equal - # - rados --pool $poolname put $objname $dir/ORIGINAL || return 1 - rados --pool $poolname get $objname $dir/COPY || return 1 - diff $dir/ORIGINAL $dir/COPY || return 1 - rm $dir/COPY - - # - # take out two OSDs used to store the object, wait for the cluster - # to be clean (i.e. all PG are clean and active) again which - # implies the PG have been moved to use the remaining OSDs. Check - # the object can still be retrieved. - # - wait_for_clean || return 1 - local osds_list=$(get_osds $poolname $objname) - local -a osds=($osds_list) - for osd in 0 1 ; do - ceph osd out ${osds[$osd]} || return 1 - done - wait_for_clean || return 1 - # - # verify the object is no longer mapped to the osds that are out - # - for osd in 0 1 ; do - ! get_osds $poolname $objname | grep '\<'${osds[$osd]}'\>' || return 1 - done - rados --pool $poolname get $objname $dir/COPY || return 1 - diff $dir/ORIGINAL $dir/COPY || return 1 - # - # bring the osds back in, , wait for the cluster - # to be clean (i.e. all PG are clean and active) again which - # implies the PG go back to using the same osds as before - # - for osd in 0 1 ; do - ceph osd in ${osds[$osd]} || return 1 - done - wait_for_clean || return 1 - test "$osds_list" = "$(get_osds $poolname $objname)" || return 1 - rm $dir/ORIGINAL -} - -function TEST_rados_put_get_lrc_advanced() { - local dir=$1 - local poolname=pool-lrc-a - local profile=profile-lrc-a - - ceph osd erasure-code-profile set $profile \ - plugin=lrc \ - mapping=DD_ \ - crush-steps='[ [ "chooseleaf", "osd", 0 ] ]' \ - layers='[ [ "DDc", "" ] ]' || return 1 - ceph osd pool create $poolname 12 12 erasure $profile \ - || return 1 - - rados_put_get $dir $poolname || return 1 - - delete_pool $poolname - ceph osd erasure-code-profile rm $profile -} - -function TEST_rados_put_get_lrc_kml() { - local dir=$1 - local poolname=pool-lrc - local profile=profile-lrc - - ceph osd erasure-code-profile set $profile \ - plugin=lrc \ - k=4 m=2 l=3 \ - crush-failure-domain=osd || return 1 - ceph osd pool create $poolname 12 12 erasure $profile \ - || return 1 - - rados_put_get $dir $poolname || return 1 - - delete_pool $poolname - ceph osd erasure-code-profile rm $profile -} - -function TEST_rados_put_get_isa() { - if ! erasure_code_plugin_exists isa ; then - echo "SKIP because plugin isa has not been built" - return 0 - fi - local dir=$1 - local poolname=pool-isa - - ceph osd erasure-code-profile set profile-isa \ - plugin=isa \ - crush-failure-domain=osd || return 1 - ceph osd pool create $poolname 1 1 erasure profile-isa \ - || return 1 - - rados_put_get $dir $poolname || return 1 - - delete_pool $poolname -} - -function TEST_rados_put_get_jerasure() { - local dir=$1 - - rados_put_get $dir ecpool || return 1 - - local poolname=pool-jerasure - local profile=profile-jerasure - - ceph osd erasure-code-profile set $profile \ - plugin=jerasure \ - k=4 m=2 \ - crush-failure-domain=osd || return 1 - ceph osd pool create $poolname 12 12 erasure $profile \ - || return 1 - - rados_put_get $dir $poolname || return 1 - rados_osds_out_in $dir $poolname || return 1 - - delete_pool $poolname - ceph osd erasure-code-profile rm $profile -} - -function TEST_rados_put_get_shec() { - local dir=$1 - - local poolname=pool-shec - local profile=profile-shec - - ceph osd erasure-code-profile set $profile \ - plugin=shec \ - k=2 m=1 c=1 \ - crush-failure-domain=osd || return 1 - ceph osd pool create $poolname 12 12 erasure $profile \ - || return 1 - - rados_put_get $dir $poolname || return 1 - - delete_pool $poolname - ceph osd erasure-code-profile rm $profile -} - -function TEST_alignment_constraints() { - local payload=ABC - echo "$payload" > $dir/ORIGINAL - # - # Verify that the rados command enforces alignment constraints - # imposed by the stripe width - # See http://tracker.ceph.com/issues/8622 - # - local stripe_unit=$(ceph-conf --show-config-value osd_pool_erasure_code_stripe_unit) - eval local $(ceph osd erasure-code-profile get myprofile | grep k=) - local block_size=$((stripe_unit * k - 1)) - dd if=/dev/zero of=$dir/ORIGINAL bs=$block_size count=2 - rados --block-size=$block_size \ - --pool ecpool put UNALIGNED $dir/ORIGINAL || return 1 - rm $dir/ORIGINAL -} - -function chunk_size() { - echo $(ceph-conf --show-config-value osd_pool_erasure_code_stripe_unit) -} - -# -# By default an object will be split in two (k=2) with the first part -# of the object in the first OSD of the up set and the second part in -# the next OSD in the up set. This layout is defined by the mapping -# parameter and this function helps verify that the first and second -# part of the object are located in the OSD where they should be. -# -function verify_chunk_mapping() { - local dir=$1 - local poolname=$2 - local first=$3 - local second=$4 - - local payload=$(printf '%*s' $(chunk_size) FIRST$poolname ; printf '%*s' $(chunk_size) SECOND$poolname) - echo -n "$payload" > $dir/ORIGINAL - - rados --pool $poolname put SOMETHING$poolname $dir/ORIGINAL || return 1 - rados --pool $poolname get SOMETHING$poolname $dir/COPY || return 1 - local -a osds=($(get_osds $poolname SOMETHING$poolname)) - for (( i = 0; i < ${#osds[@]}; i++ )) ; do - ceph daemon osd.${osds[$i]} flush_journal - done - diff $dir/ORIGINAL $dir/COPY || return 1 - rm $dir/COPY - - local -a osds=($(get_osds $poolname SOMETHING$poolname)) - grep --quiet --recursive --text FIRST$poolname $dir/${osds[$first]} || return 1 - grep --quiet --recursive --text SECOND$poolname $dir/${osds[$second]} || return 1 -} - -function TEST_chunk_mapping() { - local dir=$1 - - # - # mapping=DD_ is the default: - # first OSD (i.e. 0) in the up set has the first part of the object - # second OSD (i.e. 1) in the up set has the second part of the object - # - verify_chunk_mapping $dir ecpool 0 1 || return 1 - - ceph osd erasure-code-profile set remap-profile \ - plugin=lrc \ - layers='[ [ "_DD", "" ] ]' \ - mapping='_DD' \ - crush-steps='[ [ "choose", "osd", 0 ] ]' || return 1 - ceph osd erasure-code-profile get remap-profile - ceph osd pool create remap-pool 12 12 erasure remap-profile \ - || return 1 - - # - # mapping=_DD - # second OSD (i.e. 1) in the up set has the first part of the object - # third OSD (i.e. 2) in the up set has the second part of the object - # - verify_chunk_mapping $dir remap-pool 1 2 || return 1 - - delete_pool remap-pool - ceph osd erasure-code-profile rm remap-profile -} - -main test-erasure-code "$@" - -# Local Variables: -# compile-command: "cd ../.. ; make -j4 && test/erasure-code/test-erasure-code.sh" -# End: diff --git a/src/test/erasure-code/test-erasure-eio.sh b/src/test/erasure-code/test-erasure-eio.sh deleted file mode 100755 index 90a225ed3cf..00000000000 --- a/src/test/erasure-code/test-erasure-eio.sh +++ /dev/null @@ -1,338 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2015 Red Hat -# -# -# Author: Kefu Chai -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# - -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7112" # git grep '\<7112\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - setup $dir || return 1 - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - # check that erasure code plugins are preloaded - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1 - grep 'load: jerasure.*lrc' $dir/mon.a.log || return 1 - $func $dir || return 1 - teardown $dir || return 1 - done -} - -function setup_osds() { - for id in $(seq 0 3) ; do - run_osd $dir $id || return 1 - done - wait_for_clean || return 1 - - # check that erasure code plugins are preloaded - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1 - grep 'load: jerasure.*lrc' $dir/osd.0.log || return 1 -} - -function create_erasure_coded_pool() { - local poolname=$1 - - ceph osd erasure-code-profile set myprofile \ - plugin=jerasure \ - k=2 m=1 \ - crush-failure-domain=osd || return 1 - ceph osd pool create $poolname 1 1 erasure myprofile \ - || return 1 - wait_for_clean || return 1 -} - -function delete_pool() { - local poolname=$1 - - ceph osd pool delete $poolname $poolname --yes-i-really-really-mean-it - ceph osd erasure-code-profile rm myprofile -} - -function rados_put() { - local dir=$1 - local poolname=$2 - local objname=${3:-SOMETHING} - - for marker in AAA BBB CCCC DDDD ; do - printf "%*s" 1024 $marker - done > $dir/ORIGINAL - # - # get and put an object, compare they are equal - # - rados --pool $poolname put $objname $dir/ORIGINAL || return 1 -} - -function rados_get() { - local dir=$1 - local poolname=$2 - local objname=${3:-SOMETHING} - local expect=${4:-ok} - - # - # Expect a failure to get object - # - if [ $expect = "fail" ]; - then - ! rados --pool $poolname get $objname $dir/COPY - return - fi - # - # get an object, compare with $dir/ORIGINAL - # - rados --pool $poolname get $objname $dir/COPY || return 1 - diff $dir/ORIGINAL $dir/COPY || return 1 - rm $dir/COPY -} - -function rados_put_get() { - local dir=$1 - local poolname=$2 - local objname=${3:-SOMETHING} - local recovery=$4 - - # - # get and put an object, compare they are equal - # - rados_put $dir $poolname $objname || return 1 - # We can read even though caller injected read error on one of the shards - rados_get $dir $poolname $objname || return 1 - - if [ -n "$recovery" ]; - then - # - # take out the last OSD used to store the object, - # bring it back, and check for clean PGs which means - # recovery didn't crash the primary. - # - local -a initial_osds=($(get_osds $poolname $objname)) - local last=$((${#initial_osds[@]} - 1)) - # Kill OSD - kill_daemons $dir TERM osd.${initial_osds[$last]} >&2 < /dev/null || return 1 - ceph osd out ${initial_osds[$last]} || return 1 - ! get_osds $poolname $objname | grep '\<'${initial_osds[$last]}'\>' || return 1 - ceph osd in ${initial_osds[$last]} || return 1 - run_osd $dir ${initial_osds[$last]} || return 1 - wait_for_clean || return 1 - fi - - rm $dir/ORIGINAL -} - -function inject_eio() { - local objname=$1 - shift - local dir=$1 - shift - local shard_id=$1 - shift - - local poolname=pool-jerasure - local -a initial_osds=($(get_osds $poolname $objname)) - local osd_id=${initial_osds[$shard_id]} - set_config osd $osd_id filestore_debug_inject_read_err true || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.$osd_id) \ - injectdataerr $poolname $objname $shard_id || return 1 -} - -function rados_get_data_eio() { - local dir=$1 - shift - local shard_id=$1 - shift - local recovery=$1 - shift - - # inject eio to speificied shard - # - local poolname=pool-jerasure - local objname=obj-eio-$$-$shard_id - inject_eio $objname $dir $shard_id || return 1 - rados_put_get $dir $poolname $objname $recovery || return 1 - - shard_id=$(expr $shard_id + 1) - inject_eio $objname $dir $shard_id || return 1 - # Now 2 out of 3 shards get EIO, so should fail - rados_get $dir $poolname $objname fail || return 1 -} - -# Change the size of speificied shard -# -function set_size() { - local objname=$1 - shift - local dir=$1 - shift - local shard_id=$1 - shift - local bytes=$1 - shift - local mode=${1} - - local poolname=pool-jerasure - local -a initial_osds=($(get_osds $poolname $objname)) - local osd_id=${initial_osds[$shard_id]} - ceph osd set noout - if [ "$mode" = "add" ]; - then - objectstore_tool $dir $osd_id $objname get-bytes $dir/CORRUPT || return 1 - dd if=/dev/urandom bs=$bytes count=1 >> $dir/CORRUPT - elif [ "$bytes" = "0" ]; - then - touch $dir/CORRUPT - else - dd if=/dev/urandom bs=$bytes count=1 of=$dir/CORRUPT - fi - objectstore_tool $dir $osd_id $objname set-bytes $dir/CORRUPT || return 1 - rm -f $dir/CORRUPT - ceph osd unset noout -} - -function rados_get_data_bad_size() { - local dir=$1 - shift - local shard_id=$1 - shift - local bytes=$1 - shift - local mode=${1:-set} - - local poolname=pool-jerasure - local objname=obj-size-$$-$shard_id-$bytes - rados_put $dir $poolname $objname || return 1 - - # Change the size of speificied shard - # - set_size $objname $dir $shard_id $bytes $mode || return 1 - - rados_get $dir $poolname $objname || return 1 - - # Leave objname and modify another shard - shard_id=$(expr $shard_id + 1) - set_size $objname $dir $shard_id $bytes $mode || return 1 - rados_get $dir $poolname $objname fail || return 1 -} - -# -# These two test cases try to validate the following behavior: -# For object on EC pool, if there is one shard having read error ( -# either primary or replica), client can still read object. -# -# If 2 shards have read errors the client will get an error. -# -function TEST_rados_get_subread_eio_shard_0() { - local dir=$1 - setup_osds || return 1 - - local poolname=pool-jerasure - create_erasure_coded_pool $poolname || return 1 - # inject eio on primary OSD (0) and replica OSD (1) - local shard_id=0 - rados_get_data_eio $dir $shard_id || return 1 - delete_pool $poolname -} - -function TEST_rados_get_subread_eio_shard_1() { - local dir=$1 - setup_osds || return 1 - - local poolname=pool-jerasure - create_erasure_coded_pool $poolname || return 1 - # inject eio into replicas OSD (1) and OSD (2) - local shard_id=1 - rados_get_data_eio $dir $shard_id || return 1 - delete_pool $poolname -} - -# -# These two test cases try to validate that following behavior: -# For object on EC pool, if there is one shard which an incorrect -# size this will cause an internal read error, client can still read object. -# -# If 2 shards have incorrect size the client will get an error. -# -function TEST_rados_get_bad_size_shard_0() { - local dir=$1 - setup_osds || return 1 - - local poolname=pool-jerasure - create_erasure_coded_pool $poolname || return 1 - # Set incorrect size into primary OSD (0) and replica OSD (1) - local shard_id=0 - rados_get_data_bad_size $dir $shard_id 10 || return 1 - rados_get_data_bad_size $dir $shard_id 0 || return 1 - rados_get_data_bad_size $dir $shard_id 256 add || return 1 - delete_pool $poolname -} - -function TEST_rados_get_bad_size_shard_1() { - local dir=$1 - setup_osds || return 1 - - local poolname=pool-jerasure - create_erasure_coded_pool $poolname || return 1 - # Set incorrect size into replicas OSD (1) and OSD (2) - local shard_id=1 - rados_get_data_bad_size $dir $shard_id 10 || return 1 - rados_get_data_bad_size $dir $shard_id 0 || return 1 - rados_get_data_bad_size $dir $shard_id 256 add || return 1 - delete_pool $poolname -} - -function TEST_rados_get_with_subreadall_eio_shard_0() { - local dir=$1 - local shard_id=0 - - setup_osds || return 1 - - local poolname=pool-jerasure - create_erasure_coded_pool $poolname || return 1 - # inject eio on primary OSD (0) - local shard_id=0 - rados_get_data_eio $dir $shard_id recovery || return 1 - - delete_pool $poolname -} - -function TEST_rados_get_with_subreadall_eio_shard_1() { - local dir=$1 - local shard_id=0 - - setup_osds || return 1 - - local poolname=pool-jerasure - create_erasure_coded_pool $poolname || return 1 - # inject eio on replica OSD (1) - local shard_id=1 - rados_get_data_eio $dir $shard_id recovery || return 1 - - delete_pool $poolname -} - -main test-erasure-eio "$@" - -# Local Variables: -# compile-command: "cd ../.. ; make -j4 && test/erasure-code/test-erasure-eio.sh" -# End: diff --git a/src/test/libradosstriper/CMakeLists.txt b/src/test/libradosstriper/CMakeLists.txt index 4f7777fb6f9..b328acb31bd 100644 --- a/src/test/libradosstriper/CMakeLists.txt +++ b/src/test/libradosstriper/CMakeLists.txt @@ -13,8 +13,6 @@ set_target_properties(ceph_test_rados_striper_api_striping PROPERTIES COMPILE_FL install(TARGETS ceph_test_rados_striper_api_striping DESTINATION ${CMAKE_INSTALL_BINDIR}) -add_ceph_test(rados-striper.sh ${CMAKE_SOURCE_DIR}/src/test/libradosstriper/rados-striper.sh) - add_executable(ceph_test_rados_striper_api_io io.cc) target_link_libraries(ceph_test_rados_striper_api_io librados radosstriper diff --git a/src/test/libradosstriper/rados-striper.sh b/src/test/libradosstriper/rados-striper.sh deleted file mode 100755 index ed0c7af65ad..00000000000 --- a/src/test/libradosstriper/rados-striper.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2014 Red Hat -# -# Author: Sebastien Ponce -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7116" # git grep '\<7116\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - - # setup - setup $dir || return 1 - - # create a cluster with one monitor and three osds - run_mon $dir a || return 1 - run_osd $dir 0 || return 1 - run_osd $dir 1 || return 1 - run_osd $dir 2 || return 1 - - # create toyfile - dd if=/dev/urandom of=$dir/toyfile bs=1234 count=1 - - # put a striped object - rados --pool rbd --striper put toyfile $dir/toyfile || return 1 - - # stat it, with and without striping - rados --pool rbd --striper stat toyfile | cut -d ',' -f 2 > $dir/stripedStat || return 1 - rados --pool rbd stat toyfile.0000000000000000 | cut -d ',' -f 2 > $dir/stat || return 1 - echo ' size 1234' > $dir/refstat - diff -w $dir/stripedStat $dir/refstat || return 1 - diff -w $dir/stat $dir/refstat || return 1 - rados --pool rbd stat toyfile >& $dir/staterror - grep -q 'No such file or directory' $dir/staterror || return 1 - - # get the file back with and without striping - rados --pool rbd --striper get toyfile $dir/stripedGroup || return 1 - diff -w $dir/toyfile $dir/stripedGroup || return 1 - rados --pool rbd get toyfile.0000000000000000 $dir/nonSTripedGroup || return 1 - diff -w $dir/toyfile $dir/nonSTripedGroup || return 1 - - # test truncate - rados --pool rbd --striper truncate toyfile 12 - rados --pool rbd --striper stat toyfile | cut -d ',' -f 2 > $dir/stripedStat || return 1 - rados --pool rbd stat toyfile.0000000000000000 | cut -d ',' -f 2 > $dir/stat || return 1 - echo ' size 12' > $dir/reftrunc - diff -w $dir/stripedStat $dir/reftrunc || return 1 - diff -w $dir/stat $dir/reftrunc || return 1 - - # test xattrs - - rados --pool rbd --striper setxattr toyfile somexattr somevalue || return 1 - rados --pool rbd --striper getxattr toyfile somexattr > $dir/xattrvalue || return 1 - rados --pool rbd getxattr toyfile.0000000000000000 somexattr > $dir/xattrvalue2 || return 1 - echo 'somevalue' > $dir/refvalue - diff -w $dir/xattrvalue $dir/refvalue || return 1 - diff -w $dir/xattrvalue2 $dir/refvalue || return 1 - rados --pool rbd --striper listxattr toyfile > $dir/xattrlist || return 1 - echo 'somexattr' > $dir/reflist - diff -w $dir/xattrlist $dir/reflist || return 1 - rados --pool rbd listxattr toyfile.0000000000000000 | grep -v striper > $dir/xattrlist2 || return 1 - diff -w $dir/xattrlist2 $dir/reflist || return 1 - rados --pool rbd --striper rmxattr toyfile somexattr || return 1 - - local attr_not_found_str="No data available" - [ `uname` = FreeBSD ] && \ - attr_not_found_str="Attribute not found" - expect_failure $dir "$attr_not_found_str" \ - rados --pool rbd --striper getxattr toyfile somexattr || return 1 - expect_failure $dir "$attr_not_found_str" \ - rados --pool rbd getxattr toyfile.0000000000000000 somexattr || return 1 - - # test rm - rados --pool rbd --striper rm toyfile || return 1 - expect_failure $dir 'No such file or directory' \ - rados --pool rbd --striper stat toyfile || return 1 - expect_failure $dir 'No such file or directory' \ - rados --pool rbd stat toyfile.0000000000000000 || return 1 - - # cleanup - teardown $dir || return 1 -} - -main rados-striper "$@" diff --git a/src/test/mgr/mgr-dashboard-smoke.sh b/src/test/mgr/mgr-dashboard-smoke.sh index 5c52215beba..353c4d45a03 100755 --- a/src/test/mgr/mgr-dashboard-smoke.sh +++ b/src/test/mgr/mgr-dashboard-smoke.sh @@ -13,7 +13,7 @@ # GNU Library Public License for more details. # source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh function run() { local dir=$1 diff --git a/src/test/mon/CMakeLists.txt b/src/test/mon/CMakeLists.txt index 6aa126c95da..4bc5b1cca4a 100644 --- a/src/test/mon/CMakeLists.txt +++ b/src/test/mon/CMakeLists.txt @@ -20,19 +20,6 @@ target_link_libraries(ceph_test_mon_msg os osdc global ${UNITTEST_LIBS}) set_target_properties(ceph_test_mon_msg PROPERTIES COMPILE_FLAGS ${UNITTEST_CXX_FLAGS}) -#scripts -add_ceph_test(misc.sh ${CMAKE_CURRENT_SOURCE_DIR}/misc.sh) -add_ceph_test(mkfs.sh ${CMAKE_CURRENT_SOURCE_DIR}/mkfs.sh) -add_ceph_test(mon-bind.sh ${CMAKE_CURRENT_SOURCE_DIR}/mon-bind.sh) -add_ceph_test(mon-created-time.sh ${CMAKE_CURRENT_SOURCE_DIR}/mon-created-time.sh) -add_ceph_test(mon-handle-forward.sh ${CMAKE_CURRENT_SOURCE_DIR}/mon-handle-forward.sh) -add_ceph_test(mon-ping.sh ${CMAKE_CURRENT_SOURCE_DIR}/mon-ping.sh) -add_ceph_test(mon-scrub.sh ${CMAKE_CURRENT_SOURCE_DIR}/mon-scrub.sh) -add_ceph_test(osd-crush.sh ${CMAKE_CURRENT_SOURCE_DIR}/osd-crush.sh) -add_ceph_test(osd-erasure-code-profile.sh ${CMAKE_CURRENT_SOURCE_DIR}/osd-erasure-code-profile.sh) -add_ceph_test(osd-pool-create.sh ${CMAKE_CURRENT_SOURCE_DIR}/osd-pool-create.sh) -add_ceph_test(test_pool_quota.sh ${CMAKE_CURRENT_SOURCE_DIR}/test_pool_quota.sh) - # unittest_mon_moncap add_executable(unittest_mon_moncap moncap.cc diff --git a/src/test/mon/misc.sh b/src/test/mon/misc.sh deleted file mode 100755 index 0ffccd76921..00000000000 --- a/src/test/mon/misc.sh +++ /dev/null @@ -1,238 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2014 Cloudwatt -# Copyright (C) 2014, 2015 Red Hat -# -# Author: Loic Dachary -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7102" # git grep '\<7102\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - $func $dir || return 1 - done -} - -TEST_POOL=rbd - -function TEST_osd_pool_get_set() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a || return 1 - ceph osd pool create $TEST_POOL 8 - - local flag - for flag in nodelete nopgchange nosizechange write_fadvise_dontneed noscrub nodeep-scrub; do - ceph osd pool set $TEST_POOL $flag 0 || return 1 - ! ceph osd dump | grep 'pool ' | grep $flag || return 1 - ceph osd pool set $TEST_POOL $flag 1 || return 1 - ceph osd dump | grep 'pool ' | grep $flag || return 1 - ceph osd pool set $TEST_POOL $flag false || return 1 - ! ceph osd dump | grep 'pool ' | grep $flag || return 1 - ceph osd pool set $TEST_POOL $flag false || return 1 - # check that setting false twice does not toggle to true (bug) - ! ceph osd dump | grep 'pool ' | grep $flag || return 1 - ceph osd pool set $TEST_POOL $flag true || return 1 - ceph osd dump | grep 'pool ' | grep $flag || return 1 - # cleanup - ceph osd pool set $TEST_POOL $flag 0 || return 1 - done - - local size=$(ceph osd pool get $TEST_POOL size|awk '{print $2}') - local min_size=$(ceph osd pool get $TEST_POOL min_size|awk '{print $2}') - - ceph osd pool set $TEST_POOL scrub_min_interval 123456 || return 1 - ceph osd dump | grep 'pool ' | grep 'scrub_min_interval 123456' || return 1 - ceph osd pool set $TEST_POOL scrub_min_interval 0 || return 1 - ceph osd dump | grep 'pool ' | grep 'scrub_min_interval' && return 1 - ceph osd pool set $TEST_POOL scrub_max_interval 123456 || return 1 - ceph osd dump | grep 'pool ' | grep 'scrub_max_interval 123456' || return 1 - ceph osd pool set $TEST_POOL scrub_max_interval 0 || return 1 - ceph osd dump | grep 'pool ' | grep 'scrub_max_interval' && return 1 - ceph osd pool set $TEST_POOL deep_scrub_interval 123456 || return 1 - ceph osd dump | grep 'pool ' | grep 'deep_scrub_interval 123456' || return 1 - ceph osd pool set $TEST_POOL deep_scrub_interval 0 || return 1 - ceph osd dump | grep 'pool ' | grep 'deep_scrub_interval' && return 1 - - #replicated pool size restrict in 1 and 10 - ! ceph osd pool set $TEST_POOL 11 || return 1 - #replicated pool min_size must be between in 1 and size - ! ceph osd pool set $TEST_POOL min_size $(expr $size + 1) || return 1 - ! ceph osd pool set $TEST_POOL min_size 0 || return 1 - - local ecpool=erasepool - ceph osd pool create $ecpool 12 12 erasure default || return 1 - #erasue pool size=k+m, min_size=k - local size=$(ceph osd pool get $ecpool size|awk '{print $2}') - local min_size=$(ceph osd pool get $ecpool min_size|awk '{print $2}') - local k=$(expr $min_size - 1) # default min_size=k+1 - #erasure pool size can't change - ! ceph osd pool set $ecpool size $(expr $size + 1) || return 1 - #erasure pool min_size must be between in k and size - ceph osd pool set $ecpool min_size $(expr $k + 1) || return 1 - ! ceph osd pool set $ecpool min_size $(expr $k - 1) || return 1 - ! ceph osd pool set $ecpool min_size $(expr $size + 1) || return 1 - - teardown $dir || return 1 -} - -function TEST_mon_add_to_single_mon() { - local dir=$1 - - fsid=$(uuidgen) - MONA=127.0.0.1:7117 # git grep '\<7117\>' : there must be only one - MONB=127.0.0.1:7118 # git grep '\<7118\>' : there must be only one - CEPH_ARGS_orig=$CEPH_ARGS - CEPH_ARGS="--fsid=$fsid --auth-supported=none " - CEPH_ARGS+="--mon-initial-members=a " - CEPH_ARGS+="--mon-host=$MONA " - - setup $dir || return 1 - run_mon $dir a --public-addr $MONA || return 1 - # wait for the quorum - timeout 120 ceph -s > /dev/null || return 1 - run_mon $dir b --public-addr $MONB || return 1 - teardown $dir || return 1 - - setup $dir || return 1 - run_mon $dir a --public-addr $MONA || return 1 - # without the fix of #5454, mon.a will assert failure at seeing the MMonJoin - # from mon.b - run_mon $dir b --public-addr $MONB || return 1 - # wait for the quorum - timeout 120 ceph -s > /dev/null || return 1 - local num_mons - num_mons=$(ceph mon dump --format=json 2>/dev/null | jq ".mons | length") || return 1 - [ $num_mons == 2 ] || return 1 - # no reason to take more than 120 secs to get this submitted - timeout 120 ceph mon add b $MONB || return 1 - teardown $dir || return 1 -} - -function TEST_no_segfault_for_bad_keyring() { - local dir=$1 - setup $dir || return 1 - # create a client.admin key and add it to ceph.mon.keyring - ceph-authtool --create-keyring $dir/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *' - ceph-authtool --create-keyring $dir/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' - ceph-authtool $dir/ceph.mon.keyring --import-keyring $dir/ceph.client.admin.keyring - CEPH_ARGS_TMP="--fsid=$(uuidgen) --mon-host=127.0.0.1:7102 --auth-supported=cephx " - CEPH_ARGS_orig=$CEPH_ARGS - CEPH_ARGS="$CEPH_ARGS_TMP --keyring=$dir/ceph.mon.keyring " - run_mon $dir a - # create a bad keyring and make sure no segfault occurs when using the bad keyring - echo -e "[client.admin]\nkey = BQAUlgtWoFePIxAAQ9YLzJSVgJX5V1lh5gyctg==" > $dir/bad.keyring - CEPH_ARGS="$CEPH_ARGS_TMP --keyring=$dir/bad.keyring" - ceph osd dump 2> /dev/null - # 139(11|128) means segfault and core dumped - [ $? -eq 139 ] && return 1 - CEPH_ARGS=$CEPH_ARGS_orig - teardown $dir || return 1 -} - -function TEST_mon_features() { - local dir=$1 - setup $dir || return 1 - - fsid=$(uuidgen) - MONA=127.0.0.1:7127 # git grep '\<7127\>' ; there must be only one - MONB=127.0.0.1:7128 # git grep '\<7128\>' ; there must be only one - MONC=127.0.0.1:7129 # git grep '\<7129\>' ; there must be only one - CEPH_ARGS_orig=$CEPH_ARGS - CEPH_ARGS="--fsid=$fsid --auth-supported=none " - CEPH_ARGS+="--mon-initial-members=a,b,c " - CEPH_ARGS+="--mon-host=$MONA,$MONB,$MONC " - CEPH_ARGS+="--mon-debug-no-initial-persistent-features " - CEPH_ARGS+="--mon-debug-no-require-luminous " - - run_mon $dir a --public-addr $MONA || return 1 - run_mon $dir b --public-addr $MONB || return 1 - timeout 120 ceph -s > /dev/null || return 1 - - # expect monmap to contain 3 monitors (a, b, and c) - jqinput="$(ceph mon_status --format=json 2>/dev/null)" - jq_success "$jqinput" '.monmap.mons | length == 3' || return 1 - # quorum contains two monitors - jq_success "$jqinput" '.quorum | length == 2' || return 1 - # quorum's monitor features contain kraken and luminous - jqfilter='.features.quorum_mon[]|select(. == "kraken")' - jq_success "$jqinput" "$jqfilter" "kraken" || return 1 - jqfilter='.features.quorum_mon[]|select(. == "luminous")' - jq_success "$jqinput" "$jqfilter" "luminous" || return 1 - - # monmap must have no persistent features set, because we - # don't currently have a quorum made out of all the monitors - # in the monmap. - jqfilter='.monmap.features.persistent | length == 0' - jq_success "$jqinput" "$jqfilter" || return 1 - - # nor do we have any optional features, for that matter. - jqfilter='.monmap.features.optional | length == 0' - jq_success "$jqinput" "$jqfilter" || return 1 - - # validate 'mon feature ls' - - jqinput="$(ceph mon feature ls --format=json 2>/dev/null)" - # 'kraken' and 'luminous' are supported - jqfilter='.all.supported[] | select(. == "kraken")' - jq_success "$jqinput" "$jqfilter" "kraken" || return 1 - jqfilter='.all.supported[] | select(. == "luminous")' - jq_success "$jqinput" "$jqfilter" "luminous" || return 1 - - # start third monitor - run_mon $dir c --public-addr $MONC || return 1 - - wait_for_quorum 300 3 || return 1 - - timeout 300 ceph -s > /dev/null || return 1 - - jqinput="$(ceph mon_status --format=json 2>/dev/null)" - # expect quorum to have all three monitors - jqfilter='.quorum | length == 3' - jq_success "$jqinput" "$jqfilter" || return 1 - # quorum's monitor features contain kraken and luminous - jqfilter='.features.quorum_mon[]|select(. == "kraken")' - jq_success "$jqinput" "$jqfilter" "kraken" || return 1 - jqfilter='.features.quorum_mon[]|select(. == "luminous")' - jq_success "$jqinput" "$jqfilter" "luminous" || return 1 - - # monmap must have no both 'kraken' and 'luminous' persistent - # features set. - jqfilter='.monmap.features.persistent | length == 2' - jq_success "$jqinput" "$jqfilter" || return 1 - jqfilter='.monmap.features.persistent[]|select(. == "kraken")' - jq_success "$jqinput" "$jqfilter" "kraken" || return 1 - jqfilter='.monmap.features.persistent[]|select(. == "luminous")' - jq_success "$jqinput" "$jqfilter" "luminous" || return 1 - - CEPH_ARGS=$CEPH_ARGS_orig - # that's all folks. thank you for tuning in. - teardown $dir || return 1 -} - -main misc "$@" - -# Local Variables: -# compile-command: "cd ../.. ; make -j4 && test/mon/misc.sh" -# End: diff --git a/src/test/mon/mkfs.sh b/src/test/mon/mkfs.sh deleted file mode 100755 index 045d235dc71..00000000000 --- a/src/test/mon/mkfs.sh +++ /dev/null @@ -1,192 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2013 Cloudwatt -# Copyright (C) 2014 Red Hat -# -# Author: Loic Dachary -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# -set -xe -PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: ' - -source $(dirname $0)/../detect-build-env-vars.sh - -DIR=mkfs -export CEPH_CONF=/dev/null -unset CEPH_ARGS -MON_ID=a -MON_DIR=$DIR/$MON_ID -CEPH_MON=127.0.0.1:7110 # git grep '\<7110\>' : there must be only one -TIMEOUT=360 - -function setup() { - teardown - mkdir $DIR -} - -function teardown() { - kill_daemons - rm -fr $DIR -} - -function mon_mkfs() { - local fsid=$(uuidgen) - - ceph-mon \ - --id $MON_ID \ - --fsid $fsid \ - --erasure-code-dir=$CEPH_LIB \ - --mkfs \ - --mon-data=$MON_DIR \ - --mon-initial-members=$MON_ID \ - --mon-host=$CEPH_MON \ - "$@" -} - -function mon_run() { - ceph-mon \ - --id $MON_ID \ - --chdir= \ - --mon-osd-full-ratio=.99 \ - --mon-data-avail-crit=1 \ - --erasure-code-dir=$CEPH_LIB \ - --mon-data=$MON_DIR \ - --log-file=$MON_DIR/log \ - --mon-cluster-log-file=$MON_DIR/log \ - --run-dir=$MON_DIR \ - --pid-file=$MON_DIR/pidfile \ - --public-addr $CEPH_MON \ - "$@" -} - -function kill_daemons() { - for pidfile in $(find $DIR -name pidfile) ; do - pid=$(cat $pidfile) - for try in 0 1 1 1 2 3 ; do - kill $pid || break - sleep $try - done - done -} - -function auth_none() { - mon_mkfs --auth-supported=none - - ceph-mon \ - --id $MON_ID \ - --mon-osd-full-ratio=.99 \ - --mon-data-avail-crit=1 \ - --erasure-code-dir=$CEPH_LIB \ - --mon-data=$MON_DIR \ - --extract-monmap $MON_DIR/monmap - - [ -f $MON_DIR/monmap ] || return 1 - - [ ! -f $MON_DIR/keyring ] || return 1 - - mon_run --auth-supported=none - - timeout $TIMEOUT ceph --mon-host $CEPH_MON mon stat || return 1 -} - -function auth_cephx_keyring() { - cat > $DIR/keyring <&1 | tee $DIR/makedir.log - grep 'toodeep.*No such file' $DIR/makedir.log > /dev/null - rm $DIR/makedir.log - - # an empty directory does not mean the mon exists - mkdir $MON_DIR - mon_mkfs --auth-supported=none 2>&1 | tee $DIR/makedir.log - ! grep "$MON_DIR already exists" $DIR/makedir.log || return 1 -} - -function idempotent() { - mon_mkfs --auth-supported=none - mon_mkfs --auth-supported=none 2>&1 | tee $DIR/makedir.log - grep "'$MON_DIR' already exists" $DIR/makedir.log > /dev/null || return 1 -} - -function run() { - local actions - actions+="makedir " - actions+="idempotent " - actions+="auth_cephx_key " - actions+="auth_cephx_keyring " - actions+="auth_none " - for action in $actions ; do - setup - $action || return 1 - teardown - done -} - -run - -# Local Variables: -# compile-command: "cd ../.. ; make TESTS=test/mon/mkfs.sh check" -# End: diff --git a/src/test/mon/mon-bind.sh b/src/test/mon/mon-bind.sh deleted file mode 100755 index abac7761bd0..00000000000 --- a/src/test/mon/mon-bind.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2017 Quantum Corp. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -SOCAT_PIDS=() - -function port_forward() { - local source_port=$1 - local target_port=$2 - - socat TCP-LISTEN:${source_port},fork,reuseaddr TCP:localhost:${target_port} & - SOCAT_PIDS+=( $! ) -} - -function cleanup() { - for p in "${SOCAT_PIDS[@]}"; do - kill $p - done - SOCAT_PIDS=() -} - -trap cleanup SIGTERM SIGKILL SIGQUIT SIGINT - -function run() { - local dir=$1 - shift - - export MON_IP=127.0.0.1 - export MONA_PUBLIC=7132 # git grep '\<7132\>' ; there must be only one - export MONB_PUBLIC=7133 # git grep '\<7133\>' ; there must be only one - export MONC_PUBLIC=7134 # git grep '\<7134\>' ; there must be only one - export MONA_BIND=7135 # git grep '\<7135\>' ; there must be only one - export MONB_BIND=7136 # git grep '\<7136\>' ; there must be only one - export MONC_BIND=7137 # git grep '\<7137\>' ; there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - setup $dir || return 1 - $func $dir && cleanup || { cleanup; return 1; } - teardown $dir - done -} - -function TEST_mon_client_connect_fails() { - local dir=$1 - - # start the mon with a public-bind-addr that is different - # from the public-addr. - CEPH_ARGS+="--mon-initial-members=a " - CEPH_ARGS+="--mon-host=${MON_IP}:${MONA_PUBLIC} " - run_mon_no_pool $dir a --mon-host=${MON_IP}:${MONA_PUBLIC} --public-bind-addr=${MON_IP}:${MONA_BIND} || return 1 - - # now attempt to ping it that should fail. - timeout 3 ceph ping mon.a || return 0 - return 1 -} - -function TEST_mon_client_connect() { - local dir=$1 - - # start the mon with a public-bind-addr that is different - # from the public-addr. - CEPH_ARGS+="--mon-initial-members=a " - CEPH_ARGS+="--mon-host=${MON_IP}:${MONA_PUBLIC} " - run_mon_no_pool $dir a --mon-host=${MON_IP}:${MONA_PUBLIC} --public-bind-addr=${MON_IP}:${MONA_BIND} || return 1 - - # now forward the public port to the bind port. - port_forward ${MONA_PUBLIC} ${MONA_BIND} - - # attempt to connect. we expect that to work - ceph ping mon.a || return 1 -} - -function TEST_mon_quorum() { - local dir=$1 - - # start the mon with a public-bind-addr that is different - # from the public-addr. - CEPH_ARGS+="--mon-initial-members=a,b,c " - CEPH_ARGS+="--mon-host=${MON_IP}:${MONA_PUBLIC},${MON_IP}:${MONB_PUBLIC},${MON_IP}:${MONC_PUBLIC} " - run_mon_no_pool $dir a --public-addr=${MON_IP}:${MONA_PUBLIC} --public-bind-addr=${MON_IP}:${MONA_BIND} || return 1 - run_mon_no_pool $dir b --public-addr=${MON_IP}:${MONB_PUBLIC} --public-bind-addr=${MON_IP}:${MONB_BIND} || return 1 - run_mon_no_pool $dir c --public-addr=${MON_IP}:${MONC_PUBLIC} --public-bind-addr=${MON_IP}:${MONC_BIND} || return 1 - - # now forward the public port to the bind port. - port_forward ${MONA_PUBLIC} ${MONA_BIND} - port_forward ${MONB_PUBLIC} ${MONB_BIND} - port_forward ${MONC_PUBLIC} ${MONC_BIND} - - # expect monmap to contain 3 monitors (a, b, and c) - jqinput="$(ceph mon_status --format=json 2>/dev/null)" - jq_success "$jqinput" '.monmap.mons | length == 3' || return 1 - - # quorum should form - wait_for_quorum 300 3 || return 1 - # expect quorum to have all three monitors - jqfilter='.quorum | length == 3' - jq_success "$jqinput" "$jqfilter" || return 1 -} - -function TEST_put_get() { - local dir=$1 - - # start the mon with a public-bind-addr that is different - # from the public-addr. - CEPH_ARGS+="--mon-initial-members=a,b,c " - CEPH_ARGS+="--mon-host=${MON_IP}:${MONA_PUBLIC},${MON_IP}:${MONB_PUBLIC},${MON_IP}:${MONC_PUBLIC} " - run_mon_no_pool $dir a --public-addr=${MON_IP}:${MONA_PUBLIC} --public-bind-addr=${MON_IP}:${MONA_BIND} || return 1 - run_mon_no_pool $dir b --public-addr=${MON_IP}:${MONB_PUBLIC} --public-bind-addr=${MON_IP}:${MONB_BIND} || return 1 - run_mon_no_pool $dir c --public-addr=${MON_IP}:${MONC_PUBLIC} --public-bind-addr=${MON_IP}:${MONC_BIND} || return 1 - - # now forward the public port to the bind port. - port_forward ${MONA_PUBLIC} ${MONA_BIND} - port_forward ${MONB_PUBLIC} ${MONB_BIND} - port_forward ${MONC_PUBLIC} ${MONC_BIND} - - # quorum should form - wait_for_quorum 300 3 || return 1 - - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - run_osd $dir 1 || return 1 - run_osd $dir 2 || return 1 - - ceph osd pool create hello 8 || return 1 - - echo "hello world" > $dir/hello - rados --pool hello put foo $dir/hello || return 1 - rados --pool hello get foo $dir/hello2 || return 1 - diff $dir/hello $dir/hello2 || return 1 -} - -main mon-bind "$@" diff --git a/src/test/mon/mon-created-time.sh b/src/test/mon/mon-created-time.sh deleted file mode 100755 index 719044fc930..00000000000 --- a/src/test/mon/mon-created-time.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2015 SUSE LINUX GmbH -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7125" # git grep '\<7125\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - setup $dir || return 1 - $func $dir || return 1 - teardown $dir || return 1 - done -} - -function TEST_mon_created_time() { - local dir=$1 - - run_mon $dir a || return 1 - - ceph mon dump || return 1 - - if test "$(ceph mon dump 2>/dev/null | sed -n '/created/p' | awk '{print $NF}')"x = ""x ; then - return 1 - fi - - if test "$(ceph mon dump 2>/dev/null | sed -n '/created/p' | awk '{print $NF}')"x = "0.000000"x ; then - return 1 - fi -} - -main mon-created-time "$@" - -# Local Variables: -# compile-command: "cd ../.. ; make -j4 && test/mon/mon-created-time.sh" -# End: diff --git a/src/test/mon/mon-handle-forward.sh b/src/test/mon/mon-handle-forward.sh deleted file mode 100755 index c49657004ec..00000000000 --- a/src/test/mon/mon-handle-forward.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2013 Cloudwatt -# Copyright (C) 2014,2015 Red Hat -# -# Author: Loic Dachary -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - - setup $dir || return 1 - - MONA=127.0.0.1:7300 - MONB=127.0.0.1:7301 - ( - FSID=$(uuidgen) - export CEPH_ARGS - CEPH_ARGS+="--fsid=$FSID --auth-supported=none " - CEPH_ARGS+="--mon-initial-members=a,b --mon-host=$MONA,$MONB " - run_mon $dir a --public-addr $MONA || return 1 - run_mon $dir b --public-addr $MONB || return 1 - ) - - timeout 360 ceph --mon-host $MONA mon stat || return 1 - # check that MONB is indeed a peon - ceph --admin-daemon $(get_asok_path mon.b) mon_status | - grep '"peon"' || return 1 - # when the leader ( MONA ) is used, there is no message forwarding - ceph --mon-host $MONA osd pool create POOL1 12 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1 - grep 'mon_command(.*"POOL1"' $dir/a/mon.a.log - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.b) log flush || return 1 - grep 'mon_command(.*"POOL1"' $dir/mon.b.log && return 1 - # when the peon ( MONB ) is used, the message is forwarded to the leader - ceph --mon-host $MONB osd pool create POOL2 12 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.b) log flush || return 1 - grep 'forward_request.*mon_command(.*"POOL2"' $dir/mon.b.log - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1 - grep ' forward(mon_command(.*"POOL2"' $dir/mon.a.log - # forwarded messages must retain features from the original connection - features=$(sed -n -e 's|.*127.0.0.1:0.*accept features \([0-9][0-9]*\)|\1|p' < \ - $dir/mon.b.log) - grep ' forward(mon_command(.*"POOL2".*con_features '$features $dir/mon.a.log - - teardown $dir || return 1 -} - -main mon-handle-forward "$@" - -# Local Variables: -# compile-command: "cd ../.. ; make -j4 TESTS=test/mon/mon-handle-forward.sh check" -# End: diff --git a/src/test/mon/mon-ping.sh b/src/test/mon/mon-ping.sh deleted file mode 100755 index 9fb8848896f..00000000000 --- a/src/test/mon/mon-ping.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2015 SUSE LINUX GmbH -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7119" # git grep '\<7119\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - setup $dir || return 1 - $func $dir || return 1 - teardown $dir || return 1 - done -} - -function TEST_mon_ping() { - local dir=$1 - - run_mon $dir a || return 1 - - ceph ping mon.a || return 1 -} - -main mon-ping "$@" - -# Local Variables: -# compile-command: "cd ../.. ; make -j4 && test/mon/mon-ping.sh" -# End: diff --git a/src/test/mon/mon-scrub.sh b/src/test/mon/mon-scrub.sh deleted file mode 100755 index b140912ec17..00000000000 --- a/src/test/mon/mon-scrub.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2014 Cloudwatt -# Copyright (C) 2014, 2015 Red Hat -# -# Author: Loic Dachary -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7120" # git grep '\<7120\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - setup $dir || return 1 - $func $dir || return 1 - teardown $dir || return 1 - done -} - -function TEST_mon_scrub() { - local dir=$1 - - run_mon $dir a || return 1 - - ceph mon scrub || return 1 -} - -main mon-scrub "$@" - -# Local Variables: -# compile-command: "cd ../.. ; make -j4 && test/mon/mon-scrub.sh" -# End: diff --git a/src/test/mon/osd-crush.sh b/src/test/mon/osd-crush.sh deleted file mode 100755 index 7cc84f2f8ca..00000000000 --- a/src/test/mon/osd-crush.sh +++ /dev/null @@ -1,229 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2014 Cloudwatt -# Copyright (C) 2014, 2015 Red Hat -# -# Author: Loic Dachary -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7104" # git grep '\<7104\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - - local funcs=${@:-$(set | ${SED} -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - setup $dir || return 1 - $func $dir || return 1 - teardown $dir || return 1 - done -} - -function TEST_crush_rule_create_simple() { - local dir=$1 - - run_mon $dir a || return 1 - - ceph --format xml osd crush rule dump replicated_rule | \ - egrep 'take[^<]+default' | \ - grep 'choose_firstn0osd' || return 1 - local ruleset=ruleset0 - local root=host1 - ceph osd crush add-bucket $root host - local failure_domain=osd - ceph osd crush rule create-simple $ruleset $root $failure_domain || return 1 - ceph osd crush rule create-simple $ruleset $root $failure_domain 2>&1 | \ - grep "$ruleset already exists" || return 1 - ceph --format xml osd crush rule dump $ruleset | \ - egrep 'take[^<]+'$root'' | \ - grep 'choose_firstn0'$failure_domain'' || return 1 - ceph osd crush rule rm $ruleset || return 1 -} - -function TEST_crush_rule_dump() { - local dir=$1 - - run_mon $dir a || return 1 - - local ruleset=ruleset1 - ceph osd crush rule create-erasure $ruleset || return 1 - test $(ceph --format json osd crush rule dump $ruleset | \ - jq ".rule_name == \"$ruleset\"") == true || return 1 - test $(ceph --format json osd crush rule dump | \ - jq "map(select(.rule_name == \"$ruleset\")) | length == 1") == true || return 1 - ! ceph osd crush rule dump non_existent_ruleset || return 1 - ceph osd crush rule rm $ruleset || return 1 -} - -function TEST_crush_rule_rm() { - local ruleset=erasure2 - - run_mon $dir a || return 1 - - ceph osd crush rule create-erasure $ruleset default || return 1 - ceph osd crush rule ls | grep $ruleset || return 1 - ceph osd crush rule rm $ruleset || return 1 - ! ceph osd crush rule ls | grep $ruleset || return 1 -} - -function TEST_crush_rule_create_erasure() { - local dir=$1 - - run_mon $dir a || return 1 - # should have at least one OSD - run_osd $dir 0 || return 1 - - local ruleset=ruleset3 - # - # create a new ruleset with the default profile, implicitly - # - ceph osd crush rule create-erasure $ruleset || return 1 - ceph osd crush rule create-erasure $ruleset 2>&1 | \ - grep "$ruleset already exists" || return 1 - ceph --format xml osd crush rule dump $ruleset | \ - egrep 'take[^<]+default' | \ - grep 'chooseleaf_indep0host' || return 1 - ceph osd crush rule rm $ruleset || return 1 - ! ceph osd crush rule ls | grep $ruleset || return 1 - # - # create a new ruleset with the default profile, explicitly - # - ceph osd crush rule create-erasure $ruleset default || return 1 - ceph osd crush rule ls | grep $ruleset || return 1 - ceph osd crush rule rm $ruleset || return 1 - ! ceph osd crush rule ls | grep $ruleset || return 1 - # - # create a new ruleset and the default profile, implicitly - # - ceph osd erasure-code-profile rm default || return 1 - ! ceph osd erasure-code-profile ls | grep default || return 1 - ceph osd crush rule create-erasure $ruleset || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1 - grep 'profile set default' $dir/mon.a.log || return 1 - ceph osd erasure-code-profile ls | grep default || return 1 - ceph osd crush rule rm $ruleset || return 1 - ! ceph osd crush rule ls | grep $ruleset || return 1 -} - -function check_ruleset_id_match_rule_id() { - local rule_name=$1 - rule_id=`ceph osd crush rule dump $rule_name | grep "\"rule_id\":" | awk -F ":|," '{print int($2)}'` - ruleset_id=`ceph osd crush rule dump $rule_name | grep "\"ruleset\":"| awk -F ":|," '{print int($2)}'` - test $ruleset_id = $rule_id || return 1 -} - -function generate_manipulated_rules() { - local dir=$1 - ceph osd crush add-bucket $root host - ceph osd crush rule create-simple test_rule1 $root osd firstn || return 1 - ceph osd crush rule create-simple test_rule2 $root osd firstn || return 1 - ceph osd getcrushmap -o $dir/original_map - crushtool -d $dir/original_map -o $dir/decoded_original_map - #manipulate the rulesets , to make the rule_id != ruleset_id - ${SED} -i 's/ruleset 0/ruleset 3/' $dir/decoded_original_map - ${SED} -i 's/ruleset 2/ruleset 0/' $dir/decoded_original_map - ${SED} -i 's/ruleset 1/ruleset 2/' $dir/decoded_original_map - - crushtool -c $dir/decoded_original_map -o $dir/new_map - ceph osd setcrushmap -i $dir/new_map - - ceph osd crush rule dump -} - -function TEST_crush_ruleset_match_rule_when_creating() { - local dir=$1 - - run_mon $dir a || return 1 - - local root=host1 - - generate_manipulated_rules $dir - - ceph osd crush rule create-simple special_rule_simple $root osd firstn || return 1 - - ceph osd crush rule dump - #show special_rule_simple has same rule_id and ruleset_id - check_ruleset_id_match_rule_id special_rule_simple || return 1 -} - -function TEST_add_ruleset_failed() { - local dir=$1 - - run_mon $dir a || return 1 - - local root=host1 - - ceph osd crush add-bucket $root host - ceph osd crush rule create-simple test_rule1 $root osd firstn || return 1 - ceph osd crush rule create-simple test_rule2 $root osd firstn || return 1 - ceph osd getcrushmap > $dir/crushmap || return 1 - crushtool --decompile $dir/crushmap > $dir/crushmap.txt || return 1 - for i in $(seq 3 255) - do - cat <> $dir/crushmap.txt - crushtool --compile $dir/crushmap.txt -o $dir/crushmap || return 1 - ceph osd setcrushmap -i $dir/crushmap || return 1 - ceph osd crush rule create-simple test_rule_nospace $root osd firstn 2>&1 | grep "Error ENOSPC" || return 1 - -} - -function TEST_crush_rename_bucket() { - local dir=$1 - - run_mon $dir a || return 1 - - ceph osd crush add-bucket host1 host - ceph osd tree - ! ceph osd tree | grep host2 || return 1 - ceph osd crush rename-bucket host1 host2 || return 1 - ceph osd tree - ceph osd tree | grep host2 || return 1 - ceph osd crush rename-bucket host1 host2 || return 1 # idempotency - ceph osd crush rename-bucket nonexistent something 2>&1 | grep "Error ENOENT" || return 1 -} - -function TEST_crush_reject_empty() { - local dir=$1 - run_mon $dir a || return 1 - # should have at least one OSD - run_osd $dir 0 || return 1 - - local empty_map=$dir/empty_map - :> $empty_map.txt - crushtool -c $empty_map.txt -o $empty_map.map || return 1 - expect_failure $dir "Error EINVAL" \ - ceph osd setcrushmap -i $empty_map.map || return 1 -} - -main osd-crush "$@" - -# Local Variables: -# compile-command: "cd ../.. ; make -j4 && test/mon/osd-crush.sh" -# End: diff --git a/src/test/mon/osd-erasure-code-profile.sh b/src/test/mon/osd-erasure-code-profile.sh deleted file mode 100755 index f99e091d187..00000000000 --- a/src/test/mon/osd-erasure-code-profile.sh +++ /dev/null @@ -1,230 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2014 Cloudwatt -# Copyright (C) 2014, 2015 Red Hat -# -# Author: Loic Dachary -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7220" # git grep '\<7220\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - setup $dir || return 1 - $func $dir || return 1 - teardown $dir || return 1 - done -} - -function TEST_set() { - local dir=$1 - local id=$2 - - run_mon $dir a || return 1 - - local profile=myprofile - # - # no key=value pairs : use the default configuration - # - ceph osd erasure-code-profile set $profile 2>&1 || return 1 - ceph osd erasure-code-profile get $profile | \ - grep plugin=jerasure || return 1 - ceph osd erasure-code-profile rm $profile - # - # key=value pairs override the default - # - ceph osd erasure-code-profile set $profile \ - key=value plugin=example || return 1 - ceph osd erasure-code-profile get $profile | \ - grep -e key=value -e plugin=example || return 1 - # - # --force is required to override an existing profile - # - ! ceph osd erasure-code-profile set $profile > $dir/out 2>&1 || return 1 - grep 'will not override' $dir/out || return 1 - ceph osd erasure-code-profile set $profile key=other --force || return 1 - ceph osd erasure-code-profile get $profile | \ - grep key=other || return 1 - - ceph osd erasure-code-profile rm $profile # cleanup -} - -function TEST_ls() { - local dir=$1 - local id=$2 - - run_mon $dir a || return 1 - - local profile=myprofile - ! ceph osd erasure-code-profile ls | grep $profile || return 1 - ceph osd erasure-code-profile set $profile 2>&1 || return 1 - ceph osd erasure-code-profile ls | grep $profile || return 1 - ceph --format xml osd erasure-code-profile ls | \ - grep "$profile" || return 1 - - ceph osd erasure-code-profile rm $profile # cleanup -} - -function TEST_rm() { - local dir=$1 - local id=$2 - - run_mon $dir a || return 1 - - local profile=myprofile - ceph osd erasure-code-profile set $profile 2>&1 || return 1 - ceph osd erasure-code-profile ls | grep $profile || return 1 - ceph osd erasure-code-profile rm $profile || return 1 - ! ceph osd erasure-code-profile ls | grep $profile || return 1 - ceph osd erasure-code-profile rm WRONG 2>&1 | \ - grep "WRONG does not exist" || return 1 - - ceph osd erasure-code-profile set $profile || return 1 - ceph osd pool create poolname 12 12 erasure $profile || return 1 - ! ceph osd erasure-code-profile rm $profile > $dir/out 2>&1 || return 1 - grep "poolname.*using.*$profile" $dir/out || return 1 - ceph osd pool delete poolname poolname --yes-i-really-really-mean-it || return 1 - ceph osd erasure-code-profile rm $profile || return 1 - - ceph osd erasure-code-profile rm $profile # cleanup -} - -function TEST_get() { - local dir=$1 - local id=$2 - - run_mon $dir a || return 1 - - local default_profile=default - ceph osd erasure-code-profile get $default_profile | \ - grep plugin=jerasure || return 1 - ceph --format xml osd erasure-code-profile get $default_profile | \ - grep 'jerasure' || return 1 - ! ceph osd erasure-code-profile get WRONG > $dir/out 2>&1 || return 1 - grep -q "unknown erasure code profile 'WRONG'" $dir/out || return 1 -} - -function TEST_set_idempotent() { - local dir=$1 - local id=$2 - - run_mon $dir a || return 1 - # - # The default profile is set using a code path different from - # ceph osd erasure-code-profile set: verify that it is idempotent, - # as if it was using the same code path. - # - ceph osd erasure-code-profile set default k=2 m=1 2>&1 || return 1 - local profile - # - # Because plugin=jerasure is the default, it uses a slightly - # different code path where defaults (m=1 for instance) are added - # implicitly. - # - profile=profileidempotent1 - ! ceph osd erasure-code-profile ls | grep $profile || return 1 - ceph osd erasure-code-profile set $profile k=2 crush-failure-domain=osd 2>&1 || return 1 - ceph osd erasure-code-profile ls | grep $profile || return 1 - ceph osd erasure-code-profile set $profile k=2 crush-failure-domain=osd 2>&1 || return 1 - ceph osd erasure-code-profile rm $profile # cleanup - - # - # In the general case the profile is exactly what is on - # - profile=profileidempotent2 - ! ceph osd erasure-code-profile ls | grep $profile || return 1 - ceph osd erasure-code-profile set $profile plugin=lrc k=4 m=2 l=3 crush-failure-domain=osd 2>&1 || return 1 - ceph osd erasure-code-profile ls | grep $profile || return 1 - ceph osd erasure-code-profile set $profile plugin=lrc k=4 m=2 l=3 crush-failure-domain=osd 2>&1 || return 1 - ceph osd erasure-code-profile rm $profile # cleanup -} - -function TEST_format_invalid() { - local dir=$1 - - local profile=profile - # osd_pool_default_erasure-code-profile is - # valid JSON but not of the expected type - run_mon $dir a \ - --osd_pool_default_erasure-code-profile 1 || return 1 - ! ceph osd erasure-code-profile set $profile > $dir/out 2>&1 || return 1 - cat $dir/out - grep 'must be a JSON object' $dir/out || return 1 -} - -function TEST_format_json() { - local dir=$1 - - # osd_pool_default_erasure-code-profile is JSON - expected='"plugin":"example"' - run_mon $dir a \ - --osd_pool_default_erasure-code-profile "{$expected}" || return 1 - ceph --format json osd erasure-code-profile get default | \ - grep "$expected" || return 1 -} - -function TEST_format_plain() { - local dir=$1 - - # osd_pool_default_erasure-code-profile is plain text - expected='"plugin":"example"' - run_mon $dir a \ - --osd_pool_default_erasure-code-profile "plugin=example" || return 1 - ceph --format json osd erasure-code-profile get default | \ - grep "$expected" || return 1 -} - -function TEST_profile_k_sanity() { - local dir=$1 - local profile=profile-sanity - - run_mon $dir a || return 1 - - expect_failure $dir 'k must be a multiple of (k + m) / l' \ - ceph osd erasure-code-profile set $profile \ - plugin=lrc \ - l=1 \ - k=1 \ - m=1 || return 1 - - if erasure_code_plugin_exists isa ; then - expect_failure $dir 'k=1 must be >= 2' \ - ceph osd erasure-code-profile set $profile \ - plugin=isa \ - k=1 \ - m=1 || return 1 - else - echo "SKIP because plugin isa has not been built" - fi - - expect_failure $dir 'k=1 must be >= 2' \ - ceph osd erasure-code-profile set $profile \ - plugin=jerasure \ - k=1 \ - m=1 || return 1 -} - -main osd-erasure-code-profile "$@" - -# Local Variables: -# compile-command: "cd ../.. ; make -j4 && test/mon/osd-erasure-code-profile.sh" -# End: diff --git a/src/test/mon/osd-pool-create.sh b/src/test/mon/osd-pool-create.sh deleted file mode 100755 index 53733abf804..00000000000 --- a/src/test/mon/osd-pool-create.sh +++ /dev/null @@ -1,221 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2013, 2014 Cloudwatt -# Copyright (C) 2014, 2015 Red Hat -# -# Author: Loic Dachary -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7105" # git grep '\<7105\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - setup $dir || return 1 - $func $dir || return 1 - teardown $dir || return 1 - done -} - -# Before http://tracker.ceph.com/issues/8307 the invalid profile was created -function TEST_erasure_invalid_profile() { - local dir=$1 - run_mon $dir a || return 1 - local poolname=pool_erasure - local notaprofile=not-a-valid-erasure-code-profile - ! ceph osd pool create $poolname 12 12 erasure $notaprofile || return 1 - ! ceph osd erasure-code-profile ls | grep $notaprofile || return 1 -} - -function TEST_erasure_crush_rule() { - local dir=$1 - run_mon $dir a || return 1 - # - # choose the crush ruleset used with an erasure coded pool - # - local crush_ruleset=myruleset - ! ceph osd crush rule ls | grep $crush_ruleset || return 1 - ceph osd crush rule create-erasure $crush_ruleset - ceph osd crush rule ls | grep $crush_ruleset - local poolname - poolname=pool_erasure1 - ! ceph --format json osd dump | grep '"crush_rule":1' || return 1 - ceph osd pool create $poolname 12 12 erasure default $crush_ruleset - ceph --format json osd dump | grep '"crush_rule":1' || return 1 - # - # a crush ruleset by the name of the pool is implicitly created - # - poolname=pool_erasure2 - ceph osd erasure-code-profile set myprofile - ceph osd pool create $poolname 12 12 erasure myprofile - ceph osd crush rule ls | grep $poolname || return 1 - # - # a non existent crush ruleset given in argument is an error - # http://tracker.ceph.com/issues/9304 - # - poolname=pool_erasure3 - ! ceph osd pool create $poolname 12 12 erasure myprofile INVALIDRULESET || return 1 -} - -function TEST_erasure_code_profile_default() { - local dir=$1 - run_mon $dir a || return 1 - ceph osd erasure-code-profile rm default || return 1 - ! ceph osd erasure-code-profile ls | grep default || return 1 - ceph osd pool create $poolname 12 12 erasure default - ceph osd erasure-code-profile ls | grep default || return 1 -} - -function TEST_erasure_crush_stripe_unit() { - local dir=$1 - # the default stripe unit is used to initialize the pool - run_mon $dir a --public-addr $CEPH_MON - stripe_unit=$(ceph-conf --show-config-value osd_pool_erasure_code_stripe_unit) - eval local $(ceph osd erasure-code-profile get myprofile | grep k=) - stripe_width = $((stripe_unit * k)) - ceph osd pool create pool_erasure 12 12 erasure - ceph --format json osd dump | tee $dir/osd.json - grep '"stripe_width":'$stripe_width $dir/osd.json > /dev/null || return 1 -} - -function TEST_erasure_crush_stripe_unit_padded() { - local dir=$1 - # setting osd_pool_erasure_code_stripe_unit modifies the stripe_width - # and it is padded as required by the default plugin - profile+=" plugin=jerasure" - profile+=" technique=reed_sol_van" - k=4 - profile+=" k=$k" - profile+=" m=2" - actual_stripe_unit=2048 - desired_stripe_unit=$((actual_stripe_unit - 1)) - actual_stripe_width=$((actual_stripe_unit * k)) - run_mon $dir a \ - --osd_pool_erasure_code_stripe_unit $desired_stripe_unit \ - --osd_pool_default_erasure_code_profile "$profile" || return 1 - ceph osd pool create pool_erasure 12 12 erasure - ceph osd dump | tee $dir/osd.json - grep "stripe_width $actual_stripe_width" $dir/osd.json > /dev/null || return 1 -} - -function TEST_erasure_code_pool() { - local dir=$1 - run_mon $dir a || return 1 - ceph --format json osd dump > $dir/osd.json - local expected='"erasure_code_profile":"default"' - ! grep "$expected" $dir/osd.json || return 1 - ceph osd pool create erasurecodes 12 12 erasure - ceph --format json osd dump | tee $dir/osd.json - grep "$expected" $dir/osd.json > /dev/null || return 1 - - ceph osd pool create erasurecodes 12 12 erasure 2>&1 | \ - grep 'already exists' || return 1 - ceph osd pool create erasurecodes 12 12 2>&1 | \ - grep 'cannot change to type replicated' || return 1 -} - -function TEST_replicated_pool_with_ruleset() { - local dir=$1 - run_mon $dir a - local ruleset=ruleset0 - local root=host1 - ceph osd crush add-bucket $root host - local failure_domain=osd - local poolname=mypool - ceph osd crush rule create-simple $ruleset $root $failure_domain || return 1 - ceph osd crush rule ls | grep $ruleset - ceph osd pool create $poolname 12 12 replicated $ruleset 2>&1 | \ - grep "pool 'mypool' created" || return 1 - rule_id=`ceph osd crush rule dump $ruleset | grep "rule_id" | awk -F[' ':,] '{print $4}'` - ceph osd pool get $poolname crush_rule 2>&1 | \ - grep "crush_rule: $rule_id" || return 1 - #non-existent crush ruleset - ceph osd pool create newpool 12 12 replicated non-existent 2>&1 | \ - grep "doesn't exist" || return 1 -} - -function TEST_erasure_code_pool_lrc() { - local dir=$1 - run_mon $dir a || return 1 - - ceph osd erasure-code-profile set LRCprofile \ - plugin=lrc \ - mapping=DD_ \ - layers='[ [ "DDc", "" ] ]' || return 1 - - ceph --format json osd dump > $dir/osd.json - local expected='"erasure_code_profile":"LRCprofile"' - local poolname=erasurecodes - ! grep "$expected" $dir/osd.json || return 1 - ceph osd pool create $poolname 12 12 erasure LRCprofile - ceph --format json osd dump | tee $dir/osd.json - grep "$expected" $dir/osd.json > /dev/null || return 1 - ceph osd crush rule ls | grep $poolname || return 1 -} - -function TEST_replicated_pool() { - local dir=$1 - run_mon $dir a || return 1 - ceph osd pool create replicated 12 12 replicated replicated_rule 2>&1 | \ - grep "pool 'replicated' created" || return 1 - ceph osd pool create replicated 12 12 replicated replicated_rule 2>&1 | \ - grep 'already exists' || return 1 - # default is replicated - ceph osd pool create replicated1 12 12 2>&1 | \ - grep "pool 'replicated1' created" || return 1 - # default is replicated, pgp_num = pg_num - ceph osd pool create replicated2 12 2>&1 | \ - grep "pool 'replicated2' created" || return 1 - ceph osd pool create replicated 12 12 erasure 2>&1 | \ - grep 'cannot change to type erasure' || return 1 -} - -function TEST_no_pool_delete() { - local dir=$1 - run_mon $dir a || return 1 - ceph osd pool create foo 1 || return 1 - ceph tell mon.a injectargs -- --no-mon-allow-pool-delete || return 1 - ! ceph osd pool delete foo foo --yes-i-really-really-mean-it || return 1 - ceph tell mon.a injectargs -- --mon-allow-pool-delete || return 1 - ceph osd pool delete foo foo --yes-i-really-really-mean-it || return 1 -} - -function TEST_utf8_cli() { - local dir=$1 - run_mon $dir a || return 1 - # Hopefully it's safe to include literal UTF-8 characters to test - # the fix for http://tracker.ceph.com/issues/7387. If it turns out - # to not be OK (when is the default encoding *not* UTF-8?), maybe - # the character '黄' can be replaced with the escape $'\xe9\xbb\x84' - ceph osd pool create 黄 1024 2>&1 | \ - grep "pool '黄' created" || return 1 - ceph osd lspools 2>&1 | \ - grep "黄" || return 1 - ceph -f json-pretty osd dump | \ - python -c "import json; import sys; json.load(sys.stdin)" || return 1 - ceph osd pool delete 黄 黄 --yes-i-really-really-mean-it -} - -main osd-pool-create "$@" - -# Local Variables: -# compile-command: "cd ../.. ; make -j4 && test/mon/osd-pool-create.sh" -# End: diff --git a/src/test/mon/run_test.sh b/src/test/mon/run_test.sh deleted file mode 100755 index ef2f6d063ec..00000000000 --- a/src/test/mon/run_test.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/bash -# vim: ts=8 sw=2 smarttab -# -# $0.sh - run mon workload generator - -if [[ ! -e "./ceph_ver.h" ]]; then - echo "This script must be run from the repository's src/ directory" - exit 1 -fi - -usage() { - echo "usage: $1 [options..] " - echo - echo "options:" - echo " -v, --verbose Be more verbose" - echo " -c, --conf FILE ceph.conf location" - echo " -d, --duration SECS Run test for SECS seconds (default: 300)" - echo " --debug LEVEL Set the test's debug level (default: 0)" - echo " -n, --new Make a fresh run by creating a new cluster" - echo - echo "environment variables:" - echo " EXTRA_ARGS Pass additional ceph arguments to the test" - echo -} - -stop_ceph() { - if [[ ! -e "init-ceph" ]]; then - echo "could not find 'init-ceph'; killing only by hand and may bob have" - echo "mercy on our souls" - else - ./init-ceph stop - fi - - for i in mon osd mds; do - killall -9 ceph-$i - done -} - -start_mon() { - if [[ ! -e "init-ceph" ]]; then - echo "could not find 'init-ceph'; attempting to start monitors by hand" - - for i in a b c; do - ./ceph-mon -i $i -c ceph.conf -k keyring -d - done - else - ./init-ceph start mon - fi -} - -make_fresh() { - rm -fr dev/ out/ keyring - mkdir dev - - if [[ ! -e "vstart.sh" ]]; then - echo "could not find 'vstart.sh', which is weird; what have you done?" - exit 1 - fi - - env MON=3 OSD=0 MDS=0 MGR=0 \ - ./vstart.sh -n -l -d mon -} - -DEBUG=0 -TEST_CEPH_CONF= -DURATION= -LOADGEN_NUM_OSDS= -ARGS= - -[[ ! -z $EXTRA_ARGS ]] && ARGS="$EXTRA_ARGS" - -fresh_run=0 - -while [[ $# -gt 0 ]]; -do - case "$1" in - -v | --verbose) - VERBOSE=1 - shift - ;; - -c | --conf) - if [[ "$2" == "" ]]; then - echo "'$1' expects an argument; none was given" - exit 1 - fi - TEST_CEPH_CONF="$2" - shift 2 - ;; - --debug) - if [[ -z $2 ]]; then - echo "'$1' expects an argument; none was given" - exit 1 - fi - ARGS="$ARGS --debug-none $2" - shift 2 - ;; - -d | --duration) - if [[ -z $2 ]]; then - echo "'$1' expects an argument; none was given" - exit 1 - fi - DURATION=$2 - shift 2 - ;; - -n | --new) - fresh_run=1 - shift - ;; - --) - shift - break - ;; - -*) - echo "$1: unknown option" >&2 - usage $0 - exit 1 - ;; - *) - LOADGEN_NUM_OSDS=$1 - shift - break - ;; - esac -done - -if [[ -z $LOADGEN_NUM_OSDS ]]; then - echo "must specify the number of osds" - usage $0 - exit 1 -fi - -stop_ceph ; -[[ $fresh_run -eq 1 ]] && make_fresh ; -start_mon ; - -env VERBOSE=$VERBOSE TEST_CEPH_CONF="$TEST_CEPH_CONF" \ - DURATION=$DURATION EXTRA_ARGS="$ARGS" \ - LOADGEN_NUM_OSDS=$LOADGEN_NUM_OSDS \ - PATH="$PATH:`pwd`" ../qa/workunits/mon/workloadgen.sh diff --git a/src/test/mon/test_pool_quota.sh b/src/test/mon/test_pool_quota.sh deleted file mode 100755 index 28c58bdc839..00000000000 --- a/src/test/mon/test_pool_quota.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -# -# Generic pool quota test -# - -# Includes - -source $(dirname $0)/../detect-build-env-vars.sh - -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:17108" # git grep '\<17108\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - $func $dir || return 1 - done -} - -function TEST_pool_quota() { - local dir=$1 - setup $dir || return 1 - - run_mon $dir a || return 1 - run_osd $dir 0 || return 1 - run_osd $dir 1 || return 1 - run_osd $dir 2 || return 1 - - local poolname=testquoa - ceph osd pool create $poolname 20 - local objects=`ceph df detail | grep -w $poolname|awk '{print $3}'` - local bytes=`ceph df detail | grep -w $poolname|awk '{print $4}'` - - echo $objects - echo $bytes - if [ $objects != 'N/A' ] || [ $bytes != 'N/A' ] ; - then - return 1 - fi - - ceph osd pool set-quota $poolname max_objects 1000 - ceph osd pool set-quota $poolname max_bytes 1024 - - objects=`ceph df detail | grep -w $poolname|awk '{print $3}'` - bytes=`ceph df detail | grep -w $poolname|awk '{print $4}'` - - if [ $objects != '1000' ] || [ $bytes != '1024' ] ; - then - return 1 - fi - - ceph osd pool delete $poolname $poolname --yes-i-really-really-mean-it - teardown $dir || return 1 -} - -main testpoolquota diff --git a/src/test/osd/CMakeLists.txt b/src/test/osd/CMakeLists.txt index 6297e8a03a1..79281014526 100644 --- a/src/test/osd/CMakeLists.txt +++ b/src/test/osd/CMakeLists.txt @@ -17,20 +17,6 @@ install(TARGETS ceph_test_rados DESTINATION ${CMAKE_INSTALL_BINDIR}) -# scripts -add_ceph_test(osd-bench.sh ${CMAKE_CURRENT_SOURCE_DIR}/osd-bench.sh) -add_ceph_test(osd-config.sh ${CMAKE_CURRENT_SOURCE_DIR}/osd-config.sh) -add_ceph_test(osd-markdown.sh ${CMAKE_CURRENT_SOURCE_DIR}/osd-markdown.sh) -add_ceph_test(osd-reactivate.sh ${CMAKE_CURRENT_SOURCE_DIR}/osd-reactivate.sh) -add_ceph_test(osd-reuse-id.sh ${CMAKE_CURRENT_SOURCE_DIR}/osd-reuse-id.sh) -add_ceph_test(osd-scrub-repair.sh ${CMAKE_CURRENT_SOURCE_DIR}/osd-scrub-repair.sh) -add_ceph_test(osd-scrub-snaps.sh ${CMAKE_CURRENT_SOURCE_DIR}/osd-scrub-snaps.sh) -add_ceph_test(osd-copy-from.sh ${CMAKE_CURRENT_SOURCE_DIR}/osd-copy-from.sh) -add_ceph_test(osd-fast-mark-down.sh ${CMAKE_CURRENT_SOURCE_DIR}/osd-fast-mark-down.sh) -if(HAVE_LIBAIO) - add_ceph_test(osd-dup.sh ${CMAKE_CURRENT_SOURCE_DIR}/osd-dup.sh) -endif() - # unittest_osdmap add_executable(unittest_osdmap TestOSDMap.cc diff --git a/src/test/osd/osd-bench.sh b/src/test/osd/osd-bench.sh deleted file mode 100755 index c44eb28e56a..00000000000 --- a/src/test/osd/osd-bench.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2014 Cloudwatt -# Copyright (C) 2014, 2015 Red Hat -# -# Author: Loic Dachary -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# - -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7106" # git grep '\<7106\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - setup $dir || return 1 - $func $dir || return 1 - teardown $dir || return 1 - done -} - -function TEST_bench() { - local dir=$1 - - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - - local osd_bench_small_size_max_iops=$(CEPH_ARGS='' ceph-conf \ - --show-config-value osd_bench_small_size_max_iops) - local osd_bench_large_size_max_throughput=$(CEPH_ARGS='' ceph-conf \ - --show-config-value osd_bench_large_size_max_throughput) - local osd_bench_max_block_size=$(CEPH_ARGS='' ceph-conf \ - --show-config-value osd_bench_max_block_size) - local osd_bench_duration=$(CEPH_ARGS='' ceph-conf \ - --show-config-value osd_bench_duration) - - # - # block size too high - # - expect_failure $dir osd_bench_max_block_size \ - ceph tell osd.0 bench 1024 $((osd_bench_max_block_size + 1)) || return 1 - - # - # count too high for small (< 1MB) block sizes - # - local bsize=1024 - local max_count=$(($bsize * $osd_bench_duration * $osd_bench_small_size_max_iops)) - expect_failure $dir bench_small_size_max_iops \ - ceph tell osd.0 bench $(($max_count + 1)) $bsize || return 1 - - # - # count too high for large (>= 1MB) block sizes - # - local bsize=$((1024 * 1024 + 1)) - local max_count=$(($osd_bench_large_size_max_throughput * $osd_bench_duration)) - expect_failure $dir osd_bench_large_size_max_throughput \ - ceph tell osd.0 bench $(($max_count + 1)) $bsize || return 1 - - # - # default values should work - # - ceph tell osd.0 bench || return 1 - - # - # test object_size < block_size - ceph tell osd.0 bench 10 14456 4444 3 - # - - # - # test object_size < block_size & object_size = 0(default value) - # - ceph tell osd.0 bench 1 14456 -} - -main osd-bench "$@" - -# Local Variables: -# compile-command: "cd ../.. ; make -j4 && test/osd/osd-bench.sh" -# End: diff --git a/src/test/osd/osd-config.sh b/src/test/osd/osd-config.sh deleted file mode 100755 index 4ef266ee98d..00000000000 --- a/src/test/osd/osd-config.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2014 Cloudwatt -# Copyright (C) 2014, 2015 Red Hat -# -# Author: Loic Dachary -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# - -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7100" # git grep '\<7100\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - setup $dir || return 1 - $func $dir || return 1 - teardown $dir || return 1 - done -} - -function TEST_config_init() { - local dir=$1 - - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - local advance=1000 - local stale=1000 - local cache=500 - run_osd $dir 0 \ - --osd-map-max-advance $advance \ - --osd-map-cache-size $cache \ - --osd-pg-epoch-persisted-max-stale $stale \ - || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1 - grep 'is not > osd_map_max_advance' $dir/osd.0.log || return 1 - grep 'is not > osd_pg_epoch_persisted_max_stale' $dir/osd.0.log || return 1 -} - -function TEST_config_track() { - local dir=$1 - - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - - local osd_map_cache_size=$(CEPH_ARGS='' ceph-conf \ - --show-config-value osd_map_cache_size) - local osd_map_max_advance=$(CEPH_ARGS='' ceph-conf \ - --show-config-value osd_map_max_advance) - local osd_pg_epoch_persisted_max_stale=$(CEPH_ARGS='' ceph-conf \ - --show-config-value osd_pg_epoch_persisted_max_stale) - # - # lower cache_size under max_advance to trigger the warning - # - ! grep 'is not > osd_map_max_advance' $dir/osd.0.log || return 1 - local cache=$(($osd_map_max_advance / 2)) - ceph tell osd.0 injectargs "--osd-map-cache-size $cache" || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1 - grep 'is not > osd_map_max_advance' $dir/osd.0.log || return 1 - rm $dir/osd.0.log - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log reopen || return 1 - - # - # reset cache_size to the default and assert that it does not trigger the warning - # - ! grep 'is not > osd_map_max_advance' $dir/osd.0.log || return 1 - local cache=$osd_map_cache_size - ceph tell osd.0 injectargs "--osd-map-cache-size $cache" || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1 - ! grep 'is not > osd_map_max_advance' $dir/osd.0.log || return 1 - rm $dir/osd.0.log - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log reopen || return 1 - - # - # increase the osd_map_max_advance above the default cache_size - # - ! grep 'is not > osd_map_max_advance' $dir/osd.0.log || return 1 - local advance=$(($osd_map_cache_size * 2)) - ceph tell osd.0 injectargs "--osd-map-max-advance $advance" || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1 - grep 'is not > osd_map_max_advance' $dir/osd.0.log || return 1 - rm $dir/osd.0.log - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log reopen || return 1 - - # - # increase the osd_pg_epoch_persisted_max_stale above the default cache_size - # - ! grep 'is not > osd_pg_epoch_persisted_max_stale' $dir/osd.0.log || return 1 - local stale=$(($osd_map_cache_size * 2)) - ceph tell osd.0 injectargs "--osd-pg-epoch-persisted-max-stale $stale" || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1 - grep 'is not > osd_pg_epoch_persisted_max_stale' $dir/osd.0.log || return 1 - rm $dir/osd.0.log - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log reopen || return 1 -} - -main osd-config "$@" - -# Local Variables: -# compile-command: "cd ../.. ; make -j4 && test/osd/osd-config.sh" -# End: diff --git a/src/test/osd/osd-copy-from.sh b/src/test/osd/osd-copy-from.sh deleted file mode 100755 index ce7c04dfe4c..00000000000 --- a/src/test/osd/osd-copy-from.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2014 Cloudwatt -# Copyright (C) 2014, 2015 Red Hat -# -# Author: Loic Dachary -# Author: Sage Weil -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# - -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7111" # git grep '\<7111\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - setup $dir || return 1 - $func $dir || return 1 - teardown $dir || return 1 - done -} - -function TEST_copy_from() { - local dir=$1 - - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - run_osd $dir 1 || return 1 - - # success - rados -p rbd put foo $(which rados) - rados -p rbd cp foo foo2 - rados -p rbd stat foo2 - - # failure - ceph tell osd.\* injectargs -- --osd-debug-inject-copyfrom-error - ! rados -p rbd cp foo foo3 - ! rados -p rbd stat foo3 - - # success again - ceph tell osd.\* injectargs -- --no-osd-debug-inject-copyfrom-error - ! rados -p rbd cp foo foo3 - rados -p rbd stat foo3 -} - -main osd-copy-from "$@" - -# Local Variables: -# compile-command: "cd ../.. ; make -j4 && test/osd/osd-bench.sh" -# End: diff --git a/src/test/osd/osd-dup.sh b/src/test/osd/osd-dup.sh deleted file mode 100755 index f497005d168..00000000000 --- a/src/test/osd/osd-dup.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash - -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7146" # git grep '\<7146\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - # avoid running out of fds in rados bench - CEPH_ARGS+="--filestore_wbthrottle_xfs_ios_hard_limit=900 " - CEPH_ARGS+="--filestore_wbthrottle_btrfs_ios_hard_limit=900 " - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - setup $dir || return 1 - $func $dir || return 1 - teardown $dir || return 1 - done -} - -function TEST_filestore_to_bluestore() { - local dir=$1 - - local flimit=$(ulimit -n) - if [ $flimit -lt 1536 ]; then - echo "Low open file limit ($flimit), test may fail. Increase to 1536 or higher and retry if that happens." - fi - - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - osd_pid=$(cat $dir/osd.0.pid) - run_osd $dir 1 || return 1 - run_osd $dir 2 || return 1 - - sleep 5 - - ceph osd pool create foo 16 - - # write some objects - rados bench -p foo 10 write -b 4096 --no-cleanup || return 1 - - # kill - while kill $osd_pid; do sleep 1 ; done - ceph osd down 0 - - mv $dir/0 $dir/0.old || return 1 - mkdir $dir/0 || return 1 - ofsid=$(cat $dir/0.old/fsid) - echo "osd fsid $ofsid" - O=$CEPH_ARGS - CEPH_ARGS+="--log-file $dir/cot.log --log-max-recent 0 " - ceph-objectstore-tool --type bluestore --data-path $dir/0 --fsid $ofsid \ - --op mkfs || return 1 - ceph-objectstore-tool --data-path $dir/0.old --target-data-path $dir/0 \ - --op dup || return 1 - CEPH_ARGS=$O - - run_osd_bluestore $dir 0 || return 1 - - while ! ceph osd stat | grep '3 up' ; do sleep 1 ; done - ceph osd metadata 0 | grep bluestore || return 1 - - ceph osd scrub 0 - - # give it some time - sleep 15 - # and make sure mon is sync'ed - flush_pg_stats - - wait_for_clean || return 1 -} - -main osd-dup "$@" - -# Local Variables: -# compile-command: "cd ../.. ; make -j4 && test/osd/osd-dup.sh" -# End: diff --git a/src/test/osd/osd-fast-mark-down.sh b/src/test/osd/osd-fast-mark-down.sh deleted file mode 100755 index 051b7d32815..00000000000 --- a/src/test/osd/osd-fast-mark-down.sh +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2016 Piotr Dałek -# Copyright (C) 2014, 2015 Red Hat -# -# Author: Piotr Dałek -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# - -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh -MAX_PROPAGATION_TIME=30 - -function run() { - local dir=$1 - shift - rm -f $dir/*.pid - export CEPH_MON="127.0.0.1:7126" # git grep '\<7126\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - # - # Disable device auto class feature for this testing, - # as it will automatically make root clones based on new class types - # and hence affect the down osd counting. - # E.g., - # - # ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY - # -4 3.00000 root default~hdd - # -3 3.00000 host gitbuilder-ceph-rpm-centos7-amd64-basic~hdd - # 0 1.00000 osd.0 down 1.00000 1.00000 - # 1 1.00000 osd.1 up 1.00000 1.00000 - # 2 1.00000 osd.2 up 1.00000 1.00000 - # -1 3.00000 root default - # -2 3.00000 host gitbuilder-ceph-rpm-centos7-amd64-basic - # 0 1.00000 osd.0 down 1.00000 1.00000 - # 1 1.00000 osd.1 up 1.00000 1.00000 - # 2 1.00000 osd.2 up 1.00000 1.00000 - # - CEPH_ARGS+="--osd-class-update-on-start=false " - - OLD_ARGS=$CEPH_ARGS - CEPH_ARGS+="--osd-fast-fail-on-connection-refused=false " - echo "Ensuring old behavior is there..." - test_fast_kill $dir && (echo "OSDs died too early! Old behavior doesn't work." ; return 1) - - CEPH_ARGS=$OLD_ARGS"--osd-fast-fail-on-connection-refused=true " - OLD_ARGS=$CEPH_ARGS - - CEPH_ARGS+="--ms_type=simple" - echo "Testing simple msgr..." - test_fast_kill $dir || return 1 - - CEPH_ARGS=$OLD_ARGS"--ms_type=async" - echo "Testing async msgr..." - test_fast_kill $dir || return 1 - - return 0 - -} - -function test_fast_kill() { - # create cluster with 3 osds - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=3 || return 1 - run_mgr $dir x || return 1 - for oi in {0..2}; do - run_osd $dir $oi || return 1 - pids[$oi]=$(cat $dir/osd.$oi.pid) - done - - # make some objects so osds to ensure connectivity between osds - rados -p rbd bench 10 write -b 4096 --max-objects 128 --no-cleanup - sleep 1 - - killid=0 - previd=0 - - # kill random osd and see if after max MAX_PROPAGATION_TIME, the osd count decreased. - for i in {1..2}; do - while [ $killid -eq $previd ]; do - killid=${pids[$RANDOM%${#pids[@]}]} - done - previd=$killid - - kill -9 $killid - time_left=$MAX_PROPAGATION_TIME - down_osds=0 - - while [ $time_left -gt 0 ]; do - sleep 1 - time_left=$[$time_left - 1]; - - grep -m 1 -c -F "ms_handle_refused" $dir/osd.*.log > /dev/null - if [ $? -ne 0 ]; then - continue - fi - - down_osds=$(ceph osd tree | grep -c down) - if [ $down_osds -lt $i ]; then - # osds not marked down yet, try again in a second - continue - elif [ $down_osds -gt $i ]; then - echo Too many \($down_osds\) osds died! - return 1 - else - break - fi - done - - if [ $down_osds -lt $i ]; then - echo Killed the OSD, yet it is not marked down - ceph osd tree - return 1 - fi - done - pkill -SIGTERM rados - teardown $dir || return 1 -} - -main osd-fast-mark-down "$@" - -# Local Variables: -# compile-command: "cd ../.. ; make -j4 && test/osd/osd-fast-mark-down.sh" -# End: diff --git a/src/test/osd/osd-markdown.sh b/src/test/osd/osd-markdown.sh deleted file mode 100755 index aa67d9e7706..00000000000 --- a/src/test/osd/osd-markdown.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2015 Intel -# Copyright (C) 2014, 2015 Red Hat -# -# Author: Xiaoxi Chen -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# - -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7108" # git grep '\<7108\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - setup $dir || return 1 - $func $dir || return 1 - teardown $dir || return 1 - done -} - -function markdown_N_impl() { - markdown_times=$1 - total_time=$2 - sleeptime=$3 - for i in `seq 1 $markdown_times` - do - # check the OSD is UP - ceph osd tree - ceph osd tree | grep osd.0 |grep up || return 1 - # mark the OSD down. - ceph osd down 0 - sleep $sleeptime - done -} - - -function TEST_markdown_exceed_maxdown_count() { - local dir=$1 - - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - run_osd $dir 1 || return 1 - run_osd $dir 2 || return 1 - # 3+1 times within 300s, osd should stay dead on the 4th time - local count=3 - local sleeptime=10 - local period=300 - ceph tell osd.0 injectargs '--osd_max_markdown_count '$count'' || return 1 - ceph tell osd.0 injectargs '--osd_max_markdown_period '$period'' || return 1 - - markdown_N_impl $(($count+1)) $period $sleeptime - # down N+1 times ,the osd.0 shoud die - ceph osd tree | grep down | grep osd.0 || return 1 -} - -function TEST_markdown_boot() { - local dir=$1 - - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - run_osd $dir 1 || return 1 - run_osd $dir 2 || return 1 - - # 3 times within 120s, should stay up - local count=3 - local sleeptime=10 - local period=120 - ceph tell osd.0 injectargs '--osd_max_markdown_count '$count'' || return 1 - ceph tell osd.0 injectargs '--osd_max_markdown_period '$period'' || return 1 - - markdown_N_impl $count $period $sleeptime - #down N times, osd.0 should be up - sleep 15 # give osd plenty of time to notice and come back up - ceph osd tree | grep up | grep osd.0 || return 1 -} - -function TEST_markdown_boot_exceed_time() { - local dir=$1 - - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - run_osd $dir 1 || return 1 - run_osd $dir 2 || return 1 - - - # 3+1 times, but over 40s, > 20s, so should stay up - local count=3 - local period=20 - local sleeptime=10 - ceph tell osd.0 injectargs '--osd_max_markdown_count '$count'' || return 1 - ceph tell osd.0 injectargs '--osd_max_markdown_period '$period'' || return 1 - - markdown_N_impl $(($count+1)) $period $sleeptime - sleep 15 # give osd plenty of time to notice and come back up - ceph osd tree | grep up | grep osd.0 || return 1 -} - -main osd-markdown "$@" - -# Local Variables: -# compile-command: "cd ../.. ; make -j4 && test/osd/osd-bench.sh" -# End: diff --git a/src/test/osd/osd-reactivate.sh b/src/test/osd/osd-reactivate.sh deleted file mode 100755 index 5088baf4eb4..00000000000 --- a/src/test/osd/osd-reactivate.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -# -# Author: Vicente Cheng -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# - -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7122" # git grep '\<7122\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - setup $dir || return 1 - $func $dir || return 1 - teardown $dir || return 1 - done -} - -function TEST_reactivate() { - local dir=$1 - - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - - kill_daemons $dir TERM osd || return 1 - - ready_path=$dir"/0/ready" - activate_path=$dir"/0/active" - # trigger mkfs again - rm -rf $ready_path $activate_path - activate_osd $dir 0 || return 1 - -} - -main osd-reactivate "$@" - -# Local Variables: -# compile-command: "cd ../.. ; make -j4 && test/osd/osd-reactivate.sh" -# End: diff --git a/src/test/osd/osd-reuse-id.sh b/src/test/osd/osd-reuse-id.sh deleted file mode 100755 index fb0afb13886..00000000000 --- a/src/test/osd/osd-reuse-id.sh +++ /dev/null @@ -1,53 +0,0 @@ -#! /bin/bash -# -# Copyright (C) 2015 Red Hat -# -# Author: Loic Dachary -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7123" # git grep '\<7123\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - $func $dir || return 1 - done -} - -function TEST_reuse_id() { - local dir=$1 - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - run_osd $dir 1 || return 1 - wait_for_clean || return 1 - destroy_osd $dir 1 || return 1 - run_osd $dir 1 || return 1 -} - -main osd-reuse-id "$@" - -# Local Variables: -# compile-command: "cd ../.. ; make -j4 && test/osd/osd-reuse-id.sh" -# End: - diff --git a/src/test/osd/osd-scrub-repair.sh b/src/test/osd/osd-scrub-repair.sh deleted file mode 100755 index c6648750bb6..00000000000 --- a/src/test/osd/osd-scrub-repair.sh +++ /dev/null @@ -1,2619 +0,0 @@ -#!/bin/bash -x -# -# Copyright (C) 2014 Red Hat -# -# Author: Loic Dachary -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -if [ `uname` = FreeBSD ]; then - # erasure coding overwrites are only tested on Bluestore - # erasure coding on filestore is unsafe - # http://docs.ceph.com/docs/master/rados/operations/erasure-code/#erasure-coding-with-overwrites - use_ec_overwrite=false -else - use_ec_overwrite=true -fi - -# Test development and debugging -# Set to "yes" in order to ignore diff errors and save results to update test -getjson="no" - -# Ignore the epoch and filter out the attr '_' value because it has date information and won't match -jqfilter='.inconsistents | (.[].shards[].attrs[]? | select(.name == "_") | .value) |= "----Stripped-by-test----"' -sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)' - -# Remove items are not consistent across runs, the pg interval and client -sedfilter='s/\([ ]*\"\(selected_\)*object_info\":.*head[(]\)[^[:space:]]* [^[:space:]]* \(.*\)/\1\3/' - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7107" # git grep '\<7107\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - $func $dir || return 1 - done -} - -function add_something() { - local dir=$1 - local poolname=$2 - local obj=${3:-SOMETHING} - local scrub=${4:-noscrub} - - if [ "$scrub" = "noscrub" ]; - then - ceph osd set noscrub || return 1 - ceph osd set nodeep-scrub || return 1 - else - ceph osd unset noscrub || return 1 - ceph osd unset nodeep-scrub || return 1 - fi - - local payload=ABCDEF - echo $payload > $dir/ORIGINAL - rados --pool $poolname put $obj $dir/ORIGINAL || return 1 -} - -# -# Corrupt one copy of a replicated pool -# -function TEST_corrupt_and_repair_replicated() { - local dir=$1 - local poolname=rbd - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=2 || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - run_osd $dir 1 || return 1 - wait_for_clean || return 1 - - add_something $dir $poolname || return 1 - corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1 - # Reproduces http://tracker.ceph.com/issues/8914 - corrupt_and_repair_one $dir $poolname $(get_primary $poolname SOMETHING) || return 1 - - teardown $dir || return 1 -} - -function corrupt_and_repair_two() { - local dir=$1 - local poolname=$2 - local first=$3 - local second=$4 - - # - # 1) remove the corresponding file from the OSDs - # - pids="" - run_in_background pids objectstore_tool $dir $first SOMETHING remove - run_in_background pids objectstore_tool $dir $second SOMETHING remove - wait_background pids - return_code=$? - if [ $return_code -ne 0 ]; then return $return_code; fi - - # - # 2) repair the PG - # - local pg=$(get_pg $poolname SOMETHING) - repair $pg - # - # 3) The files must be back - # - pids="" - run_in_background pids objectstore_tool $dir $first SOMETHING list-attrs - run_in_background pids objectstore_tool $dir $second SOMETHING list-attrs - wait_background pids - return_code=$? - if [ $return_code -ne 0 ]; then return $return_code; fi - - rados --pool $poolname get SOMETHING $dir/COPY || return 1 - diff $dir/ORIGINAL $dir/COPY || return 1 -} - -# -# 1) add an object -# 2) remove the corresponding file from a designated OSD -# 3) repair the PG -# 4) check that the file has been restored in the designated OSD -# -function corrupt_and_repair_one() { - local dir=$1 - local poolname=$2 - local osd=$3 - - # - # 1) remove the corresponding file from the OSD - # - objectstore_tool $dir $osd SOMETHING remove || return 1 - # - # 2) repair the PG - # - local pg=$(get_pg $poolname SOMETHING) - repair $pg - # - # 3) The file must be back - # - objectstore_tool $dir $osd SOMETHING list-attrs || return 1 - rados --pool $poolname get SOMETHING $dir/COPY || return 1 - diff $dir/ORIGINAL $dir/COPY || return 1 -} - -function corrupt_and_repair_erasure_coded() { - local dir=$1 - local poolname=$2 - - add_something $dir $poolname || return 1 - - local primary=$(get_primary $poolname SOMETHING) - local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//")) - local not_primary_first=${osds[0]} - local not_primary_second=${osds[1]} - - # Reproduces http://tracker.ceph.com/issues/10017 - corrupt_and_repair_one $dir $poolname $primary || return 1 - # Reproduces http://tracker.ceph.com/issues/10409 - corrupt_and_repair_one $dir $poolname $not_primary_first || return 1 - corrupt_and_repair_two $dir $poolname $not_primary_first $not_primary_second || return 1 - corrupt_and_repair_two $dir $poolname $primary $not_primary_first || return 1 - -} - -function create_ec_pool() { - local pool_name=$1 - local allow_overwrites=$2 - - ceph osd erasure-code-profile set myprofile crush-failure-domain=osd $3 $4 $5 $6 $7 || return 1 - - ceph osd pool create "$poolname" 1 1 erasure myprofile || return 1 - - if [ "$allow_overwrites" = "true" ]; then - ceph osd pool set "$poolname" allow_ec_overwrites true || return 1 - fi - - wait_for_clean || return 1 - return 0 -} - -function auto_repair_erasure_coded() { - local dir=$1 - local allow_overwrites=$2 - local poolname=ecpool - - # Launch a cluster with 5 seconds scrub interval - setup $dir || return 1 - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - local ceph_osd_args="--osd-scrub-auto-repair=true \ - --osd-deep-scrub-interval=5 \ - --osd-scrub-max-interval=5 \ - --osd-scrub-min-interval=5 \ - --osd-scrub-interval-randomize-ratio=0" - for id in $(seq 0 2) ; do - if [ "$allow_overwrites" = "true" ]; then - run_osd_bluestore $dir $id $ceph_osd_args || return 1 - else - run_osd $dir $id $ceph_osd_args || return 1 - fi - done - wait_for_clean || return 1 - - # Create an EC pool - create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1 - - # Put an object - local payload=ABCDEF - echo $payload > $dir/ORIGINAL - rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1 - - # Remove the object from one shard physically - # Restarted osd get $ceph_osd_args passed - objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1 - # Wait for auto repair - local pgid=$(get_pg $poolname SOMETHING) - wait_for_scrub $pgid "$(get_last_scrub_stamp $pgid)" - wait_for_clean || return 1 - # Verify - the file should be back - # Restarted osd get $ceph_osd_args passed - objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1 - rados --pool $poolname get SOMETHING $dir/COPY || return 1 - diff $dir/ORIGINAL $dir/COPY || return 1 - - # Tear down - teardown $dir || return 1 -} - -function TEST_auto_repair_erasure_coded_appends() { - auto_repair_erasure_coded $1 false -} - -function TEST_auto_repair_erasure_coded_overwrites() { - if [ "$use_ec_overwrite" = "true" ]; then - auto_repair_erasure_coded $1 true - fi -} - -function corrupt_and_repair_jerasure() { - local dir=$1 - local allow_overwrites=$2 - local poolname=ecpool - - setup $dir || return 1 - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - for id in $(seq 0 3) ; do - if [ "$allow_overwrites" = "true" ]; then - run_osd_bluestore $dir $id || return 1 - else - run_osd $dir $id || return 1 - fi - done - wait_for_clean || return 1 - - create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1 - corrupt_and_repair_erasure_coded $dir $poolname || return 1 - - teardown $dir || return 1 -} - -function TEST_corrupt_and_repair_jerasure_appends() { - corrupt_and_repair_jerasure $1 -} - -function TEST_corrupt_and_repair_jerasure_overwrites() { - if [ "$use_ec_overwrite" = "true" ]; then - corrupt_and_repair_jerasure $1 true - fi -} - -function corrupt_and_repair_lrc() { - local dir=$1 - local allow_overwrites=$2 - local poolname=ecpool - - setup $dir || return 1 - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - for id in $(seq 0 9) ; do - if [ "$allow_overwrites" = "true" ]; then - run_osd_bluestore $dir $id || return 1 - else - run_osd $dir $id || return 1 - fi - done - wait_for_clean || return 1 - - create_ec_pool $poolname $allow_overwrites k=4 m=2 l=3 plugin=lrc || return 1 - corrupt_and_repair_erasure_coded $dir $poolname || return 1 - - teardown $dir || return 1 -} - -function TEST_corrupt_and_repair_lrc_appends() { - corrupt_and_repair_jerasure $1 -} - -function TEST_corrupt_and_repair_lrc_overwrites() { - if [ "$use_ec_overwrite" = "true" ]; then - corrupt_and_repair_jerasure $1 true - fi -} - -function unfound_erasure_coded() { - local dir=$1 - local allow_overwrites=$2 - local poolname=ecpool - local payload=ABCDEF - - setup $dir || return 1 - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - for id in $(seq 0 3) ; do - if [ "$allow_overwrites" = "true" ]; then - run_osd_bluestore $dir $id || return 1 - else - run_osd $dir $id || return 1 - fi - done - wait_for_clean || return 1 - - create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1 - - add_something $dir $poolname || return 1 - - local primary=$(get_primary $poolname SOMETHING) - local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//")) - local not_primary_first=${osds[0]} - local not_primary_second=${osds[1]} - local not_primary_third=${osds[2]} - - # - # 1) remove the corresponding file from the OSDs - # - pids="" - run_in_background pids objectstore_tool $dir $not_primary_first SOMETHING remove - run_in_background pids objectstore_tool $dir $not_primary_second SOMETHING remove - run_in_background pids objectstore_tool $dir $not_primary_third SOMETHING remove - wait_background pids - return_code=$? - if [ $return_code -ne 0 ]; then return $return_code; fi - - # - # 2) repair the PG - # - local pg=$(get_pg $poolname SOMETHING) - repair $pg - # - # 3) check pg state - # - # it may take a bit to appear due to mon/mgr asynchrony - for f in `seq 1 60`; do - ceph -s | grep "1/1 unfound" && break - sleep 1 - done - ceph -s|grep "4 osds: 4 up, 4 in" || return 1 - ceph -s|grep "1/1 unfound" || return 1 - - teardown $dir || return 1 -} - -function TEST_unfound_erasure_coded_appends() { - unfound_erasure_coded $1 -} - -function TEST_unfound_erasure_coded_overwrites() { - if [ "$use_ec_overwrite" = "true" ]; then - unfound_erasure_coded $1 true - fi -} - -# -# list_missing for EC pool -# -function list_missing_erasure_coded() { - local dir=$1 - local allow_overwrites=$2 - local poolname=ecpool - - setup $dir || return 1 - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - for id in $(seq 0 2) ; do - if [ "$allow_overwrites" = "true" ]; then - run_osd_bluestore $dir $id || return 1 - else - run_osd $dir $id || return 1 - fi - done - wait_for_clean || return 1 - - create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1 - - # Put an object and remove the two shards (including primary) - add_something $dir $poolname MOBJ0 || return 1 - local -a osds0=($(get_osds $poolname MOBJ0)) - - # Put another object and remove two shards (excluding primary) - add_something $dir $poolname MOBJ1 || return 1 - local -a osds1=($(get_osds $poolname MOBJ1)) - - # Stop all osd daemons - for id in $(seq 0 2) ; do - kill_daemons $dir TERM osd.$id >&2 < /dev/null || return 1 - done - - id=${osds0[0]} - ceph-objectstore-tool --data-path $dir/$id \ - MOBJ0 remove || return 1 - id=${osds0[1]} - ceph-objectstore-tool --data-path $dir/$id \ - MOBJ0 remove || return 1 - - id=${osds1[1]} - ceph-objectstore-tool --data-path $dir/$id \ - MOBJ1 remove || return 1 - id=${osds1[2]} - ceph-objectstore-tool --data-path $dir/$id \ - MOBJ1 remove || return 1 - - for id in $(seq 0 2) ; do - activate_osd $dir $id >&2 || return 1 - done - wait_for_clean || return 1 - - # Get get - both objects should in the same PG - local pg=$(get_pg $poolname MOBJ0) - - # Repair the PG, which triggers the recovering, - # and should mark the object as unfound - repair $pg - - for i in $(seq 0 120) ; do - [ $i -lt 60 ] || return 1 - matches=$(ceph pg $pg list_missing | egrep "MOBJ0|MOBJ1" | wc -l) - [ $matches -eq 2 ] && break - done - - teardown $dir || return 1 -} - -function TEST_list_missing_erasure_coded_appends() { - list_missing_erasure_coded $1 false -} - -function TEST_list_missing_erasure_coded_overwrites() { - if [ "$use_ec_overwrite" = "true" ]; then - list_missing_erasure_coded $1 true - fi -} - -# -# Corrupt one copy of a replicated pool -# -function TEST_corrupt_scrub_replicated() { - local dir=$1 - local poolname=csr_pool - local total_objs=15 - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=2 || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - run_osd $dir 1 || return 1 - wait_for_clean || return 1 - - ceph osd pool create $poolname 1 1 || return 1 - wait_for_clean || return 1 - - for i in $(seq 1 $total_objs) ; do - objname=ROBJ${i} - add_something $dir $poolname $objname || return 1 - - rados --pool $poolname setomapheader $objname hdr-$objname || return 1 - rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1 - done - - local pg=$(get_pg $poolname ROBJ0) - - # Compute an old omap digest and save oi - CEPH_ARGS='' ceph daemon $(get_asok_path osd.0) \ - config set osd_deep_scrub_update_digest_min_age 0 - CEPH_ARGS='' ceph daemon $(get_asok_path osd.1) \ - config set osd_deep_scrub_update_digest_min_age 0 - pg_deep_scrub $pg - - for i in $(seq 1 $total_objs) ; do - objname=ROBJ${i} - - # Alternate corruption between osd.0 and osd.1 - local osd=$(expr $i % 2) - - case $i in - 1) - # Size (deep scrub data_digest too) - local payload=UVWXYZZZ - echo $payload > $dir/CORRUPT - objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 - ;; - - 2) - # digest (deep scrub only) - local payload=UVWXYZ - echo $payload > $dir/CORRUPT - objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 - ;; - - 3) - # missing - objectstore_tool $dir $osd $objname remove || return 1 - ;; - - 4) - # Modify omap value (deep scrub only) - objectstore_tool $dir $osd $objname set-omap key-$objname $dir/CORRUPT || return 1 - ;; - - 5) - # Delete omap key (deep scrub only) - objectstore_tool $dir $osd $objname rm-omap key-$objname || return 1 - ;; - - 6) - # Add extra omap key (deep scrub only) - echo extra > $dir/extra-val - objectstore_tool $dir $osd $objname set-omap key2-$objname $dir/extra-val || return 1 - rm $dir/extra-val - ;; - - 7) - # Modify omap header (deep scrub only) - echo -n newheader > $dir/hdr - objectstore_tool $dir $osd $objname set-omaphdr $dir/hdr || return 1 - rm $dir/hdr - ;; - - 8) - rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1 - rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1 - - # Break xattrs - echo -n bad-val > $dir/bad-val - objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1 - objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1 - echo -n val3-$objname > $dir/newval - objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1 - rm $dir/bad-val $dir/newval - ;; - - 9) - objectstore_tool $dir $osd $objname get-attr _ > $dir/robj9-oi - echo -n D > $dir/change - rados --pool $poolname put $objname $dir/change - objectstore_tool $dir $osd $objname set-attr _ $dir/robj9-oi - rm $dir/oi $dir/change - ;; - - # ROBJ10 must be handled after digests are re-computed by a deep scrub below - # ROBJ11 must be handled with config change before deep scrub - # ROBJ12 must be handled with config change before scrubs - # ROBJ13 must be handled before scrubs - - 14) - echo -n bad-val > $dir/bad-val - objectstore_tool $dir 0 $objname set-attr _ $dir/bad-val || return 1 - objectstore_tool $dir 1 $objname rm-attr _ || return 1 - rm $dir/bad-val - ;; - - 15) - objectstore_tool $dir $osd $objname rm-attr _ || return 1 - - esac - done - - local pg=$(get_pg $poolname ROBJ0) - - set_config osd 0 filestore_debug_inject_read_err true || return 1 - set_config osd 1 filestore_debug_inject_read_err true || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.1) \ - injectdataerr $poolname ROBJ11 || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) \ - injectmdataerr $poolname ROBJ12 || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) \ - injectmdataerr $poolname ROBJ13 || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.1) \ - injectdataerr $poolname ROBJ13 || return 1 - - pg_scrub $pg - - rados list-inconsistent-pg $poolname > $dir/json || return 1 - # Check pg count - test $(jq '. | length' $dir/json) = "1" || return 1 - # Check pgid - test $(jq -r '.[0]' $dir/json) = $pg || return 1 - - rados list-inconsistent-obj $pg > $dir/json || return 1 - # Get epoch for repair-get requests - epoch=$(jq .epoch $dir/json) - - jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson -{ - "inconsistents": [ - { - "shards": [ - { - "size": 7, - "errors": [], - "osd": 0 - }, - { - "size": 9, - "errors": [ - "size_mismatch_oi" - ], - "osd": 1 - } - ], - "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])", - "union_shard_errors": [ - "size_mismatch_oi" - ], - "errors": [ - "size_mismatch" - ], - "object": { - "version": 3, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ1" - } - }, - { - "shards": [ - { - "errors": [ - "stat_error" - ], - "osd": 0 - }, - { - "size": 7, - "errors": [], - "osd": 1 - } - ], - "selected_object_info": "3:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])", - "union_shard_errors": [ - "stat_error" - ], - "errors": [], - "object": { - "version": 36, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ12" - } - }, - { - "shards": [ - { - "errors": [ - "stat_error" - ], - "osd": 0 - }, - { - "size": 7, - "errors": [], - "osd": 1 - } - ], - "selected_object_info": "3:d60617f9:::ROBJ13:head(47'55 osd.0.0:54 dirty|omap|data_digest|omap_digest s 7 uv 39 dd 2ddbf8f5 od 6441854d alloc_hint [0 0 0])", - "union_shard_errors": [ - "stat_error" - ], - "errors": [], - "object": { - "version": 39, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ13" - } - }, - { - "shards": [ - { - "size": 7, - "errors": [ - "oi_attr_corrupted" - ], - "osd": 0 - }, - { - "size": 7, - "errors": [ - "oi_attr_missing" - ], - "osd": 1 - } - ], - "union_shard_errors": [ - "oi_attr_missing", - "oi_attr_corrupted" - ], - "errors": [], - "object": { - "version": 0, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ14" - } - }, - { - "shards": [ - { - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], - "size": 7, - "errors": [], - "osd": 0 - }, - { - "attrs": [ - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], - "size": 7, - "errors": [ - "oi_attr_missing" - ], - "osd": 1 - } - ], - "selected_object_info": "3:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])", - "union_shard_errors": [ - "oi_attr_missing" - ], - "errors": [ - "attr_name_mismatch" - ], - "object": { - "version": 45, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ15" - } - }, - { - "shards": [ - { - "size": 7, - "errors": [], - "osd": 0 - }, - { - "errors": [ - "missing" - ], - "osd": 1 - } - ], - "selected_object_info": "3:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])", - "union_shard_errors": [ - "missing" - ], - "errors": [], - "object": { - "version": 9, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ3" - } - }, - { - "shards": [ - { - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, - { - "Base64": false, - "value": "bad-val", - "name": "_key1-ROBJ8" - }, - { - "Base64": false, - "value": "val3-ROBJ8", - "name": "_key3-ROBJ8" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], - "size": 7, - "errors": [], - "osd": 0 - }, - { - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, - { - "Base64": false, - "value": "val1-ROBJ8", - "name": "_key1-ROBJ8" - }, - { - "Base64": false, - "value": "val2-ROBJ8", - "name": "_key2-ROBJ8" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], - "size": 7, - "errors": [], - "osd": 1 - } - ], - "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 62 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])", - "union_shard_errors": [], - "errors": [ - "attr_value_mismatch", - "attr_name_mismatch" - ], - "object": { - "version": 62, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ8" - } - }, - { - "shards": [ - { - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], - "object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 63 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])", - "size": 1, - "errors": [], - "osd": 0 - }, - { - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], - "object_info": "3:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])", - "size": 1, - "errors": [], - "osd": 1 - } - ], - "selected_object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 63 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])", - "union_shard_errors": [], - "errors": [ - "object_info_inconsistency", - "attr_value_mismatch" - ], - "object": { - "version": 63, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ9" - } - } - ], - "epoch": 0 -} -EOF - - jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson - diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 - if test $getjson = "yes" - then - jq '.' $dir/json > save1.json - fi - - if which jsonschema > /dev/null; - then - jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 - fi - - objname=ROBJ9 - # Change data and size again because digest was recomputed - echo -n ZZZ > $dir/change - rados --pool $poolname put $objname $dir/change - # Set one to an even older value - objectstore_tool $dir 0 $objname set-attr _ $dir/robj9-oi - rm $dir/oi $dir/change - - objname=ROBJ10 - objectstore_tool $dir 1 $objname get-attr _ > $dir/oi - rados --pool $poolname setomapval $objname key2-$objname val2-$objname - objectstore_tool $dir 0 $objname set-attr _ $dir/oi - objectstore_tool $dir 1 $objname set-attr _ $dir/oi - rm $dir/oi - - set_config osd 0 filestore_debug_inject_read_err true || return 1 - set_config osd 1 filestore_debug_inject_read_err true || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.1) \ - injectdataerr $poolname ROBJ11 || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) \ - injectmdataerr $poolname ROBJ12 || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) \ - injectmdataerr $poolname ROBJ13 || return 1 - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.1) \ - injectdataerr $poolname ROBJ13 || return 1 - pg_deep_scrub $pg - - rados list-inconsistent-pg $poolname > $dir/json || return 1 - # Check pg count - test $(jq '. | length' $dir/json) = "1" || return 1 - # Check pgid - test $(jq -r '.[0]' $dir/json) = $pg || return 1 - - rados list-inconsistent-obj $pg > $dir/json || return 1 - # Get epoch for repair-get requests - epoch=$(jq .epoch $dir/json) - - jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson -{ - "inconsistents": [ - { - "shards": [ - { - "data_digest": "0x2ddbf8f5", - "omap_digest": "0xf5fba2c6", - "size": 7, - "errors": [], - "osd": 0 - }, - { - "data_digest": "0x2d4a11c2", - "omap_digest": "0xf5fba2c6", - "size": 9, - "errors": [ - "data_digest_mismatch_oi", - "size_mismatch_oi" - ], - "osd": 1 - } - ], - "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])", - "union_shard_errors": [ - "data_digest_mismatch_oi", - "size_mismatch_oi" - ], - "errors": [ - "data_digest_mismatch", - "size_mismatch" - ], - "object": { - "version": 3, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ1" - } - }, - { - "shards": [ - { - "data_digest": "0x2ddbf8f5", - "omap_digest": "0xa8dd5adc", - "size": 7, - "errors": [ - "omap_digest_mismatch_oi" - ], - "osd": 0 - }, - { - "data_digest": "0x2ddbf8f5", - "omap_digest": "0xa8dd5adc", - "size": 7, - "errors": [ - "omap_digest_mismatch_oi" - ], - "osd": 1 - } - ], - "selected_object_info": "3:b1f19cbd:::ROBJ10:head(47'51 osd.0.0:50 dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [0 0 0])", - "union_shard_errors": [ - "omap_digest_mismatch_oi" - ], - "errors": [], - "object": { - "version": 30, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ10" - } - }, - { - "shards": [ - { - "data_digest": "0x2ddbf8f5", - "omap_digest": "0xa03cef03", - "size": 7, - "errors": [], - "osd": 0 - }, - { - "size": 7, - "errors": [ - "read_error" - ], - "osd": 1 - } - ], - "selected_object_info": "3:87abbf36:::ROBJ11:head(47'48 osd.0.0:47 dirty|omap|data_digest|omap_digest s 7 uv 33 dd 2ddbf8f5 od a03cef03 alloc_hint [0 0 0])", - "union_shard_errors": [ - "read_error" - ], - "errors": [], - "object": { - "version": 33, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ11" - } - }, - { - "shards": [ - { - "errors": [ - "stat_error" - ], - "osd": 0 - }, - { - "data_digest": "0x2ddbf8f5", - "omap_digest": "0x067f306a", - "size": 7, - "errors": [], - "osd": 1 - } - ], - "selected_object_info": "3:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])", - "union_shard_errors": [ - "stat_error" - ], - "errors": [], - "object": { - "version": 36, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ12" - } - }, - { - "shards": [ - { - "errors": [ - "stat_error" - ], - "osd": 0 - }, - { - "size": 7, - "errors": [ - "read_error" - ], - "osd": 1 - } - ], - "union_shard_errors": [ - "stat_error", - "read_error" - ], - "errors": [], - "object": { - "version": 0, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ13" - } - }, - { - "shards": [ - { - "data_digest": "0x2ddbf8f5", - "omap_digest": "0x4f14f849", - "size": 7, - "errors": [ - "oi_attr_corrupted" - ], - "osd": 0 - }, - { - "data_digest": "0x2ddbf8f5", - "omap_digest": "0x4f14f849", - "size": 7, - "errors": [ - "oi_attr_missing" - ], - "osd": 1 - } - ], - "union_shard_errors": [ - "oi_attr_missing", - "oi_attr_corrupted" - ], - "errors": [], - "object": { - "version": 0, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ14" - } - }, - { - "shards": [ - { - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], - "data_digest": "0x2ddbf8f5", - "omap_digest": "0x2d2a4d6e", - "size": 7, - "errors": [], - "osd": 0 - }, - { - "attrs": [ - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], - "data_digest": "0x2ddbf8f5", - "omap_digest": "0x2d2a4d6e", - "size": 7, - "errors": [ - "oi_attr_missing" - ], - "osd": 1 - } - ], - "selected_object_info": "3:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])", - "union_shard_errors": [ - "oi_attr_missing" - ], - "errors": [ - "attr_name_mismatch" - ], - "object": { - "version": 45, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ15" - } - }, - { - "shards": [ - { - "data_digest": "0x578a4830", - "omap_digest": "0xf8e11918", - "size": 7, - "errors": [ - "data_digest_mismatch_oi" - ], - "osd": 0 - }, - { - "data_digest": "0x2ddbf8f5", - "omap_digest": "0xf8e11918", - "size": 7, - "errors": [], - "osd": 1 - } - ], - "selected_object_info": "3:e97ce31e:::ROBJ2:head(47'56 osd.0.0:55 dirty|omap|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od f8e11918 alloc_hint [0 0 0])", - "union_shard_errors": [ - "data_digest_mismatch_oi" - ], - "errors": [ - "data_digest_mismatch" - ], - "object": { - "version": 6, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ2" - } - }, - { - "shards": [ - { - "data_digest": "0x2ddbf8f5", - "omap_digest": "0x00b35dfd", - "size": 7, - "errors": [], - "osd": 0 - }, - { - "errors": [ - "missing" - ], - "osd": 1 - } - ], - "selected_object_info": "3:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])", - "union_shard_errors": [ - "missing" - ], - "errors": [], - "object": { - "version": 9, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ3" - } - }, - { - "shards": [ - { - "data_digest": "0x2ddbf8f5", - "omap_digest": "0xd7178dfe", - "size": 7, - "errors": [ - "omap_digest_mismatch_oi" - ], - "osd": 0 - }, - { - "data_digest": "0x2ddbf8f5", - "omap_digest": "0xe2d46ea4", - "size": 7, - "errors": [], - "osd": 1 - } - ], - "selected_object_info": "3:f4981d31:::ROBJ4:head(47'58 osd.0.0:57 dirty|omap|data_digest|omap_digest s 7 uv 12 dd 2ddbf8f5 od e2d46ea4 alloc_hint [0 0 0])", - "union_shard_errors": [ - "omap_digest_mismatch_oi" - ], - "errors": [ - "omap_digest_mismatch" - ], - "object": { - "version": 12, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ4" - } - }, - { - "shards": [ - { - "data_digest": "0x2ddbf8f5", - "omap_digest": "0x1a862a41", - "size": 7, - "errors": [], - "osd": 0 - }, - { - "data_digest": "0x2ddbf8f5", - "omap_digest": "0x06cac8f6", - "size": 7, - "errors": [ - "omap_digest_mismatch_oi" - ], - "osd": 1 - } - ], - "selected_object_info": "3:f4bfd4d1:::ROBJ5:head(47'59 osd.0.0:58 dirty|omap|data_digest|omap_digest s 7 uv 15 dd 2ddbf8f5 od 1a862a41 alloc_hint [0 0 0])", - "union_shard_errors": [ - "omap_digest_mismatch_oi" - ], - "errors": [ - "omap_digest_mismatch" - ], - "object": { - "version": 15, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ5" - } - }, - { - "shards": [ - { - "data_digest": "0x2ddbf8f5", - "omap_digest": "0x689ee887", - "size": 7, - "errors": [ - "omap_digest_mismatch_oi" - ], - "osd": 0 - }, - { - "data_digest": "0x2ddbf8f5", - "omap_digest": "0x179c919f", - "size": 7, - "errors": [], - "osd": 1 - } - ], - "selected_object_info": "3:a53c12e8:::ROBJ6:head(47'50 osd.0.0:49 dirty|omap|data_digest|omap_digest s 7 uv 18 dd 2ddbf8f5 od 179c919f alloc_hint [0 0 0])", - "union_shard_errors": [ - "omap_digest_mismatch_oi" - ], - "errors": [ - "omap_digest_mismatch" - ], - "object": { - "version": 18, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ6" - } - }, - { - "shards": [ - { - "data_digest": "0x2ddbf8f5", - "omap_digest": "0xefced57a", - "size": 7, - "errors": [], - "osd": 0 - }, - { - "data_digest": "0x2ddbf8f5", - "omap_digest": "0x6a73cc07", - "size": 7, - "errors": [ - "omap_digest_mismatch_oi" - ], - "osd": 1 - } - ], - "selected_object_info": "3:8b55fa4b:::ROBJ7:head(47'49 osd.0.0:48 dirty|omap|data_digest|omap_digest s 7 uv 21 dd 2ddbf8f5 od efced57a alloc_hint [0 0 0])", - "union_shard_errors": [ - "omap_digest_mismatch_oi" - ], - "errors": [ - "omap_digest_mismatch" - ], - "object": { - "version": 21, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ7" - } - }, - { - "shards": [ - { - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, - { - "Base64": false, - "value": "bad-val", - "name": "_key1-ROBJ8" - }, - { - "Base64": false, - "value": "val3-ROBJ8", - "name": "_key3-ROBJ8" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], - "data_digest": "0x2ddbf8f5", - "omap_digest": "0xd6be81dc", - "size": 7, - "errors": [], - "osd": 0 - }, - { - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, - { - "Base64": false, - "value": "val1-ROBJ8", - "name": "_key1-ROBJ8" - }, - { - "Base64": false, - "value": "val2-ROBJ8", - "name": "_key2-ROBJ8" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], - "data_digest": "0x2ddbf8f5", - "omap_digest": "0xd6be81dc", - "size": 7, - "errors": [], - "osd": 1 - } - ], - "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 62 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])", - "union_shard_errors": [], - "errors": [ - "attr_value_mismatch", - "attr_name_mismatch" - ], - "object": { - "version": 62, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ8" - } - }, - { - "shards": [ - { - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], - "object_info": "3:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])", - "data_digest": "0x1f26fb26", - "omap_digest": "0x2eecc539", - "size": 3, - "errors": [], - "osd": 0 - }, - { - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], - "object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 64 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])", - "data_digest": "0x1f26fb26", - "omap_digest": "0x2eecc539", - "size": 3, - "errors": [], - "osd": 1 - } - ], - "selected_object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 64 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])", - "union_shard_errors": [], - "errors": [ - "object_info_inconsistency", - "attr_value_mismatch" - ], - "object": { - "version": 64, - "snap": "head", - "locator": "", - "nspace": "", - "name": "ROBJ9" - } - } - ], - "epoch": 0 -} -EOF - - jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson - diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 - if test $getjson = "yes" - then - jq '.' $dir/json > save2.json - fi - - if which jsonschema > /dev/null; - then - jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 - fi - - rados rmpool $poolname $poolname --yes-i-really-really-mean-it - teardown $dir || return 1 -} - - -# -# Test scrub errors for an erasure coded pool -# -function corrupt_scrub_erasure() { - local dir=$1 - local allow_overwrites=$2 - local poolname=ecpool - local total_objs=5 - - setup $dir || return 1 - run_mon $dir a || return 1 - run_mgr $dir x || return 1 - for id in $(seq 0 2) ; do - if [ "$allow_overwrites" = "true" ]; then - run_osd_bluestore $dir $id || return 1 - else - run_osd $dir $id || return 1 - fi - done - wait_for_clean || return 1 - - create_ec_pool $poolname $allow_overwrites k=2 m=1 stripe_unit=2K --force || return 1 - - for i in $(seq 1 $total_objs) ; do - objname=EOBJ${i} - add_something $dir $poolname $objname || return 1 - - local osd=$(expr $i % 2) - - case $i in - 1) - # Size (deep scrub data_digest too) - local payload=UVWXYZZZ - echo $payload > $dir/CORRUPT - objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 - ;; - - 2) - # Corrupt EC shard - dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=1 - objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 - ;; - - 3) - # missing - objectstore_tool $dir $osd $objname remove || return 1 - ;; - - 4) - rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1 - rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1 - - # Break xattrs - echo -n bad-val > $dir/bad-val - objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1 - objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1 - echo -n val3-$objname > $dir/newval - objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1 - rm $dir/bad-val $dir/newval - ;; - - 5) - # Corrupt EC shard - dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=2 - objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 - ;; - - esac - done - - local pg=$(get_pg $poolname EOBJ0) - - pg_scrub $pg - - rados list-inconsistent-pg $poolname > $dir/json || return 1 - # Check pg count - test $(jq '. | length' $dir/json) = "1" || return 1 - # Check pgid - test $(jq -r '.[0]' $dir/json) = $pg || return 1 - - rados list-inconsistent-obj $pg > $dir/json || return 1 - # Get epoch for repair-get requests - epoch=$(jq .epoch $dir/json) - - jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson -{ - "inconsistents": [ - { - "shards": [ - { - "size": 2048, - "errors": [], - "shard": 2, - "osd": 0 - }, - { - "size": 9, - "shard": 0, - "errors": [ - "size_mismatch_oi" - ], - "osd": 1 - }, - { - "size": 2048, - "shard": 1, - "errors": [], - "osd": 2 - } - ], - "selected_object_info": "3:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", - "union_shard_errors": [ - "size_mismatch_oi" - ], - "errors": [ - "size_mismatch" - ], - "object": { - "version": 1, - "snap": "head", - "locator": "", - "nspace": "", - "name": "EOBJ1" - } - }, - { - "shards": [ - { - "size": 2048, - "errors": [], - "shard": 2, - "osd": 0 - }, - { - "shard": 0, - "errors": [ - "missing" - ], - "osd": 1 - }, - { - "size": 2048, - "shard": 1, - "errors": [], - "osd": 2 - } - ], - "selected_object_info": "3:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", - "union_shard_errors": [ - "missing" - ], - "errors": [], - "object": { - "version": 3, - "snap": "head", - "locator": "", - "nspace": "", - "name": "EOBJ3" - } - }, - { - "shards": [ - { - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, - { - "Base64": false, - "value": "bad-val", - "name": "_key1-EOBJ4" - }, - { - "Base64": false, - "value": "val3-EOBJ4", - "name": "_key3-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], - "size": 2048, - "errors": [], - "shard": 2, - "osd": 0 - }, - { - "osd": 1, - "shard": 0, - "errors": [], - "size": 2048, - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, - { - "Base64": false, - "value": "val1-EOBJ4", - "name": "_key1-EOBJ4" - }, - { - "Base64": false, - "value": "val2-EOBJ4", - "name": "_key2-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ] - }, - { - "osd": 2, - "shard": 1, - "errors": [], - "size": 2048, - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, - { - "Base64": false, - "value": "val1-EOBJ4", - "name": "_key1-EOBJ4" - }, - { - "Base64": false, - "value": "val2-EOBJ4", - "name": "_key2-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ] - } - ], - "selected_object_info": "3:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", - "union_shard_errors": [], - "errors": [ - "attr_value_mismatch", - "attr_name_mismatch" - ], - "object": { - "version": 6, - "snap": "head", - "locator": "", - "nspace": "", - "name": "EOBJ4" - } - }, - { - "shards": [ - { - "size": 2048, - "errors": [], - "shard": 2, - "osd": 0 - }, - { - "size": 4096, - "shard": 0, - "errors": [ - "size_mismatch_oi" - ], - "osd": 1 - }, - { - "size": 2048, - "shard": 1, - "errors": [], - "osd": 2 - } - ], - "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", - "union_shard_errors": [ - "size_mismatch_oi" - ], - "errors": [ - "size_mismatch" - ], - "object": { - "version": 7, - "snap": "head", - "locator": "", - "nspace": "", - "name": "EOBJ5" - } - } - ], - "epoch": 0 -} -EOF - - jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson - diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 - if test $getjson = "yes" - then - jq '.' $dir/json > save3.json - fi - - if which jsonschema > /dev/null; - then - jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 - fi - - pg_deep_scrub $pg - - rados list-inconsistent-pg $poolname > $dir/json || return 1 - # Check pg count - test $(jq '. | length' $dir/json) = "1" || return 1 - # Check pgid - test $(jq -r '.[0]' $dir/json) = $pg || return 1 - - rados list-inconsistent-obj $pg > $dir/json || return 1 - # Get epoch for repair-get requests - epoch=$(jq .epoch $dir/json) - - if [ "$allow_overwrites" = "true" ] - then - jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson -{ - "inconsistents": [ - { - "shards": [ - { - "data_digest": "0x00000000", - "omap_digest": "0xffffffff", - "size": 2048, - "errors": [], - "shard": 2, - "osd": 0 - }, - { - "size": 9, - "shard": 0, - "errors": [ - "read_error", - "size_mismatch_oi" - ], - "osd": 1 - }, - { - "data_digest": "0x00000000", - "omap_digest": "0xffffffff", - "size": 2048, - "shard": 1, - "errors": [], - "osd": 2 - } - ], - "selected_object_info": "3:9175b684:::EOBJ1:head(27'1 client.4155.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", - "union_shard_errors": [ - "read_error", - "size_mismatch_oi" - ], - "errors": [ - "size_mismatch" - ], - "object": { - "version": 1, - "snap": "head", - "locator": "", - "nspace": "", - "name": "EOBJ1" - } - }, - { - "shards": [ - { - "data_digest": "0x00000000", - "omap_digest": "0xffffffff", - "size": 2048, - "errors": [], - "shard": 2, - "osd": 0 - }, - { - "shard": 0, - "errors": [ - "missing" - ], - "osd": 1 - }, - { - "data_digest": "0x00000000", - "omap_digest": "0xffffffff", - "size": 2048, - "shard": 1, - "errors": [], - "osd": 2 - } - ], - "selected_object_info": "3:b197b25d:::EOBJ3:head(41'3 client.4199.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", - "union_shard_errors": [ - "missing" - ], - "errors": [], - "object": { - "version": 3, - "snap": "head", - "locator": "", - "nspace": "", - "name": "EOBJ3" - } - }, - { - "shards": [ - { - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, - { - "Base64": false, - "value": "bad-val", - "name": "_key1-EOBJ4" - }, - { - "Base64": false, - "value": "val3-EOBJ4", - "name": "_key3-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], - "data_digest": "0x00000000", - "omap_digest": "0xffffffff", - "size": 2048, - "errors": [], - "shard": 2, - "osd": 0 - }, - { - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, - { - "Base64": false, - "value": "val1-EOBJ4", - "name": "_key1-EOBJ4" - }, - { - "Base64": false, - "value": "val2-EOBJ4", - "name": "_key2-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], - "data_digest": "0x00000000", - "omap_digest": "0xffffffff", - "size": 2048, - "errors": [], - "shard": 0, - "osd": 1 - }, - { - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, - { - "Base64": false, - "value": "val1-EOBJ4", - "name": "_key1-EOBJ4" - }, - { - "Base64": false, - "value": "val2-EOBJ4", - "name": "_key2-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], - "data_digest": "0x00000000", - "omap_digest": "0xffffffff", - "size": 2048, - "errors": [], - "shard": 1, - "osd": 2 - } - ], - "selected_object_info": "3:5e723e06:::EOBJ4:head(48'6 client.4223.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", - "union_shard_errors": [], - "errors": [ - "attr_value_mismatch", - "attr_name_mismatch" - ], - "object": { - "version": 6, - "snap": "head", - "locator": "", - "nspace": "", - "name": "EOBJ4" - } - }, - { - "shards": [ - { - "data_digest": "0x00000000", - "omap_digest": "0xffffffff", - "size": 2048, - "errors": [], - "shard": 2, - "osd": 0 - }, - { - "data_digest": "0x00000000", - "omap_digest": "0xffffffff", - "size": 4096, - "errors": [ - "size_mismatch_oi" - ], - "shard": 0, - "osd": 1 - }, - { - "data_digest": "0x00000000", - "omap_digest": "0xffffffff", - "size": 2048, - "errors": [], - "shard": 1, - "osd": 2 - } - ], - "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4288.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", - "union_shard_errors": [ - "size_mismatch_oi" - ], - "errors": [ - "size_mismatch" - ], - "object": { - "version": 7, - "snap": "head", - "locator": "", - "nspace": "", - "name": "EOBJ5" - } - } - ], - "epoch": 0 -} -EOF - - else - - jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson -{ - "inconsistents": [ - { - "shards": [ - { - "data_digest": "0x04cfa72f", - "omap_digest": "0xffffffff", - "size": 2048, - "errors": [], - "shard": 2, - "osd": 0 - }, - { - "size": 9, - "shard": 0, - "errors": [ - "read_error", - "size_mismatch_oi" - ], - "osd": 1 - }, - { - "data_digest": "0x04cfa72f", - "omap_digest": "0xffffffff", - "size": 2048, - "shard": 1, - "errors": [], - "osd": 2 - } - ], - "selected_object_info": "3:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", - "union_shard_errors": [ - "read_error", - "size_mismatch_oi" - ], - "errors": [ - "size_mismatch" - ], - "object": { - "version": 1, - "snap": "head", - "locator": "", - "nspace": "", - "name": "EOBJ1" - } - }, - { - "shards": [ - { - "size": 2048, - "errors": [ - "ec_hash_error" - ], - "shard": 2, - "osd": 0 - }, - { - "data_digest": "0x04cfa72f", - "omap_digest": "0xffffffff", - "size": 2048, - "errors": [], - "shard": 0, - "osd": 1 - }, - { - "data_digest": "0x04cfa72f", - "omap_digest": "0xffffffff", - "size": 2048, - "errors": [], - "shard": 1, - "osd": 2 - } - ], - "selected_object_info": "3:9babd184:::EOBJ2:head(29'2 client.4217.0:1 dirty|data_digest|omap_digest s 7 uv 2 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", - "union_shard_errors": [ - "ec_hash_error" - ], - "errors": [], - "object": { - "version": 2, - "snap": "head", - "locator": "", - "nspace": "", - "name": "EOBJ2" - } - }, - { - "shards": [ - { - "data_digest": "0x04cfa72f", - "omap_digest": "0xffffffff", - "size": 2048, - "errors": [], - "shard": 2, - "osd": 0 - }, - { - "osd": 1, - "shard": 0, - "errors": [ - "missing" - ] - }, - { - "data_digest": "0x04cfa72f", - "omap_digest": "0xffffffff", - "size": 2048, - "shard": 1, - "errors": [], - "osd": 2 - } - ], - "selected_object_info": "3:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", - "union_shard_errors": [ - "missing" - ], - "errors": [], - "object": { - "version": 3, - "snap": "head", - "locator": "", - "nspace": "", - "name": "EOBJ3" - } - }, - { - "shards": [ - { - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, - { - "Base64": false, - "value": "bad-val", - "name": "_key1-EOBJ4" - }, - { - "Base64": false, - "value": "val3-EOBJ4", - "name": "_key3-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], - "data_digest": "0x04cfa72f", - "omap_digest": "0xffffffff", - "size": 2048, - "errors": [], - "shard": 2, - "osd": 0 - }, - { - "osd": 1, - "shard": 0, - "errors": [], - "size": 2048, - "omap_digest": "0xffffffff", - "data_digest": "0x04cfa72f", - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, - { - "Base64": false, - "value": "val1-EOBJ4", - "name": "_key1-EOBJ4" - }, - { - "Base64": false, - "value": "val2-EOBJ4", - "name": "_key2-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ] - }, - { - "osd": 2, - "shard": 1, - "errors": [], - "size": 2048, - "omap_digest": "0xffffffff", - "data_digest": "0x04cfa72f", - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, - { - "Base64": false, - "value": "val1-EOBJ4", - "name": "_key1-EOBJ4" - }, - { - "Base64": false, - "value": "val2-EOBJ4", - "name": "_key2-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ] - } - ], - "selected_object_info": "3:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", - "union_shard_errors": [], - "errors": [ - "attr_value_mismatch", - "attr_name_mismatch" - ], - "object": { - "version": 6, - "snap": "head", - "locator": "", - "nspace": "", - "name": "EOBJ4" - } - }, - { - "shards": [ - { - "data_digest": "0x04cfa72f", - "omap_digest": "0xffffffff", - "size": 2048, - "errors": [], - "shard": 2, - "osd": 0 - }, - { - "size": 4096, - "shard": 0, - "errors": [ - "size_mismatch_oi", - "ec_size_error" - ], - "osd": 1 - }, - { - "data_digest": "0x04cfa72f", - "omap_digest": "0xffffffff", - "size": 2048, - "shard": 1, - "errors": [], - "osd": 2 - } - ], - "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", - "union_shard_errors": [ - "size_mismatch_oi", - "ec_size_error" - ], - "errors": [ - "size_mismatch" - ], - "object": { - "version": 7, - "snap": "head", - "locator": "", - "nspace": "", - "name": "EOBJ5" - } - } - ], - "epoch": 0 -} -EOF - - fi - - jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson - diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 - if test $getjson = "yes" - then - if [ "$allow_overwrites" = "true" ] - then - num=4 - else - num=5 - fi - jq '.' $dir/json > save${num}.json - fi - - if which jsonschema > /dev/null; - then - jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 - fi - - rados rmpool $poolname $poolname --yes-i-really-really-mean-it - teardown $dir || return 1 -} - -function TEST_corrupt_scrub_erasure_appends() { - corrupt_scrub_erasure $1 false -} - -function TEST_corrupt_scrub_erasure_overwrites() { - if [ "$use_ec_overwrite" = "true" ]; then - corrupt_scrub_erasure $1 true - fi -} - -# -# Test to make sure that a periodic scrub won't cause deep-scrub info to be lost -# -function TEST_periodic_scrub_replicated() { - local dir=$1 - local poolname=psr_pool - local objname=POBJ - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=2 || return 1 - run_mgr $dir x || return 1 - local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0" - run_osd $dir 0 $ceph_osd_args || return 1 - run_osd $dir 1 $ceph_osd_args || return 1 - wait_for_clean || return 1 - - ceph osd pool create $poolname 1 1 || return 1 - wait_for_clean || return 1 - - local osd=0 - add_something $dir $poolname $objname scrub || return 1 - local primary=$(get_primary $poolname $objname) - local pg=$(get_pg $poolname $objname) - - # Add deep-scrub only error - local payload=UVWXYZ - echo $payload > $dir/CORRUPT - # Uses $ceph_osd_args for osd restart - objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 - - # No scrub information available, so expect failure - set -o pipefail - ! rados list-inconsistent-obj $pg | jq '.' || return 1 - set +o pipefail - - pg_deep_scrub $pg || return 1 - - # Make sure bad object found - rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1 - - local last_scrub=$(get_last_scrub_stamp $pg) - # Fake a schedule scrub - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \ - trigger_scrub $pg || return 1 - # Wait for schedule regular scrub - wait_for_scrub $pg "$last_scrub" - - # It needed to be upgraded - grep -q "Deep scrub errors, upgrading scrub to deep-scrub" $dir/osd.${primary}.log || return 1 - - # Bad object still known - rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1 - - # Can't upgrade with this set - ceph osd set nodeep-scrub - # Let map change propagate to OSDs - sleep 2 - - # Fake a schedule scrub - local last_scrub=$(get_last_scrub_stamp $pg) - CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \ - trigger_scrub $pg || return 1 - # Wait for schedule regular scrub - # to notice scrub and skip it - local found=false - for i in $(seq 14 -1 0) - do - sleep 1 - ! grep -q "Regular scrub skipped due to deep-scrub errors and nodeep-scrub set" $dir/osd.${primary}.log || { found=true ; break; } - echo Time left: $i seconds - done - test $found = "true" || return 1 - - # Bad object still known - rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1 - - # Request a regular scrub and it will be done - local scrub_backoff_ratio=$(get_config osd ${primary} osd_scrub_backoff_ratio) - set_config osd ${primary} osd_scrub_backoff_ratio 0 - pg_scrub $pg - sleep 1 - set_config osd ${primary} osd_scrub_backoff_ratio $scrub_backoff_ratio - grep -q "Regular scrub request, losing deep-scrub details" $dir/osd.${primary}.log || return 1 - - # deep-scrub error is no longer present - rados list-inconsistent-obj $pg | jq '.' | grep -qv $objname || return 1 -} - - -main osd-scrub-repair "$@" - -# Local Variables: -# compile-command: "cd ../.. ; make -j4 && \ -# test/osd/osd-scrub-repair.sh # TEST_corrupt_and_repair_replicated" -# End: diff --git a/src/test/osd/osd-scrub-snaps.sh b/src/test/osd/osd-scrub-snaps.sh deleted file mode 100755 index 5328e5e009e..00000000000 --- a/src/test/osd/osd-scrub-snaps.sh +++ /dev/null @@ -1,480 +0,0 @@ -#! /bin/bash -# -# Copyright (C) 2015 Red Hat -# -# Author: David Zafman -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# -source $(dirname $0)/../detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh - -function run() { - local dir=$1 - shift - - export CEPH_MON="127.0.0.1:7121" # git grep '\<7121\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - - local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} - for func in $funcs ; do - $func $dir || return 1 - done -} - -function TEST_scrub_snaps() { - local dir=$1 - local poolname=test - - TESTDATA="testdata.$$" - - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 - run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - - wait_for_clean || return 1 - - # Create a pool with a single pg - ceph osd pool create $poolname 1 1 - poolid=$(ceph osd dump | grep "^pool.*[']test[']" | awk '{ print $2 }') - - dd if=/dev/urandom of=$TESTDATA bs=1032 count=1 - for i in `seq 1 15` - do - rados -p $poolname put obj${i} $TESTDATA - done - - SNAP=1 - rados -p $poolname mksnap snap${SNAP} - dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP} - rados -p $poolname put obj1 $TESTDATA - rados -p $poolname put obj5 $TESTDATA - rados -p $poolname put obj3 $TESTDATA - for i in `seq 6 14` - do rados -p $poolname put obj${i} $TESTDATA - done - - SNAP=2 - rados -p $poolname mksnap snap${SNAP} - dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP} - rados -p $poolname put obj5 $TESTDATA - - SNAP=3 - rados -p $poolname mksnap snap${SNAP} - dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP} - rados -p $poolname put obj3 $TESTDATA - - SNAP=4 - rados -p $poolname mksnap snap${SNAP} - dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP} - rados -p $poolname put obj5 $TESTDATA - rados -p $poolname put obj2 $TESTDATA - - SNAP=5 - rados -p $poolname mksnap snap${SNAP} - SNAP=6 - rados -p $poolname mksnap snap${SNAP} - dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP} - rados -p $poolname put obj5 $TESTDATA - - SNAP=7 - rados -p $poolname mksnap snap${SNAP} - - rados -p $poolname rm obj4 - rados -p $poolname rm obj2 - - kill_daemons $dir TERM osd || return 1 - - # Don't need to ceph_objectstore_tool function because osd stopped - - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj1)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" --force remove - - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --op list obj5 | grep \"snapid\":2)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" remove - - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --op list obj5 | grep \"snapid\":1)" - OBJ5SAVE="$JSON" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" remove - - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --op list obj5 | grep \"snapid\":4)" - dd if=/dev/urandom of=$TESTDATA bs=256 count=18 - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" set-bytes $TESTDATA - - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj3)" - dd if=/dev/urandom of=$TESTDATA bs=256 count=15 - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" set-bytes $TESTDATA - - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --op list obj4 | grep \"snapid\":7)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" remove - - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj2)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" rm-attr snapset - - # Create a clone which isn't in snapset and doesn't have object info - JSON="$(echo "$OBJ5SAVE" | sed s/snapid\":1/snapid\":7/)" - dd if=/dev/urandom of=$TESTDATA bs=256 count=7 - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" set-bytes $TESTDATA - - rm -f $TESTDATA - - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj6)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj7)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset corrupt - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj8)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset seq - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj9)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset clone_size - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj10)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset clone_overlap - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj11)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset clones - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj12)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset head - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj13)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset snaps - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj14)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset size - - echo "garbage" > $dir/bad - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj15)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" set-attr snapset $dir/bad - rm -f $dir/bad - - run_osd $dir 0 || return 1 - wait_for_clean || return 1 - - local pgid="${poolid}.0" - if ! pg_scrub "$pgid" ; then - cat $dir/osd.0.log - return 1 - fi - grep 'log_channel' $dir/osd.0.log - - rados list-inconsistent-pg $poolname > $dir/json || return 1 - # Check pg count - test $(jq '. | length' $dir/json) = "1" || return 1 - # Check pgid - test $(jq -r '.[0]' $dir/json) = $pgid || return 1 - - rados list-inconsistent-snapset $pgid > $dir/json || return 1 - test $(jq '.inconsistents | length' $dir/json) = "21" || return 1 - - local jqfilter='.inconsistents' - local sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)' - - jq "$jqfilter" << EOF | python -c "$sortkeys" > $dir/checkcsjson -{ - "inconsistents": [ - { - "errors": [ - "headless" - ], - "snap": 1, - "locator": "", - "nspace": "", - "name": "obj1" - }, - { - "errors": [ - "size_mismatch" - ], - "snap": 1, - "locator": "", - "nspace": "", - "name": "obj10" - }, - { - "errors": [ - "headless" - ], - "snap": 1, - "locator": "", - "nspace": "", - "name": "obj11" - }, - { - "errors": [ - "size_mismatch" - ], - "snap": 1, - "locator": "", - "nspace": "", - "name": "obj14" - }, - { - "errors": [ - "headless" - ], - "snap": 1, - "locator": "", - "nspace": "", - "name": "obj6" - }, - { - "errors": [ - "headless" - ], - "snap": 1, - "locator": "", - "nspace": "", - "name": "obj7" - }, - { - "errors": [ - "size_mismatch" - ], - "snap": 1, - "locator": "", - "nspace": "", - "name": "obj9" - }, - { - "errors": [ - "headless" - ], - "snap": 4, - "locator": "", - "nspace": "", - "name": "obj2" - }, - { - "errors": [ - "size_mismatch" - ], - "snap": 4, - "locator": "", - "nspace": "", - "name": "obj5" - }, - { - "errors": [ - "headless" - ], - "snap": 7, - "locator": "", - "nspace": "", - "name": "obj2" - }, - { - "errors": [ - "oi_attr_missing", - "headless" - ], - "snap": 7, - "locator": "", - "nspace": "", - "name": "obj5" - }, - { - "extra clones": [ - 1 - ], - "errors": [ - "extra_clones" - ], - "snap": "head", - "locator": "", - "nspace": "", - "name": "obj11" - }, - { - "errors": [ - "head_mismatch" - ], - "snap": "head", - "locator": "", - "nspace": "", - "name": "obj12" - }, - { - "errors": [ - "ss_attr_corrupted" - ], - "snap": "head", - "locator": "", - "nspace": "", - "name": "obj15" - }, - { - "extra clones": [ - 7, - 4 - ], - "errors": [ - "ss_attr_missing", - "extra_clones" - ], - "snap": "head", - "locator": "", - "nspace": "", - "name": "obj2" - }, - { - "errors": [ - "size_mismatch" - ], - "snap": "head", - "locator": "", - "nspace": "", - "name": "obj3" - }, - { - "missing": [ - 7 - ], - "errors": [ - "clone_missing" - ], - "snap": "head", - "locator": "", - "nspace": "", - "name": "obj4" - }, - { - "missing": [ - 2, - 1 - ], - "extra clones": [ - 7 - ], - "errors": [ - "extra_clones", - "clone_missing" - ], - "snap": "head", - "locator": "", - "nspace": "", - "name": "obj5" - }, - { - "extra clones": [ - 1 - ], - "errors": [ - "extra_clones" - ], - "snap": "head", - "locator": "", - "nspace": "", - "name": "obj6" - }, - { - "extra clones": [ - 1 - ], - "errors": [ - "head_mismatch", - "extra_clones" - ], - "snap": "head", - "locator": "", - "nspace": "", - "name": "obj7" - }, - { - "errors": [ - "snapset_mismatch" - ], - "snap": "head", - "locator": "", - "nspace": "", - "name": "obj8" - } - ], - "epoch": 20 -} -EOF - - jq "$jqfilter" $dir/json | python -c "$sortkeys" > $dir/csjson - diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || return 1 - - if which jsonschema > /dev/null; - then - jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-snap.json || return 1 - fi - - for i in `seq 1 7` - do - rados -p $poolname rmsnap snap$i - done - - ERRORS=0 - - pidfile=$(find $dir 2>/dev/null | grep $name_prefix'[^/]*\.pid') - pid=$(cat $pidfile) - if ! kill -0 $pid - then - echo "OSD crash occurred" - tail -100 $dir/osd.0.log - ERRORS=$(expr $ERRORS + 1) - fi - - kill_daemons $dir || return 1 - - declare -a err_strings - err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj10:.* is missing in clone_overlap" - err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj5:7 no '_' attr" - err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj5:7 is an unexpected clone" - err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj5:4 on disk size [(]4608[)] does not match object info size [(]512[)] adjusted for ondisk to [(]512[)]" - err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj5:head expected clone .*:::obj5:2" - err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj5:head expected clone .*:::obj5:1" - err_strings[6]="log_channel[(]cluster[)] log [[]INF[]] : scrub [0-9]*[.]0 .*:::obj5:head 2 missing clone[(]s[)]" - err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj12:head snapset.head_exists=false, but head exists" - err_strings[8]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj8:head snaps.seq not set" - err_strings[9]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj7:head snapset.head_exists=false, but head exists" - err_strings[10]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj7:1 is an unexpected clone" - err_strings[11]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj3:head on disk size [(]3840[)] does not match object info size [(]768[)] adjusted for ondisk to [(]768[)]" - err_strings[12]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj6:1 is an unexpected clone" - err_strings[13]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj2:head no 'snapset' attr" - err_strings[14]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj2:7 clone ignored due to missing snapset" - err_strings[15]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj2:4 clone ignored due to missing snapset" - err_strings[16]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj4:head expected clone .*:::obj4:7" - err_strings[17]="log_channel[(]cluster[)] log [[]INF[]] : scrub [0-9]*[.]0 .*:::obj4:head 1 missing clone[(]s[)]" - err_strings[18]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj1:1 is an unexpected clone" - err_strings[19]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj9:1 is missing in clone_size" - err_strings[20]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj11:1 is an unexpected clone" - err_strings[21]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj14:1 size 1032 != clone_size 1033" - err_strings[22]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 23 errors" - err_strings[23]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj15:head can't decode 'snapset' attr buffer" - err_strings[24]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj12:1 has no oi or legacy_snaps; cannot convert 1=[[]1[]]:[[]1[]].stray_clone_snaps=[{]1=[[]1[]][}]" - - for i in `seq 0 ${#err_strings[@]}` - do - if ! grep "${err_strings[$i]}" $dir/osd.0.log > /dev/null; - then - echo "Missing log message '${err_strings[$i]}'" - ERRORS=$(expr $ERRORS + 1) - fi - done - - teardown $dir || return 1 - - if [ $ERRORS != "0" ]; - then - echo "TEST FAILED WITH $ERRORS ERRORS" - return 1 - fi - - echo "TEST PASSED" - return 0 -} - -main osd-scrub-snaps "$@" - -# Local Variables: -# compile-command: "cd ../.. ; make -j4 && \ -# test/osd/osd-scrub-snaps.sh" diff --git a/src/test/test-ceph-helpers.sh b/src/test/test-ceph-helpers.sh deleted file mode 100755 index 488bed02318..00000000000 --- a/src/test/test-ceph-helpers.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2013,2014 Cloudwatt -# Copyright (C) 2014 Red Hat -# Copyright (C) 2014 Federico Gimenez -# -# Author: Loic Dachary -# Author: Federico Gimenez -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Library Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library Public License for more details. -# -source $(dirname $0)/detect-build-env-vars.sh - -$CEPH_ROOT/qa/workunits/ceph-helpers.sh TESTS diff --git a/src/test/test_crush_bucket.sh b/src/test/test_crush_bucket.sh index ead183e213b..837fd0a54b2 100755 --- a/src/test/test_crush_bucket.sh +++ b/src/test/test_crush_bucket.sh @@ -6,7 +6,7 @@ # Includes source $(dirname $0)/detect-build-env-vars.sh -source ../qa/workunits/ceph-helpers.sh +source ../qa/standalone/ceph-helpers.sh function run() { local dir=$1 shift diff --git a/src/test/test_pidfile.sh b/src/test/test_pidfile.sh index 3fd003239be..b76fa9b64a6 100755 --- a/src/test/test_pidfile.sh +++ b/src/test/test_pidfile.sh @@ -6,7 +6,7 @@ # Includes source $(dirname $0)/detect-build-env-vars.sh -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh function run() { local dir=$1 diff --git a/src/test/test_pool_create.sh b/src/test/test_pool_create.sh index 8f122192b24..613e205f813 100755 --- a/src/test/test_pool_create.sh +++ b/src/test/test_pool_create.sh @@ -5,7 +5,7 @@ # # Includes -source ../qa/workunits/ceph-helpers.sh +source ../qa/standalone/ceph-helpers.sh function run() { local dir=$1 diff --git a/src/test/vstart_wrapper.sh b/src/test/vstart_wrapper.sh index 9af1cf9b410..235a81fb467 100755 --- a/src/test/vstart_wrapper.sh +++ b/src/test/vstart_wrapper.sh @@ -16,7 +16,7 @@ # GNU Library Public License for more details. # -source $CEPH_ROOT/qa/workunits/ceph-helpers.sh +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh export CEPH_VSTART_WRAPPER=1 export CEPH_DIR="${TMPDIR:-$PWD}/td/t-$CEPH_PORT"