From a8da9fd0771c5b708b2863f7bb5f5646d3397e89 Mon Sep 17 00:00:00 2001 From: Bassam Tabbara Date: Fri, 7 Jul 2017 01:13:35 -0700 Subject: [PATCH] test,qa/workunits: add tests for public_bind_addr Add a set of new tests for the case when public_addr and public_bind_addr are different for a mon. In order to test this properly I had to employ port forwarding with socat. This helps simulate what would happen in a environment like Kubernetes. socat is now a build dependency. Also, moved jq_success to ceph-helpers.sh and refactored run_mon to enable creating the mons without creating the rbd pool immediately. Signed-off-by: Bassam Tabbara --- ceph.spec.in | 1 + debian/control | 1 + install-deps.sh | 1 + qa/workunits/ceph-helpers.sh | 56 ++++++++++++- src/test/mon/CMakeLists.txt | 1 + src/test/mon/misc.sh | 41 ---------- src/test/mon/mon-bind.sh | 148 +++++++++++++++++++++++++++++++++++ 7 files changed, 205 insertions(+), 44 deletions(-) create mode 100755 src/test/mon/mon-bind.sh diff --git a/ceph.spec.in b/ceph.spec.in index 91297331c248..07231917f0ac 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -131,6 +131,7 @@ BuildRequires: python-pecan BuildRequires: python-requests BuildRequires: python-virtualenv BuildRequires: python-werkzeug +BuildRequires: socat BuildRequires: snappy-devel BuildRequires: udev BuildRequires: util-linux diff --git a/debian/control b/debian/control index 4f77e3f48523..b9e4c67962ac 100644 --- a/debian/control +++ b/debian/control @@ -58,6 +58,7 @@ Build-Depends: bc, python-werkzeug, python3-all-dev, python3-setuptools, + socat, uuid-runtime, valgrind, virtualenv | python-virtualenv, diff --git a/install-deps.sh b/install-deps.sh index 626cbc9e3978..315848d90526 100755 --- a/install-deps.sh +++ b/install-deps.sh @@ -43,6 +43,7 @@ if [ x`uname`x = xFreeBSDx ]; then ftp/curl \ misc/e2fsprogs-libuuid \ misc/getopt \ + net/socat \ textproc/expat2 \ textproc/gsed \ textproc/libxml2 \ diff --git a/qa/workunits/ceph-helpers.sh b/qa/workunits/ceph-helpers.sh index 7b61d3684391..2c260255d5e2 100755 --- a/qa/workunits/ceph-helpers.sh +++ b/qa/workunits/ceph-helpers.sh @@ -363,7 +363,7 @@ function test_kill_daemons() { # @param ... can be any option valid for ceph-mon # @return 0 on success, 1 on error # -function run_mon() { +function run_mon_no_pool() { local dir=$1 shift local id=$1 @@ -398,13 +398,23 @@ function run_mon() { --mon-allow-pool-delete \ "$@" || return 1 - ceph osd pool create rbd 8 - cat > $dir/ceph.conf < /dev/null || return 1 - # NOTE: - # jq only support --exit-status|-e from version 1.4 forwards, which makes - # returning on error waaaay prettier and straightforward. - # However, the current automated upstream build is running with v1.3, - # which has no idea what -e is. Hence the convoluted error checking we - # need. Sad. - # The next time someone changes this code, please check if v1.4 is now - # a thing, and, if so, please change these to use -e. Thanks. - - # jq '.all.supported | select([.[] == "foo"] | any)' - # expect monmap to contain 3 monitors (a, b, and c) jqinput="$(ceph mon_status --format=json 2>/dev/null)" jq_success "$jqinput" '.monmap.mons | length == 3' || return 1 diff --git a/src/test/mon/mon-bind.sh b/src/test/mon/mon-bind.sh new file mode 100755 index 000000000000..abac7761bd04 --- /dev/null +++ b/src/test/mon/mon-bind.sh @@ -0,0 +1,148 @@ +#!/bin/bash +# +# Copyright (C) 2017 Quantum Corp. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# +source $(dirname $0)/../detect-build-env-vars.sh +source $CEPH_ROOT/qa/workunits/ceph-helpers.sh + +SOCAT_PIDS=() + +function port_forward() { + local source_port=$1 + local target_port=$2 + + socat TCP-LISTEN:${source_port},fork,reuseaddr TCP:localhost:${target_port} & + SOCAT_PIDS+=( $! ) +} + +function cleanup() { + for p in "${SOCAT_PIDS[@]}"; do + kill $p + done + SOCAT_PIDS=() +} + +trap cleanup SIGTERM SIGKILL SIGQUIT SIGINT + +function run() { + local dir=$1 + shift + + export MON_IP=127.0.0.1 + export MONA_PUBLIC=7132 # git grep '\<7132\>' ; there must be only one + export MONB_PUBLIC=7133 # git grep '\<7133\>' ; there must be only one + export MONC_PUBLIC=7134 # git grep '\<7134\>' ; there must be only one + export MONA_BIND=7135 # git grep '\<7135\>' ; there must be only one + export MONB_BIND=7136 # git grep '\<7136\>' ; there must be only one + export MONC_BIND=7137 # git grep '\<7137\>' ; there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir && cleanup || { cleanup; return 1; } + teardown $dir + done +} + +function TEST_mon_client_connect_fails() { + local dir=$1 + + # start the mon with a public-bind-addr that is different + # from the public-addr. + CEPH_ARGS+="--mon-initial-members=a " + CEPH_ARGS+="--mon-host=${MON_IP}:${MONA_PUBLIC} " + run_mon_no_pool $dir a --mon-host=${MON_IP}:${MONA_PUBLIC} --public-bind-addr=${MON_IP}:${MONA_BIND} || return 1 + + # now attempt to ping it that should fail. + timeout 3 ceph ping mon.a || return 0 + return 1 +} + +function TEST_mon_client_connect() { + local dir=$1 + + # start the mon with a public-bind-addr that is different + # from the public-addr. + CEPH_ARGS+="--mon-initial-members=a " + CEPH_ARGS+="--mon-host=${MON_IP}:${MONA_PUBLIC} " + run_mon_no_pool $dir a --mon-host=${MON_IP}:${MONA_PUBLIC} --public-bind-addr=${MON_IP}:${MONA_BIND} || return 1 + + # now forward the public port to the bind port. + port_forward ${MONA_PUBLIC} ${MONA_BIND} + + # attempt to connect. we expect that to work + ceph ping mon.a || return 1 +} + +function TEST_mon_quorum() { + local dir=$1 + + # start the mon with a public-bind-addr that is different + # from the public-addr. + CEPH_ARGS+="--mon-initial-members=a,b,c " + CEPH_ARGS+="--mon-host=${MON_IP}:${MONA_PUBLIC},${MON_IP}:${MONB_PUBLIC},${MON_IP}:${MONC_PUBLIC} " + run_mon_no_pool $dir a --public-addr=${MON_IP}:${MONA_PUBLIC} --public-bind-addr=${MON_IP}:${MONA_BIND} || return 1 + run_mon_no_pool $dir b --public-addr=${MON_IP}:${MONB_PUBLIC} --public-bind-addr=${MON_IP}:${MONB_BIND} || return 1 + run_mon_no_pool $dir c --public-addr=${MON_IP}:${MONC_PUBLIC} --public-bind-addr=${MON_IP}:${MONC_BIND} || return 1 + + # now forward the public port to the bind port. + port_forward ${MONA_PUBLIC} ${MONA_BIND} + port_forward ${MONB_PUBLIC} ${MONB_BIND} + port_forward ${MONC_PUBLIC} ${MONC_BIND} + + # expect monmap to contain 3 monitors (a, b, and c) + jqinput="$(ceph mon_status --format=json 2>/dev/null)" + jq_success "$jqinput" '.monmap.mons | length == 3' || return 1 + + # quorum should form + wait_for_quorum 300 3 || return 1 + # expect quorum to have all three monitors + jqfilter='.quorum | length == 3' + jq_success "$jqinput" "$jqfilter" || return 1 +} + +function TEST_put_get() { + local dir=$1 + + # start the mon with a public-bind-addr that is different + # from the public-addr. + CEPH_ARGS+="--mon-initial-members=a,b,c " + CEPH_ARGS+="--mon-host=${MON_IP}:${MONA_PUBLIC},${MON_IP}:${MONB_PUBLIC},${MON_IP}:${MONC_PUBLIC} " + run_mon_no_pool $dir a --public-addr=${MON_IP}:${MONA_PUBLIC} --public-bind-addr=${MON_IP}:${MONA_BIND} || return 1 + run_mon_no_pool $dir b --public-addr=${MON_IP}:${MONB_PUBLIC} --public-bind-addr=${MON_IP}:${MONB_BIND} || return 1 + run_mon_no_pool $dir c --public-addr=${MON_IP}:${MONC_PUBLIC} --public-bind-addr=${MON_IP}:${MONC_BIND} || return 1 + + # now forward the public port to the bind port. + port_forward ${MONA_PUBLIC} ${MONA_BIND} + port_forward ${MONB_PUBLIC} ${MONB_BIND} + port_forward ${MONC_PUBLIC} ${MONC_BIND} + + # quorum should form + wait_for_quorum 300 3 || return 1 + + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + run_osd $dir 1 || return 1 + run_osd $dir 2 || return 1 + + ceph osd pool create hello 8 || return 1 + + echo "hello world" > $dir/hello + rados --pool hello put foo $dir/hello || return 1 + rados --pool hello get foo $dir/hello2 || return 1 + diff $dir/hello $dir/hello2 || return 1 +} + +main mon-bind "$@" -- 2.47.3