From: Sage Weil Date: Mon, 18 Aug 2014 15:39:14 +0000 (-0700) Subject: qa/workunits/hadoop: move all hadoop tests into a hadoop/ dir X-Git-Tag: v0.86~243^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F2279%2Fhead;p=ceph.git qa/workunits/hadoop: move all hadoop tests into a hadoop/ dir Signed-off-by: Sage Weil --- diff --git a/qa/workunits/hadoop-internal-tests/test.sh b/qa/workunits/hadoop-internal-tests/test.sh deleted file mode 100755 index 48a9224dac17..000000000000 --- a/qa/workunits/hadoop-internal-tests/test.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash -e - -# bail if $TESTDIR is not set as this test will fail in that scenario -[ -z $TESTDIR ] && { echo "\$TESTDIR needs to be set, but is not. Exiting."; exit 1; } - -# configure CEPH_CONF and LD_LIBRARY_PATH if they're not already set -conf="$CEPH_CONF" -if [ -z "$conf" ] ; then - echo "Setting conf to /etc/ceph/ceph.conf" - conf="/etc/ceph/ceph.conf" -else - echo "conf is set to $conf" -fi - -ld_lib_path="$LD_LIBRARY_PATH" -if [ -z "$ld_lib_path" ] ; then - echo "Setting ld_lib_path to /usr/lib/jni" - ld_lib_path="/usr/lib/jni" -else - echo "ld_lib_path was set to $ld_lib_path" -fi - -POOL_SIZES=`seq 1 8` -POOL_BASE=hadoop -POOL_NAMES=`echo -n $POOL_SIZES | sed "s/\([0-9]*\)/$POOL_BASE\1/g" | sed "s/ /,/g"` - -function gen_hadoop_conf() { -local outfile=$1 -local poolnames=$2 -local conf=$3 -cat << EOF > $outfile - - - - - ceph.conf.file - $conf - - - ceph.data.pools - $poolnames - - -EOF -} - -echo creating hadoop test pools -for size in $POOL_SIZES; do - name=${POOL_BASE}$size - echo creating pool $name - ceph osd pool create $name 100 100 - ceph osd pool set $name size $size - - echo making pool $name a data pool - poolid=`ceph osd dump | sed -n "s/^pool \([0-9]*\) '$name'.*/\1/p"` - ceph mds add_data_pool $poolid -done - -def_repl_conf=`mktemp` -echo generating default replication hadoop config $def_repl_conf -gen_hadoop_conf $def_repl_conf "" $conf - -cust_repl_conf=`mktemp` -echo generating custom replication hadoop config $cust_repl_conf -gen_hadoop_conf $cust_repl_conf $POOL_NAMES $conf - -echo running default replication hadoop tests -java -Dhadoop.conf.file=$def_repl_conf -Djava.library.path=$ld_lib_path -cp /usr/share/java/junit4.jar:$TESTDIR/apache_hadoop/build/hadoop-core-1.0.4-SNAPSHOT.jar:$TESTDIR/inktank_hadoop/build/hadoop-cephfs.jar:$TESTDIR/inktank_hadoop/build/hadoop-cephfs-test.jar:$TESTDIR/apache_hadoop/build/hadoop-test-1.0.4-SNAPSHOT.jar:$TESTDIR/apache_hadoop/build/ivy/lib/Hadoop/common/commons-logging-1.1.1.jar:/usr/share/java/libcephfs.jar org.junit.runner.JUnitCore org.apache.hadoop.fs.ceph.TestCephDefaultReplication - -echo running custom replication hadoop tests -java -Dhadoop.conf.file=$cust_repl_conf -Djava.library.path=$ld_lib_path -cp /usr/share/java/junit4.jar:$TESTDIR/apache_hadoop/build/hadoop-core-1.0.4-SNAPSHOT.jar:$TESTDIR/inktank_hadoop/build/hadoop-cephfs.jar:$TESTDIR/inktank_hadoop/build/hadoop-cephfs-test.jar:$TESTDIR/apache_hadoop/build/hadoop-test-1.0.4-SNAPSHOT.jar:$TESTDIR/apache_hadoop/build/ivy/lib/Hadoop/common/commons-logging-1.1.1.jar:/usr/share/java/libcephfs.jar org.junit.runner.JUnitCore org.apache.hadoop.fs.ceph.TestCephCustomReplication - -echo "completed hadoop-internal-tests tests" -exit 0 diff --git a/qa/workunits/hadoop-wordcount/test.sh b/qa/workunits/hadoop-wordcount/test.sh deleted file mode 100755 index 66fac10ddf7c..000000000000 --- a/qa/workunits/hadoop-wordcount/test.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -ex - -echo "starting hadoop-wordcount test" - -# bail if $TESTDIR is not set as this test will fail in that scenario -[ -z $TESTDIR ] && { echo "\$TESTDIR needs to be set, but is not. Exiting."; exit 1; } - -# if HADOOP_HOME is not set, use default -[ -z $HADOOP_HOME ] && { HADOOP_HOME=$TESTDIR/apache_hadoop; } - -# if HADOOP_MR_HOME is not set, use default -[ -z $HADOOP_MR_HOME ] && { HADOOP_MR_HOME=$TESTDIR/apache_hadoop/build; } - -export JAVA_HOME=/usr/lib/jvm/default-java -mkdir -p $TESTDIR/hadoop_input -wget http://ceph.com/qa/hadoop_input_files.tar -O $TESTDIR/hadoop_input/files.tar -cd $TESTDIR/hadoop_input -tar -xf $TESTDIR/hadoop_input/files.tar -$HADOOP_HOME/bin/hadoop fs -mkdir /wordcount_input -$HADOOP_HOME/bin/hadoop fs -rmr /wordcount_output -$HADOOP_HOME/bin/hadoop fs -put $TESTDIR/hadoop_input/*txt /wordcount_input/ -$HADOOP_HOME/bin/hadoop jar $HADOOP_MR_HOME/hadoop-*examples.jar wordcount /wordcount_input /wordcount_output -rm -rf $TESTDIR/hadoop_input - -echo "completed hadoop-wordcount test" -exit 0 diff --git a/qa/workunits/hadoop/internal-tests.sh b/qa/workunits/hadoop/internal-tests.sh new file mode 100755 index 000000000000..48a9224dac17 --- /dev/null +++ b/qa/workunits/hadoop/internal-tests.sh @@ -0,0 +1,74 @@ +#!/bin/bash -e + +# bail if $TESTDIR is not set as this test will fail in that scenario +[ -z $TESTDIR ] && { echo "\$TESTDIR needs to be set, but is not. Exiting."; exit 1; } + +# configure CEPH_CONF and LD_LIBRARY_PATH if they're not already set +conf="$CEPH_CONF" +if [ -z "$conf" ] ; then + echo "Setting conf to /etc/ceph/ceph.conf" + conf="/etc/ceph/ceph.conf" +else + echo "conf is set to $conf" +fi + +ld_lib_path="$LD_LIBRARY_PATH" +if [ -z "$ld_lib_path" ] ; then + echo "Setting ld_lib_path to /usr/lib/jni" + ld_lib_path="/usr/lib/jni" +else + echo "ld_lib_path was set to $ld_lib_path" +fi + +POOL_SIZES=`seq 1 8` +POOL_BASE=hadoop +POOL_NAMES=`echo -n $POOL_SIZES | sed "s/\([0-9]*\)/$POOL_BASE\1/g" | sed "s/ /,/g"` + +function gen_hadoop_conf() { +local outfile=$1 +local poolnames=$2 +local conf=$3 +cat << EOF > $outfile + + + + + ceph.conf.file + $conf + + + ceph.data.pools + $poolnames + + +EOF +} + +echo creating hadoop test pools +for size in $POOL_SIZES; do + name=${POOL_BASE}$size + echo creating pool $name + ceph osd pool create $name 100 100 + ceph osd pool set $name size $size + + echo making pool $name a data pool + poolid=`ceph osd dump | sed -n "s/^pool \([0-9]*\) '$name'.*/\1/p"` + ceph mds add_data_pool $poolid +done + +def_repl_conf=`mktemp` +echo generating default replication hadoop config $def_repl_conf +gen_hadoop_conf $def_repl_conf "" $conf + +cust_repl_conf=`mktemp` +echo generating custom replication hadoop config $cust_repl_conf +gen_hadoop_conf $cust_repl_conf $POOL_NAMES $conf + +echo running default replication hadoop tests +java -Dhadoop.conf.file=$def_repl_conf -Djava.library.path=$ld_lib_path -cp /usr/share/java/junit4.jar:$TESTDIR/apache_hadoop/build/hadoop-core-1.0.4-SNAPSHOT.jar:$TESTDIR/inktank_hadoop/build/hadoop-cephfs.jar:$TESTDIR/inktank_hadoop/build/hadoop-cephfs-test.jar:$TESTDIR/apache_hadoop/build/hadoop-test-1.0.4-SNAPSHOT.jar:$TESTDIR/apache_hadoop/build/ivy/lib/Hadoop/common/commons-logging-1.1.1.jar:/usr/share/java/libcephfs.jar org.junit.runner.JUnitCore org.apache.hadoop.fs.ceph.TestCephDefaultReplication + +echo running custom replication hadoop tests +java -Dhadoop.conf.file=$cust_repl_conf -Djava.library.path=$ld_lib_path -cp /usr/share/java/junit4.jar:$TESTDIR/apache_hadoop/build/hadoop-core-1.0.4-SNAPSHOT.jar:$TESTDIR/inktank_hadoop/build/hadoop-cephfs.jar:$TESTDIR/inktank_hadoop/build/hadoop-cephfs-test.jar:$TESTDIR/apache_hadoop/build/hadoop-test-1.0.4-SNAPSHOT.jar:$TESTDIR/apache_hadoop/build/ivy/lib/Hadoop/common/commons-logging-1.1.1.jar:/usr/share/java/libcephfs.jar org.junit.runner.JUnitCore org.apache.hadoop.fs.ceph.TestCephCustomReplication + +echo "completed hadoop-internal-tests tests" +exit 0 diff --git a/qa/workunits/hadoop/wordcount.sh b/qa/workunits/hadoop/wordcount.sh new file mode 100755 index 000000000000..66fac10ddf7c --- /dev/null +++ b/qa/workunits/hadoop/wordcount.sh @@ -0,0 +1,26 @@ +#!/bin/sh -ex + +echo "starting hadoop-wordcount test" + +# bail if $TESTDIR is not set as this test will fail in that scenario +[ -z $TESTDIR ] && { echo "\$TESTDIR needs to be set, but is not. Exiting."; exit 1; } + +# if HADOOP_HOME is not set, use default +[ -z $HADOOP_HOME ] && { HADOOP_HOME=$TESTDIR/apache_hadoop; } + +# if HADOOP_MR_HOME is not set, use default +[ -z $HADOOP_MR_HOME ] && { HADOOP_MR_HOME=$TESTDIR/apache_hadoop/build; } + +export JAVA_HOME=/usr/lib/jvm/default-java +mkdir -p $TESTDIR/hadoop_input +wget http://ceph.com/qa/hadoop_input_files.tar -O $TESTDIR/hadoop_input/files.tar +cd $TESTDIR/hadoop_input +tar -xf $TESTDIR/hadoop_input/files.tar +$HADOOP_HOME/bin/hadoop fs -mkdir /wordcount_input +$HADOOP_HOME/bin/hadoop fs -rmr /wordcount_output +$HADOOP_HOME/bin/hadoop fs -put $TESTDIR/hadoop_input/*txt /wordcount_input/ +$HADOOP_HOME/bin/hadoop jar $HADOOP_MR_HOME/hadoop-*examples.jar wordcount /wordcount_input /wordcount_output +rm -rf $TESTDIR/hadoop_input + +echo "completed hadoop-wordcount test" +exit 0