From de6b53aba7168d841ae17e221aa13f635ae79f25 Mon Sep 17 00:00:00 2001
From: Noah Watkins <noahwatkins@gmail.com>
Date: Wed, 11 Feb 2015 16:02:49 -0800
Subject: [PATCH] qa: hadoop plays nice with new teuthology task

This brings the hadoop wordcount up-to-date with the new teuthology
hadoop task.

Signed-off-by: Noah Watkins <noahwatkins@gmail.com>
---
 qa/workunits/hadoop/wordcount.sh | 57 +++++++++++++-------------------
 1 file changed, 23 insertions(+), 34 deletions(-)

diff --git a/qa/workunits/hadoop/wordcount.sh b/qa/workunits/hadoop/wordcount.sh
index 1b1c20702431b..da91929fcdc74 100755
--- a/qa/workunits/hadoop/wordcount.sh
+++ b/qa/workunits/hadoop/wordcount.sh
@@ -1,4 +1,11 @@
-#!/bin/sh -ex
+#!/bin/bash
+
+set -e
+set -x
+
+WC_INPUT=/wc_input
+WC_OUTPUT=/wc_output
+DATA_INPUT=$(mktemp -d)
 
 echo "starting hadoop-wordcount test"
 
@@ -6,43 +13,25 @@ echo "starting hadoop-wordcount test"
 [ -z $TESTDIR ] && { echo "\$TESTDIR needs to be set, but is not. Exiting."; exit 1; }
 
 # if HADOOP_PREFIX is not set, use default
-[ -z $HADOOP_PREFIX ] && { HADOOP_PREFIX=$TESTDIR/apache_hadoop; }
-
-# if HADOOP_MR_HOME is not set, use default
-[ -z $HADOOP_MR_HOME ] && { HADOOP_MR_HOME=$TESTDIR/apache_hadoop/build; }
+[ -z $HADOOP_PREFIX ] && { HADOOP_PREFIX=$TESTDIR/hadoop; }
 
 export JAVA_HOME=/usr/lib/jvm/default-java
 
-set -e
-set -x
+# Nuke hadoop directories
+$HADOOP_PREFIX/bin/hadoop fs -rm -r $WC_INPUT $WC_OUTPUT || true
+
+# Fetch and import testing data set
+curl http://ceph.com/qa/hadoop_input_files.tar | tar xf - -C $DATA_INPUT
+$HADOOP_PREFIX/bin/hadoop fs -copyFromLocal $DATA_INPUT $WC_INPUT
+rm -rf $DATA_INPUT
+
+# Run the job
+$HADOOP_PREFIX/bin/hadoop jar \
+  $HADOOP_PREFIX/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar \
+  wordcount $WC_INPUT $WC_OUTPUT
 
-# Clear out in case there was a previous run (idempotency)
-if $HADOOP_PREFIX/bin/hadoop fs -ls /wordcount_output 2>/dev/null ; then
-    $HADOOP_PREFIX/bin/hadoop fs -rmr /wordcount_output
-fi
-if $HADOOP_PREFIX/bin/hadoop fs -ls /wordcount_input 2>/dev/null ; then
-    $HADOOP_PREFIX/bin/hadoop fs -rmr /wordcount_input
-fi
-rm -rf $TESTDIR/hadoop_input
-
-# Load input files into local filesystem
-mkdir -p $TESTDIR/hadoop_input
-wget http://ceph.com/qa/hadoop_input_files.tar -O $TESTDIR/hadoop_input/files.tar
-cd $TESTDIR/hadoop_input
-tar -xf $TESTDIR/hadoop_input/files.tar
-
-# Load input files into hadoop filesystem
-$HADOOP_PREFIX/bin/hadoop fs -mkdir /wordcount_input
-$HADOOP_PREFIX/bin/hadoop fs -put $TESTDIR/hadoop_input/*txt /wordcount_input/
-
-# Execute job
-$HADOOP_PREFIX/bin/hadoop jar $HADOOP_MR_HOME/hadoop*examples*jar wordcount /wordcount_input /wordcount_output
-
-# Clean up
-$HADOOP_PREFIX/bin/hadoop fs -rmr /wordcount_output
-$HADOOP_PREFIX/bin/hadoop fs -rmr /wordcount_input
-cd $TESTDIR
-rm -rf $TESTDIR/hadoop_input
+# Cleanup
+$HADOOP_PREFIX/bin/hadoop fs -rm -r $WC_INPUT $WC_OUTPUT || true
 
 echo "completed hadoop-wordcount test"
 exit 0
-- 
2.39.5