hadoop: update gitbuilder for cephfs-hadoop

Replaces the old hadoop gitbuilders with the current
cephfs-hadoop repository.

Signed-off-by: Noah Watkins <noahwatkins@gmail.com>

diff --git a/build-hadoop.sh b/build-hadoop.sh
index 1f2f1cf713e16eae57d8aae81173eb2280f5f440..d921b9707c07b694d23d61b96e799776e21b3da6 100755 (executable)
--- a/build-hadoop.sh
+++ b/build-hadoop.sh
@@ -1,43 +1,33 @@
 #!/bin/bash -x
 
+set -x
 set -e
 
-HADOOP_ERRORS_IGNORE=""
-
-#using Java6 openjdk for now.
-HADOOP_JAVA_HOME="/usr/lib/jvm/java-6-openjdk"
-export JAVA_HOME=$HADOOP_JAVA_HOME
-
-#get the libcephfs jar and so files so the build works
-GETLIBSOUTPUT=`python ../../get-libcephfs-java-jar.py`
-
-echo $GETLIBSOUTPUT
+# maven 3.0.4 on ubuntu 12.04 has some problems. fabfile will install a copy
+# of maven 3.1.1 at /srv/apache-maven-3.1.1. setup the necessary pointers.
+export M2_HOME=/srv/apache-maven-3.1.1
+export M2=$M2_HOME/bin
+export PATH=$M2:$PATH
+
+# maven3 creates a local repository of downloaded artificats in the current
+# user's home directory. the autobuild-ceph user doesn't have a home
+# directory. use a different location for this repository.
+export LOCAL_MAVEN_REPO=/tmp/autobuild-ceph-m3home
+if [ ! -d "$LOCAL_MAVEN_REPO" ]; then
+  mkdir $LOCAL_MAVEN_REPO
+fi
+export MAVEN_OPTS="-Dmaven.repo.local=$LOCAL_MAVEN_REPO"
 
-HADOOP_ERRORS_IGNORE="\
-grep -vi \"warning\"" #| \
-#grep -v \"is not a pointer or array, skip client functions\" | \
-#grep -v \"is a pointer to type 'string', skip client functions\""
+# skip tests when building; tests need ceph cluster
+mvn package -Dmaven.test.skip=true
 
 REV="$(git rev-parse HEAD)"
-
-DESTDIR_TMP="install.tmp"
 OUTDIR="../out/output/sha1/$REV"
-CURRENT_DIR=`pwd`
-
-install -d -m0766 -- "$DESTDIR_TMP"
-
-NCPU=$(( 2 * `grep -c processor /proc/cpuinfo` ))
-
-echo "$0: building..."
-echo --START-IGNORE-WARNINGS
-# filter out idl errors "Unable to determine origin..." to avoid gitbuilder failing
-ionice -c3 nice -n20 ant -Divy.default.ivy.user.dir=$CURRENT_DIR cephfs cephfs-test 2> >( eval ${HADOOP_ERRORS_IGNORE} ) || exit 4
-
 OUTDIR_TMP="${OUTDIR}.tmp"
-
 install -d -m0755 -- "$OUTDIR_TMP"
-tar czf "${OUTDIR_TMP}/hadoop.tgz" -C "${CURRENT_DIR}" .
+cp target/*.jar $OUTDIR_TMP
 printf '%s\n' "$REV" >"$OUTDIR_TMP/sha1"
+printf '%s\n' "cephfs-hadoop" >"$OUTDIR_TMP/name"
 
 # we're successful, the files are ok to be published; try to be as
 # atomic as possible about replacing potentially existing OUTDIR
@@ -48,6 +38,4 @@ fi
 mv -- "$OUTDIR_TMP" "$OUTDIR"
 rm -rf -- "$OUTDIR.old"
 
-
 exit 0
-

diff --git a/fabfile.py b/fabfile.py
index e58dc208575d56624e1bf0427cc3971f3b0233cc..c242627cdd342d6a59d7511352795e35c4250ff1 100644 (file)
--- a/fabfile.py
+++ b/fabfile.py
@@ -62,11 +62,7 @@ env.roledefs['gitbuilder_samba'] = [
     ]
 
 env.roledefs['gitbuilder_hadoop'] = [
-    'ubuntu@gitbuilder-precise-hadoop-amd64.front.sepia.ceph.com',
-    ]
-
-env.roledefs['gitbuilder_apache_hadoop'] = [
-    'ubuntu@gitbuilder-precise-apache-hadoop-amd64.front.sepia.ceph.com',
+    'ubuntu@gitbuilder-hadoop-jar-precise-amd64-basic.front.sepia.ceph.com',
     ]
 
 
@@ -523,16 +519,6 @@ def gitbuilder_kernel_rpm():
     _sync_to_gitbuilder('kernel','rpm','basic')
     sudo('start autobuild-ceph || /etc/init.d/autobuild-ceph start')
 
-
-def _hadoop_deps():
-    #_apt_add_testing_repo('master')
-    _apt_install(
-               'openjdk-6-jdk',
-        'ant',
-        'automake',
-        'libtool',
-        )
-
 def _samba_deps():
     _apt_add_testing_repo('master')
     _apt_install(
@@ -579,33 +565,25 @@ def gitbuilder_samba():
 
 @roles('gitbuilder_hadoop')
 def gitbuilder_hadoop():
-    _hadoop_deps()
-    _gitbuilder(
-        flavor='hadoop',
-        git_repo='https://github.com/ceph/hadoop-common.git',
-        extra_packages=[
-            'fakeroot',
-            'reprepro',
-            ],
-        branches_local_name='branches-local-hadoop',
+    _apt_install(
+               'openjdk-6-jdk',
+        'zlib1g-dev',
         )
-    _sync_to_gitbuilder('hadoop', 'jar', 'basic')
-    sudo('start autobuild-ceph || /etc/init.d/autobuild-ceph start')
-
-@roles('gitbuilder_apache_hadoop')
-def gitbuilder_apache_hadoop():
-    _hadoop_deps()
+    if not exists('/srv/apache-maven-3.1.1'):
+        with cd('/srv'):
+            sudo('wget http://apache.osuosl.org/maven/maven-3/3.1.1/binaries/apache-maven-3.1.1-bin.tar.gz')
+            sudo('tar xzf apache-maven-3.1.1-bin.tar.gz')
+            sudo('rm -f apache-maven-3.1.1-bin.tar.gz')
     _gitbuilder(
-        flavor='apache-hadoop',
-        git_repo='git://git.apache.org/hadoop-common.git',
+        flavor='hadoop',
+        git_repo='https://github.com/ceph/cephfs-hadoop.git',
         extra_packages=[
             'fakeroot',
             'reprepro',
             ],
-        branches_local_name='branches-local-apache-hadoop',
         )
-    _sync_to_gitbuilder('apache-hadoop', 'jar', 'basic')
     sudo('start autobuild-ceph || /etc/init.d/autobuild-ceph start')
+    _sync_to_gitbuilder('hadoop', 'jar', 'basic')
 
 @roles('gitbuilder_ceph')
 def gitbuilder_ceph():