From 22f3f906eba99ef21bd2844ed0bb0abb0884a7c1 Mon Sep 17 00:00:00 2001 From: Greg Farnum Date: Sun, 1 Nov 2009 20:18:29 -0800 Subject: [PATCH] Now passes all unit tests! Changes made to rename and between error codes/exception throwing. --- src/client/hadoop/CephFSInterface.cc | 6 ++-- src/client/hadoop/ceph/CephFS.java | 1 + src/client/hadoop/ceph/CephFaker.java | 20 ++++++----- src/client/hadoop/ceph/CephFileSystem.java | 39 ++++++++++++++++------ 4 files changed, 45 insertions(+), 21 deletions(-) diff --git a/src/client/hadoop/CephFSInterface.cc b/src/client/hadoop/CephFSInterface.cc index fdcbbfdfafbd7..a379e31d5b2e8 100644 --- a/src/client/hadoop/CephFSInterface.cc +++ b/src/client/hadoop/CephFSInterface.cc @@ -181,8 +181,10 @@ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_fs_ceph_CephTalker_ceph_1renam env->ReleaseStringUTFChars(j_from, c_from); return false; } - - jboolean success = (0 <= ceph_rename(c_from, c_to)) ? JNI_TRUE : JNI_FALSE; + jboolean success = false; + struct stat stbuf; + if (ceph_lstat(c_to, &stbuf) < 0) //Hadoop doesn't want to overwrite files in a rename + success = (0 <= ceph_rename(c_from, c_to)) ? JNI_TRUE : JNI_FALSE; env->ReleaseStringUTFChars(j_from, c_from); env->ReleaseStringUTFChars(j_to, c_to); return success; diff --git a/src/client/hadoop/ceph/CephFS.java b/src/client/hadoop/ceph/CephFS.java index 0111ebd7f6df0..0de7f44575fb1 100644 --- a/src/client/hadoop/ceph/CephFS.java +++ b/src/client/hadoop/ceph/CephFS.java @@ -34,6 +34,7 @@ abstract class CephFS { protected static final int TRACE = 5; protected static final int NOLOG = 6; + protected static final int ENOTDIR = 20; protected static final int EEXIST = 17; protected static final int ENOENT = 2; diff --git a/src/client/hadoop/ceph/CephFaker.java b/src/client/hadoop/ceph/CephFaker.java index 5fa612af73ed9..4257320ddd08a 100644 --- a/src/client/hadoop/ceph/CephFaker.java +++ b/src/client/hadoop/ceph/CephFaker.java @@ -29,6 +29,7 @@ import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsStatus; @@ -131,12 +132,14 @@ class CephFaker extends CephFS { protected boolean ceph_rename(String oldName, String newName) { oldName = prepare_path(oldName); newName = prepare_path(newName); - boolean ret = false; try { - ret = localFS.rename(new Path(oldName), new Path(newName)); + Path parent = new Path(newName).getParent(); + Path newPath = new Path(newName); + if (localFS.exists(parent) && !localFS.exists(newPath)) + return localFS.rename(new Path(oldName), newPath); + return false; } - catch (IOException e) { } - return ret; + catch (IOException e) { return false; } } protected boolean ceph_exists(String path) { @@ -165,13 +168,11 @@ class CephFaker extends CephFS { protected boolean ceph_isdirectory(String path) { path = prepare_path(path); - boolean ret = false; try { FileStatus status = localFS.getFileStatus(new Path(path)); - ret = status.isDir(); + return status.isDir(); } - catch (IOException e) {} - return ret; + catch (IOException e) { return false; } } protected boolean ceph_isfile(String path) { @@ -211,7 +212,8 @@ class CephFaker extends CephFS { if(localFS.mkdirs(new Path(path), new FsPermission((short)mode))) return 0; } - catch (IOException e) { debug(e.toString(), ERROR); } + catch (FileAlreadyExistsException fe) { return ENOTDIR; } + catch (IOException e) {} if (ceph_isdirectory(path)) return -EEXIST; //apparently it already existed return -1; diff --git a/src/client/hadoop/ceph/CephFileSystem.java b/src/client/hadoop/ceph/CephFileSystem.java index dbdb63f9c3b28..fa571a04afafb 100644 --- a/src/client/hadoop/ceph/CephFileSystem.java +++ b/src/client/hadoop/ceph/CephFileSystem.java @@ -28,6 +28,7 @@ import java.lang.Math; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -142,7 +143,7 @@ public class CephFileSystem extends FileSystem { arguments += " --client-readahead-max-periods=" + conf.get("fs.ceph.readahead", "1"); //make sure they gave us a ceph monitor address or conf file - ceph.debug("initialize:Ceph intialization arguments: " + arguments, ceph.INFO); + ceph.debug("initialize:Ceph initialization arguments: " + arguments, ceph.INFO); if ( (conf.get("fs.ceph.monAddr") == null) && (arguments.indexOf("-m") == -1) && (arguments.indexOf("-c") == -1) ) { @@ -234,7 +235,7 @@ public class CephFileSystem extends FileSystem { Path abs_path = makeAbsolute(dir); ceph.debug("setWorkingDirectory:calling ceph_setcwd from Java", ceph.TRACE); if (!ceph.ceph_setcwd(abs_path.toString())) - ceph.debug("setWorkingDirectory: WARNING! ceph_setcwd failed for some reason on path " + abs_path, ceph.ERROR); + ceph.debug("setWorkingDirectory: WARNING! ceph_setcwd failed for some reason on path " + abs_path, ceph.WARN); ceph.debug("setWorkingDirectory:exit", ceph.DEBUG); } @@ -273,6 +274,7 @@ public class CephFileSystem extends FileSystem { * @return true if successful, false otherwise * @throws IOException if initialize() hasn't been called. */ + @Override public boolean mkdirs(Path path, FsPermission perms) throws IOException { if (!initialized) throw new IOException ("You have to initialize the " +"CephFileSystem before calling other methods."); @@ -280,10 +282,17 @@ public class CephFileSystem extends FileSystem { Path abs_path = makeAbsolute(path); ceph.debug("mkdirs:calling ceph_mkdirs from Java", ceph.TRACE); int result = ceph.ceph_mkdirs(abs_path.toString(), (int)perms.toShort()); - ceph.debug("mkdirs:exit with result " + result, ceph.DEBUG); - if (result != 0) + if (result != 0) { + ceph.debug("mkdirs: make directory " + abs_path + + "Failing with result " + result, ceph.WARN); + if (ceph.ENOTDIR == result) + throw new FileAlreadyExistsException("Parent path is not a directory"); return false; - else return true; + } + else { + ceph.debug("mkdirs:exiting succesfully", ceph.DEBUG); + return true; + } } /** @@ -386,7 +395,7 @@ public class CephFileSystem extends FileSystem { public FileStatus[] listStatus(Path path) throws IOException { if (!initialized) throw new IOException ("You have to initialize the " +"CephFileSystem before calling other methods."); - ceph.debug("listStatus:enter with path " + path, ceph.DEBUG); + ceph.debug("listStatus:enter with path " + path, ceph.WARN); Path abs_path = makeAbsolute(path); Path[] paths = listPaths(abs_path); if (paths != null) { @@ -573,6 +582,16 @@ public class CephFileSystem extends FileSystem { Path abs_dst = makeAbsolute(dst); ceph.debug("calling ceph_rename from Java", ceph.TRACE); boolean result = ceph.ceph_rename(abs_src.toString(), abs_dst.toString()); + if (!result) { + if (isDirectory(abs_dst)) { //move the srcdir into destdir + ceph.debug("ceph_rename failed but dst is a directory!", ceph.NOLOG); + Path new_dst = new Path(abs_dst, abs_src.getName()); + result = rename(abs_src, new_dst); + ceph.debug("attempt to move " + abs_src.toString() + + " to " + new_dst.toString() + + "has result:" + result, ceph.NOLOG); + } + } ceph.debug("rename:exit with result: " + result, ceph.DEBUG); return result; } @@ -701,9 +720,6 @@ public class CephFileSystem extends FileSystem { /* The path is a directory, so recursively try to delete its contents, and then delete the directory. */ - if (!recursive) { - throw new IOException("Directories must be deleted recursively!"); - } //get the entries; listPaths will remove . and .. for us Path[] contents = listPaths(abs_path); if (contents == null) { @@ -712,6 +728,9 @@ public class CephFileSystem extends FileSystem { "\" while trying to delete it, BAILING", ceph.ERROR); return false; } + if (!recursive && contents.length > 0) { + throw new IOException("Directories must be deleted recursively!"); + } // delete the entries ceph.debug("delete: recursively calling delete on contents of " + abs_path, ceph.DEBUG); @@ -791,7 +810,7 @@ public class CephFileSystem extends FileSystem { Path[] paths = new Path[dirlist.length]; for (int i = 0; i < dirlist.length; ++i) { ceph.debug("Raw enumeration of paths in \"" + abs_path.toString() + "\": \"" + - dirlist[i] + "\"", ceph.NOLOG); + dirlist[i] + "\"", ceph.TRACE); // convert each listing to an absolute path Path raw_path = new Path(dirlist[i]); if (raw_path.isAbsolute()) -- 2.39.5