From: Colin Patrick McCabe Date: Tue, 10 May 2011 23:45:39 +0000 (-0700) Subject: rados tool: integrate rados_sync with rados tool X-Git-Tag: v0.29~139^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=3425a8e5031a4f0c9c0eb85e8a329b02d05b9420;p=ceph.git rados tool: integrate rados_sync with rados tool * integrate rados_sync with rados_tool * Improve rados tool usage a bit * Rename test_rados_sync.sh to test_rados_tool.sh Signed-off-by: Colin McCabe --- diff --git a/src/Makefile.am b/src/Makefile.am index 4a786bcfbea0..05ba14702f53 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -260,14 +260,10 @@ librbd_a_SOURCES = ${librbd_SOURCES} librbd_a_CFLAGS = ${AM_CFLAGS} librbd_a_CXXFLAGS = ${AM_CXXFLAGS} -rados_SOURCES = rados.cc +rados_SOURCES = rados.cc rados_sync.cc rados_LDADD = librados.la -lpthread -lm $(CRYPTO_LIBS) $(EXTRALIBS) bin_PROGRAMS += rados -rados_sync_SOURCES = rados_sync.cc -rados_sync_LDADD = librados.la -lpthread -lm $(CRYPTO_LIBS) $(EXTRALIBS) -bin_PROGRAMS += rados_sync - testrados_SOURCES = testrados.c testrados_LDADD = librados.la -lpthread -lm $(CRYPTO_LIBS) $(EXTRALIBS) testradospp_SOURCES = testradospp.cc diff --git a/src/rados.cc b/src/rados.cc index 758c1a537d10..e3f4ded00520 100644 --- a/src/rados.cc +++ b/src/rados.cc @@ -33,58 +33,63 @@ using namespace librados; #include #include +int rados_tool_sync(const std::map < std::string, std::string > &opts, + std::vector &args); + void usage() { - cerr << "usage: rados [options] [commands]" << std::endl; - /* cerr << "If no commands are specified, enter interactive mode.\n"; - cerr << "Commands:" << std::endl; - cerr << " stop -- cleanly shut down file system" << std::endl - << " (osd|pg|mds) stat -- get monitor subsystem status" << std::endl - << " ..." << std::endl; - */ - cerr << "Commands:\n"; - cerr << " lspools list pools\n"; - cerr << " df show per-pool and total usage\n\n"; - - cerr << "Pool commands:\n"; - cerr << " get [outfile] fetch object\n"; - cerr << " put [infile] write object\n"; - cerr << " create create object\n"; - cerr << " rm remove object\n"; - cerr << " listxattr \n"; - cerr << " getxattr attr\n"; - cerr << " setxattr attr val\n"; - cerr << " rmxattr attr\n"; - cerr << " stat objname stat the named object\n"; - cerr << " ls list objects in pool\n\n"; - cerr << " chown 123 change the pool owner to auid 123\n"; - cerr << " mapext \n"; - cerr << " mkpool [123[ 4]] create pool '\n" - << " [with auid 123[and using crush rule 4]]\n"; - cerr << " rmpool remove pool '\n"; - cerr << " mkpool create the pool \n"; - cerr << " lssnap list snaps\n"; - cerr << " mksnap create snap \n"; - cerr << " rmsnap remove snap \n"; - cerr << " rollback roll back object to snap \n\n"; - cerr << " bench write|seq|rand [-t concurrent_operations]\n"; - cerr << " default is 16 concurrent IOs and 4 MB op size\n\n"; - - cerr << "Options:\n"; - cerr << " -p pool\n"; - cerr << " --pool=pool\n"; - cerr << " select given pool by name\n"; - cerr << " -b op_size\n"; - cerr << " set the size of write ops for put or benchmarking"; - cerr << " -s name\n"; - cerr << " --snap name\n"; - cerr << " select given snap name for (read) IO\n"; - cerr << " -i infile\n"; - cerr << " -o outfile\n"; - cerr << " specify input or output file (for certain commands)\n"; - cerr << " --create-pool\n"; - cerr << " create the pool that was specified\n"; - exit(1); + cerr << \ +"usage: rados [options] [commands]\n" +"POOL COMMANDS\n" +" lspools list pools\n" +" mkpool [123[ 4]] create pool '\n" +" [with auid 123[and using crush rule 4]]\n" +" rmpool remove pool '\n" +" mkpool create the pool \n" +" df show per-pool and total usage\n" +" ls list objects in pool\n\n" +" chown 123 change the pool owner to auid 123\n" +"OBJECT COMMANDS\n" +" get [outfile] fetch object\n" +" put [infile] write object\n" +" create create object\n" +" rm remove object\n" +" listxattr \n" +" getxattr attr\n" +" setxattr attr val\n" +" rmxattr attr\n" +" stat objname stat the named object\n" +" mapext \n" +" lssnap list snaps\n" +" mksnap create snap \n" +" rmsnap remove snap \n" +" rollback roll back object to snap \n\n" +" bench write|seq|rand [-t concurrent_operations]\n" +" default is 16 concurrent IOs and 4 MB op size\n\n" +"IMPORT AND EXPORT\n" +" import [options] \n" +" Upload to \n" +" export [options] rados-pool> \n" +" Download to \n" +" options:\n" +" -f / --force Copy everything, even if it hasn't changed.\n" +" -d / --delete-after After synchronizing, delete unreferenced\n" +" files or objects from the target bucket\n" +" or directory.\n" +"GLOBAL OPTIONS:\n" +" -p pool\n" +" --pool=pool\n" +" select given pool by name\n" +" -b op_size\n" +" set the size of write ops for put or benchmarking" +" -s name\n" +" --snap name\n" +" select given snap name for (read) IO\n" +" -i infile\n" +" -o outfile\n" +" specify input or output file (for certain commands)\n" +" --create\n" +" create the pool or directory that was specified\n"; } static int do_get(IoCtx& io_ctx, const char *objname, const char *outfile, bool check_stdio) @@ -196,11 +201,6 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts, snapid = strtoll(i->second.c_str(), NULL, 10); } - if (nargs.empty()) { - usage(); - return 1; - } - // open rados Rados rados; ret = rados.init_with_config(&g_conf); @@ -261,6 +261,8 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts, cout << "selected snap " << snapid << " '" << snapname << "'" << std::endl; } + assert(!nargs.empty()); + // list pools? if (strcmp(nargs[0], "lspools") == 0) { list vec; @@ -666,6 +668,10 @@ int main(int argc, const char **argv) if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) { usage(); exit(0); + } else if (ceph_argparse_flag(args, i, "-f", "--force", (char*)NULL)) { + opts["force"] = "true"; + } else if (ceph_argparse_flag(args, i, "-d", "--delete-after", (char*)NULL)) { + opts["delete-after"] = "true"; } else if (ceph_argparse_flag(args, i, "-C", "--create", "--create-pool", (char*)NULL)) { opts["create"] = "true"; @@ -683,5 +689,12 @@ int main(int argc, const char **argv) } } - return rados_tool_common(opts, args); + if (args.empty()) { + cerr << "rados: you must give an action. Try --help" << std::endl; + return 1; + } + if ((strcmp(args[0], "import") == 0) || (strcmp(args[0], "export") == 0)) + return rados_tool_sync(opts, args); + else + return rados_tool_common(opts, args); } diff --git a/src/rados_sync.cc b/src/rados_sync.cc index 54db7c09416d..0c8d3ca56054 100644 --- a/src/rados_sync.cc +++ b/src/rados_sync.cc @@ -46,22 +46,6 @@ static const char ERR_PREFIX[] = "[ERROR] "; #define ENOATTR ENODATA #endif -static void usage() -{ - cerr << "usage:\n\ -\n\ -Importing data from a local directory to a rados pool:\n\ -rados_sync [options] import \n\ -\n\ -Exporting data from a rados pool to a local directory:\n\ -rados_sync [options] export \n\ -\n\ -options:\n\ --h or --help This help message\n\ --c or --create Create destination pools or directories that don't exist\n\ -"; -} - static int xattr_test(const char *dir_name) { int ret; @@ -961,41 +945,22 @@ static int do_import(IoCtx& io_ctx, const char *dir_name, return 0; } -int main(int argc, const char **argv) +int rados_tool_sync(const std::map < std::string, std::string > &opts, + std::vector &args) { int ret; - bool create = false; - bool force = false; - bool delete_after = false; - vector args; + bool force = opts.count("force"); + bool delete_after = opts.count("delete-after"); + bool create = opts.count("create"); std::string action, src, dst; - argv_to_vec(argc, argv, args); - env_to_vec(args); - common_init(args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); - - std::vector::iterator i; - for (i = args.begin(); i != args.end(); ) { - if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) { - usage(); - exit(1); - } else if (ceph_argparse_flag(args, i, "-C", "--create", (char*)NULL)) { - create = true; - } else if (ceph_argparse_flag(args, i, "-f", "--force", (char*)NULL)) { - force = true; - } else if (ceph_argparse_flag(args, i, "-d", "--delete-after", (char*)NULL)) { - delete_after = true; - } else { - // begin positional arguments - break; - } - } + std::vector::iterator i = args.begin(); if ((i != args.end()) && ((strcmp(*i, "import") == 0) || (strcmp(*i, "export") == 0))) { action = *i; ++i; } else { - cerr << argv[0] << ": You must specify either 'import' or 'export'.\n"; + cerr << "rados" << ": You must specify either 'import' or 'export'.\n"; cerr << "Use --help to show help.\n"; exit(1); } @@ -1004,7 +969,7 @@ int main(int argc, const char **argv) ++i; } else { - cerr << argv[0] << ": You must give a source.\n"; + cerr << "rados" << ": You must give a source.\n"; cerr << "Use --help to show help.\n"; exit(1); } @@ -1013,7 +978,7 @@ int main(int argc, const char **argv) ++i; } else { - cerr << argv[0] << ": You must give a destination.\n"; + cerr << "rados" << ": You must give a destination.\n"; cerr << "Use --help to show help.\n"; exit(1); } @@ -1021,11 +986,11 @@ int main(int argc, const char **argv) // open rados Rados rados; if (rados.init_with_config(&g_conf) < 0) { - cerr << argv[0] << ": failed to initialize Rados!" << std::endl; + cerr << "rados" << ": failed to initialize Rados!" << std::endl; exit(1); } if (rados.connect() < 0) { - cerr << argv[0] << ": failed to connect to Rados cluster!" << std::endl; + cerr << "rados" << ": failed to connect to Rados cluster!" << std::endl; exit(1); } IoCtx io_ctx; @@ -1035,20 +1000,20 @@ int main(int argc, const char **argv) if (create) { ret = rados.pool_create(pool_name.c_str()); if (ret) { - cerr << argv[0] << ": pool_create failed with error " << ret + cerr << "rados" << ": pool_create failed with error " << ret << std::endl; exit(ret); } ret = rados.ioctx_create(pool_name.c_str(), io_ctx); } else { - cerr << argv[0] << ": pool '" << pool_name << "' does not exist. Use " + cerr << "rados" << ": pool '" << pool_name << "' does not exist. Use " << "--create to try to create it." << std::endl; exit(ENOENT); } } if (ret < 0) { - cerr << argv[0] << ": error opening pool " << pool_name << ": " + cerr << "rados" << ": error opening pool " << pool_name << ": " << cpp_strerror(ret) << std::endl; exit(ret); } @@ -1057,7 +1022,7 @@ int main(int argc, const char **argv) if (action == "import") { if (access(dir_name.c_str(), R_OK)) { - cerr << argv[0] << ": source directory '" << dst + cerr << "rados" << ": source directory '" << dst << "' appears to be inaccessible." << std::endl; exit(ENOENT); } @@ -1072,13 +1037,13 @@ int main(int argc, const char **argv) ret = mkdir(dst.c_str(), 0700); if (ret < 0) { ret = errno; - cerr << argv[0] << ": mkdir(" << dst << ") failed with error " << ret + cerr << "rados" << ": mkdir(" << dst << ") failed with error " << ret << std::endl; exit(ret); } } else { - cerr << argv[0] << ": directory '" << dst << "' is not accessible. Use " + cerr << "rados" << ": directory '" << dst << "' is not accessible. Use " << "--create to try to create it.\n"; exit(ENOENT); } diff --git a/src/test/test_rados_sync.sh b/src/test/test_rados_sync.sh deleted file mode 100755 index cc3b7ca1e647..000000000000 --- a/src/test/test_rados_sync.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/bash - -die() { - echo "$@" - exit 1 -} - -usage() { - cat </dev/null -[ $? -eq 0 ] || die "you must install the 'attr' tool to manipulate \ -extended attributes." - -while getopts "c:hkp:" flag; do - case $flag in - c) RADOS_SYNC="$RADOS_SYNC -c $OPTARG";; - k) KEEP_TEMP_FILES=1;; - h) usage; exit 0;; - p) POOL=$OPTARG;; - *) echo; usage; exit 1;; - esac -done - -TDIR=`mktemp -d -t test_rados_sync.XXXXXXXXXX` || die "mktemp failed" -[ $KEEP_TEMP_FILES -eq 0 ] && trap "rm -rf ${TDIR}; exit" INT TERM EXIT - -mkdir "$TDIR/dira" -touch "$TDIR/dira/000029c4_foo" -attr -q -s rados_full_name -V "foo" "$TDIR/dira/000029c4_foo" -touch "$TDIR/dira/00003036_foo2" -attr -q -s rados_full_name -V "foo2" "$TDIR/dira/00003036_foo2" -touch "$TDIR/dira/000027d5_bar" -attr -q -s rados_full_name -V "bar" "$TDIR/dira/000027d5_bar" -mkdir "$TDIR/dirb" -mkdir "$TDIR/dirc" -touch "$TDIR/dirc/000029c4_foo" -attr -q -s rados_full_name -V "foo" "$TDIR/dirc/000029c4_foo" -attr -q -s "rados.toothbrush" -V "toothbrush" "$TDIR/dirc/000029c4_foo" -attr -q -s "rados.toothpaste" -V "crest" "$TDIR/dirc/000029c4_foo" -attr -q -s "rados.floss" -V "myfloss" "$TDIR/dirc/000029c4_foo" -touch "$TDIR/dirc/00003036_foo2" -attr -q -s "rados.toothbrush" -V "green" "$TDIR/dirc/00003036_foo2" -attr -q -s rados_full_name -V "foo2" "$TDIR/dirc/00003036_foo2" - -# make sure that --create works -run "$RADOS_TOOL" rmpool "$POOL" -run_expect_succ "$RADOS_SYNC" --create import "$TDIR/dira" "$POOL" - -# make sure that lack of --create fails -run_expect_succ "$RADOS_TOOL" rmpool "$POOL" -run_expect_fail "$RADOS_SYNC" import "$TDIR/dira" "$POOL" - -run_expect_succ "$RADOS_SYNC" --create import "$TDIR/dira" "$POOL" - -# inaccessible import src should fail -run_expect_fail "$RADOS_SYNC" import "$TDIR/dir_nonexistent" "$POOL" - -# export some stuff -run_expect_succ "$RADOS_SYNC" --create export "$POOL" "$TDIR/dirb" -diff -q -r "$TDIR/dira" "$TDIR/dirb" \ - || die "failed to export the same stuff we imported!" - -# import some stuff with extended attributes on it -run_expect_succ "$RADOS_SYNC" import "$TDIR/dirc" "$POOL" | tee "$TDIR/out" -run_expect_succ grep -q '\[xattr\]' $TDIR/out - -# the second time, the xattrs should match, so there should be nothing to do. -run_expect_succ "$RADOS_SYNC" import "$TDIR/dirc" "$POOL" | tee "$TDIR/out" -run_expect_fail grep -q '\[xattr\]' "$TDIR/out" - -# now force it to copy everything -run_expect_succ "$RADOS_SYNC" --force import "$TDIR/dirc" "$POOL" | tee "$TDIR/out2" -run_expect_succ grep '\[force\]' "$TDIR/out2" - -# export some stuff with extended attributes on it -run_expect_succ "$RADOS_SYNC" -C export "$POOL" "$TDIR/dirc_copy" - -# check to make sure extended attributes were preserved -PRE_EXPORT=`attr -qg rados.toothbrush "$TDIR/dirc/000029c4_foo"` -POST_EXPORT=`attr -qg rados.toothbrush "$TDIR/dirc_copy/000029c4_foo"` -if [ "$PRE_EXPORT" != "$POST_EXPORT" ]; then - die "xattr not preserved across import/export! \ -\$PRE_EXPORT = $PRE_EXPORT, \$POST_EXPORT = $POST_EXPORT" -fi - -# trigger a rados delete using --delete-after -run_expect_succ "$RADOS_SYNC" --create export "$POOL" "$TDIR/dird" -rm -f "$TDIR/dird/000029c4_foo" -run_expect_succ "$RADOS_SYNC" --delete-after import "$TDIR/dird" "$POOL" | tee "$TDIR/out3" -run_expect_succ grep '\[deleted\]' "$TDIR/out3" - -# trigger a local delete using --delete-after -run_expect_succ "$RADOS_SYNC" --delete-after export "$POOL" "$TDIR/dirc" | tee "$TDIR/out4" -run_expect_succ grep '\[deleted\]' "$TDIR/out4" -[ -e "$TDIR/dird/000029c4_foo" ] && die "--delete-after failed to delete a file!" - -echo "SUCCESS!" -exit 0 diff --git a/src/test/test_rados_tool.sh b/src/test/test_rados_tool.sh new file mode 100755 index 000000000000..83621506380c --- /dev/null +++ b/src/test/test_rados_tool.sh @@ -0,0 +1,130 @@ +#!/bin/bash + +die() { + echo "$@" + exit 1 +} + +usage() { + cat </dev/null +[ $? -eq 0 ] || die "you must install the 'attr' tool to manipulate \ +extended attributes." + +while getopts "c:hkp:" flag; do + case $flag in + c) RADOS_TOOL="$RADOS_TOOL -c $OPTARG";; + k) KEEP_TEMP_FILES=1;; + h) usage; exit 0;; + p) POOL=$OPTARG;; + *) echo; usage; exit 1;; + esac +done + +TDIR=`mktemp -d -t test_rados_tool.XXXXXXXXXX` || die "mktemp failed" +[ $KEEP_TEMP_FILES -eq 0 ] && trap "rm -rf ${TDIR}; exit" INT TERM EXIT + +mkdir "$TDIR/dira" +touch "$TDIR/dira/000029c4_foo" +attr -q -s rados_full_name -V "foo" "$TDIR/dira/000029c4_foo" +touch "$TDIR/dira/00003036_foo2" +attr -q -s rados_full_name -V "foo2" "$TDIR/dira/00003036_foo2" +touch "$TDIR/dira/000027d5_bar" +attr -q -s rados_full_name -V "bar" "$TDIR/dira/000027d5_bar" +mkdir "$TDIR/dirb" +mkdir "$TDIR/dirc" +touch "$TDIR/dirc/000029c4_foo" +attr -q -s rados_full_name -V "foo" "$TDIR/dirc/000029c4_foo" +attr -q -s "rados.toothbrush" -V "toothbrush" "$TDIR/dirc/000029c4_foo" +attr -q -s "rados.toothpaste" -V "crest" "$TDIR/dirc/000029c4_foo" +attr -q -s "rados.floss" -V "myfloss" "$TDIR/dirc/000029c4_foo" +touch "$TDIR/dirc/00003036_foo2" +attr -q -s "rados.toothbrush" -V "green" "$TDIR/dirc/00003036_foo2" +attr -q -s rados_full_name -V "foo2" "$TDIR/dirc/00003036_foo2" + +# make sure that --create works +run "$RADOS_TOOL" rmpool "$POOL" +run_expect_succ "$RADOS_TOOL" --create import "$TDIR/dira" "$POOL" + +# make sure that lack of --create fails +run_expect_succ "$RADOS_TOOL" rmpool "$POOL" +run_expect_fail "$RADOS_TOOL" import "$TDIR/dira" "$POOL" + +run_expect_succ "$RADOS_TOOL" --create import "$TDIR/dira" "$POOL" + +# inaccessible import src should fail +run_expect_fail "$RADOS_TOOL" import "$TDIR/dir_nonexistent" "$POOL" + +# export some stuff +run_expect_succ "$RADOS_TOOL" --create export "$POOL" "$TDIR/dirb" +diff -q -r "$TDIR/dira" "$TDIR/dirb" \ + || die "failed to export the same stuff we imported!" + +# import some stuff with extended attributes on it +run_expect_succ "$RADOS_TOOL" import "$TDIR/dirc" "$POOL" | tee "$TDIR/out" +run_expect_succ grep -q '\[xattr\]' $TDIR/out + +# the second time, the xattrs should match, so there should be nothing to do. +run_expect_succ "$RADOS_TOOL" import "$TDIR/dirc" "$POOL" | tee "$TDIR/out" +run_expect_fail grep -q '\[xattr\]' "$TDIR/out" + +# now force it to copy everything +run_expect_succ "$RADOS_TOOL" --force import "$TDIR/dirc" "$POOL" | tee "$TDIR/out2" +run_expect_succ grep '\[force\]' "$TDIR/out2" + +# export some stuff with extended attributes on it +run_expect_succ "$RADOS_TOOL" -C export "$POOL" "$TDIR/dirc_copy" + +# check to make sure extended attributes were preserved +PRE_EXPORT=`attr -qg rados.toothbrush "$TDIR/dirc/000029c4_foo"` +POST_EXPORT=`attr -qg rados.toothbrush "$TDIR/dirc_copy/000029c4_foo"` +if [ "$PRE_EXPORT" != "$POST_EXPORT" ]; then + die "xattr not preserved across import/export! \ +\$PRE_EXPORT = $PRE_EXPORT, \$POST_EXPORT = $POST_EXPORT" +fi + +# trigger a rados delete using --delete-after +run_expect_succ "$RADOS_TOOL" --create export "$POOL" "$TDIR/dird" +rm -f "$TDIR/dird/000029c4_foo" +run_expect_succ "$RADOS_TOOL" --delete-after import "$TDIR/dird" "$POOL" | tee "$TDIR/out3" +run_expect_succ grep '\[deleted\]' "$TDIR/out3" + +# trigger a local delete using --delete-after +run_expect_succ "$RADOS_TOOL" --delete-after export "$POOL" "$TDIR/dirc" | tee "$TDIR/out4" +run_expect_succ grep '\[deleted\]' "$TDIR/out4" +[ -e "$TDIR/dird/000029c4_foo" ] && die "--delete-after failed to delete a file!" + +echo "SUCCESS!" +exit 0