From 3425a8e5031a4f0c9c0eb85e8a329b02d05b9420 Mon Sep 17 00:00:00 2001 From: Colin Patrick McCabe Date: Tue, 10 May 2011 16:45:39 -0700 Subject: [PATCH] rados tool: integrate rados_sync with rados tool * integrate rados_sync with rados_tool * Improve rados tool usage a bit * Rename test_rados_sync.sh to test_rados_tool.sh Signed-off-by: Colin McCabe --- src/Makefile.am | 6 +- src/rados.cc | 125 ++++++++++-------- src/rados_sync.cc | 69 +++------- ...{test_rados_sync.sh => test_rados_tool.sh} | 33 +++-- 4 files changed, 103 insertions(+), 130 deletions(-) rename src/test/{test_rados_sync.sh => test_rados_tool.sh} (76%) diff --git a/src/Makefile.am b/src/Makefile.am index 4a786bcfbea0f..05ba14702f536 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -260,14 +260,10 @@ librbd_a_SOURCES = ${librbd_SOURCES} librbd_a_CFLAGS = ${AM_CFLAGS} librbd_a_CXXFLAGS = ${AM_CXXFLAGS} -rados_SOURCES = rados.cc +rados_SOURCES = rados.cc rados_sync.cc rados_LDADD = librados.la -lpthread -lm $(CRYPTO_LIBS) $(EXTRALIBS) bin_PROGRAMS += rados -rados_sync_SOURCES = rados_sync.cc -rados_sync_LDADD = librados.la -lpthread -lm $(CRYPTO_LIBS) $(EXTRALIBS) -bin_PROGRAMS += rados_sync - testrados_SOURCES = testrados.c testrados_LDADD = librados.la -lpthread -lm $(CRYPTO_LIBS) $(EXTRALIBS) testradospp_SOURCES = testradospp.cc diff --git a/src/rados.cc b/src/rados.cc index 758c1a537d105..e3f4ded005201 100644 --- a/src/rados.cc +++ b/src/rados.cc @@ -33,58 +33,63 @@ using namespace librados; #include #include +int rados_tool_sync(const std::map < std::string, std::string > &opts, + std::vector &args); + void usage() { - cerr << "usage: rados [options] [commands]" << std::endl; - /* cerr << "If no commands are specified, enter interactive mode.\n"; - cerr << "Commands:" << std::endl; - cerr << " stop -- cleanly shut down file system" << std::endl - << " (osd|pg|mds) stat -- get monitor subsystem status" << std::endl - << " ..." << std::endl; - */ - cerr << "Commands:\n"; - cerr << " lspools list pools\n"; - cerr << " df show per-pool and total usage\n\n"; - - cerr << "Pool commands:\n"; - cerr << " get [outfile] fetch object\n"; - cerr << " put [infile] write object\n"; - cerr << " create create object\n"; - cerr << " rm remove object\n"; - cerr << " listxattr \n"; - cerr << " getxattr attr\n"; - cerr << " setxattr attr val\n"; - cerr << " rmxattr attr\n"; - cerr << " stat objname stat the named object\n"; - cerr << " ls list objects in pool\n\n"; - cerr << " chown 123 change the pool owner to auid 123\n"; - cerr << " mapext \n"; - cerr << " mkpool [123[ 4]] create pool '\n" - << " [with auid 123[and using crush rule 4]]\n"; - cerr << " rmpool remove pool '\n"; - cerr << " mkpool create the pool \n"; - cerr << " lssnap list snaps\n"; - cerr << " mksnap create snap \n"; - cerr << " rmsnap remove snap \n"; - cerr << " rollback roll back object to snap \n\n"; - cerr << " bench write|seq|rand [-t concurrent_operations]\n"; - cerr << " default is 16 concurrent IOs and 4 MB op size\n\n"; - - cerr << "Options:\n"; - cerr << " -p pool\n"; - cerr << " --pool=pool\n"; - cerr << " select given pool by name\n"; - cerr << " -b op_size\n"; - cerr << " set the size of write ops for put or benchmarking"; - cerr << " -s name\n"; - cerr << " --snap name\n"; - cerr << " select given snap name for (read) IO\n"; - cerr << " -i infile\n"; - cerr << " -o outfile\n"; - cerr << " specify input or output file (for certain commands)\n"; - cerr << " --create-pool\n"; - cerr << " create the pool that was specified\n"; - exit(1); + cerr << \ +"usage: rados [options] [commands]\n" +"POOL COMMANDS\n" +" lspools list pools\n" +" mkpool [123[ 4]] create pool '\n" +" [with auid 123[and using crush rule 4]]\n" +" rmpool remove pool '\n" +" mkpool create the pool \n" +" df show per-pool and total usage\n" +" ls list objects in pool\n\n" +" chown 123 change the pool owner to auid 123\n" +"OBJECT COMMANDS\n" +" get [outfile] fetch object\n" +" put [infile] write object\n" +" create create object\n" +" rm remove object\n" +" listxattr \n" +" getxattr attr\n" +" setxattr attr val\n" +" rmxattr attr\n" +" stat objname stat the named object\n" +" mapext \n" +" lssnap list snaps\n" +" mksnap create snap \n" +" rmsnap remove snap \n" +" rollback roll back object to snap \n\n" +" bench write|seq|rand [-t concurrent_operations]\n" +" default is 16 concurrent IOs and 4 MB op size\n\n" +"IMPORT AND EXPORT\n" +" import [options] \n" +" Upload to \n" +" export [options] rados-pool> \n" +" Download to \n" +" options:\n" +" -f / --force Copy everything, even if it hasn't changed.\n" +" -d / --delete-after After synchronizing, delete unreferenced\n" +" files or objects from the target bucket\n" +" or directory.\n" +"GLOBAL OPTIONS:\n" +" -p pool\n" +" --pool=pool\n" +" select given pool by name\n" +" -b op_size\n" +" set the size of write ops for put or benchmarking" +" -s name\n" +" --snap name\n" +" select given snap name for (read) IO\n" +" -i infile\n" +" -o outfile\n" +" specify input or output file (for certain commands)\n" +" --create\n" +" create the pool or directory that was specified\n"; } static int do_get(IoCtx& io_ctx, const char *objname, const char *outfile, bool check_stdio) @@ -196,11 +201,6 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts, snapid = strtoll(i->second.c_str(), NULL, 10); } - if (nargs.empty()) { - usage(); - return 1; - } - // open rados Rados rados; ret = rados.init_with_config(&g_conf); @@ -261,6 +261,8 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts, cout << "selected snap " << snapid << " '" << snapname << "'" << std::endl; } + assert(!nargs.empty()); + // list pools? if (strcmp(nargs[0], "lspools") == 0) { list vec; @@ -666,6 +668,10 @@ int main(int argc, const char **argv) if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) { usage(); exit(0); + } else if (ceph_argparse_flag(args, i, "-f", "--force", (char*)NULL)) { + opts["force"] = "true"; + } else if (ceph_argparse_flag(args, i, "-d", "--delete-after", (char*)NULL)) { + opts["delete-after"] = "true"; } else if (ceph_argparse_flag(args, i, "-C", "--create", "--create-pool", (char*)NULL)) { opts["create"] = "true"; @@ -683,5 +689,12 @@ int main(int argc, const char **argv) } } - return rados_tool_common(opts, args); + if (args.empty()) { + cerr << "rados: you must give an action. Try --help" << std::endl; + return 1; + } + if ((strcmp(args[0], "import") == 0) || (strcmp(args[0], "export") == 0)) + return rados_tool_sync(opts, args); + else + return rados_tool_common(opts, args); } diff --git a/src/rados_sync.cc b/src/rados_sync.cc index 54db7c09416dc..0c8d3ca56054f 100644 --- a/src/rados_sync.cc +++ b/src/rados_sync.cc @@ -46,22 +46,6 @@ static const char ERR_PREFIX[] = "[ERROR] "; #define ENOATTR ENODATA #endif -static void usage() -{ - cerr << "usage:\n\ -\n\ -Importing data from a local directory to a rados pool:\n\ -rados_sync [options] import \n\ -\n\ -Exporting data from a rados pool to a local directory:\n\ -rados_sync [options] export \n\ -\n\ -options:\n\ --h or --help This help message\n\ --c or --create Create destination pools or directories that don't exist\n\ -"; -} - static int xattr_test(const char *dir_name) { int ret; @@ -961,41 +945,22 @@ static int do_import(IoCtx& io_ctx, const char *dir_name, return 0; } -int main(int argc, const char **argv) +int rados_tool_sync(const std::map < std::string, std::string > &opts, + std::vector &args) { int ret; - bool create = false; - bool force = false; - bool delete_after = false; - vector args; + bool force = opts.count("force"); + bool delete_after = opts.count("delete-after"); + bool create = opts.count("create"); std::string action, src, dst; - argv_to_vec(argc, argv, args); - env_to_vec(args); - common_init(args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); - - std::vector::iterator i; - for (i = args.begin(); i != args.end(); ) { - if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) { - usage(); - exit(1); - } else if (ceph_argparse_flag(args, i, "-C", "--create", (char*)NULL)) { - create = true; - } else if (ceph_argparse_flag(args, i, "-f", "--force", (char*)NULL)) { - force = true; - } else if (ceph_argparse_flag(args, i, "-d", "--delete-after", (char*)NULL)) { - delete_after = true; - } else { - // begin positional arguments - break; - } - } + std::vector::iterator i = args.begin(); if ((i != args.end()) && ((strcmp(*i, "import") == 0) || (strcmp(*i, "export") == 0))) { action = *i; ++i; } else { - cerr << argv[0] << ": You must specify either 'import' or 'export'.\n"; + cerr << "rados" << ": You must specify either 'import' or 'export'.\n"; cerr << "Use --help to show help.\n"; exit(1); } @@ -1004,7 +969,7 @@ int main(int argc, const char **argv) ++i; } else { - cerr << argv[0] << ": You must give a source.\n"; + cerr << "rados" << ": You must give a source.\n"; cerr << "Use --help to show help.\n"; exit(1); } @@ -1013,7 +978,7 @@ int main(int argc, const char **argv) ++i; } else { - cerr << argv[0] << ": You must give a destination.\n"; + cerr << "rados" << ": You must give a destination.\n"; cerr << "Use --help to show help.\n"; exit(1); } @@ -1021,11 +986,11 @@ int main(int argc, const char **argv) // open rados Rados rados; if (rados.init_with_config(&g_conf) < 0) { - cerr << argv[0] << ": failed to initialize Rados!" << std::endl; + cerr << "rados" << ": failed to initialize Rados!" << std::endl; exit(1); } if (rados.connect() < 0) { - cerr << argv[0] << ": failed to connect to Rados cluster!" << std::endl; + cerr << "rados" << ": failed to connect to Rados cluster!" << std::endl; exit(1); } IoCtx io_ctx; @@ -1035,20 +1000,20 @@ int main(int argc, const char **argv) if (create) { ret = rados.pool_create(pool_name.c_str()); if (ret) { - cerr << argv[0] << ": pool_create failed with error " << ret + cerr << "rados" << ": pool_create failed with error " << ret << std::endl; exit(ret); } ret = rados.ioctx_create(pool_name.c_str(), io_ctx); } else { - cerr << argv[0] << ": pool '" << pool_name << "' does not exist. Use " + cerr << "rados" << ": pool '" << pool_name << "' does not exist. Use " << "--create to try to create it." << std::endl; exit(ENOENT); } } if (ret < 0) { - cerr << argv[0] << ": error opening pool " << pool_name << ": " + cerr << "rados" << ": error opening pool " << pool_name << ": " << cpp_strerror(ret) << std::endl; exit(ret); } @@ -1057,7 +1022,7 @@ int main(int argc, const char **argv) if (action == "import") { if (access(dir_name.c_str(), R_OK)) { - cerr << argv[0] << ": source directory '" << dst + cerr << "rados" << ": source directory '" << dst << "' appears to be inaccessible." << std::endl; exit(ENOENT); } @@ -1072,13 +1037,13 @@ int main(int argc, const char **argv) ret = mkdir(dst.c_str(), 0700); if (ret < 0) { ret = errno; - cerr << argv[0] << ": mkdir(" << dst << ") failed with error " << ret + cerr << "rados" << ": mkdir(" << dst << ") failed with error " << ret << std::endl; exit(ret); } } else { - cerr << argv[0] << ": directory '" << dst << "' is not accessible. Use " + cerr << "rados" << ": directory '" << dst << "' is not accessible. Use " << "--create to try to create it.\n"; exit(ENOENT); } diff --git a/src/test/test_rados_sync.sh b/src/test/test_rados_tool.sh similarity index 76% rename from src/test/test_rados_sync.sh rename to src/test/test_rados_tool.sh index cc3b7ca1e647a..83621506380cc 100755 --- a/src/test/test_rados_sync.sh +++ b/src/test/test_rados_tool.sh @@ -7,7 +7,7 @@ die() { usage() { cat </dev/null [ $? -eq 0 ] || die "you must install the 'attr' tool to manipulate \ @@ -46,7 +45,7 @@ extended attributes." while getopts "c:hkp:" flag; do case $flag in - c) RADOS_SYNC="$RADOS_SYNC -c $OPTARG";; + c) RADOS_TOOL="$RADOS_TOOL -c $OPTARG";; k) KEEP_TEMP_FILES=1;; h) usage; exit 0;; p) POOL=$OPTARG;; @@ -54,7 +53,7 @@ while getopts "c:hkp:" flag; do esac done -TDIR=`mktemp -d -t test_rados_sync.XXXXXXXXXX` || die "mktemp failed" +TDIR=`mktemp -d -t test_rados_tool.XXXXXXXXXX` || die "mktemp failed" [ $KEEP_TEMP_FILES -eq 0 ] && trap "rm -rf ${TDIR}; exit" INT TERM EXIT mkdir "$TDIR/dira" @@ -77,36 +76,36 @@ attr -q -s rados_full_name -V "foo2" "$TDIR/dirc/00003036_foo2" # make sure that --create works run "$RADOS_TOOL" rmpool "$POOL" -run_expect_succ "$RADOS_SYNC" --create import "$TDIR/dira" "$POOL" +run_expect_succ "$RADOS_TOOL" --create import "$TDIR/dira" "$POOL" # make sure that lack of --create fails run_expect_succ "$RADOS_TOOL" rmpool "$POOL" -run_expect_fail "$RADOS_SYNC" import "$TDIR/dira" "$POOL" +run_expect_fail "$RADOS_TOOL" import "$TDIR/dira" "$POOL" -run_expect_succ "$RADOS_SYNC" --create import "$TDIR/dira" "$POOL" +run_expect_succ "$RADOS_TOOL" --create import "$TDIR/dira" "$POOL" # inaccessible import src should fail -run_expect_fail "$RADOS_SYNC" import "$TDIR/dir_nonexistent" "$POOL" +run_expect_fail "$RADOS_TOOL" import "$TDIR/dir_nonexistent" "$POOL" # export some stuff -run_expect_succ "$RADOS_SYNC" --create export "$POOL" "$TDIR/dirb" +run_expect_succ "$RADOS_TOOL" --create export "$POOL" "$TDIR/dirb" diff -q -r "$TDIR/dira" "$TDIR/dirb" \ || die "failed to export the same stuff we imported!" # import some stuff with extended attributes on it -run_expect_succ "$RADOS_SYNC" import "$TDIR/dirc" "$POOL" | tee "$TDIR/out" +run_expect_succ "$RADOS_TOOL" import "$TDIR/dirc" "$POOL" | tee "$TDIR/out" run_expect_succ grep -q '\[xattr\]' $TDIR/out # the second time, the xattrs should match, so there should be nothing to do. -run_expect_succ "$RADOS_SYNC" import "$TDIR/dirc" "$POOL" | tee "$TDIR/out" +run_expect_succ "$RADOS_TOOL" import "$TDIR/dirc" "$POOL" | tee "$TDIR/out" run_expect_fail grep -q '\[xattr\]' "$TDIR/out" # now force it to copy everything -run_expect_succ "$RADOS_SYNC" --force import "$TDIR/dirc" "$POOL" | tee "$TDIR/out2" +run_expect_succ "$RADOS_TOOL" --force import "$TDIR/dirc" "$POOL" | tee "$TDIR/out2" run_expect_succ grep '\[force\]' "$TDIR/out2" # export some stuff with extended attributes on it -run_expect_succ "$RADOS_SYNC" -C export "$POOL" "$TDIR/dirc_copy" +run_expect_succ "$RADOS_TOOL" -C export "$POOL" "$TDIR/dirc_copy" # check to make sure extended attributes were preserved PRE_EXPORT=`attr -qg rados.toothbrush "$TDIR/dirc/000029c4_foo"` @@ -117,13 +116,13 @@ if [ "$PRE_EXPORT" != "$POST_EXPORT" ]; then fi # trigger a rados delete using --delete-after -run_expect_succ "$RADOS_SYNC" --create export "$POOL" "$TDIR/dird" +run_expect_succ "$RADOS_TOOL" --create export "$POOL" "$TDIR/dird" rm -f "$TDIR/dird/000029c4_foo" -run_expect_succ "$RADOS_SYNC" --delete-after import "$TDIR/dird" "$POOL" | tee "$TDIR/out3" +run_expect_succ "$RADOS_TOOL" --delete-after import "$TDIR/dird" "$POOL" | tee "$TDIR/out3" run_expect_succ grep '\[deleted\]' "$TDIR/out3" # trigger a local delete using --delete-after -run_expect_succ "$RADOS_SYNC" --delete-after export "$POOL" "$TDIR/dirc" | tee "$TDIR/out4" +run_expect_succ "$RADOS_TOOL" --delete-after export "$POOL" "$TDIR/dirc" | tee "$TDIR/out4" run_expect_succ grep '\[deleted\]' "$TDIR/out4" [ -e "$TDIR/dird/000029c4_foo" ] && die "--delete-after failed to delete a file!" -- 2.39.5