From: David Zafman Date: Wed, 6 Aug 2014 01:26:11 +0000 (-0700) Subject: ceph_objectstore_tool, test: Implement import-rados feature and unit test code X-Git-Tag: v0.80.10~69^2~63 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=4f83005bb8a615df370de7b6dfe5d926c7cfef7f;p=ceph.git ceph_objectstore_tool, test: Implement import-rados feature and unit test code Fixes: #8276 Signed-off-by: David Zafman (cherry picked from commit 23ec93a86140c4b271b45d87c62682288079cbba) --- diff --git a/src/test/ceph_objectstore_tool.py b/src/test/ceph_objectstore_tool.py index c836e8390ce86..f88a49c7591f8 100755 --- a/src/test/ceph_objectstore_tool.py +++ b/src/test/ceph_objectstore_tool.py @@ -140,6 +140,34 @@ def get_nspace(num): return "ns{num}".format(num=num) +def verify(DATADIR, POOL, NAME_PREFIX): + TMPFILE = r"/tmp/tmp.{pid}".format(pid=os.getpid()) + nullfd = open(os.devnull, "w") + ERRORS = 0 + for nsfile in [f for f in os.listdir(DATADIR) if f.split('-')[1].find(NAME_PREFIX) == 0]: + nspace = nsfile.split("-")[0] + file = nsfile.split("-")[1] + path = os.path.join(DATADIR, nsfile) + try: + os.unlink(TMPFILE) + except: + pass + cmd = "./rados -p {pool} -N '{nspace}' get {file} {out}".format(pool=POOL, file=file, out=TMPFILE, nspace=nspace) + logging.debug(cmd) + call(cmd, shell=True, stdout=nullfd, stderr=nullfd) + cmd = "diff -q {src} {result}".format(src=path, result=TMPFILE) + logging.debug(cmd) + ret = call(cmd, shell=True) + if ret != 0: + logging.error("{file} data not imported properly".format(file=file)) + ERRORS += 1 + try: + os.unlink(TMPFILE) + except: + pass + return ERRORS + + def main(): sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) nullfd = open(os.devnull, "w") @@ -630,7 +658,6 @@ def main(): ERRORS += IMP_ERRORS logging.debug(cmd) - call("/bin/rm -rf {dir}".format(dir=TESTDIR), shell=True) if EXP_ERRORS == 0 and RM_ERRORS == 0 and IMP_ERRORS == 0: print "Verify replicated import data" @@ -657,37 +684,43 @@ def main(): if ret != 0: logging.error("{file} data not imported properly into {obj}".format(file=file, obj=obj_loc)) ERRORS += 1 + else: + logging.warning("SKIPPING CHECKING IMPORT DATA DUE TO PREVIOUS FAILURES") - vstart(new=False) - wait_for_health() + vstart(new=False) + wait_for_health() + if EXP_ERRORS == 0 and RM_ERRORS == 0 and IMP_ERRORS == 0: print "Verify erasure coded import data" - for nsfile in [f for f in os.listdir(DATADIR) if f.split('-')[1].find(EC_NAME) == 0]: - nspace = nsfile.split("-")[0] - file = nsfile.split("-")[1] - path = os.path.join(DATADIR, nsfile) - try: - os.unlink(TMPFILE) - except: - pass - cmd = "./rados -p {pool} -N '{nspace}' get {file} {out}".format(pool=EC_POOL, file=file, out=TMPFILE, nspace=nspace) - logging.debug(cmd) - call(cmd, shell=True, stdout=nullfd, stderr=nullfd) - cmd = "diff -q {src} {result}".format(src=path, result=TMPFILE) - logging.debug(cmd) - ret = call(cmd, shell=True) - if ret != 0: - logging.error("{file} data not imported properly".format(file=file)) - ERRORS += 1 - try: - os.unlink(TMPFILE) - except: - pass + ERRORS += verify(DATADIR, EC_POOL, EC_NAME) + + if EXP_ERRORS == 0: + NEWPOOL = "import-rados-pool" + cmd = "./rados mkpool {pool}".format(pool=NEWPOOL) + logging.debug(cmd) + ret = call(cmd, shell=True, stdout=nullfd) + + print "Test import-rados" + for osd in [f for f in os.listdir(OSDDIR) if os.path.isdir(os.path.join(OSDDIR, f)) and string.find(f, "osd") == 0]: + dir = os.path.join(TESTDIR, osd) + for pg in [f for f in os.listdir(dir) if os.path.isfile(os.path.join(dir, f))]: + if string.find(pg, "{id}.".format(id=REPID)) != 0: + continue + file = os.path.join(dir, pg) + cmd = "./ceph_objectstore_tool import-rados {pool} {file}".format(pool=NEWPOOL, file=file) + logging.debug(cmd) + ret = call(cmd, shell=True, stdout=nullfd) + if ret != 0: + logging.error("Import-rados failed from {file} with {ret}".format(file=file, ret=ret)) + ERRORS += 1 - call("./stop.sh", stderr=nullfd) + ERRORS += verify(DATADIR, NEWPOOL, REP_NAME) else: - logging.warning("SKIPPING CHECKING IMPORT DATA DUE TO PREVIOUS FAILURES") + logging.warning("SKIPPING IMPORT-RADOS TESTS DUE TO PREVIOUS FAILURES") + + call("./stop.sh", stderr=nullfd) + call("/bin/rm -rf {dir}".format(dir=TESTDIR), shell=True) call("/bin/rm -rf {dir}".format(dir=DATADIR), shell=True) if ERRORS == 0: diff --git a/src/tools/Makefile.am b/src/tools/Makefile.am index fe4294115c538..edd9ccc06e803 100644 --- a/src/tools/Makefile.am +++ b/src/tools/Makefile.am @@ -12,7 +12,7 @@ ceph_kvstore_tool_CXXFLAGS = $(UNITTEST_CXXFLAGS) bin_DEBUGPROGRAMS += ceph-kvstore-tool ceph_objectstore_tool_SOURCES = tools/ceph_objectstore_tool.cc -ceph_objectstore_tool_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) $(BOOST_PROGRAM_OPTIONS_LIBS) +ceph_objectstore_tool_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) $(BOOST_PROGRAM_OPTIONS_LIBS) $(LIBRADOS) if LINUX ceph_objectstore_tool_LDADD += -ldl endif # LINUX diff --git a/src/tools/ceph_objectstore_tool.cc b/src/tools/ceph_objectstore_tool.cc index 4b263b8cc664d..e545d64a5a07d 100644 --- a/src/tools/ceph_objectstore_tool.cc +++ b/src/tools/ceph_objectstore_tool.cc @@ -31,6 +31,8 @@ #include "json_spirit/json_spirit_value.h" #include "json_spirit/json_spirit_reader.h" +#include "include/rados/librados.hpp" + namespace po = boost::program_options; using namespace std; @@ -347,6 +349,7 @@ hobject_t biginfo_oid, log_oid; int file_fd = fd_none; bool debug = false; super_header sh; +uint64_t testalign; template int write_section(sectiontype_t type, const T& obj, int fd) { @@ -938,6 +941,175 @@ int get_omap(ObjectStore *store, coll_t coll, ghobject_t hoid, return 0; } +int get_object_rados(librados::IoCtx &ioctx, bufferlist &bl) +{ + bufferlist::iterator ebliter = bl.begin(); + object_begin ob; + ob.decode(ebliter); + map::iterator i; + bufferlist abl; + + data_section ds; + attr_section as; + omap_hdr_section oh; + omap_section os; + + if (!ob.hoid.hobj.is_head()) { + cout << "Skipping non-head for " << ob.hoid << std::endl; + } + + ioctx.set_namespace(ob.hoid.hobj.get_namespace()); + + string msg("Write"); + int ret = ioctx.create(ob.hoid.hobj.oid.name, true); + if (ret && ret != -EEXIST) { + cerr << "create failed: " << cpp_strerror(ret) << std::endl; + return ret; + } + if (ret == -EEXIST) { + msg = "***Overwrite***"; + ret = ioctx.remove(ob.hoid.hobj.oid.name); + if (ret < 0) { + cerr << "remove failed: " << cpp_strerror(ret) << std::endl; + return ret; + } + ret = ioctx.create(ob.hoid.hobj.oid.name, true); + if (ret < 0) { + cerr << "create failed: " << cpp_strerror(ret) << std::endl; + return ret; + } + } + + cout << msg << " " << ob.hoid << std::endl; + + bool need_align = false; + uint64_t alignment = 0; + if (testalign) { + need_align = true; + alignment = testalign; + } else { + if ((need_align = ioctx.pool_requires_alignment())) + alignment = ioctx.pool_required_alignment(); + } + + if (debug && need_align) + cerr << "alignment = " << alignment << std::endl; + + bufferlist ebl, databl; + uint64_t in_offset = 0, out_offset = 0; + bool done = false; + while(!done) { + sectiontype_t type; + int ret = read_section(file_fd, &type, &ebl); + if (ret) + return ret; + + ebliter = ebl.begin(); + //cout << "\tdo_object: Section type " << hex << type << dec << std::endl; + //cout << "\t\tsection size " << ebl.length() << std::endl; + if (type >= END_OF_TYPES) { + cout << "Skipping unknown object section type" << std::endl; + continue; + } + switch(type) { + case TYPE_DATA: + ds.decode(ebliter); + if (debug) + cerr << "\tdata: offset " << ds.offset << " len " << ds.len << std::endl; + if (need_align) { + if (ds.offset != in_offset) { + cerr << "Discontiguous object data in export" << std::endl; + return EFAULT; + } + assert(ds.databl.length() == ds.len); + databl.claim_append(ds.databl); + in_offset += ds.len; + if (databl.length() >= alignment) { + uint64_t rndlen = uint64_t(databl.length() / alignment) * alignment; + if (debug) cerr << "write offset=" << out_offset << " len=" << rndlen << std::endl; + ret = ioctx.write(ob.hoid.hobj.oid.name, databl, rndlen, out_offset); + if (ret) { + cerr << "write failed: " << cpp_strerror(ret) << std::endl; + return ret; + } + out_offset += rndlen; + bufferlist n; + if (databl.length() > rndlen) { + assert(databl.length() - rndlen < alignment); + n.substr_of(databl, rndlen, databl.length() - rndlen); + } + databl = n; + } + break; + } + ret = ioctx.write(ob.hoid.hobj.oid.name, ds.databl, ds.len, ds.offset); + if (ret) { + cerr << "write failed: " << cpp_strerror(ret) << std::endl; + return ret; + } + break; + case TYPE_ATTRS: + as.decode(ebliter); + + if (debug) + cerr << "\tattrs: len " << as.data.size() << std::endl; + for (i = as.data.begin(); i != as.data.end(); i++) { + if (i->first == "_" || i->first == "snapset") + continue; + abl.clear(); + abl.push_front(i->second); + ret = ioctx.setxattr(ob.hoid.hobj.oid.name, i->first.substr(1).c_str(), abl); + if (ret) { + cerr << "setxattr failed: " << cpp_strerror(ret) << std::endl; + if (ret != -EOPNOTSUPP) + return ret; + } + } + break; + case TYPE_OMAP_HDR: + oh.decode(ebliter); + + if (debug) + cerr << "\tomap header: " << string(oh.hdr.c_str(), oh.hdr.length()) + << std::endl; + ret = ioctx.omap_set_header(ob.hoid.hobj.oid.name, oh.hdr); + if (ret) { + cerr << "omap_set_header failed: " << cpp_strerror(ret) << std::endl; + if (ret != -EOPNOTSUPP) + return ret; + } + break; + case TYPE_OMAP: + os.decode(ebliter); + + if (debug) + cerr << "\tomap: size " << os.omap.size() << std::endl; + ret = ioctx.omap_set(ob.hoid.hobj.oid.name, os.omap); + if (ret) { + cerr << "omap_set failed: " << cpp_strerror(ret) << std::endl; + if (ret != -EOPNOTSUPP) + return ret; + } + break; + case TYPE_OBJECT_END: + if (need_align && databl.length() > 0) { + assert(databl.length() < alignment); + if (debug) cerr << "END write offset=" << out_offset << " len=" << databl.length() << std::endl; + ret = ioctx.write(ob.hoid.hobj.oid.name, databl, databl.length(), out_offset); + if (ret) { + cerr << "write failed: " << cpp_strerror(ret) << std::endl; + return ret; + } + } + done = true; + break; + default: + return EFAULT; + } + } + return 0; +} + int get_object(ObjectStore *store, coll_t coll, bufferlist &bl) { ObjectStore::Transaction tran; @@ -1035,6 +1207,125 @@ int get_pg_metadata(ObjectStore *store, coll_t coll, bufferlist &bl) return 0; } +int do_import_rados(string pool) +{ + bufferlist ebl; + pg_info_t info; + PGLog::IndexedLog log; + + int ret = sh.read_super(); + if (ret) + return ret; + + if (sh.magic != super_header::super_magic) { + cerr << "Invalid magic number" << std::endl; + return EFAULT; + } + + if (sh.version > super_header::super_ver) { + cerr << "Can't handle export format version=" << sh.version << std::endl; + return EINVAL; + } + + //First section must be TYPE_PG_BEGIN + sectiontype_t type; + ret = read_section(file_fd, &type, &ebl); + if (ret) + return ret; + if (type != TYPE_PG_BEGIN) { + return EFAULT; + } + + bufferlist::iterator ebliter = ebl.begin(); + pg_begin pgb; + pgb.decode(ebliter); + spg_t pgid = pgb.pgid; + + if (!pgid.is_no_shard()) { + cerr << "Importing Erasure Coded shard is not supported" << std::endl; + exit(1); + } + + if (debug) { + cerr << "Exported features: " << pgb.superblock.compat_features << std::endl; + } + + // XXX: How to check export features? +#if 0 + if (sb.compat_features.compare(pgb.superblock.compat_features) == -1) { + cerr << "Export has incompatible features set " + << pgb.superblock.compat_features << std::endl; + return 1; + } +#endif + + librados::IoCtx ioctx; + librados::Rados cluster; + + char *id = getenv("CEPH_CLIENT_ID"); + if (id) cerr << "Client id is: " << id << std::endl; + ret = cluster.init(id); + if (ret) { + cerr << "Error " << ret << " in cluster.init" << std::endl; + return ret; + } + ret = cluster.conf_read_file(NULL); + if (ret) { + cerr << "Error " << ret << " in cluster.conf_read_file" << std::endl; + return ret; + } + ret = cluster.conf_parse_env(NULL); + if (ret) { + cerr << "Error " << ret << " in cluster.conf_read_env" << std::endl; + return ret; + } + cluster.connect(); + + ret = cluster.ioctx_create(pool.c_str(), ioctx); + if (ret < 0) { + cerr << "ioctx_create " << pool << " failed with " << ret << std::endl; + return ret; + } + + cout << "Importing from pgid " << pgid << std::endl; + + bool done = false; + bool found_metadata = false; + while(!done) { + ret = read_section(file_fd, &type, &ebl); + if (ret) + return ret; + + //cout << "do_import: Section type " << hex << type << dec << std::endl; + if (type >= END_OF_TYPES) { + cout << "Skipping unknown section type" << std::endl; + continue; + } + switch(type) { + case TYPE_OBJECT_BEGIN: + ret = get_object_rados(ioctx, ebl); + if (ret) return ret; + break; + case TYPE_PG_METADATA: + if (debug) + cout << "Don't care about the old metadata" << std::endl; + found_metadata = true; + break; + case TYPE_PG_END: + done = true; + break; + default: + return EFAULT; + } + } + + if (!found_metadata) { + cerr << "Missing metadata section, ignored" << std::endl; + } + + return 0; +} + int do_import(ObjectStore *store, OSDSuperblock& sb) { bufferlist ebl; @@ -1498,6 +1789,8 @@ void usage(po::options_description &desc) cerr << "ceph_objectstore_tool ... list-omap" << std::endl; cerr << "ceph_objectstore_tool ... remove" << std::endl; cerr << std::endl; + cerr << "ceph_objectstore_tool import-rados [file]" << std::endl; + cerr << std::endl; exit(1); } @@ -1532,6 +1825,7 @@ int main(int argc, char **argv) ("objcmd", po::value(&objcmd), "command [(get|set)-bytes, (get|set|rm)-(attr|omap), list-attrs, list-omap, remove]") ("arg1", po::value(&arg1), "arg1 based on cmd") ("arg2", po::value(&arg2), "arg2 based on cmd") + ("test-align", po::value(&testalign)->default_value(0), "hidden align option for testing") ; po::options_description all("All options"); @@ -1555,6 +1849,47 @@ int main(int argc, char **argv) usage(desc); } + if (!vm.count("debug")) { + debug = false; + } else { + debug = true; + } + + // Handle completely different operation "import-rados" + if (object == "import-rados") { + if (vm.count("objcmd") == 0) { + cerr << "ceph_objectstore_tool import-rados [file]" << std::endl; + exit(1); + } + + string pool = objcmd; + // positional argument takes precendence, but accept + // --file option too + if (!vm.count("arg1")) { + if (!vm.count("file")) + arg1 = "-"; + else + arg1 = file; + } + if (arg1 == "-") { + if (isatty(STDIN_FILENO)) { + cerr << "stdin is a tty and no file specified" << std::endl; + exit(1); + } + file_fd = STDIN_FILENO; + } else { + file_fd = open(arg1.c_str(), O_RDONLY); + if (file_fd < 0) { + perror("open"); + return 1; + } + } + int ret = do_import_rados(pool); + if (ret == 0) + cout << "Import successful" << std::endl; + return ret != 0; + } + if (!vm.count("data-path")) { cerr << "Must provide --data-path" << std::endl; usage(desc); @@ -1651,12 +1986,6 @@ int main(int argc, char **argv) ceph_options.push_back(i->c_str()); } - if (!vm.count("debug")) { - debug = false; - } else { - debug = true; - } - osflagbits_t flags = 0; if (vm.count("skip-journal-replay")) flags |= SKIP_JOURNAL_REPLAY;