From: John Spray Date: Thu, 4 Jun 2015 13:13:24 +0000 (+0100) Subject: tools: remove objectstoretool's rados-import X-Git-Tag: v9.0.3~203^2~4 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=460200c7bdcf76e37c90cb28a8c714696ba56524;p=ceph.git tools: remove objectstoretool's rados-import Same functionality now exposed as "rados import". This removes objectstoretool's librados dependency. Signed-off-by: John Spray --- diff --git a/src/tools/Makefile-client.am b/src/tools/Makefile-client.am index 8c96136f14fe..1fb78540e043 100644 --- a/src/tools/Makefile-client.am +++ b/src/tools/Makefile-client.am @@ -15,7 +15,7 @@ bin_DEBUGPROGRAMS += ceph_radosacl rados_SOURCES = \ tools/rados/rados.cc \ tools/RadosDump.cc \ - tools/RadosImport.cc \ + tools/rados/RadosImport.cc \ tools/rados/PoolDump.cc rados_SOURCES += common/obj_bencher.cc # needs cleanup so it can go in libcommon.la rados_LDADD = libcls_lock_client.la $(LIBRADOS) $(CEPH_GLOBAL) diff --git a/src/tools/Makefile-server.am b/src/tools/Makefile-server.am index 2dc27e176cc9..45f943e0bfc5 100644 --- a/src/tools/Makefile-server.am +++ b/src/tools/Makefile-server.am @@ -14,8 +14,8 @@ bin_DEBUGPROGRAMS += ceph-kvstore-tool if WITH_OSD -ceph_objectstore_tool_SOURCES = tools/ceph_objectstore_tool.cc tools/RadosImport.cc tools/RadosDump.cc -ceph_objectstore_tool_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) $(BOOST_PROGRAM_OPTIONS_LIBS) $(LIBRADOS) +ceph_objectstore_tool_SOURCES = tools/ceph_objectstore_tool.cc tools/RadosDump.cc +ceph_objectstore_tool_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) $(BOOST_PROGRAM_OPTIONS_LIBS) if LINUX ceph_objectstore_tool_LDADD += -ldl endif # LINUX diff --git a/src/tools/Makefile.am b/src/tools/Makefile.am index 5c8d86ad9c92..7fc23b87e4f2 100644 --- a/src/tools/Makefile.am +++ b/src/tools/Makefile.am @@ -55,6 +55,6 @@ noinst_HEADERS += \ tools/cephfs/TableTool.h \ tools/cephfs/MDSUtility.h \ tools/RadosDump.h \ - tools/RadosImport.h\ + tools/rados/RadosImport.h \ tools/ceph_objectstore_tool.h \ tools/rados/PoolDump.h diff --git a/src/tools/RadosImport.cc b/src/tools/RadosImport.cc deleted file mode 100644 index 9fe06c3577bb..000000000000 --- a/src/tools/RadosImport.cc +++ /dev/null @@ -1,368 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2015 Red Hat - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - * - */ - - -#include "common/errno.h" - -#include "osd/PGLog.h" -#include "RadosImport.h" - -#define dout_subsys ceph_subsys_rados - -int RadosImport::import(std::string pool, bool no_overwrite) -{ - librados::IoCtx ioctx; - librados::Rados cluster; - - char *id = getenv("CEPH_CLIENT_ID"); - if (id) cerr << "Client id is: " << id << std::endl; - int ret = cluster.init(id); - if (ret) { - cerr << "Error " << ret << " in cluster.init" << std::endl; - return ret; - } - ret = cluster.conf_read_file(NULL); - if (ret) { - cerr << "Error " << ret << " in cluster.conf_read_file" << std::endl; - return ret; - } - ret = cluster.conf_parse_env(NULL); - if (ret) { - cerr << "Error " << ret << " in cluster.conf_read_env" << std::endl; - return ret; - } - cluster.connect(); - - ret = cluster.ioctx_create(pool.c_str(), ioctx); - if (ret < 0) { - cerr << "ioctx_create " << pool << " failed with " << ret << std::endl; - return ret; - } - - return import(ioctx, no_overwrite); -} - -int RadosImport::import(librados::IoCtx &io_ctx, bool no_overwrite) -{ - bufferlist ebl; - pg_info_t info; - PGLog::IndexedLog log; - - int ret = read_super(); - if (ret) - return ret; - - if (sh.magic != super_header::super_magic) { - cerr << "Invalid magic number: 0x" - << std::hex << sh.magic << " vs. 0x" << super_header::super_magic - << std::dec << std::endl; - return -EFAULT; - } - - if (sh.version > super_header::super_ver) { - cerr << "Can't handle export format version=" << sh.version << std::endl; - return -EINVAL; - } - - //First section must be TYPE_PG_BEGIN - sectiontype_t type; - ret = read_section(&type, &ebl); - if (ret) - return ret; - - bool pool_mode = false; - if (type == TYPE_POOL_BEGIN) { - pool_mode = true; - cout << "Importing pool" << std::endl; - } else if (type == TYPE_PG_BEGIN) { - bufferlist::iterator ebliter = ebl.begin(); - pg_begin pgb; - pgb.decode(ebliter); - spg_t pgid = pgb.pgid;; - if (!pgid.is_no_shard()) { - cerr << "Importing Erasure Coded shard is not supported" << std::endl; - return -EOPNOTSUPP; - } - dout(10) << "Exported features: " << pgb.superblock.compat_features << dendl; - cout << "Importing from pgid " << pgid << std::endl; - } else { - cerr << "Invalid initial section code " << type << std::endl; - return -EFAULT; - } - - // XXX: How to check export features? -#if 0 - if (sb.compat_features.compare(pgb.superblock.compat_features) == -1) { - cerr << "Export has incompatible features set " - << pgb.superblock.compat_features << std::endl; - return -EINVAL; - } -#endif - - bool done = false; - bool found_metadata = false; - while(!done) { - ret = read_section(&type, &ebl); - if (ret) - return ret; - - //cout << "do_import: Section type " << hex << type << dec << std::endl; - if (type >= END_OF_TYPES) { - cout << "Skipping unknown section type" << std::endl; - continue; - } - switch(type) { - case TYPE_OBJECT_BEGIN: - ret = get_object_rados(io_ctx, ebl, no_overwrite); - if (ret) { - cerr << "Error inserting object: " << ret << std::endl; - return ret; - } - break; - case TYPE_PG_METADATA: - dout(10) << "Don't care about the old metadata" << dendl; - found_metadata = true; - break; - case TYPE_PG_END: - done = true; - break; - case TYPE_POOL_END: - done = true; - break; - default: - return -EFAULT; - } - } - - if (!(pool_mode || found_metadata)) { - cerr << "Missing metadata section!" << std::endl; - } - - return 0; -} - -int RadosImport::get_object_rados(librados::IoCtx &ioctx, bufferlist &bl, bool no_overwrite) -{ - bufferlist::iterator ebliter = bl.begin(); - object_begin ob; - ob.decode(ebliter); - map::iterator i; - bufferlist abl; - bool skipping; - - data_section ds; - attr_section as; - omap_hdr_section oh; - omap_section os; - - assert(g_ceph_context); - if (ob.hoid.hobj.nspace == g_ceph_context->_conf->osd_hit_set_namespace) { - cout << "Skipping internal object " << ob.hoid << std::endl; - skip_object(bl); - return 0; - } - - if (!ob.hoid.hobj.is_head()) { - cout << "Skipping non-head for " << ob.hoid << std::endl; - skip_object(bl); - return 0; - } - - ioctx.set_namespace(ob.hoid.hobj.get_namespace()); - - string msg("Write"); - skipping = false; - if (dry_run) { - uint64_t psize; - time_t pmtime; - int ret = ioctx.stat(ob.hoid.hobj.oid.name, &psize, &pmtime); - if (ret == 0) { - if (no_overwrite) - // Could set skipping, but dry-run doesn't change anything either - msg = "Skipping existing"; - else - msg = "***Overwrite***"; - } - } else { - int ret = ioctx.create(ob.hoid.hobj.oid.name, true); - if (ret && ret != -EEXIST) { - cerr << "create failed: " << cpp_strerror(ret) << std::endl; - return ret; - } - if (ret == -EEXIST) { - if (no_overwrite) { - msg = "Skipping existing"; - skipping = true; - } else { - msg = "***Overwrite***"; - ret = ioctx.remove(ob.hoid.hobj.oid.name); - if (ret < 0) { - cerr << "remove failed: " << cpp_strerror(ret) << std::endl; - return ret; - } - ret = ioctx.create(ob.hoid.hobj.oid.name, true); - // If object re-appeared after removal, let's just skip it - if (ret == -EEXIST) { - skipping = true; - msg = "Skipping in-use object"; - ret = 0; - } - if (ret < 0) { - cerr << "create failed: " << cpp_strerror(ret) << std::endl; - return ret; - } - } - } - } - - cout << msg << " " << ob.hoid << std::endl; - - bool need_align = false; - uint64_t alignment = 0; - if (align) { - need_align = true; - alignment = align; - } else { - if ((need_align = ioctx.pool_requires_alignment())) - alignment = ioctx.pool_required_alignment(); - } - - if (need_align) { - dout(10) << "alignment = " << alignment << dendl; - } - - bufferlist ebl, databl; - uint64_t in_offset = 0, out_offset = 0; - bool done = false; - while(!done) { - sectiontype_t type; - int ret = read_section(&type, &ebl); - if (ret) { - cerr << "Error reading section: " << ret << std::endl; - return ret; - } - - ebliter = ebl.begin(); - //cout << "\tdo_object: Section type " << hex << type << dec << std::endl; - //cout << "\t\tsection size " << ebl.length() << std::endl; - if (type >= END_OF_TYPES) { - cout << "Skipping unknown object section type" << std::endl; - continue; - } - switch(type) { - case TYPE_DATA: - ds.decode(ebliter); - dout(10) << "\tdata: offset " << ds.offset << " len " << ds.len << dendl; - if (need_align) { - if (ds.offset != in_offset) { - cerr << "Discontiguous object data in export" << std::endl; - return -EFAULT; - } - assert(ds.databl.length() == ds.len); - databl.claim_append(ds.databl); - in_offset += ds.len; - if (databl.length() >= alignment) { - uint64_t rndlen = uint64_t(databl.length() / alignment) * alignment; - dout(10) << "write offset=" << out_offset << " len=" << rndlen << dendl; - if (!dry_run && !skipping) { - ret = ioctx.write(ob.hoid.hobj.oid.name, databl, rndlen, out_offset); - if (ret) { - cerr << "write failed: " << cpp_strerror(ret) << std::endl; - return ret; - } - } - out_offset += rndlen; - bufferlist n; - if (databl.length() > rndlen) { - assert(databl.length() - rndlen < alignment); - n.substr_of(databl, rndlen, databl.length() - rndlen); - } - databl = n; - } - break; - } - if (!dry_run && !skipping) { - ret = ioctx.write(ob.hoid.hobj.oid.name, ds.databl, ds.len, ds.offset); - if (ret) { - cerr << "write failed: " << cpp_strerror(ret) << std::endl; - return ret; - } - } - break; - case TYPE_ATTRS: - as.decode(ebliter); - - dout(10) << "\tattrs: len " << as.data.size() << dendl; - if (dry_run || skipping) - break; - for (std::map::iterator i = as.data.begin(); - i != as.data.end(); ++i) { - if (i->first == "_" || i->first == "snapset") - continue; - ret = ioctx.setxattr(ob.hoid.hobj.oid.name, i->first.substr(1).c_str(), i->second); - if (ret) { - cerr << "setxattr failed: " << cpp_strerror(ret) << std::endl; - if (ret != -EOPNOTSUPP) - return ret; - } - } - break; - case TYPE_OMAP_HDR: - oh.decode(ebliter); - - dout(10) << "\tomap header: " << string(oh.hdr.c_str(), oh.hdr.length()) - << dendl; - if (dry_run || skipping) - break; - ret = ioctx.omap_set_header(ob.hoid.hobj.oid.name, oh.hdr); - if (ret) { - cerr << "omap_set_header failed: " << cpp_strerror(ret) << std::endl; - if (ret != -EOPNOTSUPP) - return ret; - } - break; - case TYPE_OMAP: - os.decode(ebliter); - - dout(10) << "\tomap: size " << os.omap.size() << dendl; - if (dry_run || skipping) - break; - ret = ioctx.omap_set(ob.hoid.hobj.oid.name, os.omap); - if (ret) { - cerr << "omap_set failed: " << cpp_strerror(ret) << std::endl; - if (ret != -EOPNOTSUPP) - return ret; - } - break; - case TYPE_OBJECT_END: - done = true; - if (need_align && databl.length() > 0) { - assert(databl.length() < alignment); - dout(10) << "END write offset=" << out_offset << " len=" << databl.length() << dendl; - if (dry_run || skipping) - break; - ret = ioctx.write(ob.hoid.hobj.oid.name, databl, databl.length(), out_offset); - if (ret) { - cerr << "write failed: " << cpp_strerror(ret) << std::endl; - return ret; - } - } - break; - default: - cerr << "Unexpected section type " << type << std::endl; - return -EFAULT; - } - } - return 0; -} diff --git a/src/tools/RadosImport.h b/src/tools/RadosImport.h deleted file mode 100644 index 4a2696a51b38..000000000000 --- a/src/tools/RadosImport.h +++ /dev/null @@ -1,45 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2015 Red Hat - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - * - */ - -#ifndef RADOS_IMPORT_H_ -#define RADOS_IMPORT_H_ - -#include - -#include "include/rados/librados.hpp" -#include "include/buffer.h" - -#include "RadosDump.h" - -/** - * Specialization of RadosDump that adds - * methods for importing objects from a stream - * to a live cluster. - */ -class RadosImport : public RadosDump -{ - protected: - uint64_t align; - int get_object_rados(librados::IoCtx &ioctx, bufferlist &bl, bool no_overwrite); - - public: - RadosImport(int file_fd_, uint64_t align_, bool dry_run_) - : RadosDump(file_fd_, dry_run_), align(align_) - {} - - int import(std::string pool, bool no_overwrite); - int import(librados::IoCtx &io_ctx, bool no_overwrite); -}; - -#endif // RADOS_IMPORT_H_ diff --git a/src/tools/ceph_objectstore_tool.cc b/src/tools/ceph_objectstore_tool.cc index 298487991982..36362229a19c 100644 --- a/src/tools/ceph_objectstore_tool.cc +++ b/src/tools/ceph_objectstore_tool.cc @@ -35,10 +35,7 @@ #include "json_spirit/json_spirit_value.h" #include "json_spirit/json_spirit_reader.h" -#include "include/rados/librados.hpp" - #include "ceph_objectstore_tool.h" -#include "RadosImport.h" namespace po = boost::program_options; using namespace std; @@ -1733,8 +1730,6 @@ void usage(po::options_description &desc) cerr << "ceph-objectstore-tool ... remove" << std::endl; cerr << "ceph-objectstore-tool ... dump-info" << std::endl; cerr << std::endl; - cerr << "ceph-objectstore-tool import-rados [file]" << std::endl; - cerr << std::endl; cerr << " can be a JSON object description as displayed" << std::endl; cerr << "by --op list." << std::endl; cerr << " can be an object name which will be looked up in all" << std::endl; @@ -1770,7 +1765,7 @@ int main(int argc, char **argv) string dpath, jpath, pgidstr, op, file, object, objcmd, arg1, arg2, type, format; spg_t pgid; ghobject_t ghobj; - bool human_readable, no_overwrite; + bool human_readable; bool force; Formatter *formatter; @@ -1796,7 +1791,6 @@ int main(int argc, char **argv) ("skip-journal-replay", "Disable journal replay") ("skip-mount-omap", "Disable mounting of omap") ("dry-run", "Don't modify the objectstore") - ("no-overwrite", "For import-rados don't overwrite existing files") ; po::options_description positional("Positional options"); @@ -1845,9 +1839,6 @@ int main(int argc, char **argv) force = true; } - no_overwrite = false; - if (vm.count("no-overwrite")) - no_overwrite = true; if (vm.count("dry-run")) dry_run = true; osflagbits_t flags = 0; @@ -1865,45 +1856,6 @@ int main(int argc, char **argv) ceph_options.push_back(i->c_str()); } - // Handle completely different operation "import-rados" - if (object == "import-rados") { - if (vm.count("objcmd") == 0) { - cerr << "ceph-objectstore-tool import-rados [file]" << std::endl; - myexit(1); - } - - string pool = objcmd; - // positional argument takes precendence, but accept - // --file option too - if (!vm.count("arg1")) { - if (!vm.count("file")) - arg1 = "-"; - else - arg1 = file; - } - if (arg1 == "-") { - if (isatty(STDIN_FILENO)) { - cerr << "stdin is a tty and no file specified" << std::endl; - myexit(1); - } - file_fd = STDIN_FILENO; - } else { - file_fd = open(arg1.c_str(), O_RDONLY); - if (file_fd < 0) { - perror("open"); - myexit(1); - } - } - - global_init(NULL, ceph_options, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); - common_init_finish(g_ceph_context); - - int ret = RadosImport(file_fd, testalign, dry_run).import(pool, no_overwrite); - if (ret == 0) - cout << "Import successful" << std::endl; - myexit(ret != 0); - } - if (!vm.count("type")) { type = "filestore"; } diff --git a/src/tools/rados/RadosImport.cc b/src/tools/rados/RadosImport.cc new file mode 100644 index 000000000000..9fe06c3577bb --- /dev/null +++ b/src/tools/rados/RadosImport.cc @@ -0,0 +1,368 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2015 Red Hat + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#include "common/errno.h" + +#include "osd/PGLog.h" +#include "RadosImport.h" + +#define dout_subsys ceph_subsys_rados + +int RadosImport::import(std::string pool, bool no_overwrite) +{ + librados::IoCtx ioctx; + librados::Rados cluster; + + char *id = getenv("CEPH_CLIENT_ID"); + if (id) cerr << "Client id is: " << id << std::endl; + int ret = cluster.init(id); + if (ret) { + cerr << "Error " << ret << " in cluster.init" << std::endl; + return ret; + } + ret = cluster.conf_read_file(NULL); + if (ret) { + cerr << "Error " << ret << " in cluster.conf_read_file" << std::endl; + return ret; + } + ret = cluster.conf_parse_env(NULL); + if (ret) { + cerr << "Error " << ret << " in cluster.conf_read_env" << std::endl; + return ret; + } + cluster.connect(); + + ret = cluster.ioctx_create(pool.c_str(), ioctx); + if (ret < 0) { + cerr << "ioctx_create " << pool << " failed with " << ret << std::endl; + return ret; + } + + return import(ioctx, no_overwrite); +} + +int RadosImport::import(librados::IoCtx &io_ctx, bool no_overwrite) +{ + bufferlist ebl; + pg_info_t info; + PGLog::IndexedLog log; + + int ret = read_super(); + if (ret) + return ret; + + if (sh.magic != super_header::super_magic) { + cerr << "Invalid magic number: 0x" + << std::hex << sh.magic << " vs. 0x" << super_header::super_magic + << std::dec << std::endl; + return -EFAULT; + } + + if (sh.version > super_header::super_ver) { + cerr << "Can't handle export format version=" << sh.version << std::endl; + return -EINVAL; + } + + //First section must be TYPE_PG_BEGIN + sectiontype_t type; + ret = read_section(&type, &ebl); + if (ret) + return ret; + + bool pool_mode = false; + if (type == TYPE_POOL_BEGIN) { + pool_mode = true; + cout << "Importing pool" << std::endl; + } else if (type == TYPE_PG_BEGIN) { + bufferlist::iterator ebliter = ebl.begin(); + pg_begin pgb; + pgb.decode(ebliter); + spg_t pgid = pgb.pgid;; + if (!pgid.is_no_shard()) { + cerr << "Importing Erasure Coded shard is not supported" << std::endl; + return -EOPNOTSUPP; + } + dout(10) << "Exported features: " << pgb.superblock.compat_features << dendl; + cout << "Importing from pgid " << pgid << std::endl; + } else { + cerr << "Invalid initial section code " << type << std::endl; + return -EFAULT; + } + + // XXX: How to check export features? +#if 0 + if (sb.compat_features.compare(pgb.superblock.compat_features) == -1) { + cerr << "Export has incompatible features set " + << pgb.superblock.compat_features << std::endl; + return -EINVAL; + } +#endif + + bool done = false; + bool found_metadata = false; + while(!done) { + ret = read_section(&type, &ebl); + if (ret) + return ret; + + //cout << "do_import: Section type " << hex << type << dec << std::endl; + if (type >= END_OF_TYPES) { + cout << "Skipping unknown section type" << std::endl; + continue; + } + switch(type) { + case TYPE_OBJECT_BEGIN: + ret = get_object_rados(io_ctx, ebl, no_overwrite); + if (ret) { + cerr << "Error inserting object: " << ret << std::endl; + return ret; + } + break; + case TYPE_PG_METADATA: + dout(10) << "Don't care about the old metadata" << dendl; + found_metadata = true; + break; + case TYPE_PG_END: + done = true; + break; + case TYPE_POOL_END: + done = true; + break; + default: + return -EFAULT; + } + } + + if (!(pool_mode || found_metadata)) { + cerr << "Missing metadata section!" << std::endl; + } + + return 0; +} + +int RadosImport::get_object_rados(librados::IoCtx &ioctx, bufferlist &bl, bool no_overwrite) +{ + bufferlist::iterator ebliter = bl.begin(); + object_begin ob; + ob.decode(ebliter); + map::iterator i; + bufferlist abl; + bool skipping; + + data_section ds; + attr_section as; + omap_hdr_section oh; + omap_section os; + + assert(g_ceph_context); + if (ob.hoid.hobj.nspace == g_ceph_context->_conf->osd_hit_set_namespace) { + cout << "Skipping internal object " << ob.hoid << std::endl; + skip_object(bl); + return 0; + } + + if (!ob.hoid.hobj.is_head()) { + cout << "Skipping non-head for " << ob.hoid << std::endl; + skip_object(bl); + return 0; + } + + ioctx.set_namespace(ob.hoid.hobj.get_namespace()); + + string msg("Write"); + skipping = false; + if (dry_run) { + uint64_t psize; + time_t pmtime; + int ret = ioctx.stat(ob.hoid.hobj.oid.name, &psize, &pmtime); + if (ret == 0) { + if (no_overwrite) + // Could set skipping, but dry-run doesn't change anything either + msg = "Skipping existing"; + else + msg = "***Overwrite***"; + } + } else { + int ret = ioctx.create(ob.hoid.hobj.oid.name, true); + if (ret && ret != -EEXIST) { + cerr << "create failed: " << cpp_strerror(ret) << std::endl; + return ret; + } + if (ret == -EEXIST) { + if (no_overwrite) { + msg = "Skipping existing"; + skipping = true; + } else { + msg = "***Overwrite***"; + ret = ioctx.remove(ob.hoid.hobj.oid.name); + if (ret < 0) { + cerr << "remove failed: " << cpp_strerror(ret) << std::endl; + return ret; + } + ret = ioctx.create(ob.hoid.hobj.oid.name, true); + // If object re-appeared after removal, let's just skip it + if (ret == -EEXIST) { + skipping = true; + msg = "Skipping in-use object"; + ret = 0; + } + if (ret < 0) { + cerr << "create failed: " << cpp_strerror(ret) << std::endl; + return ret; + } + } + } + } + + cout << msg << " " << ob.hoid << std::endl; + + bool need_align = false; + uint64_t alignment = 0; + if (align) { + need_align = true; + alignment = align; + } else { + if ((need_align = ioctx.pool_requires_alignment())) + alignment = ioctx.pool_required_alignment(); + } + + if (need_align) { + dout(10) << "alignment = " << alignment << dendl; + } + + bufferlist ebl, databl; + uint64_t in_offset = 0, out_offset = 0; + bool done = false; + while(!done) { + sectiontype_t type; + int ret = read_section(&type, &ebl); + if (ret) { + cerr << "Error reading section: " << ret << std::endl; + return ret; + } + + ebliter = ebl.begin(); + //cout << "\tdo_object: Section type " << hex << type << dec << std::endl; + //cout << "\t\tsection size " << ebl.length() << std::endl; + if (type >= END_OF_TYPES) { + cout << "Skipping unknown object section type" << std::endl; + continue; + } + switch(type) { + case TYPE_DATA: + ds.decode(ebliter); + dout(10) << "\tdata: offset " << ds.offset << " len " << ds.len << dendl; + if (need_align) { + if (ds.offset != in_offset) { + cerr << "Discontiguous object data in export" << std::endl; + return -EFAULT; + } + assert(ds.databl.length() == ds.len); + databl.claim_append(ds.databl); + in_offset += ds.len; + if (databl.length() >= alignment) { + uint64_t rndlen = uint64_t(databl.length() / alignment) * alignment; + dout(10) << "write offset=" << out_offset << " len=" << rndlen << dendl; + if (!dry_run && !skipping) { + ret = ioctx.write(ob.hoid.hobj.oid.name, databl, rndlen, out_offset); + if (ret) { + cerr << "write failed: " << cpp_strerror(ret) << std::endl; + return ret; + } + } + out_offset += rndlen; + bufferlist n; + if (databl.length() > rndlen) { + assert(databl.length() - rndlen < alignment); + n.substr_of(databl, rndlen, databl.length() - rndlen); + } + databl = n; + } + break; + } + if (!dry_run && !skipping) { + ret = ioctx.write(ob.hoid.hobj.oid.name, ds.databl, ds.len, ds.offset); + if (ret) { + cerr << "write failed: " << cpp_strerror(ret) << std::endl; + return ret; + } + } + break; + case TYPE_ATTRS: + as.decode(ebliter); + + dout(10) << "\tattrs: len " << as.data.size() << dendl; + if (dry_run || skipping) + break; + for (std::map::iterator i = as.data.begin(); + i != as.data.end(); ++i) { + if (i->first == "_" || i->first == "snapset") + continue; + ret = ioctx.setxattr(ob.hoid.hobj.oid.name, i->first.substr(1).c_str(), i->second); + if (ret) { + cerr << "setxattr failed: " << cpp_strerror(ret) << std::endl; + if (ret != -EOPNOTSUPP) + return ret; + } + } + break; + case TYPE_OMAP_HDR: + oh.decode(ebliter); + + dout(10) << "\tomap header: " << string(oh.hdr.c_str(), oh.hdr.length()) + << dendl; + if (dry_run || skipping) + break; + ret = ioctx.omap_set_header(ob.hoid.hobj.oid.name, oh.hdr); + if (ret) { + cerr << "omap_set_header failed: " << cpp_strerror(ret) << std::endl; + if (ret != -EOPNOTSUPP) + return ret; + } + break; + case TYPE_OMAP: + os.decode(ebliter); + + dout(10) << "\tomap: size " << os.omap.size() << dendl; + if (dry_run || skipping) + break; + ret = ioctx.omap_set(ob.hoid.hobj.oid.name, os.omap); + if (ret) { + cerr << "omap_set failed: " << cpp_strerror(ret) << std::endl; + if (ret != -EOPNOTSUPP) + return ret; + } + break; + case TYPE_OBJECT_END: + done = true; + if (need_align && databl.length() > 0) { + assert(databl.length() < alignment); + dout(10) << "END write offset=" << out_offset << " len=" << databl.length() << dendl; + if (dry_run || skipping) + break; + ret = ioctx.write(ob.hoid.hobj.oid.name, databl, databl.length(), out_offset); + if (ret) { + cerr << "write failed: " << cpp_strerror(ret) << std::endl; + return ret; + } + } + break; + default: + cerr << "Unexpected section type " << type << std::endl; + return -EFAULT; + } + } + return 0; +} diff --git a/src/tools/rados/RadosImport.h b/src/tools/rados/RadosImport.h new file mode 100644 index 000000000000..3ce369084294 --- /dev/null +++ b/src/tools/rados/RadosImport.h @@ -0,0 +1,45 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2015 Red Hat + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef RADOS_IMPORT_H_ +#define RADOS_IMPORT_H_ + +#include + +#include "include/rados/librados.hpp" +#include "include/buffer.h" + +#include "tools/RadosDump.h" + +/** + * Specialization of RadosDump that adds + * methods for importing objects from a stream + * to a live cluster. + */ +class RadosImport : public RadosDump +{ + protected: + uint64_t align; + int get_object_rados(librados::IoCtx &ioctx, bufferlist &bl, bool no_overwrite); + + public: + RadosImport(int file_fd_, uint64_t align_, bool dry_run_) + : RadosDump(file_fd_, dry_run_), align(align_) + {} + + int import(std::string pool, bool no_overwrite); + int import(librados::IoCtx &io_ctx, bool no_overwrite); +}; + +#endif // RADOS_IMPORT_H_ diff --git a/src/tools/rados/rados.cc b/src/tools/rados/rados.cc index 8ebb587aed13..cf75b7a30c68 100644 --- a/src/tools/rados/rados.cc +++ b/src/tools/rados/rados.cc @@ -44,7 +44,7 @@ #include "common/hobject.h" #include "PoolDump.h" -#include "tools/RadosImport.h" +#include "RadosImport.h" int rados_tool_sync(const std::map < std::string, std::string > &opts, std::vector &args);