From: David Zafman Date: Fri, 23 Jan 2015 15:34:11 +0000 (-0800) Subject: ceph-objectstore-tool: Add --dry-run option X-Git-Tag: v9.0.2~168^2~10 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=2795161f470ab87cf4a82f455773c9b1b82946ef;p=ceph.git ceph-objectstore-tool: Add --dry-run option Fixes: #10290 Signed-off-by: David Zafman --- diff --git a/src/tools/ceph_objectstore_tool.cc b/src/tools/ceph_objectstore_tool.cc index 691259832476..113b798f19b9 100644 --- a/src/tools/ceph_objectstore_tool.cc +++ b/src/tools/ceph_objectstore_tool.cc @@ -92,6 +92,7 @@ const uint16_t shortmagic = 0xffce; //goes into stream as "ceff" const mymagic_t endmagic = (0xecff << 16) | shortmagic; const int fd_none = INT_MIN; bool outistty; +bool dry_run = false; //The first FIXED_LENGTH bytes are a fixed //portion of the export output. This includes the overall @@ -555,6 +556,8 @@ uint64_t testalign; template int write_section(sectiontype_t type, const T& obj, int fd) { + if (dry_run) + return 0; bufferlist blhdr, bl, blftr; obj.encode(bl); header hdr(type, bl.length()); @@ -595,6 +598,8 @@ static void cleanbin(string &str) int write_simple(sectiontype_t type, int fd) { + if (dry_run) + return 0; bufferlist hbl; header hdr(type, 0); @@ -786,10 +791,14 @@ int mark_pg_for_removal(ObjectStore *fs, spg_t pgid, ObjectStore::Transaction *t int initiate_new_remove_pg(ObjectStore *store, spg_t r_pgid) { + if (!dry_run) + finish_remove_pgs(store); if (!store->collection_exists(coll_t(r_pgid))) return -ENOENT; cout << " marking collection for removal" << std::endl; + if (dry_run) + return 0; ObjectStore::Transaction *rmt = new ObjectStore::Transaction; int r = mark_pg_for_removal(store, r_pgid, rmt); if (r < 0) { @@ -797,6 +806,7 @@ int initiate_new_remove_pg(ObjectStore *store, spg_t r_pgid) return r; } store->apply_transaction(*rmt); + finish_remove_pgs(store); return r; } @@ -1045,6 +1055,8 @@ int add_osdmap(ObjectStore *store, metadata_section &ms) //Write super_header with its fixed 16 byte length void write_super() { + if (dry_run) + return; bufferlist superbl; super_header sh; footer ft; @@ -1296,23 +1308,30 @@ int get_object_rados(librados::IoCtx &ioctx, bufferlist &bl) ioctx.set_namespace(ob.hoid.hobj.get_namespace()); string msg("Write"); - int ret = ioctx.create(ob.hoid.hobj.oid.name, true); - if (ret && ret != -EEXIST) { - cerr << "create failed: " << cpp_strerror(ret) << std::endl; - return ret; - } - if (ret == -EEXIST) { - msg = "***Overwrite***"; - ret = ioctx.remove(ob.hoid.hobj.oid.name); - if (ret < 0) { - cerr << "remove failed: " << cpp_strerror(ret) << std::endl; - return ret; - } - ret = ioctx.create(ob.hoid.hobj.oid.name, true); - if (ret < 0) { + if (dry_run) { + uint64_t psize; + time_t pmtime; + int ret = ioctx.stat(ob.hoid.hobj.oid.name, &psize, &pmtime); + if (ret == 0) + msg = "***Overwrite***"; + } else { + int ret = ioctx.create(ob.hoid.hobj.oid.name, true); + if (ret && ret != -EEXIST) { cerr << "create failed: " << cpp_strerror(ret) << std::endl; return ret; } + if (ret == -EEXIST) { + msg = "***Overwrite***"; + ret = ioctx.remove(ob.hoid.hobj.oid.name); + if (ret < 0) { + cerr << "remove failed: " << cpp_strerror(ret) << std::endl; + return ret; + } + ret = ioctx.create(ob.hoid.hobj.oid.name, true); + if (ret < 0) { + cerr << "create failed: " << cpp_strerror(ret) << std::endl; + return ret; + } } cout << msg << " " << ob.hoid << std::endl; @@ -1362,10 +1381,12 @@ int get_object_rados(librados::IoCtx &ioctx, bufferlist &bl) if (databl.length() >= alignment) { uint64_t rndlen = uint64_t(databl.length() / alignment) * alignment; if (debug) cerr << "write offset=" << out_offset << " len=" << rndlen << std::endl; - ret = ioctx.write(ob.hoid.hobj.oid.name, databl, rndlen, out_offset); - if (ret) { - cerr << "write failed: " << cpp_strerror(ret) << std::endl; - return ret; + if (!dry_run) { + ret = ioctx.write(ob.hoid.hobj.oid.name, databl, rndlen, out_offset); + if (ret) { + cerr << "write failed: " << cpp_strerror(ret) << std::endl; + return ret; + } } out_offset += rndlen; bufferlist n; @@ -1377,10 +1398,12 @@ int get_object_rados(librados::IoCtx &ioctx, bufferlist &bl) } break; } - ret = ioctx.write(ob.hoid.hobj.oid.name, ds.databl, ds.len, ds.offset); - if (ret) { - cerr << "write failed: " << cpp_strerror(ret) << std::endl; - return ret; + if (!dry_run) { + ret = ioctx.write(ob.hoid.hobj.oid.name, ds.databl, ds.len, ds.offset); + if (ret) { + cerr << "write failed: " << cpp_strerror(ret) << std::endl; + return ret; + } } break; case TYPE_ATTRS: @@ -1388,6 +1411,8 @@ int get_object_rados(librados::IoCtx &ioctx, bufferlist &bl) if (debug) cerr << "\tattrs: len " << as.data.size() << std::endl; + if (dry_run) + break; for (i = as.data.begin(); i != as.data.end(); ++i) { if (i->first == "_" || i->first == "snapset") continue; @@ -1407,6 +1432,8 @@ int get_object_rados(librados::IoCtx &ioctx, bufferlist &bl) if (debug) cerr << "\tomap header: " << string(oh.hdr.c_str(), oh.hdr.length()) << std::endl; + if (dry_run) + break; ret = ioctx.omap_set_header(ob.hoid.hobj.oid.name, oh.hdr); if (ret) { cerr << "omap_set_header failed: " << cpp_strerror(ret) << std::endl; @@ -1419,6 +1446,8 @@ int get_object_rados(librados::IoCtx &ioctx, bufferlist &bl) if (debug) cerr << "\tomap: size " << os.omap.size() << std::endl; + if (dry_run) + break; ret = ioctx.omap_set(ob.hoid.hobj.oid.name, os.omap); if (ret) { cerr << "omap_set failed: " << cpp_strerror(ret) << std::endl; @@ -1427,16 +1456,18 @@ int get_object_rados(librados::IoCtx &ioctx, bufferlist &bl) } break; case TYPE_OBJECT_END: + done = true; if (need_align && databl.length() > 0) { assert(databl.length() < alignment); if (debug) cerr << "END write offset=" << out_offset << " len=" << databl.length() << std::endl; + if (dry_run) + break; ret = ioctx.write(ob.hoid.hobj.oid.name, databl, databl.length(), out_offset); if (ret) { cerr << "write failed: " << cpp_strerror(ret) << std::endl; return ret; } } - done = true; break; default: return -EFAULT; @@ -1486,7 +1517,8 @@ int get_object(ObjectStore *store, coll_t coll, bufferlist &bl, OSDMap &curmap) } } - t->touch(coll, ob.hoid); + if (!dry_run) + t->touch(coll, ob.hoid); cout << "Write " << ob.hoid << std::endl; @@ -1506,18 +1538,22 @@ int get_object(ObjectStore *store, coll_t coll, bufferlist &bl, OSDMap &curmap) } switch(type) { case TYPE_DATA: + if (dry_run) break; ret = get_data(store, coll, ob.hoid, t, ebl); if (ret) return ret; break; case TYPE_ATTRS: + if (dry_run) break; ret = get_attrs(store, coll, ob.hoid, t, ebl, driver, mapper); if (ret) return ret; break; case TYPE_OMAP_HDR: + if (dry_run) break; ret = get_omap_hdr(store, coll, ob.hoid, t, ebl); if (ret) return ret; break; case TYPE_OMAP: + if (dry_run) break; ret = get_omap(store, coll, ob.hoid, t, ebl); if (ret) return ret; break; @@ -1529,7 +1565,8 @@ int get_object(ObjectStore *store, coll_t coll, bufferlist &bl, OSDMap &curmap) return -EFAULT; } } - store->apply_transaction(*t); + if (!dry_run) + store->apply_transaction(*t); return 0; } @@ -1784,7 +1821,8 @@ int do_import(ObjectStore *store, OSDSuperblock& sb, bool force) pg_info_t info; PGLog::IndexedLog log; - finish_remove_pgs(store); + if (!dry_run) + finish_remove_pgs(store); int ret = sh.read_super(); if (ret) @@ -1872,17 +1910,19 @@ int do_import(ObjectStore *store, OSDSuperblock& sb, bool force) return -EEXIST; } - ObjectStore::Transaction *t = new ObjectStore::Transaction; - PG::_create(*t, pgid); - PG::_init(*t, pgid, NULL); + if (!dry_run) { + ObjectStore::Transaction *t = new ObjectStore::Transaction; + PG::_create(*t, pgid); + PG::_init(*t, pgid, NULL); - // mark this coll for removal until we're done - map values; - ::encode((char)1, values["_remove"]); - t->omap_setkeys(coll, pgid.make_pgmeta_oid(), values); + // mark this coll for removal until we're done + map values; + ::encode((char)1, values["_remove"]); + t->omap_setkeys(coll, pgid.make_pgmeta_oid(), values); - store->apply_transaction(*t); - delete t; + store->apply_transaction(*t); + delete t; + } cout << "Importing pgid " << pgid << std::endl; @@ -1923,43 +1963,47 @@ int do_import(ObjectStore *store, OSDSuperblock& sb, bool force) return -EFAULT; } - pg_log_t newlog, reject; - pg_log_t::filter_log(pgid, curmap, g_ceph_context->_conf->osd_hit_set_namespace, - ms.log, newlog, reject); - if (debug) { - for (list::iterator i = newlog.log.begin(); - i != newlog.log.end(); ++i) - cerr << "Keeping log entry " << *i << std::endl; - for (list::iterator i = reject.log.begin(); - i != reject.log.end(); ++i) - cerr << "Skipping log entry " << *i << std::endl; - } - - divergent_priors_t newdp, rejectdp; - filter_divergent_priors(pgid, curmap, g_ceph_context->_conf->osd_hit_set_namespace, - ms.divergent_priors, newdp, rejectdp); - ms.divergent_priors = newdp; - if (debug) { - for (divergent_priors_t::iterator i = newdp.begin(); - i != newdp.end(); ++i) - cerr << "Keeping divergent_prior " << *i << std::endl; - for (divergent_priors_t::iterator i = rejectdp.begin(); - i != rejectdp.end(); ++i) - cerr << "Skipping divergent_prior " << *i << std::endl; + ObjectStore::Transaction t; + if (!dry_run) { + pg_log_t newlog, reject; + pg_log_t::filter_log(pgid, curmap, g_ceph_context->_conf->osd_hit_set_namespace, + ms.log, newlog, reject); + if (debug) { + for (list::iterator i = newlog.log.begin(); + i != newlog.log.end(); ++i) + cerr << "Keeping log entry " << *i << std::endl; + for (list::iterator i = reject.log.begin(); + i != reject.log.end(); ++i) + cerr << "Skipping log entry " << *i << std::endl; + } + + divergent_priors_t newdp, rejectdp; + filter_divergent_priors(pgid, curmap, g_ceph_context->_conf->osd_hit_set_namespace, + ms.divergent_priors, newdp, rejectdp); + ms.divergent_priors = newdp; + if (debug) { + for (divergent_priors_t::iterator i = newdp.begin(); + i != newdp.end(); ++i) + cerr << "Keeping divergent_prior " << *i << std::endl; + for (divergent_priors_t::iterator i = rejectdp.begin(); + i != rejectdp.end(); ++i) + cerr << "Skipping divergent_prior " << *i << std::endl; + } + + ret = write_pg(t, ms.map_epoch, ms.info, newlog, ms.past_intervals, ms.divergent_priors); + if (ret) return ret; } - t = new ObjectStore::Transaction; - ret = write_pg(*t, ms.map_epoch, ms.info, newlog, ms.past_intervals, ms.divergent_priors); - if (ret) return ret; - // done, clear removal flag if (debug) cerr << "done, clearing removal flag" << std::endl; - set remove; - remove.insert("_remove"); - t->omap_rmkeys(coll, pgid.make_pgmeta_oid(), remove); - store->apply_transaction(*t); - delete t; + + if (!dry_run) { + set remove; + remove.insert("_remove"); + t.omap_rmkeys(coll, pgid.make_pgmeta_oid(), remove); + store->apply_transaction(t); + } return 0; } @@ -2000,9 +2044,11 @@ int do_remove_object(ObjectStore *store, coll_t coll, ghobject_t &ghobj) return r; } + cout << "remove " << ghobj << std::endl; + if (dry_run) + return 0; ObjectStore::Transaction *t = new ObjectStore::Transaction; OSDriver::OSTransaction _t(driver.get_transaction(t)); - cout << "remove " << ghobj << std::endl; r = mapper.remove_oid(ghobj.hobj, &_t); if (r < 0 && r != -ENOENT) { cerr << "remove_oid returned " << cpp_strerror(r) << std::endl; @@ -2109,8 +2155,10 @@ int do_set_bytes(ObjectStore *store, coll_t coll, ghobject_t &ghobj, int fd) if (debug) cerr << "Write " << ghobj << std::endl; - t->touch(coll, ghobj); - t->truncate(coll, ghobj, 0); + if (!dry_run) { + t->touch(coll, ghobj); + t->truncate(coll, ghobj, 0); + } uint64_t offset = 0; bufferlist rawdatabl; @@ -2127,13 +2175,15 @@ int do_set_bytes(ObjectStore *store, coll_t coll, ghobject_t &ghobj, int fd) if (debug) cerr << "\tdata: offset " << offset << " bytes " << bytes << std::endl; - t->write(coll, ghobj, offset, bytes, rawdatabl); + if (!dry_run) + t->write(coll, ghobj, offset, bytes, rawdatabl); offset += bytes; // XXX: Should we apply_transaction() every once in a while for very large files } while(true); - store->apply_transaction(*t); + if (!dry_run) + store->apply_transaction(*t); return 0; } @@ -2317,6 +2367,8 @@ struct do_fix_lost : public action_on_object_t { virtual int call(ObjectStore *store, coll_t coll, ghobject_t &ghobj, object_info_t &oi) { if (oi.is_lost()) { cout << coll << "/" << ghobj << " is lost, fixing" << std::endl; + if (dry_run) + return 0; oi.clear_flag(object_info_t::FLAG_LOST); bufferlist bl; ::encode(oi, bl); @@ -2440,6 +2492,7 @@ int main(int argc, char **argv) ("force", "Ignore some types of errors and proceed with operation - USE WITH CAUTION: CORRUPTION POSSIBLE NOW OR IN THE FUTURE") ("skip-journal-replay", "Disable journal replay") ("skip-mount-omap", "Disable mounting of omap") + ("dry-run", "Don't modify the objectstore") ; po::options_description positional("Positional options"); @@ -2488,6 +2541,14 @@ int main(int argc, char **argv) force = true; } + if (vm.count("dry-run")) + dry_run = true; + osflagbits_t flags = 0; + if (dry_run || vm.count("skip-journal-replay")) + flags |= SKIP_JOURNAL_REPLAY; + if (vm.count("skip-mount-omap")) + flags |= SKIP_MOUNT_OMAP; + vector ceph_options; env_to_vec(ceph_options); ceph_options.reserve(ceph_options.size() + ceph_option_strings.size()); @@ -2568,7 +2629,7 @@ int main(int argc, char **argv) outistty = isatty(STDOUT_FILENO); file_fd = fd_none; - if (op == "export") { + if (op == "export" && !dry_run) { if (!vm.count("file") || file == "-") { if (outistty) { cerr << "stdout is a tty and no --file filename specified" << std::endl; @@ -2590,7 +2651,7 @@ int main(int argc, char **argv) } } - if (vm.count("file") && file_fd == fd_none) { + if (vm.count("file") && file_fd == fd_none && !dry_run) { cerr << "--file option only applies to import or export" << std::endl; myexit(1); } @@ -2600,12 +2661,6 @@ int main(int argc, char **argv) myexit(1); } - osflagbits_t flags = 0; - if (vm.count("skip-journal-replay")) - flags |= SKIP_JOURNAL_REPLAY; - if (vm.count("skip-mount-omap")) - flags |= SKIP_MOUNT_OMAP; - global_init( NULL, ceph_options, CEPH_ENTITY_TYPE_OSD, CODE_ENVIRONMENT_UTILITY_NODOUT, 0); @@ -2890,19 +2945,20 @@ int main(int argc, char **argv) goto out; } - superblock.compat_features.incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SHARDS); - ObjectStore::Transaction t; - bl.clear(); - ::encode(superblock, bl); - t.write(META_COLL, OSD_SUPERBLOCK_POBJECT, 0, bl.length(), bl); - ret = fs->apply_transaction(t); - if (ret < 0) { - cerr << "Error writing OSD superblock: " << cpp_strerror(ret) << std::endl; - goto out; - } - - fs->set_allow_sharded_objects(); + if (!dry_run) { + superblock.compat_features.incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SHARDS); + ObjectStore::Transaction t; + bl.clear(); + ::encode(superblock, bl); + t.write(META_COLL, OSD_SUPERBLOCK_POBJECT, 0, bl.length(), bl); + ret = fs->apply_transaction(t); + if (ret < 0) { + cerr << "Error writing OSD superblock: " << cpp_strerror(ret) << std::endl; + goto out; + } + fs->set_allow_sharded_objects(); + } cout << "Enabled on-disk sharded objects" << std::endl; ret = 0; @@ -2959,13 +3015,11 @@ int main(int argc, char **argv) biginfo_oid = OSD::make_pg_biginfo_oid(pgid); if (op == "remove") { - finish_remove_pgs(fs); ret = initiate_new_remove_pg(fs, pgid); if (ret < 0) { cerr << "PG '" << pgid << "' not found" << std::endl; goto out; } - finish_remove_pgs(fs); cout << "Remove successful" << std::endl; goto out; } @@ -3328,6 +3382,14 @@ out: ret = r; } + if (dry_run) { + // Export output can go to stdout, so put this message on stderr + if (op == "export") + cerr << "dry-run: Nothing changed" << std::endl; + else + cout << "dry-run: Nothing changed" << std::endl; + } + if (ret < 0) ret = 1; myexit(ret);