From cda62de5e55dc1125254758d8bbeef7a611ac42c Mon Sep 17 00:00:00 2001 From: sageweil Date: Wed, 29 Aug 2007 00:16:23 +0000 Subject: [PATCH] dupstore, ObjectStore, Ebofs and Fakestore interface fixups git-svn-id: https://ceph.svn.sf.net/svnroot/ceph@1723 29311d96-e01e-0410-9327-a35deaab8ce9 --- trunk/ceph/Makefile | 4 +- trunk/ceph/dupstore.cc | 102 +++++++++++++++++++++ trunk/ceph/ebofs/Ebofs.cc | 5 +- trunk/ceph/ebofs/Ebofs.h | 4 +- trunk/ceph/ebofs/dup.ebofs.cc | 82 ----------------- trunk/ceph/osd/Fake.h | 12 +++ trunk/ceph/osd/FakeStore.cc | 99 +++++++++++++++++++-- trunk/ceph/osd/FakeStore.h | 11 ++- trunk/ceph/osd/OSD.cc | 4 +- trunk/ceph/osd/ObjectStore.h | 9 ++ trunk/ceph/script/study_lookups.pl | 137 +++++++++++++++++++++++++++++ 11 files changed, 366 insertions(+), 103 deletions(-) create mode 100644 trunk/ceph/dupstore.cc delete mode 100644 trunk/ceph/ebofs/dup.ebofs.cc create mode 100644 trunk/ceph/script/study_lookups.pl diff --git a/trunk/ceph/Makefile b/trunk/ceph/Makefile index b2b1bc5db2c0e..70b196135fb08 100644 --- a/trunk/ceph/Makefile +++ b/trunk/ceph/Makefile @@ -119,7 +119,7 @@ endif # targets -TARGETS = cmon cosd cmds csyn mkmonmap cmonctl fakesyn +TARGETS = cmon cosd cmds csyn mkmonmap cmonctl fakesyn dupstore SRCS=*.cc */*.cc *.h */*.h */*/*.h ifneq ($(fuse),no) @@ -204,7 +204,7 @@ mkfs.ebofs: ebofs/mkfs.ebofs.cc config.cc common/Clock.o ebofs.o test.ebofs: ebofs/test.ebofs.cc config.cc common/Clock.o ebofs.o ${CC} -pg ${CFLAGS} ${LIBS} $^ -o $@ -dup.ebofs: ebofs/dup.ebofs.cc config.cc common/Clock.o ebofs.o +dupstore: dupstore.cc config.cc ebofs.o common/Clock.o common/Timer.o osd/FakeStore.o ${CC} -pg ${CFLAGS} ${LIBS} $^ -o $@ diff --git a/trunk/ceph/dupstore.cc b/trunk/ceph/dupstore.cc new file mode 100644 index 0000000000000..d43f935cb50cc --- /dev/null +++ b/trunk/ceph/dupstore.cc @@ -0,0 +1,102 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2006 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include +#include "ebofs/Ebofs.h" +#include "osd/FakeStore.h" + + +int dupstore(ObjectStore* src, ObjectStore* dst) +{ + if (src->mount() < 0) return 1; + if (dst->mkfs() < 0) return 1; + if (dst->mount() < 0) return 1; + + // objects + list objects; + src->list_objects(objects); + int num = objects.size(); + cout << num << " objects" << std::endl; + int i = 1; + for (list::iterator p = objects.begin(); p != objects.end(); ++p) { + bufferlist bl; + src->read(*p, 0, 0, bl); + cout << "object " << i++ << "/" << num << " " << *p << " = " << bl.length() << " bytes" << std::endl; + dst->write(*p, 0, bl.length(), bl, 0); + map attrs; + src->getattrs(*p, attrs); + dst->setattrs(*p, attrs); + } + + // collections + list collections; + src->list_collections(collections); + num = collections.size(); + cout << num << " collections" << std::endl; + i = 1; + for (list::iterator p = collections.begin(); + p != collections.end(); + ++p) { + dst->create_collection(*p, 0); + map attrs; + src->collection_getattrs(*p, attrs); + dst->collection_setattrs(*p, attrs); + list o; + src->collection_list(*p, o); + int numo = 0; + for (list::iterator q = o.begin(); q != o.end(); q++) { + dst->collection_add(*p, *q, 0); + numo++; + } + cout << "collection " << i++ << "/" << num << " " << hex << *p << dec << " = " << numo << " objects" << std::endl; + } + + + src->umount(); + dst->umount(); + return 0; +} + +void usage() +{ + cerr << "usage: dup.ebofs (ebofs|fakestore) src (ebofs|fakestore) dst" << std::endl; + exit(0); +} + +int main(int argc, char **argv) +{ + vector args; + argv_to_vec(argc, argv, args); + parse_config_options(args); + + // args + if (args.size() != 4) + usage(); + + ObjectStore *src, *dst; + + if (strcmp(args[0], "ebofs") == 0) + src = new Ebofs(args[1]); + else if (strcmp(args[0], "fakestore") == 0) + src = new FakeStore(args[1]); + else usage(); + + if (strcmp(args[2], "ebofs") == 0) + dst = new Ebofs(args[3]); + else if (strcmp(args[2], "fakestore") == 0) + dst = new FakeStore(args[3]); + else usage(); + + return dupstore(src, dst); +} diff --git a/trunk/ceph/ebofs/Ebofs.cc b/trunk/ceph/ebofs/Ebofs.cc index 5374057ad9aca..97cba8285b463 100644 --- a/trunk/ceph/ebofs/Ebofs.cc +++ b/trunk/ceph/ebofs/Ebofs.cc @@ -3401,16 +3401,15 @@ int Ebofs::_collection_getattrs(coll_t cid, map &aset) return 0; } -int Ebofs::collection_setattrs(coll_t cid, const map &aset, Context *onsafe) +int Ebofs::collection_setattrs(coll_t cid, map &aset) { ebofs_lock.Lock(); - assert(onsafe == 0); // der i am lazy int r = _collection_setattrs(cid, aset); ebofs_lock.Unlock(); return r; } -int Ebofs::_collection_setattrs(coll_t cid, const map &aset) +int Ebofs::_collection_setattrs(coll_t cid, map &aset) { dout(8) << "_collection_setattrs " << cid << dendl; diff --git a/trunk/ceph/ebofs/Ebofs.h b/trunk/ceph/ebofs/Ebofs.h index 4ed6afb515d8b..1bcccafe05567 100644 --- a/trunk/ceph/ebofs/Ebofs.h +++ b/trunk/ceph/ebofs/Ebofs.h @@ -310,7 +310,7 @@ protected: int collection_list(coll_t c, list& o); int collection_setattr(coll_t cid, const char *name, const void *value, size_t size, Context *onsafe); - int collection_setattrs(coll_t cid, const map &aset, Context *onsafe); + int collection_setattrs(coll_t cid, map &aset); int collection_getattr(coll_t cid, const char *name, void *value, size_t size); int collection_getattrs(coll_t cid, map &aset); int collection_rmattr(coll_t cid, const char *name, Context *onsafe); @@ -362,7 +362,7 @@ private: int _collection_remove(coll_t c, object_t o); int _collection_getattrs(coll_t oid, map &aset); int _collection_setattr(coll_t oid, const char *name, const void *value, size_t size); - int _collection_setattrs(coll_t oid, const map &aset); + int _collection_setattrs(coll_t oid, map &aset); int _collection_rmattr(coll_t cid, const char *name); diff --git a/trunk/ceph/ebofs/dup.ebofs.cc b/trunk/ceph/ebofs/dup.ebofs.cc deleted file mode 100644 index 05e034a4331d7..0000000000000 --- a/trunk/ceph/ebofs/dup.ebofs.cc +++ /dev/null @@ -1,82 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2004-2006 Sage Weil - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - * - */ - -#include -#include "ebofs/Ebofs.h" - - -int main(int argc, char **argv) -{ - vector args; - argv_to_vec(argc, argv, args); - parse_config_options(args); - - // args - if (args.size() != 2) { - cerr << "usage: dup.ebofs src dst" << std::endl; - return -1; - } - char *srcfn = args[0]; - char *dstfn = args[1]; - - cout << "src " << srcfn << std::endl; - cout << "dst " << dstfn << std::endl; - - Ebofs src(srcfn); - Ebofs dst(dstfn); - - if (src.mount() < 0) return 1; - if (dst.mkfs() < 0) return 1; - if (dst.mount() < 0) return 1; - - // collections - list collections; - src.list_collections(collections); - int num = collections.size(); - cout << num << " collections" << std::endl; - int i = 1; - for (list::iterator p = collections.begin(); - p != collections.end(); - ++p) { - cout << "collection " << i++ << "/" << num << " " << hex << *p << dec << std::endl; - dst.create_collection(*p, 0); - map attrs; - src.collection_getattrs(*p, attrs); - dst.collection_setattrs(*p, attrs, 0); - } - - // objects - list objects; - src.list_objects(objects); - num = objects.size(); - cout << num << " objects" << std::endl; - i = 1; - for (list::iterator p = objects.begin(); p != objects.end(); ++p) { - bufferlist bl; - src.read(*p, 0, 0, bl); - cout << "object " << i++ << "/" << num << " " << *p << " = " << bl.length() << " bytes" << std::endl; - dst.write(*p, 0, bl.length(), bl, 0); - map attrs; - src.getattrs(*p, attrs); - dst.setattrs(*p, attrs); - set c; - src.get_object_collections(*p, c); - for (set::iterator q = c.begin(); q != c.end(); q++) - dst.collection_add(*q, *p, 0); - } - - src.umount(); - dst.umount(); - return 0; -} diff --git a/trunk/ceph/osd/Fake.h b/trunk/ceph/osd/Fake.h index 2155c46de3673..342c153c25cfd 100644 --- a/trunk/ceph/osd/Fake.h +++ b/trunk/ceph/osd/Fake.h @@ -223,6 +223,18 @@ class FakeStoreAttrs { faker_lock.Unlock(); return r; } + int collection_setattrs(coll_t cid, map& aset) { + faker_lock.Lock(); + int r = fakecattrs[cid].setattrs(aset); + faker_lock.Unlock(); + return r; + } + int collection_getattrs(coll_t cid, map& aset) { + faker_lock.Lock(); + int r = fakecattrs[cid].getattrs(aset); + faker_lock.Unlock(); + return r; + } int collection_rmattr(coll_t c, const char *name, Context *onsafe=0) { faker_lock.Lock(); diff --git a/trunk/ceph/osd/FakeStore.cc b/trunk/ceph/osd/FakeStore.cc index 336383e236b91..b28311f741578 100644 --- a/trunk/ceph/osd/FakeStore.cc +++ b/trunk/ceph/osd/FakeStore.cc @@ -41,8 +41,8 @@ #include "config.h" -#define dout(l) if (l<=g_conf.debug) *_dout << dbeginl << g_clock.now() << " osd" << whoami << ".fakestore " -#define derr(l) if (l<=g_conf.debug) *_derr << dbeginl << g_clock.now() << " osd" << whoami << ".fakestore " +#define dout(l) if (l<=g_conf.debug) *_dout << dbeginl << g_clock.now() << " fakestore(" << basedir << ") " +#define derr(l) if (l<=g_conf.debug) *_derr << dbeginl << g_clock.now() << " fakestore(" << basedir << ") " #include "include/buffer.h" @@ -51,8 +51,8 @@ using namespace __gnu_cxx; // crap-a-crap hash -#define HASH_DIRS 0x80 -#define HASH_MASK 0x7f +//#define HASH_DIRS 0x80 +//#define HASH_MASK 0x7f // end crap hash @@ -69,14 +69,16 @@ int FakeStore::statfs(struct statfs *buf) */ void FakeStore::get_oname(object_t oid, char *s) { - static hash H; + //static hash H; assert(sizeof(oid) == 16); #ifdef __LP64__ - sprintf(s, "%s/objects/%02lx/%016lx.%016lx", basedir.c_str(), H(oid) & HASH_MASK, + //sprintf(s, "%s/objects/%02lx/%016lx.%016lx", basedir.c_str(), H(oid) & HASH_MASK, + sprintf(s, "%s/objects/%016lx.%016lx", basedir.c_str(), *((uint64_t*)&oid), *(((uint64_t*)&oid) + 1)); #else - sprintf(s, "%s/objects/%02x/%016llx.%016llx", basedir.c_str(), H(oid) & HASH_MASK, + //sprintf(s, "%s/objects/%02x/%016llx.%016llx", basedir.c_str(), H(oid) & HASH_MASK, + sprintf(s, "%s/objects/%016llx.%016llx", basedir.c_str(), *((uint64_t*)&oid), *(((uint64_t*)&oid) + 1)); #endif @@ -130,6 +132,7 @@ int FakeStore::mkfs() system(cmd); // hashed bits too + /* for (int i=0; i& aset) p != aset.end(); ++p) { r = ::setxattr(fn, p->first.c_str(), p->second.c_str(), p->second.length(), 0); - if (r < 0) break; + if (r < 0) { + cerr << "error setxattr " << strerror(errno) << std::endl; + break; + } } #endif return r; @@ -524,6 +531,49 @@ int FakeStore::collection_getattr(coll_t c, const char *name, return 0; } +int FakeStore::collection_setattrs(coll_t cid, map& aset) +{ + if (fake_attrs) return attrs.collection_setattrs(cid, aset); + + char fn[100]; + get_cdir(cid, fn); + int r = 0; +#ifndef __CYGWIN__ + for (map::iterator p = aset.begin(); + p != aset.end(); + ++p) { + r = ::setxattr(fn, p->first.c_str(), p->second.c_str(), p->second.length(), 0); + if (r < 0) break; + } +#endif + return r; +} + +int FakeStore::collection_getattrs(coll_t cid, map& aset) +{ + if (fake_attrs) return attrs.collection_getattrs(cid, aset); + +#ifndef __CYGWIN__ + char fn[100]; + get_cdir(cid, fn); + + char val[1000]; + char names[1000]; + int num = ::listxattr(fn, names, 1000); + + char *name = names; + for (int i=0; i& ls) +{ + char fn[200]; + sprintf(fn, "%s/objects", basedir.c_str()); + + DIR *dir = ::opendir(fn); + assert(dir); + + struct dirent *de; + while ((de = ::readdir(dir)) != 0) { + if (de->d_name[0] == '.') continue; + // parse + object_t o; + assert(sizeof(o) == 16); + //cout << " got object " << de->d_name << std::endl; + *(((uint64_t*)&o) + 0) = strtoll(de->d_name, 0, 16); + assert(de->d_name[16] == '.'); + *(((uint64_t*)&o) + 1) = strtoll(de->d_name+17, 0, 16); + //dout(0) << " got " << o << " errno " << errno << " on " << de->d_name << dendl; + if (errno) continue; + ls.push_back(o); + } + + ::closedir(dir); + return 0; +} + + // -------------------------- // collections @@ -645,6 +724,8 @@ int FakeStore::collection_list(coll_t c, list& ls) struct dirent *de; while ((de = ::readdir(dir)) != 0) { // parse + if (de->d_name[0] == '.') continue; + //cout << " got object " << de->d_name << std::endl; object_t o; assert(sizeof(o) == 16); *(((uint64_t*)&o) + 0) = strtoll(de->d_name, 0, 16); diff --git a/trunk/ceph/osd/FakeStore.h b/trunk/ceph/osd/FakeStore.h index e88c205315bc0..5828c27c14d96 100644 --- a/trunk/ceph/osd/FakeStore.h +++ b/trunk/ceph/osd/FakeStore.h @@ -35,7 +35,6 @@ using namespace __gnu_cxx; class FakeStore : public ObjectStore { string basedir; - int whoami; Mutex synclock; Cond synccond; @@ -55,9 +54,8 @@ class FakeStore : public ObjectStore { void get_coname(coll_t cid, object_t oid, char *s); public: - FakeStore(char *base, int w) : + FakeStore(char *base) : basedir(base), - whoami(w), unsync(0), attrs(this), fake_attrs(false), collections(this), fake_collections(false) { } @@ -83,6 +81,8 @@ class FakeStore : public ObjectStore { void sync(); void sync(Context *onsafe); + int list_objects(list& ls); + // attrs int setattr(object_t oid, const char *name, const void *value, size_t size, Context *onsafe=0); int setattrs(object_t oid, map& aset); @@ -94,7 +94,8 @@ class FakeStore : public ObjectStore { int collection_rmattr(coll_t c, const char *name, Context *onsafe=0); int collection_getattr(coll_t c, const char *name, void *value, size_t size); //int collection_listattr(coll_t c, char *attrs, size_t size); - + int collection_getattrs(coll_t cid, map &aset); + int collection_setattrs(coll_t cid, map &aset); // collections int list_collections(list& ls); @@ -106,6 +107,8 @@ class FakeStore : public ObjectStore { int collection_remove(coll_t c, object_t o, Context *onsafe=0); int collection_list(coll_t c, list& o); + + }; #endif diff --git a/trunk/ceph/osd/OSD.cc b/trunk/ceph/osd/OSD.cc index 507ce7a7af21d..c08f7c0024bc6 100644 --- a/trunk/ceph/osd/OSD.cc +++ b/trunk/ceph/osd/OSD.cc @@ -170,7 +170,7 @@ OSD::OSD(int id, Messenger *m, MonMap *mm, char *dev) : #endif // USE_OSBDB else { sprintf(dev_path, "osddata/osd%d", whoami); - store = new FakeStore(dev_path, whoami); + store = new FakeStore(dev_path); } } @@ -298,6 +298,8 @@ int OSD::shutdown() state = STATE_STOPPING; // cancel timers + timer.cancel_event(send_pg_stats_event); + send_pg_stats_event = 0; timer.cancel_all(); timer.join(); diff --git a/trunk/ceph/osd/ObjectStore.h b/trunk/ceph/osd/ObjectStore.h index 14c1e19af2d58..c8df5d8218fed 100644 --- a/trunk/ceph/osd/ObjectStore.h +++ b/trunk/ceph/osd/ObjectStore.h @@ -562,6 +562,10 @@ public: virtual int clone(object_t oid, object_t noid) { return -1; } + + virtual int list_objects(list& ls) = 0;//{ return -1; } + + virtual int get_object_collections(object_t oid, set& ls) { return -1; } //virtual int listattr(object_t oid, char *attrs, size_t size) {return 0;} //= 0; @@ -586,6 +590,11 @@ public: Context *onsafe=0) {return 0;} //= 0; virtual int collection_getattr(coll_t cid, const char *name, void *value, size_t size) {return 0;} //= 0; + + virtual int collection_getattrs(coll_t cid, map &aset) = 0;//{ return -1; } + virtual int collection_setattrs(coll_t cid, map &aset) = 0;//{ return -1; } + + //virtual int collection_listattr(coll_t cid, char *attrs, size_t size) {return 0;} //= 0; virtual void sync(Context *onsync) {} diff --git a/trunk/ceph/script/study_lookups.pl b/trunk/ceph/script/study_lookups.pl new file mode 100644 index 0000000000000..7a0784f3210a4 --- /dev/null +++ b/trunk/ceph/script/study_lookups.pl @@ -0,0 +1,137 @@ +#!/usr/bin/perl + +use strict; + +my @buckets = (1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096); + +my %dir_miss; # dir(ino) -> last lookup miss +my %dir_has; # ino -> dentries we have + + +my %ns; # parent -> fn -> ino +my %nlink; # num links to each ino +my %since; # when it got its second link + +my @ignore = ('ll_getattr','ll_setattr','ll_forget','ll_fsync','ll_readlink','ll_statfs','ll_opendir','ll_releasedir','ll_flush','ll_release','ll_open','ll_read','ll_write'); + +my $when; + +my $sumage; +my $numage; + +sub unlink { + my ($p,$n) = @_; + my $i = $ns{$p}->{$n}; + my $new = --$nlink{$i}; + if ($new == 1) { + my $age = $when - $since{$i}; + #print "$since{$i} to $when on $i\t$age\n"; + delete $since{$i}; + + $numage++; + $sumage += $age; + + } elsif ($new == 0) { + delete $nlink{$i}; + } + delete $ns{$p}->{$n}; +} + + +my ($sec, $usec, $cmd); +$_ = <>; +while (1) { + # read trace record + chomp; + last unless $_ eq '@'; + + chomp(my $sec = <>); + chomp(my $usec = <>); + $when = sprintf("%d.%06d",$sec,$usec);# + ($usec / 1000000); + #$when = "$sec.$usec"; + + chomp($cmd = <>); + + #print "cmd $cmd\n"; + + if ($cmd eq 'll_lookup') { + chomp(my $p = <>); + chomp(my $n = <>); + chomp(my $r = <>); + $ns{$p}->{$n} = $r; + } + + elsif ($cmd eq 'll_create') { + chomp(my $p = <>); + chomp(my $n = <>); + <>; <>; <>; + chomp(my $r = <>); + $ns{$p}->{$n} = $r; + $nlink{$r} = 1; + } + elsif ($cmd eq 'll_mknod') { + chomp(my $p = <>); + chomp(my $n = <>); + <>; <>; + chomp(my $r = <>); + $ns{$p}->{$n} = $r; + $nlink{$r} = 1; + } + elsif ($cmd eq 'll_mkdir') { + chomp(my $p = <>); + chomp(my $n = <>); + <>; + chomp(my $r = <>); + $ns{$p}->{$n} = $r; + $nlink{$r} = 1; + } + elsif ($cmd eq 'll_symlink') { + chomp(my $p = <>); + chomp(my $n = <>); + <>; + chomp(my $r = <>); + $ns{$p}->{$n} = $r; + $nlink{$r} = 1; + } + elsif ($cmd eq 'll_link') { + chomp(my $i = <>); + chomp(my $p = <>); + chomp(my $n = <>); + $ns{$p}->{$n} = $i; + if (++$nlink{$i} == 2) { + $since{$i} = $when; + } + } + elsif ($cmd eq 'll_unlink' || + $cmd eq 'll_rmdir') { + chomp(my $p = <>); + chomp(my $n = <>); + &unlink($p, $n); + } + elsif ($cmd eq 'll_rename') { + chomp(my $p = <>); + chomp(my $n = <>); + chomp(my $np = <>); + chomp(my $nn = <>); + if ($ns{$np}->{$nn}) { + &unlink($np, $nn); + } + $ns{$np}->{$nn} = $ns{$p}->{$n}; + delete $ns{$p}->{$n}; + } + + # skip to @ + while (<>) { + last if $_ eq "@\n"; + print "$cmd: $_" + unless grep {$_ eq $cmd} @ignore; + } +} + +print "num $numage .. sum $sumage .. avg lifetime " . ($sumage / $numage) . "\n"; + +# dump hard link inos +for my $ino (keys %nlink) { + next if $nlink{$ino} < 2; + print "$ino\t$nlink{$ino}\n"; +} -- 2.39.5