From 54ce20a4ad9256022eafddc55e15f48a93cea3a1 Mon Sep 17 00:00:00 2001 From: sageweil Date: Sun, 25 Feb 2007 00:53:41 +0000 Subject: [PATCH] fakestore rewrite. new dir layout. now only optionally fake collections or attrs. verify xattr support on mount. git-svn-id: https://ceph.svn.sf.net/svnroot/ceph@1127 29311d96-e01e-0410-9327-a35deaab8ce9 --- branches/riccardo/monitor2/config.cc | 7 +- branches/riccardo/monitor2/config.h | 3 +- branches/riccardo/monitor2/osd/FakeStore.cc | 518 +++++++++++++++----- branches/riccardo/monitor2/osd/FakeStore.h | 76 ++- branches/riccardo/monitor2/osd/OSD.cc | 5 +- 5 files changed, 455 insertions(+), 154 deletions(-) diff --git a/branches/riccardo/monitor2/config.cc b/branches/riccardo/monitor2/config.cc index 9219d3b9d16bf..b164218d81a8e 100644 --- a/branches/riccardo/monitor2/config.cc +++ b/branches/riccardo/monitor2/config.cc @@ -221,7 +221,8 @@ md_config_t g_conf = { fakestore_fsync: false,//true, fakestore_writesync: false, fakestore_syncthreads: 4, - fakestore_fakeattr: true, + fakestore_fake_attrs: false, + fakestore_fake_collections: false, fakestore_dev: 0, // --- ebofs --- @@ -685,6 +686,10 @@ void parse_config_options(std::vector& args) g_conf.fakestore_writesync = atoi(args[++i]); else if (strcmp(args[i], "--fakestore_dev") == 0) g_conf.fakestore_dev = args[++i]; + else if (strcmp(args[i], "--fakestore_fake_attrs") == 0) + g_conf.fakestore_fake_attrs = true;//atoi(args[++i]); + else if (strcmp(args[i], "--fakestore_fake_collections") == 0) + g_conf.fakestore_fake_collections = true;//atoi(args[++i]); else if (strcmp(args[i], "--obfs") == 0) { g_conf.uofs = 1; diff --git a/branches/riccardo/monitor2/config.h b/branches/riccardo/monitor2/config.h index b0edea33ffacb..cf0b4540b07c8 100644 --- a/branches/riccardo/monitor2/config.h +++ b/branches/riccardo/monitor2/config.h @@ -211,7 +211,8 @@ struct md_config_t { bool fakestore_fsync; bool fakestore_writesync; int fakestore_syncthreads; // such crap - bool fakestore_fakeattr; + bool fakestore_fake_attrs; + bool fakestore_fake_collections; char *fakestore_dev; // ebofs diff --git a/branches/riccardo/monitor2/osd/FakeStore.cc b/branches/riccardo/monitor2/osd/FakeStore.cc index 36dc01127107e..1c8ff259e08e9 100644 --- a/branches/riccardo/monitor2/osd/FakeStore.cc +++ b/branches/riccardo/monitor2/osd/FakeStore.cc @@ -28,7 +28,7 @@ #include #include #include -//#include +#include //#include #ifdef DARWIN @@ -38,7 +38,8 @@ #include "config.h" #undef dout -#define dout(l) if (l<=g_conf.debug) cout << "osd" << whoami << ".fakestore " +#define dout(l) if (l<=g_conf.debug) cout << g_clock.now() << " osd" << whoami << ".fakestore " +#define derr(l) if (l<=g_conf.debug) cerr << g_clock.now() << " osd" << whoami << ".fakestore " #include "include/buffer.h" @@ -54,159 +55,149 @@ using namespace __gnu_cxx; +int FakeStore::statfs(struct statfs *buf) +{ + return ::statfs(basedir.c_str(), buf); +} +/* + * sorry, these are sentitive to the object_t and coll_t typing. + */ +void FakeStore::get_oname(object_t oid, char *s) +{ + static hash H; + assert(sizeof(oid) == 16); + sprintf(s, "%s/objects/%02x/%016llx.%016llx", basedir.c_str(), H(oid) & HASH_MASK, + *((__uint64_t*)&oid), + *(((__uint64_t*)&oid) + 1)); +} -int FakeStore::mount() +void FakeStore::get_cdir(coll_t cid, char *s) +{ + assert(sizeof(cid) == 8); + sprintf(s, "%s/collections/%016llx", basedir.c_str(), + cid); +} + +void FakeStore::get_coname(coll_t cid, object_t oid, char *s) +{ + assert(sizeof(oid) == 16); + sprintf(s, "%s/collections/%016llx/%016llx.%016llx", basedir.c_str(), cid, + *((__uint64_t*)&oid), + *(((__uint64_t*)&oid) + 1)); +} + + + + +int FakeStore::mkfs() { + char cmd[200]; if (g_conf.fakestore_dev) { dout(0) << "mounting" << endl; - char cmd[100]; sprintf(cmd,"mount %s", g_conf.fakestore_dev); system(cmd); } - string mydir; - get_dir(mydir); - - dout(5) << "init with basedir " << mydir << endl; - - // make sure global base dir exists - struct stat st; - int r = ::stat(basedir.c_str(), &st); - if (r != 0) { - dout(1) << "unable to stat basedir " << basedir << ", r = " << r << endl; - return r; - } + dout(1) << "mkfs in " << basedir << endl; - // all okay. - return 0; -} + // wipe + sprintf(cmd, "test -d %s && rm -r %s ; mkdir -p %s/collections && mkdir -p %s/objects", + basedir.c_str(), basedir.c_str(), basedir.c_str(), basedir.c_str()); + + dout(5) << "wipe: " << cmd << endl; + system(cmd); -int FakeStore::umount() -{ - dout(5) << "finalize" << endl; + // hashed bits too + for (int i=0; i H; - sprintf(s, "%d/%02x/%016llx.%08x.%d", whoami, H(oid) & HASH_MASK, oid.ino, oid.bno, oid.rev); - fn = basedir + "/" + s; - // dout(1) << "oname is " << fn << endl; -} - - + dout(1) << "mkfs done in " << basedir << endl; -void FakeStore::wipe_dir(string mydir) -{ - DIR *dir = ::opendir(mydir.c_str()); - if (dir) { - dout(10) << "wiping " << mydir << endl; - struct dirent *ent = 0; - - while ((ent = ::readdir(dir)) != 0) { - if (ent->d_name[0] == '.') continue; - dout(25) << "mkfs unlinking " << ent->d_name << endl; - string fn = mydir + "/" + ent->d_name; - ::unlink(fn.c_str()); - } - - ::closedir(dir); - } else { - dout(1) << "mkfs couldn't read dir " << mydir << endl; - } + return 0; } -int FakeStore::mkfs() +int FakeStore::mount() { if (g_conf.fakestore_dev) { dout(0) << "mounting" << endl; char cmd[100]; sprintf(cmd,"mount %s", g_conf.fakestore_dev); - system(cmd); + //system(cmd); } - - int r = 0; + dout(5) << "basedir " << basedir << endl; + + // make sure global base dir exists struct stat st; - string mydir; - get_dir(mydir); - - dout(1) << "mkfs in " << mydir << endl; - - - // make sure my dir exists - r = ::stat(mydir.c_str(), &st); + int r = ::stat(basedir.c_str(), &st); if (r != 0) { - dout(10) << "creating " << mydir << endl; - mkdir(mydir.c_str(), 0755); - r = ::stat(mydir.c_str(), &st); - if (r != 0) { - dout(1) << "couldnt create dir, r = " << r << endl; - return r; - } + derr(0) << "unable to stat basedir " << basedir << ", r = " << r << endl; + return r; + } + + if (g_conf.fakestore_fake_collections) { + dout(0) << "faking collections (in memory)" << endl; + fake_collections = true; } - else wipe_dir(mydir); - // hashed bits too - for (int i=0; i 0) did += r; else { - dout(1) << "couldn't write to " << fn.c_str() << " len " << len << " off " << offset << " errno " << errno << " " << strerror(errno) << endl; + derr(0) << "couldn't write to " << fn << " len " << len << " off " << offset << " errno " << errno << " " << strerror(errno) << endl; } } if (did < 0) { - dout(1) << "couldn't write to " << fn.c_str() << " len " << len << " off " << offset << " errno " << errno << " " << strerror(errno) << endl; + derr(0) << "couldn't write to " << fn << " len " << len << " off " << offset << " errno " << errno << " " << strerror(errno) << endl; } ::flock(fd, LOCK_UN); @@ -341,24 +332,44 @@ int FakeStore::write(object_t oid, class C_FakeSync : public Context { -public: Context *c; int *n; - C_FakeSync(Context *c_, int *n_) : c(c_), n(n_) { + Mutex *lock; + Cond *cond; + +public: + C_FakeSync(Context *c_, int *n_, Mutex *lo, Cond *co) : + c(c_), n(n_), + lock(lo), cond(co) { + lock->Lock(); ++*n; + lock->Unlock(); } void finish(int r) { c->finish(r); + + lock->Lock(); --(*n); - //cout << "sync, " << *n << " still unsync" << endl; + if (*n == 0) cond->Signal(); + lock->Unlock(); } }; +void FakeStore::sync() +{ + synclock.Lock(); + while (unsync > 0) { + dout(0) << "sync waiting for " << unsync << " items to (fake) sync" << endl; + synccond.Wait(synclock); + } + synclock.Unlock(); +} + void FakeStore::sync(Context *onsafe) { if (g_conf.fakestore_fake_sync) { g_timer.add_event_after((float)g_conf.fakestore_fake_sync, - new C_FakeSync(onsafe, &unsync)); + new C_FakeSync(onsafe, &unsync, &synclock, &synccond)); } else { assert(0); // der..no implemented anymore @@ -366,4 +377,247 @@ void FakeStore::sync(Context *onsafe) } +// ------------------------------- +// attributes + +// objects + +int FakeStore::setattr(object_t oid, const char *name, + const void *value, size_t size, + Context *onsafe) +{ + if (fake_attrs) return attrs.setattr(oid, name, value, size, onsafe); + + char fn[100]; + get_oname(oid, fn); + int r = ::setxattr(fn, name, value, size, 0); + return r; +} + +int FakeStore::setattrs(object_t oid, map& aset) +{ + if (fake_attrs) return attrs.setattrs(oid, aset); + + char fn[100]; + get_oname(oid, fn); + int r = 0; + for (map::iterator p = aset.begin(); + p != aset.end(); + ++p) { + r = ::setxattr(fn, p->first.c_str(), p->second.c_str(), p->second.length(), 0); + if (r < 0) break; + } + return r; +} + +int FakeStore::getattr(object_t oid, const char *name, + void *value, size_t size) +{ + if (fake_attrs) return attrs.getattr(oid, name, value, size); + char fn[100]; + get_oname(oid, fn); + int r = ::getxattr(fn, name, value, size); + return r; +} + +int FakeStore::getattrs(object_t oid, map& aset) +{ + if (fake_attrs) return attrs.getattrs(oid, aset); + + char fn[100]; + get_oname(oid, fn); + + char val[1000]; + char names[1000]; + int num = ::listxattr(fn, names, 1000); + + char *name = names; + for (int i=0; i& ls) +{ + if (fake_collections) return collections.list_collections(ls); + + char fn[200]; + sprintf(fn, "%s/collections", basedir.c_str()); + + DIR *dir = ::opendir(fn); + assert(dir); + + struct dirent *de; + while ((de = ::readdir(dir)) != 0) { + // parse + coll_t c = strtoll(de->d_name, 0, 16); + dout(0) << " got " << c << " errno " << errno << " on " << de->d_name << endl; + if (errno) continue; + ls.push_back(c); + } + + ::closedir(dir); + return 0; +} + +int FakeStore::create_collection(coll_t c, + Context *onsafe) +{ + if (fake_collections) return collections.create_collection(c, onsafe); + + char fn[200]; + get_cdir(c, fn); + + int r = ::mkdir(fn, 0755); + + if (onsafe) sync(onsafe); + return r; +} + +int FakeStore::destroy_collection(coll_t c, + Context *onsafe) +{ + if (fake_collections) return collections.destroy_collection(c, onsafe); + + char fn[200]; + get_cdir(c, fn); + char cmd[200]; + sprintf(cmd, "test -d %s && rm -r %s", fn, fn); + system(cmd); + + if (onsafe) sync(onsafe); + return 0; +} + +int FakeStore::collection_stat(coll_t c, struct stat *st) +{ + if (fake_collections) return collections.collection_stat(c, st); + + char fn[200]; + get_cdir(c, fn); + return ::lstat(fn, st); +} + +bool FakeStore::collection_exists(coll_t c) +{ + if (fake_collections) return collections.collection_exists(c); + + struct stat st; + return collection_stat(c, &st) == 0; +} + + +int FakeStore::collection_add(coll_t c, object_t o, + Context *onsafe) +{ + if (fake_collections) return collections.collection_add(c, o, onsafe); + + char cof[200]; + get_coname(c, o, cof); + char of[200]; + get_oname(o, of); + + int r = ::link(of, cof); + if (onsafe) sync(onsafe); + return r; +} + +int FakeStore::collection_remove(coll_t c, object_t o, + Context *onsafe) +{ + if (fake_collections) return collections.collection_remove(c, o, onsafe); + + char cof[200]; + get_coname(c, o, cof); + + int r = ::unlink(cof); + if (onsafe) sync(onsafe); + return r; +} + +int FakeStore::collection_list(coll_t c, list& ls) +{ + if (fake_collections) return collections.collection_list(c, ls); + + char fn[200]; + get_cdir(c, fn); + + DIR *dir = ::opendir(fn); + assert(dir); + + struct dirent *de; + while ((de = ::readdir(dir)) != 0) { + // parse + object_t o; + assert(sizeof(o) == 16); + *(((__uint64_t*)&o) + 0) = strtoll(de->d_name, 0, 16); + assert(de->d_name[16] == '.'); + *(((__uint64_t*)&o) + 1) = strtoll(de->d_name+17, 0, 16); + dout(0) << " got " << o << " errno " << errno << " on " << de->d_name << endl; + if (errno) continue; + ls.push_back(o); + } + + ::closedir(dir); + return 0; +} +// eof. diff --git a/branches/riccardo/monitor2/osd/FakeStore.h b/branches/riccardo/monitor2/osd/FakeStore.h index eaa4126e84e46..b4bf822aa294a 100644 --- a/branches/riccardo/monitor2/osd/FakeStore.h +++ b/branches/riccardo/monitor2/osd/FakeStore.h @@ -32,31 +32,34 @@ using namespace __gnu_cxx; // fake attributes in memory, if we need to. - -class FakeStore : public ObjectStore, - public FakeStoreAttrs, - public FakeStoreCollections { +class FakeStore : public ObjectStore { string basedir; int whoami; - - int unsync; - Mutex lock; + Mutex synclock; + Cond synccond; + int unsync; - // fns - void get_dir(string& dir); - void get_oname(object_t oid, string& fn); - void wipe_dir(string mydir); + // fake attrs? + FakeStoreAttrs attrs; + bool fake_attrs; + // fake collections? + FakeStoreCollections collections; + bool fake_collections; + + // helper fns + void get_oname(object_t oid, char *s); + void get_cdir(coll_t cid, char *s); + void get_coname(coll_t cid, object_t oid, char *s); public: - FakeStore(char *base, int whoami) : FakeStoreAttrs(this), FakeStoreCollections(this) - { - this->basedir = base; - this->whoami = whoami; - unsync = 0; - } - + FakeStore(char *base, int w) : + basedir(base), + whoami(w), + unsync(0), + attrs(this), fake_attrs(false), + collections(this), fake_collections(false) { } int mount(); int umount(); @@ -81,7 +84,44 @@ class FakeStore : public ObjectStore, bufferlist& bl, Context *onsafe); + void sync(); void sync(Context *onsafe); + + // attrs + int setattr(object_t oid, const char *name, + const void *value, size_t size, + Context *onsafe=0); + int setattrs(object_t oid, map& aset); + int getattr(object_t oid, const char *name, + void *value, size_t size); + int getattrs(object_t oid, map& aset); + int rmattr(object_t oid, const char *name, + Context *onsafe=0); + int listattr(object_t oid, char *attrs, size_t size); + int collection_setattr(coll_t c, const char *name, + void *value, size_t size, + Context *onsafe=0); + int collection_rmattr(coll_t c, const char *name, + Context *onsafe=0); + int collection_getattr(coll_t c, const char *name, + void *value, size_t size); + int collection_listattr(coll_t c, char *attrs, size_t size); + + + // collections + int list_collections(list& ls); + int create_collection(coll_t c, + Context *onsafe=0); + int destroy_collection(coll_t c, + Context *onsafe=0); + int collection_stat(coll_t c, struct stat *st); + bool collection_exists(coll_t c); + int collection_add(coll_t c, object_t o, + Context *onsafe=0); + int collection_remove(coll_t c, object_t o, + Context *onsafe=0); + int collection_list(coll_t c, list& o); + }; #endif diff --git a/branches/riccardo/monitor2/osd/OSD.cc b/branches/riccardo/monitor2/osd/OSD.cc index 1799a4109cb75..b1290b132cac9 100644 --- a/branches/riccardo/monitor2/osd/OSD.cc +++ b/branches/riccardo/monitor2/osd/OSD.cc @@ -167,7 +167,8 @@ OSD::OSD(int id, Messenger *m, MonMap *mm, char *dev) : timer(osd_lock) } #endif // USE_OSBDB else { - store = new FakeStore(osd_base_path, whoami); + sprintf(dev_path, "osddata/osd%d", whoami); + store = new FakeStore(dev_path, whoami); } } @@ -1737,7 +1738,7 @@ void OSD::handle_pg_log(MOSDPGLog *m) assert(pg->missing.num_lost() == 0); // ok activate! - pg->activate(t); + pg->activate(t); } unsigned tr = store->apply_transaction(t); -- 2.39.5