From e8d257488048193f13d3a477a9eecc7ba3fb77b6 Mon Sep 17 00:00:00 2001 From: sage Date: Wed, 8 Jun 2005 23:55:19 +0000 Subject: [PATCH] *** empty log message *** git-svn-id: https://ceph.svn.sf.net/svnroot/ceph@283 29311d96-e01e-0410-9327-a35deaab8ce9 --- ceph/Makefile | 5 +- ceph/TODO | 15 ++- ceph/client/SyntheticClient.cc | 75 ++++++++++++++- ceph/client/SyntheticClient.h | 17 +++- ceph/config.cc | 2 +- ceph/fakesyn.cc | 171 +++++++++++++++++++++++++++++++++ ceph/msg/FakeMessenger.cc | 11 ++- ceph/msg/FakeMessenger.h | 3 +- ceph/osd/FakeStore.cc | 102 ++++++++++++++++++-- ceph/osd/FakeStore.h | 26 ++++- 10 files changed, 400 insertions(+), 27 deletions(-) create mode 100644 ceph/fakesyn.cc diff --git a/ceph/Makefile b/ceph/Makefile index 0862e519f535e..5fec193a6a433 100644 --- a/ceph/Makefile +++ b/ceph/Makefile @@ -74,9 +74,12 @@ mttest: test/mttest.cc msg/MTMessenger.cc ${COMMON_OBJS} mpifuse: mpifuse.cc mds/allmds.o client/Client.o client/SyntheticClient.o osd/OSD.o client/fuse.o msg/MPIMessenger.cc msg/CheesySerializer.o ${COMMON_OBJS} ${MPICC} ${CFLAGS} ${LIBS} -lfuse $^ -o $@ -mpisyn: mpisyn.cc mds/allmds.o client/Client.o client/SyntheticClient.o osd/OSD.o client/fuse.o msg/MPIMessenger.cc msg/CheesySerializer.o ${COMMON_OBJS} +mpisyn: mpisyn.cc mds/allmds.o client/Client.o client/SyntheticClient.o osd/OSD.o msg/MPIMessenger.cc msg/CheesySerializer.o ${COMMON_OBJS} ${MPICC} ${CFLAGS} ${LIBS} -lfuse $^ -o $@ +fakesyn: fakesyn.cc mds/allmds.o client/Client.o client/SyntheticClient.o osd/OSD.o msg/FakeMessenger.o msg/CheesySerializer.o ${COMMON_OBJS} + ${CC} ${CFLAGS} ${LIBS} $^ -o $@ + fakefuse: fakefuse.cc mds/allmds.o client/Client.o osd/OSD.o client/fuse.o msg/FakeMessenger.cc msg/CheesySerializer.o ${COMMON_OBJS} ${CC} ${CFLAGS} ${LIBS} -lfuse $^ -o $@ diff --git a/ceph/TODO b/ceph/TODO index abeb4e28fe06c..b54acc4934bba 100644 --- a/ceph/TODO +++ b/ceph/TODO @@ -7,24 +7,33 @@ big fast todo's: - mds benchmarks - synthetic client stable, - pseudo-mega-filesystem - - osd copy-on-write? + - osd copy-on-write.. + +- osd failure structures +- heartbeatmonitor vs pingmonitor + + ask tyce+bill: - obfs on alc? +md tests: +- log length versus cache size, workload + + notes and todos. - SyntheticClient + - virtual mega-filesystem (metadata only) -HARD LINKS +finish HARD LINKS - reclaim danglers from inode file on discover... - fix rename - MDS TODO - fix hard links - they mostly work, but they're fragile diff --git a/ceph/client/SyntheticClient.cc b/ceph/client/SyntheticClient.cc index e537cc57c14a5..1bd306b97720a 100644 --- a/ceph/client/SyntheticClient.cc +++ b/ceph/client/SyntheticClient.cc @@ -20,7 +20,19 @@ void *synthetic_client_thread_entry(void *ptr) { SyntheticClient *sc = (SyntheticClient*)ptr; - sc->run(); + switch (sc->mode) { + case SYNCLIENT_MODE_RANDOMWALK: + sc->random_walk(sc->iarg1); + break; + case SYNCLIENT_MODE_MAKEDIRS: + sc->make_dirs(sc->sarg1.c_str(), sc->iarg1, sc->iarg2, sc->iarg3); + break; + case SYNCLIENT_MODE_FULLWALK: + sc->full_walk(sc->sarg1); + break; + default: + assert(0); + } return 0; } @@ -84,11 +96,68 @@ void SyntheticClient::up() } -int SyntheticClient::run() + + +int SyntheticClient::full_walk(string& basedir) +{ + // read dir + map contents; + int r = client->getdir(basedir.c_str(), contents); + if (r < 0) { + dout(1) << "readdir on " << basedir << " returns " << r << endl; + return r; + } + + for (map::iterator it = contents.begin(); + it != contents.end(); + it++) { + string file = basedir + "/" + it->first; + + struct stat st; + int r = client->lstat(file.c_str(), &st); + if (r < 0) { + dout(1) << "stat error on " << file << " r=" << r << endl; + continue; + } + + if (st.st_mode & INODE_MODE_DIR) full_walk(file); + } + + return 0; +} + +int SyntheticClient::make_dirs(const char *basedir, int dirs, int files, int depth) +{ + // make sure base dir exists + int r = client->mkdir(basedir, 0755); + if (r != 0) { + dout(1) << "can't make base dir? " << basedir << endl; + return -1; + } + + if (depth == 0) return 0; + + // children + char d[500]; + dout(5-depth) << "make_dirs " << basedir << " dirs " << dirs << " files " << files << " depth " << depth << endl; + for (int i=0; imknod(d, 0644); + } + + for (int i=0; iclient = client; - this->num_req = num_req; thread_id = 0; } int start_thread(); int join_thread(); - int run(); + string sarg1; + int iarg1, iarg2, iarg3; + int mode; + + int full_walk(string& fromdir); + int random_walk(int n); + int make_dirs(const char *basedir, int dirs, int files, int depth); }; #endif diff --git a/ceph/config.cc b/ceph/config.cc index 6510a0b9b2f2b..248c69b3bdcf6 100644 --- a/ceph/config.cc +++ b/ceph/config.cc @@ -25,7 +25,7 @@ md_config_t g_conf = { fake_clock: false, fakemessenger_serialize: true, - debug: 15, + debug: 5, // --- client --- client_cache_size: 400, diff --git a/ceph/fakesyn.cc b/ceph/fakesyn.cc new file mode 100644 index 0000000000000..314160e2cc86f --- /dev/null +++ b/ceph/fakesyn.cc @@ -0,0 +1,171 @@ + + +#include +#include +#include +using namespace std; + +#include "include/config.h" + +#include "mds/MDCluster.h" +#include "mds/MDS.h" +#include "osd/OSD.h" +#include "client/Client.h" +#include "client/SyntheticClient.h" + +#include "msg/FakeMessenger.h" +#include "msg/CheesySerializer.h" + +#include "common/Timer.h" + +#define NUMMDS g_conf.num_mds +#define NUMOSD g_conf.num_osd +#define NUMCLIENT g_conf.num_client + +class C_Test : public Context { +public: + void finish(int r) { + cout << "C_Test->finish(" << r << ")" << endl; + } +}; + + +int main(int argc, char **argv) { + + //cerr << "mpisyn starting " << myrank << "/" << world << endl; + + int start = 0; + + // build new argc+argv for fuse + typedef char* pchar; + int nargc = 0; + char **nargv = new pchar[argc]; + nargv[nargc++] = argv[0]; + + int synthetic = 100; + int mkfs = 0; + for (int i=1; iinit(); + } + + for (int i=0; iinit(); + } + + // create client + for (int i=0; iinit(); + + // use my argc, argv (make sure you pass a mount point!) + cout << "mounting" << endl; + client[i]->mount(mkfs); + + cout << "starting synthatic client " << endl; + syn[i] = new SyntheticClient(client[i]); + + syn[i]->mode = SYNCLIENT_MODE_MAKEDIRS; + char s[20]; + sprintf(s,"syn.%d", i); + syn[i]->sarg1 = s; + syn[i]->iarg1 = 5; + syn[i]->iarg2 = 5; + syn[i]->iarg3 = 2; + + syn[i]->start_thread(); + } + for (int i=0; ijoin_thread(); + delete syn[i]; + + client[i]->unmount(); + cout << "unmounted" << endl; + client[i]->shutdown(); + } + + + // wait for it to finish + fakemessenger_wait(); + + // cleanup + for (int i=0; isecond->get_message(); if (m) { //dout(18) << "got " << m << endl; - dout(5) << "---- do_loop dispatching '" << m->get_type_name() << + dout(5) << "---- '" << m->get_type_name() << "' from " << MSG_ADDR_NICE(m->get_source()) << ':' << m->get_source_port() << " to " << MSG_ADDR_NICE(m->get_dest()) << ':' << m->get_dest_port() << " ---- " << m << endl; @@ -198,6 +203,8 @@ int FakeMessenger::shutdown() //cout << "shutdown on messenger " << this << " has " << num_incoming() << " queued" << endl; directory.erase(whoami); + if (directory.empty()) + ::shutdown = true; } void FakeMessenger::trigger_timer(Timer *t) diff --git a/ceph/msg/FakeMessenger.h b/ceph/msg/FakeMessenger.h index e2200fc0eb756..0604be86ce38c 100644 --- a/ceph/msg/FakeMessenger.h +++ b/ceph/msg/FakeMessenger.h @@ -58,7 +58,6 @@ int fakemessenger_do_loop(); int fakemessenger_do_loop_2(); void fakemessenger_startthread(); void fakemessenger_stopthread(); - - +void fakemessenger_wait(); #endif diff --git a/ceph/osd/FakeStore.cc b/ceph/osd/FakeStore.cc index 5a2b201b0dc60..1a41af3c11169 100644 --- a/ceph/osd/FakeStore.cc +++ b/ceph/osd/FakeStore.cc @@ -25,10 +25,15 @@ -FakeStore::FakeStore(char *base, int whoami) +FakeStore::FakeStore(char *base, int whoami, char *shadow) { this->basedir = base; this->whoami = whoami; + + if (shadow) { + is_shadow = true; + shadowdir = shadow; + } } @@ -38,6 +43,9 @@ int FakeStore::init() get_dir(mydir); dout(5) << "init with basedir " << mydir << endl; + if (is_shadow) { + dout(5) << " SHADOW dir is " << shadowdir << endl; + } // make sure global base dir exists struct stat st; @@ -67,10 +75,13 @@ void FakeStore::get_dir(string& dir) { sprintf(s, "%d", whoami); dir = basedir + "/" + s; } -void FakeStore::get_oname(object_t oid, string& fn) { +void FakeStore::get_oname(object_t oid, string& fn, bool shadow) { static char s[100]; sprintf(s, "%d/%02lld/%lld", whoami, HASH_FUNC(oid), oid); - fn = basedir + "/" + s; + if (shadow) + fn = shadowdir + "/" + s; + else + fn = basedir + "/" + s; // dout(1) << "oname is " << fn << endl; } @@ -104,6 +115,10 @@ int FakeStore::mkfs() dout(1) << "mkfs in " << mydir << endl; + if (is_shadow) { + dout(1) << "WARNING mkfs reverting to shadow fs, which pbly isn't what MDS expects!" << endl; + } + // make sure my dir exists r = ::stat(mydir.c_str(), &st); if (r != 0) { @@ -142,32 +157,91 @@ int FakeStore::mkfs() bool FakeStore::exists(object_t oid) { struct stat st; - if (stat(oid, &st) == 0) + if (stat(oid, &st) == 0) return true; - else + else return false; } + int FakeStore::stat(object_t oid, struct stat *st) { dout(20) << "stat " << oid << endl; string fn; get_oname(oid,fn); - return ::stat(fn.c_str(), st); + int r = ::stat(fn.c_str(), st); + + if (is_shadow && + r != 0 && // primary didn't exist + ::lstat(fn.c_str(), st) != 0) { // and wasn't an intentionally bad symlink + get_oname(oid,fn,true); + return ::stat(fn.c_str(), st); + } else + return r; } + + +void FakeStore::shadow_copy_maybe(object_t oid) { + struct stat st; + string fn; + get_oname(oid, fn); + if (::lstat(fn.c_str(), &st) == 0) + return; // live copy exists, we're fine, do nothing. + + // is there a shadow object? + string sfn; + get_oname(oid, sfn, true); + if (::stat(sfn.c_str(), &st) == 0) { + // shadow exists. copy! + dout(10) << "copying shadow for " << oid << " " << st.st_size << " bytes" << endl; + char *buf = new char[1024*1024]; + int left = st.st_size; + + int sfd = ::open(sfn.c_str(), O_RDONLY); + int fd = ::open(fn.c_str(), O_WRONLY); + assert(sfd && fd); + while (left) { + int howmuch = left; + if (howmuch > 1024*1024) howmuch = 1024*1024; + int got = ::read(sfd, buf, howmuch); + int wrote = ::write(fd, buf, got); + assert(wrote == got); + left -= got; + } + ::close(fd); + ::close(sfd); + } +} + int FakeStore::remove(object_t oid) { dout(20) << "remove " << oid << endl; string fn; get_oname(oid,fn); - return ::unlink(fn.c_str()); + int r = ::unlink(fn.c_str()); + + if (r == 0 && is_shadow) { + string sfn; + struct stat st; + get_oname(oid, sfn, true); + int s = ::stat(sfn.c_str(), &st); + if (s == 0) { + // shadow exists. make a bad symlink to mask it. + ::symlink(sfn.c_str(), "doesnotexist"); + r = 0; + } + } + return r; } int FakeStore::truncate(object_t oid, off_t size) { dout(20) << "truncate " << oid << " size " << size << endl; + + if (is_shadow) shadow_copy_maybe(oid); + string fn; get_oname(oid,fn); ::truncate(fn.c_str(), size); @@ -182,7 +256,17 @@ int FakeStore::read(object_t oid, get_oname(oid,fn); int fd = open(fn.c_str(), O_RDONLY); - if (fd < 0) return fd; + if (fd < 0) { + if (is_shadow) { + struct stat st; + if (::lstat(fn.c_str(), &st) == 0) return fd; // neg symlink + get_oname(oid,fn); + fd = open(fn.c_str(), O_RDONLY); + if (fd < 0) + return fd; // no shadow either. + } else + return fd; + } flock(fd, LOCK_EX); // lock for safety off_t actual = lseek(fd, offset, SEEK_SET); @@ -200,6 +284,8 @@ int FakeStore::write(object_t oid, char *buffer) { dout(20) << "write " << oid << " len " << len << " off " << offset << endl; + if (is_shadow) shadow_copy_maybe(oid); + string fn; get_oname(oid,fn); diff --git a/ceph/osd/FakeStore.h b/ceph/osd/FakeStore.h index a7f9d7082a4be..7e440ad829fcd 100644 --- a/ceph/osd/FakeStore.h +++ b/ceph/osd/FakeStore.h @@ -8,11 +8,33 @@ class FakeStore : public ObjectStore { int whoami; void get_dir(string& dir); - void get_oname(object_t oid, string& fn); + void get_oname(object_t oid, string& fn, bool shadow=false); void wipe_dir(string mydir); + /* shadow: copy-on-write behavior against a "starting" clean object store... + + if (is_shadow == true), + shadowdir has same layout as basedir + + if the (normal, live) object file: + - doesn't exist, then use the shadow file if it exists + - does exist, use the live file (in its entirety, COW is on object granularity) + - is a symlink to a nonexistant file, the object doesn't exist (even if it does in the shadow dir) + + write, truncate initiate a copy from shadow -> live. + unlink may create a bad symlink if the shadow file exists + + etc. + + wipe wipes the live dir, effectively revertiing to the shadow fs, so be careful as + this isn't what a MDS mkfs expects! + */ + string shadowdir; + bool is_shadow; + void shadow_copy_maybe(object_t oid); // do copy-on-write.. called by write(), truncate() + public: - FakeStore(char *base, int whoami); + FakeStore(char *base, int whoami, char *shadow = 0); int init(); int finalize(); -- 2.39.5