osd: revamp osd startup

diff --git a/src/cosd.cc b/src/cosd.cc
index 9291acef7466f7600b8de8fef61c3a8ec9752559..0bcb1fbbb0e11227b6ae8b1dc9122916657cfb63 100644 (file)
--- a/src/cosd.cc
+++ b/src/cosd.cc
@@ -33,6 +33,17 @@ using namespace std;
 
 #include "common/Timer.h"
 
+void usage() 
+{
+  cerr << "usage: cosd <device> [-m monitor] [--mkfs_for_osd <nodeid>]" << std::endl;
+  cerr << "   -d              daemonize" << std::endl;
+  cerr << "   --debug_osd N   set debug level (e.g. 10)" << std::endl;
+  cerr << "   --debug_ms N    set message debug level (e.g. 1)" << std::endl;
+  cerr << "   --ebofs         use EBOFS for object storage (default)" << std::endl;
+  cerr << "   --fakestore     store objects as files in directory <device>" << std::endl;
+  exit(1);
+}
+
 
 int main(int argc, const char **argv) 
 {
@@ -45,46 +56,30 @@ int main(int argc, const char **argv)
 
   // osd specific args
   const char *dev = 0;
-  char dev_default[20];
   int whoami = -1;
-  const char *monhost = 0;
+  bool mkfs;
   for (unsigned i=0; i<args.size(); i++) {
-    if (strcmp(args[i],"--dev") == 0) 
-      dev = args[++i];
-    else if (strcmp(args[i],"--osd") == 0)
+    if (strcmp(args[i],"--mkfs_for_osd") == 0) {
+      mkfs = 1; 
       whoami = atoi(args[++i]);
-    else if (!monhost)
-      monhost = args[i];
+    } else if (strcmp(args[i],"--dev") == 0) 
+      dev = args[++i];
+    else if (!dev)
+      dev = args[i];
     else {
       cerr << "unrecognized arg " << args[i] << std::endl;
-      return -1;
+      usage();
     }
   }
-  if (whoami < 0) {
-    cerr << "must specify '--osd #' where # is the osd number" << std::endl;
-  }
   if (!dev) {
-    sprintf(dev_default, "dev/osd%d", whoami);
-    dev = dev_default;
-  }
-  if (whoami < 0) {
-    // who am i?   peek at superblock!
-    OSDSuperblock sb;
-    ObjectStore *store = new Ebofs(dev);
-    bufferlist bl;
-    store->mount();
-    int r = store->read(OSD_SUPERBLOCK_POBJECT, 0, sizeof(sb), bl);
-    if (r < 0) {
-      cerr << "couldn't read superblock object on " << dev << std::endl;
-      exit(0);
-    }
-    bl.copy(0, sizeof(sb), (char*)&sb);
-    store->umount();
-    delete store;
-    whoami = sb.whoami;
+    cerr << "must specify device file" << std::endl;
+    usage();
   }
 
-  create_courtesy_output_symlink("osd", whoami);
+  if (mkfs && whoami < 0) {
+    cerr << "must specify '--osd #' where # is the osd number" << std::endl;
+    usage();
+  }
 
   // get monmap
   MonMap monmap;
@@ -92,6 +87,26 @@ int main(int argc, const char **argv)
   if (mc.get_monmap(&monmap) < 0)
     return -1;
 
+  if (mkfs) {
+    int err = OSD::mkfs(dev, monmap.fsid, whoami);
+    if (err < 0) {
+      cerr << "error creating empty object store in " << dev << ": " << strerror(-err) << std::endl;
+      exit(1);
+    }
+    cout << "created object store for osd" << whoami << " fsid " << monmap.fsid << " on " << dev << std::endl;
+    exit(0);
+  }
+
+  if (whoami < 0) {
+    whoami = OSD::peek_whoami(dev);
+    if (whoami < 0) {
+      cerr << "unable to determine OSD identity from superblock on " << dev << ": " << strerror(-whoami) << std::endl;
+      exit(1);
+    }
+  }
+
+  create_courtesy_output_symlink("osd", whoami);
+
   // start up network
   rank.bind();
 

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index f3e12b80540ba23281d59056a2b168557fa2dcdd..070f6b7eb94a93c9d7431c4e1612df1f64c732af 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -81,9 +81,99 @@
 #define  dout(l)    if (l<=g_conf.debug || l<=g_conf.debug_osd) *_dout << dbeginl << g_clock.now() << " osd" << whoami << " " << (osdmap ? osdmap->get_epoch():0) << " "
 #define  derr(l)    if (l<=g_conf.debug || l<=g_conf.debug_osd) *_derr << dbeginl << g_clock.now() << " osd" << whoami << " " << (osdmap ? osdmap->get_epoch():0) << " "
 
+
+
+
 const char *osd_base_path = "./osddata";
 const char *ebofs_base_path = "./dev";
 
+int OSD::find_osd_dev(char *result, int whoami)
+{
+  if (!g_conf.ebofs) {// || !g_conf.osbdb) {
+    sprintf(result, "%s/osddata/osd%d", osd_base_path, whoami);
+    return 0;
+  }
+
+  char hostname[100];
+  hostname[0] = 0;
+  gethostname(hostname,100);
+
+  // try in this order:
+  // dev/osd$num
+  // dev/osd.$hostname
+  // dev/osd.all
+  struct stat sta;
+
+  sprintf(result, "%s/osd%d", ebofs_base_path, whoami);
+  if (::lstat(result, &sta) == 0) return 0;
+
+  sprintf(result, "%s/osd.%s", ebofs_base_path, hostname);    
+  if (::lstat(result, &sta) == 0) return 0;
+    
+  sprintf(result, "%s/osd.all", ebofs_base_path);
+  if (::lstat(result, &sta) == 0) return 0;
+
+  return -ENOENT;
+}
+
+
+ObjectStore *OSD::create_object_store(const char *dev)
+{
+  ObjectStore *store = 0;
+  
+  if (g_conf.ebofs) 
+    store = new Ebofs(dev);
+#ifdef USE_OSBDB
+  else if (g_conf.bdbstore)
+    store = new OSBDB(dev);
+#endif // USE_OSBDB
+  else
+    store = new FakeStore(dev);
+
+  return store;
+}
+
+
+int OSD::mkfs(const char *dev, ceph_fsid fsid, int whoami)
+{
+  ObjectStore *store = create_object_store(dev);
+  int err = store->mkfs();    
+  if (err < 0) return err;
+  err = store->mount();
+  if (err < 0) return err;
+    
+  OSDSuperblock sb;
+  sb.fsid = fsid;
+  sb.whoami = whoami;
+  bufferlist bl;
+  bl.append((const char *)&sb, sizeof(sb));
+  store->write(OSD_SUPERBLOCK_POBJECT, 0, bl.length(), bl, 0);
+  store->umount();
+  delete store;
+  return 0;
+}
+
+int OSD::peek_whoami(const char *dev)
+{
+  ObjectStore *store = create_object_store(dev);
+  int err = store->mount();
+  if (err < 0) 
+    return err;
+
+  OSDSuperblock sb;
+  bufferlist bl;
+  err = store->read(OSD_SUPERBLOCK_POBJECT, 0, sizeof(sb), bl);
+  if (err < 0) 
+    return -ENOENT;
+  bl.copy(0, sizeof(sb), (char*)&sb);
+  store->umount();
+  delete store;
+
+  return sb.whoami;
+}
+
+
+
 // <hack> force remount hack for performance testing FakeStore
 class C_Remount : public Context {
   OSD *osd;
@@ -114,12 +204,12 @@ LogType osd_logtype;
 
 OSD::OSD(int id, Messenger *m, MonMap *mm, const char *dev) : 
   timer(osd_lock),
+  whoami(id), dev_name(dev),
   stat_oprate(5.0),
   read_latency_calc(g_conf.osd_max_opq<1 ? 1:g_conf.osd_max_opq),
   qlen_calc(3),
   iat_averager(g_conf.osd_flash_crowd_iat_alpha)
 {
-  whoami = id;
   messenger = m;
   monmap = mm;
 
@@ -141,45 +231,6 @@ OSD::OSD(int id, Messenger *m, MonMap *mm, const char *dev) :
 
   if (g_conf.osd_remount_at) 
     timer.add_event_after(g_conf.osd_remount_at, new C_Remount(this));
-
-
-  // init object store
-  // try in this order:
-  // dev/osd$num
-  // dev/osd.$hostname
-  // dev/osd.all
-
-  if (dev) {
-    strcpy(dev_path,dev);
-  } else {
-    char hostname[100];
-    hostname[0] = 0;
-    gethostname(hostname,100);
-    
-    sprintf(dev_path, "%s/osd%d", ebofs_base_path, whoami);
-
-    struct stat sta;
-    if (::lstat(dev_path, &sta) != 0)
-      sprintf(dev_path, "%s/osd.%s", ebofs_base_path, hostname);    
-    
-    if (::lstat(dev_path, &sta) != 0)
-      sprintf(dev_path, "%s/osd.all", ebofs_base_path);
-  }
-
-  if (g_conf.ebofs) {
-    store = new Ebofs(dev_path);
-    //store->_fake_writes(true);
-  }
-#ifdef USE_OSBDB
-  else if (g_conf.bdbstore) {
-    store = new OSBDB(dev_path);
-  }
-#endif // USE_OSBDB
-  else {
-    sprintf(dev_path, "osddata/osd%d", whoami);
-    store = new FakeStore(dev_path);
-  }
-
 }
 
 OSD::~OSD()
@@ -196,18 +247,30 @@ int OSD::init()
 {
   Mutex::Locker lock(osd_lock);
 
-  // mkfs?
-  if (g_conf.osd_mkfs) {
-    dout(2) << "mkfs on local store" << dendl;
-    if (store->mkfs() < 0)
-      return -1;
-    
-    // make up a superblock
-    superblock.whoami = whoami;
+  char dev_path[100];
+  if (dev_name) 
+    strcpy(dev_path, dev_name);
+  else {
+    // search for a suitable dev path, based on our identity
+    int r = find_osd_dev(dev_path, whoami);
+    if (r < 0) {
+      dout(0) << "*** unable to find a dev for osd" << whoami << dendl;
+      return r;
+    }
+
+    // mkfs?
+    if (g_conf.osd_mkfs) {
+      dout(2) << "mkfs on local store" << dendl;
+      r = mkfs(dev_path, monmap->fsid, whoami);
+      if (r < 0) 
+       return r;
+    }
   }
   
   // mount.
   dout(2) << "mounting " << dev_path << dendl;
+  store = create_object_store(dev_path);
+  assert(store);
   int r = store->mount();
   if (r < 0) return -1;
   
@@ -385,17 +448,19 @@ int OSD::read_superblock()
   }
 
   bl.copy(0, sizeof(superblock), (char*)&superblock);
-  
-  dout(10) << "read_superblock " << superblock << dendl;
 
+  dout(10) << "read_superblock " << superblock << dendl;
+  assert(whoami == superblock.whoami);  // fixme!
+  
   // load up "current" osdmap
   assert(!osdmap);
   osdmap = new OSDMap;
-  bl.clear();
-  get_map_bl(superblock.current_epoch, bl);
-  osdmap->decode(bl);
+  if (superblock.current_epoch) {
+    bl.clear();
+    get_map_bl(superblock.current_epoch, bl);
+    osdmap->decode(bl);
+  }
 
-  assert(whoami == superblock.whoami);  // fixme!
   return 0;
 }
 

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index aad0dadd8822725cd9dc2bb52cf2891a5c1d588b..295fe8d7d8f0fe8f1b4abe27b7d225a236f0863b 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -63,7 +63,7 @@ protected:
   MonMap      *monmap;
 
   int whoami;
-  char dev_path[100];
+  const char *dev_name;
 
 public:
   int get_nodeid() { return whoami; }
@@ -357,6 +357,12 @@ private:
   OSD(int id, Messenger *m, MonMap *mm, const char *dev = 0);
   ~OSD();
 
+  // static bits
+  static int find_osd_dev(char *result, int whoami);
+  static ObjectStore *create_object_store(const char *dev);
+  static int mkfs(const char *dev, ceph_fsid fsid, int whoami);
+  static int peek_whoami(const char *dev);
+
   // startup/shutdown
   int init();
   int shutdown();

diff --git a/src/start.sh b/src/start.sh
index 765e80dcb7111d84f5e141884c6bdd39e486ecf4..9348bf53d49d53a821bd9ab14284cdce7c6e2814 100755 (executable)
--- a/src/start.sh
+++ b/src/start.sh
@@ -35,11 +35,11 @@ $CEPH_BIN/cmon $ARGS mondata/mon0 --debug_mon 10 --debug_ms 1
 $CEPH_BIN/osdmaptool --clobber --createsimple .ceph_monmap 4 --print .ceph_osdmap
 $CEPH_BIN/cmonctl osd setmap -i .ceph_osdmap
 
-# osds
-$CEPH_BIN/cosd $ARGS --mkfs --osd 0
-$CEPH_BIN/cosd $ARGS --mkfs --osd 1
-$CEPH_BIN/cosd $ARGS --mkfs --osd 2
-$CEPH_BIN/cosd $ARGS --mkfs --osd 3
+for osd in 0 1 2 3 
+do
+ $CEPH_BIN/cosd --mkfs_for_osd $osd dev/osd$osd  # initialize empty object store
+ $CEPH_BIN/cosd $ARGS dev/osd$osd
+done
 
 # mds
 $CEPH_BIN/cmds $ARGS --debug_mds 10