{
int r;
+ // first try bluestore -- it has a crc on its header and will fail
+ // reliably.
+ r = BlueStore::get_block_device_fsid(path, fsid);
+ if (r == 0)
+ return r;
+
// okay, try FileStore (journal).
r = FileStore::get_block_device_fsid(path, fsid);
if (r == 0)
return 0;
}
-int BlueFS::mkfs(uint64_t super_offset_a, uint64_t super_offset_b)
+int BlueFS::mkfs(uuid_d osd_uuid)
{
dout(1) << __func__
- << " super offsets " << super_offset_a << " " << super_offset_b
+ << " osd_uuid " << osd_uuid
<< dendl;
assert(bdev.size() >= 1);
_init_alloc();
+ super.osd_uuid = osd_uuid;
super.uuid.generate_random();
dout(1) << __func__ << " uuid " << super.uuid << dendl;
_flush_log();
// write supers
- super.version = 0;
- super.super_a_offset = super_offset_a;
- super.super_b_offset = super_offset_b;
+ super.version = 1;
super.block_size = bdev[0]->get_block_size();
super.log_fnode = log_file->fnode;
_write_super();
_flush_bdev();
- super.version = 1;
- _write_super();
- _flush_bdev();
// clean up
super = bluefs_super_t();
}
}
-int BlueFS::mount(uint64_t super_offset_a, uint64_t super_offset_b)
+int BlueFS::mount()
{
- dout(1) << __func__ << " super offsets " << super_offset_a
- << " " << super_offset_b << dendl;
+ dout(1) << __func__ << dendl;
assert(!bdev.empty());
- int r = _open_super(super_offset_a, super_offset_b);
+ int r = _open_super();
if (r < 0) {
derr << __func__ << " failed to open super: " << cpp_strerror(r) << dendl;
goto out;
::encode(super, bl);
uint32_t crc = bl.crc32c(-1);
::encode(crc, bl);
- assert(bl.length() <= super.block_size);
- bufferptr z(super.block_size - bl.length());
+ assert(bl.length() <= get_super_length());
+ bufferptr z(get_super_length() - bl.length());
z.zero();
bl.append(z);
bl.rebuild();
- uint64_t off = (super.version & 1) ?
- super.super_b_offset : super.super_a_offset;
IOContext ioc(NULL);
- bdev[0]->aio_write(off, bl, &ioc);
+ bdev[0]->aio_write(get_super_offset(), bl, &ioc);
bdev[0]->aio_submit(&ioc);
ioc.aio_wait();
dout(20) << __func__ << " v " << super.version << " crc " << crc
- << " offset " << off << dendl;
+ << " offset " << get_super_offset() << dendl;
return 0;
}
-int BlueFS::_open_super(uint64_t super_offset_a, uint64_t super_offset_b)
+int BlueFS::_open_super()
{
dout(10) << __func__ << dendl;
- bufferlist abl, bbl, t;
- bluefs_super_t a, b;
- uint32_t a_crc, b_crc, crc;
+ bufferlist bl, t;
+ uint32_t expected_crc, crc;
int r;
- r = bdev[0]->read(super_offset_a, bdev[0]->get_block_size(), &abl, ioc[0]);
- if (r < 0)
- return r;
- r = bdev[0]->read(super_offset_b, bdev[0]->get_block_size(), &bbl, ioc[0]);
+ // always the second block
+ r = bdev[0]->read(get_super_offset(), get_super_length(),
+ &bl, ioc[0]);
if (r < 0)
return r;
- bufferlist::iterator p = abl.begin();
- ::decode(a, p);
+ bufferlist::iterator p = bl.begin();
+ ::decode(super, p);
{
bufferlist t;
- t.substr_of(abl, 0, p.get_off());
+ t.substr_of(bl, 0, p.get_off());
crc = t.crc32c(-1);
}
- ::decode(a_crc, p);
- if (crc != a_crc) {
- derr << __func__ << " bad crc on superblock a, expected " << a_crc
+ ::decode(expected_crc, p);
+ if (crc != expected_crc) {
+ derr << __func__ << " bad crc on superblock, expected " << expected_crc
<< " != actual " << crc << dendl;
return -EIO;
}
-
- p = bbl.begin();
- ::decode(b, p);
- {
- bufferlist t;
- t.substr_of(bbl, 0, p.get_off());
- crc = t.crc32c(-1);
- }
- ::decode(b_crc, p);
- if (crc != b_crc) {
- derr << __func__ << " bad crc on superblock a, expected " << a_crc
- << " != actual " << crc << dendl;
- return -EIO;
- }
-
- dout(10) << __func__ << " superblock a " << a.version
- << " b " << b.version
- << dendl;
-
- if (a.version == b.version + 1) {
- dout(10) << __func__ << " using a" << dendl;
- super = a;
- } else if (b.version == a.version + 1) {
- dout(10) << __func__ << " using b" << dendl;
- super = b;
- } else {
- derr << __func__ << " non-adjacent superblock versions "
- << a.version << " and " << b.version << dendl;
- return -EIO;
- }
+ dout(10) << __func__ << " superblock " << super.version << dendl;
dout(10) << __func__ << " log_fnode " << super.log_fnode << dendl;
return 0;
}
assert(q != dir_map.end());
map<string,FileRef>::iterator r = q->second->file_map.find(filename);
assert(r != q->second->file_map.end());
- if (--r->second->refs == 0) {
- file_map.erase(r->second->fnode.ino);
- }
+ --r->second->refs;
q->second->file_map.erase(r);
}
break;
dout(10) << __func__ << " log file size was " << log_file->fnode.size << dendl;
delete log_reader;
+ // verify file link counts are all >0
+ for (auto& p : file_map) {
+ if (p.second->refs == 0 &&
+ p.second->fnode.ino > 1) {
+ derr << __func__ << " file with link count 0: " << p.second->fnode
+ << dendl;
+ return -EIO;
+ }
+ }
+
dout(10) << __func__ << " done" << dendl;
return 0;
}
void _invalidate_cache(FileRef f, uint64_t offset, uint64_t length);
- int _open_super(uint64_t super_offset_a, uint64_t super_offset_b);
+ int _open_super();
int _write_super();
int _replay(); ///< replay journal
+ // always put the super in the second 4k block. FIXME should this be
+ // block size independent?
+ unsigned get_super_offset() {
+ return 4096;
+ }
+ unsigned get_super_length() {
+ return 4096;
+ }
+
public:
BlueFS();
~BlueFS();
// the super is always stored on bdev 0
- int mkfs(uint64_t super_offset_a, uint64_t super_offset_b);
- int mount(uint64_t super_offset_a, uint64_t super_offset_b);
+ int mkfs(uuid_d osd_uuid);
+ int mount();
void umount();
int fsck();
const string PREFIX_WAL = "L"; // write ahead log
const string PREFIX_ALLOC = "B"; // block allocator
+// write a label in the first block. always use this size. note that
+// bluefs makes a matching assumption about the location of its
+// superblock (always the second block of the device).
+#define BDEV_LABEL_BLOCK_SIZE 4096
+
/*
* object name key structure
*
// XXX
}
-int BlueStore::peek_journal_fsid(uuid_d *fsid)
+int BlueStore::get_block_device_fsid(const string& path, uuid_d *fsid)
{
+ bluestore_bdev_label_t label;
+ int r = _read_bdev_label(path, &label);
+ if (r < 0)
+ return r;
+ *fsid = label.osd_uuid;
return 0;
}
fs = NULL;
}
-int BlueStore::_open_bdev()
+int BlueStore::_write_bdev_label(string path, bluestore_bdev_label_t label)
+{
+ dout(10) << __func__ << " path " << path << " label " << label << dendl;
+ bufferlist bl;
+ ::encode(label, bl);
+ uint32_t crc = bl.crc32c(-1);
+ ::encode(crc, bl);
+ assert(bl.length() <= BDEV_LABEL_BLOCK_SIZE);
+ bufferptr z(BDEV_LABEL_BLOCK_SIZE - bl.length());
+ z.zero();
+ bl.append(z);
+
+ int fd = ::open(path.c_str(), O_WRONLY);
+ if (fd < 0) {
+ fd = errno;
+ derr << __func__ << " failed to open " << path << ": " << cpp_strerror(fd)
+ << dendl;
+ return fd;
+ }
+ int r = bl.write_fd(fd);
+ if (r < 0) {
+ derr << __func__ << " failed to write to " << path
+ << ": " << cpp_strerror(r) << dendl;
+ }
+ VOID_TEMP_FAILURE_RETRY(::close(fd));
+ return r;
+}
+
+int BlueStore::_read_bdev_label(string path, bluestore_bdev_label_t *label)
+{
+ dout(10) << __func__ << dendl;
+ int fd = ::open(path.c_str(), O_RDONLY);
+ if (fd < 0) {
+ fd = errno;
+ derr << __func__ << " failed to open " << path << ": " << cpp_strerror(fd)
+ << dendl;
+ return fd;
+ }
+ bufferlist bl;
+ int r = bl.read_fd(fd, BDEV_LABEL_BLOCK_SIZE);
+ if (r < 0) {
+ derr << __func__ << " failed to read from " << path
+ << ": " << cpp_strerror(r) << dendl;
+ }
+ VOID_TEMP_FAILURE_RETRY(::close(fd));
+
+ uint32_t crc, expected_crc;
+ bufferlist::iterator p = bl.begin();
+ try {
+ ::decode(*label, p);
+ bufferlist t;
+ t.substr_of(bl, 0, p.get_off());
+ crc = t.crc32c(-1);
+ ::decode(expected_crc, p);
+ }
+ catch (buffer::error& e) {
+ return -EINVAL;
+ }
+ if (crc != expected_crc) {
+ derr << __func__ << " bad crc on label, expected " << expected_crc
+ << " != actual " << crc << dendl;
+ return -EIO;
+ }
+ dout(10) << __func__ << " got " << *label << dendl;
+ return 0;
+}
+
+int BlueStore::_open_bdev(bool create)
{
+ bluestore_bdev_label_t label;
assert(bdev == NULL);
bdev = new BlockDevice(aio_cb, static_cast<void*>(this));
string p = path + "/block";
int r = bdev->open(p);
- if (r < 0) {
- delete bdev;
- bdev = NULL;
+ if (r < 0)
+ goto fail;
+
+ if (create) {
+ label.osd_uuid = fsid;
+ label.size = bdev->get_size();
+ label.btime = ceph_clock_now(NULL);
+ label.description = "main";
+ r = _write_bdev_label(p, label);
+ if (r < 0)
+ goto fail;
+ } else {
+ r = _read_bdev_label(p, &label);
+ if (r < 0)
+ goto fail;
+ if (label.osd_uuid != fsid) {
+ derr << __func__ << " bdev " << p << " fsid " << label.osd_uuid
+ << " does not match our fsid " << fsid << dendl;
+ r = -EIO;
+ goto fail;
+ }
}
+ return 0;
+
+ fail:
+ delete bdev;
+ bdev = NULL;
return r;
}
goto out_path;
r = _lock_fsid();
if (r < 0)
- ret = true; // if we can't lock, it is in used
+ ret = true; // if we can't lock, it is in use
_close_fsid();
out_path:
_close_path();
struct stat st;
if (::stat(bfn, &st) == 0) {
bluefs->add_block_device(1, bfn);
+ // label in first 4k
+ bluestore_bdev_label_t label;
if (create) {
- bluefs->add_block_extent(1, 0, bluefs->get_block_device_size(1));
+ label.osd_uuid = fsid;
+ label.size = bluefs->get_block_device_size(1);
+ label.btime = ceph_clock_now(NULL);
+ label.description = "bluefs wal";
+ int r = _write_bdev_label(bfn, label);
+ if (r < 0)
+ return r;
+ // reset belongs to bluefs
+ bluefs->add_block_extent(
+ 1, BDEV_LABEL_BLOCK_SIZE,
+ bluefs->get_block_device_size(1) - BDEV_LABEL_BLOCK_SIZE);
+ } else {
+ int r = _read_bdev_label(bfn, &label);
+ if (r < 0)
+ return r;
+ if (label.osd_uuid != fsid) {
+ derr << __func__ << " bdev " << bfn << " fsid " << label.osd_uuid
+ << " does not match our fsid " << fsid << dendl;
+ return -EIO;
+ }
}
}
if (create) {
- bluefs->mkfs(0, 4096);
+ bluefs->mkfs(fsid);
}
- int r = bluefs->mount(0, 4096);
+ int r = bluefs->mount();
if (r < 0) {
derr << __func__ << " failed bluefs mount: " << cpp_strerror(r) << dendl;
delete bluefs;
} else {
dout(1) << __func__ << " using provided fsid " << fsid << dendl;
}
- r = _write_fsid();
- if (r < 0)
- goto out_close_fsid;
+ // we'll write it last.
} else {
if (!fsid.is_zero() && fsid != old_fsid) {
derr << __func__ << " on-disk fsid " << old_fsid
goto out_close_fsid;
}
fsid = old_fsid;
- dout(1) << __func__ << " fsid is already set to " << fsid << dendl;
+ dout(1) << __func__ << " already created, fsid is " << fsid << dendl;
+ goto out_close_fsid;
}
// block symlink/file
}
}
- r = _open_bdev();
+ r = _open_bdev(true);
if (r < 0)
goto out_close_fsid;
db->submit_transaction_sync(t);
}
- // FIXME: superblock
-
- dout(10) << __func__ << " success" << dendl;
- r = 0;
+ // indicate mkfs completion/success by writing the fsid file
+ r = _write_fsid();
+ if (r == 0)
+ dout(10) << __func__ << " success" << dendl;
+ else
+ derr << __func__ << " error writing fsid: " << cpp_strerror(r) << dendl;
_close_alloc();
out_close_db:
if (r < 0)
goto out_fsid;
- r = _open_bdev();
+ r = _open_bdev(false);
if (r < 0)
goto out_fsid;
if (r < 0)
goto out_fsid;
- r = _open_bdev();
+ r = _open_bdev(false);
if (r < 0)
goto out_fsid;
int _read_fsid(uuid_d *f);
int _write_fsid();
void _close_fsid();
- int _open_bdev();
+ int _open_bdev(bool create);
void _close_bdev();
int _open_db(bool create);
void _close_db();
int _open_collections(int *errors=0);
void _close_collections();
+ int _write_bdev_label(string path, bluestore_bdev_label_t label);
+ static int _read_bdev_label(string path, bluestore_bdev_label_t *label);
+
int _open_super_meta();
int _reconcile_bluefs_freespace();
bool wants_journal() { return false; };
bool allows_journal() { return false; };
- int peek_journal_fsid(uuid_d *fsid);
+ static int get_block_device_fsid(const string& path, uuid_d *fsid);
bool test_mount_in_use();
fs.add_block_device(i-1, args[i]);
}
- int r = fs.mount(0, 4096);
+ int r = fs.mount();
assert(r == 0);
vector<string> dirs;
{
ENCODE_START(1, 1, bl);
::encode(uuid, bl);
+ ::encode(osd_uuid, bl);
::encode(version, bl);
- ::encode(super_a_offset, bl);
- ::encode(super_b_offset, bl);
::encode(block_size, bl);
::encode(log_fnode, bl);
ENCODE_FINISH(bl);
{
DECODE_START(1, p);
::decode(uuid, p);
+ ::decode(osd_uuid, p);
::decode(version, p);
- ::decode(super_a_offset, p);
- ::decode(super_b_offset, p);
::decode(block_size, p);
::decode(log_fnode, p);
DECODE_FINISH(p);
void bluefs_super_t::dump(Formatter *f) const
{
f->dump_stream("uuid") << uuid;
+ f->dump_stream("osd_uuid") << osd_uuid;
f->dump_unsigned("version", version);
- f->dump_unsigned("super_a_offset", super_a_offset);
- f->dump_unsigned("super_b_offset", super_b_offset);
f->dump_unsigned("block_size", block_size);
f->dump_object("log_fnode", log_fnode);
}
ls.push_back(new bluefs_super_t);
ls.back()->version = 1;
ls.back()->block_size = 4096;
- ls.back()->super_a_offset = 8192;
- ls.back()->super_a_offset = 16384;
}
ostream& operator<<(ostream& out, const bluefs_super_t& s)
{
return out << "super(" << s.uuid
+ << " osd " << s.osd_uuid
<< " v " << s.version
- << " loc " << s.super_a_offset << "," << s.super_b_offset
<< " block_size " << s.block_size
<< " log_fnode " << s.log_fnode
<< ")";
struct bluefs_super_t {
- uuid_d uuid;
+ uuid_d uuid; ///< unique to this bluefs instance
+ uuid_d osd_uuid; ///< matches the osd that owns us
uint64_t version;
- uint64_t super_a_offset;
- uint64_t super_b_offset;
uint32_t block_size;
bluefs_fnode_t log_fnode;
bluefs_super_t()
: version(0),
- super_a_offset(0),
- super_b_offset(0),
block_size(4096) { }
uint64_t block_mask() const {
#include "bluestore_types.h"
#include "common/Formatter.h"
+#include "include/stringify.h"
+
+// bluestore_bdev_label_t
+
+void bluestore_bdev_label_t::encode(bufferlist& bl) const
+{
+ // be slightly friendly to someone who looks at the device
+ bl.append("bluestore block device\n");
+ bl.append(stringify(osd_uuid));
+ bl.append("\n");
+ ENCODE_START(1, 1, bl);
+ ::encode(osd_uuid, bl);
+ ::encode(size, bl);
+ ::encode(btime, bl);
+ ::encode(description, bl);
+ ENCODE_FINISH(bl);
+}
+
+void bluestore_bdev_label_t::decode(bufferlist::iterator& p)
+{
+ p.advance(60); // see above
+ DECODE_START(1, p);
+ ::decode(osd_uuid, p);
+ ::decode(size, p);
+ ::decode(btime, p);
+ ::decode(description, p);
+ DECODE_FINISH(p);
+}
+
+void bluestore_bdev_label_t::dump(Formatter *f) const
+{
+ f->dump_stream("osd_uuid") << osd_uuid;
+ f->dump_unsigned("size", size);
+ f->dump_stream("btime") << btime;
+ f->dump_string("description", description);
+}
+
+void bluestore_bdev_label_t::generate_test_instances(
+ list<bluestore_bdev_label_t*>& o)
+{
+ o.push_back(new bluestore_bdev_label_t);
+ o.push_back(new bluestore_bdev_label_t);
+ o.back()->size = 123;
+ o.back()->btime = utime_t(4, 5);
+ o.back()->description = "fakey";
+}
+
+ostream& operator<<(ostream& out, const bluestore_bdev_label_t& l)
+{
+ return out << "bdev(osd_uuid " << l.osd_uuid
+ << " size " << l.size
+ << " btime " << l.btime
+ << " desc " << l.description << ")";
+}
// cnode_t
#include <ostream>
#include "include/types.h"
#include "include/interval_set.h"
+#include "include/utime.h"
#include "common/hobject.h"
namespace ceph {
class Formatter;
}
+/// label for block device
+struct bluestore_bdev_label_t {
+ uuid_d osd_uuid; ///< osd uuid
+ uint64_t size; ///< device size
+ utime_t btime; ///< birth time
+ string description; ///< device description
+
+ void encode(bufferlist& bl) const;
+ void decode(bufferlist::iterator& p);
+ void dump(Formatter *f) const;
+ static void generate_test_instances(list<bluestore_bdev_label_t*>& o);
+};
+WRITE_CLASS_ENCODER(bluestore_bdev_label_t)
+
+ostream& operator<<(ostream& out, const bluestore_bdev_label_t& l);
+
/// collection metadata
struct cnode_t {
uint32_t bits; ///< how many bits of coll pgid are significant
TEST(BlueFS, mkfs) {
uint64_t size = 1048476 * 128;
string fn = get_temp_bdev(size);
+ uuid_d fsid;
BlueFS fs;
fs.add_block_device(0, fn);
fs.add_block_extent(0, 1048576, size - 1048576);
- fs.mkfs(0, 4096);
+ fs.mkfs(fsid);
rm_temp_bdev(fn);
}
BlueFS fs;
ASSERT_EQ(0, fs.add_block_device(0, fn));
fs.add_block_extent(0, 1048576, size - 1048576);
- ASSERT_EQ(0, fs.mkfs(0, 4096));
- ASSERT_EQ(0, fs.mount(0, 4096));
+ uuid_d fsid;
+ ASSERT_EQ(0, fs.mkfs(fsid));
+ ASSERT_EQ(0, fs.mount());
ASSERT_EQ(fs.get_total(0), size - 1048576);
ASSERT_LT(fs.get_free(0), size - 1048576);
fs.umount();
BlueFS fs;
ASSERT_EQ(0, fs.add_block_device(0, fn));
fs.add_block_extent(0, 1048576, size - 1048576);
- ASSERT_EQ(0, fs.mkfs(0, 4096));
- ASSERT_EQ(0, fs.mount(0, 4096));
+ uuid_d fsid;
+ ASSERT_EQ(0, fs.mkfs(fsid));
+ ASSERT_EQ(0, fs.mount());
{
BlueFS::FileWriter *h;
ASSERT_EQ(0, fs.mkdir("dir"));
BlueFS fs;
ASSERT_EQ(0, fs.add_block_device(0, fn));
fs.add_block_extent(0, 1048576, size - 1048576);
- ASSERT_EQ(0, fs.mkfs(0, 4096));
- ASSERT_EQ(0, fs.mount(0, 4096));
+ uuid_d fsid;
+ ASSERT_EQ(0, fs.mkfs(fsid));
+ ASSERT_EQ(0, fs.mount());
{
BlueFS::FileWriter *h;
ASSERT_EQ(0, fs.mkdir("dir"));
projects / ceph.git / commitdiff