From: Noah Watkins Date: Wed, 6 Mar 2013 02:00:26 +0000 (-0800) Subject: libcephfs: return osd location from crush map X-Git-Tag: v0.60~125^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=ee158ed267cebf8641e5ff18294e8c2e4e76bde8;p=ceph.git libcephfs: return osd location from crush map Adds ceph_get_osd_crush_location that returns a list of (type,name) pairs for devices that exist between a given osd and the root of the bucket hierarchy. Signed-off-by: Noah Watkins --- diff --git a/src/client/Client.cc b/src/client/Client.cc index d8ba59adb600..247927fba479 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -7570,6 +7570,12 @@ int Client::get_file_extent_osds(int fd, loff_t off, loff_t *len, vector& o return 0; } +int Client::get_osd_crush_location(int id, vector >& path) +{ + Mutex::Locker lock(client_lock); + return osdmap->crush->get_full_location_ordered(id, path); +} + int Client::get_file_stripe_address(int fd, loff_t offset, vector& address) { Mutex::Locker lock(client_lock); diff --git a/src/client/Client.h b/src/client/Client.h index b0dd069bbe55..9302c3e82a42 100644 --- a/src/client/Client.h +++ b/src/client/Client.h @@ -673,6 +673,7 @@ public: int get_pool_replication(int64_t pool); int64_t get_pool_id(const char *pool_name); string get_pool_name(int64_t pool); + int get_osd_crush_location(int id, vector >& path); int enumerate_layout(int fd, vector& result, loff_t length, loff_t offset); diff --git a/src/crush/CrushWrapper.cc b/src/crush/CrushWrapper.cc index 53e3c1cc6495..b2735b31926d 100644 --- a/src/crush/CrushWrapper.cc +++ b/src/crush/CrushWrapper.cc @@ -133,12 +133,25 @@ bool CrushWrapper::check_item_loc(CephContext *cct, int item, const map CrushWrapper::get_full_location(int id){ - +map CrushWrapper::get_full_location(int id) +{ + vector > full_location_ordered; map full_location; + + get_full_location_ordered(id, full_location_ordered); + + std::copy(full_location_ordered.begin(), + full_location_ordered.end(), + std::inserter(full_location, full_location.begin())); + + return full_location; +} + +int CrushWrapper::get_full_location_ordered(int id, vector >& path) +{ + int parent_id, ret; pair parent_coord; - parent_coord = get_immediate_parent(id); - int parent_id; + parent_coord = get_immediate_parent(id, &ret); // read the type map and get the name of the type with the largest ID int high_type = 0; @@ -149,19 +162,19 @@ map CrushWrapper::get_full_location(int id){ string high_type_name = type_map[high_type]; - full_location[ parent_coord.first ] = parent_coord.second; + path.push_back(parent_coord); parent_id = get_item_id( (parent_coord.second).c_str() ); while (parent_coord.first != high_type_name) { parent_coord = get_immediate_parent(parent_id); - full_location[ parent_coord.first ] = parent_coord.second; + path.push_back(parent_coord); if ( parent_coord.first != high_type_name ){ parent_id = get_item_id( (parent_coord.second).c_str() ); } } - return full_location; + return ret; } @@ -439,9 +452,10 @@ bool CrushWrapper::check_item_present(int id) } -pair CrushWrapper::get_immediate_parent(int id) +pair CrushWrapper::get_immediate_parent(int id, int *_ret) { pair loc; + int ret = -ENOENT; for (int bidx = 0; bidx < crush->max_buckets; bidx++) { crush_bucket *b = crush->buckets[bidx]; @@ -452,9 +466,13 @@ pair CrushWrapper::get_immediate_parent(int id) string parent_id = name_map[b->id]; string parent_bucket_type = type_map[b->type]; loc = make_pair(parent_bucket_type, parent_id); + ret = 0; } } + if (_ret) + *_ret = ret; + return loc; } diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h index 0b919cba3ec5..98406d6ce4c3 100644 --- a/src/crush/CrushWrapper.h +++ b/src/crush/CrushWrapper.h @@ -286,7 +286,7 @@ public: /** * returns the (type, name) of the parent bucket of id */ - pair get_immediate_parent(int id); + pair get_immediate_parent(int id, int *ret = NULL); int get_immediate_parent_id(int id, int *parent); /** @@ -300,6 +300,14 @@ public: */ map get_full_location(int id); + /* + * identical to get_full_location(int id) although it returns the type/name + * pairs in the order they occur in the hierarchy. + * + * returns -ENOENT if id is not found. + */ + int get_full_location_ordered(int id, vector >& path); + /** * returns (type_id, type) of all parent buckets between id and * default, can be used to check for anomolous CRUSH maps diff --git a/src/include/cephfs/libcephfs.h b/src/include/cephfs/libcephfs.h index afad8cf1e3d3..fc92b5d45b02 100644 --- a/src/include/cephfs/libcephfs.h +++ b/src/include/cephfs/libcephfs.h @@ -912,6 +912,23 @@ int ceph_get_file_stripe_address(struct ceph_mount_info *cmount, int fd, loff_t int ceph_get_file_extent_osds(struct ceph_mount_info *cmount, int fh, loff_t offset, loff_t *length, int *osds, int nosds); +/** + * Get the fully qualified CRUSH location of an OSD. + * + * Returns (type, name) string pairs for each device in the CRUSH bucket + * hierarchy starting from the given osd to the root. Each pair element is + * separated by a NULL character. + * + * @param cmount the ceph mount handle to use. + * @param osd the OSD id. + * @param path buffer to store location. + * @param len size of buffer. + * @returns the amount of bytes written into the buffer, or -ERANGE if the + * array is not large enough. + */ +int ceph_get_osd_crush_location(struct ceph_mount_info *cmount, + int osd, char *path, size_t len); + /** * Get the file layout stripe unit granularity. * @param cmount the ceph mount handle. diff --git a/src/libcephfs.cc b/src/libcephfs.cc index 0c77125fb50b..35f1fd6ad363 100644 --- a/src/libcephfs.cc +++ b/src/libcephfs.cc @@ -816,6 +816,44 @@ extern "C" int ceph_get_file_extent_osds(struct ceph_mount_info *cmount, int fh, return vosds.size(); } +extern "C" int ceph_get_osd_crush_location(struct ceph_mount_info *cmount, + int osd, char *path, size_t len) +{ + if (!cmount->is_mounted()) + return -ENOTCONN; + + if (!path) + return -EINVAL; + + vector > loc; + int ret = cmount->get_client()->get_osd_crush_location(osd, loc); + if (ret) + return ret; + + size_t needed = 0; + size_t cur = 0; + vector >::iterator it; + for (it = loc.begin(); it != loc.end(); it++) { + string& type = it->first; + string& name = it->second; + needed += type.size() + name.size() + 2; + if (needed < len) { + strcpy(path + cur, type.c_str()); + cur += type.size() + 1; + strcpy(path + cur, name.c_str()); + cur += name.size() + 1; + } + } + + if (len == 0) + return needed; + + if (needed > len) + return -ERANGE; + + return needed; +} + extern "C" int ceph_get_file_stripe_address(struct ceph_mount_info *cmount, int fh, loff_t offset, struct sockaddr_storage *addr, int naddr) { diff --git a/src/test/libcephfs/test.cc b/src/test/libcephfs/test.cc index 4afe01c32d05..c23eedbe78cf 100644 --- a/src/test/libcephfs/test.cc +++ b/src/test/libcephfs/test.cc @@ -980,3 +980,52 @@ TEST(LibCephFS, GetExtentOsds) { ceph_shutdown(cmount); } + +TEST(LibCephFS, GetOsdCrushLocation) { + struct ceph_mount_info *cmount; + ASSERT_EQ(ceph_create(&cmount, NULL), 0); + + EXPECT_EQ(-ENOTCONN, ceph_get_osd_crush_location(cmount, 0, NULL, 0)); + + ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0); + ASSERT_EQ(ceph_mount(cmount, NULL), 0); + + ASSERT_EQ(ceph_get_osd_crush_location(cmount, 0, NULL, 0), -EINVAL); + + char path[256]; + ASSERT_EQ(ceph_get_osd_crush_location(cmount, 9999999, path, 0), -ENOENT); + ASSERT_EQ(ceph_get_osd_crush_location(cmount, -1, path, 0), -ENOENT); + + char test_file[256]; + sprintf(test_file, "test_osds_loc_%d", getpid()); + int fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0666); + ASSERT_GT(fd, 0); + + /* get back how many osds > 0 */ + int ret = ceph_get_file_extent_osds(cmount, fd, 0, NULL, NULL, 0); + EXPECT_GT(ret, 0); + + /* full stripe extent */ + int osds[ret]; + EXPECT_EQ(ret, ceph_get_file_extent_osds(cmount, fd, 0, NULL, osds, ret)); + + ASSERT_GT(ceph_get_osd_crush_location(cmount, 0, path, 0), 0); + ASSERT_EQ(ceph_get_osd_crush_location(cmount, 0, path, 1), -ERANGE); + + for (int i = 0; i < ret; i++) { + int len = ceph_get_osd_crush_location(cmount, osds[i], path, sizeof(path)); + ASSERT_GT(len, 0); + int pos = 0; + while (pos < len) { + std::string type(path + pos); + ASSERT_GT((int)type.size(), 0); + pos += type.size() + 1; + + std::string name(path + pos); + ASSERT_GT((int)name.size(), 0); + pos += name.size() + 1; + } + } + + ceph_shutdown(cmount); +}