From: Sage Weil Date: Mon, 26 Jun 2017 17:22:57 +0000 (-0400) Subject: librados: allow service registrations X-Git-Tag: v12.1.1~98^2~6 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=aabe0e35a0c189190c8564d1639b832f8f4f0f9a;p=ceph.git librados: allow service registrations Signed-off-by: Sage Weil --- diff --git a/qa/workunits/rados/test.sh b/qa/workunits/rados/test.sh index 33bdd8ed0b16..cbf398fe65f1 100755 --- a/qa/workunits/rados/test.sh +++ b/qa/workunits/rados/test.sh @@ -16,6 +16,7 @@ declare -A pids for f in \ api_aio api_io api_list api_lock api_misc \ api_tier api_pool api_snapshots api_stat api_watch_notify api_cmd \ + api_service \ api_c_write_operations \ api_c_read_operations \ list_parallel \ diff --git a/src/include/rados/librados.h b/src/include/rados/librados.h index 67956636e0c3..544c2dbca393 100644 --- a/src/include/rados/librados.h +++ b/src/include/rados/librados.h @@ -3689,6 +3689,47 @@ CEPH_RADOS_API int rados_monitor_log(rados_t cluster, const char *level, CEPH_RADOS_API int rados_monitor_log2(rados_t cluster, const char *level, rados_log_callback2_t cb, void *arg); + +/** + * register daemon instance for a service + * + * Register us as a daemon providing a particular service. We identify + * the service (e.g., 'rgw') and our instance name (e.g., 'rgw.$hostname'). + * The metadata is a map of keys and values with arbitrary static metdata + * for this instance. The encoding is a series of NULL-terminated strings, + * alternating key names and values, terminating with an empty key name. + * For example, "foo\0bar\0this\0that\0\0" is the dict {foo=bar,this=that}. + * + * For the lifetime of the librados instance, regular beacons will be sent + * to the cluster to maintain our registration in the service map. + * + * @param cluster handle + * @param service service name + * @param daemon deamon instance name + * @param metadata_dict static daemon metadata dict + */ +CEPH_RADOS_API int rados_service_register( + rados_t cluster, + const char *service, + const char *daemon, + const char *metadata_dict); + +/** + * update daemon status + * + * Update our mutable status information in the service map. + * + * The status dict is encoded the same way the daemon metadata is encoded + * for rados_service_register. For example, "foo\0bar\0this\0that\0\0" is + * {foo=bar,this=that}. + * + * @param cluster rados cluster handle + * @param status_dict status dict + */ +CEPH_RADOS_API int rados_service_update_status( + rados_t cluster, + const char *status_dict); + /** @} Mon/OSD/PG commands */ /* diff --git a/src/include/rados/librados.hpp b/src/include/rados/librados.hpp index e4eb2d2506c1..0fc93a733616 100644 --- a/src/include/rados/librados.hpp +++ b/src/include/rados/librados.hpp @@ -1298,6 +1298,13 @@ namespace librados int conf_set(const char *option, const char *value); int conf_get(const char *option, std::string &val); + int service_daemon_register( + const std::string& service, ///< service name (e.g., 'rgw') + const std::string& name, ///< daemon name (e.g., 'gwfoo') + const std::map& metadata); ///< static metadata about daemon + int service_daemon_update_status( + const std::map& status); + int pool_create(const char *name); int pool_create(const char *name, uint64_t auid); int pool_create(const char *name, uint64_t auid, uint8_t crush_rule); diff --git a/src/librados/RadosClient.cc b/src/librados/RadosClient.cc index 9f43603967f5..b142267f588c 100644 --- a/src/librados/RadosClient.cc +++ b/src/librados/RadosClient.cc @@ -307,6 +307,12 @@ int librados::RadosClient::connect() monclient.sub_want("mgrmap", 0, 0); monclient.renew_subs(); + if (service_daemon) { + ldout(cct, 10) << __func__ << " registering as " << service_name << "." + << daemon_name << dendl; + mgrclient.service_daemon_register(service_name, daemon_name, + daemon_metadata); + } mgrclient.init(); objecter->set_client_incarnation(0); @@ -1014,3 +1020,49 @@ void librados::RadosClient::handle_log(MLog *m) m->put(); } + +int librados::RadosClient::service_daemon_register( + const std::string& service, ///< service name (e.g., 'rgw') + const std::string& name, ///< daemon name (e.g., 'gwfoo') + const std::map& metadata) +{ + if (service_daemon) { + return -EEXIST; + } + if (service == "osd" || + service == "mds" || + service == "client" || + service == "mon" || + service == "mgr") { + // normal ceph entity types are not allowed! + return -EINVAL; + } + if (service.empty() || name.empty()) { + return -EINVAL; + } + + ldout(cct,10) << __func__ << " " << service << "." << name << dendl; + service_daemon = true; + service_name = service; + daemon_name = name; + daemon_metadata.insert(metadata.begin(), metadata.end()); + + if (state == DISCONNECTED) { + return 0; + } + if (state == CONNECTING) { + return -EBUSY; + } + mgrclient.service_daemon_register(service_name, daemon_name, + daemon_metadata); + return 0; +} + +int librados::RadosClient::service_daemon_update_status( + const std::map& status) +{ + if (state != CONNECTED) { + return -ENOTCONN; + } + return mgrclient.service_daemon_update_status(status); +} diff --git a/src/librados/RadosClient.h b/src/librados/RadosClient.h index b3f3016310d4..6c61c7f3fb6b 100644 --- a/src/librados/RadosClient.h +++ b/src/librados/RadosClient.h @@ -78,6 +78,10 @@ private: void *log_cb_arg; string log_watch; + bool service_daemon = false; + string daemon_name, service_name; + map daemon_metadata; + int wait_for_osdmap(); public: @@ -151,6 +155,13 @@ public: void get(); bool put(); void blacklist_self(bool set); + + int service_daemon_register( + const std::string& service, ///< service name (e.g., 'rgw') + const std::string& name, ///< daemon name (e.g., 'gwfoo') + const std::map& metadata); ///< static metadata about daemon + int service_daemon_update_status( + const std::map& status); }; #endif diff --git a/src/librados/librados.cc b/src/librados/librados.cc index 1438952f8864..b1002194a4f5 100644 --- a/src/librados/librados.cc +++ b/src/librados/librados.cc @@ -2346,6 +2346,20 @@ int librados::Rados::conf_get(const char *option, std::string &val) return 0; } +int librados::Rados::service_daemon_register( + const std::string& service, ///< service name (e.g., 'rgw') + const std::string& name, ///< daemon name (e.g., 'gwfoo') + const std::map& metadata) ///< static metadata about daemon +{ + return client->service_daemon_register(service, name, metadata); +} + +int librados::Rados::service_daemon_update_status( + const std::map& status) +{ + return client->service_daemon_update_status(status); +} + int librados::Rados::pool_create(const char *name) { string str(name); diff --git a/src/test/librados/CMakeLists.txt b/src/test/librados/CMakeLists.txt index 401c8c44a4b3..bfae1d641d77 100644 --- a/src/test/librados/CMakeLists.txt +++ b/src/test/librados/CMakeLists.txt @@ -120,6 +120,15 @@ set_target_properties(ceph_test_rados_api_lock PROPERTIES COMPILE_FLAGS target_link_libraries(ceph_test_rados_api_lock rados_a ${UNITTEST_LIBS} radostest) +# ceph_test_rados_api_service +add_executable(ceph_test_rados_api_service + service.cc + ) +set_target_properties(ceph_test_rados_api_service PROPERTIES COMPILE_FLAGS + ${UNITTEST_CXX_FLAGS}) +target_link_libraries(ceph_test_rados_api_service + rados_a global ${UNITTEST_LIBS} radostest) + # ceph_test_rados_api_tier add_executable(ceph_test_rados_api_tier tier.cc diff --git a/src/test/librados/service.cc b/src/test/librados/service.cc new file mode 100644 index 000000000000..ec1ce9a09807 --- /dev/null +++ b/src/test/librados/service.cc @@ -0,0 +1,59 @@ +#include "include/rados/librados.h" +#include "include/rados/librados.hpp" +#include "test/librados/test.h" +#include "test/librados/TestCase.h" +#include "include/stringify.h" + +#include +#include +#include "gtest/gtest.h" +#include "test/unit.cc" + +using namespace librados; + +TEST(LibRadosServicePP, RegisterEarly) { + Rados cluster; + cluster.init("admin"); + ASSERT_EQ(0, cluster.conf_read_file(NULL)); + cluster.conf_parse_env(NULL); + string name = string("pid") + stringify(getpid()); + ASSERT_EQ(0, cluster.service_daemon_register( + "laundry", name, {{"foo", "bar"}, {"this", "that"}})); + ASSERT_EQ(-EEXIST, cluster.service_daemon_register( + "laundry", name, {{"foo", "bar"}, {"this", "that"}})); + ASSERT_EQ(0, cluster.connect()); + sleep(5); + cluster.shutdown(); +} + +TEST(LibRadosServicePP, RegisterLate) { + Rados cluster; + cluster.init("admin"); + ASSERT_EQ(0, cluster.conf_read_file(NULL)); + cluster.conf_parse_env(NULL); + ASSERT_EQ("", connect_cluster_pp(cluster)); + string name = string("pid") + stringify(getpid()); + ASSERT_EQ(0, cluster.service_daemon_register( + "laundry", name, {{"foo", "bar"}, {"this", "that"}})); +} + +TEST(LibRadosServicePP, Status) { + Rados cluster; + cluster.init("admin"); + ASSERT_EQ(0, cluster.conf_read_file(NULL)); + cluster.conf_parse_env(NULL); + string name = string("pid") + stringify(getpid()); + ASSERT_EQ(-ENOTCONN, cluster.service_daemon_update_status( + {{"testing", "starting"}})); + ASSERT_EQ(0, cluster.connect()); + ASSERT_EQ(0, cluster.service_daemon_register( + "laundry", name, {{"foo", "bar"}, {"this", "that"}})); + for (int i=0; i<20; ++i) { + ASSERT_EQ(0, cluster.service_daemon_update_status({ + {"testing", "running"}, + {"count", stringify(i)} + })); + sleep(1); + } + cluster.shutdown(); +}