From 964a0a6e1326d4f773c547655ebb2a5c97794268 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 19 Apr 2011 11:09:54 -0700 Subject: [PATCH] osd: load classes from a fixed location off disk Simplify the class loading code to just load out of a local directory. Do not request classes from the monitor. Signed-off-by: Sage Weil --- src/common/ClassHandler.cc | 408 +++++++++---------------------------- src/common/ClassHandler.h | 74 +++---- src/common/config.cc | 2 +- src/common/config.h | 2 +- src/objclass/class_api.cc | 40 ++-- src/osd/OSD.cc | 100 ++------- src/osd/OSD.h | 13 +- src/osd/ReplicatedPG.cc | 23 ++- src/vstart.sh | 1 + 9 files changed, 166 insertions(+), 497 deletions(-) diff --git a/src/common/ClassHandler.cc b/src/common/ClassHandler.cc index 2afb44d2667c9..6465d54ef2caf 100644 --- a/src/common/ClassHandler.cc +++ b/src/common/ClassHandler.cc @@ -4,7 +4,6 @@ #include "osd/OSD.h" #include "messages/MClass.h" #include "ClassHandler.h" -#include "common/arch.h" #include @@ -16,192 +15,113 @@ #undef dout_prefix #define dout_prefix *_dout -static ClassHandler::ClassData null_cls_data; - -int ClassHandler::_load_class(ClassData &cls) +int ClassHandler::open_class(const string& cname, ClassData **pcls) { - int ret; - dout(10) << "load_class " << cls.name << dendl; - - cls_deps_t *(*cls_deps)(); - - char fname[80]; - snprintf(fname, sizeof(fname), "%s/class-XXXXXX", - g_conf.osd_class_tmp.c_str()); - - int fd = mkstemp(fname); - if (fd < 0) { - dout(0) << "could not create temp file " << fname << dendl; - return -errno; - } - cls.impl.binary.write_fd(fd); - close(fd); - - cls.handle = dlopen(fname, RTLD_NOW); - - if (!cls.handle) { - dout(0) << "could not open class (dlopen failed): " << dlerror() << dendl; - ret = -EIO; - goto done; + Mutex::Locker lock(mutex); + ClassData *cls = _get_class(cname); + if (cls->status != ClassData::CLASS_OPEN) { + int r = _load_class(cls); + if (r) + return r; } - cls_deps = (cls_deps_t *(*)())dlsym(cls.handle, "class_deps"); - if (cls_deps) { - cls_deps_t *deps = cls_deps(); - while (deps) { - if (!deps->name) - break; - cls._add_dependency(deps); - deps++; - } - } - ret = cls.load(); -done: - unlink(fname); - return ret; -} - -int ClassHandler::load_class(const string& cname) -{ - int ret; - - ClassData& cls = get_obj(cname); - if (&cls == &null_cls_data) { - dout(0) << "ERROR: can't load null class data" << dendl; - return -EINVAL; - } - cls.mutex->Lock(); - ret = _load_class(cls); - cls.mutex->Unlock(); - - return ret; + *pcls = cls; + return 0; } -ClassHandler::ClassData& ClassHandler::get_obj(const string& cname) +ClassHandler::ClassData *ClassHandler::_get_class(const string& cname) { - Mutex::Locker locker(mutex); + ClassData *cls; map::iterator iter = classes.find(cname); - if (iter == classes.end()) { - ClassData& cls = classes[cname]; - cls.mutex = new Mutex("ClassData"); - if (!cls.mutex) { - classes[cname] = null_cls_data; - return null_cls_data; - } - dout(0) << "get_obj: adding new class name=" << cname << " ptr=" << &cls << dendl; - cls.name = cname; - cls.osd = osd; - cls.handler = this; - return cls; - } - return iter->second; + if (iter != classes.end()) { + cls = &iter->second; + } else { + cls = &classes[cname]; + dout(10) << "_get_class adding new class name " << cname << " " << cls << dendl; + cls->name = cname; + cls->handler = this; + } + return cls; } -ClassHandler::ClassData *ClassHandler::get_class(const string& cname, ClassVersion& version) +int ClassHandler::_load_class(ClassData *cls) { - ClassData *ret = NULL; - ClassData *cls = &get_obj(cname); - if (cls == &null_cls_data) - return NULL; - - Mutex::Locker lock(*cls->mutex); + // already open + if (cls->status == ClassData::CLASS_OPEN) + return 0; - switch (cls->status) { - case ClassData::CLASS_LOADED: - case ClassData::CLASS_ERROR: - case ClassData::CLASS_INVALID: - if (cls->cache_timed_out()) { - dout(0) << "class timed out going to send request for " << cname.c_str() << " v" << version << dendl; - ret = cls; - goto send; + if (cls->status == ClassData::CLASS_UNKNOWN || + cls->status == ClassData::CLASS_MISSING) { + char fname[PATH_MAX]; + snprintf(fname, sizeof(fname), "%s/libcls_%s.so", + g_conf.osd_class_dir.c_str(), + cls->name.c_str()); + dout(10) << "_load_class " << cls->name << " from " << fname << dendl; + + cls->handle = dlopen(fname, RTLD_NOW); + if (!cls->handle) { + dout(0) << "_load_class could not open class " << fname + << " (dlopen failed): " << dlerror() << dendl; + cls->status = ClassData::CLASS_MISSING; + return -EIO; } - return cls; - case ClassData::CLASS_REQUESTED: - return NULL; - - case ClassData::CLASS_UNKNOWN: - cls->set_status(ClassData::CLASS_REQUESTED); - break; - - default: - assert(0); + cls_deps_t *(*cls_deps)(); + cls_deps = (cls_deps_t *(*)())dlsym(cls->handle, "class_deps"); + if (cls_deps) { + cls_deps_t *deps = cls_deps(); + while (deps) { + if (!deps->name) + break; + ClassData *cls_dep = _get_class(deps->name); + cls->dependencies.insert(cls_dep); + if (cls_dep->status != ClassData::CLASS_OPEN) + cls->missing_dependencies.insert(cls_dep); + deps++; + } + } } - cls->version = version; -send: - osd->send_class_request(cname.c_str(), version); - return ret; -} - -void ClassHandler::handle_class(MClass *m) -{ - deque::iterator info_iter; - deque::iterator impl_iter; - deque::iterator add_iter; - - for (info_iter = m->info.begin(), add_iter = m->add.begin(), impl_iter = m->impl.begin(); - info_iter != m->info.end(); - ++info_iter, ++add_iter) { - ClassData& data = get_obj(info_iter->name); - if (&data == &null_cls_data) { - dout(1) << "couldn't get class, out of memory? continuing" << dendl; - continue; + // resolve dependencies + set::iterator p = cls->missing_dependencies.begin(); + while (p != cls->missing_dependencies.end()) { + ClassData *dc = *p; + int r = _load_class(dc); + if (r < 0) { + cls->status = ClassData::CLASS_MISSING_DEPS; + return r; } - dout(10) << "handle_class " << info_iter->name << dendl; - data.mutex->Lock(); - if (*add_iter) { - data.set_status(ClassData::CLASS_REQUESTED); - dout(10) << "added class '" << info_iter->name << "'" << dendl; - data.impl = *impl_iter; - ++impl_iter; - int ret = _load_class(data); - if (ret < 0) { - data.set_status(ClassData::CLASS_ERROR); - osd->got_class(info_iter->name); - } - } else { - dout(10) << "response of an invalid class '" << info_iter->name << "'" << dendl; - data.set_status(ClassData::CLASS_INVALID); - osd->got_class(info_iter->name); - } - data.mutex->Unlock(); + dout(10) << "_load_class " << cls->name << " satisfied dependency " << dc->name << dendl; + cls->missing_dependencies.erase(p++); } + + // initialize + void (*cls_init)() = (void (*)())dlsym(cls->handle, "__cls_init"); + if (cls_init) { + cls->status = ClassData::CLASS_INITIALIZING; + cls_init(); + } + + dout(10) << "_load_class " << cls->name << " success" << dendl; + cls->status = ClassData::CLASS_OPEN; + return 0; } -void ClassHandler::resend_class_requests() -{ - for (map::iterator p = classes.begin(); p != classes.end(); p++) { - dout(20) << "resending class request "<< p->first.c_str() << " v" << p->second.version << dendl; - osd->send_class_request(p->first.c_str(), p->second.version); - } -} ClassHandler::ClassData *ClassHandler::register_class(const char *cname) { - ClassData& class_data = get_obj(cname); + assert(mutex.is_locked()); - if (&class_data == &null_cls_data) { - dout(0) << "couldn't get class object, out of memory?" << dendl; - return NULL; - } + ClassData *cls = _get_class(cname); + dout(10) << "register_class " << cname << " status " << cls->status << dendl; - dout(0) << "&class_data=" << (void *)&class_data << " status=" << class_data.status << dendl; - - if (class_data.status != ClassData::CLASS_LOADED) { - dout(0) << "class " << cname << " can't be loaded" << dendl; + if (cls->status != ClassData::CLASS_INITIALIZING) { + dout(0) << "class " << cname << " isn't loaded; is the class registering under the wrong name?" << dendl; return NULL; } - - if (class_data.registered) { - dout(0) << "class " << cname << " already registered" << dendl; - } - - class_data.registered = true; - - return &class_data; + return cls; } void ClassHandler::unregister_class(ClassHandler::ClassData *cls) @@ -209,121 +129,17 @@ void ClassHandler::unregister_class(ClassHandler::ClassData *cls) /* FIXME: do we really need this one? */ } - -int ClassHandler::ClassData::load() -{ - int ret; - switch (status) { - case CLASS_INVALID: - ret = -EINVAL; - break; - case CLASS_ERROR: - ret = -EIO; - break; - default: - ret = 0; - } - if (ret) { - /* if we're invalid, we should just notify osd */ - osd->got_class(name); - return ret; - } - - if (!has_missing_deps()) { - set_status(CLASS_LOADED); - dout(0) << "setting class " << name << " status to CLASS_LOADED" << dendl; - init(); - osd->got_class(name); - } - - list::iterator iter; - for (iter = dependents.begin(); iter != dependents.end(); ++iter) { - ClassData *cls = *iter; - cls->satisfy_dependency(this); - } - - return 0; -} - -void ClassHandler::ClassData::init() -{ - void (*cls_init)() = (void (*)())dlsym(handle, "__cls_init"); - - if (cls_init) - cls_init(); -} - -bool ClassHandler::ClassData::_add_dependency(cls_deps_t *dep) -{ - if (!dep->name) - return false; - - ClassData& cls_dep = handler->get_obj(dep->name); - if (&cls_dep == &null_cls_data) { - dout(0) << "couldn't get class dep object, out of memory?" << dendl; - return false; - } - map::iterator iter = missing_dependencies.find(dep->name); - dependencies[dep->name] = &cls_dep; - dout(0) << "adding dependency " << dep->name << dendl; - - if (cls_dep.status != CLASS_LOADED) { - missing_dependencies[dep->name] = &cls_dep; - - if(cls_dep.status == CLASS_UNKNOWN) { - ClassVersion version; - version.set_arch(get_arch()); - handler->get_class(dep->name, version); - } - dout(0) << "adding missing dependency " << dep->name << dendl; - } - cls_dep._add_dependent(*this); - - if ((cls_dep.status == CLASS_INVALID) || - (cls_dep.status == CLASS_ERROR)) { - dout(0) << "ouch! depending on bad class" << dendl; - set_status(cls_dep.status); /* we have an invalid dependency, we're invalid */ - } - - return true; -} - -void ClassHandler::ClassData::satisfy_dependency(ClassData *cls) -{ - Mutex::Locker lock(*mutex); - map::iterator iter = missing_dependencies.find(cls->name); - - if (iter != missing_dependencies.end()) { - dout(0) << "satisfied dependency name=" << name << " dep=" << cls->name << dendl; - missing_dependencies.erase(iter); - if (missing_dependencies.size() == 0) { - dout(0) << "all dependencies are satisfied! initializing, notifying osd" << dendl; - set_status(CLASS_LOADED); - dout(0) << "this=" << (void *)this << " status=" << status << dendl; - init(); - osd->got_class(name); - } - } -} - -void ClassHandler::ClassData::_add_dependent(ClassData& dependent) -{ - Mutex::Locker lock(*mutex); - dout(0) << "class " << name << " has dependet: " << dependent.name << dendl; - dependents.push_back(&dependent); -} - ClassHandler::ClassMethod *ClassHandler::ClassData::register_method(const char *mname, int flags, cls_method_call_t func) { /* no need for locking, called under the class_init mutex */ + dout(10) << "register_method " << name << "." << mname << " flags " << flags << " " << (void*)func << dendl; ClassMethod& method = methods_map[mname]; method.func = func; method.name = mname; method.flags = flags; method.cls = this; - return &method; } @@ -332,67 +148,41 @@ ClassHandler::ClassMethod *ClassHandler::ClassData::register_cxx_method(const ch cls_method_cxx_call_t func) { /* no need for locking, called under the class_init mutex */ + dout(10) << "register_cxx_method " << name << "." << mname << " flags " << flags << " " << (void*)func << dendl; ClassMethod& method = methods_map[mname]; method.cxx_func = func; method.name = mname; method.flags = flags; method.cls = this; - return &method; } -ClassHandler::ClassMethod *ClassHandler::ClassData::get_method(const char *mname) +ClassHandler::ClassMethod *ClassHandler::ClassData::_get_method(const char *mname) { - Mutex::Locker lock(*mutex); map::iterator iter = methods_map.find(mname); - if (iter == methods_map.end()) return NULL; - return &(iter->second); } -void ClassHandler::ClassData::unregister_method(ClassHandler::ClassMethod *method) +int ClassHandler::ClassData::get_method_flags(const char *mname) { - /* no need for locking, called under the class_init mutex */ - map::iterator iter; + Mutex::Locker l(handler->mutex); + ClassMethod *method = _get_method(mname); + if (!method) + return 0; + return method->flags; +} - iter = methods_map.find(method->name); +void ClassHandler::ClassData::unregister_method(ClassHandler::ClassMethod *method) +{ + /* no need for locking, called under the class_init mutex */ + map::iterator iter = methods_map.find(method->name); if (iter == methods_map.end()) return; - methods_map.erase(iter); } -void ClassHandler::ClassData::set_status(ClassHandler::ClassData::Status _status) -{ - status = _status; - - utime_t e = g_clock.now(); - switch (status) { - case CLASS_ERROR: - case CLASS_INVALID: - e += g_conf.osd_class_error_timeout; - expires = e; - break; - - case CLASS_LOADED: - e += g_conf.osd_class_timeout; - expires = e; - break; - - default: - break; - } -} - -bool ClassHandler::ClassData::cache_timed_out() -{ - utime_t now = g_clock.now(); - dout(0) << "expires " << expires << " now " << now << dendl; - return expires != utime_t() && now > expires; -} - void ClassHandler::ClassMethod::unregister() { cls->unregister_method(this); @@ -418,15 +208,3 @@ int ClassHandler::ClassMethod::exec(cls_method_context_t ctx, bufferlist& indata return ret; } -int ClassHandler::get_method_flags(const string& cname, const string& mname) -{ - ClassData& cls = get_obj(cname); - if (&cls == &null_cls_data) - return 0; - - ClassMethod *method = cls.get_method(mname.c_str()); - if (!method) - return 0; - - return method->flags; -} diff --git a/src/common/ClassHandler.h b/src/common/ClassHandler.h index 91d18a8c2d602..bb4277ac77071 100644 --- a/src/common/ClassHandler.h +++ b/src/common/ClassHandler.h @@ -11,15 +11,8 @@ #include "common/ClassVersion.h" -class OSD; -class MClass; - - - class ClassHandler { - OSD *osd; - public: class ClassData; @@ -37,68 +30,55 @@ public: }; struct ClassData { - Mutex *mutex; - bool _add_dependency(cls_deps_t *dep); - void _add_dependent(ClassData& dependent); - - void satisfy_dependency(ClassData *cls); - enum Status { CLASS_UNKNOWN, - CLASS_INVALID, - CLASS_ERROR, - CLASS_LOADED, - CLASS_REQUESTED, + CLASS_MISSING, // missing + CLASS_MISSING_DEPS, // missing dependencies + CLASS_INITIALIZING, // calling init() right now + CLASS_OPEN, // initialized, usable } status; - ClassVersion version; - utime_t expires; - ClassImpl impl; + string name; - OSD *osd; ClassHandler *handler; void *handle; - bool registered; + map methods_map; - map dependencies; /* our dependencies */ - map missing_dependencies; /* only missing dependencies */ - list dependents; /* classes that depend on us */ + set dependencies; /* our dependencies */ + set missing_dependencies; /* only missing dependencies */ - bool has_missing_deps() { return (missing_dependencies.size() > 0); } + ClassMethod *_get_method(const char *mname); - ClassData() : mutex(NULL), status(CLASS_UNKNOWN), version(), handle(NULL), registered(false) {} - ~ClassData() { if (mutex) delete mutex; } + ClassData() : status(CLASS_UNKNOWN), + handle(NULL) {} + ~ClassData() { } ClassMethod *register_method(const char *mname, int flags, cls_method_call_t func); ClassMethod *register_cxx_method(const char *mname, int flags, cls_method_cxx_call_t func); - ClassMethod *get_method(const char *mname); void unregister_method(ClassMethod *method); - int load(); - void init(); - - void set_status(Status _status); - bool cache_timed_out(); + ClassMethod *get_method(const char *mname) { + Mutex::Locker l(handler->mutex); + return _get_method(mname); + } + int get_method_flags(const char *mname); }; + +private: Mutex mutex; map classes; - ClassData& get_obj(const string& cname); - - int load_class(const string& cname); - int _load_class(ClassData &data); - - ClassHandler(OSD *_osd) : osd(_osd), mutex("ClassHandler") {} - - ClassData *get_class(const string& cname, ClassVersion& version); - void resend_class_requests(); - - void handle_class(MClass *m); + ClassData *_get_class(const string& cname); + int _load_class(ClassData *cls); +public: + ClassHandler() : mutex("ClassHandler") {} + + int open_class(const string& cname, ClassData **pcls); + ClassData *register_class(const char *cname); void unregister_class(ClassData *cls); - - int get_method_flags(const string& cname, const string& mname); + }; diff --git a/src/common/config.cc b/src/common/config.cc index b8823e6a9b7ea..e3b80c7f84bf0 100644 --- a/src/common/config.cc +++ b/src/common/config.cc @@ -368,7 +368,7 @@ struct config_option config_optionsp[] = { OPTION(osd_auto_weight, OPT_BOOL, false), OPTION(osd_class_error_timeout, OPT_DOUBLE, 60.0), // seconds OPTION(osd_class_timeout, OPT_DOUBLE, 60*60.0), // seconds - OPTION(osd_class_tmp, OPT_STR, "/var/lib/ceph/tmp"), + OPTION(osd_class_dir, OPT_STR, "/usr/lib/rados-classes"), OPTION(osd_check_for_log_corruption, OPT_BOOL, false), OPTION(osd_use_stale_snap, OPT_BOOL, false), OPTION(osd_max_notify_timeout, OPT_U32, 30), // max notify timeout in seconds diff --git a/src/common/config.h b/src/common/config.h index e07766909a9b2..7dcd6407e275c 100644 --- a/src/common/config.h +++ b/src/common/config.h @@ -416,7 +416,7 @@ public: double osd_class_error_timeout; double osd_class_timeout; - std::string osd_class_tmp; + std::string osd_class_dir; int osd_max_scrubs; float osd_scrub_load_threshold; diff --git a/src/objclass/class_api.cc b/src/objclass/class_api.cc index 44299b8a5eb2a..757943160b92c 100644 --- a/src/objclass/class_api.cc +++ b/src/objclass/class_api.cc @@ -2,21 +2,20 @@ #include "common/config.h" #include "objclass/objclass.h" -#include "osd/OSD.h" #include "osd/ReplicatedPG.h" #include "common/ClassHandler.h" -static OSD *osd; +static ClassHandler *ch; -void cls_initialize(OSD *_osd) +void cls_initialize(ClassHandler *h) { - osd = _osd; + ch = h; } void cls_finalize() { - osd = NULL; + ch = NULL; } @@ -32,21 +31,15 @@ void cls_free(void *p) int cls_register(const char *name, cls_handle_t *handle) { - ClassHandler::ClassData *cd; - - cd = osd->class_handler->register_class(name); - - *handle = (cls_handle_t)cd; - - return (cd != NULL); + ClassHandler::ClassData *cls = ch->register_class(name); + *handle = (cls_handle_t)cls; + return (cls != NULL); } int cls_unregister(cls_handle_t handle) { - ClassHandler::ClassData *cd; - cd = (ClassHandler::ClassData *)handle; - - osd->class_handler->unregister_class(cd); + ClassHandler::ClassData *cls = (ClassHandler::ClassData *)handle; + ch->unregister_class(cls); return 1; } @@ -54,11 +47,8 @@ int cls_register_method(cls_handle_t hclass, const char *method, int flags, cls_method_call_t class_call, cls_method_handle_t *handle) { - ClassHandler::ClassData *cd; - cls_method_handle_t hmethod; - - cd = (ClassHandler::ClassData *)hclass; - hmethod = (cls_method_handle_t)cd->register_method(method, flags, class_call); + ClassHandler::ClassData *cls = (ClassHandler::ClassData *)hclass; + cls_method_handle_t hmethod =(cls_method_handle_t)cls->register_method(method, flags, class_call); if (handle) *handle = hmethod; return (hmethod != NULL); @@ -68,11 +58,8 @@ int cls_register_cxx_method(cls_handle_t hclass, const char *method, int flags, cls_method_cxx_call_t class_call, cls_method_handle_t *handle) { - ClassHandler::ClassData *cd; - cls_method_handle_t hmethod; - - cd = (ClassHandler::ClassData *)hclass; - hmethod = (cls_method_handle_t)cd->register_cxx_method(method, flags, class_call); + ClassHandler::ClassData *cls = (ClassHandler::ClassData *)hclass; + cls_method_handle_t hmethod = (cls_method_handle_t)cls->register_cxx_method(method, flags, class_call); if (handle) *handle = hmethod; return (hmethod != NULL); @@ -82,7 +69,6 @@ int cls_unregister_method(cls_method_handle_t handle) { ClassHandler::ClassMethod *method = (ClassHandler::ClassMethod *)handle; method->unregister(); - return 1; } diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index a39779f585579..05d28dc7c456d 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -454,7 +454,6 @@ OSD::OSD(int id, Messenger *internal_messenger, Messenger *external_messenger, osdmap(NULL), map_lock("OSD::map_lock"), map_cache_lock("OSD::map_cache_lock"), map_cache_keep_from(0), - class_lock("OSD::class_lock"), up_thru_wanted(0), up_thru_pending(0), pg_stat_queue_lock("OSD::pg_stat_queue_lock"), osd_stat_updated(false), @@ -514,7 +513,7 @@ void handle_signal(int signal) } } -void cls_initialize(OSD *_osd); +void cls_initialize(ClassHandler *ch); int OSD::pre_init() @@ -568,10 +567,10 @@ int OSD::init() return -1; } - class_handler = new ClassHandler(this); + class_handler = new ClassHandler(); if (!class_handler) return -ENOMEM; - cls_initialize(this); + cls_initialize(class_handler); // load up "current" osdmap assert_warn(!osdmap); @@ -1833,7 +1832,6 @@ void OSD::ms_handle_connect(Connection *con) send_pg_temp(); send_failures(); send_pg_stats(g_clock.now()); - class_handler->resend_class_requests(); } } @@ -2553,10 +2551,6 @@ void OSD::_dispatch(Message *m) handle_rep_scrub((MOSDRepScrub*)m); break; - case MSG_CLASS: - handle_class((MClass*)m); - break; - // -- need OSDMap -- default: @@ -5110,7 +5104,11 @@ void OSD::handle_op(MOSDOp *op) utime_t now = g_clock.now(); - init_op_flags(op); + int r = init_op_flags(op); + if (r) { + reply_op_error(op, r); + return; + } // calc actual pgid pg_t pgid = op->get_pg(); @@ -5471,78 +5469,7 @@ void OSD::wait_for_no_ops() // -------------------------------- -int OSD::get_class(const string& cname, ClassVersion& version, pg_t pgid, Message *m, - ClassHandler::ClassData **pcls) -{ - Mutex::Locker l(class_lock); - dout(10) << "get_class '" << cname << "' by " << m << dendl; - - ClassHandler::ClassData *cls = class_handler->get_class(cname, version); - if (cls) { - switch (cls->status) { - case ClassHandler::ClassData::CLASS_LOADED: - *pcls = cls; - return 0; - case ClassHandler::ClassData::CLASS_INVALID: - dout(1) << "class " << cname << " not supported" << dendl; - return -EOPNOTSUPP; - case ClassHandler::ClassData::CLASS_ERROR: - dout(0) << "error loading class!" << dendl; - return -EIO; - default: - assert(0); - } - } - - dout(10) << "get_class '" << cname << "' by " << m << " waiting for missing class" << dendl; - waiting_for_missing_class[cname].push_back(m); - return -EAGAIN; -} - -void OSD::got_class(const string& cname) -{ - // no lock.. this is an upcall from handle_class - dout(10) << "got_class '" << cname << "'" << dendl; - - if (waiting_for_missing_class.count(cname)) { - take_waiters(waiting_for_missing_class[cname]); - waiting_for_missing_class.erase(cname); - } -} - -void OSD::handle_class(MClass *m) -{ - if (!require_mon_peer(m)) - return; - - Mutex::Locker l(class_lock); - dout(10) << "handle_class action=" << m->action << dendl; - - switch (m->action) { - case CLASS_RESPONSE: - class_handler->handle_class(m); - break; - - default: - assert(0); - } - m->put(); -} - -void OSD::send_class_request(const char *cname, ClassVersion& version) -{ - dout(10) << "send_class_request class=" << cname << " version=" << version << dendl; - MClass *m = new MClass(monc->get_fsid(), 0); - ClassInfo info; - info.name = cname; - info.version = version; - m->info.push_back(info); - m->action = CLASS_GET; - monc->send_mon_message(m); -} - - -void OSD::init_op_flags(MOSDOp *op) +int OSD::init_op_flags(MOSDOp *op) { vector::iterator iter; @@ -5568,7 +5495,12 @@ void OSD::init_op_flags(MOSDOp *op) string cname, mname; bp.copy(iter->op.cls.class_len, cname); bp.copy(iter->op.cls.method_len, mname); - int flags = class_handler->get_method_flags(cname, mname); + + ClassHandler::ClassData *cls; + int r = class_handler->open_class(cname, &cls); + if (r) + return r; + int flags = cls->get_method_flags(mname.c_str()); is_read = flags & CLS_METHOD_RD; is_write = flags & CLS_METHOD_WR; is_public = flags & CLS_METHOD_PUBLIC; @@ -5590,4 +5522,6 @@ void OSD::init_op_flags(MOSDOp *op) break; } } + + return 0; } diff --git a/src/osd/OSD.h b/src/osd/OSD.h index e5ed939264e27..e3d7803948df1 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -491,17 +491,6 @@ protected: Watch *watch; /* notify-watch handler */ -protected: - // -- classes -- - Mutex class_lock; - map > waiting_for_missing_class; - - int get_class(const string& cname, ClassVersion& version, pg_t pgid, Message *m, ClassHandler::ClassData **cls); - void handle_class(MClass *m); -public: - void got_class(const string& cname); - void send_class_request(const char *n, ClassVersion& version); - protected: // -- placement groups -- map pool_map; @@ -1102,7 +1091,7 @@ public: void force_remount(); - void init_op_flags(MOSDOp *op); + int init_op_flags(MOSDOp *op); void put_object_context(void *_obc, pg_t pgid); diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc index 596ecb047a730..8446a67f96608 100644 --- a/src/osd/ReplicatedPG.cc +++ b/src/osd/ReplicatedPG.cc @@ -15,7 +15,6 @@ #include "OSD.h" #include "PGLS.h" -#include "common/arch.h" #include "common/errno.h" #include "common/ProfLogger.h" @@ -979,16 +978,10 @@ int ReplicatedPG::prepare_call(MOSDOp *osd_op, ceph_osd_op& op, bufferlist::iterator& bp, ClassHandler::ClassMethod **pmethod) { - int result; - ClassHandler::ClassData *cls; - ClassVersion version; - version.set_arch(get_arch()); - result = osd->get_class(cname, version, info.pgid, osd_op, &cls); - if (result) { - dout(10) << "call class " << cname << " does not exist" << dendl; - return result; - } + int result = osd->class_handler->open_class(cname, &cls); + assert(result == 0); + bufferlist outdata; ClassHandler::ClassMethod *method = cls->get_method(mname.c_str()); if (!method) { @@ -1022,6 +1015,7 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector& ops, dout(10) << "do_osd_op " << osd_op << dendl; // modify? + int flags; bool is_modify; string cname, mname; bufferlist::iterator bp = osd_op.data.begin(); @@ -1029,7 +1023,14 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector& ops, case CEPH_OSD_OP_CALL: bp.copy(op.cls.class_len, cname); bp.copy(op.cls.method_len, mname); - is_modify = osd->class_handler->get_method_flags(cname, mname) & CLS_METHOD_WR; + { + ClassHandler::ClassData *cls; + int r = osd->class_handler->open_class(cname, &cls); + assert(r == 0); + flags = cls->get_method_flags(mname.c_str()); + } + is_modify = flags & CLS_METHOD_WR; + dout(10) << " class " << cname << "." << mname << " flags " << flags << " is_modify " << is_modify << dendl; break; default: diff --git a/src/vstart.sh b/src/vstart.sh index c357d3735af49..fee4342209f9d 100755 --- a/src/vstart.sh +++ b/src/vstart.sh @@ -286,6 +286,7 @@ $CMDSDEBUG [osd] $DAEMONOPTS osd class tmp = out + osd class dir = .libs osd scrub load threshold = 5.0 $COSDDEBUG [mon] -- 2.39.5