From 8e9dae8195335b8b2679df3949fee91a433d51cc Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Thu, 1 Sep 2022 08:53:50 -0400 Subject: [PATCH] rgwlib: rework linkage Signed-off-by: Matt Benjamin --- src/global/signal_handler.h | 2 +- src/rgw/CMakeLists.txt | 2 +- src/rgw/librgw.cc | 580 +------------------------------- src/rgw/rgw_file.cc | 88 +++-- src/rgw/rgw_file.h | 4 +- src/rgw/rgw_lib.cc | 606 ++++++++++++++++++++++++++++++++++ src/rgw/rgw_lib.h | 6 +- src/rgw/rgw_main.h | 121 ++++--- src/rgw/rgw_signal.h | 18 +- src/test/librgw_file_nfsns.cc | 6 +- 10 files changed, 736 insertions(+), 697 deletions(-) create mode 100644 src/rgw/rgw_lib.cc diff --git a/src/global/signal_handler.h b/src/global/signal_handler.h index c7cb84a1066..bfdf0464413 100644 --- a/src/global/signal_handler.h +++ b/src/global/signal_handler.h @@ -22,7 +22,7 @@ typedef void (*signal_handler_t)(int); namespace ceph { - struct BackTrace; +struct BackTrace; } #if defined(HAVE_SIGDESCR_NP) diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index 26f8d37d3bf..f4460802827 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -276,7 +276,6 @@ if(WITH_RADOSGW_MOTR) endif() set(rgw_a_srcs - librgw.cc rgw_appmain.cc rgw_asio_client.cc rgw_asio_frontend.cc @@ -286,6 +285,7 @@ set(rgw_a_srcs rgw_frontend.cc rgw_http_client_curl.cc rgw_kmip_client_impl.cc + rgw_lib.cc rgw_loadgen.cc rgw_loadgen_process.cc rgw_log.cc diff --git a/src/rgw/librgw.cc b/src/rgw/librgw.cc index fbfda8f8a2f..38e43a59da1 100644 --- a/src/rgw/librgw.cc +++ b/src/rgw/librgw.cc @@ -4,7 +4,7 @@ /* * Ceph - scalable distributed file system * - * Copyright (C) 2011 New Dream Network + * Copyright (C) 2022 New Dream Network * * This is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -18,29 +18,13 @@ #include #include "include/rados/librgw.h" -#include "rgw_acl.h" #include "include/str_list.h" -#include "global/signal_handler.h" -#include "common/Timer.h" -#include "common/TracepointProvider.h" -#include "common/WorkQueue.h" #include "common/ceph_argparse.h" #include "common/ceph_context.h" -#include "common/common_init.h" #include "common/dout.h" -#include "rgw_op.h" -#include "rgw_rest.h" -#include "rgw_log.h" -#include "rgw_frontend.h" -#include "rgw_request.h" -#include "rgw_process.h" -#include "rgw_auth.h" #include "rgw_lib.h" -#include "rgw_lib_frontend.h" -#include "rgw_perf_counters.h" -#include "rgw_main.h" #include #include @@ -49,566 +33,14 @@ #define dout_subsys ceph_subsys_rgw -using namespace std; - namespace rgw { - bool global_stop = false; - - static void handle_sigterm(int signum) - { - dout(20) << __func__ << " SIGUSR1 ignored" << dendl; - } - - static std::mutex librgw_mtx; - - RGWLib rgwlib; - - class C_InitTimeout : public Context { - public: - C_InitTimeout() {} - void finish(int r) override { - derr << "Initialization timeout, failed to initialize" << dendl; - exit(1); - } - }; - - void RGWLibProcess::checkpoint() - { - m_tp.drain(&req_wq); - } - -#define MIN_EXPIRE_S 120 - - void RGWLibProcess::run() - { - /* write completion interval */ - RGWLibFS::write_completion_interval_s = - cct->_conf->rgw_nfs_write_completion_interval_s; - - /* start write timer */ - RGWLibFS::write_timer.resume(); - - /* gc loop */ - while (! shutdown) { - lsubdout(cct, rgw, 5) << "RGWLibProcess GC" << dendl; - - /* dirent invalidate timeout--basically, the upper-bound on - * inconsistency with the S3 namespace */ - auto expire_s = cct->_conf->rgw_nfs_namespace_expire_secs; - - /* delay between gc cycles */ - auto delay_s = std::max(int64_t(1), std::min(int64_t(MIN_EXPIRE_S), expire_s/2)); - - unique_lock uniq(mtx); - restart: - int cur_gen = gen; - for (auto iter = mounted_fs.begin(); iter != mounted_fs.end(); - ++iter) { - RGWLibFS* fs = iter->first->ref(); - uniq.unlock(); - fs->gc(); - const DoutPrefix dp(cct, dout_subsys, "librgw: "); - fs->update_user(&dp); - fs->rele(); - uniq.lock(); - if (cur_gen != gen) - goto restart; /* invalidated */ - } - cv.wait_for(uniq, std::chrono::seconds(delay_s)); - uniq.unlock(); - } - } - - void RGWLibProcess::handle_request(const DoutPrefixProvider *dpp, RGWRequest* r) - { - /* - * invariant: valid requests are derived from RGWLibRequst - */ - RGWLibRequest* req = static_cast(r); - - // XXX move RGWLibIO and timing setup into process_request - -#if 0 /* XXX */ - utime_t tm = ceph_clock_now(); -#endif - - RGWLibIO io_ctx; - - int ret = process_request(req, &io_ctx); - if (ret < 0) { - /* we don't really care about return code */ - dout(20) << "process_request() returned " << ret << dendl; - - } - delete req; - } /* handle_request */ - - int RGWLibProcess::process_request(RGWLibRequest* req) - { - // XXX move RGWLibIO and timing setup into process_request - -#if 0 /* XXX */ - utime_t tm = ceph_clock_now(); -#endif - - RGWLibIO io_ctx; - - int ret = process_request(req, &io_ctx); - if (ret < 0) { - /* we don't really care about return code */ - dout(20) << "process_request() returned " << ret << dendl; - } - return ret; - } /* process_request */ - - static inline void abort_req(req_state *s, RGWOp *op, int err_no) - { - if (!s) - return; - - /* XXX the dump_errno and dump_bucket_from_state behaviors in - * the abort_early (rgw_rest.cc) might be valuable, but aren't - * safe to call presently as they return HTTP data */ - - perfcounter->inc(l_rgw_failed_req); - } /* abort_req */ - - int RGWLibProcess::process_request(RGWLibRequest* req, RGWLibIO* io) - { - int ret = 0; - bool should_log = true; // XXX - - dout(1) << "====== " << __func__ - << " starting new request req=" << hex << req << dec - << " ======" << dendl; - - /* - * invariant: valid requests are derived from RGWOp--well-formed - * requests should have assigned RGWRequest::op in their descendant - * constructor--if not, the compiler can find it, at the cost of - * a runtime check - */ - RGWOp *op = (req->op) ? req->op : dynamic_cast(req); - if (! op) { - ldpp_dout(op, 1) << "failed to derive cognate RGWOp (invalid op?)" << dendl; - return -EINVAL; - } - - io->init(req->cct); - - perfcounter->inc(l_rgw_req); - - RGWEnv& rgw_env = io->get_env(); - - /* XXX - * until major refactoring of req_state and req_info, we need - * to build their RGWEnv boilerplate from the RGWLibRequest, - * pre-staging any strings (HTTP_HOST) that provoke a crash when - * not found - */ - - /* XXX for now, use ""; could be a legit hostname, or, in future, - * perhaps a tenant (Yehuda) */ - rgw_env.set("HTTP_HOST", ""); - - /* XXX and -then- bloat up req_state with string copies from it */ - req_state rstate(req->cct, &rgw_env, req->id); - req_state *s = &rstate; - - // XXX fix this - s->cio = io; - - /* XXX and -then- stash req_state pointers everywhere they are needed */ - ret = req->init(rgw_env, store, io, s); - if (ret < 0) { - ldpp_dout(op, 10) << "failed to initialize request" << dendl; - abort_req(s, op, ret); - goto done; - } - - /* req is-a RGWOp, currently initialized separately */ - ret = req->op_init(); - if (ret < 0) { - dout(10) << "failed to initialize RGWOp" << dendl; - abort_req(s, op, ret); - goto done; - } - - /* now expected by rgw_log_op() */ - rgw_env.set("REQUEST_METHOD", s->info.method); - rgw_env.set("REQUEST_URI", s->info.request_uri); - rgw_env.set("QUERY_STRING", ""); - - try { - /* XXX authorize does less here then in the REST path, e.g., - * the user's info is cached, but still incomplete */ - ldpp_dout(s, 2) << "authorizing" << dendl; - ret = req->authorize(op, null_yield); - if (ret < 0) { - dout(10) << "failed to authorize request" << dendl; - abort_req(s, op, ret); - goto done; - } - - /* FIXME: remove this after switching all handlers to the new - * authentication infrastructure. */ - if (! s->auth.identity) { - s->auth.identity = rgw::auth::transform_old_authinfo(s); - } - - ldpp_dout(s, 2) << "reading op permissions" << dendl; - ret = req->read_permissions(op, null_yield); - if (ret < 0) { - abort_req(s, op, ret); - goto done; - } - - ldpp_dout(s, 2) << "init op" << dendl; - ret = op->init_processing(null_yield); - if (ret < 0) { - abort_req(s, op, ret); - goto done; - } - - ldpp_dout(s, 2) << "verifying op mask" << dendl; - ret = op->verify_op_mask(); - if (ret < 0) { - abort_req(s, op, ret); - goto done; - } - - ldpp_dout(s, 2) << "verifying op permissions" << dendl; - ret = op->verify_permission(null_yield); - if (ret < 0) { - if (s->system_request) { - ldpp_dout(op, 2) << "overriding permissions due to system operation" << dendl; - } else if (s->auth.identity->is_admin_of(s->user->get_id())) { - ldpp_dout(op, 2) << "overriding permissions due to admin operation" << dendl; - } else { - abort_req(s, op, ret); - goto done; - } - } - - ldpp_dout(s, 2) << "verifying op params" << dendl; - ret = op->verify_params(); - if (ret < 0) { - abort_req(s, op, ret); - goto done; - } - - ldpp_dout(s, 2) << "executing" << dendl; - op->pre_exec(); - op->execute(null_yield); - op->complete(); - - } catch (const ceph::crypto::DigestException& e) { - dout(0) << "authentication failed" << e.what() << dendl; - abort_req(s, op, -ERR_INVALID_SECRET_KEY); - } - - done: - try { - io->complete_request(); - } catch (rgw::io::Exception& e) { - dout(0) << "ERROR: io->complete_request() returned " - << e.what() << dendl; - } - if (should_log) { - rgw_log_op(nullptr /* !rest */, s, (op ? op->name() : "unknown"), olog); - } - - int http_ret = s->err.http_ret; - - ldpp_dout(s, 2) << "http status=" << http_ret << dendl; - - ldpp_dout(op, 1) << "====== " << __func__ - << " req done req=" << hex << req << dec << " http_status=" - << http_ret - << " ======" << dendl; - - return (ret < 0 ? ret : s->err.ret); - } /* process_request */ - - int RGWLibProcess::start_request(RGWLibContinuedReq* req) - { - - dout(1) << "====== " << __func__ - << " starting new continued request req=" << hex << req << dec - << " ======" << dendl; - - /* - * invariant: valid requests are derived from RGWOp--well-formed - * requests should have assigned RGWRequest::op in their descendant - * constructor--if not, the compiler can find it, at the cost of - * a runtime check - */ - RGWOp *op = (req->op) ? req->op : dynamic_cast(req); - if (! op) { - ldpp_dout(op, 1) << "failed to derive cognate RGWOp (invalid op?)" << dendl; - return -EINVAL; - } - - req_state* s = req->get_state(); - RGWLibIO& io_ctx = req->get_io(); - RGWEnv& rgw_env = io_ctx.get_env(); - - rgw_env.set("HTTP_HOST", ""); - - int ret = req->init(rgw_env, store, &io_ctx, s); - if (ret < 0) { - ldpp_dout(op, 10) << "failed to initialize request" << dendl; - abort_req(s, op, ret); - goto done; - } - - /* req is-a RGWOp, currently initialized separately */ - ret = req->op_init(); - if (ret < 0) { - dout(10) << "failed to initialize RGWOp" << dendl; - abort_req(s, op, ret); - goto done; - } - - /* XXX authorize does less here then in the REST path, e.g., - * the user's info is cached, but still incomplete */ - ldpp_dout(s, 2) << "authorizing" << dendl; - ret = req->authorize(op, null_yield); - if (ret < 0) { - dout(10) << "failed to authorize request" << dendl; - abort_req(s, op, ret); - goto done; - } - - /* FIXME: remove this after switching all handlers to the new authentication - * infrastructure. */ - if (! s->auth.identity) { - s->auth.identity = rgw::auth::transform_old_authinfo(s); - } - - ldpp_dout(s, 2) << "reading op permissions" << dendl; - ret = req->read_permissions(op, null_yield); - if (ret < 0) { - abort_req(s, op, ret); - goto done; - } - - ldpp_dout(s, 2) << "init op" << dendl; - ret = op->init_processing(null_yield); - if (ret < 0) { - abort_req(s, op, ret); - goto done; - } - - ldpp_dout(s, 2) << "verifying op mask" << dendl; - ret = op->verify_op_mask(); - if (ret < 0) { - abort_req(s, op, ret); - goto done; - } - - ldpp_dout(s, 2) << "verifying op permissions" << dendl; - ret = op->verify_permission(null_yield); - if (ret < 0) { - if (s->system_request) { - ldpp_dout(op, 2) << "overriding permissions due to system operation" << dendl; - } else if (s->auth.identity->is_admin_of(s->user->get_id())) { - ldpp_dout(op, 2) << "overriding permissions due to admin operation" << dendl; - } else { - abort_req(s, op, ret); - goto done; - } - } - - ldpp_dout(s, 2) << "verifying op params" << dendl; - ret = op->verify_params(); - if (ret < 0) { - abort_req(s, op, ret); - goto done; - } - - op->pre_exec(); - req->exec_start(); - - done: - return (ret < 0 ? ret : s->err.ret); - } - - int RGWLibProcess::finish_request(RGWLibContinuedReq* req) - { - RGWOp *op = (req->op) ? req->op : dynamic_cast(req); - if (! op) { - ldpp_dout(op, 1) << "failed to derive cognate RGWOp (invalid op?)" << dendl; - return -EINVAL; - } - - int ret = req->exec_finish(); - int op_ret = op->get_ret(); +bool global_stop = false; - ldpp_dout(op, 1) << "====== " << __func__ - << " finishing continued request req=" << hex << req << dec - << " op status=" << op_ret - << " ======" << dendl; +static std::mutex librgw_mtx; +static RGWLib rgwlib; - perfcounter->inc(l_rgw_req); - - return ret; - } - - int RGWLibFrontend::init() - { - pprocess = new RGWLibProcess(g_ceph_context, &env, - g_conf()->rgw_thread_pool_size, conf); - return 0; - } - - void RGWLib::set_fe(rgw::RGWLibFrontend* fe) - { - this->fe = fe; - } - - int RGWLib::init() - { - vector args; - return init(args); - } - - int RGWLib::init(vector& args) - { - /* alternative default for module */ - map defaults = { - { "debug_rgw", "1/5" }, - { "keyring", "$rgw_data/keyring" }, - { "log_file", "/var/log/radosgw/$cluster-$name.log" }, - { "objecter_inflight_ops", "24576" }, - // require a secure mon connection by default - { "ms_mon_client_mode", "secure" }, - { "auth_client_required", "cephx" }, - }; - - cct = rgw_global_init(&defaults, args, - CEPH_ENTITY_TYPE_CLIENT, - CODE_ENVIRONMENT_DAEMON, - CINIT_FLAG_UNPRIVILEGED_DAEMON_DEFAULTS); - - ceph::mutex mutex = ceph::make_mutex("main"); - SafeTimer init_timer(g_ceph_context, mutex); - init_timer.init(); - mutex.lock(); - init_timer.add_event_after(g_conf()->rgw_init_timeout, new C_InitTimeout); - mutex.unlock(); - - /* stage all front-ends (before common-init-finish) */ - main.init_frontends1(true /* nfs */); - - common_init_finish(g_ceph_context); - - main.init_storage(); - if (! main.get_store()) { - mutex.lock(); - init_timer.cancel_all_events(); - init_timer.shutdown(); - mutex.unlock(); - - derr << "Couldn't init storage provider (RADOS)" << dendl; - return -EIO; - } - - main.init_perfcounters(); - main.init_http_clients(); - main.cond_init_apis(); - - mutex.lock(); - init_timer.cancel_all_events(); - init_timer.shutdown(); - mutex.unlock(); - - main.init_ldap(); - main.init_opslog(); - - init_async_signal_handler(); - register_async_signal_handler(SIGUSR1, handle_sigterm); - - main.init_tracepoints(); - main.init_frontends2(this /* rgwlib */); - main.init_notification_endpoints(); - main.init_lua(); - - return 0; - } /* RGWLib::init() */ - - int RGWLib::stop() - { - derr << "shutting down" << dendl; - main.shutdown(); - - return 0; - } /* RGWLib::stop() */ - - int RGWLibIO::set_uid(rgw::sal::Store* store, const rgw_user& uid) - { - const DoutPrefix dp(store->ctx(), dout_subsys, "librgw: "); - std::unique_ptr user = store->get_user(uid); - /* object exists, but policy is broken */ - int ret = user->load_user(&dp, null_yield); - if (ret < 0) { - derr << "ERROR: failed reading user info: uid=" << uid << " ret=" - << ret << dendl; - } - user_info = user->get_info(); - return ret; - } - - int RGWLibRequest::read_permissions(RGWOp* op, optional_yield y) { - /* bucket and object ops */ - int ret = - rgw_build_bucket_policies(op, rgwlib.get_store(), get_state(), y); - if (ret < 0) { - ldpp_dout(op, 10) << "read_permissions (bucket policy) on " - << get_state()->bucket << ":" - << get_state()->object - << " only_bucket=" << only_bucket() - << " ret=" << ret << dendl; - if (ret == -ENODATA) - ret = -EACCES; - } else if (! only_bucket()) { - /* object ops */ - ret = rgw_build_object_policies(op, rgwlib.get_store(), get_state(), - op->prefetch_data(), y); - if (ret < 0) { - ldpp_dout(op, 10) << "read_permissions (object policy) on" - << get_state()->bucket << ":" - << get_state()->object - << " ret=" << ret << dendl; - if (ret == -ENODATA) - ret = -EACCES; - } - } - return ret; - } /* RGWLibRequest::read_permissions */ - - int RGWHandler_Lib::authorize(const DoutPrefixProvider *dpp, optional_yield y) - { - /* TODO: handle - * 1. subusers - * 2. anonymous access - * 3. system access - * 4. ? - * - * Much or all of this depends on handling the cached authorization - * correctly (e.g., dealing with keystone) at mount time. - */ - s->perm_mask = RGW_PERM_FULL_CONTROL; - - // populate the owner info - s->owner.set_id(s->user->get_id()); - s->owner.set_name(s->user->get_display_name()); - - return 0; - } /* RGWHandler_Lib::authorize */ - -} /* namespace rgw */ +} // namespace rgw extern "C" { @@ -618,6 +50,8 @@ int librgw_create(librgw_t* rgw, int argc, char **argv) int rc = -EINVAL; + g_rgwlib = &rgwlib; + if (! g_ceph_context) { std::lock_guard lg(librgw_mtx); if (! g_ceph_context) { diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc index 500776bff1e..953cf98a199 100644 --- a/src/rgw/rgw_file.cc +++ b/src/rgw/rgw_file.cc @@ -39,8 +39,6 @@ using namespace rgw; namespace rgw { - extern RGWLib rgwlib; - const string RGWFileHandle::root_name = "/"; std::atomic RGWLibFS::fs_inst_counter; @@ -105,7 +103,7 @@ namespace rgw { std::string bucket_name{path}; RGWStatBucketRequest req(cct, user->clone(), bucket_name, bs); - int rc = rgwlib.get_fe()->execute_req(&req); + int rc = g_rgwlib->get_fe()->execute_req(&req); if ((rc == 0) && (req.get_ret() == 0) && (req.matched())) { @@ -201,7 +199,7 @@ namespace rgw { RGWStatObjRequest req(cct, user->clone(), parent->bucket_name(), obj_path, RGWStatObjRequest::FLAG_NONE); - int rc = rgwlib.get_fe()->execute_req(&req); + int rc = g_rgwlib->get_fe()->execute_req(&req); if ((rc == 0) && (req.get_ret() == 0)) { fhr = lookup_fh(parent, path, RGWFileHandle::FLAG_CREATE); @@ -238,7 +236,7 @@ namespace rgw { RGWStatObjRequest req(cct, user->clone(), parent->bucket_name(), obj_path, RGWStatObjRequest::FLAG_NONE); - int rc = rgwlib.get_fe()->execute_req(&req); + int rc = g_rgwlib->get_fe()->execute_req(&req); if ((rc == 0) && (req.get_ret() == 0)) { fhr = lookup_fh(parent, path, RGWFileHandle::FLAG_DIRECTORY); @@ -269,7 +267,7 @@ namespace rgw { std::string object_name{path}; RGWStatLeafRequest req(cct, user->clone(), parent, object_name); - int rc = rgwlib.get_fe()->execute_req(&req); + int rc = g_rgwlib->get_fe()->execute_req(&req); if ((rc == 0) && (req.get_ret() == 0)) { if (req.matched) { @@ -322,7 +320,7 @@ namespace rgw { RGWReadRequest req(get_context(), user->clone(), rgw_fh, offset, length, buffer); - int rc = rgwlib.get_fe()->execute_req(&req); + int rc = g_rgwlib->get_fe()->execute_req(&req); if ((rc == 0) && ((rc = req.get_ret()) == 0)) { lock_guard guard(rgw_fh->mtx); @@ -344,7 +342,7 @@ namespace rgw { RGWReadRequest req(get_context(), user->clone(), rgw_fh, offset, length, buffer); - int rc = rgwlib.get_fe()->execute_req(&req); + int rc = g_rgwlib->get_fe()->execute_req(&req); if ((rc == 0) && ((rc = req.get_ret()) == 0)) { lock_guard(rgw_fh->mtx); @@ -400,14 +398,14 @@ namespace rgw { /* delete object w/key "/" (uxattrs), if any */ string oname{"/"}; RGWDeleteObjRequest req(cct, user->clone(), bkt_fh->bucket_name(), oname); - rc = rgwlib.get_fe()->execute_req(&req); + rc = g_rgwlib->get_fe()->execute_req(&req); /* don't care if ENOENT */ unref(bkt_fh); } string bname{name}; RGWDeleteBucketRequest req(cct, user->clone(), bname); - rc = rgwlib.get_fe()->execute_req(&req); + rc = g_rgwlib->get_fe()->execute_req(&req); if (! rc) { rc = req.get_ret(); } @@ -443,7 +441,7 @@ namespace rgw { oname += "/"; } RGWDeleteObjRequest req(cct, user->clone(), parent->bucket_name(), oname); - rc = rgwlib.get_fe()->execute_req(&req); + rc = g_rgwlib->get_fe()->execute_req(&req); if (! rc) { rc = req.get_ret(); } @@ -520,7 +518,7 @@ namespace rgw { case 0: { RGWCopyObjRequest req(cct, user->clone(), src_fh, dst_fh, src_name, dst_name); - int rc = rgwlib.get_fe()->execute_req(&req); + int rc = g_rgwlib->get_fe()->execute_req(&req); if ((rc != 0) || ((rc = req.get_ret()) != 0)) { ldout(get_context(), 1) @@ -646,7 +644,7 @@ namespace rgw { req.emplace_attr(RGW_ATTR_UNIX_KEY1, std::move(ux_key)); req.emplace_attr(RGW_ATTR_UNIX1, std::move(ux_attrs)); - rc = rgwlib.get_fe()->execute_req(&req); + rc = g_rgwlib->get_fe()->execute_req(&req); rc2 = req.get_ret(); } else { /* create an object representing the directory */ @@ -672,7 +670,7 @@ namespace rgw { req.emplace_attr(RGW_ATTR_UNIX_KEY1, std::move(ux_key)); req.emplace_attr(RGW_ATTR_UNIX1, std::move(ux_attrs)); - rc = rgwlib.get_fe()->execute_req(&req); + rc = g_rgwlib->get_fe()->execute_req(&req); rc2 = req.get_ret(); } @@ -728,7 +726,7 @@ namespace rgw { RGWPutObjRequest req(cct, user->clone(), parent->bucket_name(), obj_name, bl); MkObjResult mkr{nullptr, -EINVAL}; - rc = rgwlib.get_fe()->execute_req(&req); + rc = g_rgwlib->get_fe()->execute_req(&req); rc2 = req.get_ret(); if ((rc == 0) && @@ -843,7 +841,7 @@ namespace rgw { req.emplace_attr(RGW_ATTR_UNIX_KEY1, std::move(ux_key)); req.emplace_attr(RGW_ATTR_UNIX1, std::move(ux_attrs)); - rc = rgwlib.get_fe()->execute_req(&req); + rc = g_rgwlib->get_fe()->execute_req(&req); rc2 = req.get_ret(); if (! ((rc == 0) && (rc2 == 0))) { @@ -922,7 +920,7 @@ namespace rgw { req.emplace_attr(RGW_ATTR_ETAG, std::move(etag)); req.emplace_attr(RGW_ATTR_ACL, std::move(acls)); - rc = rgwlib.get_fe()->execute_req(&req); + rc = g_rgwlib->get_fe()->execute_req(&req); rc2 = req.get_ret(); if (rc == -ENOENT) { @@ -936,7 +934,7 @@ namespace rgw { req.emplace_attr(RGW_ATTR_UNIX_KEY1, std::move(ux_key)); req.emplace_attr(RGW_ATTR_UNIX1, std::move(ux_attrs)); - rc = rgwlib.get_fe()->execute_req(&req); + rc = g_rgwlib->get_fe()->execute_req(&req); rc2 = req.get_ret(); } @@ -1008,7 +1006,7 @@ namespace rgw { } } - rc = rgwlib.get_fe()->execute_req(&req); + rc = g_rgwlib->get_fe()->execute_req(&req); rc2 = req.get_ret(); rc3 = ((rc == 0) && (rc2 == 0)) ? 0 : -EIO; @@ -1067,7 +1065,7 @@ namespace rgw { RGWGetAttrsRequest req(cct, user->clone(), rgw_fh->bucket_name(), obj_name); - rc = rgwlib.get_fe()->execute_req(&req); + rc = g_rgwlib->get_fe()->execute_req(&req); rc2 = req.get_ret(); rc3 = ((rc == 0) && (rc2 == 0)) ? 0 : -EIO; @@ -1137,7 +1135,7 @@ namespace rgw { return -EINVAL; } - rc = rgwlib.get_fe()->execute_req(&req); + rc = g_rgwlib->get_fe()->execute_req(&req); rc2 = req.get_ret(); return (((rc == 0) && (rc2 == 0)) ? 0 : -EIO); @@ -1173,7 +1171,7 @@ namespace rgw { return -EINVAL; } - rc = rgwlib.get_fe()->execute_req(&req); + rc = g_rgwlib->get_fe()->execute_req(&req); rc2 = req.get_ret(); return (((rc == 0) && (rc2 == 0)) ? 0 : -EIO); @@ -1204,7 +1202,7 @@ namespace rgw { req.emplace_attr(RGW_ATTR_UNIX_KEY1, std::move(ux_key)); req.emplace_attr(RGW_ATTR_UNIX1, std::move(ux_attrs)); - rc = rgwlib.get_fe()->execute_req(&req); + rc = g_rgwlib->get_fe()->execute_req(&req); rc2 = req.get_ret(); if ((rc != 0) || (rc2 != 0)) { @@ -1237,7 +1235,7 @@ namespace rgw { /* force cache drain, forces objects to evict */ fh_cache.drain(ObjUnref(this), RGWFileHandle::FHCache::FLAG_LOCK); - rgwlib.get_fe()->get_process()->unregister_fs(this); + g_rgwlib->get_fe()->get_process()->unregister_fs(this); rele(); } /* RGWLibFS::close */ @@ -1526,9 +1524,9 @@ namespace rgw { return false; RGWRMdirCheck req(fs->get_context(), - rgwlib.get_store()->get_user(fs->get_user()->user_id), + g_rgwlib->get_store()->get_user(fs->get_user()->user_id), this); - int rc = rgwlib.get_fe()->execute_req(&req); + int rc = g_rgwlib->get_fe()->execute_req(&req); if (! rc) { return req.valid && req.has_children; } @@ -1582,9 +1580,9 @@ namespace rgw { } if (is_root()) { - RGWListBucketsRequest req(cct, rgwlib.get_store()->get_user(fs->get_user()->user_id), + RGWListBucketsRequest req(cct, g_rgwlib->get_store()->get_user(fs->get_user()->user_id), this, rcb, cb_arg, offset); - rc = rgwlib.get_fe()->execute_req(&req); + rc = g_rgwlib->get_fe()->execute_req(&req); if (! rc) { (void) clock_gettime(CLOCK_MONOTONIC_COARSE, &now); /* !LOCKED */ lock_guard guard(mtx); @@ -1595,9 +1593,9 @@ namespace rgw { *eof = req.eof(); } } else { - RGWReaddirRequest req(cct, rgwlib.get_store()->get_user(fs->get_user()->user_id), + RGWReaddirRequest req(cct, g_rgwlib->get_store()->get_user(fs->get_user()->user_id), this, rcb, cb_arg, offset); - rc = rgwlib.get_fe()->execute_req(&req); + rc = g_rgwlib->get_fe()->execute_req(&req); if (! rc) { (void) clock_gettime(CLOCK_MONOTONIC_COARSE, &now); /* !LOCKED */ lock_guard guard(mtx); @@ -1664,10 +1662,10 @@ namespace rgw { /* start */ std::string object_name = relative_object_name(); f->write_req = - new RGWWriteRequest(rgwlib.get_store(), - rgwlib.get_store()->get_user(fs->get_user()->user_id), + new RGWWriteRequest(g_rgwlib->get_store(), + g_rgwlib->get_store()->get_user(fs->get_user()->user_id), this, bucket_name(), object_name); - rc = rgwlib.get_fe()->start_req(f->write_req); + rc = g_rgwlib->get_fe()->start_req(f->write_req); if (rc < 0) { lsubdout(fs->get_context(), rgw, 5) << __func__ @@ -1754,7 +1752,7 @@ namespace rgw { << __func__ << " finishing write trans on " << object_name() << dendl; - rc = rgwlib.get_fe()->finish_req(f->write_req); + rc = g_rgwlib->get_fe()->finish_req(f->write_req); if (! rc) { rc = f->write_req->get_ret(); } @@ -2054,15 +2052,15 @@ void rgwfile_version(int *major, int *minor, int *extra) sec_key, "/"); ceph_assert(new_fs); - const DoutPrefix dp(rgwlib.get_store()->ctx(), dout_subsys, "rgw mount: "); - rc = new_fs->authorize(&dp, rgwlib.get_store()); + const DoutPrefix dp(g_rgwlib->get_store()->ctx(), dout_subsys, "rgw mount: "); + rc = new_fs->authorize(&dp, g_rgwlib->get_store()); if (rc != 0) { delete new_fs; return -EINVAL; } /* register fs for shared gc */ - rgwlib.get_fe()->get_process()->register_fs(new_fs); + g_rgwlib->get_fe()->get_process()->register_fs(new_fs); struct rgw_fs *fs = new_fs->get_fs(); fs->rgw = rgw; @@ -2097,15 +2095,15 @@ int rgw_mount2(librgw_t rgw, const char *uid, const char *acc_key, ceph_assert(new_fs); /* should we be using ceph_assert? */ - const DoutPrefix dp(rgwlib.get_store()->ctx(), dout_subsys, "rgw mount2: "); - rc = new_fs->authorize(&dp, rgwlib.get_store()); + const DoutPrefix dp(g_rgwlib->get_store()->ctx(), dout_subsys, "rgw mount2: "); + rc = new_fs->authorize(&dp, g_rgwlib->get_store()); if (rc != 0) { delete new_fs; return -EINVAL; } /* register fs for shared gc */ - rgwlib.get_fe()->get_process()->register_fs(new_fs); + g_rgwlib->get_fe()->get_process()->register_fs(new_fs); struct rgw_fs *fs = new_fs->get_fs(); fs->rgw = rgw; @@ -2150,9 +2148,9 @@ int rgw_statfs(struct rgw_fs *rgw_fs, struct rados_cluster_stat_t stats; RGWGetClusterStatReq req(fs->get_context(), - rgwlib.get_store()->get_user(fs->get_user()->user_id), + g_rgwlib->get_store()->get_user(fs->get_user()->user_id), stats); - int rc = rgwlib.get_fe()->execute_req(&req); + int rc = g_rgwlib->get_fe()->execute_req(&req); if (rc < 0) { lderr(fs->get_context()) << "ERROR: getting total cluster usage" << cpp_strerror(-rc) << dendl; @@ -2655,7 +2653,7 @@ int rgw_readv(struct rgw_fs *rgw_fs, bl); req.do_hexdump = false; - rc = rgwlib.get_fe()->execute_req(&req); + rc = g_rgwlib->get_fe()->execute_req(&req); if (! rc) { RGWReadV* rdv = static_cast( @@ -2715,10 +2713,10 @@ int rgw_writev(struct rgw_fs *rgw_fs, struct rgw_file_handle *fh, } std::string oname = rgw_fh->relative_object_name(); - RGWPutObjRequest req(cct, rgwlib.get_store()->get_user(fs->get_user()->user_id), + RGWPutObjRequest req(cct, g_rgwlib->get_store()->get_user(fs->get_user()->user_id), rgw_fh->bucket_name(), oname, bl); - int rc = rgwlib.get_fe()->execute_req(&req); + int rc = g_rgwlib->get_fe()->execute_req(&req); /* XXX update size (in request) */ diff --git a/src/rgw/rgw_file.h b/src/rgw/rgw_file.h index 81e7ac0a346..f217b415cd5 100644 --- a/src/rgw/rgw_file.h +++ b/src/rgw/rgw_file.h @@ -1010,7 +1010,7 @@ namespace rgw { return -ERR_USER_SUSPENDED; } else { /* try external authenticators (ldap for now) */ - rgw::LDAPHelper* ldh = rgwlib.get_ldh(); /* !nullptr */ + rgw::LDAPHelper* ldh = g_rgwlib->get_ldh(); /* !nullptr */ RGWToken token; /* boost filters and/or string_ref may throw on invalid input */ try { @@ -1314,7 +1314,7 @@ namespace rgw { RGWUserInfo* get_user() { return &user->get_info(); } void update_user(const DoutPrefixProvider *dpp) { - (void) rgwlib.get_store()->get_user_by_access_key(dpp, key.id, null_yield, &user); + (void) g_rgwlib->get_store()->get_user_by_access_key(dpp, key.id, null_yield, &user); } void close(); diff --git a/src/rgw/rgw_lib.cc b/src/rgw/rgw_lib.cc new file mode 100644 index 00000000000..4bc63e869e1 --- /dev/null +++ b/src/rgw/rgw_lib.cc @@ -0,0 +1,606 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2011 New Dream Network + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include +#include +#include + +#include "include/rados/librgw.h" +#include "rgw_acl.h" + +#include "include/str_list.h" +#include "global/signal_handler.h" +#include "common/Timer.h" +#include "common/WorkQueue.h" +#include "common/ceph_argparse.h" +#include "common/ceph_context.h" +#include "common/common_init.h" +#include "common/dout.h" + +#include "rgw_op.h" +#include "rgw_rest.h" +#include "rgw_log.h" +#include "rgw_frontend.h" +#include "rgw_request.h" +#include "rgw_process.h" +#include "rgw_auth.h" +#include "rgw_lib.h" +#include "rgw_lib_frontend.h" +#include "rgw_perf_counters.h" +#include "rgw_main.h" + +#include +#include +#include +#include + +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +namespace rgw { + + RGWLib* g_rgwlib = nullptr; + + static void handle_sigterm(int signum) + { + dout(20) << __func__ << " SIGUSR1 ignored" << dendl; + } + + class C_InitTimeout : public Context { + public: + C_InitTimeout() {} + void finish(int r) override { + derr << "Initialization timeout, failed to initialize" << dendl; + exit(1); + } + }; + + void RGWLibProcess::checkpoint() + { + m_tp.drain(&req_wq); + } + +#define MIN_EXPIRE_S 120 + + void RGWLibProcess::run() + { + /* write completion interval */ + RGWLibFS::write_completion_interval_s = + cct->_conf->rgw_nfs_write_completion_interval_s; + + /* start write timer */ + RGWLibFS::write_timer.resume(); + + /* gc loop */ + while (! shutdown) { + lsubdout(cct, rgw, 5) << "RGWLibProcess GC" << dendl; + + /* dirent invalidate timeout--basically, the upper-bound on + * inconsistency with the S3 namespace */ + auto expire_s = cct->_conf->rgw_nfs_namespace_expire_secs; + + /* delay between gc cycles */ + auto delay_s = std::max(int64_t(1), std::min(int64_t(MIN_EXPIRE_S), expire_s/2)); + + unique_lock uniq(mtx); + restart: + int cur_gen = gen; + for (auto iter = mounted_fs.begin(); iter != mounted_fs.end(); + ++iter) { + RGWLibFS* fs = iter->first->ref(); + uniq.unlock(); + fs->gc(); + const DoutPrefix dp(cct, dout_subsys, "librgw: "); + fs->update_user(&dp); + fs->rele(); + uniq.lock(); + if (cur_gen != gen) + goto restart; /* invalidated */ + } + cv.wait_for(uniq, std::chrono::seconds(delay_s)); + uniq.unlock(); + } + } + + void RGWLibProcess::handle_request(const DoutPrefixProvider *dpp, RGWRequest* r) + { + /* + * invariant: valid requests are derived from RGWLibRequst + */ + RGWLibRequest* req = static_cast(r); + + // XXX move RGWLibIO and timing setup into process_request + +#if 0 /* XXX */ + utime_t tm = ceph_clock_now(); +#endif + + RGWLibIO io_ctx; + + int ret = process_request(req, &io_ctx); + if (ret < 0) { + /* we don't really care about return code */ + dout(20) << "process_request() returned " << ret << dendl; + + } + delete req; + } /* handle_request */ + + int RGWLibProcess::process_request(RGWLibRequest* req) + { + // XXX move RGWLibIO and timing setup into process_request + +#if 0 /* XXX */ + utime_t tm = ceph_clock_now(); +#endif + + RGWLibIO io_ctx; + + int ret = process_request(req, &io_ctx); + if (ret < 0) { + /* we don't really care about return code */ + dout(20) << "process_request() returned " << ret << dendl; + } + return ret; + } /* process_request */ + + static inline void abort_req(req_state *s, RGWOp *op, int err_no) + { + if (!s) + return; + + /* XXX the dump_errno and dump_bucket_from_state behaviors in + * the abort_early (rgw_rest.cc) might be valuable, but aren't + * safe to call presently as they return HTTP data */ + + perfcounter->inc(l_rgw_failed_req); + } /* abort_req */ + + int RGWLibProcess::process_request(RGWLibRequest* req, RGWLibIO* io) + { + int ret = 0; + bool should_log = true; // XXX + + dout(1) << "====== " << __func__ + << " starting new request req=" << hex << req << dec + << " ======" << dendl; + + /* + * invariant: valid requests are derived from RGWOp--well-formed + * requests should have assigned RGWRequest::op in their descendant + * constructor--if not, the compiler can find it, at the cost of + * a runtime check + */ + RGWOp *op = (req->op) ? req->op : dynamic_cast(req); + if (! op) { + ldpp_dout(op, 1) << "failed to derive cognate RGWOp (invalid op?)" << dendl; + return -EINVAL; + } + + io->init(req->cct); + + perfcounter->inc(l_rgw_req); + + RGWEnv& rgw_env = io->get_env(); + + /* XXX + * until major refactoring of req_state and req_info, we need + * to build their RGWEnv boilerplate from the RGWLibRequest, + * pre-staging any strings (HTTP_HOST) that provoke a crash when + * not found + */ + + /* XXX for now, use ""; could be a legit hostname, or, in future, + * perhaps a tenant (Yehuda) */ + rgw_env.set("HTTP_HOST", ""); + + /* XXX and -then- bloat up req_state with string copies from it */ + req_state rstate(req->cct, &rgw_env, req->id); + req_state *s = &rstate; + + // XXX fix this + s->cio = io; + + /* XXX and -then- stash req_state pointers everywhere they are needed */ + ret = req->init(rgw_env, store, io, s); + if (ret < 0) { + ldpp_dout(op, 10) << "failed to initialize request" << dendl; + abort_req(s, op, ret); + goto done; + } + + /* req is-a RGWOp, currently initialized separately */ + ret = req->op_init(); + if (ret < 0) { + dout(10) << "failed to initialize RGWOp" << dendl; + abort_req(s, op, ret); + goto done; + } + + /* now expected by rgw_log_op() */ + rgw_env.set("REQUEST_METHOD", s->info.method); + rgw_env.set("REQUEST_URI", s->info.request_uri); + rgw_env.set("QUERY_STRING", ""); + + try { + /* XXX authorize does less here then in the REST path, e.g., + * the user's info is cached, but still incomplete */ + ldpp_dout(s, 2) << "authorizing" << dendl; + ret = req->authorize(op, null_yield); + if (ret < 0) { + dout(10) << "failed to authorize request" << dendl; + abort_req(s, op, ret); + goto done; + } + + /* FIXME: remove this after switching all handlers to the new + * authentication infrastructure. */ + if (! s->auth.identity) { + s->auth.identity = rgw::auth::transform_old_authinfo(s); + } + + ldpp_dout(s, 2) << "reading op permissions" << dendl; + ret = req->read_permissions(op, null_yield); + if (ret < 0) { + abort_req(s, op, ret); + goto done; + } + + ldpp_dout(s, 2) << "init op" << dendl; + ret = op->init_processing(null_yield); + if (ret < 0) { + abort_req(s, op, ret); + goto done; + } + + ldpp_dout(s, 2) << "verifying op mask" << dendl; + ret = op->verify_op_mask(); + if (ret < 0) { + abort_req(s, op, ret); + goto done; + } + + ldpp_dout(s, 2) << "verifying op permissions" << dendl; + ret = op->verify_permission(null_yield); + if (ret < 0) { + if (s->system_request) { + ldpp_dout(op, 2) << "overriding permissions due to system operation" << dendl; + } else if (s->auth.identity->is_admin_of(s->user->get_id())) { + ldpp_dout(op, 2) << "overriding permissions due to admin operation" << dendl; + } else { + abort_req(s, op, ret); + goto done; + } + } + + ldpp_dout(s, 2) << "verifying op params" << dendl; + ret = op->verify_params(); + if (ret < 0) { + abort_req(s, op, ret); + goto done; + } + + ldpp_dout(s, 2) << "executing" << dendl; + op->pre_exec(); + op->execute(null_yield); + op->complete(); + + } catch (const ceph::crypto::DigestException& e) { + dout(0) << "authentication failed" << e.what() << dendl; + abort_req(s, op, -ERR_INVALID_SECRET_KEY); + } + + done: + try { + io->complete_request(); + } catch (rgw::io::Exception& e) { + dout(0) << "ERROR: io->complete_request() returned " + << e.what() << dendl; + } + if (should_log) { + rgw_log_op(nullptr /* !rest */, s, (op ? op->name() : "unknown"), olog); + } + + int http_ret = s->err.http_ret; + + ldpp_dout(s, 2) << "http status=" << http_ret << dendl; + + ldpp_dout(op, 1) << "====== " << __func__ + << " req done req=" << hex << req << dec << " http_status=" + << http_ret + << " ======" << dendl; + + return (ret < 0 ? ret : s->err.ret); + } /* process_request */ + + int RGWLibProcess::start_request(RGWLibContinuedReq* req) + { + + dout(1) << "====== " << __func__ + << " starting new continued request req=" << hex << req << dec + << " ======" << dendl; + + /* + * invariant: valid requests are derived from RGWOp--well-formed + * requests should have assigned RGWRequest::op in their descendant + * constructor--if not, the compiler can find it, at the cost of + * a runtime check + */ + RGWOp *op = (req->op) ? req->op : dynamic_cast(req); + if (! op) { + ldpp_dout(op, 1) << "failed to derive cognate RGWOp (invalid op?)" << dendl; + return -EINVAL; + } + + req_state* s = req->get_state(); + RGWLibIO& io_ctx = req->get_io(); + RGWEnv& rgw_env = io_ctx.get_env(); + + rgw_env.set("HTTP_HOST", ""); + + int ret = req->init(rgw_env, store, &io_ctx, s); + if (ret < 0) { + ldpp_dout(op, 10) << "failed to initialize request" << dendl; + abort_req(s, op, ret); + goto done; + } + + /* req is-a RGWOp, currently initialized separately */ + ret = req->op_init(); + if (ret < 0) { + dout(10) << "failed to initialize RGWOp" << dendl; + abort_req(s, op, ret); + goto done; + } + + /* XXX authorize does less here then in the REST path, e.g., + * the user's info is cached, but still incomplete */ + ldpp_dout(s, 2) << "authorizing" << dendl; + ret = req->authorize(op, null_yield); + if (ret < 0) { + dout(10) << "failed to authorize request" << dendl; + abort_req(s, op, ret); + goto done; + } + + /* FIXME: remove this after switching all handlers to the new authentication + * infrastructure. */ + if (! s->auth.identity) { + s->auth.identity = rgw::auth::transform_old_authinfo(s); + } + + ldpp_dout(s, 2) << "reading op permissions" << dendl; + ret = req->read_permissions(op, null_yield); + if (ret < 0) { + abort_req(s, op, ret); + goto done; + } + + ldpp_dout(s, 2) << "init op" << dendl; + ret = op->init_processing(null_yield); + if (ret < 0) { + abort_req(s, op, ret); + goto done; + } + + ldpp_dout(s, 2) << "verifying op mask" << dendl; + ret = op->verify_op_mask(); + if (ret < 0) { + abort_req(s, op, ret); + goto done; + } + + ldpp_dout(s, 2) << "verifying op permissions" << dendl; + ret = op->verify_permission(null_yield); + if (ret < 0) { + if (s->system_request) { + ldpp_dout(op, 2) << "overriding permissions due to system operation" << dendl; + } else if (s->auth.identity->is_admin_of(s->user->get_id())) { + ldpp_dout(op, 2) << "overriding permissions due to admin operation" << dendl; + } else { + abort_req(s, op, ret); + goto done; + } + } + + ldpp_dout(s, 2) << "verifying op params" << dendl; + ret = op->verify_params(); + if (ret < 0) { + abort_req(s, op, ret); + goto done; + } + + op->pre_exec(); + req->exec_start(); + + done: + return (ret < 0 ? ret : s->err.ret); + } + + int RGWLibProcess::finish_request(RGWLibContinuedReq* req) + { + RGWOp *op = (req->op) ? req->op : dynamic_cast(req); + if (! op) { + ldpp_dout(op, 1) << "failed to derive cognate RGWOp (invalid op?)" << dendl; + return -EINVAL; + } + + int ret = req->exec_finish(); + int op_ret = op->get_ret(); + + ldpp_dout(op, 1) << "====== " << __func__ + << " finishing continued request req=" << hex << req << dec + << " op status=" << op_ret + << " ======" << dendl; + + perfcounter->inc(l_rgw_req); + + return ret; + } + + int RGWLibFrontend::init() + { + pprocess = new RGWLibProcess(g_ceph_context, &env, + g_conf()->rgw_thread_pool_size, conf); + return 0; + } + + void RGWLib::set_fe(rgw::RGWLibFrontend* fe) + { + this->fe = fe; + } + + int RGWLib::init() + { + vector args; + return init(args); + } + + int RGWLib::init(vector& args) + { + /* alternative default for module */ + map defaults = { + { "debug_rgw", "1/5" }, + { "keyring", "$rgw_data/keyring" }, + { "log_file", "/var/log/radosgw/$cluster-$name.log" }, + { "objecter_inflight_ops", "24576" }, + // require a secure mon connection by default + { "ms_mon_client_mode", "secure" }, + { "auth_client_required", "cephx" }, + }; + + cct = rgw_global_init(&defaults, args, + CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_DAEMON, + CINIT_FLAG_UNPRIVILEGED_DAEMON_DEFAULTS); + + ceph::mutex mutex = ceph::make_mutex("main"); + SafeTimer init_timer(g_ceph_context, mutex); + init_timer.init(); + mutex.lock(); + init_timer.add_event_after(g_conf()->rgw_init_timeout, new C_InitTimeout); + mutex.unlock(); + + /* stage all front-ends (before common-init-finish) */ + main.init_frontends1(true /* nfs */); + + common_init_finish(g_ceph_context); + + main.init_storage(); + if (! main.get_store()) { + mutex.lock(); + init_timer.cancel_all_events(); + init_timer.shutdown(); + mutex.unlock(); + + derr << "Couldn't init storage provider (RADOS)" << dendl; + return -EIO; + } + + main.init_perfcounters(); + main.init_http_clients(); + main.cond_init_apis(); + + mutex.lock(); + init_timer.cancel_all_events(); + init_timer.shutdown(); + mutex.unlock(); + + main.init_ldap(); + main.init_opslog(); + + init_async_signal_handler(); + register_async_signal_handler(SIGUSR1, handle_sigterm); + + main.init_tracepoints(); + main.init_frontends2(this /* rgwlib */); + main.init_notification_endpoints(); + main.init_lua(); + + return 0; + } /* RGWLib::init() */ + + int RGWLib::stop() + { + derr << "shutting down" << dendl; + main.shutdown(); + + return 0; + } /* RGWLib::stop() */ + + int RGWLibIO::set_uid(rgw::sal::Store* store, const rgw_user& uid) + { + const DoutPrefix dp(store->ctx(), dout_subsys, "librgw: "); + std::unique_ptr user = store->get_user(uid); + /* object exists, but policy is broken */ + int ret = user->load_user(&dp, null_yield); + if (ret < 0) { + derr << "ERROR: failed reading user info: uid=" << uid << " ret=" + << ret << dendl; + } + user_info = user->get_info(); + return ret; + } + + int RGWLibRequest::read_permissions(RGWOp* op, optional_yield y) { + /* bucket and object ops */ + int ret = + rgw_build_bucket_policies(op, g_rgwlib->get_store(), get_state(), y); + if (ret < 0) { + ldpp_dout(op, 10) << "read_permissions (bucket policy) on " + << get_state()->bucket << ":" + << get_state()->object + << " only_bucket=" << only_bucket() + << " ret=" << ret << dendl; + if (ret == -ENODATA) + ret = -EACCES; + } else if (! only_bucket()) { + /* object ops */ + ret = rgw_build_object_policies(op, g_rgwlib->get_store(), get_state(), + op->prefetch_data(), y); + if (ret < 0) { + ldpp_dout(op, 10) << "read_permissions (object policy) on" + << get_state()->bucket << ":" + << get_state()->object + << " ret=" << ret << dendl; + if (ret == -ENODATA) + ret = -EACCES; + } + } + return ret; + } /* RGWLibRequest::read_permissions */ + + int RGWHandler_Lib::authorize(const DoutPrefixProvider *dpp, optional_yield y) + { + /* TODO: handle + * 1. subusers + * 2. anonymous access + * 3. system access + * 4. ? + * + * Much or all of this depends on handling the cached authorization + * correctly (e.g., dealing with keystone) at mount time. + */ + s->perm_mask = RGW_PERM_FULL_CONTROL; + + // populate the owner info + s->owner.set_id(s->user->get_id()); + s->owner.set_name(s->user->get_display_name()); + + return 0; + } /* RGWHandler_Lib::authorize */ + +} /* namespace rgw */ diff --git a/src/rgw/rgw_lib.h b/src/rgw/rgw_lib.h index 600a7bcedcb..c104fad18af 100644 --- a/src/rgw/rgw_lib.h +++ b/src/rgw/rgw_lib.h @@ -45,7 +45,7 @@ namespace rgw { int stop(); }; - extern RGWLib rgwlib; + extern RGWLib* g_rgwlib; /* request interface */ @@ -130,7 +130,7 @@ namespace rgw { inline req_state* get_state() { return this->RGWRequest::s; } RGWLibRequest(CephContext* _cct, std::unique_ptr _user) - : RGWRequest(rgwlib.get_store()->get_new_req_id()), + : RGWRequest(g_rgwlib->get_store()->get_new_req_id()), tuser(std::move(_user)), cct(_cct) {} @@ -188,7 +188,7 @@ namespace rgw { io_ctx.init(_cct); RGWRequest::init_state(&rstate); - RGWHandler::init(rgwlib.get_store(), &rstate, &io_ctx); + RGWHandler::init(g_rgwlib->get_store(), &rstate, &io_ctx); get_state()->req_id = store->zone_unique_id(id); get_state()->trans_id = store->zone_unique_trans_id(id); diff --git a/src/rgw/rgw_main.h b/src/rgw/rgw_main.h index d7425ad9ebf..09b23250179 100644 --- a/src/rgw/rgw_main.h +++ b/src/rgw/rgw_main.h @@ -51,67 +51,66 @@ public: namespace rgw { - class RGWLib; - - class AppMain { - /* several components should be initalized only if librgw is - * also serving HTTP */ - bool have_http_frontend{false}; - bool nfs{false}; - - std::vector fes; - std::vector fe_configs; - std::multimap fe_map; - std::unique_ptr ldh; - OpsLogSink* olog; - RGWREST rest; - std::unique_ptr lua_background; - std::unique_ptr implicit_tenant_context; - std::unique_ptr sched_ctx; - std::unique_ptr ratelimiter; - // wow, realm reloader has a lot of parts - std::unique_ptr reloader; - std::unique_ptr pusher; - std::unique_ptr fe_pauser; - std::unique_ptr realm_watcher; - std::unique_ptr rgw_pauser; - rgw::sal::Store* store; - DoutPrefixProvider* dpp; - - public: - AppMain(DoutPrefixProvider* dpp) - : dpp(dpp) - {} - - void shutdown(); - - rgw::sal::Store* get_store() { - return store; - } - - rgw::LDAPHelper* get_ldh() { - return ldh.get(); - } - - void init_frontends1(bool nfs = false); - void init_numa(); - void init_storage(); - void init_perfcounters(); - void init_http_clients(); - void cond_init_apis(); - void init_ldap(); - void init_opslog(); - int init_frontends2(RGWLib* rgwlib = nullptr); - void init_tracepoints(); - void init_notification_endpoints(); - void init_lua(); - - bool have_http() { - return have_http_frontend; - } - - static OpsLogFile* ops_log_file; - }; /* AppMain */ +class RGWLib; +class AppMain { + /* several components should be initalized only if librgw is + * also serving HTTP */ + bool have_http_frontend{false}; + bool nfs{false}; + + std::vector fes; + std::vector fe_configs; + std::multimap fe_map; + std::unique_ptr ldh; + OpsLogSink* olog; + RGWREST rest; + std::unique_ptr lua_background; + std::unique_ptr implicit_tenant_context; + std::unique_ptr sched_ctx; + std::unique_ptr ratelimiter; + // wow, realm reloader has a lot of parts + std::unique_ptr reloader; + std::unique_ptr pusher; + std::unique_ptr fe_pauser; + std::unique_ptr realm_watcher; + std::unique_ptr rgw_pauser; + rgw::sal::Store* store; + DoutPrefixProvider* dpp; + +public: + AppMain(DoutPrefixProvider* dpp) + : dpp(dpp) + {} + + void shutdown(); + + rgw::sal::Store* get_store() { + return store; + } + + rgw::LDAPHelper* get_ldh() { + return ldh.get(); + } + + void init_frontends1(bool nfs = false); + void init_numa(); + void init_storage(); + void init_perfcounters(); + void init_http_clients(); + void cond_init_apis(); + void init_ldap(); + void init_opslog(); + int init_frontends2(RGWLib* rgwlib = nullptr); + void init_tracepoints(); + void init_notification_endpoints(); + void init_lua(); + + bool have_http() { + return have_http_frontend; + } + + static OpsLogFile* ops_log_file; +}; /* AppMain */ } // namespace rgw static inline RGWRESTMgr *set_logging(RGWRESTMgr* mgr) diff --git a/src/rgw/rgw_signal.h b/src/rgw/rgw_signal.h index d6c1994678f..68fc4f614a3 100644 --- a/src/rgw/rgw_signal.h +++ b/src/rgw/rgw_signal.h @@ -17,15 +17,15 @@ namespace rgw { - namespace signal { +namespace signal { - void signal_shutdown(); - void wait_shutdown(); - int signal_fd_init(); - void signal_fd_finalize(); - void handle_sigterm(int signum); - void handle_sigterm(int signum); - void sighup_handler(int signum); +void signal_shutdown(); +void wait_shutdown(); +int signal_fd_init(); +void signal_fd_finalize(); +void handle_sigterm(int signum); +void handle_sigterm(int signum); +void sighup_handler(int signum); - } // namespace signal +} // namespace signal } // namespace rgw diff --git a/src/test/librgw_file_nfsns.cc b/src/test/librgw_file_nfsns.cc index 3adcc2d3ec6..2ce65519c01 100644 --- a/src/test/librgw_file_nfsns.cc +++ b/src/test/librgw_file_nfsns.cc @@ -21,6 +21,7 @@ #include "include/rados/librgw.h" #include "include/rados/rgw_file.h" #include "rgw/rgw_file.h" +#include "rgw_lib.h" #include "rgw/rgw_lib_frontend.h" // direct requests #include "gtest/gtest.h" @@ -31,6 +32,7 @@ #define dout_subsys ceph_subsys_rgw using namespace std; +using namespace rgw; // g_rgwlib namespace { @@ -259,9 +261,9 @@ TEST(LibRGW, SETUP_HIER1) << std::endl; } RGWPutObjRequest req(cct, - rgwlib.get_store()->get_user(fs_private->get_user()->user_id), + g_rgwlib->get_store()->get_user(fs_private->get_user()->user_id), bucket_name, obj_name, bl); - int rc = rgwlib.get_fe()->execute_req(&req); + int rc = g_rgwlib->get_fe()->execute_req(&req); int rc2 = req.get_ret(); ASSERT_EQ(rc, 0); ASSERT_EQ(rc2, 0); -- 2.39.5