From 059019c907c8387debdb8fc0fbb470a39ad0cf7c Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Mon, 11 Jul 2011 13:41:40 -0700 Subject: [PATCH] rgw: restructure runtime config env --- src/Makefile.am | 3 +- src/rgw/rgw_cache.cc | 24 +++++++-------- src/rgw/rgw_cache.h | 3 -- src/rgw/rgw_common.h | 36 +++++++++++++++++++++- src/rgw/rgw_env.cc | 70 ++++++++++++++++++++++++++++++++++++++++++ src/rgw/rgw_log.cc | 2 +- src/rgw/rgw_main.cc | 2 ++ src/rgw/rgw_op.cc | 13 ++------ src/rgw/rgw_op.h | 2 +- src/rgw/rgw_os_auth.cc | 8 ++--- src/rgw/rgw_rest.cc | 55 ++++++++++++++++----------------- src/rgw/rgw_rest_s3.cc | 8 ++--- 12 files changed, 159 insertions(+), 67 deletions(-) create mode 100644 src/rgw/rgw_env.cc diff --git a/src/Makefile.am b/src/Makefile.am index 4b90ed178d634..fca3c89929809 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -319,7 +319,8 @@ my_radosgw_src = \ rgw/rgw_formats.cc \ rgw/rgw_log.cc \ rgw/rgw_multi.cc \ - rgw/rgw_escape.c + rgw/rgw_escape.c \ + rgw/rgw_env.cc my_radosgw_ldadd = \ libglobal.la librgw.la librados.la -lfcgi -lcurl -lexpat \ diff --git a/src/rgw/rgw_cache.cc b/src/rgw/rgw_cache.cc index c6182ee8f6402..db63a35ebb9b2 100644 --- a/src/rgw/rgw_cache.cc +++ b/src/rgw/rgw_cache.cc @@ -77,24 +77,22 @@ void ObjectCache::remove(string& name) void ObjectCache::touch_lru(string& name, std::list::iterator& lru_iter) { + while (lru.size() > (size_t)rgwconf->max_cache_lru) { + list::iterator iter = lru.begin(); + map::iterator map_iter = cache_map.find(*iter); + RGW_LOG(10) << "removing entry: name=" << *iter << " from cache LRU" << dendl; + if (map_iter != cache_map.end()) + cache_map.erase(map_iter); + lru.pop_front(); + } + if (lru_iter == lru.end()) { lru.push_back(name); lru_iter--; - RGW_LOG(0) << "adding " << name << " to cache LRU end" << dendl; - - -#define MAX_CACHE_SIZE 10000 // FIXME: make this configurable - while (lru.size() > MAX_CACHE_SIZE) { - list::iterator iter = lru.begin(); - map::iterator map_iter = cache_map.find(*iter); - RGW_LOG(0) << "removing entry: name=" << *iter << " from cache LRU" << dendl; - if (map_iter != cache_map.end()) - cache_map.erase(map_iter); - lru.pop_front(); - } + RGW_LOG(10) << "adding " << name << " to cache LRU end" << dendl; } else { string name = *lru_iter; - RGW_LOG(0) << "moving " << name << " to cache LRU end" << dendl; + RGW_LOG(10) << "moving " << name << " to cache LRU end" << dendl; lru.erase(lru_iter); lru.push_back(name); lru_iter = lru.end(); diff --git a/src/rgw/rgw_cache.h b/src/rgw/rgw_cache.h index 6b1d7fd9a1087..dd8e17dc62fe4 100644 --- a/src/rgw/rgw_cache.h +++ b/src/rgw/rgw_cache.h @@ -284,9 +284,6 @@ int RGWCache::watch_cb(int opcode, uint64_t ver, bufferlist& bl) return -EIO; } - string& bucket = info.obj.bucket; - string& oid = info.obj.object; - switch (info.op) { case UPDATE_OBJ: cache.put(info.ns, info.obj_info); diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index b965d03c85308..cc9eb4da3ff02 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -155,6 +155,40 @@ class XMLArgs map& get_sub_resources() { return sub_resources; } }; +class RGWConf; + +class RGWEnv { + std::map env_map; +public: + RGWConf *conf; + + RGWEnv() : conf(NULL) {} + ~RGWEnv(); + void reinit(char **envp); + const char *get(const char *name, const char *def_val = NULL); + int get_int(const char *name, int def_val = 0); + size_t get_size(const char *name, size_t def_val = 0); +}; + +class RGWConf { + friend class RGWEnv; +protected: + void init(RGWEnv * env); +public: + RGWConf() : + max_cache_lru(0), + log_level(0), + should_log(0) {} + + size_t max_cache_lru; + int log_level; + int should_log; +}; + +extern RGWEnv rgw_env; + +#define rgwconf rgw_env.conf + enum http_op { OP_GET, OP_PUT, @@ -398,7 +432,7 @@ struct req_state { int prot_flags; - char *os_auth_token; + const char *os_auth_token; char *os_user; char *os_groups; diff --git a/src/rgw/rgw_env.cc b/src/rgw/rgw_env.cc new file mode 100644 index 0000000000000..68bd646baa85c --- /dev/null +++ b/src/rgw/rgw_env.cc @@ -0,0 +1,70 @@ +#include "rgw_common.h" +#include "rgw_log.h" + +#include +#include + +RGWEnv rgw_env; + +RGWEnv::~RGWEnv() +{ + delete conf; +} + +void RGWEnv::reinit(char **envp) +{ + const char *p; + + env_map.clear(); + + for (int i=0; (p = envp[i]); ++i) { + string s(p); + int pos = s.find('='); + if (pos <= 0) // should never be 0 + continue; + string name = s.substr(0, pos); + string val = s.substr(pos + 1); + env_map[name] = val; + } + + if (!conf) + conf = new RGWConf; + + conf->init(this); +} + +const char *RGWEnv::get(const char *name, const char *def_val) +{ + map::iterator iter = env_map.find(name); + if (iter == env_map.end()) + return def_val; + + return iter->second.c_str(); +} + +int RGWEnv::get_int(const char *name, int def_val) +{ + map::iterator iter = env_map.find(name); + if (iter == env_map.end()) + return def_val; + + const char *s = iter->second.c_str(); + return atoi(s); +} + +size_t RGWEnv::get_size(const char *name, size_t def_val) +{ + map::iterator iter = env_map.find(name); + if (iter == env_map.end()) + return def_val; + + const char *s = iter->second.c_str(); + return atoll(s); +} + +void RGWConf::init(RGWEnv *env) +{ + max_cache_lru = env->get_size("RGW_MAX_CACHE_LRU", 10000); + log_level = env->get_int("RGW_LOG_LEVEL", g_conf->rgw_log); + should_log = env->get_int("RGW_SHOULD_LOG", RGW_SHOULD_LOG_DEFAULT); +} diff --git a/src/rgw/rgw_log.cc b/src/rgw/rgw_log.cc index 96aa4a15617e4..75447462af0d3 100644 --- a/src/rgw/rgw_log.cc +++ b/src/rgw/rgw_log.cc @@ -6,7 +6,7 @@ static void set_param_str(struct req_state *s, const char *name, string& str) { - const char *p = FCGX_GetParam(name, s->fcgx->envp); + const char *p = rgw_env.get(name); if (p) str = p; } diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc index eded7648884b2..d378360b4d164 100644 --- a/src/rgw/rgw_main.cc +++ b/src/rgw/rgw_main.cc @@ -87,6 +87,8 @@ int main(int argc, const char **argv) while (FCGX_Accept(&fcgx.in, &fcgx.out, &fcgx.err, &fcgx.envp) >= 0) { + rgw_env.reinit(fcgx.envp); + RGWOp *op; int init_error = 0; RGWHandler *handler = RGWHandler_REST::init_handler(&s, &fcgx, &init_error); diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 678aa4c952f47..638f1b1530a92 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -1373,17 +1373,10 @@ done: void RGWHandler::init_state(struct req_state *s, struct fcgx_state *fcgx) { - /* Retrieve the loglevel from the CGI envirioment (if set) */ - const char *cgi_env_level = FCGX_GetParam("RGW_LOG_LEVEL", fcgx->envp); - if (cgi_env_level != NULL) { - int level = atoi(cgi_env_level); - if (level >= 0) { - g_conf->rgw_log = level; - } - } + if (rgwconf->log_level >= 0) + g_conf->rgw_log = rgwconf->log_level; - const char *cgi_should_log = FCGX_GetParam("RGW_SHOULD_LOG", fcgx->envp); - s->should_log = rgw_str_to_bool(cgi_should_log, RGW_SHOULD_LOG_DEFAULT); + s->should_log = rgwconf->should_log; if (g_conf->rgw_log >= 20) { char *p; diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 8f91cc604d83f..a5f4abfae8aad 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -215,7 +215,7 @@ protected: size_t len; off_t ofs; char *data; - char *supplied_md5_b64; + const char *supplied_md5_b64; string etag; public: diff --git a/src/rgw/rgw_os_auth.cc b/src/rgw/rgw_os_auth.cc index ab0044215eed1..ce5e7125979be 100644 --- a/src/rgw/rgw_os_auth.cc +++ b/src/rgw/rgw_os_auth.cc @@ -123,10 +123,10 @@ void RGW_OS_Auth_Get::execute() RGW_LOG(0) << "RGW_OS_Auth_Get::execute()" << dendl; - const char *key = FCGX_GetParam("HTTP_X_AUTH_KEY", s->fcgx->envp); - const char *user = FCGX_GetParam("HTTP_X_AUTH_USER", s->fcgx->envp); - const char *url_prefix = FCGX_GetParam("RGW_OPENSTACK_URL_PREFIX", s->fcgx->envp); - const char *os_url = FCGX_GetParam("RGW_OPENSTACK_URL", s->fcgx->envp); + const char *key = rgw_env.get("HTTP_X_AUTH_KEY"); + const char *user = rgw_env.get("HTTP_X_AUTH_USER"); + const char *url_prefix = rgw_env.get("RGW_OPENSTACK_URL_PREFIX"); + const char *os_url = rgw_env.get("RGW_OPENSTACK_URL"); string user_str = user; RGWUserInfo info; diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index d99e8df01be24..e87f7d27bca07 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -209,11 +209,11 @@ void dump_range(struct req_state *s, off_t ofs, off_t end) int RGWGetObj_REST::get_params() { - range_str = FCGX_GetParam("HTTP_RANGE", s->fcgx->envp); - if_mod = FCGX_GetParam("HTTP_IF_MODIFIED_SINCE", s->fcgx->envp); - if_unmod = FCGX_GetParam("HTTP_IF_UNMODIFIED_SINCE", s->fcgx->envp); - if_match = FCGX_GetParam("HTTP_IF_MATCH", s->fcgx->envp); - if_nomatch = FCGX_GetParam("HTTP_IF_NONE_MATCH", s->fcgx->envp); + range_str = rgw_env.get("HTTP_RANGE"); + if_mod = rgw_env.get("HTTP_IF_MODIFIED_SINCE"); + if_unmod = rgw_env.get("HTTP_IF_UNMODIFIED_SINCE"); + if_match = rgw_env.get("HTTP_IF_MATCH"); + if_nomatch = rgw_env.get("HTTP_IF_NONE_MATCH"); return 0; } @@ -221,7 +221,7 @@ int RGWGetObj_REST::get_params() int RGWPutObj_REST::get_params() { - supplied_md5_b64 = FCGX_GetParam("HTTP_CONTENT_MD5", s->fcgx->envp); + supplied_md5_b64 = rgw_env.get("HTTP_CONTENT_MD5"); return 0; } @@ -247,17 +247,17 @@ int RGWPutObj_REST::get_data() } if (!ofs) - supplied_md5_b64 = FCGX_GetParam("HTTP_CONTENT_MD5", s->fcgx->envp); + supplied_md5_b64 = rgw_env.get("HTTP_CONTENT_MD5"); return 0; } int RGWCopyObj_REST::get_params() { - if_mod = FCGX_GetParam("HTTP_X_AMZ_COPY_IF_MODIFIED_SINCE", s->fcgx->envp); - if_unmod = FCGX_GetParam("HTTP_X_AMZ_COPY_IF_UNMODIFIED_SINCE", s->fcgx->envp); - if_match = FCGX_GetParam("HTTP_X_AMZ_COPY_IF_MATCH", s->fcgx->envp); - if_nomatch = FCGX_GetParam("HTTP_X_AMZ_COPY_IF_NONE_MATCH", s->fcgx->envp); + if_mod = rgw_env.get("HTTP_X_AMZ_COPY_IF_MODIFIED_SINCE"); + if_unmod = rgw_env.get("HTTP_X_AMZ_COPY_IF_UNMODIFIED_SINCE"); + if_match = rgw_env.get("HTTP_X_AMZ_COPY_IF_MATCH"); + if_nomatch = rgw_env.get("HTTP_X_AMZ_COPY_IF_NONE_MATCH"); return 0; } @@ -377,9 +377,7 @@ void init_entities_from_header(struct req_state *s) string req; string first; - gateway_dns_name = FCGX_GetParam("RGW_DNS_NAME", s->fcgx->envp); - if (!gateway_dns_name) - gateway_dns_name = "s3."; + gateway_dns_name = rgw_env.get("RGW_DNS_NAME", "s3."); RGW_LOG(20) << "gateway_dns_name = " << gateway_dns_name << dendl; @@ -438,7 +436,7 @@ void init_entities_from_header(struct req_state *s) pos = req.find('/'); if (pos >= 0) { - const char *openstack_url_prefix = FCGX_GetParam("RGW_OPENSTACK_URL_PREFIX", s->fcgx->envp); + const char *openstack_url_prefix = rgw_env.get("RGW_OPENSTACK_URL_PREFIX"); bool cut_url = (openstack_url_prefix != NULL); if (!openstack_url_prefix) openstack_url_prefix = "v1"; @@ -470,12 +468,11 @@ void init_entities_from_header(struct req_state *s) string ver; string auth_key; - RGW_LOG(10) << "before2" << dendl; next_tok(req, ver, '/'); RGW_LOG(10) << "ver=" << ver << dendl; next_tok(req, auth_key, '/'); RGW_LOG(10) << "auth_key=" << auth_key << dendl; - s->os_auth_token = FCGX_GetParam("HTTP_X_AUTH_TOKEN", s->fcgx->envp); + s->os_auth_token = rgw_env.get("HTTP_X_AUTH_TOKEN"); next_tok(req, first, '/'); RGW_LOG(10) << "ver=" << ver << " auth_key=" << auth_key << " first=" << first << " req=" << req << dendl; @@ -682,16 +679,16 @@ int RGWHandler_REST::init_rest(struct req_state *s, struct fcgx_state *fcgx) int ret = 0; RGWHandler::init_state(s, fcgx); - s->path_name = FCGX_GetParam("SCRIPT_NAME", s->fcgx->envp); - s->path_name_url = FCGX_GetParam("REQUEST_URI", s->fcgx->envp); + s->path_name = rgw_env.get("SCRIPT_NAME"); + s->path_name_url = rgw_env.get("REQUEST_URI"); int pos = s->path_name_url.find('?'); if (pos >= 0) s->path_name_url = s->path_name_url.substr(0, pos); - s->method = FCGX_GetParam("REQUEST_METHOD", s->fcgx->envp); - s->host = FCGX_GetParam("HTTP_HOST", s->fcgx->envp); - s->query = FCGX_GetParam("QUERY_STRING", s->fcgx->envp); - s->length = FCGX_GetParam("CONTENT_LENGTH", s->fcgx->envp); - s->content_type = FCGX_GetParam("CONTENT_TYPE", s->fcgx->envp); + s->method = rgw_env.get("REQUEST_METHOD"); + s->host = rgw_env.get("HTTP_HOST"); + s->query = rgw_env.get("QUERY_STRING"); + s->length = rgw_env.get("CONTENT_LENGTH"); + s->content_type = rgw_env.get("CONTENT_TYPE"); s->prot_flags = 0; if (!s->method) @@ -720,16 +717,16 @@ int RGWHandler_REST::init_rest(struct req_state *s, struct fcgx_state *fcgx) init_auth_info(s); - const char *cacl = FCGX_GetParam("HTTP_X_AMZ_ACL", s->fcgx->envp); + const char *cacl = rgw_env.get("HTTP_X_AMZ_ACL"); if (cacl) s->canned_acl = cacl; - s->copy_source = FCGX_GetParam("HTTP_X_AMZ_COPY_SOURCE", s->fcgx->envp); - s->http_auth = FCGX_GetParam("HTTP_AUTHORIZATION", s->fcgx->envp); + s->copy_source = rgw_env.get("HTTP_X_AMZ_COPY_SOURCE"); + s->http_auth = rgw_env.get("HTTP_AUTHORIZATION"); - const char *cgi_env_continue = FCGX_GetParam("RGW_PRINT_CONTINUE", s->fcgx->envp); + const char *cgi_env_continue = rgw_env.get("RGW_PRINT_CONTINUE"); if (rgw_str_to_bool(cgi_env_continue, 0)) { - const char *expect = FCGX_GetParam("HTTP_EXPECT", s->fcgx->envp); + const char *expect = rgw_env.get("HTTP_EXPECT"); s->expect_cont = (expect && !strcasecmp(expect, "100-continue")); } return ret; diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 83183fd12bb7e..4acdc3f75ca70 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -270,7 +270,7 @@ void RGWCompleteMultipart_REST_S3::send_response() if (ret == 0) { dump_start(s); s->formatter->open_obj_section("CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\""); - const char *gateway_dns_name = FCGX_GetParam("RGW_DNS_NAME", s->fcgx->envp); + const char *gateway_dns_name = rgw_env.get("RGW_DNS_NAME"); if (gateway_dns_name) s->formatter->dump_value_str("Location", "%s.%s", s->bucket, gateway_dns_name); s->formatter->dump_value_str("Bucket", s->bucket); @@ -531,12 +531,12 @@ static void get_auth_header(struct req_state *s, string& dest, bool qsr) dest = s->method; dest.append("\n"); - const char *md5 = FCGX_GetParam("HTTP_CONTENT_MD5", s->fcgx->envp); + const char *md5 = rgw_env.get("HTTP_CONTENT_MD5"); if (md5) dest.append(md5); dest.append("\n"); - const char *type = FCGX_GetParam("CONTENT_TYPE", s->fcgx->envp); + const char *type = rgw_env.get("CONTENT_TYPE"); if (type) dest.append(type); dest.append("\n"); @@ -545,7 +545,7 @@ static void get_auth_header(struct req_state *s, string& dest, bool qsr) if (qsr) { date = s->args.get("Expires"); } else { - const char *str = FCGX_GetParam("HTTP_DATE", s->fcgx->envp); + const char *str = rgw_env.get("HTTP_DATE"); if (str) date = str; } -- 2.39.5