From 07bf9b821c802501056528236ff034f68bec1e3f Mon Sep 17 00:00:00 2001 From: Greg Farnum Date: Wed, 2 Mar 2011 14:13:38 -0800 Subject: [PATCH] tcmalloc: switch the interface. Previously, we used function pointers. Fun for me to learn about, icky to actually have! Now we use our own wrapper functions with two implementations -- one for with tcmalloc and one without. Make those programs which are tcmalloc-aware build with the appropriate implementation source at compile-time, but leave the wrapper function stubs in no matter what. While we're at it, implement two of the "MallocExtension" calls in the OSD. Signed-off-by: Greg Farnum --- src/Makefile.am | 7 +++ src/cmds.cc | 7 --- src/common/common_init.cc | 14 ----- src/common/config.h | 9 ---- src/cosd.cc | 10 ++-- src/mds/MDS.cc | 14 +++-- src/osd/OSD.cc | 32 ++++++----- src/perfglue/disabled_heap_profiler.cc | 30 +++++++++++ src/perfglue/heap_profiler.cc | 75 ++++++++++++++++++++++++++ src/perfglue/heap_profiler.h | 44 +++++++++++++++ 10 files changed, 184 insertions(+), 58 deletions(-) create mode 100644 src/perfglue/disabled_heap_profiler.cc create mode 100644 src/perfglue/heap_profiler.cc create mode 100644 src/perfglue/heap_profiler.h diff --git a/src/Makefile.am b/src/Makefile.am index cd9f6c36c54e7..76a052fcc8571 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -129,12 +129,17 @@ if WITH_TCMALLOC tcmalloc_safety_flags = -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free cosd_LDADD += -ltcmalloc cosd_CXXFLAGS += ${tcmalloc_safety_flags} +cosd_SOURCES += perfglue/heap_profiler.cc cmds_LDADD += -ltcmalloc cmds_CXXFLAGS += ${tcmalloc_safety_flags} +cmds_SOURCES += perfglue/heap_profiler.cc if WITH_FUSE cfuse_LDADD += -ltcmalloc cfuse_CXXFLAGS += ${tcmalloc_safety_flags} endif #WITH_FUSE +else +cosd_SOURCES += perfglue/disabled_heap_profiler.cc +cmds_SOURCES += perfglue/disabled_heap_profiler.cc endif # WITH_TCMALLOC # debug targets? @@ -561,6 +566,7 @@ else libcommon_files += perfglue/disabled_stubs.cc endif + libcrush_a_SOURCES = \ crush/builder.c \ crush/mapper.c \ @@ -974,6 +980,7 @@ noinst_HEADERS = \ osdc/ObjectCacher.h\ osdc/Objecter.h\ perfglue/cpu_profiler.h\ + perfglue/heap_profiler.h\ rgw/rgw_access.h\ rgw/rgw_acl.h\ rgw/rgw_fs.h\ diff --git a/src/cmds.cc b/src/cmds.cc index c058b51b5a0c6..ebd9796c9864c 100644 --- a/src/cmds.cc +++ b/src/cmds.cc @@ -65,13 +65,6 @@ int main(int argc, const char **argv) argv_to_vec(argc, argv, args); env_to_vec(args); -#ifdef HAVE_LIBTCMALLOC - g_conf.profiler_start = HeapProfilerStart; - g_conf.profiler_running = IsHeapProfilerRunning; - g_conf.profiler_stop = HeapProfilerStop; - g_conf.profiler_dump = HeapProfilerDump; - g_conf.tcmalloc_have = true; -#endif //HAVE_LIBTCMALLOC common_init(args, "mds", STARTUP_FLAG_INIT_KEYS | STARTUP_FLAG_DAEMON); // mds specific args diff --git a/src/common/common_init.cc b/src/common/common_init.cc index 9bd3cb96e9c4f..748e0385590eb 100644 --- a/src/common/common_init.cc +++ b/src/common/common_init.cc @@ -168,20 +168,6 @@ void common_init(std::vector& args, const char *module_type, int fl parse_config_options(args); install_standard_sighandlers(); -#ifdef HAVE_LIBTCMALLOC - if (g_conf.tcmalloc_profiler_run && g_conf.tcmalloc_have) { - char profile_name[PATH_MAX]; - sprintf(profile_name, "%s/%s", g_conf.log_dir, g_conf.name); - char *val = new char[sizeof(int)*8+1]; - sprintf(val, "%i", g_conf.profiler_allocation_interval); - setenv("HEAP_PROFILE_ALLOCATION_INTERVAL", val, g_conf.profiler_allocation_interval); - sprintf(val, "%i", g_conf.profiler_highwater_interval); - setenv("HEAP_PROFILE_INUSE_INTERVAL", val, g_conf.profiler_highwater_interval); - generic_dout(0) << "turning on heap profiler with prefix " << profile_name << dendl; - g_conf.profiler_start(profile_name); - } -#endif //HAVE_LIBTCMALLOC - if (flags & STARTUP_FLAG_INIT_KEYS) { if (is_supported_auth(CEPH_AUTH_CEPHX)) keyring_init(g_conf.keyring); diff --git a/src/common/config.h b/src/common/config.h index 923cb649ca212..cb0e2f49d48b0 100644 --- a/src/common/config.h +++ b/src/common/config.h @@ -31,10 +31,6 @@ extern struct ceph_file_layout g_default_file_layout; #include "msg/msg_types.h" -#ifdef HAVE_LIBTCMALLOC -#include -#endif //HAVE_LIBTCMALLOC - struct EntityName; enum log_to_stderr_t { @@ -68,12 +64,7 @@ struct md_config_t { bool daemonize; //profiling - bool tcmalloc_have; bool tcmalloc_profiler_run; - void (*profiler_start)(const char*); - bool (*profiler_running)(); - void (*profiler_stop)(); - void (*profiler_dump)(const char*); int profiler_allocation_interval; int profiler_highwater_interval; diff --git a/src/cosd.cc b/src/cosd.cc index edffb329adbad..09ef59ad73e4b 100644 --- a/src/cosd.cc +++ b/src/cosd.cc @@ -37,6 +37,8 @@ using namespace std; #include "include/color.h" #include "common/errno.h" +#include "perfglue/heap_profiler.h" + void usage() { derr << "usage: cosd -i osdid [--osd-data=path] [--osd-journal=path] " @@ -62,14 +64,8 @@ int main(int argc, const char **argv) } } -#ifdef HAVE_LIBTCMALLOC - g_conf.profiler_start = HeapProfilerStart; - g_conf.profiler_running = IsHeapProfilerRunning; - g_conf.profiler_stop = HeapProfilerStop; - g_conf.profiler_dump = HeapProfilerDump; - g_conf.tcmalloc_have = true; -#endif //HAVE_LIBTCMALLOC common_init(args, "osd", startup_flags); + ceph_heap_profiler_init(); // osd specific args bool mkfs = false; diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc index 6eba78e90e897..413e35e703b6d 100644 --- a/src/mds/MDS.cc +++ b/src/mds/MDS.cc @@ -71,6 +71,7 @@ #include "common/config.h" #include "perfglue/cpu_profiler.h" +#include "perfglue/heap_profiler.h" #define DOUT_SUBSYS mds @@ -791,23 +792,20 @@ void MDS::handle_command(MMonCommand *m) clog.info() << g_conf.name << " set heap variables from current config\n"; } else if (m->cmd.size() == 1 && m->cmd[0] == "heap_profiler_start") { - char location[PATH_MAX]; - snprintf(location, sizeof(location), - "%s/%s", g_conf.log_dir, g_conf.name); - g_conf.profiler_start(location); + ceph_heap_profiler_start(); clog.info() << g_conf.name << " started profiler\n"; } else if (m->cmd.size() == 1 && m->cmd[0] == "heap_profiler_stop") { - g_conf.profiler_stop(); + ceph_heap_profiler_stop(); clog.info() << g_conf.name << " stopped profiler\n"; } else if (m->cmd.size() == 1 && m->cmd[0] == "heap_profiler_dump"){ - if (g_conf.tcmalloc_have) { - if (!g_conf.profiler_running()) { + if (ceph_using_tcmalloc()) { + if (!ceph_heap_profiler_running()) { clog.info() << g_conf.name << " can't dump heap: profiler not running\n"; } else { clog.info() << g_conf.name << " dumping heap profile now\n"; - g_conf.profiler_dump("admin request"); + ceph_heap_profiler_dump("admin request"); } } else { clog.info() << "tcmalloc not enabled, can't use profiler\n"; diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 76e2021a10d77..baa277f39eff6 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -82,6 +82,7 @@ #include "common/LogClient.h" #include "common/safe_io.h" #include "perfglue/cpu_profiler.h" +#include "perfglue/heap_profiler.h" #include "common/ClassHandler.h" @@ -2083,12 +2084,15 @@ void OSD::handle_command(MMonCommand *m) } else if (m->cmd.size() == 2 && m->cmd[0] == "logger" && m->cmd[1] == "reopen") { logger_reopen_all(); } else if (m->cmd.size() == 1 && m->cmd[0] == "heapdump") { - if (g_conf.tcmalloc_have) { - if (!g_conf.profiler_running()) { + if (ceph_using_tcmalloc()) { + if (!ceph_heap_profiler_running()) { clog.info() << "can't dump heap: profiler not running\n"; } else { - clog.info() << g_conf.name << "dumping heap profile now\n"; - g_conf.profiler_dump("admin request"); + char *heap_stats = new char[1024]; + ceph_heap_profiler_stats(heap_stats, 1024); + clog.info() << g_conf.name << "dumping heap profile now.\n" + << heap_stats << std::endl; + ceph_heap_profiler_dump("admin request"); } } else { clog.info() << g_conf.name << " does not have tcmalloc, " @@ -2102,17 +2106,19 @@ void OSD::handle_command(MMonCommand *m) setenv("HEAP_PROFILE_INUSE_INTERVAL", val, g_conf.profiler_highwater_interval); clog.info() << g_conf.name << " set heap variables from current config"; } else if (m->cmd.size() == 1 && m->cmd[0] == "start_profiler") { - char location[PATH_MAX]; - snprintf(location, sizeof(location), - "%s/%s", g_conf.log_dir, g_conf.name); - g_conf.profiler_start(location); - clog.info() << g_conf.name << " started profiler with output " - << location << "\n"; + ceph_heap_profiler_start(); + clog.info() << g_conf.name << " started profiler \n"; } else if (m->cmd.size() == 1 && m->cmd[0] == "stop_profiler") { - g_conf.profiler_stop(); + ceph_heap_profiler_stop(); clog.info() << g_conf.name << " stopped profiler\n"; - } - else if (m->cmd.size() > 1 && m->cmd[0] == "debug") { + } else if (m->cmd.size() == 1 && m->cmd[0] == "release_heap") { + if (ceph_using_tcmalloc()) { + ceph_heap_release_free_memory(); + clog.info() << g_conf.name << " releasing free RAM back to system.\n"; + } else { + clog.warn() << "can't release RAM: not using tcmalloc!"; + } + } else if (m->cmd.size() > 1 && m->cmd[0] == "debug") { if (m->cmd.size() == 3 && m->cmd[1] == "dump_missing") { const string &file_name(m->cmd[2]); std::ofstream fout(file_name.c_str()); diff --git a/src/perfglue/disabled_heap_profiler.cc b/src/perfglue/disabled_heap_profiler.cc new file mode 100644 index 0000000000000..7f4d890025af5 --- /dev/null +++ b/src/perfglue/disabled_heap_profiler.cc @@ -0,0 +1,30 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2011 New Dream Network/Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ +#include "heap_profiler.h" + +bool ceph_using_tcmalloc() { return false; } + +void ceph_heap_profiler_init() { return; } + +void ceph_heap_profiler_stats(char *buf, int length) { return; } + +void ceph_heap_release_free_memory() { return; } + +bool ceph_heap_profiler_running() { return false; } + +void ceph_heap_profiler_start() { return; } + +void ceph_heap_profiler_stop() { return; } + +void ceph_heap_profiler_dump(const char *reason) { return; } diff --git a/src/perfglue/heap_profiler.cc b/src/perfglue/heap_profiler.cc new file mode 100644 index 0000000000000..6e064f74f5ab2 --- /dev/null +++ b/src/perfglue/heap_profiler.cc @@ -0,0 +1,75 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2011 New Dream Network/Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include +#include +#include "heap_profiler.h" + +bool ceph_using_tcmalloc() +{ + return true; +} + +void ceph_heap_profiler_init() +{ + char profile_name[PATH_MAX]; + sprintf(profile_name, "%s/%s", g_conf.log_dir, g_conf.name); + char *val = new char[sizeof(int)*8+1]; + sprintf(val, "%i", g_conf.profiler_allocation_interval); + setenv("HEAP_PROFILE_ALLOCATION_INTERVAL", val, g_conf.profiler_allocation_interval); + sprintf(val, "%i", g_conf.profiler_highwater_interval); + setenv("HEAP_PROFILE_INUSE_INTERVAL", val, g_conf.profiler_highwater_interval); + if (g_conf.tcmalloc_profiler_run) { + generic_dout(0) << "turning on heap profiler with prefix " << profile_name << dendl; + HeapProfilerStart(profile_name); + } +} + +void ceph_heap_profiler_stats(char *buf, int length) +{ + MallocExtension::instance()->GetStats(buf, length); +} + +void ceph_heap_release_free_memory() +{ + MallocExtension::instance()->ReleaseFreeMemory(); +} + +bool ceph_heap_profiler_running() +{ + return IsHeapProfilerRunning(); +} + +void ceph_heap_profiler_start() +{ + char profile_name[PATH_MAX]; + sprintf(profile_name, "%s/%s", g_conf.log_dir, g_conf.name); + char *val = new char[sizeof(int)*8+1]; + sprintf(val, "%i", g_conf.profiler_allocation_interval); + setenv("HEAP_PROFILE_ALLOCATION_INTERVAL", val, g_conf.profiler_allocation_interval); + sprintf(val, "%i", g_conf.profiler_highwater_interval); + setenv("HEAP_PROFILE_INUSE_INTERVAL", val, g_conf.profiler_highwater_interval); + generic_dout(0) << "turning on heap profiler with prefix " << profile_name << dendl; + HeapProfilerStart(profile_name); +} + +void ceph_heap_profiler_stop() +{ + HeapProfilerStop(); +} + +void ceph_heap_profiler_dump(const char *reason) +{ + HeapProfilerDump(reason); +} diff --git a/src/perfglue/heap_profiler.h b/src/perfglue/heap_profiler.h new file mode 100644 index 0000000000000..4eaabc1349939 --- /dev/null +++ b/src/perfglue/heap_profiler.h @@ -0,0 +1,44 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2011 New Dream Network/Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ +#ifndef HEAP_PROFILER_H_ +#define HEAP_PROFILER_H_ + +#include "common/config.h" + +/* + * Ceph glue for the Google perftools heap profiler, included + * as part of tcmalloc. This replaces ugly function pointers + * and #ifdef hacks! + */ +bool ceph_using_tcmalloc(); + +/* + * Configure the heap profiler + */ +void ceph_heap_profiler_init(); + +void ceph_heap_profiler_stats(char *buf, int length); + +void ceph_heap_release_free_memory(); + +bool ceph_heap_profiler_running(); + +void ceph_heap_profiler_start(); + +void ceph_heap_profiler_stop(); + +void ceph_heap_profiler_dump(const char *reason); + + + +#endif /* HEAP_PROFILER_H_ */ -- 2.39.5