mon: Monitor: log every administrative action in an 'audit log'

Fixes: #7988
Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index bc22e4ab3b55f199e9f6df4d47bd6d5f8643ddec..6de269e8b2334e06ed9a57590aa2316b71ee7bd7 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -44,10 +44,15 @@ OPTION(err_to_syslog, OPT_BOOL, false)
 OPTION(log_flush_on_exit, OPT_BOOL, true) // default changed by common_preinit()
 OPTION(log_stop_at_utilization, OPT_FLOAT, .97)  // stop logging at (near) full
 
-OPTION(clog_to_monitors, OPT_BOOL, true)
-OPTION(clog_to_syslog, OPT_BOOL, false)
-OPTION(clog_to_syslog_level, OPT_STR, "info")         // this level and above
-OPTION(clog_to_syslog_facility, OPT_STR, "daemon")
+// options will take k/v pairs, or single-item that will be assumed as general
+// default for all, regardless of channel.
+// e.g., "info" would be taken as the same as "default=info"
+// also, "default=daemon audit=local0" would mean
+//    "default all to 'daemon', override 'audit' with 'local0'
+OPTION(clog_to_monitors, OPT_STR, "default=true")
+OPTION(clog_to_syslog, OPT_STR, "false")
+OPTION(clog_to_syslog_level, OPT_STR, "info") // this level and above
+OPTION(clog_to_syslog_facility, OPT_STR, "default=daemon audit=local0")
 
 OPTION(mon_cluster_log_to_syslog, OPT_BOOL, false)
 OPTION(mon_cluster_log_to_syslog_level, OPT_STR, "info")   // this level and above

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index ac68700b5eea95353dc3eb568b2b591a64ce7246..68486667a0ee951b2be72a6e49ef4e7c835ccc03 100644 (file)
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -65,6 +65,7 @@
 #include "include/color.h"
 #include "include/ceph_fs.h"
 #include "include/str_list.h"
+#include "include/str_map.h"
 
 #include "OSDMonitor.h"
 #include "MDSMonitor.h"
@@ -141,7 +142,7 @@ Monitor::Monitor(CephContext* cct_, string nm, MonitorDBStore *s,
   has_ever_joined(false),
   logger(NULL), cluster_logger(NULL), cluster_logger_registered(false),
   monmap(map),
-  clog(cct_, messenger, monmap, LogClient::FLAG_MON),
+  log_client(cct_, messenger, monmap, LogClient::FLAG_MON),
   key_server(cct, &keyring),
   auth_cluster_required(cct,
                        cct->_conf->auth_supported.length() ?
@@ -181,6 +182,11 @@ Monitor::Monitor(CephContext* cct_, string nm, MonitorDBStore *s,
 {
   rank = -1;
 
+  clog = log_client.create_channel(CLOG_CHANNEL_CLUSTER);
+  audit_clog = log_client.create_channel(CLOG_CHANNEL_AUDIT);
+
+  update_log_clients();
+
   paxos = new Paxos(this, "paxos");
 
   paxos_service[PAXOS_MDSMAP] = new MDSMonitor(this, paxos, "mdsmap");
@@ -397,6 +403,78 @@ void Monitor::handle_conf_change(const struct md_config_t *conf,
   sanitize_options();
 }
 
+void Monitor::update_log_client(
+    LogChannelRef lc, const string &name,
+    map<string,string> &log_to_monitors,
+    map<string,string> &log_to_syslog,
+    map<string,string> &log_channels,
+    map<string,string> &log_prios)
+{
+  bool to_monitors = (get_str_map_key(log_to_monitors, name,
+                                      &CLOG_CHANNEL_DEFAULT) == "true");
+  bool to_syslog = (get_str_map_key(log_to_syslog, name,
+                                    &CLOG_CHANNEL_DEFAULT) == "true");
+  string syslog_facility = get_str_map_key(log_channels, name,
+                                           &CLOG_CHANNEL_DEFAULT);
+  string prio = get_str_map_key(log_prios, name, &CLOG_CHANNEL_DEFAULT);
+
+  lc->set_log_to_monitors(to_monitors);
+  lc->set_log_to_syslog(to_syslog);
+  lc->set_syslog_facility(syslog_facility);
+  lc->set_log_channel(name);
+  lc->set_log_prio(prio);
+
+  dout(15) << __func__ << " " << name << "("
+           << " to_monitors: " << (to_monitors ? "true" : "false")
+           << " to_syslog: " << (to_syslog ? "true" : "false")
+           << " syslog_facility: " << syslog_facility
+           << " prio: " << prio << ")" << dendl;
+}
+
+void Monitor::update_log_clients()
+{
+  map<string,string> log_to_monitors;
+  map<string,string> log_to_syslog;
+  map<string,string> log_channel;
+  map<string,string> log_prio;
+  ostringstream oss;
+
+  int r = get_conf_str_map_helper(g_conf->clog_to_monitors, oss,
+                                  &log_to_monitors, CLOG_CHANNEL_DEFAULT);
+  if (r < 0) {
+    derr << __func__ << " error parsing 'clog_to_monitors'" << dendl;
+    return;
+  }
+
+  r = get_conf_str_map_helper(g_conf->clog_to_syslog, oss,
+                              &log_to_syslog, CLOG_CHANNEL_DEFAULT);
+  if (r < 0) {
+    derr << __func__ << " error parsing 'clog_to_syslog'" << dendl;
+    return;
+  }
+
+  r = get_conf_str_map_helper(g_conf->clog_to_syslog_facility, oss,
+                              &log_channel, CLOG_CHANNEL_DEFAULT);
+  if (r < 0) {
+    derr << __func__ << " error parsing 'clog_to_syslog_facility'" << dendl;
+    return;
+  }
+
+  r = get_conf_str_map_helper(g_conf->clog_to_syslog_level, oss,
+                              &log_prio, CLOG_CHANNEL_DEFAULT);
+  if (r < 0) {
+    derr << __func__ << " error parsing 'clog_to_syslog_level'" << dendl;
+    return;
+  }
+
+  update_log_client(clog, CLOG_CHANNEL_CLUSTER,
+                    log_to_monitors, log_to_syslog,
+                    log_channel, log_prio);
+  update_log_client(audit_clog, CLOG_CHANNEL_AUDIT,
+                    log_to_monitors, log_to_syslog,
+                    log_channel, log_prio);
+}
+
 int Monitor::sanitize_options()
 {
   int r = 0;
@@ -404,7 +482,7 @@ int Monitor::sanitize_options()
   // mon_lease must be greater than mon_lease_renewal; otherwise we
   // may incur in leases expiring before they are renewed.
   if (g_conf->mon_lease <= g_conf->mon_lease_renew_interval) {
-    clog.error() << "mon_lease (" << g_conf->mon_lease
+    clog->error() << "mon_lease (" << g_conf->mon_lease
                  << ") must be greater "
                  << "than mon_lease_renew_interval ("
                  << g_conf->mon_lease_renew_interval << ")";
@@ -417,7 +495,7 @@ int Monitor::sanitize_options()
   // the monitors happened to be overloaded -- or even under normal load for
   // a small enough value.
   if (g_conf->mon_lease_ack_timeout <= g_conf->mon_lease) {
-    clog.error() << "mon_lease_ack_timeout ("
+    clog->error() << "mon_lease_ack_timeout ("
                  << g_conf->mon_lease_ack_timeout
                  << ") must be greater than mon_lease ("
                  << g_conf->mon_lease << ")";
@@ -773,6 +851,8 @@ void Monitor::shutdown()
     cluster_logger = NULL;
   }
 
+  log_client.shutdown();
+
   // unlock before msgr shutdown...
   lock.Unlock();
 
@@ -1671,7 +1751,7 @@ void Monitor::start_election()
 
   cancel_probe_timeout();
 
-  clog.info() << "mon." << name << " calling new monitor election\n";
+  clog->info() << "mon." << name << " calling new monitor election\n";
   elector.call_election();
 }
 
@@ -1719,7 +1799,7 @@ void Monitor::win_election(epoch_t epoch, set<int>& active, uint64_t features,
   quorum_features = features;
   outside_quorum.clear();
 
-  clog.info() << "mon." << name << "@" << rank
+  clog->info() << "mon." << name << "@" << rank
                << " won leader election with quorum " << quorum << "\n";
 
   set_leader_supported_commands(cmdset, cmdsize);
@@ -2330,10 +2410,20 @@ void Monitor::handle_command(MMonCommand *m)
   if (!_allowed_command(session, module, prefix, cmdmap,
                         param_str_map, mon_cmd)) {
     dout(1) << __func__ << " access denied" << dendl;
+    audit_clog->info() << "from='" << session->inst << "' "
+                      << "entity='" << session->auth_handler->get_entity_name()
+                      << "' cmd=" << m->cmd << ":  access denied";
     reply_command(m, -EACCES, "access denied", 0);
     return;
   }
 
+  audit_clog->info() << "from='" << session->inst << "' "
+    << "entity='"
+    << (session->auth_handler ?
+        stringify(session->auth_handler->get_entity_name())
+        : "forwarded-request")
+    << "' cmd=" << m->cmd << ": dispatch";
+
   if (module == "mds" || module == "fs") {
     mdsmon()->dispatch(m);
     return;
@@ -3094,7 +3184,7 @@ void Monitor::dispatch(MonSession *s, Message *m, const bool src_is_mon)
       break;
 
     case MSG_LOGACK:
-      clog.handle_log_ack((MLogAck*)m);
+      log_client.handle_log_ack((MLogAck*)m);
       m->put();
       break;
 
@@ -3488,9 +3578,9 @@ void Monitor::handle_timecheck_leader(MTimeCheck *m)
   ostringstream ss;
   health_status_t status = timecheck_status(ss, skew_bound, latency);
   if (status == HEALTH_ERR)
-    clog.error() << other << " " << ss.str() << "\n";
+    clog->error() << other << " " << ss.str() << "\n";
   else if (status == HEALTH_WARN)
-    clog.warn() << other << " " << ss.str() << "\n";
+    clog->warn() << other << " " << ss.str() << "\n";
 
   dout(10) << __func__ << " from " << other << " ts " << m->timestamp
           << " delta " << delta << " skew_bound " << skew_bound
@@ -3759,12 +3849,12 @@ int Monitor::scrub()
   assert(is_leader());
 
   if ((get_quorum_features() & CEPH_FEATURE_MON_SCRUB) == 0) {
-    clog.warn() << "scrub not supported by entire quorum\n";
+    clog->warn() << "scrub not supported by entire quorum\n";
     return -EOPNOTSUPP;
   }
 
   if (!scrub_result.empty()) {
-    clog.info() << "scrub already in progress\n";
+    clog->info() << "scrub already in progress\n";
     return -EBUSY;
   }
 
@@ -3859,13 +3949,13 @@ void Monitor::scrub_finish()
       continue;
     if (p->second != mine) {
       ++errors;
-      clog.error() << "scrub mismatch" << "\n";
-      clog.error() << " mon." << rank << " " << mine << "\n";
-      clog.error() << " mon." << p->first << " " << p->second << "\n";
+      clog->error() << "scrub mismatch" << "\n";
+      clog->error() << " mon." << rank << " " << mine << "\n";
+      clog->error() << " mon." << p->first << " " << p->second << "\n";
     }
   }
   if (!errors)
-    clog.info() << "scrub ok on " << quorum << ": " << mine << "\n";
+    clog->info() << "scrub ok on " << quorum << ": " << mine << "\n";
 
   scrub_reset();
 }

diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index 2f079116d22fc167564d5ef05d3cecb579a2d8ad..8a523aec37f47694534ed0d39137bdcf1b1bf957 100644 (file)
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -52,6 +52,7 @@
 
 #include <memory>
 #include "include/memory.h"
+#include "include/str_map.h"
 #include <errno.h>
 
 
@@ -147,7 +148,9 @@ public:
 
   set<entity_addr_t> extra_probe_peers;
 
-  LogClient clog;
+  LogClient log_client;
+  LogChannelRef clog;
+  LogChannelRef audit_clog;
   KeyRing keyring;
   KeyServer key_server;
 
@@ -724,8 +727,31 @@ public:
     C_Command(Monitor *_mm, MMonCommand *_m, int r, string s, bufferlist rd, version_t v) :
       mon(_mm), m(_m), rc(r), rs(s), rdata(rd), version(v){}
     void finish(int r) {
-      if (r >= 0)
+      if (r >= 0) {
+        ostringstream ss;
+        if (!m->get_connection()) {
+          ss << "connection dropped for command ";
+        } else {
+          MonSession *s = m->get_session();
+
+          // if client drops we may not have a session to draw information from.
+          if (s) {
+            ss << "from='" << s->inst << "' "
+              << "entity='";
+            if (s->auth_handler)
+              ss << s->auth_handler->get_entity_name();
+            else
+              ss << "forwarded-request";
+            ss << "' ";
+          } else {
+            ss << "session dropped for command ";
+          }
+        }
+        ss << "cmd='" << m->cmd << "': finished";
+
+        mon->audit_clog->info() << ss.str();
        mon->reply_command(m, rc, rs, rdata, version);
+      }
       else if (r == -ECANCELED)
        m->put();
       else if (r == -EAGAIN)
@@ -794,6 +820,12 @@ public:
   virtual void handle_conf_change(const struct md_config_t *conf,
                                   const std::set<std::string> &changed);
 
+  void update_log_client(LogChannelRef lc, const string &name,
+                         map<string,string> &log_to_monitors,
+                         map<string,string> &log_to_syslog,
+                         map<string,string> &log_channels,
+                         map<string,string> &log_prios);
+  void update_log_clients();
   int sanitize_options();
   int preinit();
   int init();
@@ -983,4 +1015,47 @@ struct MonCommand {
 };
 WRITE_CLASS_ENCODER(MonCommand)
 
+// Having this here is less than optimal, but we needed to keep it
+// somewhere as to avoid code duplication, as it will be needed both
+// on the Monitor class and the LogMonitor class.
+//
+// We are attempting to avoid code duplication in the event that
+// changing how the mechanisms currently work will lead to unnecessary
+// issues, resulting from the need of changing this function in multiple
+// places.
+//
+// This function is just a helper to perform a task that should not be
+// needed anywhere else besides the two functions that shall call it.
+//
+// This function's only purpose is to check whether a given map has only
+// ONE key with an empty value (which would mean that 'get_str_map()' read
+// a map in the form of 'VALUE', without any KEY/VALUE pairs) and, in such
+// event, to assign said 'VALUE' to a given 'def_key', such that we end up
+// with a map of the form "m = { 'def_key' : 'VALUE' }" instead of the
+// original "m = { 'VALUE' : '' }".
+static inline int get_conf_str_map_helper(
+    const string &str,
+    ostringstream &oss,
+    map<string,string> *m,
+    const string &def_key)
+{
+  int r = get_str_map(str, m);
+
+  if (r < 0) {
+    generic_derr << __func__ << " error: " << oss.str() << dendl;
+    return r;
+  }
+
+  if (r >= 0 && m->size() == 1) {
+    map<string,string>::iterator p = m->begin();
+    if (p->second.empty()) {
+      string s = p->first;
+      m->erase(s);
+      (*m)[def_key] = s;
+    }
+  }
+  return r;
+}
+
+
 #endif