From: Sage Weil <sage@redhat.com>
Date: Sun, 12 Aug 2018 18:18:09 +0000 (-0500)
Subject: mon/AuthMonitor: raise health warning on invalid caps
X-Git-Tag: v14.0.1~431^2~14
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=35820f4b88436b294e4aacecebaf1652099115e8;p=ceph.git

mon/AuthMonitor: raise health warning on invalid caps

Raise a health warning if we have invalid (unparsable) caps in the auth
database.  Include a simple test.

Signed-off-by: Sage Weil <sage@redhat.com>
---

diff --git a/qa/suites/rados/monthrash/workloads/rados_mon_workunits.yaml b/qa/suites/rados/monthrash/workloads/rados_mon_workunits.yaml
index b05eb38be358..63b88c0dda9d 100644
--- a/qa/suites/rados/monthrash/workloads/rados_mon_workunits.yaml
+++ b/qa/suites/rados/monthrash/workloads/rados_mon_workunits.yaml
@@ -5,6 +5,7 @@ overrides:
     - overall HEALTH_
     - \(PG_
     - \(MON_DOWN\)
+    - \(AUTH_BAD_CAPS\)
 tasks:
 - workunit:
     clients:
diff --git a/qa/suites/rados/singleton-bluestore/all/cephtool.yaml b/qa/suites/rados/singleton-bluestore/all/cephtool.yaml
index 7f7da99af03b..0567b603d1d6 100644
--- a/qa/suites/rados/singleton-bluestore/all/cephtool.yaml
+++ b/qa/suites/rados/singleton-bluestore/all/cephtool.yaml
@@ -36,6 +36,7 @@ tasks:
     - \(SMALLER_PGP_NUM\)
     - \(POOL_NEARFULL\)
     - \(POOL_APP_NOT_ENABLED\)
+    - \(AUTH_BAD_CAPS\)
 - workunit:
     clients:
       all:
diff --git a/qa/suites/rados/singleton/all/mon-auth-caps.yaml b/qa/suites/rados/singleton/all/mon-auth-caps.yaml
index 318af5ee6642..ae4a5d2e3a07 100644
--- a/qa/suites/rados/singleton/all/mon-auth-caps.yaml
+++ b/qa/suites/rados/singleton/all/mon-auth-caps.yaml
@@ -8,6 +8,9 @@ roles:
 tasks:
 - install:
 - ceph:
+    log-whitelist:
+    - overall HEALTH_
+    - \(AUTH_BAD_CAPS\)
 - workunit:
     clients:
       all:
diff --git a/qa/workunits/mon/caps.sh b/qa/workunits/mon/caps.sh
index 3951e9976c33..5d4c130e4ecb 100755
--- a/qa/workunits/mon/caps.sh
+++ b/qa/workunits/mon/caps.sh
@@ -66,4 +66,16 @@ ceph auth del client.bar
 
 rm $tmp.bazar.keyring $tmp.foo.keyring $tmp.bar.keyring
 
+# invalid caps health warning
+cat <<EOF | ceph auth import -i -
+[client.bad]
+  caps mon = this is wrong
+  caps osd = does not parse
+  caps mds = also does not parse
+EOF
+ceph health | grep AUTH_BAD_CAP
+ceph health detail | grep client.bad
+ceph auth rm client.bad
+expect "ceph auth health | grep AUTH_BAD_CAP" 1
+
 echo OK
diff --git a/src/mon/AuthMonitor.cc b/src/mon/AuthMonitor.cc
index 60d0bf5e1934..7611152095ce 100644
--- a/src/mon/AuthMonitor.cc
+++ b/src/mon/AuthMonitor.cc
@@ -198,6 +198,8 @@ void AuthMonitor::create_initial()
 void AuthMonitor::update_from_paxos(bool *need_bootstrap)
 {
   dout(10) << __func__ << dendl;
+  load_health();
+
   version_t version = get_last_committed();
   version_t keys_ver = mon->key_server.get_ver();
   if (version == keys_ver)
@@ -322,6 +324,52 @@ void AuthMonitor::encode_pending(MonitorDBStore::TransactionRef t)
   version_t version = get_last_committed() + 1;
   put_version(t, version, bl);
   put_last_committed(t, version);
+
+  // health
+  health_check_map_t next;
+  map<string,list<string>> bad_detail;  // entity -> details
+  for (auto i = mon->key_server.secrets_begin();
+       i != mon->key_server.secrets_end();
+       ++i) {
+    for (auto& p : i->second.caps) {
+      ostringstream ss;
+      if (!valid_caps(p.first, p.second, &ss)) {
+	ostringstream ss2;
+	ss2 << i->first << " " << ss.str();
+	bad_detail[i->first.to_str()].push_back(ss2.str());
+      }
+    }
+  }
+  for (auto& inc : pending_auth) {
+    if (inc.inc_type == AUTH_DATA) {
+      KeyServerData::Incremental auth_inc;
+      auto iter = inc.auth_data.cbegin();
+      decode(auth_inc, iter);
+      if (auth_inc.op == KeyServerData::AUTH_INC_DEL) {
+	bad_detail.erase(auth_inc.name.to_str());
+      } else if (auth_inc.op == KeyServerData::AUTH_INC_ADD) {
+	for (auto& p : auth_inc.auth.caps) {
+	  ostringstream ss;
+	  if (!valid_caps(p.first, p.second, &ss)) {
+	    ostringstream ss2;
+	    ss2 << auth_inc.name << " " << ss.str();
+	    bad_detail[auth_inc.name.to_str()].push_back(ss2.str());
+	  }
+	}
+      }
+    }
+  }
+  if (bad_detail.size()) {
+    ostringstream ss;
+    ss << bad_detail.size() << " auth entities have invalid capabilities";
+    health_check_t *check = &next.add("AUTH_BAD_CAPS", HEALTH_ERR, ss.str());
+    for (auto& i : bad_detail) {
+      for (auto& j : i.second) {
+	check->detail.push_back(j);
+      }
+    }
+  }
+  encode_health(next, t);
 }
 
 void AuthMonitor::encode_full(MonitorDBStore::TransactionRef t)
diff --git a/src/mon/AuthMonitor.h b/src/mon/AuthMonitor.h
index 4b6812428b6e..89c548cf079a 100644
--- a/src/mon/AuthMonitor.h
+++ b/src/mon/AuthMonitor.h
@@ -121,6 +121,17 @@ private:
 
   /* validate mon/osd/mds caps; fail on unrecognized service/type */
   bool valid_caps(const string& type, const string& caps, ostream *out);
+  bool valid_caps(const string& type, const bufferlist& bl, ostream *out) {
+    auto p = bl.begin();
+    string v;
+    try {
+      decode(v, p);
+    } catch (buffer::error& e) {
+      *out << "corrupt capability encoding";
+      return false;
+    }
+    return valid_caps(type, v, out);
+  }
   bool valid_caps(const vector<string>& caps, ostream *out);
 
   void on_active() override;