mon/AuthMonitor: raise health warning on invalid caps

Raise a health warning if we have invalid (unparsable) caps in the auth
database.  Include a simple test.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/qa/suites/rados/monthrash/workloads/rados_mon_workunits.yaml b/qa/suites/rados/monthrash/workloads/rados_mon_workunits.yaml
index b05eb38be358b23c2feed5b5d7bc907f299c568b..63b88c0dda9d6d1d84acfa1f6a4fdbd63795d711 100644 (file)
--- a/qa/suites/rados/monthrash/workloads/rados_mon_workunits.yaml
+++ b/qa/suites/rados/monthrash/workloads/rados_mon_workunits.yaml
@@ -5,6 +5,7 @@ overrides:
     - overall HEALTH_
     - \(PG_
     - \(MON_DOWN\)
+    - \(AUTH_BAD_CAPS\)
 tasks:
 - workunit:
     clients:

diff --git a/qa/suites/rados/singleton-bluestore/all/cephtool.yaml b/qa/suites/rados/singleton-bluestore/all/cephtool.yaml
index 7f7da99af03b557c323652408f8ea873b4523793..0567b603d1d647f33951bb83d5217fa5e1dd916a 100644 (file)
--- a/qa/suites/rados/singleton-bluestore/all/cephtool.yaml
+++ b/qa/suites/rados/singleton-bluestore/all/cephtool.yaml
@@ -36,6 +36,7 @@ tasks:
     - \(SMALLER_PGP_NUM\)
     - \(POOL_NEARFULL\)
     - \(POOL_APP_NOT_ENABLED\)
+    - \(AUTH_BAD_CAPS\)
 - workunit:
     clients:
       all:

diff --git a/qa/suites/rados/singleton/all/mon-auth-caps.yaml b/qa/suites/rados/singleton/all/mon-auth-caps.yaml
index 318af5ee66429bda7a2fff9a686af847e21b4fae..ae4a5d2e3a07abe64855d4c1edb1f0bc32abf57b 100644 (file)
--- a/qa/suites/rados/singleton/all/mon-auth-caps.yaml
+++ b/qa/suites/rados/singleton/all/mon-auth-caps.yaml
@@ -8,6 +8,9 @@ roles:
 tasks:
 - install:
 - ceph:
+    log-whitelist:
+    - overall HEALTH_
+    - \(AUTH_BAD_CAPS\)
 - workunit:
     clients:
       all:

diff --git a/qa/workunits/mon/caps.sh b/qa/workunits/mon/caps.sh
index 3951e9976c338101abb6ada1fd7c9d9be491fbd1..5d4c130e4ecb342566dc169f641f81974dab9141 100755 (executable)
--- a/qa/workunits/mon/caps.sh
+++ b/qa/workunits/mon/caps.sh
@@ -66,4 +66,16 @@ ceph auth del client.bar
 
 rm $tmp.bazar.keyring $tmp.foo.keyring $tmp.bar.keyring
 
+# invalid caps health warning
+cat <<EOF | ceph auth import -i -
+[client.bad]
+  caps mon = this is wrong
+  caps osd = does not parse
+  caps mds = also does not parse
+EOF
+ceph health | grep AUTH_BAD_CAP
+ceph health detail | grep client.bad
+ceph auth rm client.bad
+expect "ceph auth health | grep AUTH_BAD_CAP" 1
+
 echo OK

diff --git a/src/mon/AuthMonitor.cc b/src/mon/AuthMonitor.cc
index 60d0bf5e19345d17beb102e06a0f3b4b0d2b85b1..7611152095cebc032bc26430b4a7a94654656660 100644 (file)
--- a/src/mon/AuthMonitor.cc
+++ b/src/mon/AuthMonitor.cc
@@ -198,6 +198,8 @@ void AuthMonitor::create_initial()
 void AuthMonitor::update_from_paxos(bool *need_bootstrap)
 {
   dout(10) << __func__ << dendl;
+  load_health();
+
   version_t version = get_last_committed();
   version_t keys_ver = mon->key_server.get_ver();
   if (version == keys_ver)
@@ -322,6 +324,52 @@ void AuthMonitor::encode_pending(MonitorDBStore::TransactionRef t)
   version_t version = get_last_committed() + 1;
   put_version(t, version, bl);
   put_last_committed(t, version);
+
+  // health
+  health_check_map_t next;
+  map<string,list<string>> bad_detail;  // entity -> details
+  for (auto i = mon->key_server.secrets_begin();
+       i != mon->key_server.secrets_end();
+       ++i) {
+    for (auto& p : i->second.caps) {
+      ostringstream ss;
+      if (!valid_caps(p.first, p.second, &ss)) {
+       ostringstream ss2;
+       ss2 << i->first << " " << ss.str();
+       bad_detail[i->first.to_str()].push_back(ss2.str());
+      }
+    }
+  }
+  for (auto& inc : pending_auth) {
+    if (inc.inc_type == AUTH_DATA) {
+      KeyServerData::Incremental auth_inc;
+      auto iter = inc.auth_data.cbegin();
+      decode(auth_inc, iter);
+      if (auth_inc.op == KeyServerData::AUTH_INC_DEL) {
+       bad_detail.erase(auth_inc.name.to_str());
+      } else if (auth_inc.op == KeyServerData::AUTH_INC_ADD) {
+       for (auto& p : auth_inc.auth.caps) {
+         ostringstream ss;
+         if (!valid_caps(p.first, p.second, &ss)) {
+           ostringstream ss2;
+           ss2 << auth_inc.name << " " << ss.str();
+           bad_detail[auth_inc.name.to_str()].push_back(ss2.str());
+         }
+       }
+      }
+    }
+  }
+  if (bad_detail.size()) {
+    ostringstream ss;
+    ss << bad_detail.size() << " auth entities have invalid capabilities";
+    health_check_t *check = &next.add("AUTH_BAD_CAPS", HEALTH_ERR, ss.str());
+    for (auto& i : bad_detail) {
+      for (auto& j : i.second) {
+       check->detail.push_back(j);
+      }
+    }
+  }
+  encode_health(next, t);
 }
 
 void AuthMonitor::encode_full(MonitorDBStore::TransactionRef t)

diff --git a/src/mon/AuthMonitor.h b/src/mon/AuthMonitor.h
index 4b6812428b6e503017bc96321422fafb8f8764f8..89c548cf079a00c6f0e4bd3f35453b2702a65df0 100644 (file)
--- a/src/mon/AuthMonitor.h
+++ b/src/mon/AuthMonitor.h
@@ -121,6 +121,17 @@ private:
 
   /* validate mon/osd/mds caps; fail on unrecognized service/type */
   bool valid_caps(const string& type, const string& caps, ostream *out);
+  bool valid_caps(const string& type, const bufferlist& bl, ostream *out) {
+    auto p = bl.begin();
+    string v;
+    try {
+      decode(v, p);
+    } catch (buffer::error& e) {
+      *out << "corrupt capability encoding";
+      return false;
+    }
+    return valid_caps(type, v, out);
+  }
   bool valid_caps(const vector<string>& caps, ostream *out);
 
   void on_active() override;