log: do not repeat errors to stderr

If we get an error writing to the log, log it only once to stderr.
This avoids generating, say, 72 GB of ENOSPC errors in
teuthology.log when /var/log fills up.

Fixes: #14616
Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/log/Log.cc b/src/log/Log.cc
index 7d34bd7ef6d15fe81df14fc5de822305d4b2af72..bf46e20c7cf60fd6650d6a0fb157a5b0c50fa94f 100644 (file)
--- a/src/log/Log.cc
+++ b/src/log/Log.cc
@@ -53,6 +53,7 @@ Log::Log(SubsystemMap *s)
     m_fd(-1),
     m_uid(0),
     m_gid(0),
+    m_fd_last_error(0),
     m_syslog_log(-2), m_syslog_crash(-2),
     m_stderr_log(1), m_stderr_crash(-1),
     m_graylog_log(-3), m_graylog_crash(-3),
@@ -337,9 +338,13 @@ void Log::_flush(EntryQueue *t, EntryQueue *requeue, bool crash)
       if (do_fd) {
         buf[buflen] = '\n';
         int r = safe_write(m_fd, buf, buflen+1);
-        if (r < 0)
-          cerr << "problem writing to " << m_log_file << ": " << cpp_strerror(r)
-              << std::endl;
+       if (r != m_fd_last_error) {
+         if (r < 0)
+           cerr << "problem writing to " << m_log_file
+                << ": " << cpp_strerror(r)
+                << std::endl;
+         m_fd_last_error = r;
+       }
       }
       if (need_dynamic)
         delete[] buf;

diff --git a/src/log/Log.h b/src/log/Log.h
index 07df6e39fc5c72c1fa81dd43dc4b8fb9bcf1a4cc..687cb9b7b19817b3003e7f0ae79e8418cbb4fcb1 100644 (file)
--- a/src/log/Log.h
+++ b/src/log/Log.h
@@ -39,6 +39,8 @@ class Log : private Thread
   uid_t m_uid;
   gid_t m_gid;
 
+  int m_fd_last_error;  ///< last error we say writing to fd (if any)
+
   int m_syslog_log, m_syslog_crash;
   int m_stderr_log, m_stderr_crash;
   int m_graylog_log, m_graylog_crash;