qa/standalone/mon/mon_cluster_log.sh: retry check for log line

Issue: The test was failing as we were checking for the osd boot
log before it was actually emitted in the log file.

Solution: We retry checking for the desired string in the log file
for a duration of 60s after OSD has come up successfully.

Fixes: https://tracker.ceph.com/issues/67282
Signed-off-by: Shraddha Agrawal <shraddha.agrawal000@gmail.com>
Signed-off-by: Naveen Naidu <naveennaidu479@gmail.com>
(cherry picked from commit 67928a27357e2d600114db1891db5e7b30c8d1a9)

diff --git a/qa/standalone/mon/mon-cluster-log.sh b/qa/standalone/mon/mon-cluster-log.sh
index 6e969596b203744dc53c3ab0037dcbef522ab4e1..710ca97f282d5c4136907b2742a037d03057ac98 100755 (executable)
--- a/qa/standalone/mon/mon-cluster-log.sh
+++ b/qa/standalone/mon/mon-cluster-log.sh
@@ -61,7 +61,7 @@ function TEST_cluster_log_level() {
 
     ceph osd down 0
     TIMEOUT=20 wait_for_osd up 0 || return 1
-    grep -q "cluster [[]INF[]] osd.0.*boot" $dir/log
+    TIMEOUT=60 wait_for_string $dir/log "cluster [[]INF[]] osd.0.*boot" 
     return_code=$?
     if [ $return_code -ne 0 ]; then
       echo "Failed : Could not find INF log in the cluster log file"
@@ -145,9 +145,17 @@ function TEST_journald_cluster_log_level() {
     ceph osd down 0
     TIMEOUT=20 wait_for_osd up 0 || return 1
     search_str="osd.0.*boot"
-    journalctl _COMM=ceph-mon CEPH_CHANNEL=cluster PRIORITY=6 --output=json-pretty --since "60 seconds ago" |jq '.MESSAGE' > $dir/journal.log
-    grep -q "$search_str" $dir/journal.log
-    return_code=$?
+    return_code=1
+    RETRY_DURATION=60 
+    for ((i=0; i < $RETRY_DURATION; i++)); do
+      sudo journalctl _COMM=ceph-mon CEPH_CHANNEL=cluster PRIORITY=6 --output=json-pretty --since "60 seconds ago" |jq '.MESSAGE' > $dir/journal.log
+      if ! grep "$search_str" $dir/journal.log; then
+        sleep 1
+      else
+        return_code=0
+        break
+      fi
+    done
     if [ $return_code -ne 0 ]; then
       echo "Failed : Could not find INF log in the journalctl log file"
       ERRORS=$(($ERRORS + 1))