Added file locking logic to avoid possible collisions when appending to the log file

Fixes http://tracker.ceph.com/issues/15563
Signed-off-by: Yuri Weinstein <yweinste@redhat.com>
Signed-off-by: Yuri Weinstein <yweinste@redhat.com>

diff --git a/nightlies/cron_wrapper b/nightlies/cron_wrapper
index d2f6ca312e3397fa61224488716611675794286a..7a197544f9499300ec132265a51ee49e80ab588d 100755 (executable)
--- a/nightlies/cron_wrapper
+++ b/nightlies/cron_wrapper
@@ -8,7 +8,10 @@ if [ -z $1 ]; then
 fi
 
 # set permanent $LOG file var
+# set per-job $LOG_XX file
+# set $LOG_LOCKED_ERR in case locking failed
 LOG="/var/log/crontab-nightlies-log/crontab.log"
+LOG_LOCK_ERR="/var/log/crontab-nightlies-log/crontab_lock_problem.$$"
 
 # temp files to store sdtout and stderr
 # named with the PID of this script in their name so they'll be unique
@@ -19,7 +22,7 @@ STDOUT="/var/tmp/stdout.$$"
 trap  "rm -f $STDOUT $STDERR" 0
 
 # log a name of the command about to be run
-echo "Running command: $@" >> $LOG
+echo "Running command: $@" >> $STDOUT
 
 # run a command from this script's argument
 # redirect stdout to $STDOUT file and redirect stderr to $STDERR file
@@ -37,6 +40,16 @@ if [ $code != 0 ] ; then
 else
         # normal exit: just log stdout
         DATE=$(date)
-        echo -n "$DATE: " >> $LOG
-        cat $STDOUT  >> $LOG
+        echo -n "$DATE: "  >> $STDOUT
+        cat $STDOUT   >> $STDOUT
 fi
+
+# lock $LOG with file descriptor 200
+# if $LOC is locked by other process - wait for 20 sec
+# if lock failed, write to $LOG_LOCK_ERR, e.g. all information already in $LOG_XX
+exec 200>>$LOG
+flock -w 20 200 || LOG=$LOG_LOCK_ERR
+# write from per-job $LOG_XX file to the commonn $LOG
+cat $LOG_XX >> $LOG
+# unlock 
+flock -u 200