From 2cc9b5469ec90dcf3811fdc5817cc587e755ac5f Mon Sep 17 00:00:00 2001 From: Yuri Weinstein Date: Thu, 26 May 2016 13:22:57 -0700 Subject: [PATCH] Added file locking logic to avoid possible collisions when appending to the log file Fixes http://tracker.ceph.com/issues/15563 Signed-off-by: Yuri Weinstein Signed-off-by: Yuri Weinstein --- nightlies/cron_wrapper | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/nightlies/cron_wrapper b/nightlies/cron_wrapper index d2f6ca312e339..7a197544f9499 100755 --- a/nightlies/cron_wrapper +++ b/nightlies/cron_wrapper @@ -8,7 +8,10 @@ if [ -z $1 ]; then fi # set permanent $LOG file var +# set per-job $LOG_XX file +# set $LOG_LOCKED_ERR in case locking failed LOG="/var/log/crontab-nightlies-log/crontab.log" +LOG_LOCK_ERR="/var/log/crontab-nightlies-log/crontab_lock_problem.$$" # temp files to store sdtout and stderr # named with the PID of this script in their name so they'll be unique @@ -19,7 +22,7 @@ STDOUT="/var/tmp/stdout.$$" trap "rm -f $STDOUT $STDERR" 0 # log a name of the command about to be run -echo "Running command: $@" >> $LOG +echo "Running command: $@" >> $STDOUT # run a command from this script's argument # redirect stdout to $STDOUT file and redirect stderr to $STDERR file @@ -37,6 +40,16 @@ if [ $code != 0 ] ; then else # normal exit: just log stdout DATE=$(date) - echo -n "$DATE: " >> $LOG - cat $STDOUT >> $LOG + echo -n "$DATE: " >> $STDOUT + cat $STDOUT >> $STDOUT fi + +# lock $LOG with file descriptor 200 +# if $LOC is locked by other process - wait for 20 sec +# if lock failed, write to $LOG_LOCK_ERR, e.g. all information already in $LOG_XX +exec 200>>$LOG +flock -w 20 200 || LOG=$LOG_LOCK_ERR +# write from per-job $LOG_XX file to the commonn $LOG +cat $LOG_XX >> $LOG +# unlock +flock -u 200 -- 2.39.5