Addressing this issue was a little harder than actually expected and
resulted in a major refactoring of the daily cron job code. Can you
please try with the attached new cron script which I have commited to
svn?
Greetings
Marc
--
-----------------------------------------------------------------------------
Marc Haber | "I don't trust Computers. They | Mailadresse im Header
Mannheim, Germany | lose things." Winona Ryder | Fon: *49 621 72739834
Nordisch by Nature | How to make an American Quilt | Fax: *49 621 72739835
#!/bin/bash
set -e
set -C
# trap handler
traphandler() {
trap - INT ERR
if [ -n "${LOCKED:-}" ]; then
# we have the lock,
pidof aide | xargs --no-run-if-empty kill -9
fi
onexit signal $1
return 0
}
trap ' traphandler INT; trap - INT ERR' INT
trap ' traphandler ERR; trap - INT ERR' ERR
# bail if no aide binary found
[ -f "/usr/bin/aide" ] || exit 0
# default variables
PATH="/sbin:/usr/sbin:/bin:/usr/bin"
LOGDIR="/var/log/aide"
LOGFILE="$LOGDIR/aide.log"
CONFFILE="/var/lib/aide/aide.conf.autogenerated"
PREFIX="aide"
TMPBASE="/var/run/aide"
LOCKFILE="$TMPBASE/cron.daily.lock"
TMPDIRIN="$TMPBASE/cron.daily"
AIDEARGS="-V4"
FQDN="$(hostname -f)"
MAILSUBJ="Daily AIDE report for $FQDN"
DATE="$(date +"%Y-%m-%d %H:%M")"
# have /etc/default/aide override variables
if [ -f "/etc/default/aide" ]; then
. "/etc/default/aide"
fi
# from here on, we're going to bail on unbound variables
set -u
# umask
umask 077
# grep aide configuration data from aide config
DATABASE="$(< "$CONFFILE" grep "^database=file:/" | head -n 1 | cut
--delimiter=: --fields=2)"
DATABASE_OUT="$(< "$CONFFILE" grep "^database_out=file:/" | head -n 1 | cut
--delimiter=: --fields=2)"
# default values
MAILTO="${MAILTO:-root}"
eval MAILTO="$MAILTO"
DATABASE="${DATABASE:-/var/lib/aide/aide.db}"
LINES="${LINES:-1000}"
COMMAND="${COMMAND:-check}"
COPYNEWDB="${COPYNEWDB:-no}"
QUIETREPORTS="${QUIETREPORTS:-no}"
ONEXIT=""
# functions
mytempfile() {
NAME="$1"
echo "$TMPDIR/$NAME"
touch "$TMPDIR/$NAME"
}
frame() {
WIDTH=78
STARS="*******************************************************************************"
SPACES="
"
printf "%s\n" "${STARS:1:$WIDTH}"
while read line ; do
HALF="${SPACES:1:$((($WIDTH-${#line})/2))}"
LINE="$HALF$line$SPACES"
printf "*%s*\n" "${LINE:1:$(($WIDTH-2))}"
done
printf "%s\n" "${STARS:1:$WIDTH}"
}
onexit() {
if [ "$ONEXIT" = "running" ]; then
return 1
fi
ONEXIT="running"
local LOGHEAD
local MAILHEAD
case "$1" in
signal)
LOGHEAD="$(printf "terminated with signal %s" "$2")"
MAILHEAD="$(printf "The cron job was terminated with signal %s"
"$2")"
;;
fatal)
LOGHEAD="$(printf "terminated by fatal error.")"
MAILHEAD="$(printf "The cron job was terminated by a fatal
error.")"
;;
nolock)
LOGHEAD="$(printf "terminated because lock %s could not be
obtaiend." "$LOCKFILE")"
MAILHEAD="$(printf "The cron job was terminated because lock %s
could not be obtained." "$LOCKFILE")"
;;
cantmovetmp)
LOGHEAD="$(printf "terminated: Cannot move away %s."
"$TMPDIRIN")"
MAILHEAD="$(printf "The cron job was terminated: Cannot move
away %s." "$TMPDIRIN")"
;;
cantcreatetmp)
LOGHEAD="$(printf "terminated: Cannot create temporary
directory %s." "$TMPDIRIN")"
MAILHEAD="$(printf "The cron job was terminated: Cannot create
temporary directory %s." "$TMPDIRIN")"
;;
success)
;;
*)
LOGHEAD="$(printf "wrong parameter (\"%s\") to onexit." "$1")"
MAILHEAD="$(printf "The cron job was terminated for unknown
reasons, and a wrong parameter (\"%s\")was given to onexit." "$1")"
;;
esac
if [ -z "${TMPDIR:-}" ]; then
# we are being called so early that no TMPDIR exists yet
# LOGHEAD goes to syslog instead of LOGFILE since we do not know
# what's up with LOGFILE
logger -t aide-cron-daily "$LOGHEAD"
echo "$MAILHEAD" | /usr/bin/mail -s "premature termination - $MAILSUBJ"
"$MAILTO"
else
# we are being called after the cron job was properly set up.
# To the full works.
[ -f "$LOGFILE" ] && savelog -t -g adm -m 640 -u root -c 7 "$LOGFILE" >
/dev/null
printf >> "$MAILFILE" \
"This is an automated report generated by the Advanced Intrusion Detection
Environment on %s started at %s.\n\n" "$FQDN" "$BEGINSTAMP"
printf >> "$LOGFILE" \
"aide run on %s started at %s.\n" "$FQDN" "$BEGINSTAMP"
if [ -n "$LOGHEAD" ]; then
printf "$LOGHEAD\n" | frame >> "$LOGFILE"
printf "\n" >> "$LOGFILE"
fi
if [ -n "$MAILHEAD" ]; then
printf "$MAILHEAD\n" | frame >> "$MAILFILE"
printf "\n\n" >> "$MAILFILE"
fi
# script errors
if [ -n "${ERRORLOG:-}" ] && [ -s "$ERRORLOG" ]; then
printf "script errors\n" | frame >> "$MAILFILE"
< "$ERRORLOG" cat >> "$MAILFILE"
printf "End of script errors\n\n" >> "$MAILFILE"
printf "script errors\n" | frame >> "$LOGFILE"
< "$ERRORLOG" cat >> "$LOGFILE"
printf "End of script errors\n" >> "$LOGFILE"
fi
# aide post run information
if [ -n "${POSTRUNLOG:-}" ] && [ -s "$POSTRUNLOG" ]; then
printf "AIDE post run information\n" >> "$MAILFILE"
< "$POSTRUNLOG" cat >> "$MAILFILE"
printf "End of AIDE post run information\n\n" >> "$MAILFILE"
printf "AIDE post run information\n" >> "$LOGFILE"
< "$POSTRUNLOG" cat >> "$LOGFILE"
printf "End of AIDE post run information\n" >> "$LOGFILE"
fi
# include error log in daily report e-mail
if [ -n "${ARETVAL:-}" ] && [ "$ARETVAL" != "0" ]; then
printf "AIDE returned a non-zero exit value\nexit value is %d\n\n"
"$ARETVAL" | frame >> "$MAILFILE"
printf "AIDE returned a non-zero exit value\nexit value is %d\n"
"$ARETVAL" | frame >> "$LOGFILE"
fi
if [ -n "${AERRLOG:-}" ] && [ -s "$AERRLOG" ]; then
errorlines="$(wc -l "$AERRLOG" | awk '{ print $1 }')"
if [ "${errorlines:=0}" -gt "$LINES" ]; then
printf "AIDE has returned many errors.\nthe error log output
has been truncated in this mail\n" | \
frame >> "$MAILFILE"
printf >> "$MAILFILE" "Error output is %d lines, truncated to
%d.\n" "$errorlines" "$LINES"
< "$AERRLOG" head -n "$LINES" >> "$MAILFILE"
printf >> "$MAILFILE" "\nEnd of truncated AIDE error output.
The full output can be found in %s.\n\n" "$LOGFILE"
else
printf >> "$MAILFILE" "Errors produced (%d lines):\n"
"$errorlines"
< "$AERRLOG" cat >> "$MAILFILE"
printf >> "$MAILFILE" "\nEnd of AIDE error output.\n\n"
fi
printf >> "$LOGFILE" "AIDE error output (%d lines):\n" "$errorlines"
< "$AERRLOG" cat >> "$LOGFILE"
printf >> "$LOGFILE" "End of AIDE error output\n"
else
printf >> "$MAILFILE" "AIDE produced no errors.\n\n"
printf >> "$LOGFILE" "AIDE produced no errors.\n"
fi
# include de-noised log into mail
if [ -n "${NOISE:-}" ]; then
NOISETMP="$(tempfile --directory "/tmp" --prefix "aidenoise")"
NOISETMP2="$(tempfile --directory "/tmp" --prefix "aidenoise")"
< "$ARUNLOG" sed -n '1,/^Detailed information about changes:/p' | \
grep '^\(changed\|removed\|added\):' | \
grep -v "^added: THERE WERE ALSO [0-9]\+ FILES ADDED UNDER THIS
DIRECTORY" > "$NOISETMP2"
if [ -n "$NOISE" ]; then
< "$NOISETMP2" grep -v "^\(changed\|removed\|added\):$NOISE" >
"$NOISETMP"
printf >> "$MAILFILE" "De-Noised output removes everything
matching %s.\n" "$NOISE"
fi
if [ -s "$NOISETMP" ]; then
loglines="$(< $NOISETMP wc -l | awk '{ print $1 }')"
if [ "${loglines:=0}" -gt "$LINES" ]; then
printf "AIDE has returned long output which has been
truncated in this mail\n" | \
frame >> "$MAILFILE"
printf >> "$MAILFILE" \
"De-Noised output is %d lines, truncated to %d.\n"
"$loglines" "$LINES"
< "$NOISETMP" head -n "$LINES" >> "$MAILFILE"
printf >> "$MAILFILE" "\nEnd of truncated De-Noised
AIDE output. The full output can be found in %s.\n\n" "$LOGFILE"
else
printf >> "$MAILFILE" "De-Noised output of the daily
AIDE run (%d lines):\n" "$loglines"
< "$NOISETMP" cat >> "$MAILFILE"
printf >> "$MAILFILE" "\nEnd of De-Noised AIDE
output.\n\n"
fi
else
printf >> "$MAILFILE" "AIDE detected no changes after removing
noise.\n\n"
fi
printf >> "$MAILFILE"
"============================================================================\n"
fi
# include non-de-noised log into mail
if [ -n "${ARUNLOG:-}" ] && [ -s "$ARUNLOG" ]; then
loglines="$(wc -l "$ARUNLOG" | awk '{ print $1 }')"
if [ "${loglines:=0}" -gt "$LINES" ]; then
printf "AIDE has returned long output which has been truncated
in this mail\n" | \
frame >> "$MAILFILE"
printf >> "$MAILFILE" \
"Output is %d lines, truncated to %d.\n" "$loglines" "$LINES"
< "$ARUNLOG" head -n "$LINES" >> "$MAILFILE"
printf >> "$MAILFILE" "\nEnd of truncated AIDE output. The full
output can be found in %s.\n\n" "$LOGFILE"
else
printf >> "$MAILFILE" "Output of the daily AIDE run (%d
lines):\n" "$loglines"
< "$ARUNLOG" cat >> "$MAILFILE"
printf >> "$MAILFILE" "\nEnd of AIDE output.\n\n"
fi
printf >> "$LOGFILE" "AIDE output (%d lines):\n" "$loglines"
< "$ARUNLOG" cat >> "$LOGFILE"
printf >> "$LOGFILE" "End of AIDE output.\n\n"
else
printf >> "$MAILFILE" "AIDE detected no changes.\n\n"
printf >> "$LOGFILE" "AIDE detected no changes.\n\n"
fi
if [ -n "${DBCHECKLOG:-}" ] && [ -s "$DBCHECKLOG" ]; then
< "$DBCHECKLOG" cat >> "$MAILFILE"
printf >> "$MAILFILE" "\n"
< "$DBCHECKLOG" cat >> "$LOGFILE"
fi
printf >> "$MAILFILE" "End of AIDE daily cron job at %s, run time %d
seconds\n" "$(date +"at %Y-%m-%d %H:%M")" "$(( $(date +%s) - $BEGINTIME ))"
printf >> "$LOGFILE" "End of AIDE daily cron job at %s, run time %d
seconds\n" "$(date +"at %Y-%m-%d %H:%M")" "$(( $(date +%s) - $BEGINTIME ))"
# send mail if changes or errors were detected or quiet reports not
requested
if [ "$QUIETREPORTS" = "no" ] || [ "$CHANGES" != "0" ] || [ $(< "$ERRORLOG"
wc -l) -ne 0 ]; then
< "$MAILFILE" /usr/bin/mail -s "$MAILSUBJ" "$MAILTO"
fi
# clean up temp files
rm -rf $TMPDIR
fi
# clear lock
if [ -n "${LOCKED:-}" ] && command -v dotlockfile >/dev/null 2>&1; then
dotlockfile -u "$LOCKFILE" || true
fi
unset LOCKED
return 0
}
BEGINTIME="$(date +%s)"
if command -v dotlockfile >/dev/null 2>&1; then
if ! dotlockfile -p -l "$LOCKFILE"; then
onexit nolock
exit 1
fi
else
PREERRLOG="no dotlockfile binary in path, not checking for already running
aide cron job\n"
fi
LOCKED=yes
# prepare temp dir
if [ -e "$TMPDIRIN" ]; then
if ! NEWNAME="$(mktemp -d $TMPBASE/cron.daily.old.XXXXXXXXXX)"; then
onexit cantmovetmp
exit 1
fi
mv "$TMPDIRIN" "$NEWNAME"
OLDTMPDIRFOUND="yes"
fi
if ! mkdir -p $TMPDIRIN; then
onexit cantcreatetmp
exit 1
fi
# we can now directly use file names inside $TMPDIR: It is only
# writeable for us (umask 077), so we're safe against symlink attacks.
# We use invariant file names here since our work files need to be
# excluded from aide.
TMPDIR="$TMPDIRIN"
# now, with $TMPDIR having been created, we can use onexit.
# LOGFILE: /var/log/aide/aide.log - all logs untruncated (not temp)
# ERRORLOG: Error messages from script. Gets written to $LOGFILE first
ERRORLOG="$(mytempfile errorlog)"
if [ -n "${PREERRORLOG:-}" ]; then
printf >> "$ERRORLOG" "$PREERRORLOG"
fi
unset PREERRORLOG
# MAILFILE: Contents gets mailed. Built and handled from inside onexit()
MAILFILE="$(mytempfile mailfile)"
ARETVAL=0
if [ ! -f "$DATABASE" ]; then
printf >> "$ERRORLOG" "Fatal error: The AIDE database does not exist!\n"
printf >> "$ERRORLOG" "This may mean you haven't created it, or it may
mean that someone has removed it.\n"
onexit fatal
exit 0
fi
# code
BEGINSTAMP="$(date +"%Y-%m-%d %H:%M:%S")"
# ARUNLOG: standard output of aide run
ARUNLOG="$(mytempfile arunlog)"
# AERRLOG: standard error of aide run
AERRLOG="$(mytempfile aerrlog)"
printf "begin timestamp %s\n" "$BEGINSTAMP" >> "$ARUNLOG"
update-aide.conf
aide.wrapper $AIDEARGS "--$COMMAND" >|"$ARUNLOG" 2>|"$AERRLOG"
ARETVAL="$?"
# POSTRUNLOG: summary of aide execution and cron job log
POSTRUNLOG="$(mytempfile postrunlog)"
# DBCHECKLOG: Output of the database checksums
DBCHECKLOG="$(mytempfile dbchecklog)"
# NOISETMP: completely de-noised log
# NOISETMP2: pre-filtered ARUNLOG, containing only changed, removed and added
lines
NOISETMP="$(mytempfile noisetmp)"
NOISETMP2="$(mytempfile noisetmp2)"
# find out checksums and other data for the database we were comparing against.
DBCHECKDB="$(mytempfile dbcheckdb)"
printf >> "$DBCHECKLOG" "The check was done against %s with the following
characteristics:\n" "$DATABASE"
DBCHECKFILE="$DBCHECKDB"
DBCHECKOLD="$DATABASE"
DBCHECKNEW="$DBCHECKDB"
printf "database=file:%s\ndatabase_out=file:%s\n%s\$
p+u+g+n+i+s+b+m+c+md5+sha1+rmd160+haval+gost+crc32+tiger" \
"${DBCHECKOLD}" "${DBCHECKNEW}" "${DBCHECKFILE}" | \
aide --config=- --init > /dev/null
sed -i "s|^${DBCHECKFILE}[[:space:]]|${DATABASE} |" "$DBCHECKDB"
DBCHECKFILE="$DATABASE"
DBCHECKOLD="$DBCHECKDB"
DBCHECKNEW="$DBCHECKDB"
printf "database=file:%s\ndatabase_out=file:%s\n%s\$
p+u+g+n+i+s+b+m+c+md5+sha1+rmd160+haval+gost+crc32+tiger" \
"${DBCHECKOLD}" "${DBCHECKNEW}" "${DBCHECKFILE}" | \
aide --config=- --check 2>/dev/null | \
sed -n '/^
\(Size\|Bcount\|Mtime\|Ctime\|Inode\|MD5\|SHA1\|RMD160\|TIGER\|CRC32\|HAVAL\|GOST\)/{s/\([^:]*\)[^,]*,[[:space:]]*\(.*\)/\1:
\2/;p;}' \
>> "$DBCHECKLOG"
if [ "$COMMAND" = "update" ]; then
printf >> "$DBCHECKLOG" "\nThe AIDE run created a new database %s with
the following characteristics:\n" "$DATABASE_OUT"
sed -i "s|^${DATABASE}[[:space:]]|${DATABASE_OUT} |" "$DBCHECKDB"
DBCHECKFILE="$DATABASE_OUT"
DBCHECKOLD="$DBCHECKDB"
DBCHECKNEW="$DBCHECKDB"
printf "database=file:%s\ndatabase_out=file:%s\n%s\$
p+u+g+n+i+s+b+m+c+md5+sha1+rmd160+haval+gost+crc32+tiger" \
"${DBCHECKOLD}" "${DBCHECKNEW}" "${DBCHECKFILE}" | \
aide --config=- --check 2>/dev/null | \
sed -n '/^
\(Size\|Bcount\|Mtime\|Ctime\|Inode\|MD5\|SHA1\|RMD160\|TIGER\|CRC32\|HAVAL\|GOST\)/{s/\([^:]*\)[^,]*,[[:space:]]*\(.*\)/\1:
\2/;p;}' \
>> "$DBCHECKLOG"
fi
rm -f "$DBCHECKDB"
# find out whether we neeed to copy the new database over the old one
COPYDB="0"
CHANGES="1"
if < "$ARUNLOG" grep '^###' | head -n 1 | grep -q '### All files match AIDE
database. Looks okay!' && \
[ "$(< $ARUNLOG wc -l)" -lt 20 ]; then
CHANGES="0"
fi
if [ "$COPYNEWDB" = "ifnochange" ] && [ "$CHANGES" = "0" ]; then
COPYDB="1"
fi
if [ "$COPYNEWDB" = "yes" ]; then
COPYDB=1
fi
if [ "$COPYDB" = "1" ] && [ "$COMMAND" = "update" ]; then
cp -f "$DATABASE_OUT" "$DATABASE"
printf >> "$POSTRUNLOG" "no significant changes detected.\n"
printf >> "$POSTRUNLOG" "output database %s was copied to %s as
requested by cron job configuration\n" "$DATABASE_OUT" "$DATABASE"
fi
onexit success
# end of file
