52mb/sec inbound traffic
Hadoop stream is showing: 25k msg per second.. i do not know how accurate
this is.
$OptimizeForUniprocessor off
$MaxMessageSize 2048k
# Rsyslog plugins
$ModLoad immark # provides --MARK-- message capability
$ModLoad imudp # provides UDP syslog reception
$ModLoad imtcp # provides TCP syslog reception
$ModLoad imuxsock # provides support for local system logging (e.g.
via logger command)
$ModLoad imklog # provides kernel logging support (previously done
by rklogd)
$ModLoad imrelp # Provides RELP syslog reception
$ModLoad omrelp # Provides RELP syslog transmission
# Rsyslog Stats
$ModLoad impstats
$PStatInterval 60
$PStatSeverity 7
# Queue configuration
$ActionQueueSize 2000000
$MainMsgQueueSize 40000000
# File Creation Permissions
$umask 0000
$DirCreateMode 0755
$FileCreateMode 0644
# Remote Log Processing Ruleset
$PreserveFQDN on
$template
appLogDynFile,"/log/app-logs/%programname:R,ERE,0,DFLT:[A-Za-z0-9]+--end%/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/%PROGRAMNAME%.log"
$template
currLogStatsDynFile,"/log/app-logs/logstats/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/logstats.log.%$HOUR%00"
$template
currAppLogDynFile,"/log/app-logs/%msg:R,ERE,1,DFLT:^([A-Za-z0-9._-]+)\|([A-Za-z0-9._-]+)\|([A-Za-z0-9._]+)[-_]*([A-Za-z0-9]*)([\^])--end%/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/%msg:R,ERE,1,DFLT:^([A-Za-z0-9._-]+)\|([A-Za-z0-9._-]+)\|([A-Za-z0-9._]+)[-_]*([A-Za-z0-9]*)([\^])--end%-%msg:R,ERE,2,DFLT:^([A-Za-z0-9._-]+)\|([A-Za-z0-9._-]+)\|([A-Za-z0-9._]+)[-_]*([A-Za-z0-9]*)([\^])--end%-%msg:R,ERE,3,DFLT:^([A-Za-z0-9._-]+)\|([A-Za-z0-9._-]+)\|([A-Za-z0-9._]+)[-_]*([A-Za-z0-9]*)([\^])--end%-%msg:R,ERE,4,DFLT:^([A-Za-z0-9._-]+)\|([A-Za-z0-9._-]+)\|([A-Za-z0-9._]+)[-_]*([A-Za-z0-9]*)([\^])--end%.log.%$HOUR%00"
$template
currAppLoggTemplate,"%msg:R,ERE,1,DFLT:^[A-Za-z0-9._-]+\|[A-Za-z0-9._-]+\|[A-Za-z0-9._]+[-_]*[A-Za-z0-9]*[\^](.*)--end%\n"
$template currAppLoggTemplate2,"%msg%\n"
$template currentappLogHadoopTemplate,"<%PRI%>%TIMESTAMP:date-rfc3164%
%FROMHOST% %msg%\n"
$template currentappLogNewHadoopTemplate,"<%PRI%>%TIMESTAMP% %FROMHOST%
app=%msg:R,ERE,1,DFLT:^([A-Za-z0-9._-]+)\|([A-Za-z0-9._-]+)\|([A-Za-z0-9.]+)[-_]*([A-Za-z0-9]*)--end%|bucket=%msg:R,ERE,4,DFLT:^([A-Za-z0-9._-]+)\|([A-Za-z0-9._-]+)\|([A-Za-z0-9.]+)[-_]*([A-Za-z0-9]*)--end%
%msg%\n"
$template appLogHadoopTemplate,"<%PRI%>%TIMESTAMP% %FROMHOST%
app=%programname:R,ERE,1,DFLT:([A-Za-z0-9]+)-.*-.*_.*--end%|bucket=%programname:R,ERE,1,DFLT:.*-.*-.*_([A-Za-z0-9]+)--end%%msg%\n"
$template
remoteMessagesDynFile,"/log/system-logs/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/messages"
$template
remoteSecureDynFile,"/log/secure-system-logs/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/secure"
$template
remoteMaillogDynFile,"/log/system-logs/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/maillog"
$template
remoteEmergDynFile,"/log/system-logs/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/emergency"
$template
remoteCronDynFile,"/log/system-logs/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/cron"
$template
remoteSpoolerDynFile,"/log/system-logs/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/spooler"
$template
remoteBootDynFile,"/log/system-logs/%FROMHOST%/%$YEAR%/%$MONTH%/%$DAY%/boot.log"
$Ruleset appLog
*.*
?appLogDynFile;appLogHadoopTemplate
# Forward to Hadoop
#*.* @@
wmhdcollector01s.stag.timstesting.net:5003;
$Ruleset currAppLog
*.*
?currAppLogDynFile;currAppLoggTemplate
# Forward to Hadoop
*.*
@@hadoopcollectors.prod.timstesting.net:5003;currentappLogHadoopTemplate
$Ruleset currLogStats
*.* ?currLogStatsDynFile
# Forward to Hadoop
#*.*
@@hadoopcollectors.prod.timstesting.net:5003;currentappLogHadoopTemplate
# Remote System Log Processing Ruleset
$Ruleset remoteSysLogs
# Log all kernel messages to the console.
# Logging much else clutters up the screen.
#kern.* /dev/console
# Log anything (except mail) of level info or higher.
# Don't log private authentication messages!
$DirCreateMode 0755
$FileCreateMode 0644
*.info;local1.none;local6.none;mail.none;authpriv.none;cron.none
?remoteMessagesDynFile
# The authpriv file has restricted access.
$DirCreateMode 0700
$FileCreateMode 0600
authpriv.*
?remoteSecureDynFile
# Log all the mail messages in one place.
$DirCreateMode 0755
$FileCreateMode 0644
mail.*
?remoteMaillogDynFile
# Log cron stuff
$DirCreateMode 0755
$FileCreateMode 0644
cron.* ?remoteCronDynFile
# Everybody gets emergency messages
$DirCreateMode 0755
$FileCreateMode 0644
*.emerg ?remoteEmergDynFile
# Save news errors of level crit and higher in a special file.
$DirCreateMode 0755
$FileCreateMode 0644
uucp,news.crit
?remoteSpoolerDynFile
# Save boot messages also to boot.log
$DirCreateMode 0755
$FileCreateMode 0644
local7.* ?remoteBootDynFile
# Local Log Processing Ruleset
$Ruleset local
# Log all kernel messages to the console.
# Logging much else clutters up the screen.
#kern.* /dev/console
# Log anything (except mail) of level info or higher.
# Don't log private authentication messages!
*.info;local1.none;local6.none;mail.none;authpriv.none;cron.none
/var/log/messages
syslog.=debug
/log/rsyslog-stats
# The authpriv file has restricted access.
authpriv.* /var/log/secure
# Log all the mail messages in one place.
mail.* -/var/log/maillog
# Log cron stuff
cron.* /var/log/cron
# Everybody gets emergency messages
*.emerg :omusrmsg:*
# Save news errors of level crit and higher in a special file.
uucp,news.crit /var/log/spooler
# Save boot messages also to boot.log
local7.* /var/log/boot.log
# Assign default Ruleset
$DefaultRuleset local
# New AppLog Process RELP Collector
$InputRELPServerBindRuleset appLog
$InputRELPServerRun 20514
# Current AppLog TCP Collector
$InputTCPServerBindRuleset currAppLog
$InputTCPServerRun 20516
# Current LogStats TCP Collector
$InputTCPServerBindRuleset currLogStats
$InputTCPServerRun 20518
# SystemLog TCP Collector
$InputTCPServerBindRuleset remoteSysLogs
$InputTCPServerRun 20515
# SystemLog UDP Collector
$InputUDPServerBindRuleset remoteSysLogs
$UDPServerRun 514
