Date: Thu, 12 May 2022 20:10:27 +0200
From: Johan Ryberg <jo...@securit.se>
To: David Lang <da...@lang.hm>
Cc: Johan Ryberg via rsyslog <rsyslog@lists.adiscon.com>,
Rainer Gerhards <rgerha...@hq.adiscon.com>
Subject: Re: [rsyslog] Ensure unixtimestamp with milliseconds?
Hi,
This is what I got from RSYSLOG_DebugFormat, sorry about newlines. I'm
using omprog and are expecting to receive a json for each message without
LF so each newline is taken from my error logs
PS, the data is anonymized
'Debug line with all properties:\n'
"FROMHOST: '172.18.0.2', fromhost-ip: '172.18.0.2', HOSTNAME: '172.18.0.2',
PRI: 15,\n"
"syslogtag '', programname: '', APP-NAME: '', PROCID: '-', MSGID: '-',\n"
"TIMESTAMP: 'May 12 17:53:30', STRUCTURED-DATA: '-',\n"
'msg: \'2022-05-12T17:53:30 Feb 23 22:09:40 laptop 1,2020/02/23
22:09:40,fcc5dd0de335,SYSTEM,general,0,2020/02/23
22:09:40,,general,,0,0,general,informational,"Connection to Update server:
updates.paloaltonetworks.com completed successfully, initiated by
172.28.125.10",139576,0x0,0,0,0,0,,laptop -\'\n'
'escaped msg: \'2022-05-12T17:53:30 Feb 23 22:09:40 laptop 1,2020/02/23
22:09:40,fcc5dd0de335,SYSTEM,general,0,2020/02/23
22:09:40,,general,,0,0,general,informational,"Connection to Update server:
updates.paloaltonetworks.com completed successfully, initiated by
172.28.125.10",139576,0x0,0,0,0,0,,laptop -\'\n'
'inputname: udp rawmsg: \'<15>1 2022-05-12T17:53:30 Feb 23 22:09:40 laptop
1,2020/02/23 22:09:40,fcc5dd0de335,SYSTEM,general,0,2020/02/23
22:09:40,,general,,0,0,general,informational,"Connection to Update server:
updates.paloaltonetworks.com completed successfully, initiated by
172.28.125.10",139576,0x0,0,0,0,0,,laptop -\'\n'
'$!:\n'
'$.:\n'
'$/:\n'
'\n'
It seems rsyslog fails to set variables since they are all empty?
I tried to run with minimal configuration with the exact same result.
Is there a global parameter to turn off variable support?
Down below is my entire configuration
Best regards Johan Ryberg
global(
WorkDirectory="/var/cache/syslog"
)
module(load="omprog")
module(load="mmutf8fix")
module(load="imtcp")
module(load="imudp")
input(type="imtcp"
name="tcp"
port="514"
ruleset="azure_syslog")
input(type="imudp"
name="udp"
port="514"
ruleset="azure_syslog")
module(load="imfile"
mode="inotify")
input(type="imfile"
file="/var/lib/docker/containers/*/*.log"
tag="docker"
reopenOnTruncate="on"
ruleset="azure_docker")
template(name="ms" type="string"
string="%timereported:1:3:date-subseconds%")
template(name="new_unix" type="string"
string="%timereported:::date-unixtimestamp%%$.ms%")
set $.ms = exec_template("ms");
if ($.ms == "0") then {
set $.ms = "000";
}
set $.new_unix = exec_template("new_unix");
set $.test = "test string";
template(name="blobstorage" type="list" option.jsonf="on") {
property(outname="timestamp" name="timereported" dateFormat="rfc3339"
format="jsonf")
property(outname="unixtime" name="timereported"
dateFormat="unixtimestamp" format="jsonf")
property(outname="host" name="fromhost-ip" format="jsonf")
property(outname="test" name="$.test" format="jsonf")
property(outname="message" name="rawmsg-after-pri" format="jsonf")
property(outname="log_id" name="uuid" format="jsonf")
}
ruleset(name="azure_syslog") {
action(type="mmutf8fix")
action(
type="omprog"
name="azure_omprog"
action.resumeInterval="5"
binary="/usr/local/bin/syslog.py syslog"
confirmMessages="on"
confirmTimeout="180000"
killUnresponsive="on"
closeTimeout="200000"
output="/dev/stdout"
queue.type="fixedArray"
queue.size="32768"
queue.dequeueBatchSize="4096"
queue.workerThreads="1"
queue.workerThreadMinimumMessages="4096"
template="RSYSLOG_DebugFormat"
)
}
ruleset(name="azure_docker") {
action(type="mmutf8fix")
action(
type="omprog"
name="azure_omprog"
action.resumeInterval="5"
binary="/usr/local/bin/syslog.py docker"
confirmMessages="on"
confirmTimeout="180000"
killUnresponsive="on"
closeTimeout="200000"
output="/dev/stdout"
queue.type="fixedArray"
queue.size="32768"
queue.dequeueBatchSize="4096"
queue.workerThreads="1"
queue.workerThreadMinimumMessages="4096"
template="blobstorage"
)
}
module(
load="impstats"
interval="60"
format="json"
resetCounters="off"
ruleset="process_stats"
)
ruleset(name="process_stats") {
action(
type="omprog"
name="to_exporter"
binary="/usr/local/bin/rsyslog_exporter --web.listen-address=
127.0.0.1:9104"
)
On Thu, 12 May 2022 at 18:47, David Lang <da...@lang.hm> wrote:
when you have issues like this, it's a good idea to log the message with
the
template RSYSLOG_DebugFormat as it shows the contents of almost all the
variables you can be working with.
David Lang
On Thu, 12 May 2022, Johan Ryberg via rsyslog wrote:
Date: Thu, 12 May 2022 10:24:34 +0200
From: Johan Ryberg via rsyslog <rsyslog@lists.adiscon.com>
To: Rainer Gerhards <rgerha...@hq.adiscon.com>
Cc: Johan Ryberg <jo...@securit.se>, rsyslog-users <
rsyslog@lists.adiscon.com>
Subject: Re: [rsyslog] Ensure unixtimestamp with milliseconds?
Please ignore comment about missing "host", that entry came from imfile,
local file digest.
// Johan
On Thu, 12 May 2022 at 10:20, Johan Ryberg <jo...@securit.se> wrote:
Thanks for you reply.
I added ms and that is empty as well
template(name="ms" type="string"
string="%timereported:1:3:date-subseconds%")
template(name="new_unix" type="string"
string="%timereported:::date-unixtimestamp%%$.ms%")
set $.ms = exec_template("ms");
if ($.ms == "0") then {
set $.ms = "000";
}
set $.new_unix = exec_template("new_unix");
template(name="blobstorage" type="list" option.jsonf="on") {
property(outname="timestamp" name="timereported"
dateFormat="rfc3339"
format="jsonf")
property(outname="unixtime" name="timereported"
dateFormat="unixtimestamp" format="jsonf")
property(outname="ms" name="$.ms" format="jsonf")
property(outname="new_unix" name="$.new_unix" format="jsonf")
property(outname="host" name="fromhost-ip" format="jsonf")
property(outname="message" name="rawmsg-after-pri" format="jsonf")
property(outname="log_id" name="uuid" format="jsonf")
}
Output: {"timestamp":"2022-05-12T08:13:32.250744+00:00",
"unixtime":"1652343212", "ms":"", "new_unix":"", "host":"",
"message":"{\"log\":<redacted>",
"log_id":"D3E23BB3CAFE4F68BE2AE4804214228D"}
host is also empty, fun part if I move "host" above "ms" then I got
values
for host.
Any clue how to troubleshoot this?
Best regards Johan Ryberg
On Thu, 12 May 2022 at 09:37, Rainer Gerhards <rgerha...@hq.adiscon.com
wrote:
I would suggest to output $.ms as well. Also, simplify the new_unix
timeplate to just contain the default timestamp, see if it works and
than go from there with more complex processing.
all in all, it doesn't look wrong - probably a detail.
HTH
Rainer
El jue, 12 may 2022 a las 9:28, Johan Ryberg via rsyslog
(<rsyslog@lists.adiscon.com>) escribió:
Hi,
Running latest version of rsyslog (8.2204.1)
I'm trying to create a template that ensures unixtimestamp with
milliseconds.
This is as far as I got
template(name="ms" type="string"
string="%timereported:1:3:date-subseconds%")
set $!ms = exec_template("ms");
if ($.ms == "0") then {
set $.ms = "000";
}
template(name="new_unix" type="string"
string="%timereported:::date-unixtimestamp%%$.ms%")
set $!new_unix = exec_template("new_unix");
template(name="storage" type="list" option.jsonf="on") {
property(outname="timestamp" name="timereported"
dateFormat="rfc3339"
format="jsonf")
property(outname="unixtime" name="$!new_unix" format="jsonf")
property(outname="host" name="fromhost-ip" format="jsonf")
property(outname="message" name="rawmsg-after-pri"
format="jsonf")
property(outname="log_id" name="uuid" format="jsonf")
}
Sadly the output of unixtime is always empty, "unixtime": ""
I would very much appreciate some guidance how to make it work and if
possible, more efficient
Best regards Johan Ryberg
_______________________________________________
rsyslog mailing list
https://lists.adiscon.net/mailman/listinfo/rsyslog
http://www.rsyslog.com/professional-services/
What's up with rsyslog? Follow https://twitter.com/rgerhards
NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a
myriad of sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST
if
you DON'T LIKE THAT.
_______________________________________________
rsyslog mailing list
https://lists.adiscon.net/mailman/listinfo/rsyslog
http://www.rsyslog.com/professional-services/
What's up with rsyslog? Follow https://twitter.com/rgerhards
NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a myriad
of sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if you
DON'T LIKE THAT.
