Hi,
On occasion there is a problem with the byte counters for the
"link1_in_asn" aggregate defined below. The byte counters for multiple
SRC_ASN to DST_ASN will show a value 18 times greater than the actual
value. This aggregate is tracking the SRC and DST ASNs every 5 minutes.
There are additional aggregates defined which track more specific data
with an increased time window of 15 minutes, e.g. "link1_in".
What's interesting is the "link1_in" aggregate does not capture the same
spurious byte count when it occurs for the "link1_in_asn" table. This
phenomenon was also confirmed to be unique to the "link1_in_asn" table by
verifying the byte count with another netflow analysis platform.
Additionally, it seems the problem regularly involves a specific regional
ID defined in the pre-tagging map file. That file was double checked and
no errors were found.
Thanks for the software and assistance.
--TC
Config values:
./configure --prefix=/usr/local/etc --enable-threads --enable-64bit
--enable-pgsql --enable-ipv6 --enable-v4-mapped
Examples from SQL:
################################################
# From "link1_in_asn" aggregate (5 minute summaries)
################################################
SELECT stamp_inserted, as_src, as_dst, ip_proto, bytes, packets, flows
FROM pmacct.link1_in_asn
WHERE stamp_inserted = '2012-03-21 23:00:00'
AND ip_proto = 17
AND agent_id = 123
ORDER BY stamp_inserted, bytes DESC LIMIT 10;
stamp_inserted | as_src | as_dst | ip_proto | bytes | packets
| flows
------------------------+--------+--------+----------+------------+---------+-------
2012-03-21 23:00:00+00 | werad | YYYYY | 17 | 3214956083 | 3500975
| 14499
2012-03-21 23:00:00+00 | dslakf | YYYYY | 17 | 2321315742 |
1583493 | 149
2012-03-21 23:00:00+00 | owier | YYYYY | 17 | 1930275726 | 1318249
| 144
2012-03-21 23:00:00+00 | odfja | YYYYY | 17 | 1646701744 | 1132506
| 6829
2012-03-21 23:00:00+00 | abwe | ZZZZZ | 17 | 1610509378 | 1122283
| 11174
2012-03-21 23:00:00+00 | dde | YYYYY | 17 | 1499006716 | 1612566
| 38509
2012-03-21 23:00:00+00 | bd | YYYYY | 17 | 1386283090 | 969709 |
143
2012-03-21 23:00:00+00 | wef | YYYYY | 17 | 1339588617 | 932396
| 1582
2012-03-21 23:00:00+00 | lkjd | YYYYY | 17 | 1256466227 | 1197856
| 28710
2012-03-21 23:00:00+00 | pwe | YYYYY | 17 | 1218225269 | 2108718
| 30506
(10 rows)
############################################
# From "link1_in" aggregate (15 minute summaries)
############################################
SELECT stamp_inserted, as_src, as_dst, ip_proto, sum(bytes) AS bytes_sum,
sum(packets) AS packets_sum, sum(flows) AS flows_sum
FROM pmacct.link1_ingress
WHERE stamp_inserted = '2012-03-21 23:00:00'
AND ip_proto = 17
AND agent_id = 123
GROUP BY stamp_inserted, as_src, as_dst, ip_proto
ORDER BY stamp_inserted, bytes_sum DESC LIMIT 10;
stamp_inserted | as_src | as_dst | ip_proto | bytes_sum |
packets_sum | flows_sum
------------------------+--------+--------+----------+-----------+-------------+-----------
2012-03-21 23:00:00+00 | abcd | YYYYY | 17 | 251521852 |
383070 | 2702
2012-03-21 23:00:00+00 | eeff | YYYYY | 17 | 206594948 |
160698 | 260
2012-03-21 23:00:00+00 | gadsf | ZZZZZ | 17 | 188309949 |
133074 | 1273
2012-03-21 23:00:00+00 | zowes | YYYYY | 17 | 178528807 |
251929 | 1671
2012-03-21 23:00:00+00 | asdfl | YYYYY | 17 | 173423708 |
256535 | 3810
2012-03-21 23:00:00+00 | bws | YYYYY | 17 | 152331951 |
176804 | 75
2012-03-21 23:00:00+00 | ppw | YYYYY | 17 | 139594690 |
95350 | 10
2012-03-21 23:00:00+00 | xwz | YYYYY | 17 | 132966829 |
145129 | 3056
2012-03-21 23:00:00+00 | owkd | YYYYY | 17 | 125959489 |
88111 | 8
2012-03-21 23:00:00+00 | llkjs | YYYYY | 17 | 121121281 |
82647 | 16
(10 rows)
##### partial config file ##########
#
daemonize: true
syslog: local0
!
aggregate[link1_in]: src_as, dst_as, src_net, src_mask, dst_net, dst_mask,
proto, flows, tag
aggregate[link1_out]: src_as, dst_as, src_net, src_mask, dst_net, dst_mask,
proto, std_comm, local_pref, med, flows, tag
aggregate[link1_in_asn]: src_as, dst_as, proto, flows, tag
aggregate[link1_out_asn]: src_as, dst_as, std_comm, as_path, proto, flows,
tag
!
aggregate_filter[link1_in]:
aggregate_filter[link1_out]:
aggregate_filter[link1_in_asn]:
aggregate_filter[link1_out_asn]:
!
plugin_pipe_size: 20480000
plugin_buffer_size: 20480
plugins: pgsql[link1_in], pgsql[link1_out], pgsql[link1_in_asn],
pgsql[link1_out_asn]
!
nfacctd_ip: 10.10.10.1
nfacctd_port: xxxxx
nfacctd_time_new: true
nfacctd_as_new: fallback
!
sql_host[link1_in_asn]: 10.10.10.2
sql_db[link1_in_asn]: database
sql_table[link1_in_asn]: pmacct.link1_in_asn_%Y%m%d_%H
sql_table_type[link1_in_asn]: bgp
sql_user[link1_in_asn]: xxxx
sql_password[link1_in_asn]: xxxx
sql_refresh_time[link1_in_asn]: 300
sql_optimize_clauses[link1_in_asn]: true
sql_history[link1_in_asn]: 5m
sql_history_roundoff[link1_in_asn]: m
sql_dont_try_update[link1_in_asn]: true
sql_use_copy[link1_in_asn]: true
sql_cache_entries[link1_in_asn]: 512009
!
sql_host[link1_in]: 10.10.10.2
sql_db[link1_in]: database
sql_table[link1_in]: pmacct.link1_ingress_%Y%m%d_%H
sql_table_type[link1_in]: bgp
sql_user[link1_in]: xxxx
sql_password[link1_in]: xxxx
sql_refresh_time[link1_in]: 900
sql_optimize_clauses[link1_in]: true
sql_history[link1_in]: 15m
sql_history_roundoff[link1_in]: m
sql_dont_try_update[link1_in]: true
sql_use_copy[link1_in]: true
sql_cache_entries[link1_in]: 2048063
!
!
refresh_maps: true
pre_tag_map: /usr/local/etc/pmacct/premap_regional_id.map
networks_mask: 24
##### premap file ##########
#
! Pre-tagging map for regional ID -- Cannot place all networks on a single
line due to
! static buffer limit so break up /24s into groups of five.
!
id=3 ip=10.10.10.5 filter="net 192.168.0.0/24 or net 192.168.41.0/24 or net
192.168.22.0/24 ... "
id=3 ip=10.10.10.5 filter="net 192.168.51.0/24 or net 192.168.64.0/24 or
net 192.168.77.0/24 ... "
id=3 ip=10.10.10.5 filter="net 192.168.120.0/24 or net 192.168.1.0/24 or
net 192.168.62.0/24 ... "
(...)
id=177 ip=10.10.10.5 filter="net ...."
(...)
_______________________________________________
pmacct-discussion mailing list
http://www.pmacct.net/#mailinglists
