[solved]Graph cut after bandwidth reach 800MB (64b counters)

[AlexDe]

Hi!
I try to monitoring my 1G and 10G channels (of cause with 64-bit counters) and have similar pictures for all channels - in attachment.
Real bandwith in maximum is little less than 1G (900-950 MBit/s), but more than in picture (800 MBit/s)

How can I see real picture, not cutted?

Thanks!

[BSOD2600]

Post the graph debug output. Make sure the graph template max isn't ~800 or something. Also make sure the rrd file DS MAX isn't too small (rrdtool info filename.rrd).

If there is no max set, then looks like you're possibly running into the snmp counter rolling over. Using 1 min polling intervals?

[lcano]

It is not an issue with the 64 bits counter, but more an issue with the graph. If the counter is not big enough, you will get an extrange graph when it goes over the maximun, and starts again from zero. But this is not your problem, you get the right data, but not correcltly graphed. Check that parameters...

[TheWitness]

You may use the RRDtool tune command to increase the size of your Data Sources max values. You need also adjust the Data Template so that future graphs do not suffer from this issue.

TheWitness

[AlexDe]

Post the graph debug output. Make sure the graph template max isn't ~800 or something. Also make sure the rrd file DS MAX isn't too small (rrdtool info filename.rrd).

Code: Select all

RRDTool Command:

/usr/local/bin/rrdtool graph - \
--imgformat=PNG \
--start=-86400 \
--end=-300 \
--title="............ Traffic ........." \
--rigid \
--base=1000 \
--height=150 \
--width=500 \
--alt-autoscale-max \
--lower-limit=0 \
--vertical-label="bits per second" \
--slope-mode \
DEF:a="/usr/src/cacti-0.8.6h/rra/10_25_8_13_traffic_in_5884.rrd":traffic_in:AVERAGE \
DEF:b="/usr/src/cacti-0.8.6h/rra/10_25_8_13_traffic_in_5884.rrd":traffic_out:AVERAGE \
CDEF:cdefa=a,8,* \
CDEF:cdefe=b,8,* \
AREA:cdefa#00CF00:"Inbound"  \
GPRINT:cdefa:LAST:" Current\:%8.2lf %s"  \
GPRINT:cdefa:AVERAGE:"Average\:%8.2lf %s"  \
GPRINT:cdefa:MAX:"Maximum\:%8.2lf %s\n"  \
LINE1:cdefe#002A97:"Outbound"  \
GPRINT:cdefe:LAST:"Current\:%8.2lf %s"  \
GPRINT:cdefe:AVERAGE:"Average\:%8.2lf %s"  \
GPRINT:cdefe:MAX:"Maximum\:%8.2lf %s" 

RRDTool Says:

OK

Code: Select all

rrdtool info /usr/src/cacti-0.8.6h/rra/10_25_8_13_traffic_in_5884.rrd

rrd_version = "0003"
step = 300
last_update = 1274154602
ds[traffic_in].type = "COUNTER"
ds[traffic_in].minimal_heartbeat = 600
ds[traffic_in].min = 0.0000000000e+00
ds[traffic_in].max = 1.0000000000e+08
ds[traffic_in].last_ds = "21217170432023"
ds[traffic_in].value = 5.8061872333e+07
ds[traffic_in].unknown_sec = 0
ds[traffic_out].type = "COUNTER"
ds[traffic_out].minimal_heartbeat = 600
ds[traffic_out].min = 0.0000000000e+00
ds[traffic_out].max = 1.0000000000e+08
ds[traffic_out].last_ds = "51638866533333"
ds[traffic_out].value = 1.7756961648e+08
ds[traffic_out].unknown_sec = 0
rra[0].cf = "AVERAGE"
rra[0].rows = 600
rra[0].pdp_per_row = 1
rra[0].xff = 5.0000000000e-01
rra[0].cdp_prep[0].value = NaN
rra[0].cdp_prep[0].unknown_datapoints = 0
rra[0].cdp_prep[1].value = NaN
rra[0].cdp_prep[1].unknown_datapoints = 0
rra[1].cf = "AVERAGE"
rra[1].rows = 700
rra[1].pdp_per_row = 6
rra[1].xff = 5.0000000000e-01
rra[1].cdp_prep[0].value = 1.1060636106e+08
rra[1].cdp_prep[0].unknown_datapoints = 0
rra[1].cdp_prep[1].value = 3.3320702216e+08
rra[1].cdp_prep[1].unknown_datapoints = 0
rra[2].cf = "AVERAGE"
rra[2].rows = 775
rra[2].pdp_per_row = 24
rra[2].xff = 5.0000000000e-01
rra[2].cdp_prep[0].value = 5.3159573168e+08
rra[2].cdp_prep[0].unknown_datapoints = 0
rra[2].cdp_prep[1].value = 1.5178887199e+09
rra[2].cdp_prep[1].unknown_datapoints = 0
rra[3].cf = "AVERAGE"
rra[3].rows = 797
rra[3].pdp_per_row = 288
rra[3].xff = 5.0000000000e-01
rra[3].cdp_prep[0].value = 9.9311853787e+08
rra[3].cdp_prep[0].unknown_datapoints = 0
rra[3].cdp_prep[1].value = 2.5630308043e+09
rra[3].cdp_prep[1].unknown_datapoints = 0
rra[4].cf = "MIN"
rra[4].rows = 600
rra[4].pdp_per_row = 1
rra[4].xff = 5.0000000000e-01
rra[4].cdp_prep[0].value = NaN
rra[4].cdp_prep[0].unknown_datapoints = 0
rra[4].cdp_prep[1].value = NaN
rra[4].cdp_prep[1].unknown_datapoints = 0
rra[5].cf = "MIN"
rra[5].rows = 700
rra[5].pdp_per_row = 6
rra[5].xff = 5.0000000000e-01
rra[5].cdp_prep[0].value = 2.6045022138e+07
rra[5].cdp_prep[0].unknown_datapoints = 0
rra[5].cdp_prep[1].value = 7.8146003027e+07
rra[5].cdp_prep[1].unknown_datapoints = 0
rra[6].cf = "MIN"
rra[6].rows = 775
rra[6].pdp_per_row = 24
rra[6].xff = 5.0000000000e-01
rra[6].cdp_prep[0].value = 2.0702501128e+07
rra[6].cdp_prep[0].unknown_datapoints = 0
rra[6].cdp_prep[1].value = 5.2869409487e+07
rra[6].cdp_prep[1].unknown_datapoints = 0
rra[7].cf = "MIN"
rra[7].rows = 797
rra[7].pdp_per_row = 288
rra[7].xff = 5.0000000000e-01
rra[7].cdp_prep[0].value = 1.6775588221e+07
rra[7].cdp_prep[0].unknown_datapoints = 0
rra[7].cdp_prep[1].value = 3.8207428519e+07
rra[7].cdp_prep[1].unknown_datapoints = 0
rra[8].cf = "MAX"
rra[8].rows = 600
rra[8].pdp_per_row = 1
rra[8].xff = 5.0000000000e-01
rra[8].cdp_prep[0].value = NaN
rra[8].cdp_prep[0].unknown_datapoints = 0
rra[8].cdp_prep[1].value = NaN
rra[8].cdp_prep[1].unknown_datapoints = 0
rra[9].cf = "MAX"
rra[9].rows = 700
rra[9].pdp_per_row = 6
rra[9].xff = 5.0000000000e-01
rra[9].cdp_prep[0].value = 2.9052349150e+07
rra[9].cdp_prep[0].unknown_datapoints = 0
rra[9].cdp_prep[1].value = 8.8749376662e+07
rra[9].cdp_prep[1].unknown_datapoints = 0
rra[10].cf = "MAX"
rra[10].rows = 775
rra[10].pdp_per_row = 24
rra[10].xff = 5.0000000000e-01
rra[10].cdp_prep[0].value = 2.9052349150e+07
rra[10].cdp_prep[0].unknown_datapoints = 0
rra[10].cdp_prep[1].value = 8.8749376662e+07
rra[10].cdp_prep[1].unknown_datapoints = 0
rra[11].cf = "MAX"
rra[11].rows = 797
rra[11].pdp_per_row = 288
rra[11].xff = 5.0000000000e-01
rra[11].cdp_prep[0].value = 2.9052349150e+07
rra[11].cdp_prep[0].unknown_datapoints = 0
rra[11].cdp_prep[1].value = 8.8749376662e+07
rra[11].cdp_prep[1].unknown_datapoints = 0
rra[12].cf = "LAST"
rra[12].rows = 600
rra[12].pdp_per_row = 1
rra[12].xff = 5.0000000000e-01
rra[12].cdp_prep[0].value = NaN
rra[12].cdp_prep[0].unknown_datapoints = 0
rra[12].cdp_prep[1].value = NaN
rra[12].cdp_prep[1].unknown_datapoints = 0
rra[13].cf = "LAST"
rra[13].rows = 700
rra[13].pdp_per_row = 6
rra[13].xff = 5.0000000000e-01
rra[13].cdp_prep[0].value = 2.9024983705e+07
rra[13].cdp_prep[0].unknown_datapoints = 0
rra[13].cdp_prep[1].value = 8.8749376662e+07
rra[13].cdp_prep[1].unknown_datapoints = 0
rra[14].cf = "LAST"
rra[14].rows = 775
rra[14].pdp_per_row = 24
rra[14].xff = 5.0000000000e-01
rra[14].cdp_prep[0].value = 2.9024983705e+07
rra[14].cdp_prep[0].unknown_datapoints = 0
rra[14].cdp_prep[1].value = 8.8749376662e+07
rra[14].cdp_prep[1].unknown_datapoints = 0
rra[15].cf = "LAST"
rra[15].rows = 797
rra[15].pdp_per_row = 288
rra[15].xff = 5.0000000000e-01
rra[15].cdp_prep[0].value = 2.9024983705e+07
rra[15].cdp_prep[0].unknown_datapoints = 0
rra[15].cdp_prep[1].value = 8.8749376662e+07
rra[15].cdp_prep[1].unknown_datapoints = 0

If there is no max set, then looks like you're possibly running into the snmp counter rolling over. Using 1 min polling intervals?

max set? Is that "Upper Limit" in Graph Template "Interface - Traffic (bits/sec)"? Value is 10000000000. In this template and in all any templates about traffic.
I can't use 1 min polling interval - only 5 min - too many devices must be requested.

You may use the RRDtool tune command to increase the size of your Data Sources max values. You need also adjust the Data Template so that future graphs do not suffer from this issue.

TheWitness

Code: Select all

rrdtool tune
RRDtool 1.2.15  Copyright 1997-2006 by Tobias Oetiker <tobi@oetiker.ch>
               Compiled Oct  4 2006 15:21:11

Usage: rrdtool [options] command command_options

 * tune -  Modify some basic properties of an RRD

        rrdtool tune filename
                [--heartbeat|-h ds-name:heartbeat]
                [--data-source-type|-d ds-name:DST]
                [--data-source-rename|-r old-name:new-name]
                [--minimum|-i ds-name:min] [--maximum|-a ds-name:max]
                [--deltapos scale-value] [--deltaneg scale-value]
                [--failure-threshold integer]
                [--window-length integer]
                [--alpha adaptation-parameter]
 * tune -  Modify some basic properties of an RRD

                [--beta adaptation-parameter]
                [--gamma adaptation-parameter]
                [--gamma-deviation adaptation-parameter]
                [--aberrant-reset ds-name]

RRDtool is distributed under the Terms of the GNU General
Public License Version 2. (www.gnu.org/copyleft/gpl.html)

For more information read the RRD manpages

oh..... those (alfa, beta, gamma etc) adaptation-parameters.... I couldn't put all my life for studying them.... Could you give examples? Sorry, maintenance of CACTi-service is not my base duty. I can (and do) base settings, create templates, add devices for monitoring, but those parameters ....

It is not an issue with the 64 bits counter, but more an issue with the graph. If the counter is not big enough, you will get an extrange graph when it goes over the maximun, and starts again from zero. But this is not your problem, you get the right data, but not correcltly graphed. Check that parameters...

I agree, problem in graphical output. But I don't know, what parameter(s) should be corrected.

[BSOD2600]

Looks like the rrd DS MAX fields are too small. Notice the .MAX is 1.0000000000e+08 while the .last_DS = "21217170432023"

[AlexDe]

Looks like the rrd DS MAX fields are too small. Notice the .MAX is 1.0000000000e+08 while the .last_DS = "21217170432023"

hm.....
I think you are talking about that:

Code: Select all

rrdtool info /usr/src/cacti-0.8.6h/rra/10_25_8_13_traffic_in_5884.rrd 
.....
ds[traffic_in].max = 1.0000000000e+08   <= Must be  1.0e+9
....
ds[traffic_in].value = 5.8061872333e+07  <= Less than  max, graph OK
.....
ds[traffic_out].max = 1.0000000000e+08    <= Must be  1.0e+9
....
ds[traffic_out].value = 1.7756961648e+08  <= More than  max, graph cutted
.....

Then next question - HOW to change
ds[traffic_in].max and ds[traffic_out].max?
All similar parameters I've changed, but

Code: Select all

rrdtool info /usr/src/cacti-0.8.6h/rra/10_25_8_13_traffic_in_5884.rrd

say the same (1.0e+8)

After 5 minuts for search.....

Code: Select all

$ rrdtool info /usr/src/cacti-0.8.6h/rra/10_25_8_13_traffic_in_5884.rrd | grep max
ds[traffic_in].max = 1.0000000000e+08
ds[traffic_out].max = 1.0000000000e+08
$ rrdtool tune /usr/src/cacti-0.8.6h/rra/10_25_8_13_traffic_in_5884.rrd -a traffic_in:1.0000000000e+09
$ rrdtool tune /usr/src/cacti-0.8.6h/rra/10_25_8_13_traffic_in_5884.rrd -a traffic_out:1.0000000000e+09
$ rrdtool info /usr/src/cacti-0.8.6h/rra/10_25_8_13_traffic_in_5884.rrd | grep max
ds[traffic_in].max = 1.0000000000e+09
ds[traffic_out].max = 1.0000000000e+09

Well done! I'll see results during 2-4 hours and write them here.


After 2-3 hours
Yepppppp!!!!!!!!!!!!! It works! I have full picture of channel bandwith.

BSOD2600, lcano, TheWitness, Thanks a lot of!

[lcano]

Nice to read you solved your problem. But you changed it manually (rrdtool tune command), so you will have again the same problem with new graphs. You should change the data template, in order to adjust the MAX parameter to the proper one.

[AlexDe]

I've changed all founded maximal parameters in all templates.
I don't know, why they didn't apply to already existed RRD-files.
May be they'll apply to newly created files? I'll see that (and write down here).
As about existing RRD-files.... At least I can change MAX values "manually" - something about that:

Code: Select all

find -name *.rrd /usr/src/cacti-0.8.6h/rra/ | rrdtool tune  -a traffic_in:1.0000000000e+09

[BSOD2600]

I've changed all founded maximal parameters in all templates.

Won't matter, as the Data Input method automatically takes the ifSpeed and uses that for the MAX DS value on rrd creation. This can be disabled if you wish, but use caution.

I don't know, why they didn't apply to already existed RRD-files.
May be they'll apply to newly created files? I'll see that (and write down here).

Cacti does not touch existing files when data templates are changed. Yes, new files will use the changes.

[AlexDe]

Won't matter, as the Data Input method automatically takes the ifSpeed and uses that for the MAX DS value on rrd creation. This can be disabled if you wish, but use caution.

How can I do that - make disable?

[BSOD2600]

Data queries -> snmp - interface -> select an associated graph template -> suggested data values. Remove the rra max entry.

[AlexDe]

Thanks!
I'll try that