Distributed in time execution of poller

[juampe]

Distributed (in time) data poller daemon for cacti
Executes polling for all sources, not a the same time.
This can resolve problems with high loads in large poller machines
and network

Code: Select all

The polling is deviated in time randomly
Example
We consiredered execution of S1 and S2 each 5 minutes and S3 each 10
Classical cron works like this
TIME   0. . . .5. . . .10. . . 15. . . 20
S1     *       *       *       *       *
S2     *       *       *       *       *
S3     *               *               *
LOAD   30000000200000003000000020000000300000
Distributed cron works like this
TIME   0       5       10      15      20
S1      *       *       *       *       *
S2         *       *       *       *       *
S3        *               *               *
LOAD   01011000010010000101100001001000010110

We can see that distribute the execution help to optimize and support
more load.

Proff of concept in perl based in a personal poller

Code: Select all

#!/usr/bin/perl
#Juan Pedro Paredes <juampe@iquis.com>
#GPL licensed
#Cron disperso en el tiempo
#Distributed crontab
use Time::HiRes qw(usleep);
  

print "$0\n";

#Daemon mode - detach
$daemon=1;
if($daemon){
	$SIG{HUP}=sub{};
	$SIG{CHLD}='IGNORE';
	my $pid=fork;
	if($pid<0){
	        print STDERR "${program}: can not detach $!\n";
	        exit -1;
	}elsif($pid){
	        exit 0;
	}
}

#Gestion de señales HUP,SIGUSR1,TERM,KILL,ABRT,INT
$SIG{HUP}='initialize';
$SIG{SIGUSR1}='initialize';
$SIG{TERM}='done';
$SIG{KILL}='done';
$SIG{ABRT}='done';
$SIG{INT}='done';
#Gestion Anti-Zombies
#$SIG{CHLD}=sub{wait};
$SIG{CHLD}=sub{while(waitpid(-1,WNOHANG)>0){}};


#Gestion E/S standard para logger
#$logger="logger -p daemon.info -t collector";
#open(STDOUT,"| $logger");
#open(STDERR,"| $logger");
#open(STDIN,"</dev/null");

#Renice
setpriority(0,$$,20);
#PID
system("echo \"$$\">$home/dcron.pid");

#Variables globales y esenciales para el cron
my @cmds;
my @steps;
my @rnds;
my @stamps;
my @counter;

#Inicializa y refresca el sistema de cron 
#Initialize and reread configuration from dcrontab
sub initialize{
	print "reseting\n";
	@cmds=();
	@steps=();
	@rnds=();
	@stamps=();
	@counter=();
	srand;
	
	#recoger de dcrontab
	#Read the crontab an fill
	open DCRONTAB,"/etc/dcrontab";
	@dcrontab=<DCRONTAB>;
	foreach $entry (@dcrontab){
		chomp $entry;
		if($entry!~/^#/){
			my ($step,$user,$cmd)=split(/:/,$entry);
			#my ($step,$user,$cmd)=$entry=~/(.+)[\t ]+(.+)[\t ]+(.+)$+/;
			print "$step $cmd\n";
			push(@steps,$step);
			push(@rnds,int(rand($step)));
			push(@cmds,$cmd);
			push(@stamps,0);
			push(@counter,0);
		}
	}
	close DCRONTAB;
}

sub min{@_[0]<@_[1]?return @_[0]:return @_[1]}
sub max{@_[0]>@_[1]?return @_[0]:return @_[1]}
sub dcic{min(abs(@_[0]-@_[1]),min(@_[0],@_[1])+abs(max(@_[0],@_[1])-@_[2]))}

print "started\n";
initialize;
#Bucle de programa
my $shift=60;
while(1){
	$t=time;
	#Comandos que se ejecutan cada cierto tiempo
	#Cmd's that are executed each second
	for($i=0;$i<=$#steps;$i++){
		$r= $t % $steps[$i];
		#If the machine is very loaded
		#do not forks more process 
		#we have a time shift to try again
                 #Fork if the second match
		#Esto permite que si la maquina esta muy cargada
		#no se lancen mas forks de recoleccion con un desplazamiento
		if($stamps[$i] < $t and dcic($r,$rnds[$i],$steps[$i])<=$shift){
			$stamps[$i]=$t+($shift*2)+1;
			$counter[$i]++;
			if($pid=fork()){
				#Father
			}elsif(defined $pid){
				#Child
				if($steps[$i]==300){
				print "$r -> $rnds[$i] ".dcic($r,$rnds[$i],$steps[$i])." ($counter[$i])\n";
				}
				`$cmds[$i]`;
				exit 0;
			}
		}
	}
	#If the for statement waits more than 750000 usec maybe the
	#poller is not executed
	#Si el bucle dura mas de 750000 mirosegundos puede que no se lancen
	#recolectores para ese segundo
	usleep(10000);
}

sub done{
	`rm $home/dcron.pid`;
	print "stopped\n";
	closelog;
	exit 0;
}