octopussy 1.0.6-0ubuntu1 / usr / sbin / octo_reporter

#!/usr/bin/perl -w

=head1 NAME

octo_reporter - Octopussy Reporter program

=head1 SYNOPSIS

octo_reporter --report <report> --device <device> --service <service>
  --taxonomy <taxonomy> --begin YYYYMMDDHHMM --end YYYYMMDDHHMM
  [ --pid_param <string> ] --output <output_file>

Mail options:
  --mail_subject <subject> --mail_recipients <recipients>
 	
FTP options:
  --ftp_host <host> --ftp_dir <dir>
  --ftp_user <user> --ftp_password <password>

SCP options: (SSH Public Key required)
  --scp_host <host> --scp_dir <dir> --scp_user <user>

=head1 DESCRIPTION

octo_reporter is the program used by the Octopussy Project to generate Reports

=cut

use strict;
use warnings;
use Readonly;

use Date::Manip;
use Getopt::Long;
Getopt::Long::Configure('bundling');
use Term::ProgressBar 2.00;

use AAT::DB;
use AAT::Syslog;
use AAT::Translation;
use AAT::Utils qw( NOT_NULL );
use Octopussy;
use Octopussy::Cache;
use Octopussy::Loglevel;
use Octopussy::Logs;
use Octopussy::Message;
use Octopussy::Plugin;
use Octopussy::Report;
use Octopussy::Table;
use Octopussy::Taxonomy;
use Octopussy::Type;

Readonly my $APPLI         => 'Octopussy';
Readonly my $PROG_NAME     => 'octo_reporter';
Readonly my $VERSION       => Octopussy::Version();
Readonly my $FILE_DB       => '/var/run/octopussy/file.db';
Readonly my $MIN_DB_INSERT => 10_000;

my $file_pid = undef;

my ($help,         $quiet);
my (@devices,      @services) = ((), ());
my ($report,       $rc, $loglevel, $taxonomy);
my ($begin,        $end, $pid_param, $lang);
my ($mail_subject, $mail_recipients);
my ($host_ftp,     $dir_ftp, $user_ftp, $pwd_ftp);
my ($host_scp, $dir_scp, $user_scp);
my ($type, $title) = qw(bars test);
my $output = undef;
my $cache  = undef;

=head1 FUNCTIONS

=head2 Help()

Prints Help

=cut

sub Help
{
    my $help_str = <<"EOF";

$PROG_NAME (version $VERSION)

 Usage: $PROG_NAME --report <report> --device <device> --service <service> 
          --loglevel <loglevel> --taxonomy <taxonomy>
          --begin YYYYMMDDHHMM --end YYYYMMDDHHMM
          [ --pid_param <string> ] --output <output_file>
 Mail options:
   --mail_subject <subject> --mail_recipients <recipients>
 Ftp options:
   --ftp_host <host> --ftp_dir <dir>
   --ftp_user <user> --ftp_password <password>
 Scp options:
   --scp_host <host> --scp_dir <dir> --scp_user <user>

EOF

    print $help_str;

    if (!defined $report)
    {
        my @reps = Octopussy::Report::List(undef, undef);
        print ' Report list: ' . join(', ', @reps) . "\n";
    }
    else
    {
        $rc = Octopussy::Report::Configuration($report);
        if (!defined $rc)
        {
            print "Error: Unable to get configuration for Report [$report]\n";
            exit;
        }
        my ($dev_groups, $devs, $servs) =
            Octopussy::Table::Devices_and_Services_With($rc->{table});
        if ((!@devices) && (defined $devs))
        {
            print ' Device list: ' . join(', ', @{$devs}) . "\n";
        }
        elsif ((!@services) && (defined $servs))
        {
            print ' Service list: ' . join(', ', sort @{$servs}) . "\n";
        }
        elsif (!defined $loglevel)
        {
            print ' ' . Octopussy::Loglevel::String_List($devs, $servs) . "\n";
        }
        elsif (!defined $taxonomy)
        {
            print ' ' . Octopussy::Taxonomy::String_List($devs, $servs) . "\n";
        }
    }
    print "\n";

    exit;
}

=head2 Progress($msg, $num)

Sets progress status

=cut

sub Progress
{
    my ($msg, $num) = @_;

    my $str_progress = AAT::Translation::Get('EN', $msg) . " [$num]";
    $cache->set("status_${pid_param}", $str_progress) if (defined $pid_param);

    return ($str_progress);
}

=head2 SQL($ref_pos, @args)

Generates SQL line

=cut

sub SQL
{
    my ($ref_pos, @args) = @_;
    my $line = '';

    foreach my $p (@{$ref_pos})
    {
        if (defined $p)
        {
            if (defined $p->{pos})
            {
                my $f = $p->{function};
                return (undef)
                    if ((defined $f)
                    && (!defined &{$f}($args[$p->{pos}])));
                my $value = (
                    defined $f
                    ? &{$f}($args[$p->{pos}])
                    : $args[$p->{pos}]
                );
                $value =~ s/\t+/ /g;
                $line .= Octopussy::Type::SQL_Datetime($value);
            }
            else
            {
                $line .= 'NULL';
            }
        }
        $line .= "\t";
    }

    return ($line);
}

=head2 Get_Messages_To_Parse($services, $taxo, $table, $query, $fields_list)

Returns list of Messages to parse

=cut

sub Get_Messages_To_Parse
{
    my ($services, $taxo, $table, $query, $fields_list) = @_;
    my @fields = Octopussy::Report::Table_Creation($table, $query);
    my $fields_regexp = Octopussy::Message::Regexped_Fields($query);
    my @msg_to_parse =
        Octopussy::Message::Parse_List($services, $loglevel, $taxo, $table,
        \@fields, $fields_regexp, $fields_list);

    return (@msg_to_parse);
}

=head2 Get_TimePeriod_Files($devices, $services, $begin, $end)

Returns list of Files for Devices $devices, Services $services
and Period $begin-$end

=cut

sub Get_TimePeriod_Files
{
    my ($devices, $services, $begin, $end) = @_;

    my $re_date = qr/^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})$/;
    my ($y1, $m1, $d1, $hour1, $min1);
    my ($y2, $m2, $d2, $hour2, $min2);

    if (   (($y1, $m1, $d1, $hour1, $min1) = $begin =~ $re_date)
        && (($y2, $m2, $d2, $hour2, $min2) = $end =~ $re_date))
    {
        my %start = (
            year  => $y1,
            month => $m1,
            day   => $d1,
            hour  => $hour1,
            min   => $min1,
        );
        my %finish = (
            year  => $y2,
            month => $m2,
            day   => $d2,
            hour  => $hour2,
            min   => $min2,
        );
        my $files =
            Octopussy::Logs::Files($devices, $services, \%start, \%finish);

        return ($files);
    }

    return (undef);
}

=head2 Insert_Data($devices, $services, $taxo, $begin, $end, $table, 
 $query, $field_list)

Inserts data

=cut

sub Insert_Data
{
    my ($devices, $services, $taxo, $begin, $end, $table, $query, $field_list) =
        @_;
    Progress('_MSG_REPORT_PROGRESS_LISTING_FILES', '1/1');
    my @msg_to_parse =
        Get_Messages_To_Parse($services, $taxo, $table, $query, $field_list);
    my $files      = Get_TimePeriod_Files($devices, $services, $begin, $end);
    my $total      = scalar @{$files};
    my @known_msgs = ();
    AAT::Syslog::Message('octo_core',
              "$total file(s) to parse with "
            . scalar @msg_to_parse
            . ' message(s).');
    my $i        = 1;
    my $nb_lines = 0;
    `touch "$FILE_DB.$$"`;
    chmod 0644, "$FILE_DB.$$";

    my $progressbar = Term::ProgressBar->new(
        {name => 'Inserting Data...', count => $total, ETA => 'linear'})
        if (!defined $pid_param);
    foreach my $f (@{$files})
    {
        Progress('_MSG_REPORT_PROGRESS_INSERTING_DATA', $i . "/$total");
        $progressbar->update($i) if (!defined $pid_param);
		my $cat = ($f =~ /.+\.gz$/ ? 'zcat' : 'cat');
		if (defined open my $FILE, '-|', "$cat \"$f\"")
        {
            while (my $line = <$FILE>)
            {
                chomp $line;
                $nb_lines++;
                foreach my $msg (@msg_to_parse)
                {
                    if (my (@args) = $line =~ $msg->{re})
                    {
                        my $sql = SQL($msg->{positions}, @args);
                        push @known_msgs, $sql if (defined $sql);
                        last;
                    }
                }
            }
            close $FILE;
        }
        else
        {
            print "Unable to open file '$f'\n";
            AAT::Syslog::Message($APPLI, 'UNABLE_OPEN_FILE', $f);
        }
        if (scalar @known_msgs > $MIN_DB_INSERT)
        {    # Insert in DB only every $MIN_DB_INSERT+ lines
            AAT::DB::Load_Infile($APPLI, $table, "$FILE_DB.$$", \@known_msgs);
            @known_msgs = ();
        }
        $i++;
    }
    AAT::DB::Load_Infile($APPLI, $table, "$FILE_DB.$$", \@known_msgs);
    unlink "$FILE_DB.$$";

    return ($total, $nb_lines);
}

=head2 End()

Ends Reporter

=cut

sub End
{
    AAT::Syslog::Message($PROG_NAME, 'REPORTER_GENERATION_STATUS', 'Aborted');
    AAT::DB::Table_Destruction($APPLI, $rc->{table} . "_$$");
    unlink $file_pid;
    unlink "$FILE_DB.$$";
    $cache->remove("info_$$");
    $cache->remove("status_$$");
    exit;
}

#
# MAIN
#
exit if (!Octopussy::Valid_User($PROG_NAME));

$SIG{USR2} = \&End;

my $status = GetOptions(
    'h|help'          	=> \$help,
    'q|quiet'           => \$quiet,
    'report=s'          => \$report,
    'devices=s'         => \@devices,
    'services=s'        => \@services,
    'loglevel=s'        => \$loglevel,
    'taxonomy=s'        => \$taxonomy,
    'begin=s'           => \$begin,
    'end=s'             => \$end,
    'lang=s'            => \$lang,
    'pid_param=s'       => \$pid_param,
    'output=s'          => \$output,
    'mail_subject=s'    => \$mail_subject,
    'mail_recipients=s' => \$mail_recipients,
    'ftp_host=s'        => \$host_ftp,
    'ftp_dir=s'         => \$dir_ftp,
    'ftp_user=s'        => \$user_ftp,
    'ftp_pwd=s'         => \$pwd_ftp,
    'scp_host=s'        => \$host_scp,
    'scp_dir=s'         => \$dir_scp,
    'scp_user=s'        => \$user_scp,
);

Help()
    if ((!$status)
    || ($help)
    || (!defined $report)
    || (!@devices)
    || (!@services)
    || (!defined $begin)
    || (!defined $end)
    || (!defined $output));

my $pid = $$;
$cache = Octopussy::Cache::Init('octo_reporter');

Progress('_MSG_REPORT_PROGRESS_REPORT_CONFIG', '1/1');
$rc = (defined $report ? Octopussy::Report::Configuration($report) : undef);
print "Error: Unable to get configuration for Report [$report]\n"
    if (!defined $rc);

$lang ||= 'EN';

my $time = time;
if (!$quiet)
{
    print "Report: $rc->{name}\n";
    print "Description: $rc->{description}\n";
}
my $pid_name = $PROG_NAME . (defined $pid_param ? "_$pid_param" : '');
$file_pid = Octopussy::PID_File($pid_name);
my $started = Date::Manip::ParseDateString("epoch $time");
my %info    = (
    report    => $rc->{name},
    started   => Date::Manip::UnixDate($started, '%Y/%m/%d %H:%M'),
    devices   => \@devices,
    services  => \@services,
    pid_param => $pid_param,
);
$cache->set("info_$pid", \%info);

my $str_report_gen = sprintf
    'Report Generation: D=[%s] S=[%s] T=[%s]',
    join(',', @devices),
    join(',', @services),
    $rc->{table};
my $str_report_begin_end = "Report Generation: B=[$begin] E=[$end]";
my $str_report_type =
    "Report Generation: type=[$rc->{graph_type}] title=[$rc->{name}]";

AAT::Syslog::Message($PROG_NAME, $str_report_gen);
AAT::Syslog::Message($PROG_NAME, $str_report_begin_end);
AAT::Syslog::Message($PROG_NAME, $str_report_type);

if (!$quiet)
{
    print "$str_report_gen\n";
    print "$str_report_begin_end\n";
    print "$str_report_type\n";
}

my @field_list = ();
push @field_list, split /\s*,\s*/, $rc->{columns}
    if (NOT_NULL($rc->{columns}));
foreach my $ds ('datasource1', 'datasource2', 'datasource3')
{
    push @field_list, $rc->{$ds} if (NOT_NULL($rc->{$ds}));
}

Progress('_MSG_REPORT_PROGRESS_INIT_PLUGIN', '1/1');
Octopussy::Plugin::Init({lang => $lang}, @field_list);

#foreach my $e ($rc->{datasource1}, $rc->{datasource2}, $rc->{datasource3})
#	{ push(@field_list, $e)	if ($e ne ""); }
Progress('_MSG_REPORT_PROGRESS_INIT_DB', '1/1');

my ($nb_files, $nb_lines) =
    Insert_Data(\@devices, \@services, $taxonomy, $begin, $end, $rc->{table},
    $rc->{query}, \@field_list);

Progress('_MSG_REPORT_PROGRESS_QUERYING_DB', '0/1');
my $query = $rc->{query};
$query =~    #s/FROM(\.*[, ])($rc->{table})([, ]\.*)/FROM$1$2_$pid$3/i;
    s/FROM ($rc->{table})(.*)?/FROM $1_$pid$2/i;

#$query =~ s/ (\w+) JOIN(\.*[, ])($rc->{table})([, ]\.*)/ $1 JOIN$2$3_$pid$4/gi;

my @data = AAT::DB::Query($APPLI, $query);
Progress('_MSG_REPORT_PROGRESS_QUERYING_DB', '1/1');

AAT::DB::Table_Destruction($APPLI, $rc->{table} . "_$pid");

Progress('_MSG_REPORT_PROGRESS_BUILDING_REPORT', '0/1');
my %conf_mail = (recipients => $mail_recipients, subject => $mail_subject);
my %conf_ftp =
    (host => $host_ftp, dir => $dir_ftp, user => $user_ftp, pwd => $pwd_ftp);
my %conf_scp = (host => $host_scp, dir => $dir_scp, user => $user_scp);
my %report_stats = (
    nb_files        => $nb_files,
    nb_lines        => $nb_lines,
    nb_result_lines => scalar(@data),
    seconds         => (time() - $time),
);

Octopussy::Report::Generate(
    $rc,        $begin,     $end,           $output,
    \@devices,  \@services, \@data,         \%conf_mail,
    \%conf_ftp, \%conf_scp, \%report_stats, $lang
);
Progress('_MSG_REPORT_PROGRESS_BUILDING_REPORT', '1/1');
AAT::Syslog::Message($PROG_NAME, 'REPORTER_GENERATION_STATUS', 'Completed');
unlink $file_pid;
$cache->remove("info_$$");
$cache->remove("status_${pid_param}") if (defined $pid_param);

=head1 AUTHOR

Sebastien Thebert <octo.devel@gmail.com>

=head1 SEE ALSO

octo_dispatcher, octo_extractor, octo_parser, octo_uparser, octo_scheduler

=cut