xmltv-util 0.5.69-1 / usr / bin / tv_grab_es_laguiatv

#!/usr/bin/perl -w

=pod

=head1 NAME

tv_grab_es_laguiatv - Alternative TV grabber for Spain.

=head1 SYNOPSIS

tv_grab_es_laguiatv --help

tv_grab_es_laguiatv [--config-file FILE] --configure [--gui OPTION]

tv_grab_es_laguiatv [--config-file FILE] [--output FILE] [--days N]
           [--offset N] [--quiet]

tv_grab_es_laguiatv --list-channels

tv_grab_es_laguiatv --capabilities

tv_grab_es_laguiatv --version

=head1 DESCRIPTION

Output TV listings for spanish channels from www.laguiatv.com.
Supports analogue and digital (D+) channels.
The grabber relies on parsing HTML so it might stop working at any time.

First run B<tv_grab_es_laguiatv --configure> to choose, which channels you want
to download. Then running B<tv_grab_es_laguiatv> with no arguments will output
listings in XML format to standard output.

B<--configure> Prompt for which channels,
and write the configuration file.

B<--config-file FILE> Set the name of the configuration file, the
default is B<~/.xmltv/tv_grab_es_laguiatv.conf>.  This is the file written by
B<--configure> and read when grabbing.

B<--gui OPTION> Use this option to enable a graphical interface to be used.
OPTION may be 'Tk', or left blank for the best available choice.
Additional allowed values of OPTION are 'Term' for normal terminal output
(default) and 'TermNoProgressBar' to disable the use of XMLTV::ProgressBar.

B<--output FILE> Write to FILE rather than standard output.

B<--days N> Grab N days.  The default is 3.

B<--offset N> Start N days in the future.  The default is to start
from today.

B<--quiet> Suppress the progress messages normally written to standard
error.

B<--capabilities> Show which capabilities the grabber supports. For more
information, see L<http://wiki.xmltv.org/index.php/XmltvCapabilities>

B<--version> Show the version of the grabber.

B<--help> Print a help message and exit.

=head1 SEE ALSO

L<xmltv(5)>.

=head1 AUTHOR

CandU, candu_sf@sourceforge.net, based on tv_grab_es, from Ramon Roca.

=head1 BUGS

=cut

# 


######################################################################
# initializations

use strict;
use XMLTV::Version '$Id: tv_grab_es_laguiatv,v 1.25 2015/03/24 17:56:38 bilbo_uk Exp $ ';
use XMLTV::Capabilities qw/baseline manualconfig cache/;
use XMLTV::Description 'Spain (laguiatv.com)';
use Getopt::Long;
use Date::Manip;
use HTML::TreeBuilder;
use HTML::Entities; # parse entities
use IO::File;
use DateTime;

use LWP::Simple;
use Encode;

use XMLTV;
use XMLTV::Memoize;
use XMLTV::ProgressBar;
use XMLTV::Ask;
use XMLTV::Config_file;
use XMLTV::DST;
use XMLTV::Get_nice 0.005065;
use XMLTV::Mode;
use XMLTV::Date;
# Todo: perhaps we should internationalize messages and docs?
use XMLTV::Usage <<END
$0: get Spanish television listings in XMLTV format
To configure: $0 --configure [--config-file FILE]
To grab listings: $0 [--config-file FILE] [--output FILE] [--days N]
        [--offset N] [--quiet]
To list channels: $0 --list-channels
To show capabilities: $0 --capabilities
To show version: $0 --version
END
  ;

# Attributes of the root element in output.
my $HEAD = { 'source-info-url'     => 'http://www.laguiatv.com/programacion/',
	     'source-data-url'     => "http://www.laguiatv.com/programacion/",
	     'generator-info-name' => 'XMLTV',
	     'generator-info-url'  => 'http://xmltv.org/',
	   };
		   
my $WRITE_ZERO_LENGTH = 0;  # whether zero-length programmes should be included in the output.
my $DO_SLOWER_DESC_GET = 0;
my $CONFIG_VERSION = 1; # default to v1 (v1 doesnt have version info)
my $EXPECTED_CONFIG_VERSION = 3;
my $CONFIG_USECACHE = 0;  # whether to use a disc cache for web pages
my $CONFIG_CACHEDIR;	# directory to store cached web pages

# default language
my $LANG="es";

# default web page encoding
my $WEB_ENCODING = 'iso-8859-15';

# Global channel_data
our @ch_all;

my @hide_channels = (
    "canal-bar.a", # currently gives 404 not found
);


######################################################################
# get options

# Get options, including undocumented --cache option.
XMLTV::Memoize::check_argv('XMLTV::Get_nice::get_nice_aux');
my ($opt_days, $opt_offset, $opt_help, $opt_output,
    $opt_configure, $opt_config_file, $opt_gui,
    $opt_quiet, $opt_list_channels, $opt_debug);
$opt_days  = 4; # default
$opt_offset = 0; # default
$opt_quiet  = 0; # default
$opt_debug  = 0; # default
GetOptions('days=i'        => \$opt_days,
	   'offset=i'      => \$opt_offset,
	   'help'          => \$opt_help,
	   'configure'     => \$opt_configure,
	   'config-file=s' => \$opt_config_file,
       'gui:s'         => \$opt_gui,
	   'output=s'      => \$opt_output,
	   'quiet'         => \$opt_quiet,
	   'list-channels' => \$opt_list_channels,
	   'debug'         => \$opt_debug,
	  )
  or usage(0);

# Force days to be 1, since we get all days at once
#		$opt_days = 1;
die 'number of days must not be negative'
  if (defined $opt_days && $opt_days < 0);
usage(1) if $opt_help;

# [mod Jan 2014 - max days is 4
die 'max days available is 4 (today + 3)'
  if ( $opt_offset + $opt_days > 4 );

XMLTV::Ask::init($opt_gui);


# Although we use HTTP::Cache::Transparent, this undocumented --cache
# option for debugging is still useful since it will _always_ use a
# cached copy of a page, without contacting the server at all.
#
use XMLTV::Memoize; XMLTV::Memoize::check_argv('XMLTV::Get_nice::get_nice_aux');


# debug print function
sub debug_print
{
	print STDERR $_[0]."\n" if $opt_debug;
}

my $mode = XMLTV::Mode::mode('grab', # default
			     $opt_configure => 'configure',
			     $opt_list_channels => 'list-channels',
			    );

# File that stores which channels to download.
my $config_file
  = XMLTV::Config_file::filename($opt_config_file, 'tv_grab_es_laguiatv', $opt_quiet);

my @config_lines; # used only in grab mode
if ($mode eq 'configure') {
    XMLTV::Config_file::check_no_overwrite($config_file);
}
elsif ($mode eq 'grab') {
    @config_lines = XMLTV::Config_file::read_lines($config_file);
}
elsif ($mode eq 'list-channels') {
    # Config file not used.
}
else { die }

# Whatever we are doing, we need the channels data.
my %channels = get_channels(); # sets @ch_all
my @channels;

my %icons;

my %categories = (
    "tag-a" => "Cine",
    "tag-b" => "Deportes",
    "tag-c" => "Programas",
    "tag-d" => "Series",
    "tag-e" => "Noticias"
);

######################################################################
# write configuration

if ($mode eq 'configure') {
    %channels = get_channels();
    
    open(CONF, ">$config_file") or die "cannot write to $config_file: $!";

    print CONF "configversion 3\n";
	
    # Ask about using a cache
    my $usecache = ask_boolean("Do you want to use a cache for web pages (recommended)", 'yes');
    warn("cannot read input, using default")
        if not defined $usecache;

    print CONF "usecache ";
    print CONF "yes\n" if $usecache;
    print CONF "no\n" if not $usecache;
	
	my $cachedir = "$ENV{HOME}/.xmltv/cache";
	if ($usecache)
	{
		my $cachedir = ask("Directory for cache (default=$cachedir)");
		warn("cannot read input, using default")
			if not defined $cachedir;
	} 
	print CONF "cachedir ".$cachedir."\n";
	
    # Ask about getting descs
    my $getdescs = ask_boolean("Do you want to get descriptions (very slow)", 'yes');
    warn("cannot read input, using default")
        if not defined $getdescs;

    print CONF "getdescriptions ";
    print CONF "yes\n" if $getdescs;
    print CONF "no\n" if not $getdescs;

    #my $cacheicons = ask_boolean('Do you want to get and cache icons during configure', 'yes');
    #warn("cannot read input, using default")
    #    if not defined $cacheicons;

    # Ask about each channel.
    my @chs = sort { $channels{$a} cmp $channels{$b} } keys %channels;
    my @names = map { $channels{$_} } @chs;
    my @qs = map { "Add channel $_?" } @names;
    my @want = ask_many_boolean(1, @qs);

    #my $iconbar = new XMLTV::ProgressBar({name => 'getting icon urls', count => scalar @chs})
    #if ((not $opt_quiet) && $cacheicons);

    foreach (@chs) {
	my $w = shift @want;
	warn("cannot read input, stopping channel questions"), last
	  if not defined $w;
	# No need to print to user - XMLTV::Ask is verbose enough.

	# Print a config line, but comment it out if channel not wanted.
	print CONF '#' if not $w;
	my $name = shift @names;
#        if ($cacheicons)
#        {
#            my $icon = get_icon($_);
#	    print CONF "channel $_ $name icon:$icon\n";
#        }
#        else
#        {
            print CONF "channel $_ ".encode($WEB_ENCODING, $name)."\n";
#        }
	# TODO don't store display-name in config file.

#        update $iconbar if ((not $opt_quiet) && $cacheicons);
    }

    close CONF or warn "cannot close $config_file: $!";
    say("Finished configuration.");

    exit();
}


# Not configuration, we must be writing something, either full
# listings or just channels.
#
die if $mode ne 'grab' and $mode ne 'list-channels';

# Options to be used for XMLTV::Writer.
my %w_args;
if (defined $opt_output) {
    my $fh = new IO::File(">$opt_output");
    die "cannot write to $opt_output: $!" if not defined $fh;
    $w_args{OUTPUT} = $fh;
}
$w_args{encoding} = 'UTF-8';
my $writer = new XMLTV::Writer(%w_args);
$writer->start($HEAD);

if ($mode eq 'list-channels') {
    $writer->write_channel($_) foreach @ch_all;
    $writer->end();
    exit();
}

######################################################################
# We are producing full listings.
die if $mode ne 'grab';

# Read configuration
my $line_num = 1;
foreach (@config_lines) {
    ++ $line_num;
    next if not defined;

    if (/configversion:?\s+(\S+)/)
    {
        $CONFIG_VERSION = $1;
    }
    elsif (/usecache:?\s+(\S+)/)
    {
		if($1 eq "yes")
        {
            $CONFIG_USECACHE = 1;
        }
    }
    elsif (/cachedir:?\s+(\S+)/)
    {
		$CONFIG_CACHEDIR = $1;
    }
    elsif (/getdescriptions:?\s+(\S+)/)
    {
        if("$CONFIG_VERSION" ne "$EXPECTED_CONFIG_VERSION")
        {
            die "Config file is out of date, please rerun with --configure\n";
        }
        if($1 eq "yes")
        {
            $DO_SLOWER_DESC_GET = 1;
        }
    }
    elsif (/^channel:?\s+(\S+)\s+([^#]+)icon\:([^#]+)/)
    {
        my $ch_did = $1;
        my $ch_name = $2;
        my $ch_icon = $3;


        #debug_print "Got channel $ch_name icon $ch_icon\n";
        $ch_name =~ s/\s*$//;
        push @channels, $ch_did;
        $channels{$ch_did} = $ch_name;
        $icons{$ch_did} = $ch_icon;
    }
    elsif (/^channel:?\s+(\S+)\s+([^#]+)/)
    {
        my $ch_did = $1;
        my $ch_name = $2;

        debug_print "Fetching channel $ch_name";
        $ch_name =~ s/\s*$//;
        push @channels, $ch_did;
        $channels{$ch_did} = $ch_name;
    }
    else {
	warn "$config_file:$line_num: bad line\n";
    }
}



if ($CONFIG_USECACHE) {
use HTTP::Cache::Transparent;
HTTP::Cache::Transparent::init( { 
    BasePath => $CONFIG_CACHEDIR,
    NoUpdate => 60*60,			# cache time in seconds
	MaxAge => 4,				# flush time in hours
    Verbose => $opt_debug,
} );
}




######################################################################
# begin main program

# Assume the listings source uses CET (see BUGS above).
my $now = DateCalc(parse_date('now'), "$opt_offset days");
die "No channels specified, run me with --configure\n"
  if not keys %channels;
my @to_get;

my $iconbar = new XMLTV::ProgressBar({name => 'getting channel info', count => scalar @channels})
  if not $opt_quiet;
# the order in which we fetch the channels matters
foreach my $ch_did (@channels) {
    my $ch_name=$channels{$ch_did};
    my $ch_xid="$ch_did.laguiatv.com";
#    my $ch_icon=$icons{$ch_did};
#    if (!$ch_icon)
#    {
#        $ch_icon = get_icon($ch_did);
#    }
#
#    if(index($ch_icon, "shim.gif") < 0)
#    {
#		$writer->write_channel({ id => $ch_xid,
#					 'display-name' => [ [ $ch_name ] ] ,
#					 'icon' => [ { 'src' => $ch_icon } ] });
#	}
#	else
#	{
		$writer->write_channel({ id => $ch_xid,
					 'display-name' => [ [ $ch_name ] ] });
#	}

	# [Jan 2014] - current website offers a fixed 4 days of data
	#	my $day=UnixDate($now,'%Q');
	#	for (my $i=0;$i<$opt_days;$i++) {
	#		push @to_get, [ $day, $ch_xid, $ch_did ];
	#		#for each day
	#		$day=nextday($day); die if not defined $day;
	#	}
	#
	push @to_get, [ '', $ch_xid, $ch_did ];
	
	update $iconbar if not $opt_quiet;
}

# This progress bar is for both downloading and parsing.  Maybe
# they could be separate.
#
my $bar = new XMLTV::ProgressBar({name => 'getting listings', count => scalar @to_get})
  if not $opt_quiet;
foreach (@to_get) {
	debug_print "process $_->[0], $_->[1], $_->[2]\n";
	foreach (process_table($_->[0], $_->[1], $_->[2])) {
		$writer->write_programme($_);
	}
	update $bar if not $opt_quiet;
}
$bar->finish() if not $opt_quiet;
$writer->end();

######################################################################
# subroutine definitions

# Use Log::TraceMessages if installed.
BEGIN {
    eval { require Log::TraceMessages };
    if ($@) {
	*t = sub {};
	*d = sub { '' };
    }
    else {
	*t = \&Log::TraceMessages::t;
	*d = \&Log::TraceMessages::d;
	Log::TraceMessages::check_argv();
    }
}

####
# process_table: fetch a URL and process it
#
# arguments:
#    Date::Manip object giving the day to grab
#    xmltv id of channel
#    elpais.es id of channel
#
# returns: list of the programme hashes to write
#
sub process_table {

    my ($date, $ch_xmltv_id, $ch_es_id) = @_;
		
	my $ch_conv_id = convert_id_to_laguiatvid($ch_es_id);
    my $today = UnixDate($date, '%d/%m/%Y');

    my $url = 'http://www.laguiatv.com/programacion/'.$ch_es_id.'.html';
	debug_print "Getting $url\n";
    t $url;
    local $SIG{__WARN__} = sub 
	{
		warn "$url: $_[0]";
	};

    # parse the page to a document object
	my $tree;
	#  HTML::Parse keeps reporting "Parsing of undecoded UTF-8 will give garbage when decoding entities" yet I can see no UTF8 in the pages!
	#    Save the page and run it again and you don't get the warning!
	#    You can't even supress the warning!  What a crock.
	{
		local $SIG{__WARN__} = sub {
			warn @_ unless (defined $_[0] && $_[0] =~ /^Parsing of undecoded UTF-/);
		};
		$tree = get_nice_tree($url,'',$WEB_ENCODING);
	}

    my @program_data = get_program_data($tree);
    my $bump_start_day=0;

    my @r;
    while (@program_data) {
	my $cur = shift @program_data;
	my $next = shift @program_data;
	unshift @program_data,$next if $next;
	
	my $p = make_programme_hash($date, $ch_xmltv_id, $ch_es_id, $cur, $next);
	if (not $p) {
	    require Data::Dumper;
	    my $d = Data::Dumper::Dumper($cur);
	    warn "cannot write programme on $ch_xmltv_id on $date:\n$d\n";
	}
	else {
	    push @r, $p;
	}

#	if (!$bump_start_day && bump_start_day($cur,$next)) {
#	    #$bump_start_day=1;
#	    $date = UnixDate(DateCalc($date,"+ 1 day"),'%Q');
#	}
    }
    return @r;
}

sub make_programme_hash {
    my ($date, $ch_xmltv_id, $ch_es_id, $cur, $next) = @_;

	#require Data::Dumper; debug_print Data::Dumper::Dumper($cur);
	
    my %prog;

    $prog{channel}=$ch_xmltv_id;
    $prog{title}=[ [ encode( 'UTF-8', $cur->{title} ), $LANG ] ];
    $prog{"sub-title"}=[ [ encode( 'UTF-8', $cur->{subtitle} ), $LANG ] ] if defined $cur->{subtitle};
    # $prog{category}=[ [ $cur->{category}, $LANG ] ];
	$prog{start}=$cur->{stime};
	$prog{stop} =$cur->{etime} if defined $cur->{etime};
    $prog{desc}=[ [ encode( 'UTF-8', $cur->{desc} ), $LANG ] ] if defined $cur->{desc};
    # $prog{category}=[ [ encode( 'UTF-8', $cur->{category} ), $LANG ] ] if defined $cur->{category};
	$prog{'date'} = $cur->{year}  if defined $cur->{year};
	$prog{'star-rating'} =  [ $cur->{rating} . '/5' ] if defined $cur->{rating};
	$prog{'rating'} =  [[ $cur->{classification}, '' ]] if defined $cur->{classification};

	if (defined $cur->{genres})
	{
		foreach ( @{ $cur->{genres} } )
		{
			push @{$prog{'category'}}, [ encode('UTF-8', $_), $LANG ]  if $_ ne '';
		}
	}
	if (defined $cur->{directors})
	{
		foreach ( @{ $cur->{directors} } )
		{
			push @{$prog{'credits'}{'director'}}, encode('UTF-8', $_)  if $_ ne '';
		}
	}
	if (defined $cur->{actors})
	{
		foreach ( @{ $cur->{actors} } )
		{
			push @{$prog{'credits'}{'actor'}}, encode('UTF-8', $_)  if $_ ne '';
    }
	}
						

    return \%prog;
}
sub bump_start_day {
    my ($cur,$next) = @_;
    if (!defined($next)) {
	return undef;
    }
    my $start = UnixDate($cur->{stime},'%H:%M');
    my $stop = UnixDate($next->{stime},'%H:%M');
    if (Date_Cmp($start,$stop)>0) {
	return 1;
    } else {
	return 0;
    }
}


#
sub get_program_data 
{
    my ($tree) = @_;
    my @data;

	my $today = DateTime->today->set_time_zone('Europe/Madrid');

	# - current website offers a fixed 4 days of data
	# ignore any programmes outside requested range
	my $startgrab = $today->clone->add('days' => $opt_offset)->epoch();
	my $stopgrab = $today->clone->add('days' => ($opt_offset + $opt_days))->epoch();
	debug_print 'Grab times: start: '.DateTime->from_epoch(epoch=>$startgrab)->strftime("%Y %m %d %H%M %S %z").' stop: '.DateTime->from_epoch(epoch=>$stopgrab)->strftime("%Y %m %d %H%M %S %z");
    # find schedule table

	# the following could could do with some error checking but I don't have time to do that right now  :-(
	
    my @divs = $tree->look_down('_tag' => 'div', 'id' => qr/dia1|nad2|nad3|nad4/);

    foreach my $div (@divs)
    {
		my ($i) = $div->attr('id') =~ /(?:dia|nad)(\d)/;
		#'debugtime'  debug_print "i= $i ".$div->attr('id');
		#'debugtime'  debug_print 'today: '.$today->strftime("%Y %m %d %H%M %S %z");
		my $theday = $today->clone->add(days => ($i - 1));
		#'debugtime'  debug_print 'theday: '.$theday->strftime("%Y %m %d %H%M %S %z");
		
		my @trs = $div->look_down('_tag' => 'tr');
	
		foreach my $tr (@trs)
        {

			my $stime = $tr->look_down('_tag' => 'th')->as_text;
			trim($stime);
	
			my $p_div = $tr->look_down('_tag' => 'div', 'class' => 'programa');
			next if !$p_div;
			
			my $a = $p_div->look_down('_tag' => 'a');
			
			my $p_url = $a->attr('href');
			my $p_title = $a->as_text;
			
			my $p_times = $p_div->look_down('_tag' => 'p')->as_text;
			
			my ($h, $i, $h2, $i2) = $p_times =~ /(\d*):(\d*)(?: *a *(\d*):(\d*))?/;
			
			my $showtime = $theday->clone->set(hour => $h, minute => $i, second => 0);
			
			# - current website offers a fixed 4 days of data
			# ignore any programmes outside requested range
			#'debugtime'  debug_print 'this: '.$showtime->strftime("%Y %m %d %H%M %S %z");
			next if ( $showtime->epoch() < $startgrab ) || ( $showtime->epoch() >= $stopgrab );
			
			my $p_stime = $theday->clone->set(hour => $h, minute => $i, second => 0)->strftime("%Y%m%d%H%M%S %z");
			
			my $p_etime;
			# this will probably fail around DST times
			if (defined $h2 && $h2 >= 0) 
            {
				$showtime->add(days => 1) if $h2 < $h; 
				eval {        # try
					$showtime->set(hour => $h2, minute => $i2, second => 0);
					$p_etime = $showtime->strftime("%Y%m%d%H%M%S %z");
				} or do {     # catch
					# no output prog 'stop' time
            }
			}
			

			# get descriptions?  Kinda compulsory now since there is no longer *any* description on the schedule page
			#
			my ($p_description, $p_rating, $p_classification, $p_year, @p_genres, @p_actors, @p_directors) = ('', '', '', '', (), (), ());		

			#
			{ # begin code block
			if ($DO_SLOWER_DESC_GET) 	# get descriptions 
            {

				my $url = $p_url;
				debug_print "Getting $url";
				t $url;

				last if $url eq 'javascript:void(0);' ;
				
				# handle no programme info situation (probably means "Close"?) :
				# <tr>
				#  <th scope="col"> 04:00</th>
				#  <td class="" data-type="programas">
				#    <div class="programa">
				#      <h2><a href="http://laguiatv.abc.es/programas/-72840/" title="*">*</a></h2>
				#      <p> 04:00 a  06:00</p>
				#    </div>
				#  </td>
			  #</tr>
				last if $url =~ m%programas/-\d*/$%;
				
				# parse the page to a document object
				#  HTML::Parse keeps reporting "Parsing of undecoded UTF-8 will give garbage when decoding entities" yet I can see no UTF8 in the pages!
				# Often on the http://hoycinema.abc.es/ pages. (Could be due to the <script> they have *before* the <meta Content-Type> ?)
				{
					local $SIG{__WARN__} = sub {
						warn @_ unless (defined $_[0] && $_[0] =~ /^Parsing of undecoded UTF-/);
					};
					$tree = get_nice_tree($url,'',$WEB_ENCODING);
				}
				
				my $div = $tree->look_down('_tag' => 'div', 'id' => 'contenedor');		# container
				$tree->dump if !$div;
				exit if !$div;
				
				# see if the title has a year
				# <h1 itemprop="name">&laquo;Fin de semana al desnudo&raquo; 
				#	<span><a href="/peliculas/1974.html" title="Películas del año 1974">(1974)</a></span>
				# </h1>
				my $h1 = $div->look_down('_tag' => 'h1', 'itemprop' => 'name');
				if ($h1)
				{
					my $a = $h1->look_down('_tag' => 'a');
					($p_year) = $a->as_text =~ /^\((19\d\d|20\d\d)\)$/  if $a;
				}	

				$div = $div->look_down('_tag' => 'div', 'class' => 'modulo');
				last if !$div;

				# get the various <dl> blocks
				my @dls = $div->look_down('_tag' => 'dl');
				foreach my $dl (@dls)
				{
					my $dt = $dl->look_down('_tag' => 'dt');
					next if !$dt;
					
					if ($dt->as_text =~ /Informaci.n/)
                {
						#<dl class="datos">
						#	<dt>Información:</dt>
						#	<dd class="calificacion">SC</dd>
						#	<dd itemprop="genre">Comedia</dd>
						#	<dd itemprop="duration">93 minutos</dd>
						#	<dd>Sin especificar</dd>    OR e.g.   <dd>EE.UU.</dd>
						#</dl>
						my $t = $dl->look_down('_tag' => 'dd', 'class' => 'calificacion');
						$p_classification = $t->as_text  if $t;
						#
						$t = $dl->look_down('_tag' => 'dd', 'itemprop' => 'genre');
						@p_genres = split(/,/, $t->as_text)  if $t;
					}
								
					if ($dt->as_text =~ /Director/)
					{	
						#<dl>
						#	<dt>Director:</dt>
						#	<dd itemprop="director" itemscope itemtype="http://schema.org/Person">
						#		<a href="/perfil-cine/peter-webber-97721/" title="Peter Webber"><span itemprop="name">Peter Webber</span></a>
						#		<a href="/perfil-cine/mariano-ozores-20092/" title="Mariano Ozores"><span itemprop="name">Mariano Ozores</span></a>
						#	</dd>
						#</dl>
						my @t = $dl->look_down('_tag' => 'span', 'itemprop' => 'name');
						foreach (@t)
						{
							push @p_directors, $_->as_text;
						}
					}
					
					if ($dt->as_text =~ /Int.rpretes/)
					{		
						#<dl class="interpretes" itemprop="actor" itemscope itemtype="http://schema.org/Person">
						#	<dt>Intérpretes:</dt>
						#	<dd>
						#		<a href="/perfil-cine/mandy-patinkin-4279/" title="Mandy Patinkin"><span itemprop="name">Mandy Patinkin</span></a>,
						#		<a href="/perfil-cine/alfredo-landa-15923/" title="Alfredo Landa"><span itemprop="name">Alfredo Landa</span></a>,
						#		<a href="/perfil-cine/thomas-gibson-18922/" title="Thomas Gibson"><span itemprop="name">Thomas Gibson</span></a>,
						#	</dd>
						#	<dd class="enlace"><a href="/peliculas/1974/fin-de-semana-al-desnudo-9814/reparto.html" title="Reparto completo">Reparto completo</a></dd>
						#</dl>
						my @t = $dl->look_down('_tag' => 'span', 'itemprop' => 'name');
						foreach (@t)
						{
							push @p_actors, $_->as_text;
						}
						# note: we could follow the "Reparto completo" link for the complete cast list
					}
					
					if ($dt->as_text =~ /Descripci.n/)
					{	
						#<dl>
						#	<dt>Descripción:</dt>
						#	<dd>Programa que repasa todas las noticias de interés general, nacionales, internacionales, así como deportivas. Incluye El tiempo. Presentador: Albert Martínez.</dd>
						#</dl>
						my $t = $dl->look_down('_tag' => 'dd');
						$p_description = $t->as_text  if $t;
					}
					
					if ($dt->as_text =~ /Sinopsis/)
					{	
						#<dl class="sinopsis">
						#	<dt>Sinopsis:</dt>
						#	<dd itemprop="description">La historia de la psicóloga Virginia Johnson (Lizzy Caplan) y el tímido ginecólogo William Masters (Michael Sheen),...<a href="/series/masters-of-sex-25058/sinopsis.html" title="Leer sinopsis completa">Leer sinopsis completa</a>.</dd>
						#</dl>
						#
						# For a *really* long description we could follow the 'Leer sinopsis completa' link but we won't do that until someone asks for it!  ;-)
						#
						if ($p_description eq '')
						{
							my $t = $dl->look_down('_tag' => 'dd', 'itemprop' => 'description');
							if ($t)
							{
								my $a = $t->look_down('_tag' => 'a');
								$a->detach  if $a;
								$p_description = $t->as_text;
							}
						}
					}
								

				}
				
				# if no description then try for a synopis
				#	<aside id="sinopsis"><h2>Sinopsis</h2><p itemprop="description">...
				if ($p_description eq '')
				{
					my $h2 = $tree->look_down('_tag' => 'h2', sub { $_[0]->as_text =~ /Sinopsis/ } );
					if ($h2) 
                    {
						my $p = $h2->right();
						$p_description = $p->as_text  if ( $p->tag() eq 'p' && $p->attr('itemprop') eq 'description' );
						#
						# website sometimes has invalid html (nested <p>) which treebuilder flattens
						# so append <p> siblings
						while (1)
						{
							$p = $p->right();
							last if ( $p->tag() ne 'p' );
							$p_description .= $p->as_text  if ( $p->tag() eq 'p' );
						}
					} 
				}
				
				# rating  (x/5)
				#	<meta itemprop="ratingValue" content="2.2"/>
				my $meta = $div->look_down('_tag' => 'meta', 'itemprop' => 'ratingValue');
				if ($meta) 
                        {
					$p_rating = $meta->attr('content');
                        }
				

                    }
			} # end code block

            #debug_print("title: $p_title start: $p_stime end: $p_etime cat: $p_category c2: " . $categories{$p_category} . "\n");
            debug_print("title: $p_title start: $p_stime end: ".(defined $p_etime?$p_etime:''));
			
			# 2014-04-02 ignore programme where title = *
			#     <h2>  <a href="javascript:void(0);" title="*">*</a>  </h2>
			#
			if($p_title && $p_title ne "" && $p_title ne "*" && $p_stime && $p_stime ne "")
			{
				my %h = ('stime' =>        $p_stime,
						 'etime' =>        $p_etime,
						 'title' =>        $p_title,
						 );
				$h{year} 			= $p_year if defined $p_year && $p_year ne "";
				$h{rating} 			= $p_rating if $p_rating ne "";
                    $h{desc} = $p_description if $p_description ne "";
				$h{classification} 	= $p_classification if $p_classification ne "";
				$h{directors} 		= \@p_directors if scalar @p_directors > 0;
				$h{actors} 			= \@p_actors if scalar @p_actors > 0;
				$h{genres} 			= \@p_genres if scalar @p_genres > 0;

                    push @data, \%h;

            }
            
        }
		
    }
    return @data;
}

sub get_icon 
{
    my ($ch_did) = @_;

    return "";
	
    my $url = "http://www.laguiatv.com/programacion/$ch_did";
	debug_print "Getting $url\n";
    t $url;
    local $SIG{__WARN__} = sub 
	{
		warn "$url: $_[0]";
	};

    my $content = get $url;
    my $pos = index($content, '<table class="grid cadena">');
    if($pos > 0)
    {
        $pos = index($content, '<img src="', $pos);
        if($pos > 0)
        {
            $pos += 10;
            my $end = index($content, '"', $pos);

            my $icon = 'http://www.laguiatv.com/' . substr($content, $pos, $end - $pos);

            debug_print "icon $icon\n";
            return $icon;
        }
    }

    return 'http://www.laguiatv.com/shim.gif';
}


sub get_prog_info
{
    my ($url) = @_;
    my $desc = "";
    my $cat = "";

    $url = "http://www.laguiatv.com/".$url;
    debug_print "Get proginfo $url\n";	

    my $content = get $url;
    my $pos = index($content, '<div class="intro-datasheet">');
    
    if($pos >= 0)
    {
        $pos = index($content, 'class="text">', $pos);
        if($pos >= 0)
        {
            my $divend = index($content, '</div', $pos);
            $pos = index($content, '<p', $pos);
	
            while($pos >= 0 && $pos < $divend)
            {
                $pos = index($content, '>', $pos) + 1;
                my $end = index($content, '</p>', $pos);
                if($end >= 0)
                {
                    $desc = $desc . substr($content, $pos, $end - $pos) . " ";
                }
                $pos = index($content, '<p', $pos);
            }
        }
    }

    decode_entities($desc);
    $desc =~ s/<\S+\s*\/*\/*>//g;
    $desc =~ s/\s+/ /g;
    $desc =~ s/\s+$//g;

    return ($desc, $cat);
}

sub get_txt_elems {
    my ($tree) = @_;

    my @txt_elem;
    my @txt_cont = $tree->look_down(
                        sub { ($_[0]->descendants() eq 0  ) },       
			sub { defined($_[0]->attr ("_content") ) } );
	foreach my $txt (@txt_cont) {
        	my @children=$txt->content_list;
		if (defined($children[0])) {
                  for (my $tmp=$children[0]) {
			s/^\s+//;s/\s+$//;
			push @txt_elem, $_;
                      }
                }
	}
    return @txt_elem;
}

# get channel listing
sub get_channels 
{
    my $bar = new XMLTV::ProgressBar({name => 'finding channels', count => 1})
	if not $opt_quiet;
    my %channels;
	
	# the front page is very big and slow to parse, so we'll
	# get channels via a dummy call to TVE 1 and then parse out the channel selector
    my $url="http://www.laguiatv.com/programacion/tve-1-807.html";
    t $url;

    my $channel_id;
    my $channel_name;
    my $channel_num;

    my $tree = get_nice_tree($url,'',$WEB_ENCODING);

    my @options = $tree->look_down('_tag' => 'select', 'id' => 'cadenas_programacion')->look_down('_tag' => 'option');

    foreach my $option (@options) 
            {
		next if !$option->attr('value');

		# <option value="tve-1-807">TVE 1</option>
		$channel_name = $option->as_text;
		$channel_id = $option->attr('value');
		($channel_num) = $option->attr('value') =~ /.*?-(\d+)$/;

		# remove channels that should not be listed
                my $hide = 0;
                foreach my $hide_id (@hide_channels)
                {
                    if($channel_id =~ m/$hide_id/)
                    {
                        $hide = 1;
                    }
                }

                if($hide == 0)
                { 
                    $channels{$channel_id}=$channel_name;
				debug_print "Got channel $channel_name with id $channel_id"  if $opt_list_channels;

				my $coded_chan_name=encode("utf-8",$channel_name);
				push @ch_all, { 
					'display-name' => [[ $coded_chan_name, $LANG ],[$channel_num]],
					'channel-num' => $channel_num,
					'id'=> "$channel_id.laguiatv" 
				};		
		}

    }

    die "no channels could be found" if not keys %channels;
    update $bar if not $opt_quiet;
    $bar->finish() if not $opt_quiet;
    return %channels;
}

sub convert_laguiatvid_to_id
{
    my ($str) = @_;


	$str =~ s/([^A-Za-z0-9])/sprintf("-%02X", ord("$1"))/seg;

	$str = "C" . $str;
	return $str;
}

sub convert_id_to_laguiatvid
{
    my ($str) = @_;

	# convert -20 to + (to replace spaces)
	$str =~ s/-20/+/g;

	# convert - to % for URL encoded chars
	$str =~ s/\-/%/g;

	# strip the C off the front
	$str = substr($str, 1);

	return $str;
}

# Bump a DDMMYYYY date by one.
sub nextday {
    my $d = shift;
    my $p = parse_date($d);
    my $n = DateCalc($p, '+ 1 day');
    return UnixDate($n, '%Q');
}

sub trim {
	# Remove leading & trailing spaces
	$_[0] =~ s/^\s+|\s+$//g;       
}

sub utf8 {
		# Catch the error:
		#    "Parsing of undecoded UTF-8 will give garbage when decoding entities 
		return decode('UTF-8', $_[0]); 
}