xmltv-util 0.5.69-1 / usr / bin / tv_grab_uk_tvguide

#!/usr/bin/perl -w
#
# Copyright G. Westcott - February 2013
#
# This code is distributed under the GNU General Public License v2 (GPLv2) .
#
#   For extended help information run
#         tv_grab_uk_tvguide  --info
# 

my $_version 	= '$Id: tv_grab_uk_tvguide,v 1.21 2017/01/22 22:01:26 bilbo_uk Exp $';


eval 'exec /usr/bin/perl -w -S $0 ${1+"$@"}'
    if 0; # not running under some shell

use Data::Dumper;

use strict;
use warnings;
use XMLTV::ProgressBar;
use XMLTV::Options qw/ParseOptions/;
use XMLTV::Configure::Writer;
use XMLTV::Supplement 0.005065 qw/SetSupplementRoot GetSupplementDir GetSupplementLines/;
use XMLTV::Get_nice 0.005065 qw/get_nice_tree/;

use File::Path;
use POSIX qw(strftime);
use DateTime;
use Date::Parse;
#v1.3: use DateTime::Format::DateParse;
use Encode;
use URI::Escape;
use HTML::TreeBuilder;

use HTTP::Cache::Transparent;


#require HTTP::Cookies;
#my $cookies = HTTP::Cookies->new;
#$XMLTV::Get_nice::ua->cookie_jar($cookies);


# Although we use HTTP::Cache::Transparent, this undocumented --cache
# option for debugging is still useful since it will _always_ use a
# cached copy of a page, without contacting the server at all.
#
use XMLTV::Memoize; XMLTV::Memoize::check_argv('XMLTV::Get_nice::get_nice_aux');

use subs qw(debug warning);
my $warnings = 0;

# ------------------------------------------------------------------------------------------------------------------------------------- #
# Grabber details
my $VERSION 								= $_version;
my $GRABBER_NAME 						= 'tv_grab_uk_tvguide';
my $GRABBER_DESC 						= 'UK - TV Guide (tvguide.co.uk)';
my $GRABBER_URL 						= 'http://wiki.xmltv.org/index.php/XMLTVProject';
my $ROOT_URL                = 'http://my.tvguide.co.uk/';
my $SOURCE_NAME							= 'TV Guide UK';
my $SOURCE_URL							= 'http://my.tvguide.co.uk/';
#
my $generator_info_name 		= $GRABBER_NAME;
my $generator_info_url 			= $GRABBER_URL;
my $source_info_name				= $SOURCE_NAME;
my $source_info_url					= $SOURCE_URL;



# ------------------------------------------------------------------------------------------------------------------------------------- #
# Use XMLTV::Options::ParseOptions to parse the options and take care of the basic capabilities that a tv_grabber should
my ($opt, $conf) = ParseOptions({ 
			grabber_name 			=> $GRABBER_NAME,
			capabilities 			=> [qw/baseline manualconfig apiconfig cache/],
			stage_sub 				=> \&config_stage,
			listchannels_sub 	=> \&fetch_channels,
			version 					=> $VERSION,
			description 			=> $GRABBER_DESC,
			extra_options			=> [qw/nodetailspage/],
});

#print Dumper($conf); exit;

# any overrides?
if (defined( $conf->{'generator-info-name'} )) { $generator_info_name = $conf->{'generator-info-name'}->[0]; }
if (defined( $conf->{'generator-info-url'} ))  { $generator_info_url  = $conf->{'generator-info-url'}->[0]; }
if (defined( $conf->{'source-info-name'} )) 	 { $source_info_name 		= $conf->{'source-info-name'}->[0]; }
if (defined( $conf->{'source-info-url'} ))  	 { $source_info_url 		= $conf->{'source-info-url'}->[0]; }



# ------------------------------------------------------------------------------------------------------------------------------------- #
# Initialise the web page cache
init_cachedir( $conf->{cachedir}->[0] );
HTTP::Cache::Transparent::init( { 
    BasePath => $conf->{cachedir}->[0],
    NoUpdate => 60*60,			# cache time in seconds
		MaxAge => 24,						# flush time in hours
    Verbose => $opt->{debug},
} );


# ------------------------------------------------------------------------------------------------------------------------------------- #
# Check we have all our required conf params
config_check();

# Load the conf file containing mapped channels and categories information
my %mapchannelhash;
my %mapcategoryhash;
loadmapconf();
#print Dumper(\%mapchannelhash, \%mapcategoryhash); exit;



# ------------------------------------------------------------------------------------------------------------------------------------- #
# Progress Bar :)
my $bar = new XMLTV::ProgressBar({
  name => "Fetching listings",
  count => (scalar @{$conf->{channel}}) * ($opt->{days} + 1)		# +1 added for the extra day necessary for <06:00 programmes
}) unless ($opt->{quiet} || $opt->{debug});



# ------------------------------------------------------------------------------------------------------------------------------------- #
# Data store before being written as XML
my $programmes = ();
my $channels = ();

# Get the schedule(s) from TV Guide
fetch_listings();

# print Dumper($programmes);

# Progress Bar
$bar->finish() && undef $bar if defined $bar;



# ------------------------------------------------------------------------------------------------------------------------------------- #
# Filter out programmes outside of requested period (see man page)
my %w_args;
if (($opt->{offset} != 0) || ($opt->{days} != -999)) {
  $w_args{offset} = $opt->{offset};
  $w_args{days} = ($opt->{days} == -999) ? 100 : $opt->{days};
  $w_args{cutoff} = '000000';			# e.g. '060000'
}



# ------------------------------------------------------------------------------------------------------------------------------------- #
# Generate the XML
my $encoding = 'UTF-8';
my $credits = { 'generator-info-name' => $generator_info_name,
								'generator-info-url' 	=> $generator_info_url,
								'source-info-name' 		=> $source_info_name,
								'source-info-url' 		=> $source_info_url };
	
XMLTV::write_data([ $encoding, $credits, $channels, $programmes ], %w_args);
# Finished!



# ------------------------------------------------------------------------------------------------------------------------------------- #
# Signal that something went wrong if there were warnings.
exit(1) if $warnings;

# All data fetched ok.
debug "Exiting without warnings.";
exit(0);


# #############################################################################
# # THE MEAT #####################################################################
# ------------------------------------------------------------------------------------------------------------------------------------- #

sub fetch_listings {
		# Fetch listings per channel
		
		foreach my $channel_id (@{$conf->{channel}}) {
			# Officially:
			# http://www.tvguide.co.uk/channellisting.asp?cTime=3%2F19%2F2013+06%3A00%3A00+&ch=857&go=go
			# But this works too:
			# http://www.tvguide.co.uk/channellisting.asp?ch=86&cTime=3/18/2013
			
			my $baseurl = $ROOT_URL.'channellisting.asp';
			
			# Now grab listings for each channel on each day, according to the options in $opt
			#
			# tvguide runs from 06:00 so we need to get the previous day as well just for any programmes after midnight
			#
			for (my $i=($opt->{offset} -1); $i < ($opt->{offset} + $opt->{days}); $i++) {
				my $theday = DateTime->today->add (days => $i)->set_time_zone('Europe/London');
			
				# Construct the listings url
				my $url = $baseurl . '?ch=' . $channel_id . '&cTime=' . uri_escape( $theday->strftime('%m/%d/%Y 00:00:00') );
				#debug "Fetching: $url";

				# If we need to map the fetched channel_id to a different value
				my $xmlchannel_id = $channel_id;
				if (defined(&map_channel_id)) { $xmlchannel_id = map_channel_id($channel_id); }
				
				# Fetch the page
				#   my $tree = XMLTV::Get_nice::get_nice_tree($url);
				my $tree = fetch_url($url);
				# $tree->dump; exit;

				# Scrub the page
				if ($tree) {
					my $channelname = $tree->look_down('_tag' => 'option', 'value' => $channel_id);

					# tvguide website can be very slow - try to avoid barfing when no response
					if (!defined $channelname) {
						print STDERR "Unable to retrieve web page for $channel_id \n";
						next;
					}

					$channelname = $channelname->as_text;
					
					# 	<table border="0" cellpadding="0" style="background:black;border-collapse: collapse;background-image: url(http://i.g8.tv/HighlightImages/Large/);background-repeat: no-repeat;" width="677">

					my @shows = $tree->look_down('_tag' => 'table', 'border' => '0', 'cellpadding' => '0', 'style' => qr/background:\s*black;border-collapse:\s*collapse;/);
					
					if (@shows) {
						foreach my $show (@shows) {
							#	$show->dump;
							
							# are we processing yesterday's schedule? (see above)
							if ($i == ($opt->{offset} -1)) {
								my $showstart = $show->look_down('_tag' => 'span', 'class' => 'tvchannel');
								my ($h, $i, $a) = $showstart->as_text =~ /(\d*):(\d*)\s*(am|pm)/;
								# 2014-04-02  see note below
								if (!defined $a) {
									$showstart = $show->look_down('_tag' => 'span', 'class' => 'season');
									($h, $i, $a) = $showstart->as_text =~ /(\d*):(\d*)\s*(am|pm)/;
								}
								if ($a eq 'am' && ($h < 6 || $h == 12)) {
									# continue processing of pre-6am programme
								} else {
									next;
								}
								$showstart = $h = $i = $a = undef;
							}
							
							my %prog = ();
								
							my $showtime;
							
							# see if we have a details page
							#		<a href="javascript:popup(151361219);" ...
							#		http://www.tvguide.co.uk/detail.asp?id=151451760
							# 2013-12-14 site changed
							#    <a href="javascript:popupshow('http://www.tvguide.co.uk/detail/1889599/94819598/saturday-kitchen-live');" target="_blank" ...
							# 2014-04-02 site changed
							#    <a href="http://www.tvguide.co.uk/detail/138990373/88745969/breakfast" target="_blank"...
							# 2014-11-26 site changed
							#    <a href="http://watch.tvguide.co.uk/engage/2057116/103685857-fake_britain" target="_blank"...
							# 2014-12-03 site changed
							#  my $webdetails = $show->look_down('_tag' => 'a', 'href' => qr/javascript:popup/);
							# 2014-12-03 The new website seems a bit flakey with these details pages, often returning a 500 Server Error
							#    Here's an option to disable the details pages  (  --nodetailspage )
							# 2014-12-24 site changed
							#  my $webdetails = $show->look_down('_tag' => 'a', 'href' => qr/\/engage\//);
							#
						if (!$opt->{nodetailspage}) {
							
							my $webdetails = $show->look_down('_tag' => 'a', 'href' => qr/\/detail\//);
							my $href = $webdetails->attr('href');
							#    my ($id) = $href =~ /javascript:popup\((\d*)\);/;
							#    $url = $ROOT_URL . 'detail.asp?id=' . $id;
							#    my ($url) = $href =~ /javascript:popupshow\('(.*)'\);/;
							my ($url) = $href;
							#debug "Fetching: $url";

							# Fetch the page
							#   my $treedetail = XMLTV::Get_nice::get_nice_tree($url);
							my $treedetail = fetch_url($url);
							
							my $showdetail = $treedetail;						
							#	$showdetail->dump;
							
							if ($showdetail) {
								# Details page contains Director names and a better list of Actors

									# Get the cast and extract them into a new tree
									my @lis = $showdetail->look_down('_tag' => 'div', 'class' => 'cast-entry');		

									LOOP:
									foreach my $person (@lis) {
										#
										# 30/6/16
										#   <div class="cast-entry">
										#       <span class="role">Margaret Sellinger</span> 
										#       <a href="http://www.tvguide.co.uk/actor.asp?actor=Lesley-Anne Down" target="_blank">
										#          <span itemprop="actor" itemscope itemtype="http://schema.org/Person">
										#             <span class="actor" itemprop="name">Lesley-Anne Down</span> 
										#          </span>
										#       </a>
										#       <a target="_blank" href="http://uk.imdb.com/find?s=nm&q=Lesley-Anne+Down"><span class="actor">(IMDB)</span></a> 
										#   </span>
										#   </div>

										my ($name, $role);
										if ( my ($_name) = $person->look_down('_tag' => 'span', 'itemprop' => 'name') ) {
											$name = $_name->as_text;
										}
										if ( my ($_role) = $person->look_down('_tag' => 'span', 'class' => 'role') ) {
											$role = $_role->as_text;
										}
										#	drop the "Executive Director" & "Executive Producer"   - any others we should drop?
										next LOOP  if ( $role =~ /^(Executive Director|Executive Producer)/ );

										# map the website role to an xmltv role
										my %xmltvroles = ( 'Director'=>'director', 'Producer'=>'producer', 'Series Producer'=>'producer', 'Writer'=>'writer', 'Co-Director'=>'director', 'Presenter'=>'presenter', 'Commentator'=>'commentator', 'Guest'=>'guest' );
										
										my $credit;
										if (exists $xmltvroles{$role}) {
											$credit = $xmltvroles{$role};
								} else {
											$credit = 'actor';
								}
								
										if ($credit eq 'actor' && defined $role) {
											push @{$prog{'credits'}{$credit}}, [ encode('utf-8', $name), encode('utf-8', $role) ];
										} else {
											push @{$prog{'credits'}{$credit}}, encode('utf-8', $name);
								}
								
								}
								
									
								# Get the "Left Panel" which contains the programme times and attributes
								my $lhs = $showdetail->look_down('_tag' => 'div', 'class' => qr/divLHS-section-2/);

									
								# Get the programme's "attributes" e.g. "Certificate"
								if ($lhs) {
									my @attrs = $lhs->look_down('_tag' => 'span', 'class' => 'LHS-attribute');
									if (@attrs) {
										foreach my $attr (@attrs) {
											# $attr->dump;
											if ( my $showattr = $attr->as_text() ) {												
												if ( $showattr =~ /^Certificate\s:\s(.*)\s*$/ ) { $prog{'rating'} = [[ $1, 'BBFC' ]]; }
											}
										}
									}
								}
								
								
								# start time, and stop time (actually an optional DTD element)
								#		<span class="datetime">10:00am-11:50am <span class=programmetext> (1 hour 50 minutes)</span> Wed 20 Mar</span>
								# (use the Date provided to avoid issues with the site running from 06:00-06:00)
								#
								# Note site displays stop time wrong on GMT/BST changeover, e.g.:
								#			12:45am-1:10am (25 minutes) Sun 31 Mar
								# 	this should be 12:45am-2:10am (BST)
								# 	this makes $showtime->set barf on "invalid local time for date in timezone"
								#
								#  1/Jan/17 times are now in the left panel. 'datetime' is used for user comments!
								#     my $showtimes = $showdetail->look_down('_tag' => 'span', 'class' => 'datetime');
								#  
								if ($lhs) {

									# Unfortunately the div with the date doesn't have any safe identifier. There are several ways we could remove the
									#  cruft from the container but the following, although clunky, is probably the safest
									my ($dt, $h, $i, $a, $h2, $i2, $a2) = $lhs->as_text =~ /((?:Mon|Tue|Wed|Thu|Fri|Sat|Sun|Christmas|Boxing Day|New Year).*?)(\d*):(\d*)(am|pm)(?:-(\d*):(\d*)(am|pm))?/;
									# print STDERR $dt."\n"; 

									if ($dt && $dt !~ /\D\D\D\s\d\d?\s\D\D\D/) {
										my @thedt = localtime(time);	# ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)
										my ($yr1, $yr2) = ($thedt[5]+1900, $thedt[5]+1900);
										if ($thedt[4] == 11) { $yr2++; }
										if ($thedt[4] == 0) { $yr1--; }
										SWITCH: {
											$dt =~ 'Christmas\s+Eve' 		&& do { $dt = '24 Dec '.$yr1; last SWITCH; };
											$dt =~ 'Christmas\s+Day' 		&& do { $dt = '25 Dec '.$yr1; last SWITCH; };
											$dt =~ 'Boxing\s+Day' 			&& do { $dt = '26 Dec '.$yr1; last SWITCH; };
											$dt =~ 'New\s+Years\s+Eve' 		&& do { $dt = '31 Dec '.$yr1; last SWITCH; };
											$dt =~ 'New\s+Years\s+Day' 		&& do { $dt = '1 Jan ' .$yr2; last SWITCH; };
											undef $dt;
										}
									}
									if ($dt) {
										 #v1.3: $showtime = DateTime::Format::DateParse->parse_datetime( $dt, 'Europe/London' );
										 
										 # workaround for bug in Date::Time::str2time() which generates wrong dates for future months when no year is given
										 #		https://rt.cpan.org/Public/Bug/Display.html?id=92611
										 $dt .= ' '.( (localtime(time))[5]+1900 )  if $dt !~ /(19|20)\d\d/;
									
										 $showtime = DateTime->from_epoch( epoch=>str2time( $dt, 'Europe/London' ) )->set_time_zone('Europe/London');
									} else {
										 $showtime = $theday->clone;
									}
									$h  += 12 if $a  eq 'pm' && $h   < 12;		# e.g. 12:30pm means 12:30 !
									$h  -= 12 if $a  eq 'am' && $h  == 12;		# e.g. 12:30am means 00:30 !
									$h2 += 12 if $a2 eq 'pm' && $h2  < 12;	
									$h2 -= 12 if $a2 eq 'am' && $h2 == 12;	
									$showtime->set(hour => $h, minute => $i, second => 0);
									$prog{'start'} = $showtime->strftime("%Y%m%d%H%M%S %z");
									if (defined $h2 && $h2 >= 0) {
										$showtime->add (days => 1) if $h2 < $h; 
										# see note above re errors with GMT/BST transition
										eval {				# try
											$showtime->set(hour => $h2, minute => $i2, second => 0);
											$prog{'stop'} = $showtime->strftime("%Y%m%d%H%M%S %z");
										} or do {			# catch
											# no output prog 'stop' time
										}
									} else {
										# no output prog 'stop' time
									}
								}
														
									
									undef @lis;
								
							}	# end showdetail
						}
								
								
							# channel
							$prog{'channel'} = $xmlchannel_id;
							
							# title (mandatory)
							#		<span class="programmeheading">Baywatch</span>
							my $showtitle = $show->look_down('_tag' => 'span', 'class' => 'programmeheading');
							$prog{'title'} = [[ encode('utf-8', $showtitle->as_text), 'en' ]];
							$showtitle->detach;
							
							
							# Note: <span class="tvchannel"> is used by StartTime then SubTitle then Category then Subtitles/B&W/etc
							
							# start (mandatory)
							#		<span class="tvchannel">3:00pm </span>
							# (don't add it even we already have it from the detail page but we still need to delete it from the tree)

							# @ 2014-04-02 the site has changed to 
							#                         <span class="season">6:00 am </span>
							# but this just doesn't sound right to me (i.e. I think it might change again), so let's try both ways
							#
							# @2016-08-05 looks like this is permanent
							# my $showstart = $show->look_down('_tag' => 'span', 'class' => 'tvchannel');
							my $showstart = $show->look_down('_tag' => 'span', 'class' => 'season');
							if (!$prog{'start'}) {
								my ($h, $i, $a) = $showstart->as_text =~ /(\d*):(\d*)\s*(am|pm)/;
								if (!defined $a) {
									$showstart = $show->look_down('_tag' => 'span', 'class' => 'season');
									($h, $i, $a) = $showstart->as_text =~ /(\d*):(\d*)\s*(am|pm)/;
								}
								$h += 12 if $a eq 'pm' && $h  < 12;		# e.g. 12:30pm means 12:30 !
								$h -= 12 if $a eq 'am' && $h == 12;		# e.g. 12:30am means 00:30 !
								$showtime = $theday->clone;
								$showtime->set(hour => $h, minute => $i, second => 0);
								$showtime->add (days => 1) if ($h < 6);		# site runs from 06:00-06:00 so anything <06:00 is for tomorrow
								$prog{'start'} = $showtime->strftime("%Y%m%d%H%M%S %z");
								# no prog 'stop' time available
							}
							$showstart->detach;
							
							# category
							#		<span class="tvchannel">Category </span><span class="programmetext">General Movie/Drama</span>
							my $showcategory = $show->look_down('_tag' => 'span', 'class' => 'tvchannel', sub { $_[0]->as_text =~ /^Category\s*$/ } );
							if ($showcategory) {
								$showcategory = $showcategory->right;
								my @showcategory = split(/\//, $showcategory->as_text);
								my @showcategories = ();
								foreach my $category (@showcategory) {
									# category translation?
									if (defined(&map_category)) { $category = map_category($category); }
									if ($category =~ /\|/) { 
										foreach my $cat (split(/\|/, $category)) { push @showcategories, $cat unless grep(/$cat/, @showcategories); }
									} elsif ($category ne '') {
										push @showcategories, $category unless grep(/$category/, @showcategories);
									}
								}
								foreach my $category (@showcategories) {
									push @{$prog{'category'}}, [ encode('utf-8', $category), 'en' ];
								}
								$showcategory->left->detach;
							}
							
							# desc
							#		<span class="programmetext">Dissolving bikinis cause a stir on the beach</span>
							my $showdesc = $show->look_down('_tag' => 'span', 'class' => 'programmetext');
							if ($showdesc) {
								$showdesc = $showdesc->as_text;
								$showdesc .= '.' if ( (length $showdesc) && (substr $showdesc,-1,1 ne '.') );	# addend a fullstop
								if (length $showdesc) {
									$prog{'desc'} = [[ encode('utf-8', $showdesc), 'en' ]];
								}
							}

							# year
							# 	strip this off the title e.g. "A Useful Life (2010)"
							my ($showyear) = $prog{'title'}->[0][0] =~ /.*\((\d\d\d\d)\)$/;
							if ($showyear) {
								$prog{'date'} = $showyear;
								# assume anything with a year is a film - add Films category group
								push @{$prog{'category'}}, [ 'Films', 'en' ];
							}
							
							# flags
							# 		<span class='tvchannel'>(Subtitles)</span> <span class='tvchannel'>(Black &amp; White)</span>
							my $showflags = $show->look_down('_tag' => 'span', 'class' => 'tvchannel', sub { $_[0]->as_text =~ /Subtitles/ } );
							if ($showflags) {
								push @{$prog{'subtitles'}}, {'type' => 'teletext'};
								$showflags->detach;
							}
							#$showflags = $show->look_down('_tag' => 'span', 'class' => 'tvchannel', sub { $_[0]->as_text =~ /Audio Described/ } );
							#if ($showflags) {
							#	push @{$prog{'subtitles'}}, {'type' => 'deaf-signed'};    <-- Audio Described is not deaf-signed
							#	$showflags->detach;
							#}
							$showflags = $show->look_down('_tag' => 'span', 'class' => 'tvchannel', sub { $_[0]->as_text =~ /Repeat/ } );
							if ($showflags) {
	#	push @{$prog{'previously-shown'}}, {};
								$prog{'previously-shown'} = {};
								$showflags->detach;
							}
							my $showvideo = $show->look_down('_tag' => 'span', 'class' => 'tvchannel', sub { $_[0]->as_text =~ /Black & White/ } );
							if ($showvideo) {
								$prog{'video'}->{'colour'} = '0';
								$showvideo->detach;
							}
							#if ($showflags && $showflags->as_text =~ '\[REP\]') {
							#	push @{$prog{'previously-shown'}}, {};
							#}
							$showflags = $show->look_down('_tag' => 'span', 'class' => 'tvchannel', sub { $_[0]->as_text =~ /Interactive/ } );
							if ($showflags) {
								# no flag in DTD for this
								$showflags->detach;
							}
						
						
							# episode number
							# 	<span class="season">Season 2 </span> <span class="season">Episode 3 of 22</span>
							my @showepisode = $show->look_down('_tag' => 'span', 'class' => 'season');
							my $showepisode;
							foreach my $el (@showepisode) { 
								$showepisode .= $el->as_text;
							}
							if ($showepisode) {
								my ($showsxx, $showexx, $showeof) = ( $showepisode =~ /^(?:(?:Series|Season) (\d+)(?:[., :]+)?)?(?:Episode (\d+)(?: of (\d+))?)?/ );
								# scan the description for any "Part x of x." info
								my ($showpxx, $showpof) = ('', '');
								($showpxx, $showpof) = ( $showdesc =~ /Part (one|two|three|four|five|six|seven|eight|nine|\d+)(?: of (one|two|three|four|five|six|seven|eight|nine|\d+))?/ ) if ($showdesc);
								my $showepnum = make_ns_epnum($showsxx, $showexx, $showeof, $showpxx, $showpof);
								if ($showepnum && $showepnum ne '...') {
									$prog{'episode-num'} = [[ $showepnum, 'xmltv_ns' ]];
								}
								#debug "--$showepnum-- ".$showepisode->as_text;
							}
							
							# episode title
							# 	<span class="tvchannel">The Fabulous Buchannon Boys</span>
							my $showeptitle = $show->look_down('_tag' => 'span', 'class' => 'tvchannel');
							if ($showeptitle) {
							  if ($showeptitle->as_text =~ /\(?Premiere\)?/) {
									$prog{'premiere'} = [];
							  } else {									
								$prog{'sub-title'} = [[ encode('utf-8', $showeptitle->as_text), 'en' ]];
								}
								$showeptitle->detach;
							}
						
							# director						
							# never seen one but let's assume they're in the description
							if (!$prog{'credits'}->{'director'}) {
								if ($showdesc) {
									my ($directors) = ( $showdesc =~ /(?:Directed by|Director) ([^\.]*)\.?/ );
									if ($directors) {
										$directors =~ s/ (with|and) /,/ig;
										$directors =~ s/ (singer|actor|actress) //ig;			# strip these words
										$directors =~ s/,,/,/g;	# delete empties
										$directors = encode('utf-8', $directors);	# encode names into utf-8
										#debug $directors;
										my @directors = split(/,/, $directors);
										s{^\s+|\s+$}{}g foreach @directors;	# strip leading & trailing spaces
										$prog{'credits'}->{'director'} = \@directors  if (scalar @directors > 0);
									}
								}
							}
							
							# actors
							# these are buried in the description  :-(
							if (!$prog{'credits'}->{'actor'}) {			
								if ($showdesc) {
									my ($actors) = ( $showdesc =~ /(?:starring)([^\.]*)\.?/i );
									if ($actors) {
										$actors =~ s/ (also|starring|with|and) /,/ig;		# may be used to separate names
										$actors =~ s/ (singer|actor|actress) //ig;			# strip these words
										$actors =~ s/,,/,/g;	# delete empties
										$actors = encode('utf-8', $actors);	# encode names into utf-8
										#debug $actors;
										my @actors = split(/,/, $actors);
										s{^\s+|\s+$}{}g foreach @actors;	# strip leading & trailing spaces
										$prog{'credits'}->{'actor'} = \@actors  if (scalar @actors > 0);
									}
								}
							}
							
							# rating
							#		<span class="programmetext">Rating<br></span><span class="programmeheading">3.9</span>
							my $showrating = $show->look_down('_tag' => 'span', 'class' => 'programmetext', sub { $_[0]->as_text =~ /Rating/ } );
							if ($showrating) {
								$showrating = $showrating->right;
								$showrating = $showrating->right if ($showrating->tag eq 'br');
								if ($showrating->tag eq 'span' && $showrating->attr('class') eq 'programmeheading') {
									if ($showrating->as_text) {
										$prog{'star-rating'} =  [ $showrating->as_text . '/10' ];
									}
								}
							}
							
				
							# debug Dumper \%prog;
							push(@{$programmes}, \%prog);
						}
						

					} else {
						# no schedule found
						warning 'No schedule found';
					}
					
					undef @shows;
					
					# Add to the channels hash
					$channels->{$channel_id} = { 'id'=> $xmlchannel_id , 'display-name' => [[$channelname, 'en']]  };

				} else {
					# tree conversion failed
					warning 'Could not parse the page';
				}

				$bar->update if defined $bar;
			}
		}
}


# #############################################################################
# # THE VEG ######################################################################
# ------------------------------------------------------------------------------------------------------------------------------------- #

sub make_ns_epnum {
		# Convert an episode number to its xmltv_ns compatible - i.e. reset the base to zero
		# Input = series number, episode number, total episodes,  part number, total parts,
		#  e.g. "1, 3, 6, 2, 4" >> "0.2/6.1/4",    "3, 4" >> "2.3."
		#
		my ($s, $e, $e_of, $p, $p_of) = @_;
		#	debug Dumper(@_);

		# "Part x of x" may contaain integers or words (e.g. "Part 1 of 2", or "Part one")
		$p = text_to_num($p) if defined $p;
		$p_of = text_to_num($p_of) if defined $p_of;
		
		# re-base the series/episode/part numbers
		$s-- if (defined $s && $s > 0);
		$e-- if (defined $e && $e > 0);
		$p-- if (defined $p && $p > 0);
		
		# make the xmltv_ns compliant episode-num
		my $episode_ns = '';
		$episode_ns .= $s if defined $s;
		$episode_ns .= '.';
		$episode_ns .= $e if defined $e;
		$episode_ns .= '/'.$e_of if defined $e_of;
		$episode_ns .= '.';
		$episode_ns .= $p if defined $p;
		$episode_ns .= '/'.$p_of if defined $p_of;
		
		#debug "--$episode_ns--";
		return $episode_ns;
}
	
sub text_to_num {
		# Convert a word number to int e.g. 'one' >> '1'
		#
		my ($text) = @_;
		if ($text !~ /^[+-]?\d+$/) {	# standard test for an int
			my %nums = (one => 1, two => 2, three => 3, four => 4, five => 5, six => 6, seven => 7, eight => 8, nine => 9);
			return $nums{$text} if exists $nums{$text};
		}
		return $text
}

sub map_channel_id {
		# Map the fetched channel_id to a different value (e.g. our PVR needs specific channel ids)
		# mapped channels should be stored in a file called  tv_grab_uk_guardian.map.conf
		# containing lines of the form:  map==fromchan==tochan  e.g. 'map==5-star==5STAR'
		#
		my ($channel_id) = @_;
		my $mapchannels = \%mapchannelhash;
		if (%mapchannelhash && exists $mapchannels->{$channel_id}) { 
			return $mapchannels->{$channel_id} ; 
		}
		return $channel_id;
}

sub map_category {
		# Map the fetched category to a different value (e.g. our PVR needs specific genres)
		# mapped categories should be stored in a file called  tv_grab_uk_guardian.map.conf
		# containing lines of the form:  cat==fromcategory==tocategory  e.g. 'cat==General Movie==Film'
		#
		my ($category) = @_;
		my $mapcategories = \%mapcategoryhash;
		if (%mapcategoryhash && exists $mapcategories->{$category}) { 
			return $mapcategories->{$category} ; 
		}
		return $category;
}

sub loadmapconf {
		# Load the conf file containing mapped channels and categories information
		# 
		# This file contains 2 record types:
		# 	lines starting with "map" are used to 'translate' the incoming channel id to those required by your PVR
		#			e.g. 	map==dave==DAVE     will output "DAVE" in your XML file instead of "dave"
		# 	lines starting with "cat" are used to translate categories (genres) in the incoming data to those required by your PVR
		# 		e.g.  cat==Science Fiction==Sci-fi			will output "Sci-Fi" in your XML file instead of "Science Fiction"
		# 
		my $mapchannels = \%mapchannelhash;
		my $mapcategories = \%mapcategoryhash;
		#		
		my $supplementdir = $ENV{XMLTV_SUPPLEMENT} || GetSupplementDir();
		SetSupplementRoot($supplementdir);
		my $lines = GetSupplementLines($GRABBER_NAME, $GRABBER_NAME . '.map.conf');
		foreach my $line (@$lines) { 
				my ($type, $mapfrom, $mapto, $trash) = $line =~ /^(.*)==(.*)==(.*?)([\s\t]*#.*)?$/;
				SWITCH: {
						lc($type) eq 'map' && do { $mapchannels->{$mapfrom} = $mapto; last SWITCH; };
						lc($type) eq 'cat' && do { $mapcategories->{$mapfrom} = $mapto; last SWITCH; };
					warning "Unknown type in map file: \n $line";
				}
			}
		# debug Dumper ($mapchannels, $mapcategories);
}

sub fetch_channels {
	my ($opt, $conf) = @_;
  
	# Fetch channels via a dummy call to BBC1 listings
	#		http://www.tvguide.co.uk/channellisting.asp?ch=86&cTime=
	my $channel_list = $ROOT_URL.'channellisting.asp?ch=86&cTime=';

  my $result;
  my $channels = {};

  my $bar = new XMLTV::ProgressBar({
    name => "Fetching channels",
    count => 1
  }) unless ($opt->{quiet} || $opt->{debug});

  # Get the page containing the list of channels 
  my $tree = XMLTV::Get_nice::get_nice_tree($channel_list);
	#		$tree->dump;
  my $_channels = $tree->look_down('_tag' => 'select', 'name' => 'ch');
	my @channels = $_channels->look_down('_tag' => 'option');
			#		debug $_channels->as_HTML;
			#		foreach  my $xchannel (@channels) { debug $xchannel->as_HTML; }

  $bar->update() && $bar->finish && undef $bar if defined $bar;

  $bar = new XMLTV::ProgressBar({
    name => "Parsing result",
    count => scalar @channels
  }) unless ($opt->{quiet} || $opt->{debug});

  # Browse through the downloaded list of channels and map them to a hash XMLTV::Writer would understand
  foreach my $channel (@channels) {
    if ($channel->as_text) {
      my ($id) = $channel->attr('value');
			my ($url) = 'channellisting.asp?ch=' . $channel->attr('value');
			my ($name) = $channel->as_text;

      $channels->{"$id"} = {
        id => "$id",
        'display-name' => [[ encode('utf-8', $name), 'en' ]],
        url => [ $ROOT_URL.$url ]
      };

    }

    $bar->update() if defined $bar;
  }

  $bar->finish() && undef $bar if defined $bar;

  # Notifying the user :)
  $bar = new XMLTV::ProgressBar({
    name => "Reformatting",
    count => 1
  }) unless ($opt->{quiet} || $opt->{debug});

  # Let XMLTV::Writer format the results as a valid xmltv file
  my $writer = new XMLTV::Writer(OUTPUT => \$result, encoding => 'utf-8');
  $writer->start({'generator-info-name' => $generator_info_name});
  $writer->write_channels($channels);
  $writer->end();

  $bar->update() && $bar->finish() if defined $bar;

  return $result;
}

sub config_stage {
		my( $stage, $conf ) = @_;
		die "Unknown stage $stage" if $stage ne "start";

		my $result;
		my $writer = new XMLTV::Configure::Writer( OUTPUT => \$result, encoding => 'utf-8' );
		$writer->start( { grabber => $GRABBER_NAME } );
		$writer->write_string( {
				id => 'cachedir', 
				title => [ [ 'Directory to store the cache in', 'en' ] ],
				description => [ 
				 [ $GRABBER_NAME.' uses a cache with files that it has already '. 
					 'downloaded. Please specify where the cache shall be stored. ', 
					 'en' ] ],
				default => get_default_cachedir(),
		 } );

		$writer->end( 'select-channels' );

		return $result;
}

sub config_check {
		if (not defined( $conf->{cachedir} )) {
				print STDERR "No cachedir defined in configfile " . 
										 $opt->{'config-file'} . "\n" .
										 "Please run the grabber with --configure.\n";
				exit 1;
		}

		if (not defined( $conf->{'channel'} )) {
				print STDERR "No channels selected in configfile " .
										 $opt->{'config-file'} . "\n" .
										 "Please run the grabber with --configure.\n";
				exit 1;
		}
}

sub fetch_url {
	# fetch a url with up to 5 retries
		my ($url) = @_;
		$XMLTV::Get_nice::FailOnError = 0;
		my $content;
		my $maxretry = 5;
		my $retry = 0;
		while ( (not defined($content = XMLTV::Get_nice::get_nice($url))) || (length($content) == 0) ) {
			my $r = $XMLTV::Get_nice::Response;
			print STDERR "HTTP error: ".$r->status_line."\n";
			$retry++;
			return undef if $retry > $maxretry;
			print STDERR "Retrying URL: $url (attempt $retry of $maxretry) \n";
		}
		$content = decode('UTF-8', $content);
		my $t = HTML::TreeBuilder->new();
		$t->parse($content) or die "cannot parse content of $url\n";
		$t->eof;
		return $t;
}

sub get_default_dir {
    my $winhome = $ENV{HOMEDRIVE} . $ENV{HOMEPATH} 
			if defined( $ENV{HOMEDRIVE} ) 
					and defined( $ENV{HOMEPATH} ); 
    
    my $home = $ENV{HOME} || $winhome || ".";
    return $home;
}

sub get_default_cachedir {
    return get_default_dir() . "/.xmltv/cache";
}

sub init_cachedir {
    my( $path ) = @_;
    if( not -d $path ) {
        mkpath( $path ) or die "Failed to create cache-directory $path: $@";
    }
}

sub debug ( $$ ) {
    my( $message, $nonewline ) = @_;
    print STDERR $message if $opt->{debug};
    print STDERR "\n" if $opt->{debug} && (!defined $nonewline || $nonewline != 1);
}

sub warning ( $ ) {
    my( $message ) = @_;
    print STDERR $message . "\n";
    $warnings++;
}

if (0) {		# not used
sub initialise_ua {
	my $cookies = HTTP::Cookies->new;
	#my $ua = LWP::UserAgent->new(keep_alive => 1);
	my $ua = LWP::UserAgent->new;
	# Cookies
	$ua->cookie_jar($cookies);
	# Define user agent type
	$ua->agent('Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US)');
	# Define timouts
	$ua->timeout(240);
	# Use proxy if set in http_proxy etc.
	$ua->env_proxy;
	
	return $ua;
}
}

# #############################################################################

__END__

=pod

=head1 NAME

B<tv_grab_uk_tvguide> - Grab TV listings for UK from the TV Guide UK website.

=head1 DESCRIPTION

Output TV listings in XMLTV format for many channels available in UK.
The data come from tvguide.co.uk

First you must run B<tv_grab_uk_tvguide --configure> to choose which channels
you want to receive.

Then running B<tv_grab_uk_tvguide> with no arguments will get a listings in XML
format for the channels you chose for available days including today.

=head1 OPTIONS

Please see B<tv_grab_uk_tvguide --help>

=head1 INSTALLATION

The file F<tv_grab_uk_tvguide.map.conf> has two purposes.  Firstly you can map the channel ids used by the site into something more meaningful to your PVR. E.g.

      map==74==BBC 1	

will change "74" to "BBC 1" in the output XML.

Note: the lines are of the form "map=={channel id}=={my name}".

The second purpose is to likewise translate genre names.  So if your PVR doesn"t have a category for "Science Fiction" but uses "Sci-fi" instead, then you can specify

      cat==Science Fiction==Sci-fi

and the output XML will have "Sci-fi".


IMPORTANT: the downloaded "tv_grab_uk_tvguide.map.conf" contains example lines to illustrate the format - you should edit this file to suit your own purposes!

=head1 ERROR HANDLING

If the grabber fails to download data for some channel on a specific day, 
it will print an errormessage to STDERR and then continue with the other
channels and days. The grabber will exit with a status code of 1 to indicate 
that the data is incomplete. 

=head1 ENVIRONMENT VARIABLES

The environment variable HOME can be set to change where configuration
files are stored. All configuration is stored in $HOME/.xmltv/. On Windows,
it might be necessary to set HOME to a path without spaces in it.

=head1 SUPPORTED CHANNELS

For information on supported channels, see http://tvguide.co.uk/

=head1 XMLTV VALIDATION

B<tv_validate_grabber> may report an error similar to:

      "Line 5 Invalid channel-id BBC 1"
	
This is a because ValidateFile.pm insists the channel-id adheres to RFC2838 despite the xmltv.dtd only saying "preferably" not "SHOULD".  
(Having channel ids of the form "bbc1.bbc.co.uk" will be rejected by many PVRs since they require the data to match their own list.)

It may also report:

      "tv_sort failed on the concatenated data. Probably due to overlapping data between days."
	
Both these errors can be ignored.

=head1 DISCLAIMER

The TV Guide website"s license for these data does not allow non-personal use.

Certainly any commercial use of listings data obtained by using this grabber will breach copyright law, but if you are just using the data for your own personal use then you are probably fine.

By using this grabber you aver you are using the listings data for your own personal use only and you absolve the author(s) from any liability under copyright law or otherwise.

=head1 AUTHOR

Geoff Westcott. This documentation and parts of the code
based on various other tv_grabbers from the XMLTV-project.

=head1 SEE ALSO

L<xmltv(5)>.

=cut


To Do
=====

1.  Improve the progress bar update frequency
2.  Add actor 'character' attribute  DONE 30/6/16
3.  Currently only does Actor, Director, Producer, Writer - does anyone actually use any of the others present in the DTD?