2015-07-25 14:21:42 +00:00
# coding: utf-8
2014-02-08 14:55:28 +00:00
from __future__ import unicode_literals
2015-07-25 14:21:42 +00:00
import re
2014-11-30 16:37:56 +00:00
import xml . etree . ElementTree
2014-02-08 14:55:28 +00:00
2015-02-19 15:46:41 +00:00
from . common import InfoExtractor
2015-04-30 21:59:13 +00:00
from . . utils import (
ExtractorError ,
2015-07-25 14:21:42 +00:00
float_or_none ,
2015-04-30 21:59:13 +00:00
int_or_none ,
2015-07-25 14:21:42 +00:00
parse_duration ,
parse_iso8601 ,
2015-04-30 21:59:13 +00:00
)
2014-11-30 16:37:56 +00:00
from . . compat import compat_HTTPError
2014-02-08 14:55:28 +00:00
2015-02-19 15:46:41 +00:00
class BBCCoUkIE ( InfoExtractor ) :
2014-02-08 14:55:28 +00:00
IE_NAME = ' bbc.co.uk '
2014-02-08 21:00:24 +00:00
IE_DESC = ' BBC iPlayer '
2015-01-30 17:47:09 +00:00
_VALID_URL = r ' https?://(?:www \ .)?bbc \ .co \ .uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[ \ da-z] {8} ) '
2014-02-08 14:55:28 +00:00
2015-07-25 14:21:42 +00:00
_MEDIASELECTOR_URL = ' http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/ %s '
2015-06-19 06:52:25 +00:00
2014-02-08 21:00:24 +00:00
_TESTS = [
{
2014-02-18 23:46:14 +00:00
' url ' : ' http://www.bbc.co.uk/programmes/b039g8p7 ' ,
2014-02-08 21:00:24 +00:00
' info_dict ' : {
2014-02-18 23:46:14 +00:00
' id ' : ' b039d07m ' ,
2014-02-08 21:00:24 +00:00
' ext ' : ' flv ' ,
2015-01-02 16:13:26 +00:00
' title ' : ' Kaleidoscope, Leonard Cohen ' ,
' description ' : ' The Canadian poet and songwriter reflects on his musical career. ' ,
2014-02-18 23:46:14 +00:00
' duration ' : 1740 ,
2014-02-08 21:00:24 +00:00
} ,
' params ' : {
# rtmp download
' skip_download ' : True ,
}
2014-02-08 14:55:28 +00:00
} ,
2014-02-08 21:00:24 +00:00
{
' url ' : ' http://www.bbc.co.uk/iplayer/episode/b00yng5w/The_Man_in_Black_Series_3_The_Printed_Name/ ' ,
' info_dict ' : {
' id ' : ' b00yng1d ' ,
' ext ' : ' flv ' ,
' title ' : ' The Man in Black: Series 3: The Printed Name ' ,
' description ' : " Mark Gatiss introduces Nicholas Pierpan ' s chilling tale of a writer ' s devilish pact with a mysterious man. Stars Ewan Bailey. " ,
' duration ' : 1800 ,
} ,
' params ' : {
# rtmp download
' skip_download ' : True ,
2014-02-17 17:26:12 +00:00
} ,
' skip ' : ' Episode is no longer available on BBC iPlayer Radio ' ,
2014-02-08 21:00:24 +00:00
} ,
{
' url ' : ' http://www.bbc.co.uk/iplayer/episode/b03vhd1f/The_Voice_UK_Series_3_Blind_Auditions_5/ ' ,
' info_dict ' : {
' id ' : ' b00yng1d ' ,
' ext ' : ' flv ' ,
2014-02-08 21:04:21 +00:00
' title ' : ' The Voice UK: Series 3: Blind Auditions 5 ' ,
' description ' : " Emma Willis and Marvin Humes present the fifth set of blind auditions in the singing competition, as the coaches continue to build their teams based on voice alone. " ,
' duration ' : 5100 ,
2014-02-08 21:00:24 +00:00
} ,
' params ' : {
# rtmp download
' skip_download ' : True ,
} ,
' skip ' : ' Currently BBC iPlayer TV programmes are available to play in the UK only ' ,
2014-11-30 16:37:56 +00:00
} ,
{
' url ' : ' http://www.bbc.co.uk/iplayer/episode/p026c7jt/tomorrows-worlds-the-unearthly-history-of-science-fiction-2-invasion ' ,
' info_dict ' : {
' id ' : ' b03k3pb7 ' ,
' ext ' : ' flv ' ,
' title ' : " Tomorrow ' s Worlds: The Unearthly History of Science Fiction " ,
' description ' : ' 2. Invasion ' ,
' duration ' : 3600 ,
} ,
' params ' : {
# rtmp download
' skip_download ' : True ,
} ,
' skip ' : ' Currently BBC iPlayer TV programmes are available to play in the UK only ' ,
2014-12-28 21:00:24 +00:00
} , {
' url ' : ' http://www.bbc.co.uk/programmes/b04v20dw ' ,
' info_dict ' : {
' id ' : ' b04v209v ' ,
' ext ' : ' flv ' ,
' title ' : ' Pete Tong, The Essential New Tune Special ' ,
' description ' : " Pete has a very special mix - all of 2014 ' s Essential New Tunes! " ,
' duration ' : 10800 ,
} ,
' params ' : {
# rtmp download
' skip_download ' : True ,
}
2015-01-03 14:43:40 +00:00
} , {
' url ' : ' http://www.bbc.co.uk/music/clips/p02frcc3 ' ,
' note ' : ' Audio ' ,
' info_dict ' : {
' id ' : ' p02frcch ' ,
' ext ' : ' flv ' ,
' title ' : ' Pete Tong, Past, Present and Future Special, Madeon - After Hours mix ' ,
' description ' : ' French house superstar Madeon takes us out of the club and onto the after party. ' ,
' duration ' : 3507 ,
} ,
' params ' : {
# rtmp download
' skip_download ' : True ,
}
} , {
' url ' : ' http://www.bbc.co.uk/music/clips/p025c0zz ' ,
' note ' : ' Video ' ,
' info_dict ' : {
' id ' : ' p025c103 ' ,
' ext ' : ' flv ' ,
' title ' : ' Reading and Leeds Festival, 2014, Rae Morris - Closer (Live on BBC Three) ' ,
' description ' : ' Rae Morris performs Closer for BBC Three at Reading 2014 ' ,
' duration ' : 226 ,
} ,
' params ' : {
# rtmp download
' skip_download ' : True ,
}
2015-04-30 22:02:56 +00:00
} , {
' url ' : ' http://www.bbc.co.uk/iplayer/episode/b054fn09/ad/natural-world-20152016-2-super-powered-owls ' ,
' info_dict ' : {
' id ' : ' p02n76xf ' ,
' ext ' : ' flv ' ,
' title ' : ' Natural World, 2015-2016: 2. Super Powered Owls ' ,
' description ' : ' md5:e4db5c937d0e95a7c6b5e654d429183d ' ,
' duration ' : 3540 ,
} ,
' params ' : {
# rtmp download
' skip_download ' : True ,
} ,
' skip ' : ' geolocation ' ,
2015-06-18 14:00:13 +00:00
} , {
' url ' : ' http://www.bbc.co.uk/iplayer/episode/b05zmgwn/royal-academy-summer-exhibition ' ,
' info_dict ' : {
' id ' : ' b05zmgw1 ' ,
' ext ' : ' flv ' ,
' description ' : ' Kirsty Wark and Morgan Quaintance visit the Royal Academy as it prepares for its annual artistic extravaganza, meeting people who have come together to make the show unique. ' ,
' title ' : ' Royal Academy Summer Exhibition ' ,
' duration ' : 3540 ,
} ,
' params ' : {
# rtmp download
' skip_download ' : True ,
} ,
' skip ' : ' geolocation ' ,
2015-01-02 14:37:54 +00:00
} , {
' url ' : ' http://www.bbc.co.uk/iplayer/playlist/p01dvks4 ' ,
' only_matching ' : True ,
2015-01-03 14:43:40 +00:00
} , {
' url ' : ' http://www.bbc.co.uk/music/clips#p02frcc3 ' ,
' only_matching ' : True ,
2015-01-30 17:47:09 +00:00
} , {
' url ' : ' http://www.bbc.co.uk/iplayer/cbeebies/episode/b0480276/bing-14-atchoo ' ,
' only_matching ' : True ,
2014-12-28 21:00:24 +00:00
}
2014-02-08 21:00:24 +00:00
]
def _extract_asx_playlist ( self , connection , programme_id ) :
asx = self . _download_xml ( connection . get ( ' href ' ) , programme_id , ' Downloading ASX playlist ' )
return [ ref . get ( ' href ' ) for ref in asx . findall ( ' ./Entry/ref ' ) ]
def _extract_connection ( self , connection , programme_id ) :
formats = [ ]
protocol = connection . get ( ' protocol ' )
supplier = connection . get ( ' supplier ' )
if protocol == ' http ' :
href = connection . get ( ' href ' )
2015-07-25 16:32:54 +00:00
transfer_format = connection . get ( ' transferFormat ' )
2014-02-08 21:00:24 +00:00
# ASX playlist
if supplier == ' asx ' :
for i , ref in enumerate ( self . _extract_asx_playlist ( connection , programme_id ) ) :
formats . append ( {
' url ' : ref ,
' format_id ' : ' ref %s _ %s ' % ( i , supplier ) ,
} )
2015-07-25 16:32:54 +00:00
# Skip DASH until supported
elif transfer_format == ' dash ' :
pass
2014-02-08 21:00:24 +00:00
# Direct link
else :
formats . append ( {
' url ' : href ,
' format_id ' : supplier ,
} )
elif protocol == ' rtmp ' :
application = connection . get ( ' application ' , ' ondemand ' )
auth_string = connection . get ( ' authString ' )
identifier = connection . get ( ' identifier ' )
server = connection . get ( ' server ' )
formats . append ( {
' url ' : ' %s :// %s / %s ? %s ' % ( protocol , server , application , auth_string ) ,
' play_path ' : identifier ,
' app ' : ' %s ? %s ' % ( application , auth_string ) ,
' page_url ' : ' http://www.bbc.co.uk ' ,
' player_url ' : ' http://www.bbc.co.uk/emp/releases/iplayer/revisions/617463_618125_4/617463_618125_4_emp.swf ' ,
' rtmp_live ' : False ,
' ext ' : ' flv ' ,
' format_id ' : supplier ,
} )
return formats
def _extract_items ( self , playlist ) :
return playlist . findall ( ' ./ { http://bbc.co.uk/2008/emp/playlist}item ' )
def _extract_medias ( self , media_selection ) :
2014-11-30 16:37:56 +00:00
error = media_selection . find ( ' ./ { http://bbc.co.uk/2008/mp/mediaselection}error ' )
if error is not None :
raise ExtractorError (
' %s returned error: %s ' % ( self . IE_NAME , error . get ( ' id ' ) ) , expected = True )
2014-02-08 21:00:24 +00:00
return media_selection . findall ( ' ./ { http://bbc.co.uk/2008/mp/mediaselection}media ' )
def _extract_connections ( self , media ) :
return media . findall ( ' ./ { http://bbc.co.uk/2008/mp/mediaselection}connection ' )
def _extract_video ( self , media , programme_id ) :
formats = [ ]
2015-07-23 20:56:54 +00:00
vbr = int_or_none ( media . get ( ' bitrate ' ) )
2014-02-08 21:00:24 +00:00
vcodec = media . get ( ' encoding ' )
service = media . get ( ' service ' )
2015-07-23 20:56:54 +00:00
width = int_or_none ( media . get ( ' width ' ) )
height = int_or_none ( media . get ( ' height ' ) )
file_size = int_or_none ( media . get ( ' media_file_size ' ) )
2014-02-08 21:00:24 +00:00
for connection in self . _extract_connections ( media ) :
conn_formats = self . _extract_connection ( connection , programme_id )
for format in conn_formats :
format . update ( {
' format_id ' : ' %s _ %s ' % ( service , format [ ' format_id ' ] ) ,
' width ' : width ,
' height ' : height ,
' vbr ' : vbr ,
' vcodec ' : vcodec ,
' filesize ' : file_size ,
} )
formats . extend ( conn_formats )
return formats
def _extract_audio ( self , media , programme_id ) :
formats = [ ]
2015-07-23 20:56:54 +00:00
abr = int_or_none ( media . get ( ' bitrate ' ) )
2014-02-08 21:00:24 +00:00
acodec = media . get ( ' encoding ' )
service = media . get ( ' service ' )
for connection in self . _extract_connections ( media ) :
conn_formats = self . _extract_connection ( connection , programme_id )
for format in conn_formats :
format . update ( {
' format_id ' : ' %s _ %s ' % ( service , format [ ' format_id ' ] ) ,
' abr ' : abr ,
' acodec ' : acodec ,
} )
formats . extend ( conn_formats )
return formats
2015-02-19 15:46:41 +00:00
def _get_subtitles ( self , media , programme_id ) :
2014-02-08 21:00:24 +00:00
subtitles = { }
for connection in self . _extract_connections ( media ) :
captions = self . _download_xml ( connection . get ( ' href ' ) , programme_id , ' Downloading captions ' )
lang = captions . get ( ' { http://www.w3.org/XML/1998/namespace}lang ' , ' en ' )
2015-02-19 15:46:41 +00:00
subtitles [ lang ] = [
{
' url ' : connection . get ( ' href ' ) ,
' ext ' : ' ttml ' ,
} ,
]
2014-02-08 21:00:24 +00:00
return subtitles
2014-02-08 14:55:28 +00:00
2014-11-30 16:37:56 +00:00
def _download_media_selector ( self , programme_id ) :
2015-07-25 14:21:42 +00:00
return self . _download_media_selector_url (
self . _MEDIASELECTOR_URL % programme_id , programme_id )
def _download_media_selector_url ( self , url , programme_id = None ) :
2014-11-30 16:37:56 +00:00
try :
media_selection = self . _download_xml (
2015-07-25 14:21:42 +00:00
url , programme_id , ' Downloading media selection XML ' )
2014-11-30 16:37:56 +00:00
except ExtractorError as ee :
if isinstance ( ee . cause , compat_HTTPError ) and ee . cause . code == 403 :
2015-06-18 13:40:45 +00:00
media_selection = xml . etree . ElementTree . fromstring ( ee . cause . read ( ) . decode ( ' utf-8 ' ) )
2014-02-08 21:00:24 +00:00
else :
2014-11-30 16:37:56 +00:00
raise
2015-07-25 14:21:42 +00:00
return self . _process_media_selector ( media_selection , programme_id )
2014-02-08 14:55:28 +00:00
2015-07-25 14:21:42 +00:00
def _process_media_selector ( self , media_selection , programme_id ) :
2014-02-08 14:55:28 +00:00
formats = [ ]
2014-02-08 21:00:24 +00:00
subtitles = None
2014-11-30 16:37:56 +00:00
for media in self . _extract_medias ( media_selection ) :
kind = media . get ( ' kind ' )
if kind == ' audio ' :
formats . extend ( self . _extract_audio ( media , programme_id ) )
elif kind == ' video ' :
formats . extend ( self . _extract_video ( media , programme_id ) )
elif kind == ' captions ' :
2015-02-19 15:46:41 +00:00
subtitles = self . extract_subtitles ( media , programme_id )
2014-02-08 21:00:24 +00:00
2014-11-30 16:37:56 +00:00
return formats , subtitles
2014-02-08 21:00:24 +00:00
2014-12-28 21:00:24 +00:00
def _download_playlist ( self , playlist_id ) :
try :
playlist = self . _download_json (
' http://www.bbc.co.uk/programmes/ %s /playlist.json ' % playlist_id ,
playlist_id , ' Downloading playlist JSON ' )
version = playlist . get ( ' defaultAvailableVersion ' )
if version :
smp_config = version [ ' smpConfig ' ]
title = smp_config [ ' title ' ]
description = smp_config [ ' summary ' ]
for item in smp_config [ ' items ' ] :
kind = item [ ' kind ' ]
if kind != ' programme ' and kind != ' radioProgramme ' :
continue
programme_id = item . get ( ' vpid ' )
2015-07-23 20:56:54 +00:00
duration = int_or_none ( item . get ( ' duration ' ) )
2014-12-28 21:00:24 +00:00
formats , subtitles = self . _download_media_selector ( programme_id )
return programme_id , title , description , duration , formats , subtitles
except ExtractorError as ee :
2015-02-15 10:32:38 +00:00
if not ( isinstance ( ee . cause , compat_HTTPError ) and ee . cause . code == 404 ) :
2014-12-28 21:00:24 +00:00
raise
# fallback to legacy playlist
2015-07-25 14:21:42 +00:00
return self . _process_legacy_playlist ( playlist_id )
def _process_legacy_playlist_url ( self , url , display_id ) :
playlist = self . _download_legacy_playlist_url ( url , display_id )
return self . _extract_from_legacy_playlist ( playlist , display_id )
def _process_legacy_playlist ( self , playlist_id ) :
return self . _process_legacy_playlist_url (
' http://www.bbc.co.uk/iplayer/playlist/ %s ' % playlist_id , playlist_id )
def _download_legacy_playlist_url ( self , url , playlist_id = None ) :
return self . _download_xml (
url , playlist_id , ' Downloading legacy playlist XML ' )
2014-12-28 21:00:24 +00:00
2015-07-25 14:21:42 +00:00
def _extract_from_legacy_playlist ( self , playlist , playlist_id ) :
2014-12-28 21:00:24 +00:00
no_items = playlist . find ( ' ./ { http://bbc.co.uk/2008/emp/playlist}noItems ' )
if no_items is not None :
reason = no_items . get ( ' reason ' )
if reason == ' preAvailability ' :
msg = ' Episode %s is not yet available ' % playlist_id
elif reason == ' postAvailability ' :
msg = ' Episode %s is no longer available ' % playlist_id
elif reason == ' noMedia ' :
msg = ' Episode %s is not currently available ' % playlist_id
else :
msg = ' Episode %s is not available: %s ' % ( playlist_id , reason )
raise ExtractorError ( msg , expected = True )
for item in self . _extract_items ( playlist ) :
kind = item . get ( ' kind ' )
if kind != ' programme ' and kind != ' radioProgramme ' :
continue
title = playlist . find ( ' ./ { http://bbc.co.uk/2008/emp/playlist}title ' ) . text
description = playlist . find ( ' ./ { http://bbc.co.uk/2008/emp/playlist}summary ' ) . text
2015-07-25 14:21:42 +00:00
def get_programme_id ( item ) :
def get_from_attributes ( item ) :
for p in ( ' identifier ' , ' group ' ) :
value = item . get ( p )
if value and re . match ( r ' ^[pb][ \ da-z] {7} $ ' , value ) :
return value
get_from_attributes ( item )
mediator = item . find ( ' ./ { http://bbc.co.uk/2008/emp/playlist}mediator ' )
if mediator is not None :
return get_from_attributes ( mediator )
programme_id = get_programme_id ( item )
2015-07-23 20:56:54 +00:00
duration = int_or_none ( item . get ( ' duration ' ) )
2015-07-25 14:21:42 +00:00
# TODO: programme_id can be None and media items can be incorporated right inside
# playlist's item (e.g. http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
# as f4m and m3u8
2014-12-28 21:00:24 +00:00
formats , subtitles = self . _download_media_selector ( programme_id )
return programme_id , title , description , duration , formats , subtitles
2014-11-30 16:37:56 +00:00
def _real_extract ( self , url ) :
group_id = self . _match_id ( url )
webpage = self . _download_webpage ( url , group_id , ' Downloading video page ' )
2015-04-30 21:59:13 +00:00
programme_id = None
tviplayer = self . _search_regex (
r ' mediator \ .bind \ (( { .+?}) \ s*, \ s*document \ .getElementById ' ,
webpage , ' player ' , default = None )
if tviplayer :
player = self . _parse_json ( tviplayer , group_id ) . get ( ' player ' , { } )
duration = int_or_none ( player . get ( ' duration ' ) )
programme_id = player . get ( ' vpid ' )
if not programme_id :
programme_id = self . _search_regex (
r ' " vpid " \ s*: \ s* " ([ \ da-z] {8} ) " ' , webpage , ' vpid ' , fatal = False , default = None )
2014-11-30 16:37:56 +00:00
if programme_id :
formats , subtitles = self . _download_media_selector ( programme_id )
2015-04-30 21:59:13 +00:00
title = self . _og_search_title ( webpage )
description = self . _search_regex (
2015-06-18 14:00:13 +00:00
r ' <p class= " [^ " ]*medium-description[^ " ]* " >([^<]+)</p> ' ,
2015-04-30 21:59:13 +00:00
webpage , ' description ' , fatal = False )
2014-11-30 16:37:56 +00:00
else :
2014-12-28 21:00:24 +00:00
programme_id , title , description , duration , formats , subtitles = self . _download_playlist ( group_id )
2014-02-08 21:00:24 +00:00
2014-02-08 14:55:28 +00:00
self . _sort_formats ( formats )
return {
2014-02-08 21:00:24 +00:00
' id ' : programme_id ,
2014-02-08 14:55:28 +00:00
' title ' : title ,
' description ' : description ,
2015-04-30 22:07:30 +00:00
' thumbnail ' : self . _og_search_thumbnail ( webpage , default = None ) ,
2014-02-08 14:55:28 +00:00
' duration ' : duration ,
' formats ' : formats ,
2014-02-08 21:00:24 +00:00
' subtitles ' : subtitles ,
2014-11-23 19:41:03 +00:00
}
2015-06-20 13:22:13 +00:00
2015-07-25 14:21:42 +00:00
class BBCIE ( BBCCoUkIE ) :
IE_NAME = ' bbc '
IE_DESC = ' BBC '
_VALID_URL = r ' https?://(?:www \ .)?bbc \ .(?:com|co \ .uk)/(?:[^/]+/)+(?P<id>[^/#?]+) '
2015-06-20 13:22:13 +00:00
2015-07-25 14:21:42 +00:00
# fails with notukerror for some videos
2015-07-25 16:19:54 +00:00
# _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
2015-07-25 14:21:42 +00:00
_MEDIASELECTOR_URL = ' http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/ %s '
2015-06-20 13:22:13 +00:00
_TESTS = [ {
2015-07-25 14:21:42 +00:00
# article with multiple videos embedded with data-media-meta containing
# playlist.sxml, externalId and no direct video links
2015-06-20 13:22:13 +00:00
' url ' : ' http://www.bbc.com/news/world-europe-32668511 ' ,
' info_dict ' : {
' id ' : ' world-europe-32668511 ' ,
' title ' : ' Russia stages massive WW2 parade despite Western boycott ' ,
2015-07-25 14:21:42 +00:00
' description ' : ' md5:00ff61976f6081841f759a08bf78cc9c ' ,
2015-06-20 13:22:13 +00:00
} ,
' playlist_count ' : 2 ,
2015-07-17 07:47:02 +00:00
} , {
2015-07-25 14:21:42 +00:00
# article with multiple videos embedded with data-media-meta (more videos)
2015-06-20 13:22:13 +00:00
' url ' : ' http://www.bbc.com/news/business-28299555 ' ,
' info_dict ' : {
' id ' : ' business-28299555 ' ,
' title ' : ' Farnborough Airshow: Video highlights ' ,
2015-07-25 14:21:42 +00:00
' description ' : ' BBC reports and video highlights at the Farnborough Airshow. ' ,
2015-06-20 13:22:13 +00:00
} ,
' playlist_count ' : 9 ,
2015-07-25 14:21:42 +00:00
' skip ' : ' Save time ' ,
2015-07-27 16:05:51 +00:00
} , {
# article with multiple videos embedded with `new SMP()`
' url ' : ' http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460 ' ,
' info_dict ' : {
' id ' : ' 3662a707-0af9-3149-963f-47bea720b460 ' ,
' title ' : ' BBC Blogs - Adam Curtis - BUGGER ' ,
} ,
' playlist_count ' : 18 ,
2015-07-17 07:47:02 +00:00
} , {
2015-07-25 14:21:42 +00:00
# single video embedded with mediaAssetPage.init()
2015-06-20 13:22:13 +00:00
' url ' : ' http://www.bbc.com/news/world-europe-32041533 ' ,
' info_dict ' : {
' id ' : ' p02mprgb ' ,
2015-07-25 14:21:42 +00:00
' ext ' : ' flv ' ,
2015-06-20 13:22:13 +00:00
' title ' : ' Aerial footage showed the site of the crash in the Alps - courtesy BFM TV ' ,
' duration ' : 47 ,
2015-07-25 14:21:42 +00:00
' timestamp ' : 1427219242 ,
2015-06-25 05:31:32 +00:00
' upload_date ' : ' 20150324 ' ,
2015-06-20 13:22:13 +00:00
} ,
' params ' : {
2015-07-25 14:21:42 +00:00
# rtmp download
2015-06-20 13:22:13 +00:00
' skip_download ' : True ,
}
2015-07-17 07:47:02 +00:00
} , {
2015-07-25 14:21:42 +00:00
# article with single video embedded with data-media-meta containing
# direct video links (for now these are extracted) and playlist.xml (with
# media items as f4m and m3u8 - currently unsupported)
2015-06-20 16:04:46 +00:00
' url ' : ' http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu ' ,
' info_dict ' : {
2015-07-25 14:21:42 +00:00
' id ' : ' 150615_telabyad_kentin_cogu ' ,
2015-06-20 16:04:46 +00:00
' ext ' : ' mp4 ' ,
2015-07-25 14:21:42 +00:00
' title ' : " YPG: Tel Abyad ' ı n tamamı kontrolümüzde" ,
2015-06-20 16:04:46 +00:00
' duration ' : 47 ,
2015-07-25 14:21:42 +00:00
' timestamp ' : 1434397334 ,
2015-06-25 05:31:32 +00:00
' upload_date ' : ' 20150615 ' ,
2015-06-20 16:04:46 +00:00
} ,
' params ' : {
' skip_download ' : True ,
}
2015-07-17 07:47:02 +00:00
} , {
2015-07-25 14:21:42 +00:00
# single video embedded with mediaAssetPage.init() (regional section)
2015-06-20 16:04:46 +00:00
' url ' : ' http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw ' ,
' info_dict ' : {
2015-07-25 14:21:42 +00:00
' id ' : ' 150619_video_honduras_militares_hospitales_corrupcion_aw ' ,
2015-06-20 16:04:46 +00:00
' ext ' : ' mp4 ' ,
2015-07-25 14:21:42 +00:00
' title ' : ' Honduras militariza sus hospitales por nuevo escándalo de corrupción ' ,
2015-06-20 16:04:46 +00:00
' duration ' : 87 ,
2015-07-25 14:21:42 +00:00
' timestamp ' : 1434713142 ,
2015-06-25 05:31:32 +00:00
' upload_date ' : ' 20150619 ' ,
2015-06-20 16:04:46 +00:00
} ,
' params ' : {
' skip_download ' : True ,
}
2015-07-25 14:21:42 +00:00
} , {
# single video story with digitalData
' url ' : ' http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret ' ,
' info_dict ' : {
' id ' : ' p02q6gc4 ' ,
' ext ' : ' flv ' ,
' title ' : ' Sri Lanka’ s spicy secret ' ,
' description ' : ' As a new train line to Jaffna opens up the country’ s north, travellers can experience a truly distinct slice of Tamil culture. ' ,
' timestamp ' : 1437674293 ,
' upload_date ' : ' 20150723 ' ,
} ,
' params ' : {
# rtmp download
' skip_download ' : True ,
}
} , {
# single video story without digitalData
' url ' : ' http://www.bbc.com/autos/story/20130513-hyundais-rock-star ' ,
' info_dict ' : {
' id ' : ' p018zqqg ' ,
' ext ' : ' flv ' ,
' title ' : ' Hyundai Santa Fe Sport: Rock star ' ,
' description ' : ' md5:b042a26142c4154a6e472933cf20793d ' ,
' timestamp ' : 1368473503 ,
' upload_date ' : ' 20130513 ' ,
} ,
' params ' : {
# rtmp download
' skip_download ' : True ,
}
} , {
# single video with playlist.sxml URL
' url ' : ' http://www.bbc.com/sport/0/football/33653409 ' ,
' info_dict ' : {
' id ' : ' p02xycnp ' ,
' ext ' : ' flv ' ,
' title ' : ' Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend? ' ,
' description ' : ' md5:398fca0e2e701c609d726e034fa1fc89 ' ,
' duration ' : 140 ,
} ,
' params ' : {
# rtmp download
' skip_download ' : True ,
}
} , {
# single video with playlist URL from weather section
' url ' : ' http://www.bbc.com/weather/features/33601775 ' ,
' only_matching ' : True ,
} , {
# custom redirection to www.bbc.com
' url ' : ' http://www.bbc.co.uk/news/science-environment-33661876 ' ,
' only_matching ' : True ,
2015-06-20 13:22:13 +00:00
} ]
2015-07-25 14:21:42 +00:00
@classmethod
def suitable ( cls , url ) :
return False if BBCCoUkIE . suitable ( url ) else super ( BBCIE , cls ) . suitable ( url )
def _extract_from_media_meta ( self , media_meta , video_id ) :
# Direct links to media in media metadata (e.g.
# http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
# TODO: there are also f4m and m3u8 streams incorporated in playlist.sxml
source_files = media_meta . get ( ' sourceFiles ' )
if source_files :
return [ {
' url ' : f [ ' url ' ] ,
' format_id ' : format_id ,
' ext ' : f . get ( ' encoding ' ) ,
' tbr ' : float_or_none ( f . get ( ' bitrate ' ) , 1000 ) ,
' filesize ' : int_or_none ( f . get ( ' filesize ' ) ) ,
} for format_id , f in source_files . items ( ) if f . get ( ' url ' ) ] , [ ]
programme_id = media_meta . get ( ' externalId ' )
if programme_id :
return self . _download_media_selector ( programme_id )
# Process playlist.sxml as legacy playlist
href = media_meta . get ( ' href ' )
if href :
playlist = self . _download_legacy_playlist_url ( href )
_ , _ , _ , _ , formats , subtitles = self . _extract_from_legacy_playlist ( playlist , video_id )
return formats , subtitles
return [ ] , [ ]
2015-06-20 13:22:13 +00:00
def _real_extract ( self , url ) :
2015-07-25 14:21:42 +00:00
playlist_id = self . _match_id ( url )
webpage = self . _download_webpage ( url , playlist_id )
timestamp = parse_iso8601 ( self . _search_regex (
[ r ' " datePublished " : \ s* " ([^ " ]+) ' ,
r ' <meta[^>]+property= " article:published_time " [^>]+content= " ([^ " ]+) " ' ,
r ' itemprop= " datePublished " [^>]+datetime= " ([^ " ]+) " ' ] ,
webpage , ' date ' , default = None ) )
# single video with playlist.sxml URL (e.g. http://www.bbc.com/sport/0/football/3365340ng)
playlist = self . _search_regex (
r ' <param[^>]+name= " playlist " [^>]+value= " ([^ " ]+) " ' ,
webpage , ' playlist ' , default = None )
if playlist :
programme_id , title , description , duration , formats , subtitles = \
self . _process_legacy_playlist_url ( playlist , playlist_id )
self . _sort_formats ( formats )
return {
' id ' : programme_id ,
' title ' : title ,
' description ' : description ,
' duration ' : duration ,
' timestamp ' : timestamp ,
' formats ' : formats ,
' subtitles ' : subtitles ,
}
2015-06-20 16:04:46 +00:00
2015-07-25 14:21:42 +00:00
# single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
programme_id = self . _search_regex (
[ r ' data-video-player-vpid= " ([ \ da-z] {8} ) " ' ,
r ' <param[^>]+name= " externalIdentifier " [^>]+value= " ([ \ da-z] {8} ) " ' ] ,
webpage , ' vpid ' , default = None )
if programme_id :
formats , subtitles = self . _download_media_selector ( programme_id )
self . _sort_formats ( formats )
# digitalData may be missing (e.g. http://www.bbc.com/autos/story/20130513-hyundais-rock-star)
digital_data = self . _parse_json (
self . _search_regex (
r ' var \ s+digitalData \ s*= \ s*( { .+?});? \ n ' , webpage , ' digital data ' , default = ' {} ' ) ,
programme_id , fatal = False )
page_info = digital_data . get ( ' page ' , { } ) . get ( ' pageInfo ' , { } )
title = page_info . get ( ' pageName ' ) or self . _og_search_title ( webpage )
description = page_info . get ( ' description ' ) or self . _og_search_description ( webpage )
timestamp = parse_iso8601 ( page_info . get ( ' publicationDate ' ) ) or timestamp
return {
' id ' : programme_id ,
' title ' : title ,
' description ' : description ,
' timestamp ' : timestamp ,
' formats ' : formats ,
' subtitles ' : subtitles ,
}
2015-07-17 07:47:02 +00:00
2015-07-25 14:21:42 +00:00
playlist_title = self . _html_search_regex (
r ' <title>(.*?)(?: \ s*- \ s*BBC [^ ]+)?</title> ' , webpage , ' playlist title ' )
2015-07-27 16:05:51 +00:00
playlist_description = self . _og_search_description ( webpage , default = None )
def extract_all ( pattern ) :
return list ( filter ( None , map (
lambda s : self . _parse_json ( s , playlist_id , fatal = False ) ,
re . findall ( pattern , webpage ) ) ) )
# Multiple video article (e.g.
# http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460)
EMBED_URL = r ' https?://(?:www \ .)?bbc \ .co \ .uk/(?:[^/]+/)+[ \ da-z] {8} (?: \ b[^ " ]*)? '
entries = [ ]
for match in extract_all ( r ' new \ s+SMP \ (( { .+?}) \ ) ' ) :
embed_url = match . get ( ' playerSettings ' , { } ) . get ( ' externalEmbedUrl ' )
if embed_url and re . match ( EMBED_URL , embed_url ) :
entries . append ( embed_url )
entries . extend ( re . findall (
r ' setPlaylist \ ( " ( %s ) " \ ) ' % EMBED_URL , webpage ) )
if entries :
return self . playlist_result (
[ self . url_result ( entry , ' BBCCoUk ' ) for entry in entries ] ,
playlist_id , playlist_title , playlist_description )
2015-07-25 14:21:42 +00:00
# Multiple video article (e.g. http://www.bbc.com/news/world-europe-32668511)
2015-07-27 16:05:51 +00:00
medias = extract_all ( r " data-media-meta= ' ( { [^ ' ]+}) ' " )
2015-07-25 14:21:42 +00:00
if not medias :
# Single video article (e.g. http://www.bbc.com/news/video_and_audio/international)
media_asset_page = self . _parse_json (
self . _search_regex (
r ' mediaAssetPage \ .init \ ( \ s*( { .+?}), " / ' , webpage , ' media asset ' ) ,
playlist_id )
medias = [ ]
for video in media_asset_page . get ( ' videos ' , { } ) . values ( ) :
medias . extend ( video . values ( ) )
entries = [ ]
for num , media_meta in enumerate ( medias , start = 1 ) :
formats , subtitles = self . _extract_from_media_meta ( media_meta , playlist_id )
if not formats :
continue
2015-06-20 13:22:13 +00:00
self . _sort_formats ( formats )
2015-07-25 14:21:42 +00:00
video_id = media_meta . get ( ' externalId ' )
if not video_id :
video_id = playlist_id if len ( medias ) == 1 else ' %s - %s ' % ( playlist_id , num )
title = media_meta . get ( ' caption ' )
if not title :
title = playlist_title if len ( medias ) == 1 else ' %s - Video %s ' % ( playlist_title , num )
duration = int_or_none ( media_meta . get ( ' durationInSeconds ' ) ) or parse_duration ( media_meta . get ( ' duration ' ) )
2015-06-25 05:31:32 +00:00
2015-07-25 14:21:42 +00:00
images = [ ]
for image in media_meta . get ( ' images ' , { } ) . values ( ) :
images . extend ( image . values ( ) )
if ' image ' in media_meta :
images . append ( media_meta [ ' image ' ] )
thumbnails = [ {
' url ' : image . get ( ' href ' ) ,
' width ' : int_or_none ( image . get ( ' width ' ) ) ,
' height ' : int_or_none ( image . get ( ' height ' ) ) ,
} for image in images ]
entries . append ( {
' id ' : video_id ,
2015-06-20 13:22:13 +00:00
' title ' : title ,
2015-07-25 14:21:42 +00:00
' thumbnails ' : thumbnails ,
2015-06-20 13:22:13 +00:00
' duration ' : duration ,
2015-07-25 14:21:42 +00:00
' timestamp ' : timestamp ,
2015-06-20 13:22:13 +00:00
' formats ' : formats ,
' subtitles ' : subtitles ,
2015-07-17 07:47:02 +00:00
} )
2015-06-20 13:22:13 +00:00
2015-07-25 14:21:42 +00:00
return self . playlist_result ( entries , playlist_id , playlist_title , playlist_description )