Merge pull request #7057 from remitamine/cspan

[cspan] correct the clip info extraction (fixes #7335)
This commit is contained in:
remitamine 2015-11-28 21:36:52 +01:00
commit 46b4070f3f
1 changed files with 62 additions and 52 deletions

View File

@ -9,6 +9,7 @@ from ..utils import (
find_xpath_attr, find_xpath_attr,
smuggle_url, smuggle_url,
determine_ext, determine_ext,
ExtractorError,
) )
from .senateisvp import SenateISVPIE from .senateisvp import SenateISVPIE
@ -18,33 +19,32 @@ class CSpanIE(InfoExtractor):
IE_DESC = 'C-SPAN' IE_DESC = 'C-SPAN'
_TESTS = [{ _TESTS = [{
'url': 'http://www.c-span.org/video/?313572-1/HolderonV', 'url': 'http://www.c-span.org/video/?313572-1/HolderonV',
'md5': '8e44ce11f0f725527daccc453f553eb0', 'md5': '94b29a4f131ff03d23471dd6f60b6a1d',
'info_dict': { 'info_dict': {
'id': '315139', 'id': '315139',
'ext': 'mp4', 'ext': 'mp4',
'title': 'Attorney General Eric Holder on Voting Rights Act Decision', 'title': 'Attorney General Eric Holder on Voting Rights Act Decision',
'description': 'Attorney General Eric Holder spoke to reporters following the Supreme Court decision in Shelby County v. Holder in which the court ruled that the preclearance provisions of the Voting Rights Act could not be enforced until Congress established new guidelines for review.', 'description': 'Attorney General Eric Holder speaks to reporters following the Supreme Court decision in [Shelby County v. Holder], in which the court ruled that the preclearance provisions of the Voting Rights Act could not be enforced.',
}, },
'skip': 'Regularly fails on travis, for unknown reasons', 'skip': 'Regularly fails on travis, for unknown reasons',
}, { }, {
'url': 'http://www.c-span.org/video/?c4486943/cspan-international-health-care-models', 'url': 'http://www.c-span.org/video/?c4486943/cspan-international-health-care-models',
# For whatever reason, the served video alternates between 'md5': '8e5fbfabe6ad0f89f3012a7943c1287b',
# two different ones
'info_dict': { 'info_dict': {
'id': '340723', 'id': 'c4486943',
'ext': 'mp4', 'ext': 'mp4',
'title': 'International Health Care Models', 'title': 'CSPAN - International Health Care Models',
'description': 'md5:7a985a2d595dba00af3d9c9f0783c967', 'description': 'md5:7a985a2d595dba00af3d9c9f0783c967',
} }
}, { }, {
'url': 'http://www.c-span.org/video/?318608-1/gm-ignition-switch-recall', 'url': 'http://www.c-span.org/video/?318608-1/gm-ignition-switch-recall',
'md5': '446562a736c6bf97118e389433ed88d4', 'md5': '2ae5051559169baadba13fc35345ae74',
'info_dict': { 'info_dict': {
'id': '342759', 'id': '342759',
'ext': 'mp4', 'ext': 'mp4',
'title': 'General Motors Ignition Switch Recall', 'title': 'General Motors Ignition Switch Recall',
'duration': 14848, 'duration': 14848,
'description': 'md5:70c7c3b8fa63fa60d42772440596034c' 'description': 'md5:118081aedd24bf1d3b68b3803344e7f3'
}, },
}, { }, {
# Video from senate.gov # Video from senate.gov
@ -57,67 +57,77 @@ class CSpanIE(InfoExtractor):
}] }]
def _real_extract(self, url): def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url) video_id = self._match_id(url)
page_id = mobj.group('id') webpage = self._download_webpage(url, video_id)
webpage = self._download_webpage(url, page_id) matches = re.search(r'data-(prog|clip)id=\'([0-9]+)\'', webpage)
video_id = self._search_regex(r'progid=\'?([0-9]+)\'?>', webpage, 'video id') if matches:
video_type, video_id = matches.groups()
if video_type == 'prog':
video_type = 'program'
else:
senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
if senate_isvp_url:
title = self._og_search_title(webpage)
surl = smuggle_url(senate_isvp_url, {'force_title': title})
return self.url_result(surl, 'SenateISVP', video_id, title)
description = self._html_search_regex( def get_text_attr(d, attr):
[ return d.get(attr, {}).get('#text')
# The full description
r'<div class=\'expandable\'>(.*?)<a href=\'#\'',
# If the description is small enough the other div is not
# present, otherwise this is a stripped version
r'<p class=\'initial\'>(.*?)</p>'
],
webpage, 'description', flags=re.DOTALL, default=None)
info_url = 'http://c-spanvideo.org/videoLibrary/assets/player/ajax-player.php?os=android&html5=program&id=' + video_id data = self._download_json(
data = self._download_json(info_url, video_id) 'http://www.c-span.org/assets/player/ajax-player.php?os=android&html5=%s&id=%s' % (video_type, video_id),
video_id)['video']
if data['@status'] != 'Success':
raise ExtractorError('%s said: %s' % (self.IE_NAME, get_text_attr(data, 'error')), expected=True)
doc = self._download_xml( doc = self._download_xml(
'http://www.c-span.org/common/services/flashXml.php?programid=' + video_id, 'http://www.c-span.org/common/services/flashXml.php?%sid=%s' % (video_type, video_id),
video_id) video_id)
description = self._html_search_meta('description', webpage)
title = find_xpath_attr(doc, './/string', 'name', 'title').text title = find_xpath_attr(doc, './/string', 'name', 'title').text
thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text
senate_isvp_url = SenateISVPIE._search_iframe_url(webpage) files = data['files']
if senate_isvp_url: capfile = get_text_attr(data, 'capfile')
surl = smuggle_url(senate_isvp_url, {'force_title': title})
return self.url_result(surl, 'SenateISVP', video_id, title)
files = data['video']['files'] entries = []
try: for partnum, f in enumerate(files):
capfile = data['video']['capfile']['#text'] formats = []
except KeyError: for quality in f['qualities']:
capfile = None formats.append({
'format_id': '%s-%sp' % (get_text_attr(quality, 'bitrate'), get_text_attr(quality, 'height')),
entries = [{ 'url': unescapeHTML(get_text_attr(quality, 'file')),
'height': int_or_none(get_text_attr(quality, 'height')),
'tbr': int_or_none(get_text_attr(quality, 'bitrate')),
})
self._sort_formats(formats)
entries.append({
'id': '%s_%d' % (video_id, partnum + 1), 'id': '%s_%d' % (video_id, partnum + 1),
'title': ( 'title': (
title if len(files) == 1 else title if len(files) == 1 else
'%s part %d' % (title, partnum + 1)), '%s part %d' % (title, partnum + 1)),
'url': unescapeHTML(f['path']['#text']), 'formats': formats,
'description': description, 'description': description,
'thumbnail': thumbnail, 'thumbnail': thumbnail,
'duration': int_or_none(f.get('length', {}).get('#text')), 'duration': int_or_none(get_text_attr(f, 'length')),
'subtitles': { 'subtitles': {
'en': [{ 'en': [{
'url': capfile, 'url': capfile,
'ext': determine_ext(capfile, 'dfxp') 'ext': determine_ext(capfile, 'dfxp')
}], }],
} if capfile else None, } if capfile else None,
} for partnum, f in enumerate(files)] })
if len(entries) == 1: if len(entries) == 1:
entry = dict(entries[0]) entry = dict(entries[0])
entry['id'] = video_id entry['id'] = 'c' + video_id if video_type == 'clip' else video_id
return entry return entry
else: else:
return { return {
'_type': 'playlist', '_type': 'playlist',
'entries': entries, 'entries': entries,
'title': title, 'title': title,
'id': video_id, 'id': 'c' + video_id if video_type == 'clip' else video_id,
} }