Minor changes to make it easier to merge

This commit is contained in:
pukkandan 2020-11-24 03:17:42 +05:30
parent 70c5802b5d
commit 70d5c17b08
2 changed files with 60 additions and 52 deletions

View File

@ -1153,7 +1153,7 @@
- **YourPorn** - **YourPorn**
- **YourUpload** - **YourUpload**
- **youtube**: YouTube.com - **youtube**: YouTube.com
- **youtube:favorites**: YouTube.com liked videos, ":ytfav" or "LL" for short (requires authentication) - **youtube:favorites**: YouTube.com liked videos, ":ytfav" for short (requires authentication)
- **youtube:history**: Youtube watch history, ":ythistory" for short (requires authentication) - **youtube:history**: Youtube watch history, ":ythistory" for short (requires authentication)
- **youtube:playlist**: YouTube.com playlists - **youtube:playlist**: YouTube.com playlists
- **youtube:recommended**: YouTube.com recommended videos, ":ytrec" for short (requires authentication) - **youtube:recommended**: YouTube.com recommended videos, ":ytrec" for short (requires authentication)
@ -1162,7 +1162,7 @@
- **youtube:search_url**: YouTube.com search URLs - **youtube:search_url**: YouTube.com search URLs
- **youtube:subscriptions**: YouTube.com subscriptions feed, ":ytsubs" for short (requires authentication) - **youtube:subscriptions**: YouTube.com subscriptions feed, ":ytsubs" for short (requires authentication)
- **youtube:tab**: YouTube.com tab - **youtube:tab**: YouTube.com tab
- **youtube:watchlater**: Youtube watch later list, ":ytwatchlater" or "WL" for short (requires authentication) - **youtube:watchlater**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
- **YoutubeYtUser**: YouTube.com user videos, URL or "ytuser" keyword - **YoutubeYtUser**: YouTube.com user videos, URL or "ytuser" keyword
- **Zapiks** - **Zapiks**
- **Zaq1** - **Zaq1**

View File

@ -72,7 +72,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
# If True it will raise an error if no login info is provided # If True it will raise an error if no login info is provided
_LOGIN_REQUIRED = False _LOGIN_REQUIRED = False
_PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM)' _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
_YOUTUBE_CLIENT_HEADERS = { _YOUTUBE_CLIENT_HEADERS = {
'x-youtube-client-name': '1', 'x-youtube-client-name': '1',
@ -1328,7 +1328,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
# https://github.com/ytdl-org/youtube-dl/pull/7599) # https://github.com/ytdl-org/youtube-dl/pull/7599)
r';ytplayer\.config\s*=\s*({.+?});ytplayer', r';ytplayer\.config\s*=\s*({.+?});ytplayer',
r';ytplayer\.config\s*=\s*({.+?});', r';ytplayer\.config\s*=\s*({.+?});',
r'ytInitialPlayerResponse\s*=\s*({.+?});var meta' # Needed???
) )
config = self._search_regex( config = self._search_regex(
patterns, webpage, 'ytplayer.config', default=None) patterns, webpage, 'ytplayer.config', default=None)
@ -2532,11 +2531,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
class YoutubeTabIE(YoutubeBaseInfoExtractor): class YoutubeTabIE(YoutubeBaseInfoExtractor):
IE_DESC = 'YouTube.com tab' IE_DESC = 'YouTube.com tab'
_VALID_URL = (r'''(?x) _VALID_URL = r'''(?x)
https?://(?:\w+\.)?(?:youtube(?:kids)?\.com|invidio\.us)/(?: https?://
(?!(%s)([/#?]|$))|channel/|c/|user/| (?:\w+\.)?
(?P<not_channel>playlist|watch)/?\?.*?\blist=) (?:
(?P<id>[^/?#&]+)''') % YoutubeBaseInfoExtractor._RESERVED_NAMES youtube(?:kids)?\.com|
invidio\.us
)/
(?:
(?:channel|c|user)/|
(?P<not_channel>
(?:playlist|watch)\?.*?\blist=
)|
(?!(%s)([/#?]|$)) # Direct URLs
)
(?P<id>[^/?\#&]+)
''' % YoutubeBaseInfoExtractor._RESERVED_NAMES
IE_NAME = 'youtube:tab' IE_NAME = 'youtube:tab'
_TESTS = [{ _TESTS = [{
@ -2910,7 +2920,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
def _rich_entries(self, rich_grid_renderer): def _rich_entries(self, rich_grid_renderer):
renderer = try_get( renderer = try_get(
rich_grid_renderer, lambda x: x['content']['videoRenderer'], dict) rich_grid_renderer, lambda x: x['content']['videoRenderer'], dict) or {}
video_id = renderer.get('videoId') video_id = renderer.get('videoId')
if not video_id: if not video_id:
return return
@ -3008,14 +3018,14 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
def _entries(self, tab, identity_token): def _entries(self, tab, identity_token):
def extract_entries(parent_renderer): def extract_entries(parent_renderer): # this needs to called again for continuation to work with feeds
slr_contents = try_get(parent_renderer, lambda x: x['contents'], list) or [] contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
for slr_content in slr_contents: for content in contents:
if not isinstance(slr_content, dict): if not isinstance(content, dict):
continue continue
is_renderer = try_get(slr_content, lambda x: x['itemSectionRenderer'], dict) is_renderer = try_get(content, lambda x: x['itemSectionRenderer'], dict)
if not is_renderer: if not is_renderer:
renderer = slr_content.get('richItemRenderer') renderer = content.get('richItemRenderer')
if renderer: if renderer:
for entry in self._rich_entries(renderer): for entry in self._rich_entries(renderer):
yield entry yield entry
@ -3041,7 +3051,6 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
if renderer: if renderer:
for entry in self._shelf_entries(renderer): for entry in self._shelf_entries(renderer):
yield entry yield entry
continuation_list[0] = self._extract_continuation(parent_renderer)
continue continue
renderer = isr_content.get('backstagePostThreadRenderer') renderer = isr_content.get('backstagePostThreadRenderer')
if renderer: if renderer:
@ -3054,8 +3063,10 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
entry = self._video_entry(renderer) entry = self._video_entry(renderer)
if entry: if entry:
yield entry yield entry
if not continuation_list[0]: if not continuation_list[0]:
continuation_list[0] = self._extract_continuation(is_renderer) continuation_list[0] = self._extract_continuation(is_renderer)
if not continuation_list[0]: if not continuation_list[0]:
continuation_list[0] = self._extract_continuation(parent_renderer) continuation_list[0] = self._extract_continuation(parent_renderer)
@ -3063,7 +3074,6 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
parent_renderer = ( parent_renderer = (
try_get(tab, lambda x: x['sectionListRenderer'], dict) try_get(tab, lambda x: x['sectionListRenderer'], dict)
or try_get(tab, lambda x: x['richGridRenderer'], dict) or {}) or try_get(tab, lambda x: x['richGridRenderer'], dict) or {})
if parent_renderer:
for entry in extract_entries(parent_renderer): for entry in extract_entries(parent_renderer):
yield entry yield entry
continuation = continuation_list[0] continuation = continuation_list[0]
@ -3078,8 +3088,6 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
for page_num in itertools.count(1): for page_num in itertools.count(1):
if not continuation: if not continuation:
break break
if hasattr(self, '_MAX_PAGES') and page_num > self._MAX_PAGES:
break
browse = self._download_json( browse = self._download_json(
'https://www.youtube.com/browse_ajax', None, 'https://www.youtube.com/browse_ajax', None,
'Downloading page %d' % page_num, 'Downloading page %d' % page_num,
@ -3111,7 +3119,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
yield entry yield entry
continuation = self._extract_continuation(continuation_renderer) continuation = self._extract_continuation(continuation_renderer)
continue continue
continuation_renderer = continuation_contents.get('sectionListContinuation') continuation_renderer = continuation_contents.get('sectionListContinuation') # for feeds
if continuation_renderer: if continuation_renderer:
continuation_list = [None] continuation_list = [None]
for entry in extract_entries(continuation_renderer): for entry in extract_entries(continuation_renderer):
@ -3125,19 +3133,13 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
continuation_item = continuation_items[0] continuation_item = continuation_items[0]
if not isinstance(continuation_item, dict): if not isinstance(continuation_item, dict):
continue continue
renderer = continuation_item.get('playlistVideoRenderer') renderer = continuation_item.get('playlistVideoRenderer') or continuation_item.get('itemSectionRenderer')
if renderer: if renderer:
video_list_renderer = {'contents': continuation_items} video_list_renderer = {'contents': continuation_items}
for entry in self._playlist_entries(video_list_renderer): for entry in self._playlist_entries(video_list_renderer):
yield entry yield entry
continuation = self._extract_continuation(video_list_renderer) continuation = self._extract_continuation(video_list_renderer)
continue continue
renderer = continuation_item.get('itemSectionRenderer')
if renderer:
for entry in self._itemSection_entries(renderer):
yield entry
continuation = self._extract_continuation({'contents': continuation_items})
continue
break break
@staticmethod @staticmethod
@ -3175,7 +3177,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
selected_tab = self._extract_selected_tab(tabs) selected_tab = self._extract_selected_tab(tabs)
renderer = try_get( renderer = try_get(
data, lambda x: x['metadata']['channelMetadataRenderer'], dict) data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
playlist_id = None playlist_id = title = description = None
if renderer: if renderer:
channel_title = renderer.get('title') or item_id channel_title = renderer.get('title') or item_id
tab_title = selected_tab.get('title') tab_title = selected_tab.get('title')
@ -3191,7 +3193,9 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
description = None description = None
playlist_id = item_id playlist_id = item_id
if playlist_id is None: if playlist_id is None:
return None playlist_id = item_id
if title is None:
title = "Youtube " + playlist_id.title()
playlist = self.playlist_result( playlist = self.playlist_result(
self._entries(selected_tab['content'], identity_token), self._entries(selected_tab['content'], identity_token),
playlist_id=playlist_id, playlist_title=title, playlist_id=playlist_id, playlist_title=title,
@ -3212,7 +3216,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
url = compat_urlparse.urlunparse( url = compat_urlparse.urlunparse(
compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com')) compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com'))
is_home = re.match(r'(?P<pre>%s)(?P<post>/?(?![^#?]).*$)' % self._VALID_URL, url) is_home = re.match(r'(?P<pre>%s)(?P<post>/?(?![^#?]).*$)' % self._VALID_URL, url)
if is_home is not None and is_home.group('not_channel') is None: if is_home is not None and is_home.group('not_channel') is None and item_id != 'feed':
self._downloader.report_warning( self._downloader.report_warning(
'A channel/user page was given. All the channel\'s videos will be downloaded. ' 'A channel/user page was given. All the channel\'s videos will be downloaded. '
'To download only the videos in the home page, add a "/home" to the URL') 'To download only the videos in the home page, add a "/home" to the URL')
@ -3365,6 +3369,25 @@ class YoutubeYtUserIE(InfoExtractor):
ie=YoutubeTabIE.ie_key(), video_id=user_id) ie=YoutubeTabIE.ie_key(), video_id=user_id)
class YoutubeFavouritesIE(InfoExtractor):
IE_NAME = 'youtube:favorites'
IE_DESC = 'YouTube.com liked videos, ":ytfav" for short (requires authentication)'
_VALID_URL = r':ytfav(?:ou?rite)?s?'
_LOGIN_REQUIRED = True
_TESTS = [{
'url': ':ytfav',
'only_matching': True,
}, {
'url': ':ytfavorites',
'only_matching': True,
}]
def _real_extract(self, url):
return self.url_result(
'https://www.youtube.com/playlist?list=LL',
ie=YoutubeTabIE.ie_key())
class YoutubeSearchIE(SearchInfoExtractor, YoutubeBaseInfoExtractor): class YoutubeSearchIE(SearchInfoExtractor, YoutubeBaseInfoExtractor):
IE_DESC = 'YouTube.com searches' IE_DESC = 'YouTube.com searches'
# there doesn't appear to be a real limit, for example if you search for # there doesn't appear to be a real limit, for example if you search for
@ -3536,9 +3559,9 @@ class YoutubeFeedsInfoExtractor(YoutubeTabIE):
class YoutubeWatchLaterIE(InfoExtractor): class YoutubeWatchLaterIE(InfoExtractor):
IE_NAME = 'youtube:watchlater' IE_NAME = 'youtube:watchlater'
IE_DESC = 'Youtube watch later list, ":ytwatchlater" or "WL" for short (requires authentication)'
_VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/watch_later|:ytwatchlater|WL' _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/watch_later|:ytwatchlater|WL'
IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
_TESTS = [{ _TESTS = [{
'url': 'https://www.youtube.com/feed/watch_later', 'url': 'https://www.youtube.com/feed/watch_later',
'only_matching': True, 'only_matching': True,
@ -3552,21 +3575,6 @@ class YoutubeWatchLaterIE(InfoExtractor):
'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key()) 'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
class YoutubeFavouritesIE(InfoExtractor):
IE_NAME = 'youtube:favourites'
IE_DESC = 'YouTube.com liked videos, ":ytfav" or "LL" for short (requires authentication)'
_VALID_URL = r':ytfav(?:ou?rite)?s?|LL'
_TESTS = [{
'url': ':ytfav',
'only_matching': True,
}]
def _real_extract(self, url):
return self.url_result(
'https://www.youtube.com/playlist?list=LL', ie=YoutubeTabIE.ie_key())
class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor): class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)' IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
_VALID_URL = r'https?://(?:www\.)?youtube\.com(?:/feed/recommended|/?[?#]|/?$)|:ytrec(?:ommended)?' _VALID_URL = r'https?://(?:www\.)?youtube\.com(?:/feed/recommended|/?[?#]|/?$)|:ytrec(?:ommended)?'
@ -3575,8 +3583,8 @@ class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor): class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
IE_DESC = 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
_VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/subscriptions|:ytsub(?:scription)?s?' _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/subscriptions|:ytsub(?:scription)?s?'
IE_DESC = 'YouTube.com subscriptions feed, ":ytsubs" for short (requires authentication)'
_FEED_NAME = 'subscriptions' _FEED_NAME = 'subscriptions'
_PLAYLIST_TITLE = 'Youtube Subscriptions' _PLAYLIST_TITLE = 'Youtube Subscriptions'