Option to choose different downloader for different protocols

* Renamed `--external-downloader-args` to `--downloader-args`
* Added `native` as an option for the downloader
* Use similar syntax to `--downloader-args` etc. Eg: `--downloader dash:native --downloader aria2c`
* Deprecated `--hls-prefer-native` and `--hls-prefer-ffmpeg` since the same can now be done with `--downloader "m3u8:native"` and `m3u8:ffmpeg` respectively
* Split `frag_urls` protocol into `m3u8_frag_urls` and `dash_frag_urls`
* Standardize shortening of protocol names with `downloader.shorten_protocol_name`
This commit is contained in:
pukkandan 2021-04-10 20:38:33 +05:30
parent d818eb7473
commit 52a8a1e1b9
No known key found for this signature in database
GPG Key ID: 0F00D95A001F4698
7 changed files with 95 additions and 33 deletions

View File

@ -337,10 +337,6 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
--playlist-random Download playlist videos in random order --playlist-random Download playlist videos in random order
--xattr-set-filesize Set file xattribute ytdl.filesize with --xattr-set-filesize Set file xattribute ytdl.filesize with
expected file size expected file size
--hls-prefer-native Use the native HLS downloader instead of
ffmpeg
--hls-prefer-ffmpeg Use ffmpeg instead of the native HLS
downloader
--hls-use-mpegts Use the mpegts container for HLS videos; --hls-use-mpegts Use the mpegts container for HLS videos;
allowing some players to play the video allowing some players to play the video
while downloading, and reducing the chance while downloading, and reducing the chance
@ -350,10 +346,19 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
--no-hls-use-mpegts Do not use the mpegts container for HLS --no-hls-use-mpegts Do not use the mpegts container for HLS
videos. This is default when not videos. This is default when not
downloading live streams downloading live streams
--external-downloader NAME Name or path of the external downloader to --downloader [PROTO:]NAME Name or path of the external downloader to
use. Currently supports aria2c, avconv, use (optionally) prefixed by the protocols
axel, curl, ffmpeg, httpie, wget (http, ftp, m3u8, dash, rstp, rtmp, mms) to
(Recommended: aria2c) use it for. Currently supports native,
aria2c, avconv, axel, curl, ffmpeg, httpie,
wget (Recommended: aria2c). You can use
this option multiple times to set different
downloaders for different protocols. For
example, --downloader aria2c --downloader
"dash,m3u8:native" will use aria2c for
http/ftp downloads, and the native
downloader for dash/m3u8 downloads
(Alias: --external-downloader)
--downloader-args NAME:ARGS Give these arguments to the external --downloader-args NAME:ARGS Give these arguments to the external
downloader. Specify the downloader name and downloader. Specify the downloader name and
the arguments separated by a colon ":". You the arguments separated by a colon ":". You
@ -1244,6 +1249,8 @@ These are all the deprecated options and the current alternative to achieve the
--metadata-from-title FORMAT --parse-metadata "%(title)s:FORMAT" --metadata-from-title FORMAT --parse-metadata "%(title)s:FORMAT"
--prefer-avconv avconv is no longer officially supported (Alias: --no-prefer-ffmpeg) --prefer-avconv avconv is no longer officially supported (Alias: --no-prefer-ffmpeg)
--prefer-ffmpeg Default (Alias: --no-prefer-avconv) --prefer-ffmpeg Default (Alias: --no-prefer-avconv)
--hls-prefer-native --downloader "m3u8:native"
--hls-prefer-ffmpeg --downloader "m3u8:ffmpeg"
--avconv-location avconv is no longer officially supported --avconv-location avconv is no longer officially supported
-C, --call-home Not implemented -C, --call-home Not implemented
--no-call-home Default --no-call-home Default

View File

@ -111,9 +111,17 @@ from .utils import (
process_communicate_or_kill, process_communicate_or_kill,
) )
from .cache import Cache from .cache import Cache
from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER, _PLUGIN_CLASSES from .extractor import (
gen_extractor_classes,
get_info_extractor,
_LAZY_LOADER,
_PLUGIN_CLASSES
)
from .extractor.openload import PhantomJSwrapper from .extractor.openload import PhantomJSwrapper
from .downloader import get_suitable_downloader from .downloader import (
get_suitable_downloader,
shorten_protocol_name
)
from .downloader.rtmp import rtmpdump_version from .downloader.rtmp import rtmpdump_version
from .postprocessor import ( from .postprocessor import (
FFmpegFixupM3u8PP, FFmpegFixupM3u8PP,
@ -359,9 +367,13 @@ class YoutubeDL(object):
geo_bypass_country geo_bypass_country
The following options determine which downloader is picked: The following options determine which downloader is picked:
external_downloader: Executable of the external downloader to call. external_downloader: A dictionary of protocol keys and the executable of the
None or unset for standard (built-in) downloader. external downloader to use for it. The allowed protocols
hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv are default|http|ftp|m3u8|dash|rtsp|rtmp|mms.
Set the value to 'native' to use the native downloader
hls_prefer_native: Deprecated - Use external_downloader = {'m3u8': 'native'}
or {'m3u8': 'ffmpeg'} instead.
Use the native HLS downloader instead of ffmpeg/avconv
if True, otherwise use ffmpeg/avconv if False, otherwise if True, otherwise use ffmpeg/avconv if False, otherwise
use downloader suggested by extractor if None. use downloader suggested by extractor if None.
@ -2776,7 +2788,7 @@ class YoutubeDL(object):
'|', '|',
format_field(f, 'filesize', ' %s', func=format_bytes) + format_field(f, 'filesize_approx', '~%s', func=format_bytes), format_field(f, 'filesize', ' %s', func=format_bytes) + format_field(f, 'filesize_approx', '~%s', func=format_bytes),
format_field(f, 'tbr', '%4dk'), format_field(f, 'tbr', '%4dk'),
f.get('protocol').replace('http_dash_segments', 'dash').replace("native", "n").replace('niconico_', ''), shorten_protocol_name(f.get('protocol', '').replace("native", "n")),
'|', '|',
format_field(f, 'vcodec', default='unknown').replace('none', ''), format_field(f, 'vcodec', default='unknown').replace('none', ''),
format_field(f, 'vbr', '%4dk'), format_field(f, 'vbr', '%4dk'),

View File

@ -1,5 +1,6 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from ..compat import compat_str
from ..utils import ( from ..utils import (
determine_protocol, determine_protocol,
) )
@ -42,6 +43,23 @@ PROTOCOL_MAP = {
} }
def shorten_protocol_name(proto, simplify=False):
short_protocol_names = {
'm3u8_native': 'm3u8_n',
'http_dash_segments': 'dash',
'niconico_dmc': 'dmc',
}
if simplify:
short_protocol_names.update({
'https': 'http',
'ftps': 'ftp',
'm3u8_native': 'm3u8',
'm3u8_frag_urls': 'm3u8',
'dash_frag_urls': 'dash',
})
return short_protocol_names.get(proto, proto)
def get_suitable_downloader(info_dict, params={}, default=HttpFD): def get_suitable_downloader(info_dict, params={}, default=HttpFD):
"""Get the downloader class that can handle the info dict.""" """Get the downloader class that can handle the info dict."""
protocol = determine_protocol(info_dict) protocol = determine_protocol(info_dict)
@ -50,8 +68,14 @@ def get_suitable_downloader(info_dict, params={}, default=HttpFD):
# if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict): # if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict):
# return FFmpegFD # return FFmpegFD
external_downloader = params.get('external_downloader') downloaders = params.get('external_downloader')
if external_downloader is not None: external_downloader = (
downloaders if isinstance(downloaders, compat_str)
else downloaders.get(shorten_protocol_name(protocol, True), downloaders.get('default')))
if external_downloader and external_downloader.lower() == 'native':
external_downloader = 'native'
if external_downloader not in (None, 'native'):
ed = get_external_downloader(external_downloader) ed = get_external_downloader(external_downloader)
if ed.can_download(info_dict, external_downloader): if ed.can_download(info_dict, external_downloader):
return ed return ed
@ -59,6 +83,8 @@ def get_suitable_downloader(info_dict, params={}, default=HttpFD):
if protocol.startswith('m3u8'): if protocol.startswith('m3u8'):
if info_dict.get('is_live'): if info_dict.get('is_live'):
return FFmpegFD return FFmpegFD
elif external_downloader == 'native':
return HlsFD
elif _get_real_downloader(info_dict, 'frag_urls', params, None): elif _get_real_downloader(info_dict, 'frag_urls', params, None):
return HlsFD return HlsFD
elif params.get('hls_prefer_native') is True: elif params.get('hls_prefer_native') is True:
@ -70,6 +96,7 @@ def get_suitable_downloader(info_dict, params={}, default=HttpFD):
__all__ = [ __all__ = [
'get_suitable_downloader',
'FileDownloader', 'FileDownloader',
'get_suitable_downloader',
'shorten_protocol_name',
] ]

View File

@ -20,7 +20,7 @@ from ..utils import (
class DashSegmentsFD(FragmentFD): class DashSegmentsFD(FragmentFD):
""" """
Download segments in a DASH manifest. External downloaders can take over Download segments in a DASH manifest. External downloaders can take over
the fragment downloads by supporting the 'frag_urls' protocol the fragment downloads by supporting the 'dash_frag_urls' protocol
""" """
FD_NAME = 'dashsegments' FD_NAME = 'dashsegments'
@ -30,7 +30,7 @@ class DashSegmentsFD(FragmentFD):
fragments = info_dict['fragments'][:1] if self.params.get( fragments = info_dict['fragments'][:1] if self.params.get(
'test', False) else info_dict['fragments'] 'test', False) else info_dict['fragments']
real_downloader = _get_real_downloader(info_dict, 'frag_urls', self.params, None) real_downloader = _get_real_downloader(info_dict, 'dash_frag_urls', self.params, None)
ctx = { ctx = {
'filename': filename, 'filename': filename,

View File

@ -81,11 +81,15 @@ class ExternalFD(FileDownloader):
@property @property
def exe(self): def exe(self):
return self.params.get('external_downloader') return self.get_basename()
@classmethod @classmethod
def available(cls, path=None): def available(cls, path=None):
return check_executable(path or cls.get_basename(), [cls.AVAILABLE_OPT]) path = check_executable(path or cls.get_basename(), [cls.AVAILABLE_OPT])
if path:
cls.exe = path
return path
return False
@classmethod @classmethod
def supports(cls, info_dict): def supports(cls, info_dict):
@ -259,7 +263,7 @@ class WgetFD(ExternalFD):
class Aria2cFD(ExternalFD): class Aria2cFD(ExternalFD):
AVAILABLE_OPT = '-v' AVAILABLE_OPT = '-v'
SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'frag_urls') SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'dash_frag_urls', 'm3u8_frag_urls')
@staticmethod @staticmethod
def supports_manifest(manifest): def supports_manifest(manifest):
@ -310,9 +314,11 @@ class Aria2cFD(ExternalFD):
class HttpieFD(ExternalFD): class HttpieFD(ExternalFD):
AVAILABLE_OPT = '--version'
@classmethod @classmethod
def available(cls, path=None): def available(cls, path=None):
return check_executable(path or 'http', ['--version']) return ExternalFD.available(cls, path or 'http')
def _make_cmd(self, tmpfilename, info_dict): def _make_cmd(self, tmpfilename, info_dict):
cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']] cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
@ -327,7 +333,8 @@ class FFmpegFD(ExternalFD):
SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms') SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms')
@classmethod @classmethod
def available(cls, path=None): # path is ignored for ffmpeg def available(cls, path=None):
# TODO: Fix path for ffmpeg
return FFmpegPostProcessor().available return FFmpegPostProcessor().available
def _call_downloader(self, tmpfilename, info_dict): def _call_downloader(self, tmpfilename, info_dict):
@ -484,4 +491,4 @@ def get_external_downloader(external_downloader):
downloader . """ downloader . """
# Drop .exe extension on Windows # Drop .exe extension on Windows
bn = os.path.splitext(os.path.basename(external_downloader))[0] bn = os.path.splitext(os.path.basename(external_downloader))[0]
return _BY_NAME[bn] return _BY_NAME.get(bn)

View File

@ -32,7 +32,7 @@ from ..utils import (
class HlsFD(FragmentFD): class HlsFD(FragmentFD):
""" """
Download segments in a m3u8 manifest. External downloaders can take over Download segments in a m3u8 manifest. External downloaders can take over
the fragment downloads by supporting the 'frag_urls' protocol and the fragment downloads by supporting the 'm3u8_frag_urls' protocol and
re-defining 'supports_manifest' function re-defining 'supports_manifest' function
""" """
@ -95,7 +95,7 @@ class HlsFD(FragmentFD):
# fd.add_progress_hook(ph) # fd.add_progress_hook(ph)
return fd.real_download(filename, info_dict) return fd.real_download(filename, info_dict)
real_downloader = _get_real_downloader(info_dict, 'frag_urls', self.params, None) real_downloader = _get_real_downloader(info_dict, 'm3u8_frag_urls', self.params, None)
if real_downloader and not real_downloader.supports_manifest(s): if real_downloader and not real_downloader.supports_manifest(s):
real_downloader = None real_downloader = None
if real_downloader: if real_downloader:

View File

@ -639,11 +639,11 @@ def parseOpts(overrideArguments=None):
downloader.add_option( downloader.add_option(
'--hls-prefer-native', '--hls-prefer-native',
dest='hls_prefer_native', action='store_true', default=None, dest='hls_prefer_native', action='store_true', default=None,
help='Use the native HLS downloader instead of ffmpeg') help=optparse.SUPPRESS_HELP)
downloader.add_option( downloader.add_option(
'--hls-prefer-ffmpeg', '--hls-prefer-ffmpeg',
dest='hls_prefer_native', action='store_false', default=None, dest='hls_prefer_native', action='store_false', default=None,
help='Use ffmpeg instead of the native HLS downloader') help=optparse.SUPPRESS_HELP)
downloader.add_option( downloader.add_option(
'--hls-use-mpegts', '--hls-use-mpegts',
dest='hls_use_mpegts', action='store_true', default=None, dest='hls_use_mpegts', action='store_true', default=None,
@ -659,11 +659,20 @@ def parseOpts(overrideArguments=None):
'Do not use the mpegts container for HLS videos. ' 'Do not use the mpegts container for HLS videos. '
'This is default when not downloading live streams')) 'This is default when not downloading live streams'))
downloader.add_option( downloader.add_option(
'--external-downloader', '--downloader', '--external-downloader',
dest='external_downloader', metavar='NAME', dest='external_downloader', metavar='[PROTO:]NAME', default={}, type='str',
action='callback', callback=_dict_from_multiple_values_options_callback,
callback_kwargs={
'allowed_keys': 'http|ftp|m3u8|dash|rtsp|rtmp|mms',
'default_key': 'default', 'process': lambda x: x.strip()},
help=( help=(
'Name or path of the external downloader to use. ' 'Name or path of the external downloader to use (optionally) prefixed by '
'Currently supports %s (Recommended: aria2c)' % ', '.join(list_external_downloaders()))) 'the protocols (http, ftp, m3u8, dash, rstp, rtmp, mms) to use it for. '
'Currently supports native, %s (Recommended: aria2c). '
'You can use this option multiple times to set different downloaders for different protocols. '
'For example, --downloader aria2c --downloader "dash,m3u8:native" will use '
'aria2c for http/ftp downloads, and the native downloader for dash/m3u8 downloads '
'(Alias: --external-downloader)' % ', '.join(list_external_downloaders())))
downloader.add_option( downloader.add_option(
'--downloader-args', '--external-downloader-args', '--downloader-args', '--external-downloader-args',
metavar='NAME:ARGS', dest='external_downloader_args', default={}, type='str', metavar='NAME:ARGS', dest='external_downloader_args', default={}, type='str',