from __future__ import unicode_literals import os.path import optparse import re import sys from .downloader.external import list_external_downloaders from .compat import ( compat_expanduser, compat_get_terminal_size, compat_getenv, compat_kwargs, compat_shlex_split, ) from .utils import ( expand_path, get_executable_path, OUTTMPL_TYPES, preferredencoding, REMUX_EXTENSIONS, write_string, ) from .version import __version__ def _hide_login_info(opts): PRIVATE_OPTS = set(['-p', '--password', '-u', '--username', '--video-password', '--ap-password', '--ap-username']) eqre = re.compile('^(?P' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$') def _scrub_eq(o): m = eqre.match(o) if m: return m.group('key') + '=PRIVATE' else: return o opts = list(map(_scrub_eq, opts)) for idx, opt in enumerate(opts): if opt in PRIVATE_OPTS and idx + 1 < len(opts): opts[idx + 1] = 'PRIVATE' return opts def parseOpts(overrideArguments=None): def _readOptions(filename_bytes, default=[]): try: optionf = open(filename_bytes) except IOError: return default # silently skip if file is not present try: # FIXME: https://github.com/ytdl-org/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56 contents = optionf.read() if sys.version_info < (3,): contents = contents.decode(preferredencoding()) res = compat_shlex_split(contents, comments=True) finally: optionf.close() return res def _readUserConf(package_name, default=[]): # .config xdg_config_home = compat_getenv('XDG_CONFIG_HOME') or compat_expanduser('~/.config') userConfFile = os.path.join(xdg_config_home, package_name, 'config') if not os.path.isfile(userConfFile): userConfFile = os.path.join(xdg_config_home, '%s.conf' % package_name) userConf = _readOptions(userConfFile, default=None) if userConf is not None: return userConf, userConfFile # appdata appdata_dir = compat_getenv('appdata') if appdata_dir: userConfFile = os.path.join(appdata_dir, package_name, 'config') userConf = _readOptions(userConfFile, default=None) if userConf is None: userConfFile += '.txt' userConf = _readOptions(userConfFile, default=None) if userConf is not None: return userConf, userConfFile # home userConfFile = os.path.join(compat_expanduser('~'), '%s.conf' % package_name) userConf = _readOptions(userConfFile, default=None) if userConf is None: userConfFile += '.txt' userConf = _readOptions(userConfFile, default=None) if userConf is not None: return userConf, userConfFile return default, None def _format_option_string(option): ''' ('-o', '--option') -> -o, --format METAVAR''' opts = [] if option._short_opts: opts.append(option._short_opts[0]) if option._long_opts: opts.append(option._long_opts[0]) if len(opts) > 1: opts.insert(1, ', ') if option.takes_value(): opts.append(' %s' % option.metavar) return ''.join(opts) def _comma_separated_values_options_callback(option, opt_str, value, parser, prepend=True): setattr( parser.values, option.dest, value.split(',') if not prepend else value.split(',') + getattr(parser.values, option.dest)) def _dict_from_multiple_values_options_callback( option, opt_str, value, parser, allowed_keys=r'[\w-]+', delimiter=':', default_key=None, process=None, multiple_keys=True): out_dict = getattr(parser.values, option.dest) if multiple_keys: allowed_keys = r'(%s)(,(%s))*' % (allowed_keys, allowed_keys) mobj = re.match(r'(?i)(?P%s)%s(?P.*)$' % (allowed_keys, delimiter), value) if mobj is not None: keys = [k.strip() for k in mobj.group('keys').lower().split(',')] val = mobj.group('val') elif default_key is not None: keys, val = [default_key], value else: raise optparse.OptionValueError( 'wrong %s formatting; it should be %s, not "%s"' % (opt_str, option.metavar, value)) val = process(val) if callable(process) else val for key in keys: out_dict[key] = val # No need to wrap help messages if we're on a wide console columns = compat_get_terminal_size().columns max_width = columns if columns else 80 max_help_position = 80 fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position) fmt.format_option_strings = _format_option_string kw = { 'version': __version__, 'formatter': fmt, 'usage': '%prog [OPTIONS] URL [URL...]', 'conflict_handler': 'resolve', } parser = optparse.OptionParser(**compat_kwargs(kw)) general = optparse.OptionGroup(parser, 'General Options') general.add_option( '-h', '--help', action='help', help='Print this help text and exit') general.add_option( '--version', action='version', help='Print program version and exit') general.add_option( '-U', '--update', action='store_true', dest='update_self', help='Update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)') general.add_option( '-i', '--ignore-errors', '--no-abort-on-error', action='store_true', dest='ignoreerrors', default=True, help='Continue on download errors, for example to skip unavailable videos in a playlist (default) (Alias: --no-abort-on-error)') general.add_option( '--abort-on-error', '--no-ignore-errors', action='store_false', dest='ignoreerrors', help='Abort downloading of further videos if an error occurs (Alias: --no-ignore-errors)') general.add_option( '--dump-user-agent', action='store_true', dest='dump_user_agent', default=False, help='Display the current browser identification') general.add_option( '--list-extractors', action='store_true', dest='list_extractors', default=False, help='List all supported extractors') general.add_option( '--extractor-descriptions', action='store_true', dest='list_extractor_descriptions', default=False, help='Output descriptions of all supported extractors') general.add_option( '--force-generic-extractor', action='store_true', dest='force_generic_extractor', default=False, help='Force extraction to use the generic extractor') general.add_option( '--default-search', dest='default_search', metavar='PREFIX', help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching') general.add_option( '--ignore-config', '--no-config', action='store_true', help=( 'Disable loading any configuration files except the one provided by --config-location. ' 'When given inside a configuration file, no further configuration files are loaded. ' 'Additionally, (for backward compatibility) if this option is found inside the ' 'system configuration file, the user configuration is not loaded')) general.add_option( '--config-location', dest='config_location', metavar='PATH', help='Location of the main configuration file; either the path to the config or its containing directory') general.add_option( '--flat-playlist', action='store_const', dest='extract_flat', const='in_playlist', default=False, help='Do not extract the videos of a playlist, only list them') general.add_option( '--flat-videos', action='store_true', dest='extract_flat', # help='Do not resolve the video urls') # doesn't work help=optparse.SUPPRESS_HELP) general.add_option( '--no-flat-playlist', action='store_false', dest='extract_flat', help='Extract the videos of a playlist') general.add_option( '--mark-watched', action='store_true', dest='mark_watched', default=False, help='Mark videos watched (YouTube only)') general.add_option( '--no-mark-watched', action='store_false', dest='mark_watched', help='Do not mark videos watched (default)') general.add_option( '--no-colors', action='store_true', dest='no_color', default=False, help='Do not emit color codes in output') network = optparse.OptionGroup(parser, 'Network Options') network.add_option( '--proxy', dest='proxy', default=None, metavar='URL', help=( 'Use the specified HTTP/HTTPS/SOCKS proxy. To enable ' 'SOCKS proxy, specify a proper scheme. For example ' 'socks5://127.0.0.1:1080/. Pass in an empty string (--proxy "") ' 'for direct connection')) network.add_option( '--socket-timeout', dest='socket_timeout', type=float, default=None, metavar='SECONDS', help='Time to wait before giving up, in seconds') network.add_option( '--source-address', metavar='IP', dest='source_address', default=None, help='Client-side IP address to bind to', ) network.add_option( '-4', '--force-ipv4', action='store_const', const='0.0.0.0', dest='source_address', help='Make all connections via IPv4', ) network.add_option( '-6', '--force-ipv6', action='store_const', const='::', dest='source_address', help='Make all connections via IPv6', ) geo = optparse.OptionGroup(parser, 'Geo-restriction') geo.add_option( '--geo-verification-proxy', dest='geo_verification_proxy', default=None, metavar='URL', help=( 'Use this proxy to verify the IP address for some geo-restricted sites. ' 'The default proxy specified by --proxy (or none, if the option is not present) is used for the actual downloading')) geo.add_option( '--cn-verification-proxy', dest='cn_verification_proxy', default=None, metavar='URL', help=optparse.SUPPRESS_HELP) geo.add_option( '--geo-bypass', action='store_true', dest='geo_bypass', default=True, help='Bypass geographic restriction via faking X-Forwarded-For HTTP header') geo.add_option( '--no-geo-bypass', action='store_false', dest='geo_bypass', default=True, help='Do not bypass geographic restriction via faking X-Forwarded-For HTTP header') geo.add_option( '--geo-bypass-country', metavar='CODE', dest='geo_bypass_country', default=None, help='Force bypass geographic restriction with explicitly provided two-letter ISO 3166-2 country code') geo.add_option( '--geo-bypass-ip-block', metavar='IP_BLOCK', dest='geo_bypass_ip_block', default=None, help='Force bypass geographic restriction with explicitly provided IP block in CIDR notation') selection = optparse.OptionGroup(parser, 'Video Selection') selection.add_option( '--playlist-start', dest='playliststart', metavar='NUMBER', default=1, type=int, help='Playlist video to start at (default is %default)') selection.add_option( '--playlist-end', dest='playlistend', metavar='NUMBER', default=None, type=int, help='Playlist video to end at (default is last)') selection.add_option( '--playlist-items', dest='playlist_items', metavar='ITEM_SPEC', default=None, help='Playlist video items to download. Specify indices of the videos in the playlist separated by commas like: "--playlist-items 1,2,5,8" if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13') selection.add_option( '--match-title', dest='matchtitle', metavar='REGEX', help='Download only matching titles (regex or caseless sub-string)') selection.add_option( '--reject-title', dest='rejecttitle', metavar='REGEX', help='Skip download for matching titles (regex or caseless sub-string)') selection.add_option( '--max-downloads', dest='max_downloads', metavar='NUMBER', type=int, default=None, help='Abort after downloading NUMBER files') selection.add_option( '--min-filesize', metavar='SIZE', dest='min_filesize', default=None, help='Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)') selection.add_option( '--max-filesize', metavar='SIZE', dest='max_filesize', default=None, help='Do not download any videos larger than SIZE (e.g. 50k or 44.6m)') selection.add_option( '--date', metavar='DATE', dest='date', default=None, help=( 'Download only videos uploaded in this date. ' 'The date can be "YYYYMMDD" or in the format ' '"(now|today)[+-][0-9](day|week|month|year)(s)?"')) selection.add_option( '--datebefore', metavar='DATE', dest='datebefore', default=None, help=( 'Download only videos uploaded on or before this date. ' 'The date formats accepted is the same as --date')) selection.add_option( '--dateafter', metavar='DATE', dest='dateafter', default=None, help=( 'Download only videos uploaded on or after this date. ' 'The date formats accepted is the same as --date')) selection.add_option( '--min-views', metavar='COUNT', dest='min_views', default=None, type=int, help='Do not download any videos with less than COUNT views') selection.add_option( '--max-views', metavar='COUNT', dest='max_views', default=None, type=int, help='Do not download any videos with more than COUNT views') selection.add_option( '--match-filter', metavar='FILTER', dest='match_filter', default=None, help=( 'Generic video filter. ' 'Specify any key (see "OUTPUT TEMPLATE" for a list of available keys) to ' 'match if the key is present, ' '!key to check if the key is not present, ' 'key>NUMBER (like "view_count > 12", also works with ' '>=, <, <=, !=, =) to compare against a number, ' 'key = \'LITERAL\' (like "uploader = \'Mike Smith\'", also works with !=) ' 'to match against a string literal ' 'and & to require multiple matches. ' 'Values which are not known are excluded unless you ' 'put a question mark (?) after the operator. ' 'For example, to only match videos that have been liked more than ' '100 times and disliked less than 50 times (or the dislike ' 'functionality is not available at the given service), but who ' 'also have a description, use --match-filter ' '"like_count > 100 & dislike_count m4a/mov>mp4/mkv" will remux aac to m4a, mov to mp4 ' 'and anything else to mkv.' % '|'.join(REMUX_EXTENSIONS))) postproc.add_option( '--recode-video', metavar='FORMAT', dest='recodevideo', default=None, help=( 'Re-encode the video into another format if re-encoding is necessary. ' 'The supported formats are the same as --remux-video')) postproc.add_option( '--postprocessor-args', '--ppa', metavar='NAME:ARGS', dest='postprocessor_args', default={}, type='str', action='callback', callback=_dict_from_multiple_values_options_callback, callback_kwargs={ 'allowed_keys': r'\w+(?:\+\w+)?', 'default_key': 'default-compat', 'process': compat_shlex_split, 'multiple_keys': False}, help=( 'Give these arguments to the postprocessors. ' 'Specify the postprocessor/executable name and the arguments separated by a colon ":" ' 'to give the argument to the specified postprocessor/executable. Supported PP are: ' 'Merger, ExtractAudio, SplitChapters, Metadata, EmbedSubtitle, EmbedThumbnail, ' 'SubtitlesConvertor, ThumbnailsConvertor, VideoRemuxer, VideoConvertor, ' 'SponSkrub, FixupStretched, FixupM4a and FixupM3u8. ' 'The supported executables are: AtomicParsley, FFmpeg, FFprobe, and SponSkrub. ' 'You can also specify "PP+EXE:ARGS" to give the arguments to the specified executable ' 'only when being used by the specified postprocessor. Additionally, for ffmpeg/ffprobe, ' '"_i"/"_o" can be appended to the prefix optionally followed by a number to pass the argument ' 'before the specified input/output file. Eg: --ppa "Merger+ffmpeg_i1:-v quiet". ' 'You can use this option multiple times to give different arguments to different ' 'postprocessors. (Alias: --ppa)')) postproc.add_option( '-k', '--keep-video', action='store_true', dest='keepvideo', default=False, help='Keep the intermediate video file on disk after post-processing') postproc.add_option( '--no-keep-video', action='store_false', dest='keepvideo', help='Delete the intermediate video file after post-processing (default)') postproc.add_option( '--post-overwrites', action='store_false', dest='nopostoverwrites', help='Overwrite post-processed files (default)') postproc.add_option( '--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False, help='Do not overwrite post-processed files') postproc.add_option( '--embed-subs', action='store_true', dest='embedsubtitles', default=False, help='Embed subtitles in the video (only for mp4, webm and mkv videos)') postproc.add_option( '--no-embed-subs', action='store_false', dest='embedsubtitles', help='Do not embed subtitles (default)') postproc.add_option( '--embed-thumbnail', action='store_true', dest='embedthumbnail', default=False, help='Embed thumbnail in the audio as cover art') postproc.add_option( '--no-embed-thumbnail', action='store_false', dest='embedthumbnail', help='Do not embed thumbnail (default)') postproc.add_option( '--add-metadata', action='store_true', dest='addmetadata', default=False, help='Write metadata to the video file') postproc.add_option( '--no-add-metadata', action='store_false', dest='addmetadata', help='Do not write metadata (default)') postproc.add_option( '--metadata-from-title', metavar='FORMAT', dest='metafromtitle', help=optparse.SUPPRESS_HELP) postproc.add_option( '--parse-metadata', metavar='FROM:TO', dest='metafromfield', action='append', help=( 'Parse additional metadata like title/artist from other fields; ' 'see "MODIFYING METADATA" for details')) postproc.add_option( '--xattrs', action='store_true', dest='xattrs', default=False, help='Write metadata to the video file\'s xattrs (using dublin core and xdg standards)') postproc.add_option( '--fixup', metavar='POLICY', dest='fixup', default=None, help=( 'Automatically correct known faults of the file. ' 'One of never (do nothing), warn (only emit a warning), ' 'detect_or_warn (the default; fix file if we can, warn otherwise)')) postproc.add_option( '--prefer-avconv', '--no-prefer-ffmpeg', action='store_false', dest='prefer_ffmpeg', help=optparse.SUPPRESS_HELP) postproc.add_option( '--prefer-ffmpeg', '--no-prefer-avconv', action='store_true', dest='prefer_ffmpeg', default=True, help=optparse.SUPPRESS_HELP) postproc.add_option( '--ffmpeg-location', '--avconv-location', metavar='PATH', dest='ffmpeg_location', help='Location of the ffmpeg binary; either the path to the binary or its containing directory') postproc.add_option( '--exec', metavar='CMD', dest='exec_cmd', help=( 'Execute a command on the file after downloading and post-processing. ' 'Similar syntax to the output template can be used to pass any field as arguments to the command. ' 'An additional field "filepath" that contains the final path of the downloaded file is also available. ' 'If no fields are passed, "%(filepath)s" is appended to the end of the command')) postproc.add_option( '--convert-subs', '--convert-sub', '--convert-subtitles', metavar='FORMAT', dest='convertsubtitles', default=None, help='Convert the subtitles to another format (currently supported: srt|ass|vtt|lrc) (Alias: --convert-subtitles)') postproc.add_option( '--convert-thumbnails', metavar='FORMAT', dest='convertthumbnails', default=None, help='Convert the thumbnails to another format (currently supported: jpg)') postproc.add_option( '--split-chapters', '--split-tracks', dest='split_chapters', action='store_true', default=False, help=( 'Split video into multiple files based on internal chapters. ' 'The "chapter:" prefix can be used with "--paths" and "--output" to ' 'set the output filename for the split files. See "OUTPUT TEMPLATE" for details')) postproc.add_option( '--no-split-chapters', '--no-split-tracks', dest='split_chapters', action='store_false', help='Do not split video based on chapters (default)') sponskrub = optparse.OptionGroup(parser, 'SponSkrub (SponsorBlock) Options', description=( 'SponSkrub (https://github.com/yt-dlp/SponSkrub) is a utility to mark/remove sponsor segments ' 'from downloaded YouTube videos using SponsorBlock API (https://sponsor.ajay.app)')) sponskrub.add_option( '--sponskrub', action='store_true', dest='sponskrub', default=None, help=( 'Use sponskrub to mark sponsored sections. ' 'This is enabled by default if the sponskrub binary exists (Youtube only)')) sponskrub.add_option( '--no-sponskrub', action='store_false', dest='sponskrub', help='Do not use sponskrub') sponskrub.add_option( '--sponskrub-cut', default=False, action='store_true', dest='sponskrub_cut', help='Cut out the sponsor sections instead of simply marking them') sponskrub.add_option( '--no-sponskrub-cut', action='store_false', dest='sponskrub_cut', help='Simply mark the sponsor sections, not cut them out (default)') sponskrub.add_option( '--sponskrub-force', default=False, action='store_true', dest='sponskrub_force', help='Run sponskrub even if the video was already downloaded') sponskrub.add_option( '--no-sponskrub-force', action='store_true', dest='sponskrub_force', help='Do not cut out the sponsor sections if the video was already downloaded (default)') sponskrub.add_option( '--sponskrub-location', metavar='PATH', dest='sponskrub_path', default='', help='Location of the sponskrub binary; either the path to the binary or its containing directory') sponskrub.add_option( '--sponskrub-args', dest='sponskrub_args', metavar='ARGS', help=optparse.SUPPRESS_HELP) extractor = optparse.OptionGroup(parser, 'Extractor Options') extractor.add_option( '--extractor-retries', dest='extractor_retries', metavar='RETRIES', default=3, help='Number of retries for known extractor errors (default is %default), or "infinite"') extractor.add_option( '--allow-dynamic-mpd', '--no-ignore-dynamic-mpd', action='store_true', dest='dynamic_mpd', default=True, help='Process dynamic DASH manifests (default) (Alias: --no-ignore-dynamic-mpd)') extractor.add_option( '--ignore-dynamic-mpd', '--no-allow-dynamic-mpd', action='store_false', dest='dynamic_mpd', help='Do not process dynamic DASH manifests (Alias: --no-allow-dynamic-mpd)') extractor.add_option( '--hls-split-discontinuity', dest='hls_split_discontinuity', action='store_true', default=False, help='Split HLS playlists to different formats at discontinuities such as ad breaks' ) extractor.add_option( '--no-hls-split-discontinuity', dest='hls_split_discontinuity', action='store_false', help='Do not split HLS playlists to different formats at discontinuities such as ad breaks (default)') extractor.add_option( '--youtube-include-dash-manifest', '--no-youtube-skip-dash-manifest', action='store_true', dest='youtube_include_dash_manifest', default=True, help='Download the DASH manifests and related data on YouTube videos (default) (Alias: --no-youtube-skip-dash-manifest)') extractor.add_option( '--youtube-skip-dash-manifest', '--no-youtube-include-dash-manifest', action='store_false', dest='youtube_include_dash_manifest', help='Do not download the DASH manifests and related data on YouTube videos (Alias: --no-youtube-include-dash-manifest)') extractor.add_option( '--youtube-include-hls-manifest', '--no-youtube-skip-hls-manifest', action='store_true', dest='youtube_include_hls_manifest', default=True, help='Download the HLS manifests and related data on YouTube videos (default) (Alias: --no-youtube-skip-hls-manifest)') extractor.add_option( '--youtube-skip-hls-manifest', '--no-youtube-include-hls-manifest', action='store_false', dest='youtube_include_hls_manifest', help='Do not download the HLS manifests and related data on YouTube videos (Alias: --no-youtube-include-hls-manifest)') parser.add_option_group(general) parser.add_option_group(network) parser.add_option_group(geo) parser.add_option_group(selection) parser.add_option_group(downloader) parser.add_option_group(filesystem) parser.add_option_group(thumbnail) parser.add_option_group(link) parser.add_option_group(verbosity) parser.add_option_group(workarounds) parser.add_option_group(video_format) parser.add_option_group(subtitles) parser.add_option_group(authentication) parser.add_option_group(postproc) parser.add_option_group(sponskrub) parser.add_option_group(extractor) if overrideArguments is not None: opts, args = parser.parse_args(overrideArguments) if opts.verbose: write_string('[debug] Override config: ' + repr(overrideArguments) + '\n') else: def compat_conf(conf): if sys.version_info < (3,): return [a.decode(preferredencoding(), 'replace') for a in conf] return conf configs = { 'command-line': compat_conf(sys.argv[1:]), 'custom': [], 'home': [], 'portable': [], 'user': [], 'system': []} paths = {'command-line': False} opts, args = parser.parse_args(configs['command-line']) def get_configs(): if '--config-location' in configs['command-line']: location = compat_expanduser(opts.config_location) if os.path.isdir(location): location = os.path.join(location, 'yt-dlp.conf') if not os.path.exists(location): parser.error('config-location %s does not exist.' % location) configs['custom'] = _readOptions(location, default=None) if configs['custom'] is None: configs['custom'] = [] else: paths['custom'] = location if '--ignore-config' in configs['command-line']: return if '--ignore-config' in configs['custom']: return def read_options(path, user=False): # Multiple package names can be given here # Eg: ('yt-dlp', 'youtube-dlc', 'youtube-dl') will look for # the configuration file of any of these three packages for package in ('yt-dlp',): if user: config, current_path = _readUserConf(package, default=None) else: current_path = os.path.join(path, '%s.conf' % package) config = _readOptions(current_path, default=None) if config is not None: return config, current_path return [], None configs['portable'], paths['portable'] = read_options(get_executable_path()) if '--ignore-config' in configs['portable']: return def get_home_path(): opts = parser.parse_args(configs['portable'] + configs['custom'] + configs['command-line'])[0] return expand_path(opts.paths.get('home', '')).strip() configs['home'], paths['home'] = read_options(get_home_path()) if '--ignore-config' in configs['home']: return configs['system'], paths['system'] = read_options('/etc') if '--ignore-config' in configs['system']: return configs['user'], paths['user'] = read_options('', True) if '--ignore-config' in configs['user']: configs['system'], paths['system'] = [], None get_configs() argv = configs['system'] + configs['user'] + configs['home'] + configs['portable'] + configs['custom'] + configs['command-line'] opts, args = parser.parse_args(argv) if opts.verbose: for label in ('System', 'User', 'Portable', 'Home', 'Custom', 'Command-line'): key = label.lower() if paths.get(key) is None: continue if paths[key]: write_string('[debug] %s config file: %s\n' % (label, paths[key])) write_string('[debug] %s config: %s\n' % (label, repr(_hide_login_info(configs[key])))) return parser, opts, args