From 4bb6b02f93e633dbba7bb722c167bf3b725cd7ce Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 8 Jul 2021 21:03:13 +0530 Subject: [PATCH] Improve `extractor_args` parsing --- Changelog.md | 4 ++-- yt_dlp/extractor/common.py | 13 +++++++++++-- yt_dlp/extractor/funimation.py | 4 ++-- yt_dlp/extractor/youtube.py | 12 ++++++------ yt_dlp/options.py | 11 ++++++++--- 5 files changed, 29 insertions(+), 15 deletions(-) diff --git a/Changelog.md b/Changelog.md index 8eebff95e..d8e818b65 100644 --- a/Changelog.md +++ b/Changelog.md @@ -22,9 +22,9 @@ ### 2021.07.07 * Merge youtube-dl: Upto [commit/a803582](https://github.com/ytdl-org/youtube-dl/commit/a8035827177d6b59aca03bd717acb6a9bdd75ada) -* Add `--extractor-args` to pass extractor-specific arguments +* Add `--extractor-args` to pass some extractor-specific arguments. See [readme](https://github.com/yt-dlp/yt-dlp#extractor-arguments) * Add extractor option `skip` for `youtube`. Eg: `--extractor-args youtube:skip=hls,dash` - * Deprecates --youtube-skip-dash-manifest, --youtube-skip-hls-manifest, --youtube-include-dash-manifest, --youtube-include-hls-manifest + * Deprecates `--youtube-skip-dash-manifest`, `--youtube-skip-hls-manifest`, `--youtube-include-dash-manifest`, `--youtube-include-hls-manifest` * Allow `--list...` options to work with `--print`, `--quiet` and other `--list...` options * [youtube] Use `player` API for additional video extraction requests by [colethedj](https://github.com/colethedj) * **Fixes youtube premium music** (format 141) extraction diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index 17d2e7158..07f413733 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -3498,9 +3498,18 @@ class InfoExtractor(object): else 'public' if all_known else None) - def _configuration_arg(self, key): - return traverse_obj( + def _configuration_arg(self, key, default=NO_DEFAULT, casesense=False): + ''' + @returns A list of values for the extractor argument given by "key" + or "default" if no such key is present + @param default The default value to return when the key is not present (default: []) + @param casesense When false, the values are converted to lower case + ''' + val = traverse_obj( self._downloader.params, ('extractor_args', self.ie_key().lower(), key)) + if val is None: + return [] if default is NO_DEFAULT else default + return list(val) if casesense else [x.lower() for x in val] class SearchInfoExtractor(InfoExtractor): diff --git a/yt_dlp/extractor/funimation.py b/yt_dlp/extractor/funimation.py index 4690c5234..4c61d126b 100644 --- a/yt_dlp/extractor/funimation.py +++ b/yt_dlp/extractor/funimation.py @@ -186,8 +186,8 @@ class FunimationIE(InfoExtractor): for lang, version, fmt in self._get_experiences(episode): experience_id = str(fmt['experienceId']) if (only_initial_experience and experience_id != initial_experience_id - or requested_languages and lang not in requested_languages - or requested_versions and version not in requested_versions): + or requested_languages and lang.lower() not in requested_languages + or requested_versions and version.lower() not in requested_versions): continue thumbnails.append({'url': fmt.get('poster')}) duration = max(duration, fmt.get('duration', 0)) diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py index 1233cc399..de70fcdd3 100644 --- a/yt_dlp/extractor/youtube.py +++ b/yt_dlp/extractor/youtube.py @@ -2207,11 +2207,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor): player_url = self._extract_player_url(ytcfg, webpage) - player_client = try_get(self._configuration_arg('player_client'), lambda x: x[0], str) or '' - if player_client.upper() not in ('WEB', 'ANDROID'): - player_client = 'WEB' - force_mobile_client = player_client.upper() == 'ANDROID' - player_skip = self._configuration_arg('player_skip') or [] + player_client = (self._configuration_arg('player_client') or [''])[0] + if player_client not in ('web', 'android', ''): + self.report_warning(f'Invalid player_client {player_client} given. Falling back to WEB') + force_mobile_client = player_client == 'android' + player_skip = self._configuration_arg('player_skip') def get_text(x): if not x: @@ -2489,7 +2489,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): dct['container'] = dct['ext'] + '_dash' formats.append(dct) - skip_manifests = self._configuration_arg('skip') or [] + skip_manifests = self._configuration_arg('skip') get_dash = 'dash' not in skip_manifests and self.get_param('youtube_include_dash_manifest', True) get_hls = 'hls' not in skip_manifests and self.get_param('youtube_include_hls_manifest', True) diff --git a/yt_dlp/options.py b/yt_dlp/options.py index 5caf4cb53..64bc380e1 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -137,7 +137,11 @@ def parseOpts(overrideArguments=None): else: raise optparse.OptionValueError( 'wrong %s formatting; it should be %s, not "%s"' % (opt_str, option.metavar, value)) - val = process(val) if callable(process) else val + try: + val = process(val) if process else val + except Exception as err: + raise optparse.OptionValueError( + 'wrong %s formatting; %s' % (opt_str, err)) for key in keys: out_dict[key] = val @@ -1344,6 +1348,7 @@ def parseOpts(overrideArguments=None): '--no-hls-split-discontinuity', dest='hls_split_discontinuity', action='store_false', help='Do not split HLS playlists to different formats at discontinuities such as ad breaks (default)') + _extractor_arg_parser = lambda key, vals='': (key.strip().lower(), [val.strip() for val in vals.split(',')]) extractor.add_option( '--extractor-args', metavar='KEY:ARGS', dest='extractor_args', default={}, type='str', @@ -1351,11 +1356,11 @@ def parseOpts(overrideArguments=None): callback_kwargs={ 'multiple_keys': False, 'process': lambda val: dict( - (lambda x: (x[0], x[1].split(',')))(arg.split('=', 1) + ['', '']) for arg in val.split(';')) + _extractor_arg_parser(*arg.split('=', 1)) for arg in val.split(';')) }, help=( 'Pass these arguments to the extractor. See "EXTRACTOR ARGUMENTS" for details. ' - 'You can use this option multiple times to give different arguments to different extractors')) + 'You can use this option multiple times to give arguments for different extractors')) extractor.add_option( '--youtube-include-dash-manifest', '--no-youtube-skip-dash-manifest', action='store_true', dest='youtube_include_dash_manifest', default=True,