From 709fe21782aefa7fac7845a254b5213210ac6a96 Mon Sep 17 00:00:00 2001 From: insaneracist Date: Sat, 31 Oct 2020 15:18:40 -0700 Subject: [PATCH 1/7] [feature] heartbeat --- youtube_dlc/downloader/common.py | 7 +++++ youtube_dlc/heartbeat.py | 51 ++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 youtube_dlc/heartbeat.py diff --git a/youtube_dlc/downloader/common.py b/youtube_dlc/downloader/common.py index 460364a0b..8803c5a95 100644 --- a/youtube_dlc/downloader/common.py +++ b/youtube_dlc/downloader/common.py @@ -7,6 +7,7 @@ import time import random from ..compat import compat_os_name +from ..heartbeat import Heartbeat from ..utils import ( decodeArgument, encodeFilename, @@ -369,6 +370,12 @@ class FileDownloader(object): '[download] Sleeping %s seconds...' % ( int(sleep_interval_sub))) time.sleep(sleep_interval_sub) + + if info_dict.get('heartbeat'): + self.heartbeat = Heartbeat(self.ydl, info_dict.get('heartbeat')) + self.add_progress_hook(self.heartbeat.check_download_status) + self.heartbeat.start() + return self.real_download(filename, info_dict) def real_download(self, filename, info_dict): diff --git a/youtube_dlc/heartbeat.py b/youtube_dlc/heartbeat.py new file mode 100644 index 000000000..a7d3229a3 --- /dev/null +++ b/youtube_dlc/heartbeat.py @@ -0,0 +1,51 @@ +import time +import threading +import traceback + +from .utils import ( + sanitized_Request +) + + +class Heartbeat(object): + def __init__(self, ydl, params): + self.ydl = ydl + + data = params.get('data') + if type(data) is str: + data = data.encode() + self.request = sanitized_Request( + params.get('url'), + data=data, + headers=params.get('headers', {}), + method=params.get('method') + ) + + self.interval = params.get('interval') + self.stopped = False + self.thread = threading.Thread(target=self.__heartbeat, daemon=True) + + def start(self): + if self.ydl.params.get('verbose'): + self.ydl.to_screen('[heartbeat] Heartbeat every %s seconds' % self.interval) + self.thread.start() + + def stop(self): + self.stopped = True + + def check_download_status(self, progress): + status = progress.get('status') + if status == 'finished' or status == 'error': + self.stop() + + def __heartbeat(self): + while not self.stopped: + try: + if self.ydl.params.get('verbose'): + self.ydl.to_screen('[heartbeat]') + self.ydl.urlopen(self.request) + except Exception: + if self.ydl.params.get('verbose'): + traceback.print_exc() + self.ydl.to_screen("[heartbeat] Heartbeat failed") + time.sleep(self.interval) From 83f45661471e56a83a59af8c467094f34e744b5f Mon Sep 17 00:00:00 2001 From: insaneracist Date: Sat, 31 Oct 2020 15:19:57 -0700 Subject: [PATCH 2/7] [niconico] fix: download video while sending heartbeat (#72) default interval --- youtube_dlc/extractor/niconico.py | 14 ++++++++++++++ youtube_dlc/heartbeat.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/youtube_dlc/extractor/niconico.py b/youtube_dlc/extractor/niconico.py index eb07ca776..6b4726bae 100644 --- a/youtube_dlc/extractor/niconico.py +++ b/youtube_dlc/extractor/niconico.py @@ -254,6 +254,19 @@ class NiconicoIE(InfoExtractor): } }).encode()) + heartbeat_url = '{}/{}?_format=json&_method=PUT'.format(session_api_endpoint['url'], session_response['data']['session']['id']) + heartbeat_headers = { + 'Accept': 'application/json', + 'Content-Type': 'application/json', + } + heartbeat = { + 'data': json.dumps(session_response['data']), + 'headers': heartbeat_headers, + 'interval': session_api_data['heartbeat_lifetime'] / 2000, + 'method': 'POST', + 'url': heartbeat_url, + } + resolution = video_quality.get('resolution', {}) return { @@ -264,6 +277,7 @@ class NiconicoIE(InfoExtractor): 'vbr': float_or_none(video_quality.get('bitrate'), 1000), 'height': resolution.get('height'), 'width': resolution.get('width'), + 'heartbeat': heartbeat, } def _real_extract(self, url): diff --git a/youtube_dlc/heartbeat.py b/youtube_dlc/heartbeat.py index a7d3229a3..7c6a234de 100644 --- a/youtube_dlc/heartbeat.py +++ b/youtube_dlc/heartbeat.py @@ -21,7 +21,7 @@ class Heartbeat(object): method=params.get('method') ) - self.interval = params.get('interval') + self.interval = params.get('interval', 30) self.stopped = False self.thread = threading.Thread(target=self.__heartbeat, daemon=True) From fa8cb27ceb04ac1599ce66f5e79a63daca7f7b79 Mon Sep 17 00:00:00 2001 From: insaneracist Date: Sat, 31 Oct 2020 15:58:53 -0700 Subject: [PATCH 3/7] import unicode_literals --- youtube_dlc/heartbeat.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/youtube_dlc/heartbeat.py b/youtube_dlc/heartbeat.py index 7c6a234de..25c0881ed 100644 --- a/youtube_dlc/heartbeat.py +++ b/youtube_dlc/heartbeat.py @@ -1,3 +1,6 @@ +# coding: utf-8 +from __future__ import unicode_literals + import time import threading import traceback From a926bd1c089feef7d7ccaa97a686e5abdd667aa6 Mon Sep 17 00:00:00 2001 From: insaneracist Date: Sun, 1 Nov 2020 04:23:39 -0800 Subject: [PATCH 4/7] prefer `cancel` to `stop` --- youtube_dlc/heartbeat.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/youtube_dlc/heartbeat.py b/youtube_dlc/heartbeat.py index 25c0881ed..94573f0f3 100644 --- a/youtube_dlc/heartbeat.py +++ b/youtube_dlc/heartbeat.py @@ -25,7 +25,7 @@ class Heartbeat(object): ) self.interval = params.get('interval', 30) - self.stopped = False + self.cancelled = False self.thread = threading.Thread(target=self.__heartbeat, daemon=True) def start(self): @@ -33,16 +33,16 @@ class Heartbeat(object): self.ydl.to_screen('[heartbeat] Heartbeat every %s seconds' % self.interval) self.thread.start() - def stop(self): - self.stopped = True + def cancel(self): + self.cancelled = True def check_download_status(self, progress): status = progress.get('status') if status == 'finished' or status == 'error': - self.stop() + self.cancel() def __heartbeat(self): - while not self.stopped: + while not self.cancelled: try: if self.ydl.params.get('verbose'): self.ydl.to_screen('[heartbeat]') From 729b407626b2a726ce90b3d7045817ab8eed73a1 Mon Sep 17 00:00:00 2001 From: insaneracist Date: Mon, 2 Nov 2020 15:22:17 -0800 Subject: [PATCH 5/7] python2: compat_str --- youtube_dlc/heartbeat.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dlc/heartbeat.py b/youtube_dlc/heartbeat.py index 94573f0f3..26d62b769 100644 --- a/youtube_dlc/heartbeat.py +++ b/youtube_dlc/heartbeat.py @@ -6,6 +6,7 @@ import threading import traceback from .utils import ( + compat_str, sanitized_Request ) @@ -15,7 +16,7 @@ class Heartbeat(object): self.ydl = ydl data = params.get('data') - if type(data) is str: + if isinstance(data, compat_str): data = data.encode() self.request = sanitized_Request( params.get('url'), From 89ada4406af61c05f4259b10e2b1d26db92653f7 Mon Sep 17 00:00:00 2001 From: insaneracist Date: Tue, 3 Nov 2020 14:00:30 -0800 Subject: [PATCH 6/7] [heartbeat] replace daemon=True with join, send exception traceback to stderr --- youtube_dlc/heartbeat.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/youtube_dlc/heartbeat.py b/youtube_dlc/heartbeat.py index 26d62b769..c67566963 100644 --- a/youtube_dlc/heartbeat.py +++ b/youtube_dlc/heartbeat.py @@ -1,12 +1,12 @@ # coding: utf-8 from __future__ import unicode_literals -import time import threading import traceback from .utils import ( compat_str, + encode_compat_str, sanitized_Request ) @@ -27,11 +27,11 @@ class Heartbeat(object): self.interval = params.get('interval', 30) self.cancelled = False - self.thread = threading.Thread(target=self.__heartbeat, daemon=True) + self.parent_thread = threading.current_thread() + self.thread = threading.Thread(target=self.__heartbeat) def start(self): - if self.ydl.params.get('verbose'): - self.ydl.to_screen('[heartbeat] Heartbeat every %s seconds' % self.interval) + self.ydl.to_screen('[heartbeat] Heartbeat every %s seconds' % self.interval) self.thread.start() def cancel(self): @@ -49,7 +49,9 @@ class Heartbeat(object): self.ydl.to_screen('[heartbeat]') self.ydl.urlopen(self.request) except Exception: + self.ydl.report_warning("[heartbeat] Heartbeat failed") if self.ydl.params.get('verbose'): - traceback.print_exc() - self.ydl.to_screen("[heartbeat] Heartbeat failed") - time.sleep(self.interval) + self.ydl.to_stderr(encode_compat_str(traceback.format_exc())) + self.parent_thread.join(self.interval) + if not self.parent_thread.is_alive(): + break From 12ab8cf6c5ed2dcaa3f912425b25f1651861f5a6 Mon Sep 17 00:00:00 2001 From: insaneracist Date: Tue, 3 Nov 2020 14:41:27 -0800 Subject: [PATCH 7/7] [heartbeat] python2: remove explicit http method --- youtube_dlc/extractor/niconico.py | 1 - youtube_dlc/heartbeat.py | 7 ++++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/youtube_dlc/extractor/niconico.py b/youtube_dlc/extractor/niconico.py index 6b4726bae..1c6172794 100644 --- a/youtube_dlc/extractor/niconico.py +++ b/youtube_dlc/extractor/niconico.py @@ -263,7 +263,6 @@ class NiconicoIE(InfoExtractor): 'data': json.dumps(session_response['data']), 'headers': heartbeat_headers, 'interval': session_api_data['heartbeat_lifetime'] / 2000, - 'method': 'POST', 'url': heartbeat_url, } diff --git a/youtube_dlc/heartbeat.py b/youtube_dlc/heartbeat.py index c67566963..0b0bb838c 100644 --- a/youtube_dlc/heartbeat.py +++ b/youtube_dlc/heartbeat.py @@ -18,11 +18,12 @@ class Heartbeat(object): data = params.get('data') if isinstance(data, compat_str): data = data.encode() + # Python 2 does not allow us to set HTTP method + # it is POST if Request has data, otherwise GET self.request = sanitized_Request( params.get('url'), data=data, - headers=params.get('headers', {}), - method=params.get('method') + headers=params.get('headers', {}) ) self.interval = params.get('interval', 30) @@ -49,7 +50,7 @@ class Heartbeat(object): self.ydl.to_screen('[heartbeat]') self.ydl.urlopen(self.request) except Exception: - self.ydl.report_warning("[heartbeat] Heartbeat failed") + self.ydl.report_warning("Heartbeat failed") if self.ydl.params.get('verbose'): self.ydl.to_stderr(encode_compat_str(traceback.format_exc())) self.parent_thread.join(self.interval)