mirror of https://github.com/morpheus65535/bazarr
408 lines
15 KiB
Python
408 lines
15 KiB
Python
# ------------------------------------------------------------------------------- #
|
|
|
|
import logging
|
|
import requests
|
|
import sys
|
|
import ssl
|
|
|
|
from requests.adapters import HTTPAdapter
|
|
from requests.sessions import Session
|
|
from requests_toolbelt.utils import dump
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
try:
|
|
import brotli
|
|
except ImportError:
|
|
pass
|
|
|
|
try:
|
|
import copyreg
|
|
except ImportError:
|
|
import copy_reg as copyreg
|
|
|
|
try:
|
|
from urlparse import urlparse
|
|
except ImportError:
|
|
from urllib.parse import urlparse
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
from .exceptions import (
|
|
CloudflareLoopProtection,
|
|
CloudflareIUAMError
|
|
)
|
|
|
|
from .cloudflare import Cloudflare
|
|
from .user_agent import User_Agent
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
__version__ = '1.2.71'
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
|
|
class CipherSuiteAdapter(HTTPAdapter):
|
|
|
|
__attrs__ = [
|
|
'ssl_context',
|
|
'max_retries',
|
|
'config',
|
|
'_pool_connections',
|
|
'_pool_maxsize',
|
|
'_pool_block',
|
|
'source_address'
|
|
]
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
self.ssl_context = kwargs.pop('ssl_context', None)
|
|
self.cipherSuite = kwargs.pop('cipherSuite', None)
|
|
self.source_address = kwargs.pop('source_address', None)
|
|
self.server_hostname = kwargs.pop('server_hostname', None)
|
|
self.ecdhCurve = kwargs.pop('ecdhCurve', 'prime256v1')
|
|
|
|
if self.source_address:
|
|
if isinstance(self.source_address, str):
|
|
self.source_address = (self.source_address, 0)
|
|
|
|
if not isinstance(self.source_address, tuple):
|
|
raise TypeError(
|
|
"source_address must be IP address string or (ip, port) tuple"
|
|
)
|
|
|
|
if not self.ssl_context:
|
|
self.ssl_context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
|
|
|
|
self.ssl_context.orig_wrap_socket = self.ssl_context.wrap_socket
|
|
self.ssl_context.wrap_socket = self.wrap_socket
|
|
|
|
if self.server_hostname:
|
|
self.ssl_context.server_hostname = self.server_hostname
|
|
|
|
self.ssl_context.set_ciphers(self.cipherSuite)
|
|
self.ssl_context.set_ecdh_curve(self.ecdhCurve)
|
|
|
|
self.ssl_context.minimum_version = ssl.TLSVersion.TLSv1_2
|
|
self.ssl_context.maximum_version = ssl.TLSVersion.TLSv1_3
|
|
|
|
super(CipherSuiteAdapter, self).__init__(**kwargs)
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
def wrap_socket(self, *args, **kwargs):
|
|
if hasattr(self.ssl_context, 'server_hostname') and self.ssl_context.server_hostname:
|
|
kwargs['server_hostname'] = self.ssl_context.server_hostname
|
|
self.ssl_context.check_hostname = False
|
|
else:
|
|
self.ssl_context.check_hostname = True
|
|
|
|
return self.ssl_context.orig_wrap_socket(*args, **kwargs)
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
def init_poolmanager(self, *args, **kwargs):
|
|
kwargs['ssl_context'] = self.ssl_context
|
|
kwargs['source_address'] = self.source_address
|
|
return super(CipherSuiteAdapter, self).init_poolmanager(*args, **kwargs)
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
def proxy_manager_for(self, *args, **kwargs):
|
|
kwargs['ssl_context'] = self.ssl_context
|
|
kwargs['source_address'] = self.source_address
|
|
return super(CipherSuiteAdapter, self).proxy_manager_for(*args, **kwargs)
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
|
|
class CloudScraper(Session):
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
self.debug = kwargs.pop('debug', False)
|
|
|
|
self.disableCloudflareV1 = kwargs.pop('disableCloudflareV1', False)
|
|
self.delay = kwargs.pop('delay', None)
|
|
self.captcha = kwargs.pop('captcha', {})
|
|
self.doubleDown = kwargs.pop('doubleDown', True)
|
|
self.interpreter = kwargs.pop('interpreter', 'native')
|
|
|
|
self.requestPreHook = kwargs.pop('requestPreHook', None)
|
|
self.requestPostHook = kwargs.pop('requestPostHook', None)
|
|
|
|
self.cipherSuite = kwargs.pop('cipherSuite', None)
|
|
self.ecdhCurve = kwargs.pop('ecdhCurve', 'prime256v1')
|
|
self.source_address = kwargs.pop('source_address', None)
|
|
self.server_hostname = kwargs.pop('server_hostname', None)
|
|
self.ssl_context = kwargs.pop('ssl_context', None)
|
|
|
|
self.allow_brotli = kwargs.pop(
|
|
'allow_brotli',
|
|
True if 'brotli' in sys.modules.keys() else False
|
|
)
|
|
|
|
self.user_agent = User_Agent(
|
|
allow_brotli=self.allow_brotli,
|
|
browser=kwargs.pop('browser', None)
|
|
)
|
|
|
|
self._solveDepthCnt = 0
|
|
self.solveDepth = kwargs.pop('solveDepth', 3)
|
|
|
|
super(CloudScraper, self).__init__(*args, **kwargs)
|
|
|
|
# pylint: disable=E0203
|
|
if 'requests' in self.headers['User-Agent']:
|
|
# ------------------------------------------------------------------------------- #
|
|
# Set a random User-Agent if no custom User-Agent has been set
|
|
# ------------------------------------------------------------------------------- #
|
|
self.headers = self.user_agent.headers
|
|
if not self.cipherSuite:
|
|
self.cipherSuite = self.user_agent.cipherSuite
|
|
|
|
if isinstance(self.cipherSuite, list):
|
|
self.cipherSuite = ':'.join(self.cipherSuite)
|
|
|
|
self.mount(
|
|
'https://',
|
|
CipherSuiteAdapter(
|
|
cipherSuite=self.cipherSuite,
|
|
ecdhCurve=self.ecdhCurve,
|
|
server_hostname=self.server_hostname,
|
|
source_address=self.source_address,
|
|
ssl_context=self.ssl_context
|
|
)
|
|
)
|
|
|
|
# purely to allow us to pickle dump
|
|
copyreg.pickle(ssl.SSLContext, lambda obj: (obj.__class__, (obj.protocol,)))
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
# Allow us to pickle our session back with all variables
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
def __getstate__(self):
|
|
return self.__dict__
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
# Allow replacing actual web request call via subclassing
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
def perform_request(self, method, url, *args, **kwargs):
|
|
return super(CloudScraper, self).request(method, url, *args, **kwargs)
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
# Raise an Exception with no stacktrace and reset depth counter.
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
def simpleException(self, exception, msg):
|
|
self._solveDepthCnt = 0
|
|
sys.tracebacklimit = 0
|
|
raise exception(msg)
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
# debug the request via the response
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
@staticmethod
|
|
def debugRequest(req):
|
|
try:
|
|
print(dump.dump_all(req).decode('utf-8', errors='backslashreplace'))
|
|
except ValueError as e:
|
|
print(f"Debug Error: {getattr(e, 'message', e)}")
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
# Decode Brotli on older versions of urllib3 manually
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
def decodeBrotli(self, resp):
|
|
if requests.packages.urllib3.__version__ < '1.25.1' and resp.headers.get('Content-Encoding') == 'br':
|
|
if self.allow_brotli and resp._content:
|
|
resp._content = brotli.decompress(resp.content)
|
|
else:
|
|
logging.warning(
|
|
f'You\'re running urllib3 {requests.packages.urllib3.__version__}, Brotli content detected, '
|
|
'Which requires manual decompression, '
|
|
'But option allow_brotli is set to False, '
|
|
'We will not continue to decompress.'
|
|
)
|
|
|
|
return resp
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
# Our hijacker request function
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
def request(self, method, url, *args, **kwargs):
|
|
# pylint: disable=E0203
|
|
if kwargs.get('proxies') and kwargs.get('proxies') != self.proxies:
|
|
self.proxies = kwargs.get('proxies')
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
# Pre-Hook the request via user defined function.
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
if self.requestPreHook:
|
|
(method, url, args, kwargs) = self.requestPreHook(
|
|
self,
|
|
method,
|
|
url,
|
|
*args,
|
|
**kwargs
|
|
)
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
# Make the request via requests.
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
response = self.decodeBrotli(
|
|
self.perform_request(method, url, *args, **kwargs)
|
|
)
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
# Debug the request via the Response object.
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
if self.debug:
|
|
self.debugRequest(response)
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
# Post-Hook the request aka Post-Hook the response via user defined function.
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
if self.requestPostHook:
|
|
newResponse = self.requestPostHook(self, response)
|
|
|
|
if response != newResponse: # Give me walrus in 3.7!!!
|
|
response = newResponse
|
|
if self.debug:
|
|
print('==== requestPostHook Debug ====')
|
|
self.debugRequest(response)
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
if not self.disableCloudflareV1:
|
|
cloudflareV1 = Cloudflare(self)
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
# Check if Cloudflare v1 anti-bot is on
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
if cloudflareV1.is_Challenge_Request(response):
|
|
# ------------------------------------------------------------------------------- #
|
|
# Try to solve the challenge and send it back
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
if self._solveDepthCnt >= self.solveDepth:
|
|
_ = self._solveDepthCnt
|
|
self.simpleException(
|
|
CloudflareLoopProtection,
|
|
f"!!Loop Protection!! We have tried to solve {_} time(s) in a row."
|
|
)
|
|
|
|
self._solveDepthCnt += 1
|
|
|
|
response = cloudflareV1.Challenge_Response(response, **kwargs)
|
|
else:
|
|
if not response.is_redirect and response.status_code not in [429, 503]:
|
|
self._solveDepthCnt = 0
|
|
|
|
return response
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
@classmethod
|
|
def create_scraper(cls, sess=None, **kwargs):
|
|
"""
|
|
Convenience function for creating a ready-to-go CloudScraper object.
|
|
"""
|
|
scraper = cls(**kwargs)
|
|
|
|
if sess:
|
|
for attr in ['auth', 'cert', 'cookies', 'headers', 'hooks', 'params', 'proxies', 'data']:
|
|
val = getattr(sess, attr, None)
|
|
if val is not None:
|
|
setattr(scraper, attr, val)
|
|
|
|
return scraper
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
# Functions for integrating cloudscraper with other applications and scripts
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
@classmethod
|
|
def get_tokens(cls, url, **kwargs):
|
|
scraper = cls.create_scraper(
|
|
**{
|
|
field: kwargs.pop(field, None) for field in [
|
|
'allow_brotli',
|
|
'browser',
|
|
'debug',
|
|
'delay',
|
|
'doubleDown',
|
|
'captcha',
|
|
'interpreter',
|
|
'source_address',
|
|
'requestPreHook',
|
|
'requestPostHook'
|
|
] if field in kwargs
|
|
}
|
|
)
|
|
|
|
try:
|
|
resp = scraper.get(url, **kwargs)
|
|
resp.raise_for_status()
|
|
except Exception:
|
|
logging.error(f'"{url}" returned an error. Could not collect tokens.')
|
|
raise
|
|
|
|
domain = urlparse(resp.url).netloc
|
|
# noinspection PyUnusedLocal
|
|
cookie_domain = None
|
|
|
|
for d in scraper.cookies.list_domains():
|
|
if d.startswith('.') and d in (f'.{domain}'):
|
|
cookie_domain = d
|
|
break
|
|
else:
|
|
cls.simpleException(
|
|
cls,
|
|
CloudflareIUAMError,
|
|
"Unable to find Cloudflare cookies. Does the site actually "
|
|
"have Cloudflare IUAM (I'm Under Attack Mode) enabled?"
|
|
)
|
|
|
|
return (
|
|
{
|
|
'cf_clearance': scraper.cookies.get('cf_clearance', '', domain=cookie_domain)
|
|
},
|
|
scraper.headers['User-Agent']
|
|
)
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
@classmethod
|
|
def get_cookie_string(cls, url, **kwargs):
|
|
"""
|
|
Convenience function for building a Cookie HTTP header value.
|
|
"""
|
|
tokens, user_agent = cls.get_tokens(url, **kwargs)
|
|
return '; '.join('='.join(pair) for pair in tokens.items()), user_agent
|
|
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
if ssl.OPENSSL_VERSION_INFO < (1, 1, 1):
|
|
print(
|
|
f"DEPRECATION: The OpenSSL being used by this python install ({ssl.OPENSSL_VERSION}) does not meet the minimum supported "
|
|
"version (>= OpenSSL 1.1.1) in order to support TLS 1.3 required by Cloudflare, "
|
|
"You may encounter an unexpected Captcha or cloudflare 1020 blocks."
|
|
)
|
|
|
|
# ------------------------------------------------------------------------------- #
|
|
|
|
create_scraper = CloudScraper.create_scraper
|
|
session = CloudScraper.create_scraper
|
|
get_tokens = CloudScraper.get_tokens
|
|
get_cookie_string = CloudScraper.get_cookie_string
|