bazarr/libs/tld/tests/test_core.py

843 lines
27 KiB
Python
Raw Normal View History

# -*- coding: utf-8 -*-
import copy
import logging
from os.path import abspath, join
import unittest
from tempfile import gettempdir
from typing import Type
from urllib.parse import urlsplit
from faker import Faker # type: ignore
from .. import defaults
from ..base import BaseTLDSourceParser
from ..conf import get_setting, reset_settings, set_setting
from ..exceptions import (
TldBadUrl,
TldDomainNotFound,
TldImproperlyConfigured,
TldIOError,
)
from ..helpers import project_dir
from ..registry import Registry
from ..utils import (
get_fld,
get_tld,
get_tld_names,
get_tld_names_container,
is_tld,
MozillaTLDSourceParser,
BaseMozillaTLDSourceParser,
parse_tld,
reset_tld_names,
update_tld_names,
update_tld_names_cli,
)
from .base import internet_available_only, log_info
__author__ = 'Artur Barseghyan'
__copyright__ = '2013-2020 Artur Barseghyan'
__license__ = 'MPL-1.1 OR GPL-2.0-only OR LGPL-2.1-or-later'
__all__ = ('TestCore',)
LOGGER = logging.getLogger(__name__)
class TestCore(unittest.TestCase):
"""Core tld functionality tests."""
@classmethod
def setUpClass(cls):
cls.faker = Faker()
cls.temp_dir = gettempdir()
def setUp(self):
"""Set up."""
self.good_patterns = [
{
'url': 'http://www.google.co.uk',
'fld': 'google.co.uk',
'subdomain': 'www',
'domain': 'google',
'suffix': 'co.uk',
'tld': 'co.uk',
'kwargs': {'fail_silently': True},
},
{
'url': 'http://www.v2.google.co.uk',
'fld': 'google.co.uk',
'subdomain': 'www.v2',
'domain': 'google',
'suffix': 'co.uk',
'tld': 'co.uk',
'kwargs': {'fail_silently': True},
},
# No longer valid
# {
# 'url': 'http://www.me.congresodelalengua3.ar',
# 'tld': 'me.congresodelalengua3.ar',
# 'subdomain': 'www',
# 'domain': 'me',
# 'suffix': 'congresodelalengua3.ar',
# },
{
'url': u'http://хром.гугл.рф',
'fld': u'гугл.рф',
'subdomain': u'хром',
'domain': u'гугл',
'suffix': u'рф',
'tld': u'рф',
'kwargs': {'fail_silently': True},
},
{
'url': 'http://www.google.co.uk:8001/lorem-ipsum/',
'fld': 'google.co.uk',
'subdomain': 'www',
'domain': 'google',
'suffix': 'co.uk',
'tld': 'co.uk',
'kwargs': {'fail_silently': True},
},
{
'url': 'http://www.me.cloudfront.net',
'fld': 'me.cloudfront.net',
'subdomain': 'www',
'domain': 'me',
'suffix': 'cloudfront.net',
'tld': 'cloudfront.net',
'kwargs': {'fail_silently': True},
},
{
'url': 'http://www.v2.forum.tech.google.co.uk:8001/'
'lorem-ipsum/',
'fld': 'google.co.uk',
'subdomain': 'www.v2.forum.tech',
'domain': 'google',
'suffix': 'co.uk',
'tld': 'co.uk',
'kwargs': {'fail_silently': True},
},
{
'url': 'https://pantheon.io/',
'fld': 'pantheon.io',
'subdomain': '',
'domain': 'pantheon',
'suffix': 'io',
'tld': 'io',
'kwargs': {'fail_silently': True},
},
{
'url': 'v2.www.google.com',
'fld': 'google.com',
'subdomain': 'v2.www',
'domain': 'google',
'suffix': 'com',
'tld': 'com',
'kwargs': {'fail_silently': True, 'fix_protocol': True},
},
{
'url': '//v2.www.google.com',
'fld': 'google.com',
'subdomain': 'v2.www',
'domain': 'google',
'suffix': 'com',
'tld': 'com',
'kwargs': {'fail_silently': True, 'fix_protocol': True},
},
{
'url': 'http://foo@bar.com',
'fld': 'bar.com',
'subdomain': '',
'domain': 'bar',
'suffix': 'com',
'tld': 'com',
'kwargs': {'fail_silently': True},
},
{
'url': 'http://user:foo@bar.com',
'fld': 'bar.com',
'subdomain': '',
'domain': 'bar',
'suffix': 'com',
'tld': 'com',
'kwargs': {'fail_silently': True},
},
{
'url': 'https://faguoren.xn--fiqs8s',
'fld': 'faguoren.xn--fiqs8s',
'subdomain': '',
'domain': 'faguoren',
'suffix': 'xn--fiqs8s',
'tld': 'xn--fiqs8s',
'kwargs': {'fail_silently': True},
},
{
'url': 'blogs.lemonde.paris',
'fld': 'lemonde.paris',
'subdomain': 'blogs',
'domain': 'lemonde',
'suffix': 'paris',
'tld': 'paris',
'kwargs': {'fail_silently': True, 'fix_protocol': True},
},
{
'url': 'axel.brighton.ac.uk',
'fld': 'brighton.ac.uk',
'subdomain': 'axel',
'domain': 'brighton',
'suffix': 'ac.uk',
'tld': 'ac.uk',
'kwargs': {'fail_silently': True, 'fix_protocol': True},
},
{
'url': 'm.fr.blogspot.com.au',
'fld': 'fr.blogspot.com.au',
'subdomain': 'm',
'domain': 'fr',
'suffix': 'blogspot.com.au',
'tld': 'blogspot.com.au',
'kwargs': {'fail_silently': True, 'fix_protocol': True},
},
{
'url': u'help.www.福岡.jp',
'fld': u'www.福岡.jp',
'subdomain': 'help',
'domain': 'www',
'suffix': u'福岡.jp',
'tld': u'福岡.jp',
'kwargs': {'fail_silently': True, 'fix_protocol': True},
},
{
'url': u'syria.arabic.variant.سوريا',
'fld': u'variant.سوريا',
'subdomain': 'syria.arabic',
'domain': 'variant',
'suffix': u'سوريا',
'tld': u'سوريا',
'kwargs': {'fail_silently': True, 'fix_protocol': True},
},
{
'url': u'http://www.help.kawasaki.jp',
'fld': u'www.help.kawasaki.jp',
'subdomain': '',
'domain': 'www',
'suffix': u'help.kawasaki.jp',
'tld': u'help.kawasaki.jp',
'kwargs': {'fail_silently': True},
},
{
'url': u'http://www.city.kawasaki.jp',
'fld': u'city.kawasaki.jp',
'subdomain': 'www',
'domain': 'city',
'suffix': u'kawasaki.jp',
'tld': u'kawasaki.jp',
'kwargs': {'fail_silently': True},
},
{
'url': u'http://fedoraproject.org',
'fld': u'fedoraproject.org',
'subdomain': '',
'domain': 'fedoraproject',
'suffix': u'org',
'tld': u'org',
'kwargs': {'fail_silently': True},
},
{
'url': u'http://www.cloud.fedoraproject.org',
'fld': u'www.cloud.fedoraproject.org',
'subdomain': '',
'domain': 'www',
'suffix': u'cloud.fedoraproject.org',
'tld': u'cloud.fedoraproject.org',
'kwargs': {'fail_silently': True},
},
{
'url': u'https://www.john.app.os.fedoraproject.org',
'fld': u'john.app.os.fedoraproject.org',
'subdomain': 'www',
'domain': 'john',
'suffix': u'app.os.fedoraproject.org',
'tld': u'app.os.fedoraproject.org',
'kwargs': {'fail_silently': True},
},
{
'url': 'ftp://www.xn--mxail5aa.xn--11b4c3d',
'fld': 'xn--mxail5aa.xn--11b4c3d',
'subdomain': 'www',
'domain': 'xn--mxail5aa',
'suffix': 'xn--11b4c3d',
'tld': 'xn--11b4c3d',
'kwargs': {'fail_silently': True},
},
{
'url': 'http://cloud.fedoraproject.org',
'fld': 'cloud.fedoraproject.org',
'subdomain': '',
'domain': 'cloud.fedoraproject.org',
'suffix': 'cloud.fedoraproject.org',
'tld': 'cloud.fedoraproject.org',
'kwargs': {'fail_silently': True}
},
{
'url': 'github.io',
'fld': 'github.io',
'subdomain': '',
'domain': 'github.io',
'suffix': 'github.io',
'tld': 'github.io',
'kwargs': {'fail_silently': True, 'fix_protocol': True}
},
{
'url': urlsplit('http://lemonde.fr/article.html'),
'fld': 'lemonde.fr',
'subdomain': '',
'domain': 'lemonde',
'suffix': 'fr',
'tld': 'fr',
'kwargs': {'fail_silently': True}
},
{
'url': 'https://github.com....../barseghyanartur/tld/',
'fld': 'github.com',
'subdomain': '',
'domain': 'github',
'suffix': 'com',
'tld': 'com',
'kwargs': {'fail_silently': True}
},
]
self.bad_patterns = {
'v2.www.google.com': {
'exception': TldBadUrl,
},
'/index.php?a=1&b=2': {
'exception': TldBadUrl,
},
'http://www.tld.doesnotexist': {
'exception': TldDomainNotFound,
},
'https://2001:0db8:0000:85a3:0000:0000:ac1f:8001': {
'exception': TldDomainNotFound,
},
'http://192.169.1.1': {
'exception': TldDomainNotFound,
},
'http://localhost:8080': {
'exception': TldDomainNotFound,
},
'https://localhost': {
'exception': TldDomainNotFound,
},
'https://localhost2': {
'exception': TldImproperlyConfigured,
'kwargs': {'search_public': False, 'search_private': False},
},
}
self.invalid_tlds = {
'v2.www.google.com',
'tld.doesnotexist',
'2001:0db8:0000:85a3:0000:0000:ac1f',
'192.169.1.1',
'localhost',
'google.com',
}
self.tld_names_local_path_custom = project_dir(
join(
'tests',
'res',
'effective_tld_names_custom.dat.txt'
)
)
self.good_patterns_custom_parser = [
{
'url': 'http://www.foreverchild',
'fld': 'www.foreverchild',
'subdomain': '',
'domain': 'www',
'suffix': 'foreverchild',
'tld': 'foreverchild',
'kwargs': {
'fail_silently': True,
# 'parser_class': self.get_custom_parser_class(),
},
},
{
'url': 'http://www.v2.foreverchild',
'fld': 'v2.foreverchild',
'subdomain': 'www',
'domain': 'v2',
'suffix': 'foreverchild',
'tld': 'foreverchild',
'kwargs': {
'fail_silently': True,
# 'parser_class': self.get_custom_parser_class(),
},
},
]
reset_settings()
def tearDown(self):
"""Tear down."""
reset_settings()
Registry.reset()
@property
def good_url(self):
return self.good_patterns[0]['url']
@property
def bad_url(self):
return list(self.bad_patterns.keys())[0]
def get_custom_parser_class(
self,
uid: str = 'custom_mozilla',
source_url: str = None,
local_path: str = 'tests/res/effective_tld_names_custom.dat.txt'
) -> Type[BaseTLDSourceParser]:
# Define a custom TLD source parser class
parser_class = type(
'CustomMozillaTLDSourceParser',
(BaseMozillaTLDSourceParser,),
{
'uid': uid,
'source_url': source_url,
'local_path': local_path,
}
)
return parser_class
@log_info
def test_0_tld_names_loaded(self):
"""Test if tld names are loaded."""
get_fld('http://www.google.co.uk')
from ..utils import tld_names
res = len(tld_names) > 0
self.assertTrue(res)
return res
@internet_available_only
@log_info
def test_1_update_tld_names(self):
"""Test updating the tld names (re-fetch mozilla source)."""
res = update_tld_names(fail_silently=False)
self.assertTrue(res)
return res
@log_info
def test_2_fld_good_patterns_pass(self):
"""Test good URL patterns."""
res = []
for data in self.good_patterns:
_res = get_fld(data['url'], **data['kwargs'])
self.assertEqual(_res, data['fld'])
res.append(_res)
return res
@log_info
def test_3_fld_bad_patterns_pass(self):
"""Test bad URL patterns."""
res = []
for url, params in self.bad_patterns.items():
_res = get_fld(url, fail_silently=True)
self.assertEqual(_res, None)
res.append(_res)
return res
@log_info
def test_4_override_settings(self):
"""Testing settings override."""
def override_settings():
"""Override settings."""
return get_setting('DEBUG')
self.assertEqual(defaults.DEBUG, override_settings())
set_setting('DEBUG', True)
self.assertEqual(True, override_settings())
return override_settings()
@log_info
def test_5_tld_good_patterns_pass_parsed_object(self):
"""Test good URL patterns."""
res = []
for data in self.good_patterns:
kwargs = copy.copy(data['kwargs'])
kwargs['as_object'] = True
_res = get_tld(data['url'], **kwargs)
self.assertEqual(_res.tld, data['tld'])
self.assertEqual(_res.subdomain, data['subdomain'])
self.assertEqual(_res.domain, data['domain'])
self.assertEqual(_res.suffix, data['suffix'])
self.assertEqual(_res.fld, data['fld'])
self.assertEqual(
str(_res).encode('utf8'),
data['tld'].encode('utf8')
)
self.assertEqual(
_res.__dict__,
{
'tld': _res.tld,
'domain': _res.domain,
'subdomain': _res.subdomain,
'fld': _res.fld,
'parsed_url': _res.parsed_url,
}
)
res.append(_res)
return res
@log_info
def test_6_override_full_names_path(self):
default = project_dir('dummy.txt')
override_base = '/tmp/test'
set_setting('NAMES_LOCAL_PATH_PARENT', override_base)
modified = project_dir('dummy.txt')
self.assertNotEqual(default, modified)
self.assertEqual(modified, abspath('/tmp/test/dummy.txt'))
@log_info
def test_7_public_private(self):
res = get_fld(
'http://silly.cc.ua',
fail_silently=True,
search_private=False
)
self.assertEqual(res, None)
res = get_fld(
'http://silly.cc.ua',
fail_silently=True,
search_private=True
)
self.assertEqual(res, 'silly.cc.ua')
res = get_fld(
'mercy.compute.amazonaws.com',
fail_silently=True,
search_private=False,
fix_protocol=True
)
self.assertEqual(res, None)
res = get_fld(
'http://whatever.com',
fail_silently=True,
search_public=False
)
self.assertEqual(res, None)
@log_info
def test_8_fld_bad_patterns_exceptions(self):
"""Test exceptions."""
res = []
for url, params in self.bad_patterns.items():
kwargs = params['kwargs'] if 'kwargs' in params else {}
kwargs['fail_silently'] = False
with self.assertRaises(params['exception']):
_res = get_fld(url, **kwargs)
res.append(_res)
return res
@log_info
def test_9_tld_good_patterns_pass(self):
"""Test `get_tld` good URL patterns."""
res = []
for data in self.good_patterns:
_res = get_tld(data['url'], **data['kwargs'])
self.assertEqual(_res, data['tld'])
res.append(_res)
return res
@log_info
def test_10_tld_bad_patterns_pass(self):
"""Test `get_tld` bad URL patterns."""
res = []
for url, params in self.bad_patterns.items():
_res = get_tld(url, fail_silently=True)
self.assertEqual(_res, None)
res.append(_res)
return res
@log_info
def test_11_parse_tld_good_patterns(self):
"""Test `parse_tld` good URL patterns."""
res = []
for data in self.good_patterns:
_res = parse_tld(data['url'], **data['kwargs'])
self.assertEqual(
_res,
(data['tld'], data['domain'], data['subdomain'])
)
res.append(_res)
return res
@log_info
def test_12_is_tld_good_patterns(self):
"""Test `is_tld` good URL patterns."""
for data in self.good_patterns:
self.assertTrue(is_tld(data['tld']))
@log_info
def test_13_is_tld_bad_patterns(self):
"""Test `is_tld` bad URL patterns."""
for _tld in self.invalid_tlds:
self.assertFalse(is_tld(_tld))
@log_info
def test_14_fail_update_tld_names(self):
"""Test fail `update_tld_names`."""
parser_class = self.get_custom_parser_class(
uid='custom_mozilla_2',
source_url='i-do-not-exist'
)
# Assert raise TldIOError on wrong NAMES_SOURCE_URL
with self.assertRaises(TldIOError):
update_tld_names(fail_silently=False, parser_uid=parser_class.uid)
# Assert return False on wrong NAMES_SOURCE_URL
self.assertFalse(
update_tld_names(fail_silently=True, parser_uid=parser_class.uid)
)
@log_info
def test_15_fail_get_tld_names(self):
"""Test fail `update_tld_names`."""
parser_class = self.get_custom_parser_class(
uid='custom_mozilla_3',
source_url='i-do-not-exist',
local_path='/srv/tests/res/effective_tld_names_custom_3.dat.txt'
)
reset_tld_names()
# Assert raise TldIOError on wrong NAMES_SOURCE_URL
for params in self.good_patterns:
kwargs = {'url': params['url']}
kwargs.update(params['kwargs'])
kwargs['fail_silently'] = False
kwargs['parser_class'] = parser_class
with self.assertRaises(TldIOError):
get_tld(**kwargs)
@log_info
def test_15_fail_get_fld_wrong_kwargs(self):
"""Test fail `get_fld` with wrong kwargs."""
with self.assertRaises(TldImproperlyConfigured):
get_fld(self.good_url, as_object=True)
@log_info
def test_16_fail_parse_tld(self):
"""Test fail `parse_tld`.
Assert raise TldIOError on wrong `NAMES_SOURCE_URL` for `parse_tld`.
"""
parser_class = self.get_custom_parser_class(
source_url='i-do-not-exist'
)
parsed_tld = parse_tld(
self.bad_url,
fail_silently=False,
parser_class=parser_class
)
self.assertEqual(parsed_tld, (None, None, None))
@log_info
def test_17_get_tld_names_and_reset_tld_names(self):
"""Test fail `get_tld_names` and repair using `reset_tld_names`."""
tmp_filename = join(
gettempdir(),
f'{self.faker.uuid4()}.dat.txt'
)
parser_class = self.get_custom_parser_class(
source_url='i-do-not-exist',
local_path=tmp_filename
)
reset_tld_names()
with self.subTest('Assert raise TldIOError'):
# Assert raise TldIOError on wrong NAMES_SOURCE_URL for
# `get_tld_names`
with self.assertRaises(TldIOError):
get_tld_names(
fail_silently=False,
parser_class=parser_class
)
tmp_filename = join(
gettempdir(),
f'{self.faker.uuid4()}.dat.txt'
)
parser_class_2 = self.get_custom_parser_class(
source_url='i-do-not-exist-2',
local_path=tmp_filename
)
reset_tld_names()
with self.subTest('Assert get None'):
# Assert get None on wrong `NAMES_SOURCE_URL` for `get_tld_names`
self.assertIsNone(
get_tld_names(
fail_silently=True,
parser_class=parser_class_2
)
)
@internet_available_only
@log_info
def test_18_update_tld_names_cli(self):
"""Test the return code of the CLI version of `update_tld_names`."""
reset_tld_names()
res = update_tld_names_cli()
self.assertEqual(res, 0)
@log_info
def test_19_parse_tld_custom_tld_names_good_patterns(self):
"""Test `parse_tld` good URL patterns for custom tld names."""
res = []
for data in self.good_patterns_custom_parser:
kwargs = copy.copy(data['kwargs'])
kwargs['parser_class'] = self.get_custom_parser_class()
_res = parse_tld(data['url'], **kwargs)
self.assertEqual(
_res,
(data['tld'], data['domain'], data['subdomain'])
)
res.append(_res)
return res
@log_info
def test_20_tld_custom_tld_names_good_patterns_pass_parsed_object(self):
"""Test `get_tld` good URL patterns for custom tld names."""
res = []
for data in self.good_patterns_custom_parser:
kwargs = copy.copy(data['kwargs'])
kwargs.update({
'as_object': True,
'parser_class': self.get_custom_parser_class(),
})
_res = get_tld(data['url'], **kwargs)
self.assertEqual(_res.tld, data['tld'])
self.assertEqual(_res.subdomain, data['subdomain'])
self.assertEqual(_res.domain, data['domain'])
self.assertEqual(_res.suffix, data['suffix'])
self.assertEqual(_res.fld, data['fld'])
self.assertEqual(
str(_res).encode('utf8'),
data['tld'].encode('utf8')
)
self.assertEqual(
_res.__dict__,
{
'tld': _res.tld,
'domain': _res.domain,
'subdomain': _res.subdomain,
'fld': _res.fld,
'parsed_url': _res.parsed_url,
}
)
res.append(_res)
return res
@log_info
def test_21_reset_tld_names_for_custom_parser(self):
"""Test `reset_tld_names` for `tld_names_local_path`."""
res = []
parser_class = self.get_custom_parser_class()
for data in self.good_patterns_custom_parser:
kwargs = copy.copy(data['kwargs'])
kwargs.update({
'as_object': True,
'parser_class': self.get_custom_parser_class(),
})
_res = get_tld(data['url'], **kwargs)
self.assertEqual(_res.tld, data['tld'])
self.assertEqual(_res.subdomain, data['subdomain'])
self.assertEqual(_res.domain, data['domain'])
self.assertEqual(_res.suffix, data['suffix'])
self.assertEqual(_res.fld, data['fld'])
self.assertEqual(
str(_res).encode('utf8'),
data['tld'].encode('utf8')
)
self.assertEqual(
_res.__dict__,
{
'tld': _res.tld,
'domain': _res.domain,
'subdomain': _res.subdomain,
'fld': _res.fld,
'parsed_url': _res.parsed_url,
}
)
res.append(_res)
tld_names = get_tld_names_container()
self.assertIn(parser_class.local_path, tld_names)
reset_tld_names(parser_class.local_path)
self.assertNotIn(parser_class.local_path, tld_names)
return res
@log_info
def test_22_fail_define_custom_parser_class_without_uid(self):
"""Test fail define custom parser class without `uid`."""
class CustomParser(BaseTLDSourceParser):
pass
class AnotherCustomParser(BaseTLDSourceParser):
uid = 'another-custom-parser'
# Assert raise TldImproperlyConfigured
with self.assertRaises(TldImproperlyConfigured):
CustomParser.get_tld_names()
# Assert raise NotImplementedError
with self.assertRaises(NotImplementedError):
AnotherCustomParser.get_tld_names()
@log_info
def test_23_len_trie_nodes(self):
"""Test len of the trie nodes."""
get_tld('http://delusionalinsanity.com')
tld_names = get_tld_names_container()
self.assertGreater(
len(tld_names[MozillaTLDSourceParser.local_path]),
0
)
@log_info
def test_24_get_tld_names_no_arguments(self):
"""Test len of the trie nodes."""
tld_names = get_tld_names()
self.assertGreater(
len(tld_names),
0
)
if __name__ == '__main__':
unittest.main()