From 7789f8c1a03f15bdc23d74c511000832f52091ed Mon Sep 17 00:00:00 2001 From: Michiel van Baak Date: Sat, 13 Mar 2021 15:18:16 +0100 Subject: [PATCH 1/7] Use dogpile.cache sha1_mangle_key to mangle cache keys When using the subtitle hashes as cache keys, sometimes they come out as filenames of 270 characters. Not a lot of filesystems out there support filenames with over 250 characters. This behaviour was reported with ext4 and zfs on both linux and FreeBSD. The dogpile.cache utils come with a function that returns a hex digest of the key, limiting the filename to a fixed length of 40 characters. --- bazarr/init.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/bazarr/init.py b/bazarr/init.py index 4ef90ffdb..bfe2d07f9 100644 --- a/bazarr/init.py +++ b/bazarr/init.py @@ -11,7 +11,9 @@ from get_args import args from logger import configure_logging from helper import path_mappings +from dogpile.cache import make_region from dogpile.cache.region import register_backend as register_cache_backend +from dogpile.cache.util import sha1_mangle_key import subliminal import datetime @@ -109,12 +111,19 @@ if not os.path.exists(os.path.join(args.config_dir, 'db', 'bazarr.db')): # upgrade database schema from database import db_upgrade + db_upgrade() # Configure dogpile file caching for Subliminal request register_cache_backend("subzero.cache.file", "subzero.cache_backends.file", "SZFileBackend") -subliminal.region.configure('subzero.cache.file', expiration_time=datetime.timedelta(days=30), - arguments={'appname': "sz_cache", 'app_cache_dir': args.config_dir}) + +subliminal.region = make_region( + key_mangler=sha1_mangle_key +).configure( + 'subzero.cache.file', + expiration_time=datetime.timedelta(days=30), + arguments={'appname': "sz_cache", 'app_cache_dir': args.config_dir} +) subliminal.region.backend.sync() if not os.path.exists(os.path.join(args.config_dir, 'config', 'releases.txt')): @@ -177,20 +186,20 @@ with open(os.path.normpath(os.path.join(args.config_dir, 'config', 'config.ini') def init_binaries(): from utils import get_binary exe = get_binary("unrar") - + rarfile.UNRAR_TOOL = exe rarfile.ORIG_UNRAR_TOOL = exe try: rarfile.custom_check([rarfile.UNRAR_TOOL], True) except: logging.debug("custom check failed for: %s", exe) - + rarfile.OPEN_ARGS = rarfile.ORIG_OPEN_ARGS rarfile.EXTRACT_ARGS = rarfile.ORIG_EXTRACT_ARGS rarfile.TEST_ARGS = rarfile.ORIG_TEST_ARGS logging.debug("Using UnRAR from: %s", exe) unrar = exe - + return unrar From c68377f40dbbd0f0e656ba588979158bf5e4fcaf Mon Sep 17 00:00:00 2001 From: release-it Date: Sat, 13 Mar 2021 14:19:07 +0000 Subject: [PATCH 2/7] Release 0.9.3-beta.10 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7530b95fc..98ba7a4c6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.9.3-beta.9 \ No newline at end of file +0.9.3-beta.10 \ No newline at end of file From 87fbd196a404dbc7c6e12caf39b309c692f8e7e8 Mon Sep 17 00:00:00 2001 From: Michiel van Baak Date: Sun, 14 Mar 2021 00:04:18 +0100 Subject: [PATCH 3/7] Make subliminal always mangle cache keys to prevent long filenames * Use dogpile.cache sha1_mangle_key to mangle cache keys When using the subtitle hashes as cache keys, sometimes they come out as filenames of 270 characters. Not a lot of filesystems out there support filenames with over 250 characters. This behaviour was reported with ext4 and zfs on both linux and FreeBSD. The dogpile.cache utils come with a function that returns a hex digest of the key, limiting the filename to a fixed length of 40 characters. * Set region to subliminal.region and only then configure it, instead of assigning the result of configure to the property * Make sure subliminal ALWAYS mangles keys with the sha1 to prevent filenames that are too long --- bazarr/init.py | 7 +------ libs/subliminal/cache.py | 5 +++-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/bazarr/init.py b/bazarr/init.py index bfe2d07f9..cb107d890 100644 --- a/bazarr/init.py +++ b/bazarr/init.py @@ -11,9 +11,7 @@ from get_args import args from logger import configure_logging from helper import path_mappings -from dogpile.cache import make_region from dogpile.cache.region import register_backend as register_cache_backend -from dogpile.cache.util import sha1_mangle_key import subliminal import datetime @@ -116,10 +114,7 @@ db_upgrade() # Configure dogpile file caching for Subliminal request register_cache_backend("subzero.cache.file", "subzero.cache_backends.file", "SZFileBackend") - -subliminal.region = make_region( - key_mangler=sha1_mangle_key -).configure( +subliminal.region.configure( 'subzero.cache.file', expiration_time=datetime.timedelta(days=30), arguments={'appname': "sz_cache", 'app_cache_dir': args.config_dir} diff --git a/libs/subliminal/cache.py b/libs/subliminal/cache.py index 3d8848e1d..40344c4e4 100644 --- a/libs/subliminal/cache.py +++ b/libs/subliminal/cache.py @@ -3,6 +3,7 @@ from __future__ import absolute_import import datetime from dogpile.cache import make_region +from dogpile.cache.util import sha1_mangle_key #: Expiration time for show caching SHOW_EXPIRATION_TIME = datetime.timedelta(weeks=3).total_seconds() @@ -13,5 +14,5 @@ EPISODE_EXPIRATION_TIME = datetime.timedelta(days=3).total_seconds() #: Expiration time for scraper searches REFINER_EXPIRATION_TIME = datetime.timedelta(weeks=1).total_seconds() - -region = make_region() +# Mangle keys to prevent long filenames +region = make_region(key_mangler=sha1_mangle_key) From 9a84cc8da3ab1096d2ab3bf903a28a321ce8478c Mon Sep 17 00:00:00 2001 From: release-it Date: Sat, 13 Mar 2021 23:05:08 +0000 Subject: [PATCH 4/7] Release 0.9.3-beta.11 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 98ba7a4c6..496cd3c63 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.9.3-beta.10 \ No newline at end of file +0.9.3-beta.11 \ No newline at end of file From f10bac1fad0c3a147483e151b4115b3d45fe1442 Mon Sep 17 00:00:00 2001 From: Michiel van Baak Date: Sun, 14 Mar 2021 11:34:12 +0100 Subject: [PATCH 5/7] Revert "Make subliminal always mangle cache keys to prevent long filenames" This reverts commit 87fbd196a404dbc7c6e12caf39b309c692f8e7e8. --- bazarr/init.py | 7 ++++++- libs/subliminal/cache.py | 5 ++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bazarr/init.py b/bazarr/init.py index cb107d890..bfe2d07f9 100644 --- a/bazarr/init.py +++ b/bazarr/init.py @@ -11,7 +11,9 @@ from get_args import args from logger import configure_logging from helper import path_mappings +from dogpile.cache import make_region from dogpile.cache.region import register_backend as register_cache_backend +from dogpile.cache.util import sha1_mangle_key import subliminal import datetime @@ -114,7 +116,10 @@ db_upgrade() # Configure dogpile file caching for Subliminal request register_cache_backend("subzero.cache.file", "subzero.cache_backends.file", "SZFileBackend") -subliminal.region.configure( + +subliminal.region = make_region( + key_mangler=sha1_mangle_key +).configure( 'subzero.cache.file', expiration_time=datetime.timedelta(days=30), arguments={'appname': "sz_cache", 'app_cache_dir': args.config_dir} diff --git a/libs/subliminal/cache.py b/libs/subliminal/cache.py index 40344c4e4..3d8848e1d 100644 --- a/libs/subliminal/cache.py +++ b/libs/subliminal/cache.py @@ -3,7 +3,6 @@ from __future__ import absolute_import import datetime from dogpile.cache import make_region -from dogpile.cache.util import sha1_mangle_key #: Expiration time for show caching SHOW_EXPIRATION_TIME = datetime.timedelta(weeks=3).total_seconds() @@ -14,5 +13,5 @@ EPISODE_EXPIRATION_TIME = datetime.timedelta(days=3).total_seconds() #: Expiration time for scraper searches REFINER_EXPIRATION_TIME = datetime.timedelta(weeks=1).total_seconds() -# Mangle keys to prevent long filenames -region = make_region(key_mangler=sha1_mangle_key) + +region = make_region() From 75d49e2f3366871bd694fe981ae271f8f1a19600 Mon Sep 17 00:00:00 2001 From: Michiel van Baak Date: Sun, 14 Mar 2021 11:34:31 +0100 Subject: [PATCH 6/7] Revert "Use dogpile.cache sha1_mangle_key to mangle cache keys" This reverts commit 7789f8c1a03f15bdc23d74c511000832f52091ed. --- bazarr/init.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/bazarr/init.py b/bazarr/init.py index bfe2d07f9..4ef90ffdb 100644 --- a/bazarr/init.py +++ b/bazarr/init.py @@ -11,9 +11,7 @@ from get_args import args from logger import configure_logging from helper import path_mappings -from dogpile.cache import make_region from dogpile.cache.region import register_backend as register_cache_backend -from dogpile.cache.util import sha1_mangle_key import subliminal import datetime @@ -111,19 +109,12 @@ if not os.path.exists(os.path.join(args.config_dir, 'db', 'bazarr.db')): # upgrade database schema from database import db_upgrade - db_upgrade() # Configure dogpile file caching for Subliminal request register_cache_backend("subzero.cache.file", "subzero.cache_backends.file", "SZFileBackend") - -subliminal.region = make_region( - key_mangler=sha1_mangle_key -).configure( - 'subzero.cache.file', - expiration_time=datetime.timedelta(days=30), - arguments={'appname': "sz_cache", 'app_cache_dir': args.config_dir} -) +subliminal.region.configure('subzero.cache.file', expiration_time=datetime.timedelta(days=30), + arguments={'appname': "sz_cache", 'app_cache_dir': args.config_dir}) subliminal.region.backend.sync() if not os.path.exists(os.path.join(args.config_dir, 'config', 'releases.txt')): @@ -186,20 +177,20 @@ with open(os.path.normpath(os.path.join(args.config_dir, 'config', 'config.ini') def init_binaries(): from utils import get_binary exe = get_binary("unrar") - + rarfile.UNRAR_TOOL = exe rarfile.ORIG_UNRAR_TOOL = exe try: rarfile.custom_check([rarfile.UNRAR_TOOL], True) except: logging.debug("custom check failed for: %s", exe) - + rarfile.OPEN_ARGS = rarfile.ORIG_OPEN_ARGS rarfile.EXTRACT_ARGS = rarfile.ORIG_EXTRACT_ARGS rarfile.TEST_ARGS = rarfile.ORIG_TEST_ARGS logging.debug("Using UnRAR from: %s", exe) unrar = exe - + return unrar From 99a529037d10f0b276396de310eafd9a0b4d2716 Mon Sep 17 00:00:00 2001 From: release-it Date: Sun, 14 Mar 2021 11:45:31 +0000 Subject: [PATCH 7/7] Release 0.9.3-beta.12 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 496cd3c63..6481142e5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.9.3-beta.11 \ No newline at end of file +0.9.3-beta.12 \ No newline at end of file