diff --git a/src/NzbDrone.Common.Test/CacheTests/CachedDictionaryFixture.cs b/src/NzbDrone.Common.Test/CacheTests/CachedDictionaryFixture.cs new file mode 100644 index 000000000..23781d033 --- /dev/null +++ b/src/NzbDrone.Common.Test/CacheTests/CachedDictionaryFixture.cs @@ -0,0 +1,102 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Common.Cache; + +namespace NzbDrone.Common.Test.CacheTests +{ + [TestFixture] + public class CachedDictionaryFixture + { + private CachedDictionary _cachedString; + private DictionaryWorker _worker; + + [SetUp] + public void SetUp() + { + _worker = new DictionaryWorker(); + _cachedString = new CachedDictionary(_worker.GetDict, TimeSpan.FromMilliseconds(100)); + } + + [Test] + public void should_not_fetch_on_create() + { + _worker.HitCount.Should().Be(0); + } + + [Test] + public void should_fetch_on_first_call() + { + var result = _cachedString.Get("Hi"); + + _worker.HitCount.Should().Be(1); + + result.Should().Be("Value"); + } + + [Test] + public void should_fetch_once() + { + var result1 = _cachedString.Get("Hi"); + var result2 = _cachedString.Get("HitCount"); + + _worker.HitCount.Should().Be(1); + } + + [Test] + public void should_auto_refresh_after_lifetime() + { + var result1 = _cachedString.Get("Hi"); + + Thread.Sleep(200); + + var result2 = _cachedString.Get("Hi"); + + _worker.HitCount.Should().Be(2); + } + + [Test] + public void should_refresh_early_if_requested() + { + var result1 = _cachedString.Get("Hi"); + + Thread.Sleep(10); + + _cachedString.RefreshIfExpired(TimeSpan.FromMilliseconds(1)); + + var result2 = _cachedString.Get("Hi"); + + _worker.HitCount.Should().Be(2); + } + + [Test] + public void should_not_refresh_early_if_not_expired() + { + var result1 = _cachedString.Get("Hi"); + + _cachedString.RefreshIfExpired(TimeSpan.FromMilliseconds(50)); + + var result2 = _cachedString.Get("Hi"); + + _worker.HitCount.Should().Be(1); + } + } + + public class DictionaryWorker + { + public int HitCount { get; private set; } + + public Dictionary GetDict() + { + HitCount++; + + var result = new Dictionary(); + result["Hi"] = "Value"; + result["HitCount"] = "Hit count is " + HitCount; + + return result; + } + } +} \ No newline at end of file diff --git a/src/NzbDrone.Common.Test/NzbDrone.Common.Test.csproj b/src/NzbDrone.Common.Test/NzbDrone.Common.Test.csproj index cfc284f3a..723fa7cb0 100644 --- a/src/NzbDrone.Common.Test/NzbDrone.Common.Test.csproj +++ b/src/NzbDrone.Common.Test/NzbDrone.Common.Test.csproj @@ -66,6 +66,7 @@ + diff --git a/src/NzbDrone.Common/Cache/CacheManager.cs b/src/NzbDrone.Common/Cache/CacheManager.cs index e702b6b45..ec44a8fa8 100644 --- a/src/NzbDrone.Common/Cache/CacheManager.cs +++ b/src/NzbDrone.Common/Cache/CacheManager.cs @@ -6,8 +6,9 @@ namespace NzbDrone.Common.Cache { public interface ICacheManager { - ICached GetCache(Type host, string name); ICached GetCache(Type host); + ICached GetCache(Type host, string name); + ICachedDictionary GetCacheDictionary(Type host, string name, Func> fetchFunc = null, TimeSpan? lifeTime = null); void Clear(); ICollection Caches { get; } } @@ -22,12 +23,6 @@ namespace NzbDrone.Common.Cache } - public ICached GetCache(Type host) - { - Ensure.That(host, () => host).IsNotNull(); - return GetCache(host, host.FullName); - } - public void Clear() { _cache.Clear(); @@ -35,6 +30,12 @@ namespace NzbDrone.Common.Cache public ICollection Caches { get { return _cache.Values; } } + public ICached GetCache(Type host) + { + Ensure.That(host, () => host).IsNotNull(); + return GetCache(host, host.FullName); + } + public ICached GetCache(Type host, string name) { Ensure.That(host, () => host).IsNotNull(); @@ -42,5 +43,13 @@ namespace NzbDrone.Common.Cache return (ICached)_cache.Get(host.FullName + "_" + name, () => new Cached()); } + + public ICachedDictionary GetCacheDictionary(Type host, string name, Func> fetchFunc = null, TimeSpan? lifeTime = null) + { + Ensure.That(host, () => host).IsNotNull(); + Ensure.That(name, () => name).IsNotNullOrWhiteSpace(); + + return (ICachedDictionary)_cache.Get("dict_" + host.FullName + "_" + name, () => new CachedDictionary(fetchFunc, lifeTime)); + } } } \ No newline at end of file diff --git a/src/NzbDrone.Common/Cache/CachedDictionary.cs b/src/NzbDrone.Common/Cache/CachedDictionary.cs new file mode 100644 index 000000000..f57ba60bd --- /dev/null +++ b/src/NzbDrone.Common/Cache/CachedDictionary.cs @@ -0,0 +1,137 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; + +namespace NzbDrone.Common.Cache +{ + + public class CachedDictionary : ICachedDictionary + { + private readonly Func> _fetchFunc; + private readonly TimeSpan? _ttl; + + private DateTime _lastRefreshed = DateTime.MinValue; + private ConcurrentDictionary _items = new ConcurrentDictionary(); + + public CachedDictionary(Func> fetchFunc = null, TimeSpan? ttl = null) + { + _fetchFunc = fetchFunc; + _ttl = ttl; + } + + public bool IsExpired(TimeSpan ttl) + { + return _lastRefreshed.Add(ttl) < DateTime.UtcNow; + } + + public void RefreshIfExpired() + { + if (_ttl.HasValue && _fetchFunc != null) + { + RefreshIfExpired(_ttl.Value); + } + } + + public void RefreshIfExpired(TimeSpan ttl) + { + if (IsExpired(ttl)) + { + Refresh(); + } + } + + public void Refresh() + { + if (_fetchFunc == null) + { + throw new InvalidOperationException("Cannot update cache without data source."); + } + + Update(_fetchFunc()); + ExtendTTL(); + } + + public void Update(IDictionary items) + { + _items = new ConcurrentDictionary(items); + ExtendTTL(); + } + + public void ExtendTTL() + { + _lastRefreshed = DateTime.UtcNow; + } + + [DebuggerBrowsable(DebuggerBrowsableState.Never)] + public ICollection Values + { + get + { + RefreshIfExpired(); + return _items.Values; + } + } + + [DebuggerBrowsable(DebuggerBrowsableState.Never)] + public int Count + { + get + { + RefreshIfExpired(); + return _items.Count; + } + } + + public TValue Get(string key) + { + RefreshIfExpired(); + + TValue result; + + if (!_items.TryGetValue(key, out result)) + { + throw new KeyNotFoundException(string.Format("Item {0} not found in cache.", key)); + } + + return result; + } + + public TValue Find(string key) + { + RefreshIfExpired(); + + TValue result; + + _items.TryGetValue(key, out result); + + return result; + } + + public void Clear() + { + _items.Clear(); + _lastRefreshed = DateTime.MinValue; + } + + public void ClearExpired() + { + if (!_ttl.HasValue) + { + throw new InvalidOperationException("Checking expiry without ttl not possible."); + } + + if (IsExpired(_ttl.Value)) + { + Clear(); + } + } + + public void Remove(string key) + { + TValue item; + _items.TryRemove(key, out item); + } + } +} diff --git a/src/NzbDrone.Common/Cache/ICachedDictionary.cs b/src/NzbDrone.Common/Cache/ICachedDictionary.cs new file mode 100644 index 000000000..164fbed8b --- /dev/null +++ b/src/NzbDrone.Common/Cache/ICachedDictionary.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace NzbDrone.Common.Cache +{ + public interface ICachedDictionary : ICached + { + void RefreshIfExpired(); + void RefreshIfExpired(TimeSpan ttl); + void Refresh(); + void Update(IDictionary items); + void ExtendTTL(); + TValue Get(string key); + TValue Find(string key); + bool IsExpired(TimeSpan ttl); + } +} diff --git a/src/NzbDrone.Common/Http/HttpClient.cs b/src/NzbDrone.Common/Http/HttpClient.cs index dc3dc7fed..2c126372b 100644 --- a/src/NzbDrone.Common/Http/HttpClient.cs +++ b/src/NzbDrone.Common/Http/HttpClient.cs @@ -28,7 +28,6 @@ namespace NzbDrone.Common.Http private readonly Logger _logger; private readonly IRateLimitService _rateLimitService; private readonly ICached _cookieContainerCache; - private readonly ICached _curlTLSFallbackCache; private readonly List _requestInterceptors; private readonly IHttpDispatcher _httpDispatcher; diff --git a/src/NzbDrone.Common/NzbDrone.Common.csproj b/src/NzbDrone.Common/NzbDrone.Common.csproj index 098a02b1d..d424ae350 100644 --- a/src/NzbDrone.Common/NzbDrone.Common.csproj +++ b/src/NzbDrone.Common/NzbDrone.Common.csproj @@ -64,7 +64,9 @@ + + diff --git a/src/NzbDrone.Core.Test/DataAugmentation/SceneNumbering/XemServiceFixture.cs b/src/NzbDrone.Core.Test/DataAugmentation/SceneNumbering/XemServiceFixture.cs index 4adfe2a19..936b48aff 100644 --- a/src/NzbDrone.Core.Test/DataAugmentation/SceneNumbering/XemServiceFixture.cs +++ b/src/NzbDrone.Core.Test/DataAugmentation/SceneNumbering/XemServiceFixture.cs @@ -11,6 +11,7 @@ using NzbDrone.Core.DataAugmentation.Xem.Model; using NzbDrone.Core.Test.Framework; using NzbDrone.Core.Tv; using NzbDrone.Core.Tv.Events; +using NzbDrone.Test.Common; namespace NzbDrone.Core.Test.DataAugmentation.SceneNumbering { @@ -144,6 +145,25 @@ namespace NzbDrone.Core.Test.DataAugmentation.SceneNumbering Mocker.GetMock() .Verify(v => v.UpdateSeries(It.IsAny()), Times.Never()); + + ExceptionVerification.ExpectedWarns(1); + } + + [Test] + public void should_not_clear_scenenumbering_if_thexem_throws() + { + GivenExistingMapping(); + + Mocker.GetMock() + .Setup(v => v.GetXemSeriesIds()) + .Throws(new InvalidOperationException()); + + Subject.Handle(new SeriesUpdatedEvent(_series)); + + Mocker.GetMock() + .Verify(v => v.UpdateSeries(It.IsAny()), Times.Never()); + + ExceptionVerification.ExpectedWarns(1); } [Test] diff --git a/src/NzbDrone.Core/DataAugmentation/Scene/SceneMappingService.cs b/src/NzbDrone.Core/DataAugmentation/Scene/SceneMappingService.cs index 325a4ccda..9194ffcac 100644 --- a/src/NzbDrone.Core/DataAugmentation/Scene/SceneMappingService.cs +++ b/src/NzbDrone.Core/DataAugmentation/Scene/SceneMappingService.cs @@ -28,8 +28,8 @@ namespace NzbDrone.Core.DataAugmentation.Scene private readonly IEnumerable _sceneMappingProviders; private readonly IEventAggregator _eventAggregator; private readonly Logger _logger; - private readonly ICached> _getTvdbIdCache; - private readonly ICached> _findByTvdbIdCache; + private readonly ICachedDictionary> _getTvdbIdCache; + private readonly ICachedDictionary> _findByTvdbIdCache; public SceneMappingService(ISceneMappingRepository repository, ICacheManager cacheManager, @@ -40,10 +40,10 @@ namespace NzbDrone.Core.DataAugmentation.Scene _repository = repository; _sceneMappingProviders = sceneMappingProviders; _eventAggregator = eventAggregator; - - _getTvdbIdCache = cacheManager.GetCache>(GetType(), "tvdb_id"); - _findByTvdbIdCache = cacheManager.GetCache>(GetType(), "find_tvdb_id"); _logger = logger; + + _getTvdbIdCache = cacheManager.GetCacheDictionary>(GetType(), "tvdb_id"); + _findByTvdbIdCache = cacheManager.GetCacheDictionary>(GetType(), "find_tvdb_id"); } public List GetSceneNames(int tvdbId, IEnumerable seasonNumbers) @@ -143,6 +143,7 @@ namespace NzbDrone.Core.DataAugmentation.Scene } RefreshCache(); + _eventAggregator.PublishEvent(new SceneMappingsUpdatedEvent()); } @@ -184,18 +185,8 @@ namespace NzbDrone.Core.DataAugmentation.Scene { var mappings = _repository.All().ToList(); - _getTvdbIdCache.Clear(); - _findByTvdbIdCache.Clear(); - - foreach (var sceneMapping in mappings.GroupBy(v => v.ParseTerm)) - { - _getTvdbIdCache.Set(sceneMapping.Key, sceneMapping.ToList()); - } - - foreach (var sceneMapping in mappings.GroupBy(x => x.TvdbId)) - { - _findByTvdbIdCache.Set(sceneMapping.Key.ToString(), sceneMapping.ToList()); - } + _getTvdbIdCache.Update(mappings.GroupBy(v => v.ParseTerm).ToDictionary(v => v.Key, v => v.ToList())); + _findByTvdbIdCache.Update(mappings.GroupBy(v => v.TvdbId).ToDictionary(v => v.Key.ToString(), v => v.ToList())); } private List FilterNonEnglish(List titles) @@ -205,7 +196,10 @@ namespace NzbDrone.Core.DataAugmentation.Scene public void Handle(SeriesRefreshStartingEvent message) { - UpdateMappings(); + if (message.ManualTrigger && _findByTvdbIdCache.IsExpired(TimeSpan.FromMinutes(1))) + { + UpdateMappings(); + } } public void Execute(UpdateSceneMappingCommand message) diff --git a/src/NzbDrone.Core/DataAugmentation/Xem/XemService.cs b/src/NzbDrone.Core/DataAugmentation/Xem/XemService.cs index 591f5c9a0..d42ca07ea 100644 --- a/src/NzbDrone.Core/DataAugmentation/Xem/XemService.cs +++ b/src/NzbDrone.Core/DataAugmentation/Xem/XemService.cs @@ -16,7 +16,7 @@ namespace NzbDrone.Core.DataAugmentation.Xem private readonly IXemProxy _xemProxy; private readonly ISeriesService _seriesService; private readonly Logger _logger; - private readonly ICached _cache; + private readonly ICachedDictionary _cache; public XemService(IEpisodeService episodeService, IXemProxy xemProxy, @@ -26,7 +26,7 @@ namespace NzbDrone.Core.DataAugmentation.Xem _xemProxy = xemProxy; _seriesService = seriesService; _logger = logger; - _cache = cacheManager.GetCache(GetType()); + _cache = cacheManager.GetCacheDictionary(GetType(), "mappedTvdbid"); } private void PerformUpdate(Series series) @@ -40,7 +40,6 @@ namespace NzbDrone.Core.DataAugmentation.Xem if (!mappings.Any() && !series.UseSceneNumbering) { _logger.Debug("Mappings for: {0} are empty, skipping", series); - _cache.Remove(series.TvdbId.ToString()); return; } @@ -171,18 +170,25 @@ namespace NzbDrone.Core.DataAugmentation.Xem } } - private void RefreshCache() + private void UpdateXemSeriesIds() { - var ids = _xemProxy.GetXemSeriesIds(); - - if (ids.Any()) + try { - _cache.Clear(); + var ids = _xemProxy.GetXemSeriesIds(); + + if (ids.Any()) + { + _cache.Update(ids.ToDictionary(v => v.ToString(), v => true)); + return; + } + + _cache.ExtendTTL(); + _logger.Warn("Failed to update Xem series list."); } - - foreach (var id in ids) + catch (Exception ex) { - _cache.Set(id.ToString(), true, TimeSpan.FromHours(1)); + _cache.ExtendTTL(); + _logger.Warn(ex, "Failed to update Xem series list."); } } @@ -206,9 +212,9 @@ namespace NzbDrone.Core.DataAugmentation.Xem public void Handle(SeriesUpdatedEvent message) { - if (_cache.Count == 0) + if (_cache.IsExpired(TimeSpan.FromHours(3))) { - RefreshCache(); + UpdateXemSeriesIds(); } if (_cache.Count == 0) @@ -228,7 +234,10 @@ namespace NzbDrone.Core.DataAugmentation.Xem public void Handle(SeriesRefreshStartingEvent message) { - RefreshCache(); + if (message.ManualTrigger && _cache.IsExpired(TimeSpan.FromMinutes(1))) + { + UpdateXemSeriesIds(); + } } } } diff --git a/src/NzbDrone.Core/Tv/Events/SeriesRefreshStartingEvent.cs b/src/NzbDrone.Core/Tv/Events/SeriesRefreshStartingEvent.cs index d0e8cca16..e330b0004 100644 --- a/src/NzbDrone.Core/Tv/Events/SeriesRefreshStartingEvent.cs +++ b/src/NzbDrone.Core/Tv/Events/SeriesRefreshStartingEvent.cs @@ -4,5 +4,11 @@ namespace NzbDrone.Core.Tv.Events { public class SeriesRefreshStartingEvent : IEvent { + public bool ManualTrigger { get; set; } + + public SeriesRefreshStartingEvent(bool manualTrigger) + { + ManualTrigger = manualTrigger; + } } } \ No newline at end of file diff --git a/src/NzbDrone.Core/Tv/RefreshSeriesService.cs b/src/NzbDrone.Core/Tv/RefreshSeriesService.cs index 8f46dbae1..983cd4b9b 100644 --- a/src/NzbDrone.Core/Tv/RefreshSeriesService.cs +++ b/src/NzbDrone.Core/Tv/RefreshSeriesService.cs @@ -144,7 +144,7 @@ namespace NzbDrone.Core.Tv public void Execute(RefreshSeriesCommand message) { - _eventAggregator.PublishEvent(new SeriesRefreshStartingEvent()); + _eventAggregator.PublishEvent(new SeriesRefreshStartingEvent(message.Trigger == CommandTrigger.Manual)); if (message.SeriesId.HasValue) {