Fixed: Clean Library being to agressive when lists are having failures.

Fixes #2455
This commit is contained in:
Leonardo Galli 2018-01-30 18:29:07 +01:00
parent 74e0db2829
commit 95ca863697
4 changed files with 46 additions and 23 deletions

View File

@ -34,19 +34,21 @@ namespace NzbDrone.Core.NetImport
_httpClient = httpClient; _httpClient = httpClient;
} }
public override IList<Movie> Fetch() public override NetImportFetchResult Fetch()
{ {
var generator = GetRequestGenerator(); var generator = GetRequestGenerator();
return FetchMovies(generator.GetMovies()); return FetchMovies(generator.GetMovies());
} }
protected virtual IList<Movie> FetchMovies(NetImportPageableRequestChain pageableRequestChain, bool isRecent = false) protected virtual NetImportFetchResult FetchMovies(NetImportPageableRequestChain pageableRequestChain, bool isRecent = false)
{ {
var movies = new List<Movie>(); var movies = new List<Movie>();
var url = string.Empty; var url = string.Empty;
var parser = GetParser(); var parser = GetParser();
var anyFailure = false;
try try
{ {
for (int i = 0; i < pageableRequestChain.Tiers; i++) for (int i = 0; i < pageableRequestChain.Tiers; i++)
@ -73,6 +75,7 @@ namespace NzbDrone.Core.NetImport
} }
catch (WebException webException) catch (WebException webException)
{ {
anyFailure = true;
if (webException.Message.Contains("502") || webException.Message.Contains("503") || if (webException.Message.Contains("502") || webException.Message.Contains("503") ||
webException.Message.Contains("timed out")) webException.Message.Contains("timed out"))
{ {
@ -85,6 +88,7 @@ namespace NzbDrone.Core.NetImport
} }
catch (HttpException httpException) catch (HttpException httpException)
{ {
anyFailure = true;
if ((int)httpException.Response.StatusCode == 429) if ((int)httpException.Response.StatusCode == 429)
{ {
_logger.Warn("API Request Limit reached for {0}", this); _logger.Warn("API Request Limit reached for {0}", this);
@ -96,11 +100,12 @@ namespace NzbDrone.Core.NetImport
} }
catch (Exception feedEx) catch (Exception feedEx)
{ {
anyFailure = true;
feedEx.Data.Add("FeedUrl", url); feedEx.Data.Add("FeedUrl", url);
_logger.Error(feedEx, "An error occurred while processing feed. " + url); _logger.Error(feedEx, "An error occurred while processing feed. " + url);
} }
return movies; return new NetImportFetchResult {Movies = movies, AnyFailure = anyFailure};
} }
protected virtual IList<Movie> FetchPage(NetImportRequest request, IParseNetImportResponse parser) protected virtual IList<Movie> FetchPage(NetImportRequest request, IParseNetImportResponse parser)

View File

@ -9,6 +9,6 @@ namespace NzbDrone.Core.NetImport
bool Enabled { get; } bool Enabled { get; }
bool EnableAuto { get; } bool EnableAuto { get; }
IList<Movie> Fetch(); NetImportFetchResult Fetch();
} }
} }

View File

@ -9,6 +9,12 @@ using NzbDrone.Core.Tv;
namespace NzbDrone.Core.NetImport namespace NzbDrone.Core.NetImport
{ {
public class NetImportFetchResult
{
public IList<Movie> Movies { get; set; }
public bool AnyFailure { get; set; }
}
public abstract class NetImportBase<TSettings> : INetImport public abstract class NetImportBase<TSettings> : INetImport
where TSettings : IProviderConfig, new() where TSettings : IProviderConfig, new()
{ {
@ -20,7 +26,7 @@ namespace NzbDrone.Core.NetImport
public abstract bool Enabled { get; } public abstract bool Enabled { get; }
public abstract bool EnableAuto { get; } public abstract bool EnableAuto { get; }
public abstract IList<Movie> Fetch(); public abstract NetImportFetchResult Fetch();
public NetImportBase(IConfigService configService, IParsingService parsingService, Logger logger) public NetImportBase(IConfigService configService, IParsingService parsingService, Logger logger)
{ {

View File

@ -54,21 +54,22 @@ namespace NzbDrone.Core.NetImport
} }
public List<Movie> Fetch(int listId, bool onlyEnableAuto = false) public NetImportFetchResult Fetch(int listId, bool onlyEnableAuto = false)
{ {
return MovieListSearch(listId, onlyEnableAuto); return MovieListSearch(listId, onlyEnableAuto);
} }
public List<Movie> FetchAndFilter(int listId, bool onlyEnableAuto) public List<Movie> FetchAndFilter(int listId, bool onlyEnableAuto)
{ {
var movies = MovieListSearch(listId, onlyEnableAuto); var movies = MovieListSearch(listId, onlyEnableAuto).Movies;
return _movieService.FilterExistingMovies(movies); return _movieService.FilterExistingMovies(movies.ToList());
} }
public List<Movie> MovieListSearch(int listId, bool onlyEnableAuto = false) public NetImportFetchResult MovieListSearch(int listId, bool onlyEnableAuto = false)
{ {
var movies = new List<Movie>(); var movies = new List<Movie>();
var anyFailure = false;
var importLists = _netImportFactory.GetAvailableProviders(); var importLists = _netImportFactory.GetAvailableProviders();
@ -81,12 +82,17 @@ namespace NzbDrone.Core.NetImport
foreach (var list in lists) foreach (var list in lists)
{ {
movies.AddRange(list.Fetch()); var result = list.Fetch();
movies.AddRange(result.Movies);
anyFailure |= result.AnyFailure;
} }
_logger.Debug("Found {0} movies from list(s) {1}", movies.Count, string.Join(", ", lists.Select(l => l.Definition.Name))); _logger.Debug("Found {0} movies from list(s) {1}", movies.Count, string.Join(", ", lists.Select(l => l.Definition.Name)));
return movies.DistinctBy(x => { return new NetImportFetchResult
{
Movies = movies.DistinctBy(x =>
{
if (x.TmdbId != 0) if (x.TmdbId != 0)
{ {
return x.TmdbId.ToString(); return x.TmdbId.ToString();
@ -98,7 +104,9 @@ namespace NzbDrone.Core.NetImport
} }
return x.Title; return x.Title;
}).ToList(); }).ToList(),
AnyFailure = anyFailure
};
} }
@ -112,9 +120,13 @@ namespace NzbDrone.Core.NetImport
return; return;
} }
var listedMovies = Fetch(0, true); var result = Fetch(0, true);
var listedMovies = result.Movies.ToList();
if (!result.AnyFailure)
{
CleanLibrary(listedMovies); CleanLibrary(listedMovies);
}
listedMovies = listedMovies.Where(x => !_movieService.MovieExists(x)).ToList(); listedMovies = listedMovies.Where(x => !_movieService.MovieExists(x)).ToList();
if (listedMovies.Any()) if (listedMovies.Any())