Health check results are stored in memory and updated as required

This commit is contained in:
Mark McDowall 2014-04-09 17:15:13 -07:00
parent 0c71b7c5d0
commit 3f4c1a16f8
21 changed files with 188 additions and 75 deletions

View File

@ -8,7 +8,7 @@ using NzbDrone.Core.Messaging.Events;
namespace NzbDrone.Api.Health
{
public class HealthModule : NzbDroneRestModuleWithSignalR<HealthResource, HealthCheck>,
IHandle<TriggerHealthCheckEvent>
IHandle<HealthCheckCompleteEvent>
{
private readonly IHealthCheckService _healthCheckService;
@ -21,10 +21,10 @@ namespace NzbDrone.Api.Health
private List<HealthResource> GetHealth()
{
return ToListResource(_healthCheckService.PerformHealthCheck);
return ToListResource(_healthCheckService.Results);
}
public void Handle(TriggerHealthCheckEvent message)
public void Handle(HealthCheckCompleteEvent message)
{
BroadcastResourceChange(ModelAction.Sync);
}

View File

@ -2,12 +2,11 @@
using NzbDrone.Api.REST;
using NzbDrone.Core.HealthCheck;
namespace NzbDrone.Api.Health
{
public class HealthResource : RestResource
{
public HealthCheckResultType Type { get; set; }
public HealthCheckResult Type { get; set; }
public String Message { get; set; }
}
}

View File

@ -136,7 +136,7 @@
<Compile Include="Frontend\Mappers\IMapHttpRequestsToDisk.cs" />
<Compile Include="Frontend\Mappers\StaticResourceMapperBase.cs" />
<Compile Include="Frontend\StaticResourceModule.cs" />
<Compile Include="Health\HistoryResource.cs" />
<Compile Include="Health\HealthResource.cs" />
<Compile Include="Health\HealthModule.cs" />
<Compile Include="History\HistoryResource.cs" />
<Compile Include="History\HistoryModule.cs" />

View File

@ -37,7 +37,7 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks
}
[Test]
public void should_return_null_when_download_client_returns()
public void should_return_ok_when_download_client_returns()
{
var downloadClient = Mocker.GetMock<IDownloadClient>();
@ -48,7 +48,7 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks
.Setup(s => s.GetDownloadClient())
.Returns(downloadClient.Object);
Subject.Check().Should().BeNull();
Subject.Check().ShouldBeOk();
}
}
}

View File

@ -57,11 +57,11 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks
}
[Test]
public void should_return_null_when_no_issues_found()
public void should_return_ok_when_no_issues_found()
{
GivenDroneFactoryFolder(true);
Subject.Check().Should().BeNull();
Subject.Check().ShouldBeOk();
}
}
}

View File

@ -5,14 +5,19 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks
{
public static class HealthCheckFixtureExtensions
{
public static void ShouldBeOk(this Core.HealthCheck.HealthCheck result)
{
result.Type.Should().Be(HealthCheckResult.Ok);
}
public static void ShouldBeWarning(this Core.HealthCheck.HealthCheck result)
{
result.Type.Should().Be(HealthCheckResultType.Warning);
result.Type.Should().Be(HealthCheckResult.Warning);
}
public static void ShouldBeError(this Core.HealthCheck.HealthCheck result)
{
result.Type.Should().Be(HealthCheckResultType.Error);
result.Type.Should().Be(HealthCheckResult.Error);
}
}
}

View File

@ -39,7 +39,7 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks
}
[Test]
public void should_return_null_when_multiple_indexers_are_enabled()
public void should_return_ok_when_multiple_indexers_are_enabled()
{
var indexer1 = Mocker.GetMock<IIndexer>();
indexer1.SetupGet(s => s.SupportsSearching).Returns(true);
@ -51,11 +51,11 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks
.Setup(s => s.GetAvailableProviders())
.Returns(new List<IIndexer> { indexer1.Object, indexer2.Object });
Subject.Check().Should().BeNull();
Subject.Check().ShouldBeOk();
}
[Test]
public void should_return_null_when_indexer_supports_searching()
public void should_return_ok_when_indexer_supports_searching()
{
var indexer1 = Mocker.GetMock<IIndexer>();
indexer1.SetupGet(s => s.SupportsSearching).Returns(true);
@ -64,7 +64,7 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks
.Setup(s => s.GetAvailableProviders())
.Returns(new List<IIndexer> { indexer1.Object });
Subject.Check().Should().BeNull();
Subject.Check().ShouldBeOk();
}
}
}

View File

@ -48,35 +48,35 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks
}
[Test]
public void should_return_null_when_mono_3_2()
public void should_return_ok_when_mono_3_2()
{
GivenOutput("3.2.0.1");
Subject.Check().Should().BeNull();
Subject.Check().ShouldBeOk();
}
[Test]
public void should_return_null_when_mono_4_0()
public void should_return_ok_when_mono_4_0()
{
GivenOutput("4.0.0.0");
Subject.Check().Should().BeNull();
Subject.Check().ShouldBeOk();
}
[Test]
public void should_return_null_when_mono_3_2_7()
public void should_return_ok_when_mono_3_2_7()
{
GivenOutput("3.2.7");
Subject.Check().Should().BeNull();
Subject.Check().ShouldBeOk();
}
[Test]
public void should_return_null_when_mono_3_2_1()
public void should_return_ok_when_mono_3_2_1()
{
GivenOutput("3.2.1");
Subject.Check().Should().BeNull();
Subject.Check().ShouldBeOk();
}
}
}

View File

@ -42,7 +42,7 @@ namespace NzbDrone.Core.Download
_configService = configService;
_logger = logger;
_failedDownloads = cacheManager.GetCache<FailedDownload>(GetType(), "queue");
_failedDownloads = cacheManager.GetCache<FailedDownload>(GetType());
}
public void MarkAsFailed(int historyId)

View File

@ -3,7 +3,7 @@ using NzbDrone.Core.Download;
namespace NzbDrone.Core.HealthCheck.Checks
{
public class DownloadClientCheck : IProvideHealthCheck
public class DownloadClientCheck : HealthCheckBase
{
private readonly IProvideDownloadClient _downloadClientProvider;
@ -12,13 +12,13 @@ namespace NzbDrone.Core.HealthCheck.Checks
_downloadClientProvider = downloadClientProvider;
}
public HealthCheck Check()
public override HealthCheck Check()
{
var downloadClient = _downloadClientProvider.GetDownloadClient();
if (downloadClient == null)
{
return new HealthCheck(HealthCheckResultType.Warning, "No download client is available");
return new HealthCheck(GetType(), HealthCheckResult.Warning, "No download client is available");
}
try
@ -27,10 +27,10 @@ namespace NzbDrone.Core.HealthCheck.Checks
}
catch (Exception)
{
return new HealthCheck(HealthCheckResultType.Error, "Unable to communicate with download client");
return new HealthCheck(GetType(), HealthCheckResult.Error, "Unable to communicate with download client");
}
return null;
return new HealthCheck(GetType());
}
}
}

View File

@ -6,7 +6,7 @@ using NzbDrone.Core.Configuration;
namespace NzbDrone.Core.HealthCheck.Checks
{
public class DroneFactoryCheck : IProvideHealthCheck
public class DroneFactoryCheck : HealthCheckBase
{
private readonly IConfigService _configService;
private readonly IDiskProvider _diskProvider;
@ -17,18 +17,18 @@ namespace NzbDrone.Core.HealthCheck.Checks
_diskProvider = diskProvider;
}
public HealthCheck Check()
public override HealthCheck Check()
{
var droneFactoryFolder = _configService.DownloadedEpisodesFolder;
if (droneFactoryFolder.IsNullOrWhiteSpace())
{
return new HealthCheck(HealthCheckResultType.Warning, "Drone factory folder is not configured");
return new HealthCheck(GetType(), HealthCheckResult.Warning, "Drone factory folder is not configured");
}
if (!_diskProvider.FolderExists(droneFactoryFolder))
{
return new HealthCheck(HealthCheckResultType.Error, "Drone factory folder does not exist");
return new HealthCheck(GetType(), HealthCheckResult.Error, "Drone factory folder does not exist");
}
try
@ -39,12 +39,12 @@ namespace NzbDrone.Core.HealthCheck.Checks
}
catch (Exception)
{
return new HealthCheck(HealthCheckResultType.Error, "Unable to write to drone factory folder");
return new HealthCheck(GetType(), HealthCheckResult.Error, "Unable to write to drone factory folder");
}
//Todo: Unable to import one or more files/folders from
return null;
return new HealthCheck(GetType());
}
}
}

View File

@ -3,7 +3,7 @@ using NzbDrone.Core.Indexers;
namespace NzbDrone.Core.HealthCheck.Checks
{
public class IndexerCheck : IProvideHealthCheck
public class IndexerCheck : HealthCheckBase
{
private readonly IIndexerFactory _indexerFactory;
@ -12,21 +12,21 @@ namespace NzbDrone.Core.HealthCheck.Checks
_indexerFactory = indexerFactory;
}
public HealthCheck Check()
public override HealthCheck Check()
{
var enabled = _indexerFactory.GetAvailableProviders();
if (!enabled.Any())
{
return new HealthCheck(HealthCheckResultType.Error, "No indexers are enabled");
return new HealthCheck(GetType(), HealthCheckResult.Error, "No indexers are enabled");
}
if (enabled.All(i => i.SupportsSearching == false))
{
return new HealthCheck(HealthCheckResultType.Warning, "Enabled indexers do not support searching");
return new HealthCheck(GetType(), HealthCheckResult.Warning, "Enabled indexers do not support searching");
}
return null;
return new HealthCheck(GetType());
}
}
}

View File

@ -6,7 +6,7 @@ using NzbDrone.Common.Processes;
namespace NzbDrone.Core.HealthCheck.Checks
{
public class MonoVersionCheck : IProvideHealthCheck
public class MonoVersionCheck : HealthCheckBase
{
private readonly IProcessProvider _processProvider;
private readonly Logger _logger;
@ -18,11 +18,11 @@ namespace NzbDrone.Core.HealthCheck.Checks
_logger = logger;
}
public HealthCheck Check()
public override HealthCheck Check()
{
if (!OsInfo.IsMono)
{
return null;
return new HealthCheck(GetType());
}
var output = _processProvider.StartAndCapture("mono", "--version");
@ -38,12 +38,28 @@ namespace NzbDrone.Core.HealthCheck.Checks
if (version >= new Version(3, 2))
{
_logger.Debug("mono version is 3.2 or better: {0}", version.ToString());
return null;
return new HealthCheck(GetType());
}
}
}
return new HealthCheck(HealthCheckResultType.Warning, "mono version is less than 3.2, upgrade for improved stability");
return new HealthCheck(GetType(), HealthCheckResult.Warning, "mono version is less than 3.2, upgrade for improved stability");
}
public override bool CheckOnConfigChange
{
get
{
return false;
}
}
public override bool CheckOnSchedule
{
get
{
return false;
}
}
}
}

View File

@ -6,7 +6,7 @@ using NzbDrone.Core.Update;
namespace NzbDrone.Core.HealthCheck.Checks
{
public class UpdateCheck : IProvideHealthCheck
public class UpdateCheck : HealthCheckBase
{
private readonly IDiskProvider _diskProvider;
private readonly IAppFolderInfo _appFolderInfo;
@ -20,7 +20,7 @@ namespace NzbDrone.Core.HealthCheck.Checks
}
public HealthCheck Check()
public override HealthCheck Check()
{
if (OsInfo.IsWindows)
{
@ -32,7 +32,7 @@ namespace NzbDrone.Core.HealthCheck.Checks
}
catch (Exception)
{
return new HealthCheck(HealthCheckResultType.Error,
return new HealthCheck(GetType(), HealthCheckResult.Error,
"Unable to update, running from write-protected folder");
}
}
@ -41,11 +41,19 @@ namespace NzbDrone.Core.HealthCheck.Checks
{
if (_checkUpdateService.AvailableUpdate() != null)
{
return new HealthCheck(HealthCheckResultType.Warning, "New update is available");
return new HealthCheck(GetType(), HealthCheckResult.Warning, "New update is available");
}
}
return null;
return new HealthCheck(GetType());
}
public override bool CheckOnConfigChange
{
get
{
return false;
}
}
}
}

View File

@ -5,18 +5,27 @@ namespace NzbDrone.Core.HealthCheck
{
public class HealthCheck : ModelBase
{
public HealthCheckResultType Type { get; set; }
public Type Source { get; set; }
public HealthCheckResult Type { get; set; }
public String Message { get; set; }
public HealthCheck(HealthCheckResultType type, string message)
public HealthCheck(Type source)
{
Source = source;
Type = HealthCheckResult.Ok;
}
public HealthCheck(Type source, HealthCheckResult type, string message)
{
Source = source;
Type = type;
Message = message;
}
}
public enum HealthCheckResultType
public enum HealthCheckResult
{
Ok = 0,
Warning = 1,
Error = 2
}

View File

@ -0,0 +1,36 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace NzbDrone.Core.HealthCheck
{
public abstract class HealthCheckBase : IProvideHealthCheck
{
public abstract HealthCheck Check();
public virtual bool CheckOnStartup
{
get
{
return true;
}
}
public virtual bool CheckOnConfigChange
{
get
{
return true;
}
}
public virtual bool CheckOnSchedule
{
get
{
return true;
}
}
}
}

View File

@ -2,7 +2,7 @@
namespace NzbDrone.Core.HealthCheck
{
public class TriggerHealthCheckEvent : IEvent
public class HealthCheckCompleteEvent : IEvent
{
}
}

View File

@ -1,9 +1,13 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Core.Configuration.Events;
using NzbDrone.Core.Download;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.Messaging.Commands;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.ThingiProvider.Events;
@ -12,56 +16,86 @@ namespace NzbDrone.Core.HealthCheck
{
public interface IHealthCheckService
{
List<HealthCheck> PerformHealthCheck();
List<HealthCheck> Results();
}
public class HealthCheckService : IHealthCheckService,
IExecute<CheckHealthCommand>,
IHandleAsync<ApplicationStartedEvent>,
IHandleAsync<ConfigSavedEvent>,
IHandleAsync<ProviderUpdatedEvent<IIndexer>>,
IHandleAsync<ProviderUpdatedEvent<IDownloadClient>>
{
private readonly IEnumerable<IProvideHealthCheck> _healthChecks;
private readonly IEventAggregator _eventAggregator;
private readonly ICacheManager _cacheManager;
private readonly Logger _logger;
public HealthCheckService(IEnumerable<IProvideHealthCheck> healthChecks, IEventAggregator eventAggregator, Logger logger)
private readonly ICached<HealthCheck> _healthCheckResults;
public HealthCheckService(IEnumerable<IProvideHealthCheck> healthChecks,
IEventAggregator eventAggregator,
ICacheManager cacheManager,
Logger logger)
{
_healthChecks = healthChecks;
_eventAggregator = eventAggregator;
_cacheManager = cacheManager;
_logger = logger;
_healthCheckResults = _cacheManager.GetCache<HealthCheck>(GetType());
}
public List<HealthCheck> PerformHealthCheck()
public List<HealthCheck> Results()
{
_logger.Trace("Checking health");
var result = _healthChecks.Select(c => c.Check()).Where(c => c != null).ToList();
return result;
return _healthCheckResults.Values.ToList();
}
public void Execute(CheckHealthCommand message)
private void PerformHealthCheck(Func<IProvideHealthCheck, bool> predicate)
{
//Until we have stored health checks we should just trigger the complete event
//and let the clients check in
//Multiple connected clients means we're going to compute the health check multiple times
//Multiple checks feels a bit ugly, but means the most up to date information goes to the client
_eventAggregator.PublishEvent(new TriggerHealthCheckEvent());
var results = _healthChecks.Where(predicate)
.Select(c => c.Check())
.ToList();
foreach (var result in results)
{
if (result.Type == HealthCheckResult.Ok)
{
_healthCheckResults.Remove(result.Source.Name);
}
else
{
_healthCheckResults.Set(result.Source.Name, result);
}
}
_eventAggregator.PublishEvent(new HealthCheckCompleteEvent());
}
public void HandleAsync(ConfigSavedEvent message)
{
_eventAggregator.PublishEvent(new TriggerHealthCheckEvent());
PerformHealthCheck(c => c.CheckOnConfigChange);
}
public void HandleAsync(ProviderUpdatedEvent<IIndexer> message)
{
_eventAggregator.PublishEvent(new TriggerHealthCheckEvent());
PerformHealthCheck(c => c.CheckOnConfigChange);
}
public void HandleAsync(ProviderUpdatedEvent<IDownloadClient> message)
{
_eventAggregator.PublishEvent(new TriggerHealthCheckEvent());
PerformHealthCheck(c => c.CheckOnConfigChange);
}
public void HandleAsync(ApplicationStartedEvent message)
{
PerformHealthCheck(c => c.CheckOnStartup);
}
public void Execute(CheckHealthCommand message)
{
PerformHealthCheck(c => c.CheckOnSchedule);
}
}
}

View File

@ -1,7 +1,12 @@
namespace NzbDrone.Core.HealthCheck
using System;
namespace NzbDrone.Core.HealthCheck
{
public interface IProvideHealthCheck
{
HealthCheck Check();
Boolean CheckOnStartup { get; }
Boolean CheckOnConfigChange { get; }
Boolean CheckOnSchedule { get; }
}
}

View File

@ -50,10 +50,10 @@ namespace NzbDrone.Core.Jobs
{
new ScheduledTask{ Interval = 1, TypeName = typeof(TrackedCommandCleanupCommand).FullName},
new ScheduledTask{ Interval = 1, TypeName = typeof(CheckForFailedDownloadCommand).FullName},
new ScheduledTask{ Interval = 5, TypeName = typeof(CheckHealthCommand).FullName},
new ScheduledTask{ Interval = 1*60, TypeName = typeof(ApplicationUpdateCommand).FullName},
new ScheduledTask{ Interval = 1*60, TypeName = typeof(TrimLogCommand).FullName},
new ScheduledTask{ Interval = 3*60, TypeName = typeof(UpdateSceneMappingCommand).FullName},
new ScheduledTask{ Interval = 6*60, TypeName = typeof(CheckHealthCommand).FullName},
new ScheduledTask{ Interval = 12*60, TypeName = typeof(RefreshSeriesCommand).FullName},
new ScheduledTask{ Interval = 24*60, TypeName = typeof(HousekeepingCommand).FullName},

View File

@ -280,7 +280,8 @@
<Compile Include="HealthCheck\Checks\IndexerCheck.cs" />
<Compile Include="HealthCheck\Checks\UpdateCheck.cs" />
<Compile Include="HealthCheck\HealthCheck.cs" />
<Compile Include="HealthCheck\TriggerHealthCheckEvent.cs" />
<Compile Include="HealthCheck\HealthCheckBase.cs" />
<Compile Include="HealthCheck\HealthCheckCompleteEvent.cs" />
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedEpisodes.cs" />
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedHistoryItems.cs" />
<Compile Include="Housekeeping\Housekeepers\CleanupOrphanedMetadataFiles.cs" />