New: Server pushable health checks

Closes #4116
This commit is contained in:
Mark McDowall 2023-09-01 17:10:07 -07:00
parent 276352dda4
commit faecdc855f
2 changed files with 86 additions and 6 deletions

View File

@ -4,6 +4,7 @@ using System.Linq;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Messaging;
using NzbDrone.Common.Reflection;
using NzbDrone.Core.Lifecycle;
@ -27,6 +28,7 @@ namespace NzbDrone.Core.HealthCheck
private readonly IProvideHealthCheck[] _startupHealthChecks;
private readonly IProvideHealthCheck[] _scheduledHealthChecks;
private readonly Dictionary<Type, IEventDrivenHealthCheck[]> _eventDrivenHealthChecks;
private readonly IServerSideNotificationService _serverSideNotificationService;
private readonly IEventAggregator _eventAggregator;
private readonly ICacheManager _cacheManager;
private readonly Logger _logger;
@ -37,12 +39,14 @@ namespace NzbDrone.Core.HealthCheck
private bool _isRunningHealthChecksAfterGracePeriod;
public HealthCheckService(IEnumerable<IProvideHealthCheck> healthChecks,
IServerSideNotificationService serverSideNotificationService,
IEventAggregator eventAggregator,
ICacheManager cacheManager,
IRuntimeInfo runtimeInfo,
Logger logger)
{
_healthChecks = healthChecks.ToArray();
_serverSideNotificationService = serverSideNotificationService;
_eventAggregator = eventAggregator;
_cacheManager = cacheManager;
_logger = logger;
@ -74,11 +78,16 @@ namespace NzbDrone.Core.HealthCheck
.ToDictionary(g => g.Key, g => g.ToArray());
}
private void PerformHealthCheck(IProvideHealthCheck[] healthChecks)
private void PerformHealthCheck(IProvideHealthCheck[] healthChecks, bool performServerChecks)
{
var results = healthChecks.Select(c => c.Check())
.ToList();
if (performServerChecks)
{
results.AddIfNotNull(_serverSideNotificationService.GetServerChecks());
}
foreach (var result in results)
{
if (result.Type == HealthCheckResult.Ok)
@ -110,17 +119,17 @@ namespace NzbDrone.Core.HealthCheck
{
if (message.Trigger == CommandTrigger.Manual)
{
PerformHealthCheck(_healthChecks);
PerformHealthCheck(_healthChecks, true);
}
else
{
PerformHealthCheck(_scheduledHealthChecks);
PerformHealthCheck(_scheduledHealthChecks, true);
}
}
public void HandleAsync(ApplicationStartedEvent message)
{
PerformHealthCheck(_startupHealthChecks);
PerformHealthCheck(_startupHealthChecks, true);
}
public void HandleAsync(IEvent message)
@ -137,7 +146,7 @@ namespace NzbDrone.Core.HealthCheck
{
_isRunningHealthChecksAfterGracePeriod = true;
PerformHealthCheck(_startupHealthChecks);
PerformHealthCheck(_startupHealthChecks, false);
// Update after running health checks so new failure notifications aren't sent 2x.
_hasRunHealthChecksAfterGracePeriod = true;
@ -175,7 +184,7 @@ namespace NzbDrone.Core.HealthCheck
// TODO: Add debounce
PerformHealthCheck(filteredChecks.ToArray());
PerformHealthCheck(filteredChecks.ToArray(), false);
}
}
}

View File

@ -0,0 +1,71 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using NLog;
using NzbDrone.Common.Cloud;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Common.Http;
using NzbDrone.Common.Serializer;
using NzbDrone.Core.Configuration;
namespace NzbDrone.Core.HealthCheck
{
public interface IServerSideNotificationService
{
public HealthCheck GetServerChecks();
}
public class ServerSideNotificationService : IServerSideNotificationService
{
private readonly IHttpClient _client;
private readonly ISonarrCloudRequestBuilder _cloudRequestBuilder;
private readonly IConfigFileProvider _configFileProvider;
private readonly Logger _logger;
public ServerSideNotificationService(IHttpClient client, ISonarrCloudRequestBuilder cloudRequestBuilder, IConfigFileProvider configFileProvider, Logger logger)
{
_client = client;
_cloudRequestBuilder = cloudRequestBuilder;
_configFileProvider = configFileProvider;
_logger = logger;
}
public HealthCheck GetServerChecks()
{
var request = _cloudRequestBuilder.Services.Create()
.Resource("/notification")
.AddQueryParam("version", BuildInfo.Version)
.AddQueryParam("os", OsInfo.Os.ToString().ToLowerInvariant())
.AddQueryParam("arch", RuntimeInformation.OSArchitecture)
.AddQueryParam("branch", _configFileProvider.Branch)
.Build();
try
{
_logger.Trace("Getting notifications");
var response = _client.Execute(request);
var result = Json.Deserialize<List<ServerNotificationResponse>>(response.Content);
var checks = result.Select(x => new HealthCheck(GetType(), x.Type, x.Message, x.WikiUrl)).ToList();
// Only one health check is supported, services returns an ordered list, so use the first one
return checks.FirstOrDefault() ?? new HealthCheck(GetType());
}
catch (Exception ex)
{
_logger.Error(ex, "Failed to retrieve notifications");
return new HealthCheck(GetType());
}
}
}
public class ServerNotificationResponse
{
public HealthCheckResult Type { get; set; }
public string Message { get; set; }
public string WikiUrl { get; set; }
}
}