Pre-filter some events before sending to sentry (#654)

* Filter some errors from Sentry and add exception message to fingerprint

* Move to new Sentry SDK
This commit is contained in:
ta264 2019-03-11 12:21:27 +00:00 committed by GitHub
parent 7d4d923903
commit c7a772363f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 285 additions and 207 deletions

View File

@ -108,7 +108,7 @@ namespace NzbDrone.Common.Instrumentation
Layout = "${message}"
};
var loggingRule = new LoggingRule("*", updateClient ? LogLevel.Trace : LogLevel.Warn, target);
var loggingRule = new LoggingRule("*", updateClient ? LogLevel.Trace : LogLevel.Debug, target);
LogManager.Configuration.AddTarget("sentryTarget", target);
LogManager.Configuration.LoggingRules.Add(loggingRule);
@ -117,7 +117,7 @@ namespace NzbDrone.Common.Instrumentation
LogManager.Configuration.LoggingRules.Insert(0, loggingRuleSentry);
}
private static void RegisterDebugger()
private static void RegisterDebugger()
{
DebuggerTarget target = new DebuggerTarget();
target.Name = "debuggerLogger";

View File

@ -1,73 +0,0 @@
using System;
using System.Collections.Generic;
using SharpRaven.Data;
namespace NzbDrone.Common.Instrumentation.Sentry
{
public class LidarrJsonPacketFactory : IJsonPacketFactory
{
private readonly SentryPacketCleanser _cleanser;
public LidarrJsonPacketFactory()
{
_cleanser = new SentryPacketCleanser();
}
private static string ShortenPath(string path)
{
if (string.IsNullOrWhiteSpace(path))
{
return null;
}
var index = path.IndexOf("\\src\\", StringComparison.Ordinal);
if (index <= 0)
{
return path;
}
return path.Substring(index + "\\src".Length);
}
public JsonPacket Create(string project, SentryEvent @event)
{
var packet = new LidarrSentryPacket(project, @event);
try
{
foreach (var exception in packet.Exceptions)
{
foreach (var frame in exception.Stacktrace.Frames)
{
frame.Filename = ShortenPath(frame.Filename);
}
}
_cleanser.CleansePacket(packet);
}
catch (Exception)
{
}
return packet;
}
[Obsolete]
public JsonPacket Create(string project, SentryMessage message, ErrorLevel level = ErrorLevel.Info, IDictionary<string, string> tags = null,
string[] fingerprint = null, object extra = null)
{
throw new NotImplementedException();
}
[Obsolete]
public JsonPacket Create(string project, Exception exception, SentryMessage message = null, ErrorLevel level = ErrorLevel.Error,
IDictionary<string, string> tags = null, string[] fingerprint = null, object extra = null)
{
throw new NotImplementedException();
}
}
}

View File

@ -1,29 +0,0 @@
using Newtonsoft.Json;
using SharpRaven.Data;
namespace NzbDrone.Common.Instrumentation.Sentry
{
public class LidarrSentryPacket : JsonPacket
{
private readonly JsonSerializerSettings _setting;
public LidarrSentryPacket(string project, SentryEvent @event) :
base(project, @event)
{
_setting = new JsonSerializerSettings
{
DefaultValueHandling = DefaultValueHandling.Ignore
};
}
public override string ToString(Formatting formatting)
{
return JsonConvert.SerializeObject(this, formatting, _setting);
}
public override string ToString()
{
return ToString(Formatting.None);
}
}
}

View File

@ -1,12 +0,0 @@
using SharpRaven.Data;
namespace NzbDrone.Common.Instrumentation.Sentry
{
public class MachineNameUserFactory : ISentryUserFactory
{
public SentryUser Create()
{
return new SentryUser(HashUtil.AnonymousToken());
}
}
}

View File

@ -0,0 +1,80 @@
using System;
using System.Linq;
using NzbDrone.Common.EnvironmentInfo;
using Sentry;
using Sentry.Protocol;
namespace NzbDrone.Common.Instrumentation.Sentry
{
public static class SentryCleanser
{
public static SentryEvent CleanseEvent(SentryEvent sentryEvent)
{
try
{
sentryEvent.Message = CleanseLogMessage.Cleanse(sentryEvent.Message);
if (sentryEvent.Fingerprint != null)
{
var fingerprint = sentryEvent.Fingerprint.Select(x => CleanseLogMessage.Cleanse(x)).ToList();
sentryEvent.SetFingerprint(fingerprint);
}
if (sentryEvent.Extra != null)
{
var extras = sentryEvent.Extra.ToDictionary(x => x.Key, y => (object)CleanseLogMessage.Cleanse((string)y.Value));
sentryEvent.SetExtras(extras);
}
foreach (var exception in sentryEvent.SentryExceptions)
{
foreach (var frame in exception.Stacktrace.Frames)
{
frame.FileName = ShortenPath(frame.FileName);
}
}
}
catch (Exception)
{
}
return sentryEvent;
}
public static Breadcrumb CleanseBreadcrumb(Breadcrumb b)
{
try
{
var message = CleanseLogMessage.Cleanse(b.Message);
var data = b.Data?.ToDictionary(x => x.Key, y => CleanseLogMessage.Cleanse(y.Value));
return new Breadcrumb(message, b.Type, data, b.Category, b.Level);
}
catch(Exception)
{
}
return b;
}
private static string ShortenPath(string path)
{
if (string.IsNullOrWhiteSpace(path))
{
return null;
}
var rootDir = OsInfo.IsWindows ? "\\src\\" : "/src/";
var index = path.IndexOf(rootDir, StringComparison.Ordinal);
if (index <= 0)
{
return path;
}
return path.Substring(index + rootDir.Length - 1);
}
}
}

View File

@ -1,31 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Newtonsoft.Json.Linq;
namespace NzbDrone.Common.Instrumentation.Sentry
{
public class SentryPacketCleanser
{
public void CleansePacket(LidarrSentryPacket packet)
{
packet.Message = CleanseLogMessage.Cleanse(packet.Message);
if (packet.Fingerprint != null)
{
for (var i = 0; i < packet.Fingerprint.Length; i++)
{
packet.Fingerprint[i] = CleanseLogMessage.Cleanse(packet.Fingerprint[i]);
}
}
if (packet.Extra != null)
{
var target = JObject.FromObject(packet.Extra);
new CleansingJsonVisitor().Visit(target);
packet.Extra = target;
}
}
}
}

View File

@ -4,51 +4,109 @@ using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading;
using System.Data.SQLite;
using NLog;
using NLog.Common;
using NLog.Targets;
using NzbDrone.Common.EnvironmentInfo;
using SharpRaven;
using SharpRaven.Data;
using System.Globalization;
using Sentry;
using Sentry.Protocol;
using NzbDrone.Common.Extensions;
namespace NzbDrone.Common.Instrumentation.Sentry
{
[Target("Sentry")]
public class SentryTarget : TargetWithLayout
{
private readonly RavenClient _client;
private static readonly IDictionary<LogLevel, ErrorLevel> LoggingLevelMap = new Dictionary<LogLevel, ErrorLevel>
{
{LogLevel.Debug, ErrorLevel.Debug},
{LogLevel.Error, ErrorLevel.Error},
{LogLevel.Fatal, ErrorLevel.Fatal},
{LogLevel.Info, ErrorLevel.Info},
{LogLevel.Trace, ErrorLevel.Debug},
{LogLevel.Warn, ErrorLevel.Warning},
// don't report uninformative SQLite exceptions
// busy/locked are benign https://forums.sonarr.tv/t/owin-sqlite-error-5-database-is-locked/5423/11
// The others will be user configuration problems and silt up Sentry
private static readonly HashSet<SQLiteErrorCode> FilteredSQLiteErrors = new HashSet<SQLiteErrorCode> {
SQLiteErrorCode.Busy,
SQLiteErrorCode.Locked,
SQLiteErrorCode.Perm,
SQLiteErrorCode.ReadOnly,
SQLiteErrorCode.IoErr,
SQLiteErrorCode.Corrupt,
SQLiteErrorCode.Full,
SQLiteErrorCode.CantOpen,
SQLiteErrorCode.Auth
};
// use string and not Type so we don't need a reference to the project
// where these are defined
private static readonly HashSet<string> FilteredExceptionTypeNames = new HashSet<string> {
// UnauthorizedAccessExceptions will just be user configuration issues
"UnauthorizedAccessException",
// Filter out people stuck in boot loops
"CorruptDatabaseException"
};
private static readonly List<string> FilteredExceptionMessages = new List<string> {
// Swallow the many, many exceptions flowing through from Jackett
"Jackett.Common.IndexerException",
// Fix openflixr being stupid with permissions
"openflixr"
};
private static readonly IDictionary<LogLevel, SentryLevel> LoggingLevelMap = new Dictionary<LogLevel, SentryLevel>
{
{LogLevel.Debug, SentryLevel.Debug},
{LogLevel.Error, SentryLevel.Error},
{LogLevel.Fatal, SentryLevel.Fatal},
{LogLevel.Info, SentryLevel.Info},
{LogLevel.Trace, SentryLevel.Debug},
{LogLevel.Warn, SentryLevel.Warning},
};
private static readonly IDictionary<LogLevel, BreadcrumbLevel> BreadcrumbLevelMap = new Dictionary<LogLevel, BreadcrumbLevel>
{
{LogLevel.Debug, BreadcrumbLevel.Debug},
{LogLevel.Error, BreadcrumbLevel.Error},
{LogLevel.Fatal, BreadcrumbLevel.Critical},
{LogLevel.Info, BreadcrumbLevel.Info},
{LogLevel.Trace, BreadcrumbLevel.Debug},
{LogLevel.Warn, BreadcrumbLevel.Warning},
};
private readonly IDisposable _sdk;
private bool _disposed;
private readonly SentryDebounce _debounce;
private bool _unauthorized;
public bool FilterEvents { get; set; }
public SentryTarget(string dsn)
{
_client = new RavenClient(new Dsn(dsn), new LidarrJsonPacketFactory(), new SentryRequestFactory(), new MachineNameUserFactory())
{
Compression = true,
Environment = RuntimeInfo.IsProduction ? "production" : "development",
Release = BuildInfo.Release,
ErrorOnCapture = OnError
};
_client.Tags.Add("osfamily", OsInfo.Os.ToString());
_client.Tags.Add("runtime", PlatformInfo.PlatformName);
_client.Tags.Add("culture", Thread.CurrentThread.CurrentCulture.Name);
_client.Tags.Add("branch", BuildInfo.Branch);
_client.Tags.Add("version", BuildInfo.Version.ToString());
_sdk = SentrySdk.Init(o =>
{
o.Dsn = new Dsn(dsn);
o.AttachStacktrace = true;
o.MaxBreadcrumbs = 200;
o.SendDefaultPii = true;
o.Debug = false;
o.DiagnosticsLevel = SentryLevel.Debug;
o.Environment = RuntimeInfo.IsProduction ? "production" : "development";
o.Release = BuildInfo.Release;
o.BeforeSend = x => SentryCleanser.CleanseEvent(x);
o.BeforeBreadcrumb = x => SentryCleanser.CleanseBreadcrumb(x);
});
SentrySdk.ConfigureScope(scope =>
{
scope.User = new User {
Username = HashUtil.AnonymousToken()
};
scope.SetTag("osfamily", OsInfo.Os.ToString());
scope.SetTag("runtime", PlatformInfo.PlatformName);
scope.SetTag("culture", Thread.CurrentThread.CurrentCulture.Name);
scope.SetTag("branch", BuildInfo.Branch);
scope.SetTag("version", BuildInfo.Version.ToString());
});
_debounce = new SentryDebounce();
}
@ -109,6 +167,25 @@ namespace NzbDrone.Common.Instrumentation.Sentry
if (logEvent.Level >= LogLevel.Error && logEvent.Exception != null)
{
if (FilterEvents)
{
var sqlEx = logEvent.Exception as SQLiteException;
if (sqlEx != null && FilteredSQLiteErrors.Contains(sqlEx.ResultCode))
{
return false;
}
if (FilteredExceptionTypeNames.Contains(logEvent.Exception.GetType().Name))
{
return false;
}
if (FilteredExceptionMessages.Any(x => logEvent.Exception.Message.Contains(x)))
{
return false;
}
}
return true;
}
@ -125,6 +202,8 @@ namespace NzbDrone.Common.Instrumentation.Sentry
try
{
SentrySdk.AddBreadcrumb(logEvent.FormattedMessage, logEvent.LoggerName, level: BreadcrumbLevelMap[logEvent.Level]);
// don't report non-critical events without exceptions
if (!IsSentryMessage(logEvent))
{
@ -137,9 +216,8 @@ namespace NzbDrone.Common.Instrumentation.Sentry
return;
}
var extras = logEvent.Properties.ToDictionary(x => x.Key.ToString(), x => x.Value.ToString());
var extras = logEvent.Properties.ToDictionary(x => x.Key.ToString(), x => (object)x.Value.ToString());
extras.Remove("Sentry");
_client.Logger = logEvent.LoggerName;
if (logEvent.Exception != null)
{
@ -149,54 +227,95 @@ namespace NzbDrone.Common.Instrumentation.Sentry
}
}
var sentryMessage = new SentryMessage(logEvent.Message, logEvent.Parameters);
var sentryEvent = new SentryEvent(logEvent.Exception)
{
Level = LoggingLevelMap[logEvent.Level],
Message = sentryMessage,
Extra = extras,
Fingerprint =
{
Logger = logEvent.LoggerName,
Message = logEvent.FormattedMessage,
};
var sentryFingerprint = new List<string> {
logEvent.Level.ToString(),
logEvent.LoggerName,
logEvent.Message
}
};
// Fix openflixr being stupid with permissions
var serverName = sentryEvent.Contexts.Device.Name.ToLower();
if (serverName == "openflixr")
{
return;
}
sentryEvent.SetExtras(extras);
if (logEvent.Exception != null)
{
sentryEvent.Fingerprint.Add(logEvent.Exception.GetType().FullName);
sentryFingerprint.Add(logEvent.Exception.GetType().FullName);
sentryFingerprint.Add(logEvent.Exception.TargetSite.ToString());
// only try to use the exeception message to fingerprint if there's no inner
// exception and the message is short, otherwise we're in danger of getting a
// stacktrace which will break the grouping
if (logEvent.Exception.InnerException == null)
{
string message = null;
// bodge to try to get the exception message in English
// https://stackoverflow.com/questions/209133/exception-messages-in-english
// There may still be some localization but this is better than nothing.
var t = new Thread(() => {
message = logEvent.Exception?.Message;
});
t.CurrentCulture = CultureInfo.InvariantCulture;
t.CurrentUICulture = CultureInfo.InvariantCulture;
t.Start();
t.Join();
if (message.IsNotNullOrWhiteSpace() && message.Length < 200)
{
sentryFingerprint.Add(message);
}
}
}
if (logEvent.Properties.ContainsKey("Sentry"))
{
sentryEvent.Fingerprint.Clear();
Array.ForEach((string[])logEvent.Properties["Sentry"], sentryEvent.Fingerprint.Add);
sentryFingerprint = ((string[])logEvent.Properties["Sentry"]).ToList();
}
sentryEvent.SetFingerprint(sentryFingerprint);
// this can't be in the constructor as at that point OsInfo won't have
// populated these values yet
var osName = Environment.GetEnvironmentVariable("OS_NAME");
var osVersion = Environment.GetEnvironmentVariable("OS_VERSION");
var runTimeVersion = Environment.GetEnvironmentVariable("RUNTIME_VERSION");
sentryEvent.Tags.Add("os_name", osName);
sentryEvent.Tags.Add("os_version", $"{osName} {osVersion}");
sentryEvent.Tags.Add("runtime_version", $"{PlatformInfo.PlatformName} {runTimeVersion}");
sentryEvent.SetTag("os_name", osName);
sentryEvent.SetTag("os_version", $"{osName} {osVersion}");
sentryEvent.SetTag("runtime_version", $"{PlatformInfo.PlatformName} {runTimeVersion}");
_client.Capture(sentryEvent);
SentrySdk.CaptureEvent(sentryEvent);
}
catch (Exception e)
{
OnError(e);
}
}
// https://stackoverflow.com/questions/2496311/implementing-idisposable-on-a-subclass-when-the-parent-also-implements-idisposab
protected override void Dispose(bool disposing)
{
// Only do something if we're not already disposed
if (_disposed)
{
// If disposing == true, we're being called from a call to base.Dispose(). In this case, we Dispose() our logger
// If we're being called from a finalizer, our logger will get finalized as well, so no need to do anything.
if (disposing)
{
_sdk?.Dispose();
}
// Flag us as disposed. This allows us to handle multiple calls to Dispose() as well as ObjectDisposedException
_disposed = true;
}
// This should always be safe to call multiple times!
// We could include it in the check for disposed above, but I left it out to demonstrate that it's safe
base.Dispose(disposing);
}
}
}

View File

@ -54,8 +54,17 @@
<HintPath>..\packages\DotNet4.SocksProxy.1.3.4.0\lib\net40\Org.Mentalis.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="SharpRaven, Version=2.4.0.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\SharpRaven.2.4.0\lib\net45\SharpRaven.dll</HintPath>
<Reference Include="Sentry, Version=1.1.2.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\Sentry.1.1.2\lib\net461\Sentry.dll</HintPath>
</Reference>
<Reference Include="Sentry.Protocol, Version=1.0.4.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\Sentry.Protocol.1.0.4\lib\net46\Sentry.Protocol.dll</HintPath>
</Reference>
<Reference Include="Sentry.PlatformAbstractions, Version=1.0.0.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\Sentry.PlatformAbstractions.1.0.0\lib\net45\Sentry.PlatformAbstractions.dll</HintPath>
</Reference>
<Reference Include="System.Collections.Immutable, Version=1.2.3.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\System.Collections.Immutable.1.5.0\lib\netstandard2.0\System.Collections.Immutable.dll</HintPath>
</Reference>
<Reference Include="SocksWebProxy, Version=1.3.4.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\DotNet4.SocksProxy.1.3.4.0\lib\net40\SocksWebProxy.dll</HintPath>
@ -69,6 +78,9 @@
<Reference Include="System.Net.Http" />
<Reference Include="System.ServiceProcess" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data.SQLite">
<HintPath>..\Libraries\Sqlite\System.Data.SQLite.dll</HintPath>
</Reference>
<Reference Include="System.Xml" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Numerics" />
@ -193,11 +205,8 @@
<Compile Include="Instrumentation\NzbDroneFileTarget.cs" />
<Compile Include="Instrumentation\NzbDroneLogger.cs" />
<Compile Include="Instrumentation\Sentry\SentryDebounce.cs" />
<Compile Include="Instrumentation\Sentry\SentryPacketCleanser.cs" />
<Compile Include="Instrumentation\Sentry\SentryTarget.cs" />
<Compile Include="Instrumentation\Sentry\MachineNameUserFactory.cs" />
<Compile Include="Instrumentation\Sentry\LidarrJsonPacketFactory.cs" />
<Compile Include="Instrumentation\Sentry\LidarrSentryPacket.cs" />
<Compile Include="Instrumentation\Sentry\SentryCleanser.cs" />
<Compile Include="Instrumentation\VersionLayoutRenderer.cs" />
<Compile Include="Extensions\LevenstheinExtensions.cs" />
<Compile Include="Extensions\FuzzyContains.cs" />
@ -244,6 +253,12 @@
<ItemGroup>
<Content Include="Expansive\license.txt" />
</ItemGroup>
<ItemGroup>
<Content Include="..\Libraries\Sqlite\libsqlite3.0.dylib">
<Link>libsqlite3.0.dylib</Link>
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\ExternalModules\CurlSharp\CurlSharp\CurlSharp.csproj">
<Project>{74420a79-cc16-442c-8b1e-7c1b913844f0}</Project>
@ -262,4 +277,4 @@
<Target Name="AfterBuild">
</Target>
-->
</Project>
</Project>

View File

@ -4,5 +4,8 @@
<package id="ICSharpCode.SharpZipLib.Patched" version="0.86.5" targetFramework="net461" />
<package id="Newtonsoft.Json" version="11.0.2" targetFramework="net461" />
<package id="NLog" version="4.5.4" targetFramework="net461" />
<package id="SharpRaven" version="2.4.0" targetFramework="net461" />
</packages>
<package id="Sentry" version="1.1.2" targetFramework="net461" />
<package id="Sentry.PlatformAbstractions" version="1.0.0" targetFramework="net461" />
<package id="Sentry.Protocol" version="1.0.4" targetFramework="net461" />
<package id="System.Collections.Immutable" version="1.5.0" targetFramework="net461" />
</packages>

View File

@ -34,6 +34,7 @@ namespace NzbDrone.Core.Configuration
bool AnalyticsEnabled { get; }
string LogLevel { get; }
string ConsoleLogLevel { get; }
bool FilterSentryEvents { get; }
string Branch { get; }
string ApiKey { get; }
string SslCertHash { get; }
@ -181,6 +182,7 @@ namespace NzbDrone.Core.Configuration
public string LogLevel => GetValue("LogLevel", "Info");
public string ConsoleLogLevel => GetValue("ConsoleLogLevel", string.Empty, persist: false);
public bool FilterSentryEvents => GetValueBoolean("FilterSentryEvents", true, persist: false);
public string SslCertHash => GetValue("SslCertHash", "");

View File

@ -3,6 +3,7 @@ using System.Linq;
using NLog;
using NLog.Config;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Instrumentation.Sentry;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Configuration.Events;
using NzbDrone.Core.Messaging.Events;
@ -40,6 +41,9 @@ namespace NzbDrone.Core.Instrumentation
SetMinimumLogLevel(rules, "appFileDebug", minimumLogLevel <= LogLevel.Debug ? LogLevel.Debug : LogLevel.Off);
SetMinimumLogLevel(rules, "appFileTrace", minimumLogLevel <= LogLevel.Trace ? LogLevel.Trace : LogLevel.Off);
// Sentry filtering
LogManager.Configuration.FindTargetByName<SentryTarget>("sentryTarget").FilterEvents = _configFileProvider.FilterSentryEvents;
LogManager.ReconfigExistingLoggers();
}