2019-08-07 02:20:47 +00:00
|
|
|
using System;
|
|
|
|
using System.Collections;
|
|
|
|
using System.Collections.Generic;
|
2019-12-22 22:08:53 +00:00
|
|
|
using System.Data.SQLite;
|
2019-08-07 02:20:47 +00:00
|
|
|
using System.Linq;
|
|
|
|
using System.Net;
|
|
|
|
using System.Threading;
|
|
|
|
using NLog;
|
|
|
|
using NLog.Common;
|
|
|
|
using NLog.Targets;
|
2023-01-21 23:16:54 +00:00
|
|
|
using Npgsql;
|
2019-08-07 02:20:47 +00:00
|
|
|
using NzbDrone.Common.EnvironmentInfo;
|
2024-02-27 05:30:32 +00:00
|
|
|
using NzbDrone.Common.Extensions;
|
2019-08-07 02:20:47 +00:00
|
|
|
using Sentry;
|
|
|
|
|
|
|
|
namespace NzbDrone.Common.Instrumentation.Sentry
|
|
|
|
{
|
|
|
|
[Target("Sentry")]
|
|
|
|
public class SentryTarget : TargetWithLayout
|
|
|
|
{
|
2019-09-27 01:41:52 +00:00
|
|
|
// don't report uninformative SQLite exceptions
|
|
|
|
// busy/locked are benign https://forums.sonarr.tv/t/owin-sqlite-error-5-database-is-locked/5423/11
|
|
|
|
// The others will be user configuration problems and silt up Sentry
|
2019-12-22 22:08:53 +00:00
|
|
|
private static readonly HashSet<SQLiteErrorCode> FilteredSQLiteErrors = new HashSet<SQLiteErrorCode>
|
|
|
|
{
|
2019-09-27 01:41:52 +00:00
|
|
|
SQLiteErrorCode.Busy,
|
|
|
|
SQLiteErrorCode.Locked,
|
|
|
|
SQLiteErrorCode.Perm,
|
|
|
|
SQLiteErrorCode.ReadOnly,
|
|
|
|
SQLiteErrorCode.IoErr,
|
|
|
|
SQLiteErrorCode.Corrupt,
|
|
|
|
SQLiteErrorCode.Full,
|
|
|
|
SQLiteErrorCode.CantOpen,
|
|
|
|
SQLiteErrorCode.Auth
|
|
|
|
};
|
|
|
|
|
2023-01-21 23:16:54 +00:00
|
|
|
private static readonly HashSet<string> FilteredPostgresErrorCodes = new HashSet<string>
|
|
|
|
{
|
|
|
|
PostgresErrorCodes.OutOfMemory,
|
|
|
|
PostgresErrorCodes.TooManyConnections,
|
|
|
|
PostgresErrorCodes.DiskFull,
|
|
|
|
PostgresErrorCodes.ProgramLimitExceeded
|
|
|
|
};
|
|
|
|
|
2019-08-07 02:20:47 +00:00
|
|
|
// use string and not Type so we don't need a reference to the project
|
|
|
|
// where these are defined
|
2019-12-22 22:08:53 +00:00
|
|
|
private static readonly HashSet<string> FilteredExceptionTypeNames = new HashSet<string>
|
|
|
|
{
|
2019-08-07 02:20:47 +00:00
|
|
|
// UnauthorizedAccessExceptions will just be user configuration issues
|
|
|
|
"UnauthorizedAccessException",
|
2019-12-22 22:08:53 +00:00
|
|
|
|
2019-08-07 02:20:47 +00:00
|
|
|
// Filter out people stuck in boot loops
|
2023-08-05 18:35:50 +00:00
|
|
|
"CorruptDatabaseException",
|
|
|
|
|
|
|
|
// Filter SingleInstance Termination Exceptions
|
|
|
|
"TerminateApplicationException",
|
|
|
|
|
|
|
|
// User config issue, root folder missing, etc.
|
|
|
|
"DirectoryNotFoundException"
|
2019-08-07 02:20:47 +00:00
|
|
|
};
|
|
|
|
|
2019-12-22 22:08:53 +00:00
|
|
|
public static readonly List<string> FilteredExceptionMessages = new List<string>
|
|
|
|
{
|
2019-08-07 02:20:47 +00:00
|
|
|
// Swallow the many, many exceptions flowing through from Jackett
|
|
|
|
"Jackett.Common.IndexerException",
|
2019-12-22 22:08:53 +00:00
|
|
|
|
2019-08-07 02:20:47 +00:00
|
|
|
// Fix openflixr being stupid with permissions
|
|
|
|
"openflixr"
|
|
|
|
};
|
|
|
|
|
|
|
|
// exception types in this list will additionally have the exception message added to the
|
|
|
|
// sentry fingerprint. Make sure that this message doesn't vary by exception
|
|
|
|
// (e.g. containing a path or a url) so that the sentry grouping is sensible
|
2019-12-22 22:08:53 +00:00
|
|
|
private static readonly HashSet<string> IncludeExceptionMessageTypes = new HashSet<string>
|
|
|
|
{
|
2019-08-07 02:20:47 +00:00
|
|
|
"SQLiteException"
|
|
|
|
};
|
2019-12-22 21:24:11 +00:00
|
|
|
|
2019-08-07 02:20:47 +00:00
|
|
|
private static readonly IDictionary<LogLevel, SentryLevel> LoggingLevelMap = new Dictionary<LogLevel, SentryLevel>
|
|
|
|
{
|
2019-12-22 22:08:53 +00:00
|
|
|
{ LogLevel.Debug, SentryLevel.Debug },
|
|
|
|
{ LogLevel.Error, SentryLevel.Error },
|
|
|
|
{ LogLevel.Fatal, SentryLevel.Fatal },
|
|
|
|
{ LogLevel.Info, SentryLevel.Info },
|
|
|
|
{ LogLevel.Trace, SentryLevel.Debug },
|
|
|
|
{ LogLevel.Warn, SentryLevel.Warning },
|
2019-08-07 02:20:47 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
private static readonly IDictionary<LogLevel, BreadcrumbLevel> BreadcrumbLevelMap = new Dictionary<LogLevel, BreadcrumbLevel>
|
|
|
|
{
|
2019-12-22 22:08:53 +00:00
|
|
|
{ LogLevel.Debug, BreadcrumbLevel.Debug },
|
|
|
|
{ LogLevel.Error, BreadcrumbLevel.Error },
|
|
|
|
{ LogLevel.Fatal, BreadcrumbLevel.Critical },
|
|
|
|
{ LogLevel.Info, BreadcrumbLevel.Info },
|
|
|
|
{ LogLevel.Trace, BreadcrumbLevel.Debug },
|
|
|
|
{ LogLevel.Warn, BreadcrumbLevel.Warning },
|
2019-08-07 02:20:47 +00:00
|
|
|
};
|
|
|
|
|
2019-09-03 02:22:25 +00:00
|
|
|
private readonly DateTime _startTime = DateTime.UtcNow;
|
2019-08-07 02:20:47 +00:00
|
|
|
private readonly IDisposable _sdk;
|
|
|
|
private readonly SentryDebounce _debounce;
|
2019-12-22 22:08:53 +00:00
|
|
|
|
|
|
|
private bool _disposed;
|
2019-08-07 02:20:47 +00:00
|
|
|
private bool _unauthorized;
|
|
|
|
|
|
|
|
public bool FilterEvents { get; set; }
|
2019-09-03 02:22:25 +00:00
|
|
|
public bool SentryEnabled { get; set; }
|
|
|
|
|
2024-02-27 05:30:32 +00:00
|
|
|
public SentryTarget(string dsn, IAppFolderInfo appFolderInfo)
|
2019-08-07 02:20:47 +00:00
|
|
|
{
|
|
|
|
_sdk = SentrySdk.Init(o =>
|
|
|
|
{
|
2021-09-28 03:35:35 +00:00
|
|
|
o.Dsn = dsn;
|
2019-08-07 02:20:47 +00:00
|
|
|
o.AttachStacktrace = true;
|
|
|
|
o.MaxBreadcrumbs = 200;
|
2023-07-04 17:12:44 +00:00
|
|
|
o.Release = $"{BuildInfo.AppName}@{BuildInfo.Release}";
|
2024-02-27 05:30:32 +00:00
|
|
|
o.SetBeforeSend(x => SentryCleanser.CleanseEvent(x));
|
|
|
|
o.SetBeforeBreadcrumb(x => SentryCleanser.CleanseBreadcrumb(x));
|
2019-09-03 02:22:25 +00:00
|
|
|
o.Environment = BuildInfo.Branch;
|
2024-02-27 05:30:32 +00:00
|
|
|
|
|
|
|
// Crash free run statistics (sends a ping for healthy and for crashes sessions)
|
|
|
|
o.AutoSessionTracking = true;
|
|
|
|
|
|
|
|
// Caches files in the event device is offline
|
|
|
|
// Sentry creates a 'sentry' sub directory, no need to concat here
|
|
|
|
o.CacheDirectoryPath = appFolderInfo.GetAppDataPath();
|
|
|
|
|
|
|
|
// default environment is production
|
|
|
|
if (!RuntimeInfo.IsProduction)
|
|
|
|
{
|
|
|
|
if (RuntimeInfo.IsDevelopment)
|
|
|
|
{
|
|
|
|
o.Environment = "development";
|
|
|
|
}
|
|
|
|
else if (RuntimeInfo.IsTesting)
|
|
|
|
{
|
|
|
|
o.Environment = "testing";
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
o.Environment = "other";
|
|
|
|
}
|
|
|
|
}
|
2019-08-07 02:20:47 +00:00
|
|
|
});
|
|
|
|
|
2019-09-03 02:22:25 +00:00
|
|
|
InitializeScope();
|
|
|
|
|
2019-08-07 02:20:47 +00:00
|
|
|
_debounce = new SentryDebounce();
|
|
|
|
|
|
|
|
// initialize to true and reconfigure later
|
2022-10-30 22:14:08 +00:00
|
|
|
// Otherwise it will default to false and any errors occurring
|
2019-08-07 02:20:47 +00:00
|
|
|
// before config file gets read will not be filtered
|
|
|
|
FilterEvents = true;
|
2019-09-03 02:22:25 +00:00
|
|
|
SentryEnabled = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
public void InitializeScope()
|
|
|
|
{
|
|
|
|
SentrySdk.ConfigureScope(scope =>
|
|
|
|
{
|
2024-02-27 05:30:32 +00:00
|
|
|
scope.User = new SentryUser
|
2019-09-03 02:22:25 +00:00
|
|
|
{
|
|
|
|
Id = HashUtil.AnonymousToken()
|
|
|
|
};
|
|
|
|
|
|
|
|
scope.Contexts.App.Name = BuildInfo.AppName;
|
|
|
|
scope.Contexts.App.Version = BuildInfo.Version.ToString();
|
|
|
|
scope.Contexts.App.StartTime = _startTime;
|
|
|
|
scope.Contexts.App.Hash = HashUtil.AnonymousToken();
|
|
|
|
scope.Contexts.App.Build = BuildInfo.Release; // Git commit cache?
|
|
|
|
|
|
|
|
scope.SetTag("culture", Thread.CurrentThread.CurrentCulture.Name);
|
|
|
|
scope.SetTag("branch", BuildInfo.Branch);
|
2022-02-20 13:32:21 +00:00
|
|
|
scope.SetTag("runtime_identifier", System.Runtime.InteropServices.RuntimeInformation.RuntimeIdentifier);
|
2019-09-03 02:22:25 +00:00
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
public void UpdateScope(IOsInfo osInfo)
|
|
|
|
{
|
|
|
|
SentrySdk.ConfigureScope(scope =>
|
|
|
|
{
|
|
|
|
scope.SetTag("is_docker", $"{osInfo.IsDocker}");
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
public void UpdateScope(Version databaseVersion, int migration, string updateBranch, IPlatformInfo platformInfo)
|
|
|
|
{
|
|
|
|
SentrySdk.ConfigureScope(scope =>
|
|
|
|
{
|
|
|
|
scope.Environment = updateBranch;
|
|
|
|
scope.SetTag("runtime_version", $"{PlatformInfo.PlatformName} {platformInfo.Version}");
|
|
|
|
|
|
|
|
if (databaseVersion != default(Version))
|
|
|
|
{
|
|
|
|
scope.SetTag("sqlite_version", $"{databaseVersion}");
|
|
|
|
scope.SetTag("database_migration", $"{migration}");
|
|
|
|
}
|
|
|
|
});
|
2019-08-07 02:20:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
private void OnError(Exception ex)
|
|
|
|
{
|
|
|
|
var webException = ex as WebException;
|
|
|
|
|
|
|
|
if (webException != null)
|
|
|
|
{
|
|
|
|
var response = webException.Response as HttpWebResponse;
|
|
|
|
var statusCode = response?.StatusCode;
|
|
|
|
if (statusCode == HttpStatusCode.Unauthorized)
|
|
|
|
{
|
|
|
|
_unauthorized = true;
|
|
|
|
_debounce.Clear();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
InternalLogger.Error(ex, "Unable to send error to Sentry");
|
|
|
|
}
|
|
|
|
|
|
|
|
private static List<string> GetFingerPrint(LogEventInfo logEvent)
|
|
|
|
{
|
|
|
|
if (logEvent.Properties.ContainsKey("Sentry"))
|
|
|
|
{
|
|
|
|
return ((string[])logEvent.Properties["Sentry"]).ToList();
|
|
|
|
}
|
|
|
|
|
|
|
|
var fingerPrint = new List<string>
|
|
|
|
{
|
|
|
|
logEvent.Level.ToString(),
|
|
|
|
logEvent.LoggerName,
|
|
|
|
logEvent.Message
|
|
|
|
};
|
|
|
|
|
|
|
|
var ex = logEvent.Exception;
|
|
|
|
|
|
|
|
if (ex != null)
|
|
|
|
{
|
|
|
|
fingerPrint.Add(ex.GetType().FullName);
|
2021-04-12 11:19:51 +00:00
|
|
|
if (ex.TargetSite != null)
|
|
|
|
{
|
|
|
|
fingerPrint.Add(ex.TargetSite.ToString());
|
|
|
|
}
|
|
|
|
|
2019-08-07 02:20:47 +00:00
|
|
|
if (ex.InnerException != null)
|
|
|
|
{
|
|
|
|
fingerPrint.Add(ex.InnerException.GetType().FullName);
|
|
|
|
}
|
|
|
|
else if (IncludeExceptionMessageTypes.Contains(ex.GetType().Name))
|
|
|
|
{
|
|
|
|
fingerPrint.Add(ex?.Message);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return fingerPrint;
|
|
|
|
}
|
|
|
|
|
|
|
|
public bool IsSentryMessage(LogEventInfo logEvent)
|
|
|
|
{
|
|
|
|
if (logEvent.Properties.ContainsKey("Sentry"))
|
|
|
|
{
|
|
|
|
return logEvent.Properties["Sentry"] != null;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (logEvent.Level >= LogLevel.Error && logEvent.Exception != null)
|
|
|
|
{
|
|
|
|
if (FilterEvents)
|
|
|
|
{
|
2022-08-13 04:33:27 +00:00
|
|
|
var exceptions = new List<Exception>();
|
|
|
|
|
|
|
|
var aggEx = logEvent.Exception as AggregateException;
|
|
|
|
|
|
|
|
if (aggEx != null && aggEx.InnerExceptions.Count > 0)
|
2019-09-27 01:41:52 +00:00
|
|
|
{
|
2022-08-13 04:33:27 +00:00
|
|
|
exceptions.AddRange(aggEx.InnerExceptions);
|
2019-09-27 01:41:52 +00:00
|
|
|
}
|
2022-08-13 04:33:27 +00:00
|
|
|
else
|
2019-08-07 02:20:47 +00:00
|
|
|
{
|
2022-08-13 04:33:27 +00:00
|
|
|
exceptions.Add(logEvent.Exception);
|
2019-08-07 02:20:47 +00:00
|
|
|
}
|
2019-12-22 21:24:11 +00:00
|
|
|
|
2022-08-13 04:33:27 +00:00
|
|
|
// If any are sentry then send to sentry
|
|
|
|
foreach (var ex in exceptions)
|
2019-08-07 02:20:47 +00:00
|
|
|
{
|
2022-08-13 04:33:27 +00:00
|
|
|
var isSentry = true;
|
|
|
|
|
|
|
|
var sqlEx = ex as SQLiteException;
|
2022-09-25 00:45:10 +00:00
|
|
|
if (sqlEx != null && FilteredSQLiteErrors.Contains(sqlEx.ResultCode))
|
2022-08-13 04:33:27 +00:00
|
|
|
{
|
|
|
|
isSentry = false;
|
|
|
|
}
|
|
|
|
|
2023-01-21 23:16:54 +00:00
|
|
|
var pgEx = logEvent.Exception as PostgresException;
|
|
|
|
if (pgEx != null && FilteredPostgresErrorCodes.Contains(pgEx.SqlState))
|
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
// We don't care about transient network and timeout errors
|
|
|
|
var npgEx = logEvent.Exception as NpgsqlException;
|
|
|
|
if (npgEx != null && npgEx.IsTransient)
|
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2022-08-13 04:33:27 +00:00
|
|
|
if (FilteredExceptionTypeNames.Contains(ex.GetType().Name))
|
|
|
|
{
|
|
|
|
isSentry = false;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (FilteredExceptionMessages.Any(x => ex.Message.Contains(x)))
|
|
|
|
{
|
|
|
|
isSentry = false;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (isSentry)
|
|
|
|
{
|
|
|
|
return true;
|
|
|
|
}
|
2019-08-07 02:20:47 +00:00
|
|
|
}
|
2022-08-13 04:33:27 +00:00
|
|
|
|
|
|
|
// The exception or aggregate exception children were not sentry exceptions
|
|
|
|
return false;
|
2019-08-07 02:20:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
protected override void Write(LogEventInfo logEvent)
|
|
|
|
{
|
2019-09-03 02:22:25 +00:00
|
|
|
if (_unauthorized || !SentryEnabled)
|
2019-08-07 02:20:47 +00:00
|
|
|
{
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
try
|
|
|
|
{
|
|
|
|
SentrySdk.AddBreadcrumb(logEvent.FormattedMessage, logEvent.LoggerName, level: BreadcrumbLevelMap[logEvent.Level]);
|
|
|
|
|
|
|
|
// don't report non-critical events without exceptions
|
|
|
|
if (!IsSentryMessage(logEvent))
|
|
|
|
{
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
var fingerPrint = GetFingerPrint(logEvent);
|
|
|
|
if (!_debounce.Allowed(fingerPrint))
|
|
|
|
{
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
var extras = logEvent.Properties.ToDictionary(x => x.Key.ToString(), x => (object)x.Value.ToString());
|
|
|
|
extras.Remove("Sentry");
|
|
|
|
|
|
|
|
if (logEvent.Exception != null)
|
|
|
|
{
|
|
|
|
foreach (DictionaryEntry data in logEvent.Exception.Data)
|
|
|
|
{
|
|
|
|
extras.Add(data.Key.ToString(), data.Value.ToString());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-02-27 05:30:32 +00:00
|
|
|
var level = LoggingLevelMap[logEvent.Level];
|
2019-08-07 02:20:47 +00:00
|
|
|
var sentryEvent = new SentryEvent(logEvent.Exception)
|
|
|
|
{
|
2024-02-27 05:30:32 +00:00
|
|
|
Level = level,
|
2019-08-07 02:20:47 +00:00
|
|
|
Logger = logEvent.LoggerName,
|
2019-09-03 02:22:25 +00:00
|
|
|
Message = logEvent.FormattedMessage
|
2019-08-07 02:20:47 +00:00
|
|
|
};
|
|
|
|
|
2024-02-27 05:30:32 +00:00
|
|
|
if (level is SentryLevel.Fatal && logEvent.Exception is not null)
|
|
|
|
{
|
|
|
|
// Usages of 'fatal' here indicates the process will crash. In Sentry this is represented with
|
|
|
|
// the 'unhandled' exception flag
|
|
|
|
logEvent.Exception.SetSentryMechanism("Logger.Fatal", "Logger.Fatal was called", false);
|
|
|
|
}
|
|
|
|
|
2019-08-07 02:20:47 +00:00
|
|
|
sentryEvent.SetExtras(extras);
|
|
|
|
sentryEvent.SetFingerprint(fingerPrint);
|
|
|
|
|
|
|
|
SentrySdk.CaptureEvent(sentryEvent);
|
|
|
|
}
|
|
|
|
catch (Exception e)
|
|
|
|
{
|
|
|
|
OnError(e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// https://stackoverflow.com/questions/2496311/implementing-idisposable-on-a-subclass-when-the-parent-also-implements-idisposab
|
|
|
|
protected override void Dispose(bool disposing)
|
|
|
|
{
|
|
|
|
// Only do something if we're not already disposed
|
|
|
|
if (_disposed)
|
|
|
|
{
|
|
|
|
// If disposing == true, we're being called from a call to base.Dispose(). In this case, we Dispose() our logger
|
|
|
|
// If we're being called from a finalizer, our logger will get finalized as well, so no need to do anything.
|
|
|
|
if (disposing)
|
|
|
|
{
|
|
|
|
_sdk?.Dispose();
|
|
|
|
}
|
2019-12-22 22:08:53 +00:00
|
|
|
|
2019-08-07 02:20:47 +00:00
|
|
|
// Flag us as disposed. This allows us to handle multiple calls to Dispose() as well as ObjectDisposedException
|
|
|
|
_disposed = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
// This should always be safe to call multiple times!
|
|
|
|
// We could include it in the check for disposed above, but I left it out to demonstrate that it's safe
|
|
|
|
base.Dispose(disposing);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|