From 1a63b1caba85dfd0cf1d09aff177bc4355edc2f0 Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Thu, 29 May 2014 00:48:37 +0200 Subject: [PATCH] Updated migration to attempt associate old grabbed & imported events and associate drone factory imports during CompletedDownloadHandling. --- .../CompletedDownloadServiceFixture.cs | 61 ++++- .../Migration/051_download_client_import.cs | 224 ++++++++++++++++++ .../051_rename_download_client_settings.cs | 88 ------- .../Framework/NzbDroneMigrationBase.cs | 4 +- .../Download/CompletedDownloadService.cs | 33 ++- src/NzbDrone.Core/History/HistoryService.cs | 8 + .../DownloadedEpisodesImportService.cs | 12 +- src/NzbDrone.Core/NzbDrone.Core.csproj | 2 +- 8 files changed, 331 insertions(+), 101 deletions(-) create mode 100644 src/NzbDrone.Core/Datastore/Migration/051_download_client_import.cs delete mode 100644 src/NzbDrone.Core/Datastore/Migration/051_rename_download_client_settings.cs diff --git a/src/NzbDrone.Core.Test/Download/CompletedDownloadServiceFixture.cs b/src/NzbDrone.Core.Test/Download/CompletedDownloadServiceFixture.cs index df1218139..0d7b6d1e9 100644 --- a/src/NzbDrone.Core.Test/Download/CompletedDownloadServiceFixture.cs +++ b/src/NzbDrone.Core.Test/Download/CompletedDownloadServiceFixture.cs @@ -13,6 +13,8 @@ using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Test.Framework; using NzbDrone.Core.MediaFiles; using NzbDrone.Test.Common; +using NzbDrone.Core.Tv; +using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Test.Download { @@ -28,6 +30,10 @@ namespace NzbDrone.Core.Test.Download .All() .With(h => h.Status = DownloadItemStatus.Completed) .With(h => h.OutputPath = @"C:\DropFolder\MyDownload".AsOsAgnostic()) + .With(h => h.RemoteEpisode = new RemoteEpisode + { + Episodes = new List { new Episode { Id = 1 } } + }) .Build() .ToList(); @@ -255,7 +261,9 @@ namespace NzbDrone.Core.Test.Download history.First().Data.Add("downloadClient", "SabnzbdClient"); history.First().Data.Add("downloadClientId", _completed.First().DownloadClientId); - + + Subject.Execute(new CheckForFinishedDownloadCommand()); + VerifyNoImports(); } @@ -278,9 +286,60 @@ namespace NzbDrone.Core.Test.Download history.First().Data.Add("downloadClient", "SabnzbdClient"); history.First().Data.Add("downloadClientId", _completed.First().DownloadClientId); + Subject.Execute(new CheckForFinishedDownloadCommand()); + VerifyNoImports(); } + [Test] + public void should_process_as_already_imported_if_drone_factory_import_history_exists() + { + GivenCompletedDownloadClientHistory(false); + + _completed.Clear(); + _completed.AddRange(Builder.CreateListOfSize(2) + .All() + .With(h => h.Status = DownloadItemStatus.Completed) + .With(h => h.OutputPath = @"C:\DropFolder\MyDownload".AsOsAgnostic()) + .With(h => h.RemoteEpisode = new RemoteEpisode + { + Episodes = new List { new Episode { Id = 1 } } + }) + .Build()); + + var grabbedHistory = Builder.CreateListOfSize(2) + .All() + .With(d => d.Data["downloadClient"] = "SabnzbdClient") + .TheFirst(1) + .With(d => d.Data["downloadClientId"] = _completed.First().DownloadClientId) + .With(d => d.SourceTitle = "Droned.S01E01.720p-LAZY") + .TheLast(1) + .With(d => d.Data["downloadClientId"] = _completed.Last().DownloadClientId) + .With(d => d.SourceTitle = "Droned.S01E01.Proper.720p-LAZY") + .Build() + .ToList(); + + var importedHistory = Builder.CreateListOfSize(2) + .All() + .With(d => d.EpisodeId = 1) + .TheFirst(1) + .With(d => d.Data["droppedPath"] = @"C:\mydownload\Droned.S01E01.720p-LAZY\lzy-dr101.mkv".AsOsAgnostic()) + .TheLast(1) + .With(d => d.Data["droppedPath"] = @"C:\mydownload\Droned.S01E01.Proper.720p-LAZY\lzy-dr101.mkv".AsOsAgnostic()) + .Build() + .ToList(); + + GivenGrabbedHistory(grabbedHistory); + GivenImportedHistory(importedHistory); + + Subject.Execute(new CheckForFinishedDownloadCommand()); + + VerifyNoImports(); + + Mocker.GetMock() + .Verify(v => v.UpdateHistoryData(It.IsAny(), It.IsAny>()), Times.Exactly(2)); + } + [Test] public void should_not_remove_if_config_disabled() { diff --git a/src/NzbDrone.Core/Datastore/Migration/051_download_client_import.cs b/src/NzbDrone.Core/Datastore/Migration/051_download_client_import.cs new file mode 100644 index 000000000..f801eda79 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/051_download_client_import.cs @@ -0,0 +1,224 @@ +using System; +using System.Data; +using System.Linq; +using System.Collections.Generic; +using FluentMigrator; +using Newtonsoft.Json; +using NzbDrone.Common; +using NzbDrone.Common.Serializer; +using NzbDrone.Core.Datastore.Migration.Framework; +using System.IO; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(51)] + public class download_client_import : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Execute.WithConnection(ConvertFolderSettings); + + Execute.WithConnection(AssociateImportedHistoryItems); + } + + private void ConvertFolderSettings(IDbConnection conn, IDbTransaction tran) + { + using (IDbCommand downloadClientsCmd = conn.CreateCommand()) + { + downloadClientsCmd.Transaction = tran; + downloadClientsCmd.CommandText = @"SELECT Value FROM Config WHERE Key = 'downloadedepisodesfolder'"; + var downloadedEpisodesFolder = downloadClientsCmd.ExecuteScalar() as String; + + downloadClientsCmd.Transaction = tran; + downloadClientsCmd.CommandText = @"SELECT Id, Implementation, Settings, ConfigContract FROM DownloadClients WHERE ConfigContract = 'FolderSettings'"; + using (IDataReader downloadClientReader = downloadClientsCmd.ExecuteReader()) + { + while (downloadClientReader.Read()) + { + var id = downloadClientReader.GetInt32(0); + var implementation = downloadClientReader.GetString(1); + var settings = downloadClientReader.GetString(2); + var configContract = downloadClientReader.GetString(3); + + var settingsJson = JsonConvert.DeserializeObject(settings) as Newtonsoft.Json.Linq.JObject; + + if (implementation == "Blackhole") + { + var newSettings = new + { + NzbFolder = settingsJson.Value("folder"), + WatchFolder = downloadedEpisodesFolder + }.ToJson(); + + using (IDbCommand updateCmd = conn.CreateCommand()) + { + updateCmd.Transaction = tran; + updateCmd.CommandText = "UPDATE DownloadClients SET Implementation = ?, Settings = ?, ConfigContract = ? WHERE Id = ?"; + updateCmd.AddParameter("UsenetBlackhole"); + updateCmd.AddParameter(newSettings); + updateCmd.AddParameter("UsenetBlackholeSettings"); + updateCmd.AddParameter(id); + + updateCmd.ExecuteNonQuery(); + } + } + else if (implementation == "Pneumatic") + { + var newSettings = new + { + NzbFolder = settingsJson.Value("folder") + }.ToJson(); + + using (IDbCommand updateCmd = conn.CreateCommand()) + { + updateCmd.Transaction = tran; + updateCmd.CommandText = "UPDATE DownloadClients SET Settings = ?, ConfigContract = ? WHERE Id = ?"; + updateCmd.AddParameter(newSettings); + updateCmd.AddParameter("PneumaticSettings"); + updateCmd.AddParameter(id); + + updateCmd.ExecuteNonQuery(); + } + } + else + { + using (IDbCommand updateCmd = conn.CreateCommand()) + { + updateCmd.Transaction = tran; + updateCmd.CommandText = "DELETE FROM DownloadClients WHERE Id = ?"; + updateCmd.AddParameter(id); + + updateCmd.ExecuteNonQuery(); + } + } + } + } + } + } + + private sealed class MigrationHistoryItem + { + public Int32 Id { get; set; } + public Int32 EpisodeId { get; set; } + public Int32 SeriesId { get; set; } + public String SourceTitle { get; set; } + public DateTime Date { get; set; } + public Dictionary Data { get; set; } + public MigrationHistoryEventType EventType { get; set; } + } + + private enum MigrationHistoryEventType + { + Unknown = 0, + Grabbed = 1, + SeriesFolderImported = 2, + DownloadFolderImported = 3, + DownloadFailed = 4 + } + + private void AssociateImportedHistoryItems(IDbConnection conn, IDbTransaction tran) + { + var historyItems = new List(); + + using (IDbCommand historyCmd = conn.CreateCommand()) + { + historyCmd.Transaction = tran; + historyCmd.CommandText = @"SELECT Id, EpisodeId, SeriesId, SourceTitle, Date, Data, EventType FROM History WHERE EventType NOT NULL"; + using (IDataReader historyRead = historyCmd.ExecuteReader()) + { + while (historyRead.Read()) + { + historyItems.Add(new MigrationHistoryItem + { + Id = historyRead.GetInt32(0), + EpisodeId = historyRead.GetInt32(1), + SeriesId = historyRead.GetInt32(2), + SourceTitle = historyRead.GetString(3), + Date = historyRead.GetDateTime(4), + Data = Json.Deserialize>(historyRead.GetString(5)), + EventType = (MigrationHistoryEventType)historyRead.GetInt32(6) + }); + } + } + } + + var numHistoryItemsNotAssociated = historyItems.Count(v => v.EventType == MigrationHistoryEventType.DownloadFolderImported && + v.Data.GetValueOrDefault("downloadClientId") == null); + + if (numHistoryItemsNotAssociated == 0) + { + return; + } + + var historyItemsToAssociate = new Dictionary(); + + var historyItemsLookup = historyItems.ToLookup(v => v.EpisodeId); + + foreach (var historyItemGroup in historyItemsLookup) + { + var list = historyItemGroup.ToList(); + + for (int i = 0; i < list.Count - 1; i++) + { + var grabbedEvent = list[i]; + if (grabbedEvent.EventType != MigrationHistoryEventType.Grabbed) continue; + if (grabbedEvent.Data.GetValueOrDefault("downloadClient") == null || grabbedEvent.Data.GetValueOrDefault("downloadClientId") == null) continue; + + // Check if it is already associated with a failed/imported event. + int j; + for (j = i + 1; j < list.Count;j++) + { + if (list[j].EventType != MigrationHistoryEventType.DownloadFolderImported && + list[j].EventType != MigrationHistoryEventType.DownloadFailed) + { + continue; + } + + if (list[j].Data.ContainsKey("downloadClient") && list[j].Data["downloadClient"] == grabbedEvent.Data["downloadClient"] && + list[j].Data.ContainsKey("downloadClientId") && list[j].Data["downloadClientId"] == grabbedEvent.Data["downloadClientId"]) + { + break; + } + } + + if (j != list.Count) + { + list.RemoveAt(j); + list.RemoveAt(i--); + continue; + } + + var importedEvent = list[i + 1]; + if (importedEvent.EventType != MigrationHistoryEventType.DownloadFolderImported) continue; + + var droppedPath = importedEvent.Data.GetValueOrDefault("droppedPath"); + if (droppedPath != null && new FileInfo(droppedPath).Directory.Name == grabbedEvent.SourceTitle) + { + historyItemsToAssociate[importedEvent] = grabbedEvent; + + list.RemoveAt(i + 1); + list.RemoveAt(i--); + } + } + } + + foreach (var pair in historyItemsToAssociate) + { + using (IDbCommand updateHistoryCmd = conn.CreateCommand()) + { + pair.Key.Data["downloadClient"] = pair.Value.Data["downloadClient"]; + pair.Key.Data["downloadClientId"] = pair.Value.Data["downloadClientId"]; + + updateHistoryCmd.Transaction = tran; + updateHistoryCmd.CommandText = "UPDATE History SET Data = ? WHERE Id = ?"; + updateHistoryCmd.AddParameter(pair.Key.Data.ToJson()); + updateHistoryCmd.AddParameter(pair.Key.Id); + + updateHistoryCmd.ExecuteNonQuery(); + } + } + + _logger.Info("Updated old History items. {0}/{1} old ImportedEvents were associated with GrabbedEvents.", historyItemsToAssociate.Count, numHistoryItemsNotAssociated); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/Migration/051_rename_download_client_settings.cs b/src/NzbDrone.Core/Datastore/Migration/051_rename_download_client_settings.cs deleted file mode 100644 index ccc3f3527..000000000 --- a/src/NzbDrone.Core/Datastore/Migration/051_rename_download_client_settings.cs +++ /dev/null @@ -1,88 +0,0 @@ -using NzbDrone.Core.Datastore.Migration.Framework; -using FluentMigrator; -using System.Data; -using NzbDrone.Common.Serializer; -using NzbDrone.Core.Download.Clients.UsenetBlackhole; -using Newtonsoft.Json; -using System; -using NzbDrone.Core.Download.Clients.Pneumatic; - -namespace NzbDrone.Core.Datastore.Migration -{ - [Migration(51)] - public class rename_download_client_settings : NzbDroneMigrationBase - { - protected override void MainDbUpgrade() - { - Execute.WithConnection(ConvertFolderSettings); - } - - private void ConvertFolderSettings(IDbConnection conn, IDbTransaction tran) - { - using (IDbCommand downloadClientsCmd = conn.CreateCommand()) - { - downloadClientsCmd.Transaction = tran; - downloadClientsCmd.CommandText = @"SELECT Value FROM Config WHERE Key = 'downloadedepisodesfolder'"; - var downloadedEpisodesFolder = downloadClientsCmd.ExecuteScalar() as String; - - downloadClientsCmd.Transaction = tran; - downloadClientsCmd.CommandText = @"SELECT Id, Implementation, Settings, ConfigContract FROM DownloadClients WHERE ConfigContract = 'FolderSettings'"; - using (IDataReader downloadClientReader = downloadClientsCmd.ExecuteReader()) - { - while (downloadClientReader.Read()) - { - var id = downloadClientReader.GetInt32(0); - var implementation = downloadClientReader.GetString(1); - var settings = downloadClientReader.GetString(2); - var configContract = downloadClientReader.GetString(3); - - var settingsJson = JsonConvert.DeserializeObject(settings) as Newtonsoft.Json.Linq.JObject; - - if (implementation == "Blackhole") - { - var newSettings = new - { - NzbFolder = settingsJson.Value("folder"), - WatchFolder = downloadedEpisodesFolder - }.ToJson(); - - using (IDbCommand updateCmd = conn.CreateCommand()) - { - updateCmd.Transaction = tran; - updateCmd.CommandText = "UPDATE DownloadClients SET Implementation = ?, Settings = ?, ConfigContract = ? WHERE Id = ?"; - updateCmd.AddParameter("UsenetBlackhole"); - updateCmd.AddParameter(newSettings); - updateCmd.AddParameter("UsenetBlackholeSettings"); - updateCmd.AddParameter(id); - - updateCmd.ExecuteNonQuery(); - } - } - else if (implementation == "Pneumatic") - { - var newSettings = new - { - NzbFolder = settingsJson.Value("folder") - }.ToJson(); - - using (IDbCommand updateCmd = conn.CreateCommand()) - { - updateCmd.Transaction = tran; - updateCmd.CommandText = "UPDATE DownloadClients SET Settings = ?, ConfigContract = ? WHERE Id = ?"; - updateCmd.AddParameter(newSettings); - updateCmd.AddParameter("PneumaticSettings"); - updateCmd.AddParameter(id); - - updateCmd.ExecuteNonQuery(); - } - } - else - { - throw new NotSupportedException(); - } - } - } - } -} - } -} diff --git a/src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneMigrationBase.cs b/src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneMigrationBase.cs index da9dde57a..5655649cd 100644 --- a/src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneMigrationBase.cs +++ b/src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneMigrationBase.cs @@ -6,11 +6,11 @@ namespace NzbDrone.Core.Datastore.Migration.Framework { public abstract class NzbDroneMigrationBase : FluentMigrator.Migration { - private Logger _logger; + protected readonly Logger _logger; protected NzbDroneMigrationBase() { - _logger = NzbDroneLogger.GetLogger(); + _logger = NzbDroneLogger.GetLogger(this); } protected virtual void MainDbUpgrade() diff --git a/src/NzbDrone.Core/Download/CompletedDownloadService.cs b/src/NzbDrone.Core/Download/CompletedDownloadService.cs index c6aa2b0c6..7ae4b422b 100644 --- a/src/NzbDrone.Core/Download/CompletedDownloadService.cs +++ b/src/NzbDrone.Core/Download/CompletedDownloadService.cs @@ -30,18 +30,21 @@ namespace NzbDrone.Core.Download private readonly IConfigService _configService; private readonly IDiskProvider _diskProvider; private readonly IDownloadedEpisodesImportService _downloadedEpisodesImportService; + private readonly IHistoryService _historyService; private readonly Logger _logger; public CompletedDownloadService(IEventAggregator eventAggregator, IConfigService configService, IDiskProvider diskProvider, IDownloadedEpisodesImportService downloadedEpisodesImportService, + IHistoryService historyService, Logger logger) { _eventAggregator = eventAggregator; _configService = configService; _diskProvider = diskProvider; _downloadedEpisodesImportService = downloadedEpisodesImportService; + _historyService = historyService; _logger = logger; } @@ -67,7 +70,7 @@ namespace NzbDrone.Core.Download _logger.Trace("Ignoring download that wasn't grabbed by drone: " + trackedDownload.DownloadItem.Title); return; } - + var importedItems = GetHistoryItems(importedHistory, trackedDownload.DownloadItem.DownloadClientId); if (importedItems.Any()) @@ -112,6 +115,34 @@ namespace NzbDrone.Core.Download } else { + if (grabbedItems.Any()) + { + var episodeIds = trackedDownload.DownloadItem.RemoteEpisode.Episodes.Select(v => v.Id).ToList(); + + // Check if we can associate it with a previous drone factory import. + importedItems = importedHistory.Where(v => v.Data.GetValueOrDefault(DownloadTrackingService.DOWNLOAD_CLIENT_ID) == null && + episodeIds.Contains(v.EpisodeId) && + v.Data.GetValueOrDefault("droppedPath") != null && + new FileInfo(v.Data["droppedPath"]).Directory.Name == grabbedItems.First().SourceTitle + ).ToList(); + if (importedItems.Count == 1) + { + var importedFile = new FileInfo(importedItems.First().Data["droppedPath"]); + + if (importedFile.Directory.Name == grabbedItems.First().SourceTitle) + { + trackedDownload.State = TrackedDownloadState.Imported; + + importedItems.First().Data[DownloadTrackingService.DOWNLOAD_CLIENT] = grabbedItems.First().Data[DownloadTrackingService.DOWNLOAD_CLIENT]; + importedItems.First().Data[DownloadTrackingService.DOWNLOAD_CLIENT_ID] = grabbedItems.First().Data[DownloadTrackingService.DOWNLOAD_CLIENT_ID]; + _historyService.UpdateHistoryData(importedItems.First().Id, importedItems.First().Data); + + _logger.Debug("Storage path does not exist, but found probable drone factory ImportEvent: " + trackedDownload.DownloadItem.Title); + return; + } + } + } + _logger.Debug("Storage path does not exist: " + trackedDownload.DownloadItem.Title); return; } diff --git a/src/NzbDrone.Core/History/HistoryService.cs b/src/NzbDrone.Core/History/HistoryService.cs index 37824b5d1..e9846031e 100644 --- a/src/NzbDrone.Core/History/HistoryService.cs +++ b/src/NzbDrone.Core/History/HistoryService.cs @@ -25,6 +25,7 @@ namespace NzbDrone.Core.History History MostRecentForEpisode(int episodeId); History Get(int id); List FindBySourceTitle(string sourceTitle); + void UpdateHistoryData(Int32 historyId, Dictionary data); } public class HistoryService : IHistoryService, IHandle, IHandle, IHandle @@ -101,6 +102,13 @@ namespace NzbDrone.Core.History .FirstOrDefault(); } + public void UpdateHistoryData(Int32 historyId, Dictionary data) + { + var history = _historyRepository.Get(historyId); + history.Data = data; + _historyRepository.Update(history); + } + public void Handle(EpisodeGrabbedEvent message) { foreach (var episode in message.Episode.Episodes) diff --git a/src/NzbDrone.Core/MediaFiles/DownloadedEpisodesImportService.cs b/src/NzbDrone.Core/MediaFiles/DownloadedEpisodesImportService.cs index 1c7235237..a765c45df 100644 --- a/src/NzbDrone.Core/MediaFiles/DownloadedEpisodesImportService.cs +++ b/src/NzbDrone.Core/MediaFiles/DownloadedEpisodesImportService.cs @@ -160,7 +160,8 @@ namespace NzbDrone.Core.MediaFiles } } - return ProcessFiles(series, quality, videoFiles); + var decisions = _importDecisionMaker.GetImportDecisions(videoFiles.ToList(), series, true, quality); + return _importApprovedEpisodes.Import(decisions, true); } private void ProcessVideoFile(string videoFile) @@ -179,13 +180,8 @@ namespace NzbDrone.Core.MediaFiles return; } - ProcessFiles(series, null, videoFile); - } - - private List ProcessFiles(Series series, QualityModel quality, params string[] videoFiles) - { - var decisions = _importDecisionMaker.GetImportDecisions(videoFiles.ToList(), series, true, quality); - return _importApprovedEpisodes.Import(decisions, true); + var decisions = _importDecisionMaker.GetImportDecisions(new [] { videoFile }.ToList(), series, true, null); + _importApprovedEpisodes.Import(decisions, true); } private void ProcessFolder(string path) diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 4549f34a1..76f49c4cd 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -197,7 +197,7 @@ - +