From 99daa47f89cad2c2e90910a366a9f6ead76a92f4 Mon Sep 17 00:00:00 2001 From: "kay.one" Date: Thu, 4 Jul 2013 20:56:27 -0700 Subject: [PATCH] added support for removing columns --- .../Datastore/SQLiteAlterFixture.cs | 49 ------- .../Datastore/SQLiteMigrationHelperFixture.cs | 84 +++++++++++ NzbDrone.Core.Test/Framework/DbTest.cs | 11 +- NzbDrone.Core.Test/NzbDrone.Core.Test.csproj | 2 +- .../Datastore/ConnectionStringFactory.cs | 37 +++++ NzbDrone.Core/Datastore/DbFactory.cs | 29 ++-- .../Datastore/Migration/007_remove_backlog.cs | 17 +-- .../Migration/Framework/MigrationContext.cs | 8 ++ .../Framework/MigrationController.cs | 10 +- .../Framework/NzbDroneMigrationBase.cs | 7 +- .../Framework/SQLiteMigrationHelper.cs | 134 ++++++++++++++++++ .../Migration/Framework/SqliteAlter.cs | 85 ++++------- NzbDrone.Core/NzbDrone.Core.csproj | 5 +- 13 files changed, 331 insertions(+), 147 deletions(-) delete mode 100644 NzbDrone.Core.Test/Datastore/SQLiteAlterFixture.cs create mode 100644 NzbDrone.Core.Test/Datastore/SQLiteMigrationHelperFixture.cs create mode 100644 NzbDrone.Core/Datastore/ConnectionStringFactory.cs create mode 100644 NzbDrone.Core/Datastore/Migration/Framework/MigrationContext.cs create mode 100644 NzbDrone.Core/Datastore/Migration/Framework/SQLiteMigrationHelper.cs diff --git a/NzbDrone.Core.Test/Datastore/SQLiteAlterFixture.cs b/NzbDrone.Core.Test/Datastore/SQLiteAlterFixture.cs deleted file mode 100644 index a8c665ab9..000000000 --- a/NzbDrone.Core.Test/Datastore/SQLiteAlterFixture.cs +++ /dev/null @@ -1,49 +0,0 @@ -using System.Linq; -using FluentAssertions; -using NUnit.Framework; -using NzbDrone.Core.Datastore; -using NzbDrone.Core.Datastore.Migration.Framework; -using NzbDrone.Core.Test.Framework; - -namespace NzbDrone.Core.Test.Datastore -{ - [TestFixture] - public class SQLiteAlterFixture : DbTest - { - private SQLiteAlter Subject; - - [SetUp] - public void SetUp() - { - var connection = Mocker.Resolve().DataMapper.ConnectionString; - Subject = new SQLiteAlter(connection); - } - - - - [Test] - public void should_parse_existing_columns() - { - var columns = Subject.GetColumns("Series"); - - columns.Should().NotBeEmpty(); - - columns.Values.Should().NotContain(c => string.IsNullOrWhiteSpace(c.Name)); - columns.Values.Should().NotContain(c => string.IsNullOrWhiteSpace(c.Schema)); - } - - [Test] - public void should_create_table_from_column_list() - { - var columns = Subject.GetColumns("Series"); - columns.Remove("Title"); - - Subject.CreateTable("Series_New", columns.Values); - - var newColumns = Subject.GetColumns("Series_New"); - - newColumns.Values.Should().HaveSameCount(columns.Values); - newColumns.Should().NotContainKey("Title"); - } - } -} \ No newline at end of file diff --git a/NzbDrone.Core.Test/Datastore/SQLiteMigrationHelperFixture.cs b/NzbDrone.Core.Test/Datastore/SQLiteMigrationHelperFixture.cs new file mode 100644 index 000000000..c940ac7b2 --- /dev/null +++ b/NzbDrone.Core.Test/Datastore/SQLiteMigrationHelperFixture.cs @@ -0,0 +1,84 @@ +using FizzWare.NBuilder; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.Datastore.Migration.Framework; +using NzbDrone.Core.Test.Framework; +using NzbDrone.Core.Tv; + +namespace NzbDrone.Core.Test.Datastore +{ + [TestFixture] + public class SQLiteMigrationHelperFixture : DbTest + { + private SQLiteMigrationHelper _subject; + + [SetUp] + public void SetUp() + { + _subject = Mocker.Resolve(); + } + + + + [Test] + public void should_parse_existing_columns() + { + var columns = _subject.GetColumns("Series"); + + columns.Should().NotBeEmpty(); + + columns.Values.Should().NotContain(c => string.IsNullOrWhiteSpace(c.Name)); + columns.Values.Should().NotContain(c => string.IsNullOrWhiteSpace(c.Schema)); + } + + [Test] + public void should_create_table_from_column_list() + { + var columns = _subject.GetColumns("Series"); + columns.Remove("Title"); + + _subject.CreateTable("Series_New", columns.Values); + + var newColumns = _subject.GetColumns("Series_New"); + + newColumns.Values.Should().HaveSameCount(columns.Values); + newColumns.Should().NotContainKey("Title"); + } + + [Test] + public void should_get_zero_count_on_empty_table() + { + _subject.GetRowCount("Series").Should().Be(0); + } + + + [Test] + public void should_be_able_to_transfer_empty_tables() + { + var columns = _subject.GetColumns("Series"); + columns.Remove("Title"); + + _subject.CreateTable("Series_New", columns.Values); + + + _subject.CopyData("Series", "Series_New", columns.Values); + } + + [Test] + public void should_transfer_table_with_data() + { + var originalEpisodes = Builder.CreateListOfSize(10).BuildListOfNew(); + + Mocker.Resolve().InsertMany(originalEpisodes); + + var columns = _subject.GetColumns("Episodes"); + columns.Remove("Title"); + + _subject.CreateTable("Episodes_New", columns.Values); + + _subject.CopyData("Episodes", "Episodes_New", columns.Values); + + _subject.GetRowCount("Episodes_New").Should().Be(originalEpisodes.Count); + } + } +} \ No newline at end of file diff --git a/NzbDrone.Core.Test/Framework/DbTest.cs b/NzbDrone.Core.Test/Framework/DbTest.cs index d01a21836..48c520c09 100644 --- a/NzbDrone.Core.Test/Framework/DbTest.cs +++ b/NzbDrone.Core.Test/Framework/DbTest.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using FluentMigrator.Runner; using Marr.Data; using Moq; using NUnit.Framework; @@ -93,10 +94,16 @@ namespace NzbDrone.Core.Test.Framework WithTempAsAppPath(); + Mocker.SetConstant(Mocker.Resolve()); + Mocker.SetConstant(Mocker.Resolve()); + Mocker.SetConstant(Mocker.Resolve()); + Mocker.SetConstant(Mocker.Resolve()); + Mocker.SetConstant(Mocker.Resolve()); + MapRepository.Instance.EnableTraceLogging = true; - var factory = new DbFactory(new MigrationController(new MigrationLogger(TestLogger)), Mocker.GetMock().Object); - _database = factory.Create(MigrationType); + var factory = Mocker.Resolve(); + var _database = factory.Create(MigrationType); _db = new TestDatabase(_database); Mocker.SetConstant(_database); } diff --git a/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj b/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj index fa00cec5a..ef4d2357b 100644 --- a/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj +++ b/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj @@ -124,7 +124,7 @@ - + diff --git a/NzbDrone.Core/Datastore/ConnectionStringFactory.cs b/NzbDrone.Core/Datastore/ConnectionStringFactory.cs new file mode 100644 index 000000000..ec0b5b279 --- /dev/null +++ b/NzbDrone.Core/Datastore/ConnectionStringFactory.cs @@ -0,0 +1,37 @@ +using System; +using System.Data.SQLite; +using NzbDrone.Common; +using NzbDrone.Common.EnvironmentInfo; + +namespace NzbDrone.Core.Datastore +{ + public interface IConnectionStringFactory + { + string MainDbConnectionString { get; } + string LogDbConnectionString { get; } + } + + public class ConnectionStringFactory : IConnectionStringFactory + { + public ConnectionStringFactory(IAppDirectoryInfo appDirectoryInfo) + { + MainDbConnectionString = GetConnectionString(appDirectoryInfo.GetNzbDroneDatabase()); + LogDbConnectionString = GetConnectionString(appDirectoryInfo.GetLogDatabase()); + } + + public string MainDbConnectionString { get; private set; } + public string LogDbConnectionString { get; private set; } + + private static string GetConnectionString(string dbPath) + { + var connectionBuilder = new SQLiteConnectionStringBuilder(); + + connectionBuilder.DataSource = dbPath; + connectionBuilder.CacheSize = (int)-10.Megabytes(); + connectionBuilder.DateTimeKind = DateTimeKind.Utc; + connectionBuilder.JournalMode = SQLiteJournalModeEnum.Wal; + + return connectionBuilder.ConnectionString; + } + } +} \ No newline at end of file diff --git a/NzbDrone.Core/Datastore/DbFactory.cs b/NzbDrone.Core/Datastore/DbFactory.cs index aad62e5c2..fb8d43a37 100644 --- a/NzbDrone.Core/Datastore/DbFactory.cs +++ b/NzbDrone.Core/Datastore/DbFactory.cs @@ -3,10 +3,8 @@ using System.Data.SQLite; using Marr.Data; using Marr.Data.Reflection; using NzbDrone.Common.Composition; -using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Messaging; using NzbDrone.Core.Datastore.Migration.Framework; -using NzbDrone.Common; using NzbDrone.Core.Instrumentation; @@ -17,10 +15,11 @@ namespace NzbDrone.Core.Datastore IDatabase Create(MigrationType migrationType = MigrationType.Main); } + public class DbFactory : IDbFactory { private readonly IMigrationController _migrationController; - private readonly IAppDirectoryInfo _appDirectoryInfo; + private readonly IConnectionStringFactory _connectionStringFactory; static DbFactory() { @@ -39,26 +38,27 @@ namespace NzbDrone.Core.Datastore }); } - public DbFactory(IMigrationController migrationController, IAppDirectoryInfo appDirectoryInfo) + public DbFactory(IMigrationController migrationController, IConnectionStringFactory connectionStringFactory) { _migrationController = migrationController; - _appDirectoryInfo = appDirectoryInfo; + _connectionStringFactory = connectionStringFactory; } public IDatabase Create(MigrationType migrationType = MigrationType.Main) { - string dbPath; + string connectionString; + switch (migrationType) { case MigrationType.Main: { - dbPath = _appDirectoryInfo.GetNzbDroneDatabase(); + connectionString = _connectionStringFactory.MainDbConnectionString; break; } case MigrationType.Log: { - dbPath = _appDirectoryInfo.GetLogDatabase(); + connectionString = _connectionStringFactory.LogDbConnectionString; break; } default: @@ -68,7 +68,6 @@ namespace NzbDrone.Core.Datastore } - var connectionString = GetConnectionString(dbPath); _migrationController.MigrateToLatest(connectionString, migrationType); @@ -82,17 +81,5 @@ namespace NzbDrone.Core.Datastore return dataMapper; }); } - - private string GetConnectionString(string dbPath) - { - var connectionBuilder = new SQLiteConnectionStringBuilder(); - - connectionBuilder.DataSource = dbPath; - connectionBuilder.CacheSize = (int)-10.Megabytes(); - connectionBuilder.DateTimeKind = DateTimeKind.Utc; - connectionBuilder.JournalMode = SQLiteJournalModeEnum.Wal; - - return connectionBuilder.ConnectionString; - } } } diff --git a/NzbDrone.Core/Datastore/Migration/007_remove_backlog.cs b/NzbDrone.Core/Datastore/Migration/007_remove_backlog.cs index f768590be..86c914afd 100644 --- a/NzbDrone.Core/Datastore/Migration/007_remove_backlog.cs +++ b/NzbDrone.Core/Datastore/Migration/007_remove_backlog.cs @@ -3,24 +3,13 @@ using NzbDrone.Core.Datastore.Migration.Framework; namespace NzbDrone.Core.Datastore.Migration { -/* [Tags("")] + [Tags("")] [Migration(7)] public class remove_backlog : NzbDroneMigrationBase { protected override void MainDbUpgrade() { - var newSeriesTable = "CREATE TABLE [Series_new] ([Id] integer NOT NULL PRIMARY KEY AUTOINCREMENT, [TvdbId] integer NOT NULL, " + - "[TvRageId] integer NOT NULL, [ImdbId] text NOT NULL, [Title] text NOT NULL, [TitleSlug] text NOT NULL, " + - "[CleanTitle] text NOT NULL, [Status] integer NOT NULL, [Overview] text, [AirTime] text, " + - "[Images] text NOT NULL, [Path] text NOT NULL, [Monitored] integer NOT NULL, [QualityProfileId] integer NOT NULL, " + - "[SeasonFolder] integer NOT NULL, [LastInfoSync] datetime, [LastDiskSync] datetime, [Runtime] integer NOT NULL, " + - "[SeriesType] integer NOT NULL, [Network] text, [CustomStartDate] datetime, " + - "[UseSceneNumbering] integer NOT NULL, [FirstAired] datetime)"; - - Execute.Sql(newSeriesTable); - - - Execute.Sql("INSERT INTO Series_new SELECT * FROM Series"); + SQLiteAlter.DropColumns("Series", new[] { "BacklogSetting" }); } - }*/ + } } diff --git a/NzbDrone.Core/Datastore/Migration/Framework/MigrationContext.cs b/NzbDrone.Core/Datastore/Migration/Framework/MigrationContext.cs new file mode 100644 index 000000000..ebf634c15 --- /dev/null +++ b/NzbDrone.Core/Datastore/Migration/Framework/MigrationContext.cs @@ -0,0 +1,8 @@ +namespace NzbDrone.Core.Datastore.Migration.Framework +{ + public class MigrationContext + { + public MigrationType MigrationType { get; set; } + public ISQLiteAlter SQLiteAlter { get; set; } + } +} \ No newline at end of file diff --git a/NzbDrone.Core/Datastore/Migration/Framework/MigrationController.cs b/NzbDrone.Core/Datastore/Migration/Framework/MigrationController.cs index 2f3966af2..0c44259b1 100644 --- a/NzbDrone.Core/Datastore/Migration/Framework/MigrationController.cs +++ b/NzbDrone.Core/Datastore/Migration/Framework/MigrationController.cs @@ -14,12 +14,14 @@ namespace NzbDrone.Core.Datastore.Migration.Framework public class MigrationController : IMigrationController { private readonly IAnnouncer _announcer; + private readonly ISQLiteAlter _sqLiteAlter; private static readonly HashSet MigrationCache = new HashSet(); - public MigrationController(IAnnouncer announcer) + public MigrationController(IAnnouncer announcer, ISQLiteAlter sqLiteAlter) { _announcer = announcer; + _sqLiteAlter = sqLiteAlter; } public void MigrateToLatest(string connectionString, MigrationType migrationType) @@ -35,7 +37,11 @@ namespace NzbDrone.Core.Datastore.Migration.Framework var migrationContext = new RunnerContext(_announcer) { Namespace = "NzbDrone.Core.Datastore.Migration", - ApplicationContext = migrationType + ApplicationContext = new MigrationContext + { + MigrationType = migrationType, + SQLiteAlter = _sqLiteAlter + } }; var options = new MigrationOptions { PreviewOnly = false, Timeout = 60 }; diff --git a/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneMigrationBase.cs b/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneMigrationBase.cs index 02b019439..c641d5f55 100644 --- a/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneMigrationBase.cs +++ b/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneMigrationBase.cs @@ -14,7 +14,11 @@ namespace NzbDrone.Core.Datastore.Migration.Framework public override void Up() { - switch ((MigrationType)ApplicationContext) + var context = (MigrationContext)ApplicationContext; + + SQLiteAlter = context.SQLiteAlter; + + switch (context.MigrationType) { case MigrationType.Main: MainDbUpgrade(); @@ -29,6 +33,7 @@ namespace NzbDrone.Core.Datastore.Migration.Framework } } + public ISQLiteAlter SQLiteAlter { get; private set; } public override void Down() { diff --git a/NzbDrone.Core/Datastore/Migration/Framework/SQLiteMigrationHelper.cs b/NzbDrone.Core/Datastore/Migration/Framework/SQLiteMigrationHelper.cs new file mode 100644 index 000000000..4f7002664 --- /dev/null +++ b/NzbDrone.Core/Datastore/Migration/Framework/SQLiteMigrationHelper.cs @@ -0,0 +1,134 @@ +using System; +using System.Collections.Generic; +using System.Data.SQLite; +using System.Linq; +using System.Text.RegularExpressions; + +namespace NzbDrone.Core.Datastore.Migration.Framework +{ + public interface ISQLiteMigrationHelper + { + Dictionary GetColumns(string tableName); + void CreateTable(string tableName, IEnumerable values); + void CopyData(string sourceTable, string destinationTable, IEnumerable columns); + int GetRowCount(string tableName); + void DropTable(string tableName); + void RenameTable(string tableName, string newName); + SQLiteTransaction BeginTransaction(); + } + + public class SQLiteMigrationHelper : ISQLiteMigrationHelper + { + private readonly SQLiteConnection _connection; + + private static readonly Regex SchemaRegex = new Regex(@"['\""\[](?\w+)['\""\]]\s(?[\w-\s]+)", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.Multiline); + + public SQLiteMigrationHelper(IConnectionStringFactory connectionStringFactory) + { + _connection = new SQLiteConnection(connectionStringFactory.MainDbConnectionString); + _connection.Open(); + } + + private string GetOriginalSql(string tableName) + { + var command = + new SQLiteCommand(string.Format("SELECT sql FROM sqlite_master WHERE type='table' AND name ='{0}'", + tableName)); + + command.Connection = _connection; + return (string)command.ExecuteScalar(); + } + + public Dictionary GetColumns(string tableName) + { + var originalSql = GetOriginalSql(tableName); + + var matches = SchemaRegex.Matches(originalSql); + + return matches.Cast().ToDictionary( + match => match.Groups["name"].Value.Trim(), + match => new SQLiteColumn + { + Name = match.Groups["name"].Value.Trim(), + Schema = match.Groups["schema"].Value.Trim() + }); + } + + public void CreateTable(string tableName, IEnumerable values) + { + var columns = String.Join(",", values.Select(c => c.ToString())); + + var command = new SQLiteCommand(string.Format("CREATE TABLE [{0}] ({1})", tableName, columns)); + command.Connection = _connection; + + command.ExecuteNonQuery(); + } + + public void CopyData(string sourceTable, string destinationTable, IEnumerable columns) + { + var originalCount = GetRowCount(sourceTable); + + var columnsToTransfer = String.Join(",", columns.Select(c => c.Name)); + + var transferCommand = BuildCommand("INSERT INTO {0} SELECT {1} FROM {2};", destinationTable, columnsToTransfer, sourceTable); + + transferCommand.ExecuteNonQuery(); + + var transferredRows = GetRowCount(destinationTable); + + + if (transferredRows != originalCount) + { + throw new ApplicationException(string.Format("Expected {0} rows to be copied from [{1}] to [{2}]. But only copied {3}", originalCount, sourceTable, destinationTable, transferredRows)); + } + } + + + public void DropTable(string tableName) + { + var dropCommand = BuildCommand("DROP TABLE {0};", tableName); + dropCommand.ExecuteNonQuery(); + } + + + public void RenameTable(string tableName, string newName) + { + var renameCommand = BuildCommand("ALTER TABLE {0} RENAME TO {1};", tableName, newName); + renameCommand.ExecuteNonQuery(); + } + + public int GetRowCount(string tableName) + { + var countCommand = BuildCommand("SELECT COUNT(*) FROM {0};", tableName); + return Convert.ToInt32(countCommand.ExecuteScalar()); + } + + + public SQLiteTransaction BeginTransaction() + { + return _connection.BeginTransaction(); + } + + private SQLiteCommand BuildCommand(string format, params string[] args) + { + var command = new SQLiteCommand(string.Format(format, args)); + command.Connection = _connection; + return command; + } + + + public class SQLiteColumn + { + public string Name { get; set; } + public string Schema { get; set; } + + public override string ToString() + { + return string.Format("[{0}] {1}", Name, Schema); + } + } + } + + +} \ No newline at end of file diff --git a/NzbDrone.Core/Datastore/Migration/Framework/SqliteAlter.cs b/NzbDrone.Core/Datastore/Migration/Framework/SqliteAlter.cs index 943409916..ce2372d69 100644 --- a/NzbDrone.Core/Datastore/Migration/Framework/SqliteAlter.cs +++ b/NzbDrone.Core/Datastore/Migration/Framework/SqliteAlter.cs @@ -1,69 +1,42 @@ -using System; -using System.Collections.Generic; -using System.Data.SQLite; +using System.Collections.Generic; using System.Linq; -using System.Text.RegularExpressions; -using FluentMigrator.Builders.Execute; namespace NzbDrone.Core.Datastore.Migration.Framework { - public class SQLiteAlter + public interface ISQLiteAlter { - private readonly SQLiteConnection _connection; - - private static readonly Regex SchemaRegex = new Regex(@"[\""\[](?\w+)[\""\]]\s(?[\w-\s]+)", - RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.Multiline); - - public SQLiteAlter(string connectionString) - { - _connection = new SQLiteConnection(connectionString); - _connection.Open(); - } - - private string GetOriginalSql(string tableName) - { - var command = - new SQLiteCommand(string.Format("SELECT sql FROM sqlite_master WHERE type='table' AND name ='{0}'", - tableName)); - - command.Connection = _connection; - return (string)command.ExecuteScalar(); - } - - public Dictionary GetColumns(string tableName) - { - var originalSql = GetOriginalSql(tableName); - - var matches = SchemaRegex.Matches(originalSql); - - return matches.Cast().ToDictionary( - match => match.Groups["name"].Value.Trim(), - match => new SQLiteColumn - { - Name = match.Groups["name"].Value.Trim(), - Schema = match.Groups["schema"].Value.Trim() - }); - } - - public void CreateTable(string tableName, Dictionary.ValueCollection values) - { - var columns = String.Join(",", values.Select(c => c.ToString())); - - var command = new SQLiteCommand(string.Format("CREATE TABLE [{0}] ({1})", tableName, columns)); - command.Connection = _connection; - - command.ExecuteNonQuery(); - } + void DropColumns(string tableName, IEnumerable columns); } - public class SQLiteColumn + public class SQLiteAlter : ISQLiteAlter { - public string Name { get; set; } - public string Schema { get; set; } + private readonly ISQLiteMigrationHelper _sqLiteMigrationHelper; - public override string ToString() + public SQLiteAlter(ISQLiteMigrationHelper sqLiteMigrationHelper) { - return string.Format("[{0}] {1}", Name, Schema); + _sqLiteMigrationHelper = sqLiteMigrationHelper; + } + + public void DropColumns(string tableName, IEnumerable columns) + { + using (var transaction = _sqLiteMigrationHelper.BeginTransaction()) + { + var originalColumns = _sqLiteMigrationHelper.GetColumns(tableName); + + var newColumns = originalColumns.Where(c => !columns.Contains(c.Key)).Select(c => c.Value).ToList(); + + var tempTableName = tableName + "_temp"; + + _sqLiteMigrationHelper.CreateTable(tempTableName, newColumns); + + _sqLiteMigrationHelper.CopyData(tableName, tempTableName, newColumns); + + _sqLiteMigrationHelper.DropTable(tableName); + + _sqLiteMigrationHelper.RenameTable(tempTableName, tableName); + + transaction.Commit(); + } } } } \ No newline at end of file diff --git a/NzbDrone.Core/NzbDrone.Core.csproj b/NzbDrone.Core/NzbDrone.Core.csproj index bc1056aa1..6ce281081 100644 --- a/NzbDrone.Core/NzbDrone.Core.csproj +++ b/NzbDrone.Core/NzbDrone.Core.csproj @@ -194,6 +194,7 @@ + @@ -213,6 +214,7 @@ + @@ -220,7 +222,8 @@ - + +