From b9623957fdbdd8ba4c723e1d7cce4bfb4e44a64b Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Mon, 25 Aug 2014 23:29:37 +0200 Subject: [PATCH] Fixed: All migrations are now transactional and will rollback if failed --- .../AlterFixture.cs | 142 ---------- .../DuplicateFixture.cs | 41 --- .../SqliteSchemaDumperFixture.cs | 84 ++++++ src/NzbDrone.Core.Test/Framework/DbTest.cs | 2 - .../NzbDrone.Core.Test.csproj | 3 +- ...02_Remove_tvrage_imdb_unique_constraint.cs | 4 +- .../Datastore/Migration/008_remove_backlog.cs | 4 +- .../Migration/009_fix_renameEpisodes.cs | 2 +- .../Datastore/Migration/011_remove_ignored.cs | 4 +- .../Migration/012_remove_custom_start_date.cs | 2 +- .../Datastore/Migration/014_drop_air_date.cs | 2 +- .../Migration/018_remove_duplicates.cs | 91 ++++-- .../019_restore_unique_constraints.cs | 12 +- .../Migration/024_drop_tvdb_episodeid.cs | 2 +- .../031_delete_old_naming_config_columns.cs | 17 +- .../Migration/034_remove_series_contraints.cs | 7 +- .../037_add_configurable_qualities.cs | 3 +- .../Migration/055_drop_old_profile_columns.cs | 2 +- .../Migration/058_drop_epsiode_file_path.cs | 2 +- .../060_remove_enable_from_indexers.cs | 4 +- .../Migration/Framework/MigrationContext.cs | 2 - .../Framework/MigrationController.cs | 12 +- .../Framework/NzbDroneMigrationBase.cs | 6 - .../Framework/NzbDroneSqliteProcessor.cs | 109 +++++++ .../NzbDroneSqliteProcessorFactory.cs | 19 ++ .../Migration/Framework/SQLiteColumn.cs | 13 - .../Migration/Framework/SQLiteIndex.cs | 44 --- .../Framework/SQLiteMigrationHelper.cs | 226 --------------- .../Migration/Framework/SqliteAlter.cs | 103 ------- .../Migration/Framework/SqliteSchemaDumper.cs | 265 ++++++++++++++++++ .../Migration/Framework/SqliteSyntaxReader.cs | 257 +++++++++++++++++ src/NzbDrone.Core/NzbDrone.Core.csproj | 8 +- 32 files changed, 842 insertions(+), 652 deletions(-) delete mode 100644 src/NzbDrone.Core.Test/Datastore/SQLiteMigrationHelperTests/AlterFixture.cs delete mode 100644 src/NzbDrone.Core.Test/Datastore/SQLiteMigrationHelperTests/DuplicateFixture.cs create mode 100644 src/NzbDrone.Core.Test/Datastore/SqliteSchemaDumperTests/SqliteSchemaDumperFixture.cs create mode 100644 src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneSqliteProcessor.cs create mode 100644 src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneSqliteProcessorFactory.cs delete mode 100644 src/NzbDrone.Core/Datastore/Migration/Framework/SQLiteColumn.cs delete mode 100644 src/NzbDrone.Core/Datastore/Migration/Framework/SQLiteIndex.cs delete mode 100644 src/NzbDrone.Core/Datastore/Migration/Framework/SQLiteMigrationHelper.cs delete mode 100644 src/NzbDrone.Core/Datastore/Migration/Framework/SqliteAlter.cs create mode 100644 src/NzbDrone.Core/Datastore/Migration/Framework/SqliteSchemaDumper.cs create mode 100644 src/NzbDrone.Core/Datastore/Migration/Framework/SqliteSyntaxReader.cs diff --git a/src/NzbDrone.Core.Test/Datastore/SQLiteMigrationHelperTests/AlterFixture.cs b/src/NzbDrone.Core.Test/Datastore/SQLiteMigrationHelperTests/AlterFixture.cs deleted file mode 100644 index 2db04935f..000000000 --- a/src/NzbDrone.Core.Test/Datastore/SQLiteMigrationHelperTests/AlterFixture.cs +++ /dev/null @@ -1,142 +0,0 @@ -using System.Collections.Generic; -using FizzWare.NBuilder; -using FluentAssertions; -using NUnit.Framework; -using NzbDrone.Core.Datastore.Migration.Framework; -using NzbDrone.Core.Test.Framework; -using NzbDrone.Core.Tv; -using System.Linq; -using NzbDrone.Test.Common; - -namespace NzbDrone.Core.Test.Datastore.SQLiteMigrationHelperTests -{ - [TestFixture] - public class AlterFixture : DbTest - { - private SqLiteMigrationHelper _subject; - - [SetUp] - public void SetUp() - { - _subject = Mocker.Resolve(); - } - - [Test] - public void should_parse_existing_columns() - { - var columns = _subject.GetColumns("Series"); - - columns.Should().NotBeEmpty(); - - columns.Values.Should().NotContain(c => string.IsNullOrWhiteSpace(c.Name)); - columns.Values.Should().NotContain(c => string.IsNullOrWhiteSpace(c.Schema)); - } - - [Test] - public void should_create_table_from_column_list() - { - var columns = _subject.GetColumns("Series"); - columns.Remove("Title"); - - _subject.CreateTable("Series_New", columns.Values, new List()); - - var newColumns = _subject.GetColumns("Series_New"); - - newColumns.Values.Should().HaveSameCount(columns.Values); - newColumns.Should().NotContainKey("Title"); - } - - - [Test] - public void should_be_able_to_transfer_empty_tables() - { - var columns = _subject.GetColumns("Series"); - var indexes = _subject.GetIndexes("Series"); - columns.Remove("Title"); - - _subject.CreateTable("Series_New", columns.Values, indexes); - - - _subject.CopyData("Series", "Series_New", columns.Values); - } - - [Test] - public void should_transfer_table_with_data() - { - var originalEpisodes = Builder.CreateListOfSize(10).BuildListOfNew(); - - Mocker.Resolve().InsertMany(originalEpisodes); - - var columns = _subject.GetColumns("Episodes"); - var indexes = _subject.GetIndexes("Episodes"); - - columns.Remove("Title"); - - _subject.CreateTable("Episodes_New", columns.Values, indexes); - - _subject.CopyData("Episodes", "Episodes_New", columns.Values); - - _subject.GetRowCount("Episodes_New").Should().Be(originalEpisodes.Count); - } - - [Test] - public void should_read_existing_indexes() - { - var indexes = _subject.GetIndexes("QualityDefinitions"); - - indexes.Should().NotBeEmpty(); - - indexes.Should().OnlyContain(c => c != null); - indexes.Should().OnlyContain(c => !string.IsNullOrWhiteSpace(c.Column)); - indexes.Should().OnlyContain(c => c.Table == "QualityDefinitions"); - indexes.Should().OnlyContain(c => c.Unique); - } - - [Test] - public void should_add_indexes_when_creating_new_table() - { - var columns = _subject.GetColumns("QualityDefinitions"); - var indexes = _subject.GetIndexes("QualityDefinitions"); - - _subject.CreateTable("QualityDefinitionsB", columns.Values, indexes); - - var newIndexes = _subject.GetIndexes("QualityDefinitionsB"); - - newIndexes.Should().HaveSameCount(indexes); - newIndexes.Select(c=>c.Column).Should().BeEquivalentTo(indexes.Select(c=>c.Column)); - } - - [Test] - public void should_be_able_to_create_table_with_new_indexes() - { - var columns = _subject.GetColumns("Series"); - columns.Remove("Title"); - - _subject.CreateTable("Series_New", columns.Values, new List{new SQLiteIndex{Column = "AirTime", Table = "Series_New", Unique = true}}); - - var newColumns = _subject.GetColumns("Series_New"); - var newIndexes = _subject.GetIndexes("Series_New"); - - newColumns.Values.Should().HaveSameCount(columns.Values); - newIndexes.Should().Contain(i=>i.Column == "AirTime"); - } - - [Test] - public void should_create_indexes_with_the_same_uniqueness() - { - var columns = _subject.GetColumns("Series"); - var indexes = _subject.GetIndexes("Series"); - - var tempIndexes = indexes.JsonClone(); - - tempIndexes[0].Unique = false; - tempIndexes[1].Unique = true; - - _subject.CreateTable("Series_New", columns.Values, tempIndexes); - var newIndexes = _subject.GetIndexes("Series_New"); - - newIndexes.Should().HaveSameCount(tempIndexes); - newIndexes.ShouldAllBeEquivalentTo(tempIndexes, options => options.Excluding(o => o.IndexName).Excluding(o => o.Table)); - } - } -} \ No newline at end of file diff --git a/src/NzbDrone.Core.Test/Datastore/SQLiteMigrationHelperTests/DuplicateFixture.cs b/src/NzbDrone.Core.Test/Datastore/SQLiteMigrationHelperTests/DuplicateFixture.cs deleted file mode 100644 index c7e8e2ad3..000000000 --- a/src/NzbDrone.Core.Test/Datastore/SQLiteMigrationHelperTests/DuplicateFixture.cs +++ /dev/null @@ -1,41 +0,0 @@ -using System.Linq; -using FizzWare.NBuilder; -using FluentAssertions; -using NUnit.Framework; -using NzbDrone.Core.Datastore.Migration.Framework; -using NzbDrone.Core.Test.Framework; -using NzbDrone.Core.Tv; - -namespace NzbDrone.Core.Test.Datastore.SQLiteMigrationHelperTests -{ - [TestFixture] - public class DuplicateFixture : DbTest - { - private SqLiteMigrationHelper _subject; - - [SetUp] - public void SetUp() - { - _subject = Mocker.Resolve(); - } - - - [Test] - public void get_duplicates() - { - var series = Builder.CreateListOfSize(10) - .Random(3) - .With(c => c.ProfileId = 100) - .BuildListOfNew(); - - Db.InsertMany(series); - - var duplicates = _subject.GetDuplicates("series", "ProfileId").ToList(); - - - duplicates.Should().HaveCount(1); - duplicates.First().Should().HaveCount(3); - } - - } -} \ No newline at end of file diff --git a/src/NzbDrone.Core.Test/Datastore/SqliteSchemaDumperTests/SqliteSchemaDumperFixture.cs b/src/NzbDrone.Core.Test/Datastore/SqliteSchemaDumperTests/SqliteSchemaDumperFixture.cs new file mode 100644 index 000000000..0bb4e2f3c --- /dev/null +++ b/src/NzbDrone.Core.Test/Datastore/SqliteSchemaDumperTests/SqliteSchemaDumperFixture.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Text; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.Datastore.Migration.Framework; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.Datastore.SqliteSchemaDumperTests +{ + [TestFixture] + public class SqliteSchemaDumperFixture + { + public SqliteSchemaDumper Subject { get; private set; } + + [SetUp] + public void Setup() + { + Subject = new SqliteSchemaDumper(null, null); + } + + [TestCase(@"CREATE TABLE TestTable (MyId INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT)", "TestTable", "MyId")] + [TestCase(@"CREATE TABLE ""TestTable"" (""MyId"" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT)", "TestTable", "MyId")] + [TestCase(@"CREATE TABLE [TestTable] ([MyId] INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT)", "TestTable", "MyId")] + [TestCase(@"CREATE TABLE `TestTable` (`MyId` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT)", "TestTable", "MyId")] + [TestCase(@"CREATE TABLE ""Test """"Table"" (""My""""Id"" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT)", "Test \"Table", "My\"Id")] + [TestCase(@"CREATE TABLE [Test Table] ([My Id] INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT)", "Test Table", "My Id")] + [TestCase(@" CREATE TABLE `Test ``Table` ( `My`` Id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT ) ", "Test `Table", "My` Id")] + public void should_parse_table_language_flavors(String sql, String tableName, String columnName) + { + var result = Subject.ReadTableSchema(sql); + + result.Name.Should().Be(tableName); + result.Columns.Count.Should().Be(1); + result.Columns.First().Name.Should().Be(columnName); + } + + [TestCase(@"CREATE INDEX TestIndex ON TestTable (MyId)", "TestIndex", "TestTable", "MyId")] + [TestCase(@"CREATE INDEX ""TestIndex"" ON ""TestTable"" (""MyId"" ASC)", "TestIndex", "TestTable", "MyId")] + [TestCase(@"CREATE INDEX [TestIndex] ON ""TestTable"" ([MyId] DESC)", "TestIndex", "TestTable", "MyId")] + [TestCase(@"CREATE INDEX `TestIndex` ON `TestTable` (`MyId` COLLATE abc ASC)", "TestIndex", "TestTable", "MyId")] + [TestCase(@"CREATE INDEX ""Test """"Index"" ON ""TestTable"" (""My""""Id"" ASC)", "Test \"Index", "TestTable", "My\"Id")] + [TestCase(@"CREATE INDEX [Test Index] ON [TestTable] ([My Id]) ", "Test Index", "TestTable", "My Id")] + [TestCase(@" CREATE INDEX `Test ``Index` ON ""TestTable"" ( `My`` Id` ASC) ", "Test `Index", "TestTable", "My` Id")] + public void should_parse_index_language_flavors(String sql, String indexName, String tableName, String columnName) + { + var result = Subject.ReadIndexSchema(sql); + + result.Name.Should().Be(indexName); + result.TableName.Should().Be(tableName); + result.Columns.Count.Should().Be(1); + result.Columns.First().Name.Should().Be(columnName); + } + + [TestCase(@"CREATE TABLE TestTable (MyId)")] + [TestCase(@"CREATE TABLE TestTable (MyId NOT NULL PRIMARY KEY AUTOINCREMENT)")] + [TestCase("CREATE TABLE TestTable\r\n(\t`MyId`\t NOT NULL PRIMARY KEY AUTOINCREMENT\n)")] + public void should_parse_column_attributes(String sql) + { + var result = Subject.ReadTableSchema(sql); + + result.Name.Should().Be("TestTable"); + result.Columns.Count.Should().Be(1); + result.Columns.First().Name.Should().Be("MyId"); + result.Columns.First().Type.Should().BeNull(); + } + + [Test] + public void should_ignore_unknown_symbols() + { + var result = Subject.ReadTableSchema("CREATE TABLE TestTable (MyId INTEGER DEFAULT 10 CHECK (Some weir +1e3 expression), CONSTRAINT NULL, MyCol INTEGER)"); + + result.Name.Should().Be("TestTable"); + result.Columns.Count.Should().Be(2); + result.Columns.First().Name.Should().Be("MyId"); + result.Columns.First().Type.Should().Be(DbType.Int64); + result.Columns.Last().Name.Should().Be("MyCol"); + result.Columns.Last().Type.Should().Be(DbType.Int64); + + } + } +} diff --git a/src/NzbDrone.Core.Test/Framework/DbTest.cs b/src/NzbDrone.Core.Test/Framework/DbTest.cs index 4fdd67869..be9ffed86 100644 --- a/src/NzbDrone.Core.Test/Framework/DbTest.cs +++ b/src/NzbDrone.Core.Test/Framework/DbTest.cs @@ -92,8 +92,6 @@ namespace NzbDrone.Core.Test.Framework Mocker.SetConstant(Mocker.Resolve()); Mocker.SetConstant(Mocker.Resolve()); - Mocker.SetConstant(Mocker.Resolve()); - Mocker.SetConstant(Mocker.Resolve()); Mocker.SetConstant(Mocker.Resolve()); MapRepository.Instance.EnableTraceLogging = true; diff --git a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj index cf704f1db..137a49fba 100644 --- a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj +++ b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj @@ -117,8 +117,7 @@ - - + diff --git a/src/NzbDrone.Core/Datastore/Migration/002_Remove_tvrage_imdb_unique_constraint.cs b/src/NzbDrone.Core/Datastore/Migration/002_Remove_tvrage_imdb_unique_constraint.cs index 587032c7a..6fc6a6cd3 100644 --- a/src/NzbDrone.Core/Datastore/Migration/002_Remove_tvrage_imdb_unique_constraint.cs +++ b/src/NzbDrone.Core/Datastore/Migration/002_Remove_tvrage_imdb_unique_constraint.cs @@ -8,8 +8,8 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - Execute.Sql("DROP INDEX IX_Series_TvRageId;"); - Execute.Sql("DROP INDEX IX_Series_ImdbId;"); + Delete.Index().OnTable("Series").OnColumn("TvRageId"); + Delete.Index().OnTable("Series").OnColumn("ImdbId"); } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/008_remove_backlog.cs b/src/NzbDrone.Core/Datastore/Migration/008_remove_backlog.cs index 83fd62478..19e16242c 100644 --- a/src/NzbDrone.Core/Datastore/Migration/008_remove_backlog.cs +++ b/src/NzbDrone.Core/Datastore/Migration/008_remove_backlog.cs @@ -8,8 +8,8 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - SqLiteAlter.DropColumns("Series", new[] { "BacklogSetting" }); - SqLiteAlter.DropColumns("NamingConfig", new[] { "UseSceneName" }); + Delete.Column("BacklogSetting").FromTable("Series"); + Delete.Column("UseSceneName").FromTable("NamingConfig"); } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/009_fix_renameEpisodes.cs b/src/NzbDrone.Core/Datastore/Migration/009_fix_renameEpisodes.cs index f5ef5b2d1..bdc0c54e5 100644 --- a/src/NzbDrone.Core/Datastore/Migration/009_fix_renameEpisodes.cs +++ b/src/NzbDrone.Core/Datastore/Migration/009_fix_renameEpisodes.cs @@ -8,7 +8,7 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - SqLiteAlter.DropColumns("NamingConfig", new[] { "SeasonFolderFormat" }); + Delete.Column("SeasonFolderFormat").FromTable("NamingConfig"); Execute.Sql("UPDATE NamingConfig SET RenameEpisodes = 1 WHERE RenameEpisodes = -1"); Execute.Sql("UPDATE NamingConfig SET RenameEpisodes = 0 WHERE RenameEpisodes = -2"); diff --git a/src/NzbDrone.Core/Datastore/Migration/011_remove_ignored.cs b/src/NzbDrone.Core/Datastore/Migration/011_remove_ignored.cs index 8f62c3b46..193b25094 100644 --- a/src/NzbDrone.Core/Datastore/Migration/011_remove_ignored.cs +++ b/src/NzbDrone.Core/Datastore/Migration/011_remove_ignored.cs @@ -8,8 +8,8 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - SqLiteAlter.DropColumns("Episodes", new[] { "Ignored" }); - SqLiteAlter.DropColumns("Seasons", new[] { "Ignored" }); + Delete.Column("Ignored").FromTable("Seasons"); + Delete.Column("Ignored").FromTable("Episodes"); } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/012_remove_custom_start_date.cs b/src/NzbDrone.Core/Datastore/Migration/012_remove_custom_start_date.cs index a7f156c87..8b19f1a3f 100644 --- a/src/NzbDrone.Core/Datastore/Migration/012_remove_custom_start_date.cs +++ b/src/NzbDrone.Core/Datastore/Migration/012_remove_custom_start_date.cs @@ -8,7 +8,7 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - SqLiteAlter.DropColumns("Series", new[] { "CustomStartDate" }); + Delete.Column("CustomStartDate").FromTable("Series"); } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/014_drop_air_date.cs b/src/NzbDrone.Core/Datastore/Migration/014_drop_air_date.cs index b88282202..00af970b9 100644 --- a/src/NzbDrone.Core/Datastore/Migration/014_drop_air_date.cs +++ b/src/NzbDrone.Core/Datastore/Migration/014_drop_air_date.cs @@ -8,7 +8,7 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - SqLiteAlter.DropColumns("Episodes", new []{ "AirDate" }); + Delete.Column("AirDate").FromTable("Episodes"); } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/018_remove_duplicates.cs b/src/NzbDrone.Core/Datastore/Migration/018_remove_duplicates.cs index 7fb11e5e7..9b34053b6 100644 --- a/src/NzbDrone.Core/Datastore/Migration/018_remove_duplicates.cs +++ b/src/NzbDrone.Core/Datastore/Migration/018_remove_duplicates.cs @@ -1,6 +1,9 @@ using FluentMigrator; using NzbDrone.Core.Datastore.Migration.Framework; using System.Linq; +using System.Data; +using System.Collections.Generic; +using System; namespace NzbDrone.Core.Datastore.Migration { @@ -9,52 +12,88 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - using (var transaction = MigrationHelper.BeginTransaction()) + Execute.WithConnection(RemoveDuplicates); + } + + private void RemoveDuplicates(IDbConnection conn, IDbTransaction tran) + { + RemoveDuplicateSeries(conn, tran, "TvdbId"); + RemoveDuplicateSeries(conn, tran, "TitleSlug"); + + var duplicatedEpisodes = GetDuplicates(conn, tran, "Episodes", "TvDbEpisodeId"); + + foreach (var duplicate in duplicatedEpisodes) { - RemoveDuplicateSeries("TvdbId"); - RemoveDuplicateSeries("TitleSlug"); - - var duplicatedEpisodes = MigrationHelper.GetDuplicates("Episodes", "TvDbEpisodeId"); - - foreach (var duplicate in duplicatedEpisodes) + foreach (var episodeId in duplicate.OrderBy(c => c.Key).Skip(1).Select(c => c.Key)) { - foreach (var episodeId in duplicate.OrderBy(c => c.Key).Skip(1).Select(c => c.Key)) - { - RemoveEpisodeRows(episodeId); - } + RemoveEpisodeRows(conn, tran, episodeId); } - - transaction.Commit(); } } - private void RemoveDuplicateSeries(string field) + private IEnumerable>> GetDuplicates(IDbConnection conn, IDbTransaction tran, string tableName, string columnName) { - var duplicatedSeries = MigrationHelper.GetDuplicates("Series", field); + var getDuplicates = conn.CreateCommand(); + getDuplicates.Transaction = tran; + getDuplicates.CommandText = string.Format("select id, {0} from {1}", columnName, tableName); + + var result = new List>(); + + using (var reader = getDuplicates.ExecuteReader()) + { + while (reader.Read()) + { + result.Add(new KeyValuePair(reader.GetInt32(0), (T)Convert.ChangeType(reader[1], typeof(T)))); + } + } + + return result.GroupBy(c => c.Value).Where(g => g.Count() > 1); + } + + private void RemoveDuplicateSeries(IDbConnection conn, IDbTransaction tran, string field) + { + var duplicatedSeries = GetDuplicates(conn, tran, "Series", field); foreach (var duplicate in duplicatedSeries) { foreach (var seriesId in duplicate.OrderBy(c => c.Key).Skip(1).Select(c => c.Key)) { - RemoveSeriesRows(seriesId); + RemoveSeriesRows(conn, tran, seriesId); } } } - private void RemoveSeriesRows(int seriesId) + private void RemoveSeriesRows(IDbConnection conn, IDbTransaction tran, int seriesId) { - MigrationHelper.ExecuteNonQuery("DELETE FROM Series WHERE Id = {0}", seriesId.ToString()); - MigrationHelper.ExecuteNonQuery("DELETE FROM Episodes WHERE SeriesId = {0}", seriesId.ToString()); - MigrationHelper.ExecuteNonQuery("DELETE FROM Seasons WHERE SeriesId = {0}", seriesId.ToString()); - MigrationHelper.ExecuteNonQuery("DELETE FROM History WHERE SeriesId = {0}", seriesId.ToString()); - MigrationHelper.ExecuteNonQuery("DELETE FROM EpisodeFiles WHERE SeriesId = {0}", seriesId.ToString()); + var deleteCmd = conn.CreateCommand(); + deleteCmd.Transaction = tran; + + deleteCmd.CommandText = String.Format("DELETE FROM Series WHERE Id = {0}", seriesId.ToString()); + deleteCmd.ExecuteNonQuery(); + + deleteCmd.CommandText = String.Format("DELETE FROM Episodes WHERE SeriesId = {0}", seriesId.ToString()); + deleteCmd.ExecuteNonQuery(); + + deleteCmd.CommandText = String.Format("DELETE FROM Seasons WHERE SeriesId = {0}", seriesId.ToString()); + deleteCmd.ExecuteNonQuery(); + + deleteCmd.CommandText = String.Format("DELETE FROM History WHERE SeriesId = {0}", seriesId.ToString()); + deleteCmd.ExecuteNonQuery(); + + deleteCmd.CommandText = String.Format("DELETE FROM EpisodeFiles WHERE SeriesId = {0}", seriesId.ToString()); + deleteCmd.ExecuteNonQuery(); } - private void RemoveEpisodeRows(int episodeId) + private void RemoveEpisodeRows(IDbConnection conn, IDbTransaction tran, int episodeId) { - MigrationHelper.ExecuteNonQuery("DELETE FROM Episodes WHERE Id = {0}", episodeId.ToString()); - MigrationHelper.ExecuteNonQuery("DELETE FROM History WHERE EpisodeId = {0}", episodeId.ToString()); - } + var deleteCmd = conn.CreateCommand(); + deleteCmd.Transaction = tran; + deleteCmd.CommandText = String.Format("DELETE FROM Episodes WHERE Id = {0}", episodeId.ToString()); + deleteCmd.ExecuteNonQuery(); + + deleteCmd.CommandText = String.Format("DELETE FROM History WHERE EpisodeId = {0}", episodeId.ToString()); + deleteCmd.ExecuteNonQuery(); + } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/019_restore_unique_constraints.cs b/src/NzbDrone.Core/Datastore/Migration/019_restore_unique_constraints.cs index c4e304df7..bf70a9532 100644 --- a/src/NzbDrone.Core/Datastore/Migration/019_restore_unique_constraints.cs +++ b/src/NzbDrone.Core/Datastore/Migration/019_restore_unique_constraints.cs @@ -8,12 +8,14 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - SqLiteAlter.AddIndexes("Series", - new SQLiteIndex { Column = "TvdbId", Table = "Series", Unique = true }, - new SQLiteIndex { Column = "TitleSlug", Table = "Series", Unique = true }); + // During an earlier version of drone, the indexes weren't recreated during alter table. + Execute.Sql("DROP INDEX IF EXISTS \"IX_Series_TvdbId\""); + Execute.Sql("DROP INDEX IF EXISTS \"IX_Series_TitleSlug\""); + Execute.Sql("DROP INDEX IF EXISTS \"IX_Episodes_TvDbEpisodeId\""); - SqLiteAlter.AddIndexes("Episodes", - new SQLiteIndex { Column = "TvDbEpisodeId", Table = "Episodes", Unique = true }); + Create.Index().OnTable("Series").OnColumn("TvdbId").Unique(); + Create.Index().OnTable("Series").OnColumn("TitleSlug").Unique(); + Create.Index().OnTable("Episodes").OnColumn("TvDbEpisodeId").Unique(); } } diff --git a/src/NzbDrone.Core/Datastore/Migration/024_drop_tvdb_episodeid.cs b/src/NzbDrone.Core/Datastore/Migration/024_drop_tvdb_episodeid.cs index 90144bcbb..c723f462c 100644 --- a/src/NzbDrone.Core/Datastore/Migration/024_drop_tvdb_episodeid.cs +++ b/src/NzbDrone.Core/Datastore/Migration/024_drop_tvdb_episodeid.cs @@ -8,7 +8,7 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - SqLiteAlter.DropColumns("Episodes", new[] { "TvDbEpisodeId" }); + Delete.Column("TvDbEpisodeId").FromTable("Episodes"); } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/031_delete_old_naming_config_columns.cs b/src/NzbDrone.Core/Datastore/Migration/031_delete_old_naming_config_columns.cs index c570fd574..90c7571af 100644 --- a/src/NzbDrone.Core/Datastore/Migration/031_delete_old_naming_config_columns.cs +++ b/src/NzbDrone.Core/Datastore/Migration/031_delete_old_naming_config_columns.cs @@ -8,16 +8,13 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - SqLiteAlter.DropColumns("NamingConfig", - new[] - { - "Separator", - "NumberStyle", - "IncludeSeriesTitle", - "IncludeEpisodeTitle", - "IncludeQuality", - "ReplaceSpaces" - }); + Delete.Column("Separator") + .Column("NumberStyle") + .Column("IncludeSeriesTitle") + .Column("IncludeEpisodeTitle") + .Column("IncludeQuality") + .Column("ReplaceSpaces") + .FromTable("NamingConfig"); } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/034_remove_series_contraints.cs b/src/NzbDrone.Core/Datastore/Migration/034_remove_series_contraints.cs index 7874e9c38..a930e990b 100644 --- a/src/NzbDrone.Core/Datastore/Migration/034_remove_series_contraints.cs +++ b/src/NzbDrone.Core/Datastore/Migration/034_remove_series_contraints.cs @@ -1,4 +1,5 @@ -using FluentMigrator; +using System.Data; +using FluentMigrator; using NzbDrone.Core.Datastore.Migration.Framework; namespace NzbDrone.Core.Datastore.Migration @@ -8,7 +9,9 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - SqLiteAlter.Nullify("Series", new[] { "ImdbId", "TitleSlug" }); + Alter.Table("Series") + .AlterColumn("ImdbId").AsString().Nullable() + .AlterColumn("TitleSlug").AsString().Nullable(); } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/037_add_configurable_qualities.cs b/src/NzbDrone.Core/Datastore/Migration/037_add_configurable_qualities.cs index fddd8e5a5..06ced4854 100644 --- a/src/NzbDrone.Core/Datastore/Migration/037_add_configurable_qualities.cs +++ b/src/NzbDrone.Core/Datastore/Migration/037_add_configurable_qualities.cs @@ -11,7 +11,8 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - SqLiteAlter.DropColumns("QualityProfiles", new[] { "Allowed" }); + Delete.Column("Allowed").FromTable("QualityProfiles"); + Alter.Column("Items").OnTable("QualityProfiles").AsString().NotNullable(); Create.TableForModel("QualityDefinitions") diff --git a/src/NzbDrone.Core/Datastore/Migration/055_drop_old_profile_columns.cs b/src/NzbDrone.Core/Datastore/Migration/055_drop_old_profile_columns.cs index 349c10ca1..3f13f5e84 100644 --- a/src/NzbDrone.Core/Datastore/Migration/055_drop_old_profile_columns.cs +++ b/src/NzbDrone.Core/Datastore/Migration/055_drop_old_profile_columns.cs @@ -8,7 +8,7 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - SqLiteAlter.DropColumns("Series", new[] { "QualityProfileId" }); + Delete.Column("QualityProfileId").FromTable("Series"); } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/058_drop_epsiode_file_path.cs b/src/NzbDrone.Core/Datastore/Migration/058_drop_epsiode_file_path.cs index 61cd22135..d2bfbcfd9 100644 --- a/src/NzbDrone.Core/Datastore/Migration/058_drop_epsiode_file_path.cs +++ b/src/NzbDrone.Core/Datastore/Migration/058_drop_epsiode_file_path.cs @@ -8,7 +8,7 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - SqLiteAlter.DropColumns("EpisodeFiles", new [] { "Path" }); + Delete.Column("Path").FromTable("EpisodeFiles"); } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/060_remove_enable_from_indexers.cs b/src/NzbDrone.Core/Datastore/Migration/060_remove_enable_from_indexers.cs index 4977cf901..05376c1d2 100644 --- a/src/NzbDrone.Core/Datastore/Migration/060_remove_enable_from_indexers.cs +++ b/src/NzbDrone.Core/Datastore/Migration/060_remove_enable_from_indexers.cs @@ -8,8 +8,8 @@ namespace NzbDrone.Core.Datastore.Migration { protected override void MainDbUpgrade() { - SqLiteAlter.DropColumns("Indexers", new[] { "Enable" }); - SqLiteAlter.DropColumns("DownloadClients", new[] { "Protocol" }); + Delete.Column("Enable").FromTable("Indexers"); + Delete.Column("Protocol").FromTable("DownloadClients"); } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/Framework/MigrationContext.cs b/src/NzbDrone.Core/Datastore/Migration/Framework/MigrationContext.cs index 90ac77a1e..e1bb47bcf 100644 --- a/src/NzbDrone.Core/Datastore/Migration/Framework/MigrationContext.cs +++ b/src/NzbDrone.Core/Datastore/Migration/Framework/MigrationContext.cs @@ -3,7 +3,5 @@ public class MigrationContext { public MigrationType MigrationType { get; set; } - public ISQLiteAlter SQLiteAlter { get; set; } - public ISqLiteMigrationHelper MigrationHelper { get; set; } } } \ No newline at end of file diff --git a/src/NzbDrone.Core/Datastore/Migration/Framework/MigrationController.cs b/src/NzbDrone.Core/Datastore/Migration/Framework/MigrationController.cs index 8192697f5..d8c011341 100644 --- a/src/NzbDrone.Core/Datastore/Migration/Framework/MigrationController.cs +++ b/src/NzbDrone.Core/Datastore/Migration/Framework/MigrationController.cs @@ -13,14 +13,10 @@ namespace NzbDrone.Core.Datastore.Migration.Framework public class MigrationController : IMigrationController { private readonly IAnnouncer _announcer; - private readonly ISQLiteAlter _sqLiteAlter; - private readonly ISqLiteMigrationHelper _migrationHelper; - public MigrationController(IAnnouncer announcer, ISQLiteAlter sqLiteAlter, ISqLiteMigrationHelper migrationHelper) + public MigrationController(IAnnouncer announcer) { _announcer = announcer; - _sqLiteAlter = sqLiteAlter; - _migrationHelper = migrationHelper; } public void MigrateToLatest(string connectionString, MigrationType migrationType) @@ -34,14 +30,12 @@ namespace NzbDrone.Core.Datastore.Migration.Framework Namespace = "NzbDrone.Core.Datastore.Migration", ApplicationContext = new MigrationContext { - MigrationType = migrationType, - SQLiteAlter = _sqLiteAlter, - MigrationHelper = _migrationHelper, + MigrationType = migrationType } }; var options = new MigrationOptions { PreviewOnly = false, Timeout = 60 }; - var factory = new SqliteProcessorFactory(); + var factory = new NzbDroneSqliteProcessorFactory(); var processor = factory.Create(connectionString, _announcer, options); var runner = new MigrationRunner(assembly, migrationContext, processor); runner.MigrateUp(true); diff --git a/src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneMigrationBase.cs b/src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneMigrationBase.cs index 5655649cd..c2fca697a 100644 --- a/src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneMigrationBase.cs +++ b/src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneMigrationBase.cs @@ -25,9 +25,6 @@ namespace NzbDrone.Core.Datastore.Migration.Framework { var context = (MigrationContext)ApplicationContext; - SqLiteAlter = context.SQLiteAlter; - MigrationHelper = context.MigrationHelper; - switch (context.MigrationType) { case MigrationType.Main: @@ -43,9 +40,6 @@ namespace NzbDrone.Core.Datastore.Migration.Framework } } - protected ISQLiteAlter SqLiteAlter { get; private set; } - protected ISqLiteMigrationHelper MigrationHelper { get; private set; } - public override void Down() { throw new NotImplementedException(); diff --git a/src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneSqliteProcessor.cs b/src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneSqliteProcessor.cs new file mode 100644 index 000000000..2400b2395 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneSqliteProcessor.cs @@ -0,0 +1,109 @@ +using System; +using System.Data; +using System.Linq; +using FluentMigrator; +using FluentMigrator.Exceptions; +using FluentMigrator.Expressions; +using FluentMigrator.Model; +using FluentMigrator.Runner; +using FluentMigrator.Runner.Generators.SQLite; +using FluentMigrator.Runner.Processors.SQLite; + +namespace NzbDrone.Core.Datastore.Migration.Framework +{ + public class NzbDroneSqliteProcessor : SqliteProcessor + { + public NzbDroneSqliteProcessor(IDbConnection connection, IMigrationGenerator generator, IAnnouncer announcer, IMigrationProcessorOptions options, FluentMigrator.Runner.Processors.IDbFactory factory) + : base(connection, generator, announcer, options, factory) + { + + } + + public override bool SupportsTransactions + { + get + { + return true; + } + } + + public override void Process(AlterColumnExpression expression) + { + var tableDefinition = GetTableSchema(expression.TableName); + + var columnDefinitions = tableDefinition.Columns.ToList(); + var columnIndex = columnDefinitions.FindIndex(c => c.Name == expression.Column.Name); + + if (columnIndex == -1) + { + throw new ApplicationException(String.Format("Column {0} does not exist on table {1}.", expression.Column.Name, expression.TableName)); + } + + columnDefinitions[columnIndex] = expression.Column; + + tableDefinition.Columns = columnDefinitions; + + ProcessAlterTable(tableDefinition); + } + + public override void Process(DeleteColumnExpression expression) + { + var tableDefinition = GetTableSchema(expression.TableName); + + var columnDefinitions = tableDefinition.Columns.ToList(); + var indexDefinitions = tableDefinition.Indexes.ToList(); + + var columnsToRemove = expression.ColumnNames.ToList(); + + columnDefinitions.RemoveAll(c => columnsToRemove.Remove(c.Name)); + indexDefinitions.RemoveAll(i => i.Columns.Any(c => expression.ColumnNames.Contains(c.Name))); + + tableDefinition.Columns = columnDefinitions; + tableDefinition.Indexes = indexDefinitions; + + if (columnsToRemove.Any()) + { + throw new ApplicationException(String.Format("Column {0} does not exist on table {1}.", columnsToRemove.First(), expression.TableName)); + } + + ProcessAlterTable(tableDefinition); + } + + protected virtual TableDefinition GetTableSchema(String tableName) + { + var schemaDumper = new SqliteSchemaDumper(this, Announcer); + var schema = schemaDumper.ReadDbSchema(); + + return schema.Single(v => v.Name == tableName); + } + + protected virtual void ProcessAlterTable(TableDefinition tableDefinition) + { + var tableName = tableDefinition.Name; + var tempTableName = tableName + "_temp"; + + var uid = 0; + while (TableExists(null, tempTableName)) + { + tempTableName = tableName + "_temp" + uid++; + } + + // What is the cleanest way to do this? Add function to Generator? + var quoter = new SqliteQuoter(); + var columnsToTransfer = String.Join(", ", tableDefinition.Columns.Select(c => quoter.QuoteColumnName(c.Name))); + + Process(new CreateTableExpression() { TableName = tempTableName, Columns = tableDefinition.Columns.ToList() }); + + Process(String.Format("INSERT INTO {0} SELECT {1} FROM {2}", quoter.QuoteTableName(tempTableName), columnsToTransfer, quoter.QuoteTableName(tableName))); + + Process(new DeleteTableExpression() { TableName = tableName }); + + Process(new RenameTableExpression() { OldName = tempTableName, NewName = tableName }); + + foreach (var index in tableDefinition.Indexes) + { + Process(new CreateIndexExpression() { Index = index }); + } + } + } +} diff --git a/src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneSqliteProcessorFactory.cs b/src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneSqliteProcessorFactory.cs new file mode 100644 index 000000000..6581bd74e --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneSqliteProcessorFactory.cs @@ -0,0 +1,19 @@ +using System; +using FluentMigrator; +using FluentMigrator.Runner; +using FluentMigrator.Runner.Generators.SQLite; +using FluentMigrator.Runner.Processors.SQLite; + +namespace NzbDrone.Core.Datastore.Migration.Framework +{ + public class NzbDroneSqliteProcessorFactory : SqliteProcessorFactory + { + public override IMigrationProcessor Create(String connectionString, IAnnouncer announcer, IMigrationProcessorOptions options) + { + var factory = new SqliteDbFactory(); + var connection = factory.CreateConnection(connectionString); + var generator = new SqliteGenerator() { compatabilityMode = CompatabilityMode.STRICT }; + return new NzbDroneSqliteProcessor(connection, generator, announcer, options, factory); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/Migration/Framework/SQLiteColumn.cs b/src/NzbDrone.Core/Datastore/Migration/Framework/SQLiteColumn.cs deleted file mode 100644 index 976916885..000000000 --- a/src/NzbDrone.Core/Datastore/Migration/Framework/SQLiteColumn.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace NzbDrone.Core.Datastore.Migration.Framework -{ - public class SQLiteColumn - { - public string Name { get; set; } - public string Schema { get; set; } - - public override string ToString() - { - return string.Format("[{0}] {1}", Name, Schema); - } - } -} \ No newline at end of file diff --git a/src/NzbDrone.Core/Datastore/Migration/Framework/SQLiteIndex.cs b/src/NzbDrone.Core/Datastore/Migration/Framework/SQLiteIndex.cs deleted file mode 100644 index 59129eaa0..000000000 --- a/src/NzbDrone.Core/Datastore/Migration/Framework/SQLiteIndex.cs +++ /dev/null @@ -1,44 +0,0 @@ -using System; - -namespace NzbDrone.Core.Datastore.Migration.Framework -{ - public class SQLiteIndex : IEquatable - { - public string Column { get; set; } - public string Table { get; set; } - public bool Unique { get; set; } - - public bool Equals(SQLiteIndex other) - { - return IndexName == other.IndexName; - } - - public override int GetHashCode() - { - return IndexName.GetHashCode(); - } - - public override string ToString() - { - return string.Format("[{0}] Unique: {1}", Column, Unique); - } - - public string IndexName - { - get - { - return string.Format("IX_{0}_{1}", Table, Column); - } - } - - public string CreateSql(string tableName) - { - if (Unique) - { - return String.Format(@"CREATE UNIQUE INDEX ""{2}"" ON ""{0}"" (""{1}"" ASC)", tableName, Column, IndexName); - } - - return String.Format(@"CREATE INDEX ""{2}"" ON ""{0}"" (""{1}"" ASC)", tableName, Column, IndexName); - } - } -} \ No newline at end of file diff --git a/src/NzbDrone.Core/Datastore/Migration/Framework/SQLiteMigrationHelper.cs b/src/NzbDrone.Core/Datastore/Migration/Framework/SQLiteMigrationHelper.cs deleted file mode 100644 index 8258d283c..000000000 --- a/src/NzbDrone.Core/Datastore/Migration/Framework/SQLiteMigrationHelper.cs +++ /dev/null @@ -1,226 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Data.SQLite; -using System.Linq; -using System.Text.RegularExpressions; -using NLog; -using NzbDrone.Common.Exceptions; - -namespace NzbDrone.Core.Datastore.Migration.Framework -{ - public interface ISqLiteMigrationHelper - { - Dictionary GetColumns(string tableName); - void CreateTable(string tableName, IEnumerable values, IEnumerable indexes); - void CopyData(string sourceTable, string destinationTable, IEnumerable columns); - void DropTable(string tableName); - void RenameTable(string tableName, string newName); - IEnumerable>> GetDuplicates(string tableName, string columnName); - SQLiteTransaction BeginTransaction(); - List GetIndexes(string tableName); - int ExecuteScalar(string command, params string[] args); - void ExecuteNonQuery(string command, params string[] args); - } - - public class SqLiteMigrationHelper : ISqLiteMigrationHelper - { - private readonly SQLiteConnection _connection; - - private static readonly Regex SchemaRegex = new Regex(@"[`'\""\[](?\w+)[`'\""\]]\s(?[\w-\s]+)", - RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.Multiline); - - private static readonly Regex IndexRegex = new Regex(@"\((?:""|')(?.*)(?:""|')\s(?ASC|DESC)\)$", - RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.Multiline); - - public SqLiteMigrationHelper(IConnectionStringFactory connectionStringFactory, Logger logger) - { - try - { - _connection = new SQLiteConnection(connectionStringFactory.MainDbConnectionString); - _connection.Open(); - } - catch (Exception e) - { - logger.ErrorException("Couldn't open database " + connectionStringFactory.MainDbConnectionString, e); - throw; - } - - } - - private string GetOriginalSql(string tableName) - { - var command = - new SQLiteCommand(string.Format("SELECT sql FROM sqlite_master WHERE type='table' AND name ='{0}'", - tableName)); - - command.Connection = _connection; - - var sql = (string)command.ExecuteScalar(); - - if (string.IsNullOrWhiteSpace(sql)) - { - throw new TableNotFoundException(tableName); - } - - return sql; - } - - public Dictionary GetColumns(string tableName) - { - var originalSql = GetOriginalSql(tableName); - - var matches = SchemaRegex.Matches(originalSql); - - return matches.Cast().ToDictionary( - match => match.Groups["name"].Value.Trim(), - match => new SQLiteColumn - { - Name = match.Groups["name"].Value.Trim(), - Schema = match.Groups["schema"].Value.Trim() - }); - } - - - private static IEnumerable ReadArray(SQLiteDataReader reader) - { - while (reader.Read()) - { - yield return (T)Convert.ChangeType(reader[0], typeof(T)); - } - } - - public List GetIndexes(string tableName) - { - var command = new SQLiteCommand(string.Format("SELECT sql FROM sqlite_master WHERE type='index' AND tbl_name ='{0}'", tableName)); - command.Connection = _connection; - - var reader = command.ExecuteReader(); - var sqls = ReadArray(reader).ToList(); - var indexes = new List(); - - foreach (var indexSql in sqls) - { - var newIndex = new SQLiteIndex(); - var matches = IndexRegex.Match(indexSql); - - if (!matches.Success) continue;; - - newIndex.Column = matches.Groups["col"].Value; - newIndex.Unique = indexSql.Contains("UNIQUE"); - newIndex.Table = tableName; - - indexes.Add(newIndex); - } - - return indexes; - } - - public void CreateTable(string tableName, IEnumerable values, IEnumerable indexes) - { - var columns = String.Join(",", values.Select(c => c.ToString())); - - ExecuteNonQuery("CREATE TABLE [{0}] ({1})", tableName, columns); - - foreach (var index in indexes) - { - ExecuteNonQuery("DROP INDEX IF EXISTS {0}", index.IndexName); - ExecuteNonQuery(index.CreateSql(tableName)); - } - } - - public void CopyData(string sourceTable, string destinationTable, IEnumerable columns) - { - var originalCount = GetRowCount(sourceTable); - - var columnsToTransfer = String.Join(",", columns.Select(c => c.Name)); - - var transferCommand = BuildCommand("INSERT INTO {0} SELECT {1} FROM {2};", destinationTable, columnsToTransfer, sourceTable); - - transferCommand.ExecuteNonQuery(); - - var transferredRows = GetRowCount(destinationTable); - - - if (transferredRows != originalCount) - { - throw new ApplicationException(string.Format("Expected {0} rows to be copied from [{1}] to [{2}]. But only copied {3}", originalCount, sourceTable, destinationTable, transferredRows)); - } - } - - public void DropTable(string tableName) - { - var dropCommand = BuildCommand("DROP TABLE {0};", tableName); - dropCommand.ExecuteNonQuery(); - } - - public void RenameTable(string tableName, string newName) - { - var renameCommand = BuildCommand("ALTER TABLE {0} RENAME TO {1};", tableName, newName); - renameCommand.ExecuteNonQuery(); - } - - public IEnumerable>> GetDuplicates(string tableName, string columnName) - { - var getDuplicates = BuildCommand("select id, {0} from {1}", columnName, tableName); - - var result = new List>(); - - using (var reader = getDuplicates.ExecuteReader()) - { - while (reader.Read()) - { - result.Add(new KeyValuePair(reader.GetInt32(0), (T)Convert.ChangeType(reader[1], typeof(T)))); - } - } - - return result.GroupBy(c => c.Value).Where(g => g.Count() > 1); - } - - public int GetRowCount(string tableName) - { - var countCommand = BuildCommand("SELECT COUNT(*) FROM {0};", tableName); - return Convert.ToInt32(countCommand.ExecuteScalar()); - } - - public SQLiteTransaction BeginTransaction() - { - return _connection.BeginTransaction(); - } - - private SQLiteCommand BuildCommand(string format, params string[] args) - { - var command = new SQLiteCommand(string.Format(format, args)); - command.Connection = _connection; - return command; - } - - public void ExecuteNonQuery(string command, params string[] args) - { - var sqLiteCommand = new SQLiteCommand(string.Format(command, args)) - { - Connection = _connection - }; - - sqLiteCommand.ExecuteNonQuery(); - } - - public int ExecuteScalar(string command, params string[] args) - { - var sqLiteCommand = new SQLiteCommand(string.Format(command, args)) - { - Connection = _connection - }; - - return (int)sqLiteCommand.ExecuteScalar(); - } - - private class TableNotFoundException : NzbDroneException - { - public TableNotFoundException(string tableName) - : base("Table [{0}] not found", tableName) - { - - } - } - } -} \ No newline at end of file diff --git a/src/NzbDrone.Core/Datastore/Migration/Framework/SqliteAlter.cs b/src/NzbDrone.Core/Datastore/Migration/Framework/SqliteAlter.cs deleted file mode 100644 index 848470de3..000000000 --- a/src/NzbDrone.Core/Datastore/Migration/Framework/SqliteAlter.cs +++ /dev/null @@ -1,103 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; - -namespace NzbDrone.Core.Datastore.Migration.Framework -{ - public interface ISQLiteAlter - { - void DropColumns(string tableName, IEnumerable columns); - void AddIndexes(string tableName, params SQLiteIndex[] indexes); - void Nullify(string tableName, IEnumerable columns); - } - - public class SQLiteAlter : ISQLiteAlter - { - private readonly ISqLiteMigrationHelper _sqLiteMigrationHelper; - - public SQLiteAlter(ISqLiteMigrationHelper sqLiteMigrationHelper) - { - _sqLiteMigrationHelper = sqLiteMigrationHelper; - } - - public void DropColumns(string tableName, IEnumerable columns) - { - using (var transaction = _sqLiteMigrationHelper.BeginTransaction()) - { - var originalColumns = _sqLiteMigrationHelper.GetColumns(tableName); - var originalIndexes = _sqLiteMigrationHelper.GetIndexes(tableName); - - var newColumns = originalColumns.Where(c => !columns.Contains(c.Key)).Select(c => c.Value).ToList(); - var newIndexes = originalIndexes.Where(c => !columns.Contains(c.Column)); - - CreateTable(tableName, newColumns, newIndexes); - - transaction.Commit(); - } - } - - public void AddIndexes(string tableName, params SQLiteIndex[] indexes) - { - using (var transaction = _sqLiteMigrationHelper.BeginTransaction()) - { - var columns = _sqLiteMigrationHelper.GetColumns(tableName).Select(c => c.Value).ToList(); - var originalIndexes = _sqLiteMigrationHelper.GetIndexes(tableName); - - var newIndexes = originalIndexes.Union(indexes); - - CreateTable(tableName, columns, newIndexes); - - transaction.Commit(); - } - } - - public void Nullify(string tableName, IEnumerable columns) - { - using (var transaction = _sqLiteMigrationHelper.BeginTransaction()) - { - var originalColumns = _sqLiteMigrationHelper.GetColumns(tableName); - var indexes = _sqLiteMigrationHelper.GetIndexes(tableName); - - var newColumns = originalColumns.Select(c => - { - if (!columns.Contains(c.Key)) - { - return c.Value; - } - - if (!c.Value.Schema.Contains("NOT NULL") && c.Value.Schema.Contains("NULL")) - { - return c.Value; - } - - if (c.Value.Schema.Contains("NOT NULL")) - { - c.Value.Schema = c.Value.Schema.Replace("NOT NULL", "NULL"); - return c.Value; - } - - c.Value.Schema += " NULL"; - - return c.Value; - }).ToList(); - - CreateTable(tableName, newColumns, indexes); - - transaction.Commit(); - } - } - - private void CreateTable(string tableName, List newColumns, IEnumerable newIndexes) - { - var tempTableName = tableName + "_temp"; - - _sqLiteMigrationHelper.CreateTable(tempTableName, newColumns, newIndexes); - - _sqLiteMigrationHelper.CopyData(tableName, tempTableName, newColumns); - - _sqLiteMigrationHelper.DropTable(tableName); - - _sqLiteMigrationHelper.RenameTable(tempTableName, tableName); - } - } -} \ No newline at end of file diff --git a/src/NzbDrone.Core/Datastore/Migration/Framework/SqliteSchemaDumper.cs b/src/NzbDrone.Core/Datastore/Migration/Framework/SqliteSchemaDumper.cs new file mode 100644 index 000000000..0fdeba8d9 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/Framework/SqliteSchemaDumper.cs @@ -0,0 +1,265 @@ +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Data; +using System.Linq; +using FluentMigrator.Builders.Execute; +using FluentMigrator.Model; +using FluentMigrator.Runner; +using System; +using FluentMigrator.Runner.Processors.SQLite; +using System.Text; + +namespace NzbDrone.Core.Datastore.Migration.Framework +{ + // Modeled after the FluentMigrator SchemaDumper class. + // The original implementation had bad support for escaped identifiers, amongst other things. + public class SqliteSchemaDumper + { + public SqliteSchemaDumper(SqliteProcessor processor, IAnnouncer announcer) + { + Announcer = announcer; + Processor = processor; + } + + public virtual IAnnouncer Announcer { get; set; } + public SqliteProcessor Processor { get; set; } + + protected internal virtual TableDefinition ReadTableSchema(String sqlSchema) + { + var reader = new SqliteSyntaxReader(sqlSchema); + + var result = ParseCreateTableStatement(reader); + + return result; + } + + protected internal virtual IndexDefinition ReadIndexSchema(String sqlSchema) + { + var reader = new SqliteSyntaxReader(sqlSchema); + + var result = ParseCreateIndexStatement(reader); + + return result; + } + + protected virtual TableDefinition ParseCreateTableStatement(SqliteSyntaxReader reader) + { + var table = new TableDefinition(); + + while (reader.Read() != SqliteSyntaxReader.TokenType.StringToken || reader.ValueToUpper != "TABLE") ; + + if (reader.Read() == SqliteSyntaxReader.TokenType.StringToken && reader.ValueToUpper == "IF") + { + reader.Read(); // NOT + reader.Read(); // EXISTS + } + else + { + reader.Rollback(); + } + + table.Name = ParseIdentifier(reader); + + // Find Column List + reader.SkipTillToken(SqliteSyntaxReader.TokenType.ListStart); + + // Split the list. + var list = reader.ReadList(); + + foreach (var columnReader in list) + { + columnReader.SkipWhitespace(); + + if (columnReader.Read() == SqliteSyntaxReader.TokenType.StringToken) + { + if (columnReader.ValueToUpper == "CONSTRAINT" || + columnReader.ValueToUpper == "PRIMARY" || columnReader.ValueToUpper == "UNIQUE" || + columnReader.ValueToUpper == "CHECK" || columnReader.ValueToUpper == "FOREIGN") + { + continue; + } + } + + columnReader.Rollback(); + + var column = ParseColumnDefinition(columnReader); + column.TableName = table.Name; + table.Columns.Add(column); + } + + return table; + } + + protected virtual ColumnDefinition ParseColumnDefinition(SqliteSyntaxReader reader) + { + var column = new ColumnDefinition(); + + column.Name = ParseIdentifier(reader); + + reader.TrimBuffer(); + + reader.Read(); + if (reader.Type != SqliteSyntaxReader.TokenType.End) + { + column.Type = GetDbType(reader.Value); + + var upper = reader.Buffer.ToUpperInvariant(); + column.IsPrimaryKey = upper.Contains("PRIMARY KEY"); + column.IsIdentity = upper.Contains("AUTOINCREMENT"); + column.IsNullable = !upper.Contains("NOT NULL") && !upper.Contains("PRIMARY KEY"); + column.IsUnique = upper.Contains("UNIQUE") || upper.Contains("PRIMARY KEY"); + } + + return column; + } + + protected virtual IndexDefinition ParseCreateIndexStatement(SqliteSyntaxReader reader) + { + var index = new IndexDefinition(); + + reader.Read(); + + reader.Read(); + index.IsUnique = reader.ValueToUpper == "UNIQUE"; + + while (reader.ValueToUpper != "INDEX") reader.Read(); + + if (reader.Read() == SqliteSyntaxReader.TokenType.StringToken && reader.ValueToUpper == "IF") + { + reader.Read(); // NOT + reader.Read(); // EXISTS + } + else + { + reader.Rollback(); + } + + index.Name = ParseIdentifier(reader); + + reader.Read(); // ON + + index.TableName = ParseIdentifier(reader); + + // Find Column List + reader.SkipTillToken(SqliteSyntaxReader.TokenType.ListStart); + + // Split the list. + var list = reader.ReadList(); + + foreach (var columnReader in list) + { + var column = new IndexColumnDefinition(); + column.Name = ParseIdentifier(columnReader); + + while (columnReader.Read() == SqliteSyntaxReader.TokenType.StringToken) + { + if (columnReader.ValueToUpper == "COLLATE") + { + columnReader.Read(); // Skip Collation name + } + else if (columnReader.ValueToUpper == "DESC") + { + column.Direction = Direction.Descending; + } + } + + index.Columns.Add(column); + } + + return index; + } + + protected virtual String ParseIdentifier(SqliteSyntaxReader reader) + { + reader.Read(); + + if (reader.Type != SqliteSyntaxReader.TokenType.Identifier && + reader.Type != SqliteSyntaxReader.TokenType.StringToken) + { + throw reader.CreateSyntaxException("Expected Identifier but found {0}", reader.Type); + } + + return reader.Value; + } + + #region ISchemaDumper Members + + public virtual IList ReadDbSchema() + { + IList tables = ReadTables(); + foreach (var table in tables) + { + table.Indexes = ReadIndexes(table.SchemaName, table.Name); + //table.ForeignKeys = ReadForeignKeys(table.SchemaName, table.Name); + } + + return tables; + } + + #endregion + + protected virtual DataSet Read(string template, params object[] args) + { + return Processor.Read(template, args); + } + + protected virtual IList ReadTables() + { + const string sqlCommand = @"SELECT name, sql FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name;"; + var dtTable = Read(sqlCommand).Tables[0]; + + var tableDefinitionList = new List(); + + foreach (DataRow dr in dtTable.Rows) + { + var sql = dr["sql"].ToString(); + var table = ReadTableSchema(sql); + + tableDefinitionList.Add(table); + } + return tableDefinitionList; + } + + /// + /// Get DbType from string type definition + /// + /// + /// + public static DbType? GetDbType(string typeNum) + { + switch (typeNum.ToUpper()) + { + case "BLOB": + return DbType.Binary; + case "INTEGER": + return DbType.Int64; + case "NUMERIC": + return DbType.Double; + case "TEXT": + return DbType.String; + case "DATETIME": + return DbType.DateTime; + case "UNIQUEIDENTIFIER": + return DbType.Guid; + default: + return null; + } + } + + protected virtual IList ReadIndexes(string schemaName, string tableName) + { + var sqlCommand = string.Format(@"SELECT type, name, sql FROM sqlite_master WHERE tbl_name = '{0}' AND type = 'index' AND name NOT LIKE 'sqlite_auto%';", tableName); + DataTable table = Read(sqlCommand).Tables[0]; + + IList indexes = new List(); + + foreach (DataRow dr in table.Rows) + { + var sql = dr["sql"].ToString(); + var index = ReadIndexSchema(sql); + indexes.Add(index); + } + return indexes; + } + } +} \ No newline at end of file diff --git a/src/NzbDrone.Core/Datastore/Migration/Framework/SqliteSyntaxReader.cs b/src/NzbDrone.Core/Datastore/Migration/Framework/SqliteSyntaxReader.cs new file mode 100644 index 000000000..89bd097a0 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/Framework/SqliteSyntaxReader.cs @@ -0,0 +1,257 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Text; + +namespace NzbDrone.Core.Datastore.Migration.Framework +{ + + public class SqliteSyntaxReader + { + public String Buffer { get; private set; } + public Int32 Index { get; private set; } + + private Int32 _previousIndex; + + public TokenType Type { get; private set; } + public String Value { get; private set; } + + public String ValueToUpper + { + get { return Value.ToUpperInvariant(); } + } + + public Boolean IsEndOfFile + { + get { return Index >= Buffer.Length; } + } + + public enum TokenType + { + Start, + Whitespace, + End, + ListStart, + ListSeparator, + ListEnd, + Identifier, + StringToken, + StringLiteral, + UnknownSymbol + } + + public SqliteSyntaxReader(String sql) + { + Buffer = sql; + } + + public void TrimBuffer() + { + Buffer = Buffer.Substring(Index); + Index = 0; + _previousIndex = 0; + } + + public void SkipWhitespace() + { + while (!IsEndOfFile && char.IsWhiteSpace(Buffer[Index])) Index++; + } + + public void SkipTillToken(TokenType tokenType) + { + if (IsEndOfFile) + return; + + while (Read() != tokenType) + { + if (Type == TokenType.ListStart) + SkipTillToken(TokenType.ListEnd); + } + } + + public void Rollback() + { + Index = _previousIndex; + Type = TokenType.Whitespace; + } + + public TokenType Read() + { + if (!IsEndOfFile && char.IsWhiteSpace(Buffer[Index])) + { + Type = TokenType.Whitespace; + SkipWhitespace(); + } + + _previousIndex = Index; + + if (IsEndOfFile) + { + Type = TokenType.End; + return Type; + } + + if (Buffer[Index] == '(') + { + Type = TokenType.ListStart; + Value = null; + Index++; + return Type; + } + + if (Buffer[Index] == ',') + { + Type = TokenType.ListSeparator; + Value = null; + Index++; + return Type; + } + + if (Buffer[Index] == ')') + { + Type = TokenType.ListEnd; + Value = null; + Index++; + return Type; + } + + if (Buffer[Index] == '\'') + { + Type = TokenType.StringLiteral; + Value = ReadEscapedString('\''); + return Type; + } + + if (Buffer[Index] == '\"') + { + Type = TokenType.Identifier; + Value = ReadEscapedString('\"'); + return Type; + } + + if (Buffer[Index] == '`') + { + Type = TokenType.Identifier; + Value = ReadEscapedString('`'); + return Type; + } + + if (Buffer[Index] == '[') + { + Type = TokenType.Identifier; + Value = ReadTerminatedString(']'); + return Type; + } + + if (Type == TokenType.UnknownSymbol) + { + Value = Buffer[Index].ToString(); + Index++; + return Type; + } + + if (char.IsLetter(Buffer[Index])) + { + var start = Index; + var end = start + 1; + while (end < Buffer.Length && (char.IsLetter(Buffer[end]) || Buffer[end] == '_')) end++; + if (end >= Buffer.Length || Buffer[end] == ',' || Buffer[end] == ')' || char.IsWhiteSpace(Buffer[end])) + { + Index = end; + } + else if (Type == TokenType.UnknownSymbol) + { + Value = Buffer[Index].ToString(); + Index++; + return Type; + } + else + { + throw CreateSyntaxException("Unexpected sequence."); + } + Type = TokenType.StringToken; + Value = Buffer.Substring(start, end - start); + return Type; + } + + Type = TokenType.UnknownSymbol; + Value = Buffer[Index].ToString(); + Index++; + return Type; + } + + public List ReadList() + { + if (Type != TokenType.ListStart) + { + throw new InvalidOperationException(); + } + + var result = new List(); + + var start = Index; + while (Read() != TokenType.ListEnd) + { + if (Type == TokenType.End) + { + throw CreateSyntaxException("Expected ListEnd first"); + } + if (Type == TokenType.ListStart) + { + SkipTillToken(TokenType.ListEnd); + } + else if (Type == TokenType.ListSeparator) + { + result.Add(new SqliteSyntaxReader(Buffer.Substring(start, Index - start - 1))); + start = Index; + } + } + + if (Index >= start + 1) + { + result.Add(new SqliteSyntaxReader(Buffer.Substring(start, Index - start - 1))); + } + + return result; + } + + protected String ReadTerminatedString(Char terminator) + { + var start = Index + 1; + var end = Buffer.IndexOf(terminator, Index); + + if (end == -1) throw new SyntaxErrorException(); + + Index = end + 1; + return Buffer.Substring(start, end - start); + } + + protected String ReadEscapedString(Char escape) + { + var identifier = new StringBuilder(); + + while (true) + { + var start = Index + 1; + var end = Buffer.IndexOf(escape, start); + + if (end == -1) throw new SyntaxErrorException(); + + Index = end + 1; + identifier.Append(Buffer.Substring(start, end - start)); + + if (Buffer[Index] != escape) break; + + identifier.Append(escape); + } + + return identifier.ToString(); + } + + public SyntaxErrorException CreateSyntaxException(String message, params object[] args) + { + return new SyntaxErrorException(String.Format("{0}. Syntax Error near: {1}", String.Format(message, args), Buffer.Substring(_previousIndex))); + } + } +} diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 4e112cbb6..c764fd0ac 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -230,10 +230,10 @@ - - - - + + + +