re-adding indexes for series/episodes.

This commit is contained in:
Keivan Beigi 2013-09-04 17:06:24 -07:00 committed by kay.one
parent c90a96c2e0
commit 9390a00f80
19 changed files with 308 additions and 78 deletions

View File

@ -7,17 +7,17 @@ using NzbDrone.Core.Test.Framework;
using NzbDrone.Core.Tv;
using System.Linq;
namespace NzbDrone.Core.Test.Datastore
namespace NzbDrone.Core.Test.Datastore.SQLiteMigrationHelperTests
{
[TestFixture]
public class SQLiteMigrationHelperFixture : DbTest
public class AlterFixture : DbTest
{
private SQLiteMigrationHelper _subject;
private SqLiteMigrationHelper _subject;
[SetUp]
public void SetUp()
{
_subject = Mocker.Resolve<SQLiteMigrationHelper>();
_subject = Mocker.Resolve<SqLiteMigrationHelper>();
}
[Test]
@ -37,7 +37,7 @@ namespace NzbDrone.Core.Test.Datastore
var columns = _subject.GetColumns("Series");
columns.Remove("Title");
_subject.CreateTable("Series_New", columns.Values, new List<SQLiteMigrationHelper.SQLiteIndex>());
_subject.CreateTable("Series_New", columns.Values, new List<SQLiteIndex>());
var newColumns = _subject.GetColumns("Series_New");
@ -106,5 +106,22 @@ namespace NzbDrone.Core.Test.Datastore
newIndexes.Should().HaveSameCount(indexes);
newIndexes.Select(c=>c.Column).Should().BeEquivalentTo(indexes.Select(c=>c.Column));
}
[Test]
public void should_be_able_to_create_table_with_new_indexes()
{
var columns = _subject.GetColumns("Series");
columns.Remove("Title");
_subject.CreateTable("Series_New", columns.Values, new List<SQLiteIndex>{new SQLiteIndex{Column = "AirTime", Table = "Series_New", Unique = true}});
var newColumns = _subject.GetColumns("Series_New");
var newIndexes = _subject.GetIndexes("Series_New");
newColumns.Values.Should().HaveSameCount(columns.Values);
newIndexes.Should().Contain(i=>i.Column == "AirTime");
}
}
}

View File

@ -0,0 +1,42 @@
using System.Collections.Generic;
using System.Linq;
using FizzWare.NBuilder;
using FluentAssertions;
using NUnit.Framework;
using NzbDrone.Core.Datastore.Migration.Framework;
using NzbDrone.Core.Test.Framework;
using NzbDrone.Core.Tv;
namespace NzbDrone.Core.Test.Datastore.SQLiteMigrationHelperTests
{
[TestFixture]
public class DuplicateFixture : DbTest
{
private SqLiteMigrationHelper _subject;
[SetUp]
public void SetUp()
{
_subject = Mocker.Resolve<SqLiteMigrationHelper>();
}
[Test]
public void get_duplicates()
{
var series = Builder<Series>.CreateListOfSize(10)
.Random(3)
.With(c => c.QualityProfileId = 100)
.BuildListOfNew();
Db.InsertMany(series);
var duplicates = _subject.GetDuplicates<int>("series", "QualityProfileId").ToList();
duplicates.Should().HaveCount(1);
duplicates.First().Should().HaveCount(3);
}
}
}

View File

@ -93,7 +93,7 @@ namespace NzbDrone.Core.Test.Framework
Mocker.SetConstant<IAnnouncer>(Mocker.Resolve<MigrationLogger>());
Mocker.SetConstant<IConnectionStringFactory>(Mocker.Resolve<ConnectionStringFactory>());
Mocker.SetConstant<ISQLiteMigrationHelper>(Mocker.Resolve<SQLiteMigrationHelper>());
Mocker.SetConstant<ISqLiteMigrationHelper>(Mocker.Resolve<SqLiteMigrationHelper>());
Mocker.SetConstant<ISQLiteAlter>(Mocker.Resolve<SQLiteAlter>());
Mocker.SetConstant<IMigrationController>(Mocker.Resolve<MigrationController>());

View File

@ -108,7 +108,8 @@
<Compile Include="Datastore\PagingSpecExtenstionsTests\ToSortDirectionFixture.cs" />
<Compile Include="Datastore\PagingSpecExtenstionsTests\PagingOffsetFixture.cs" />
<Compile Include="Datastore\ReflectionStrategyFixture\Benchmarks.cs" />
<Compile Include="Datastore\SQLiteMigrationHelperFixture.cs" />
<Compile Include="Datastore\SQLiteMigrationHelperTests\AlterFixture.cs" />
<Compile Include="Datastore\SQLiteMigrationHelperTests\DuplicateFixture.cs" />
<Compile Include="DecisionEngineTests\NotRestrictedReleaseSpecificationFixture.cs" />
<Compile Include="Download\DownloadApprovedReportsTests\DownloadApprovedFixture.cs" />
<Compile Include="Download\DownloadApprovedReportsTests\GetQualifiedReportsFixture.cs" />

View File

@ -8,8 +8,8 @@ namespace NzbDrone.Core.Datastore.Migration
{
protected override void MainDbUpgrade()
{
SQLiteAlter.DropColumns("Series", new[] { "BacklogSetting" });
SQLiteAlter.DropColumns("NamingConfig", new[] { "UseSceneName" });
SqLiteAlter.DropColumns("Series", new[] { "BacklogSetting" });
SqLiteAlter.DropColumns("NamingConfig", new[] { "UseSceneName" });
}
}
}

View File

@ -8,7 +8,7 @@ namespace NzbDrone.Core.Datastore.Migration
{
protected override void MainDbUpgrade()
{
SQLiteAlter.DropColumns("NamingConfig", new[] { "SeasonFolderFormat" });
SqLiteAlter.DropColumns("NamingConfig", new[] { "SeasonFolderFormat" });
Execute.Sql("UPDATE NamingConfig SET RenameEpisodes = 1 WHERE RenameEpisodes = -1");
Execute.Sql("UPDATE NamingConfig SET RenameEpisodes = 0 WHERE RenameEpisodes = -2");

View File

@ -8,8 +8,8 @@ namespace NzbDrone.Core.Datastore.Migration
{
protected override void MainDbUpgrade()
{
SQLiteAlter.DropColumns("Episodes", new[] { "Ignored" });
SQLiteAlter.DropColumns("Seasons", new[] { "Ignored" });
SqLiteAlter.DropColumns("Episodes", new[] { "Ignored" });
SqLiteAlter.DropColumns("Seasons", new[] { "Ignored" });
}
}
}

View File

@ -8,7 +8,7 @@ namespace NzbDrone.Core.Datastore.Migration
{
protected override void MainDbUpgrade()
{
SQLiteAlter.DropColumns("Series", new[] { "CustomStartDate" });
SqLiteAlter.DropColumns("Series", new[] { "CustomStartDate" });
}
}
}

View File

@ -8,7 +8,7 @@ namespace NzbDrone.Core.Datastore.Migration
{
protected override void MainDbUpgrade()
{
SQLiteAlter.DropColumns("Episodes", new []{ "AirDate" });
SqLiteAlter.DropColumns("Episodes", new []{ "AirDate" });
}
}
}

View File

@ -0,0 +1,60 @@
using FluentMigrator;
using NzbDrone.Core.Datastore.Migration.Framework;
using System.Linq;
namespace NzbDrone.Core.Datastore.Migration
{
[Migration(18)]
public class remove_duplicates : NzbDroneMigrationBase
{
protected override void MainDbUpgrade()
{
using (var transaction = MigrationHelper.BeginTransaction())
{
RemoveDuplicateSeries("TvdbId");
RemoveDuplicateSeries("TitleSlug");
var duplicatedEpisodes = MigrationHelper.GetDuplicates<int>("Episodes", "TvDbEpisodeId");
foreach (var duplicate in duplicatedEpisodes)
{
foreach (var episodeId in duplicate.OrderBy(c => c.Key).Skip(1).Select(c => c.Key))
{
RemoveEpisodeRows(episodeId);
}
}
transaction.Commit();
}
}
private void RemoveDuplicateSeries(string field)
{
var duplicatedSeries = MigrationHelper.GetDuplicates<int>("Series", field);
foreach (var duplicate in duplicatedSeries)
{
foreach (var seriesId in duplicate.OrderBy(c => c.Key).Skip(1).Select(c => c.Key))
{
RemoveSeriesRows(seriesId);
}
}
}
private void RemoveSeriesRows(int seriesId)
{
MigrationHelper.ExecuteNonQuery("DELETE FROM Series WHERE Id = {0}", seriesId.ToString());
MigrationHelper.ExecuteNonQuery("DELETE FROM Episodes WHERE SeriesId = {0}", seriesId.ToString());
MigrationHelper.ExecuteNonQuery("DELETE FROM Seasons WHERE SeriesId = {0}", seriesId.ToString());
MigrationHelper.ExecuteNonQuery("DELETE FROM History WHERE SeriesId = {0}", seriesId.ToString());
MigrationHelper.ExecuteNonQuery("DELETE FROM EpisodeFiles WHERE SeriesId = {0}", seriesId.ToString());
}
private void RemoveEpisodeRows(int episodeId)
{
MigrationHelper.ExecuteNonQuery("DELETE FROM Episodes WHERE Id = {0}", episodeId.ToString());
MigrationHelper.ExecuteNonQuery("DELETE FROM History WHERE EpisodeId = {0}", episodeId.ToString());
}
}
}

View File

@ -0,0 +1,20 @@
using FluentMigrator;
using NzbDrone.Core.Datastore.Migration.Framework;
namespace NzbDrone.Core.Datastore.Migration
{
[Migration(19)]
public class restore_unique_constraints : NzbDroneMigrationBase
{
protected override void MainDbUpgrade()
{
SqLiteAlter.AddIndexes("Series",
new SQLiteIndex { Column = "TvdbId", Table = "Series", Unique = true },
new SQLiteIndex { Column = "TitleSlug", Table = "Series", Unique = true });
SqLiteAlter.AddIndexes("Episodes",
new SQLiteIndex { Column = "TvDbEpisodeId", Table = "Episodes", Unique = true });
}
}
}

View File

@ -4,5 +4,6 @@
{
public MigrationType MigrationType { get; set; }
public ISQLiteAlter SQLiteAlter { get; set; }
public ISqLiteMigrationHelper MigrationHelper { get; set; }
}
}

View File

@ -15,13 +15,15 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
{
private readonly IAnnouncer _announcer;
private readonly ISQLiteAlter _sqLiteAlter;
private readonly ISqLiteMigrationHelper _migrationHelper;
private static readonly HashSet<string> MigrationCache = new HashSet<string>();
public MigrationController(IAnnouncer announcer, ISQLiteAlter sqLiteAlter)
public MigrationController(IAnnouncer announcer, ISQLiteAlter sqLiteAlter, ISqLiteMigrationHelper migrationHelper)
{
_announcer = announcer;
_sqLiteAlter = sqLiteAlter;
_migrationHelper = migrationHelper;
}
public void MigrateToLatest(string connectionString, MigrationType migrationType)
@ -40,7 +42,8 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
ApplicationContext = new MigrationContext
{
MigrationType = migrationType,
SQLiteAlter = _sqLiteAlter
SQLiteAlter = _sqLiteAlter,
MigrationHelper = _migrationHelper,
}
};

View File

@ -1,9 +1,18 @@
using System;
using NLog;
using NzbDrone.Common.Instrumentation;
namespace NzbDrone.Core.Datastore.Migration.Framework
{
public abstract class NzbDroneMigrationBase : FluentMigrator.Migration
{
private Logger _logger;
protected NzbDroneMigrationBase()
{
_logger = NzbDroneLogger.GetLogger();
}
protected virtual void MainDbUpgrade()
{
}
@ -16,7 +25,8 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
{
var context = (MigrationContext)ApplicationContext;
SQLiteAlter = context.SQLiteAlter;
SqLiteAlter = context.SQLiteAlter;
MigrationHelper = context.MigrationHelper;
switch (context.MigrationType)
{
@ -33,7 +43,8 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
}
}
protected ISQLiteAlter SQLiteAlter { get; private set; }
protected ISQLiteAlter SqLiteAlter { get; private set; }
protected ISqLiteMigrationHelper MigrationHelper { get; private set; }
public override void Down()
{

View File

@ -0,0 +1,13 @@
namespace NzbDrone.Core.Datastore.Migration.Framework
{
public class SQLiteColumn
{
public string Name { get; set; }
public string Schema { get; set; }
public override string ToString()
{
return string.Format("[{0}] {1}", Name, Schema);
}
}
}

View File

@ -0,0 +1,39 @@
using System;
namespace NzbDrone.Core.Datastore.Migration.Framework
{
public class SQLiteIndex : IEquatable<SQLiteIndex>
{
public string Column { get; set; }
public string Table { get; set; }
public bool Unique { get; set; }
public bool Equals(SQLiteIndex other)
{
return IndexName == other.IndexName;
}
public override int GetHashCode()
{
return IndexName.GetHashCode();
}
public override string ToString()
{
return string.Format("[{0}] Unique: {1}", Column, Unique);
}
public string IndexName
{
get
{
return string.Format("IX_{0}_{1}", Table, Column);
}
}
public string CreateSql(string tableName)
{
return string.Format(@"CREATE UNIQUE INDEX ""{2}"" ON ""{0}"" (""{1}"" ASC)", tableName, Column, IndexName);
}
}
}

View File

@ -7,18 +7,21 @@ using NLog;
namespace NzbDrone.Core.Datastore.Migration.Framework
{
public interface ISQLiteMigrationHelper
public interface ISqLiteMigrationHelper
{
Dictionary<String, SQLiteMigrationHelper.SQLiteColumn> GetColumns(string tableName);
void CreateTable(string tableName, IEnumerable<SQLiteMigrationHelper.SQLiteColumn> values, IEnumerable<SQLiteMigrationHelper.SQLiteIndex> indexes);
void CopyData(string sourceTable, string destinationTable, IEnumerable<SQLiteMigrationHelper.SQLiteColumn> columns);
Dictionary<String, SQLiteColumn> GetColumns(string tableName);
void CreateTable(string tableName, IEnumerable<SQLiteColumn> values, IEnumerable<SQLiteIndex> indexes);
void CopyData(string sourceTable, string destinationTable, IEnumerable<SQLiteColumn> columns);
void DropTable(string tableName);
void RenameTable(string tableName, string newName);
IEnumerable<IGrouping<T, KeyValuePair<int, T>>> GetDuplicates<T>(string tableName, string columnName);
SQLiteTransaction BeginTransaction();
List<SQLiteMigrationHelper.SQLiteIndex> GetIndexes(string tableName);
List<SQLiteIndex> GetIndexes(string tableName);
int ExecuteScalar(string command, params string[] args);
void ExecuteNonQuery(string command, params string[] args);
}
public class SQLiteMigrationHelper : ISQLiteMigrationHelper
public class SqLiteMigrationHelper : ISqLiteMigrationHelper
{
private readonly SQLiteConnection _connection;
@ -28,7 +31,7 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
private static readonly Regex IndexRegex = new Regex(@"\(""(?<col>.*)""\s(?<direction>ASC|DESC)\)$",
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.Multiline);
public SQLiteMigrationHelper(IConnectionStringFactory connectionStringFactory, Logger logger)
public SqLiteMigrationHelper(IConnectionStringFactory connectionStringFactory, Logger logger)
{
try
{
@ -69,18 +72,22 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
});
}
private static IEnumerable<T> ReadArray<T>(SQLiteDataReader reader)
{
while (reader.Read())
{
yield return (T)Convert.ChangeType(reader[0], typeof(T));
}
}
public List<SQLiteIndex> GetIndexes(string tableName)
{
var command = new SQLiteCommand(string.Format("SELECT sql FROM sqlite_master WHERE type='index' AND tbl_name ='{0}'", tableName));
command.Connection = _connection;
var reader = command.ExecuteReader();
var sqls = new List<string>();
while (reader.Read())
{
sqls.Add(reader[0].ToString());
}
var sqls = ReadArray<string>(reader).ToList();
var indexes = new List<SQLiteIndex>();
@ -108,7 +115,7 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
foreach (var index in indexes)
{
ExecuteNonQuery("DROP INDEX {0}", index.IndexName);
ExecuteNonQuery("DROP INDEX IF EXISTS {0}", index.IndexName);
ExecuteNonQuery(index.CreateSql(tableName));
}
}
@ -146,6 +153,23 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
renameCommand.ExecuteNonQuery();
}
public IEnumerable<IGrouping<T, KeyValuePair<int, T>>> GetDuplicates<T>(string tableName, string columnName)
{
var getDuplicates = BuildCommand("select id, {0} from {1}", columnName, tableName);
var result = new List<KeyValuePair<int, T>>();
using (var reader = getDuplicates.ExecuteReader())
{
while (reader.Read())
{
result.Add(new KeyValuePair<int, T>(reader.GetInt16(0), (T)Convert.ChangeType(reader[1], typeof(T))));
}
}
return result.GroupBy(c => c.Value).Where(g => g.Count() > 1);
}
public int GetRowCount(string tableName)
{
var countCommand = BuildCommand("SELECT COUNT(*) FROM {0};", tableName);
@ -166,8 +190,7 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
}
private void ExecuteNonQuery(string command, params string[] args)
public void ExecuteNonQuery(string command, params string[] args)
{
var sqLiteCommand = new SQLiteCommand(string.Format(command, args))
{
@ -177,43 +200,17 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
sqLiteCommand.ExecuteNonQuery();
}
public class SQLiteColumn
public int ExecuteScalar(string command, params string[] args)
{
public string Name { get; set; }
public string Schema { get; set; }
public override string ToString()
var sqLiteCommand = new SQLiteCommand(string.Format(command, args))
{
return string.Format("[{0}] {1}", Name, Schema);
}
Connection = _connection
};
return (int)sqLiteCommand.ExecuteScalar();
}
public class SQLiteIndex
{
public string Column { get; set; }
public string Table { get; set; }
public bool Unique { get; set; }
public override string ToString()
{
return string.Format("[{0}] Unique: {1}", Column, Unique);
}
public string IndexName
{
get
{
return string.Format("IX_{0}_{1}", Table, Column);
}
}
public string CreateSql(string tableName)
{
return string.Format(@"CREATE UNIQUE INDEX ""{2}"" ON ""{0}"" (""{1}"" ASC)", tableName, Column, IndexName);
}
}
}
}

View File

@ -6,13 +6,14 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
public interface ISQLiteAlter
{
void DropColumns(string tableName, IEnumerable<string> columns);
void AddIndexes(string tableName, params SQLiteIndex[] indexes);
}
public class SQLiteAlter : ISQLiteAlter
{
private readonly ISQLiteMigrationHelper _sqLiteMigrationHelper;
private readonly ISqLiteMigrationHelper _sqLiteMigrationHelper;
public SQLiteAlter(ISQLiteMigrationHelper sqLiteMigrationHelper)
public SQLiteAlter(ISqLiteMigrationHelper sqLiteMigrationHelper)
{
_sqLiteMigrationHelper = sqLiteMigrationHelper;
}
@ -27,18 +28,39 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
var newColumns = originalColumns.Where(c => !columns.Contains(c.Key)).Select(c => c.Value).ToList();
var newIndexes = originalIndexes.Where(c => !columns.Contains(c.Column));
var tempTableName = tableName + "_temp";
_sqLiteMigrationHelper.CreateTable(tempTableName, newColumns, newIndexes);
_sqLiteMigrationHelper.CopyData(tableName, tempTableName, newColumns);
_sqLiteMigrationHelper.DropTable(tableName);
_sqLiteMigrationHelper.RenameTable(tempTableName, tableName);
CreateTable(tableName, newColumns, newIndexes);
transaction.Commit();
}
}
public void AddIndexes(string tableName, params SQLiteIndex[] indexes)
{
using (var transaction = _sqLiteMigrationHelper.BeginTransaction())
{
var columns = _sqLiteMigrationHelper.GetColumns(tableName).Select(c => c.Value).ToList();
var originalIndexes = _sqLiteMigrationHelper.GetIndexes(tableName);
var newIndexes = originalIndexes.Union(indexes);
CreateTable(tableName, columns, newIndexes);
transaction.Commit();
}
}
private void CreateTable(string tableName, List<SQLiteColumn> newColumns, IEnumerable<SQLiteIndex> newIndexes)
{
var tempTableName = tableName + "_temp";
_sqLiteMigrationHelper.CreateTable(tempTableName, newColumns, newIndexes);
_sqLiteMigrationHelper.CopyData(tableName, tempTableName, newColumns);
_sqLiteMigrationHelper.DropTable(tableName);
_sqLiteMigrationHelper.RenameTable(tempTableName, tableName);
}
}
}

View File

@ -160,6 +160,8 @@
<Compile Include="Datastore\Migration\015_add_air_date_as_string.cs" />
<Compile Include="Datastore\Migration\016_updated_imported_history_item.cs" />
<Compile Include="Datastore\Migration\017_reset_scene_names.cs" />
<Compile Include="Datastore\Migration\018_remove_duplicates.cs" />
<Compile Include="Datastore\Migration\019_restore_unique_constraints.cs" />
<Compile Include="Datastore\Migration\Framework\MigrationContext.cs" />
<Compile Include="Datastore\Migration\Framework\MigrationController.cs" />
<Compile Include="Datastore\Migration\Framework\MigrationExtension.cs" />
@ -169,6 +171,8 @@
<Compile Include="Datastore\Migration\Framework\NzbDroneMigrationBase.cs" />
<Compile Include="Datastore\MigrationType.cs" />
<Compile Include="Datastore\Migration\Framework\SQLiteAlter.cs" />
<Compile Include="Datastore\Migration\Framework\SQLiteColumn.cs" />
<Compile Include="Datastore\Migration\Framework\SQLiteIndex.cs" />
<Compile Include="Datastore\Migration\Framework\SQLiteMigrationHelper.cs" />
<Compile Include="Datastore\ModelBase.cs" />
<Compile Include="Datastore\BasicRepository.cs" />