From 8f6e099794e1856f26c3befc00f130ccd46d677d Mon Sep 17 00:00:00 2001
From: Robin Dadswell <19610103+RobinDadswell@users.noreply.github.com>
Date: Wed, 26 Jan 2022 00:08:27 +0000
Subject: [PATCH] New: Postgres Database Support
Co-Authored-By: Qstick <376117+Qstick@users.noreply.github.com>
---
azure-pipelines.yml | 117 ++++++
frontend/src/System/Status/About/About.js | 9 +
src/Lidarr.Api.V1/System/SystemController.cs | 3 +-
.../AutomationTest.cs | 2 +-
.../CleanseLogMessageFixture.cs | 17 +
.../ServiceFactoryFixture.cs | 5 +-
.../Instrumentation/CleanseLogMessage.cs | 1 +
.../Processes/ProcessProvider.cs | 13 +-
.../Datastore/DatabaseFixture.cs | 2 +-
.../Datastore/DatabaseRelationshipFixture.cs | 25 +-
.../Datastore/LazyLoadingFixture.cs | 12 +-
...add_various_qualities_in_profileFixture.cs | 4 +-
.../023_add_release_groups_etcFixture.cs | 26 +-
...30_add_mediafilerepository_mtimeFixture.cs | 14 +-
..._add_artistmetadataid_constraintFixture.cs | 6 +-
...036_add_download_client_priorityFixture.cs | 20 +-
.../049_email_multiple_addressesFixture.cs | 2 +-
.../051_cdh_per_downloadclientFixture.cs | 12 +-
...ture.cs => WhereBuilderPostgresFixture.cs} | 36 +-
.../Datastore/WhereBuilderSqliteFixture.cs | 203 +++++++++
src/NzbDrone.Core.Test/Framework/DbTest.cs | 56 ++-
.../Framework/DbTestCleanup.cs | 6 +-
.../Framework/TestDatabase.cs | 3 +
.../CleanupOrphanedReleasesFixture.cs | 43 ++
.../AlbumRepositoryFixture.cs | 17 +-
.../ArtistRepositoryFixture.cs | 11 +-
.../RefreshAlbumReleaseServiceFixture.cs | 6 +-
.../MusicTests/RefreshAlbumServiceFixture.cs | 12 +-
.../MusicTests/RefreshArtistServiceFixture.cs | 6 +-
.../ArtistStats/ArtistStatisticsRepository.cs | 50 ++-
.../Blocklisting/BlocklistRepository.cs | 2 +-
.../Configuration/ConfigFileProvider.cs | 19 +-
.../Datastore/BasicRepository.cs | 21 +-
.../Datastore/ConnectionStringFactory.cs | 29 +-
src/NzbDrone.Core/Datastore/Database.cs | 43 +-
src/NzbDrone.Core/Datastore/DbFactory.cs | 17 +-
.../Datastore/Extensions/BuilderExtensions.cs | 46 ++-
src/NzbDrone.Core/Datastore/LogDatabase.cs | 4 +
src/NzbDrone.Core/Datastore/MainDatabase.cs | 4 +
.../Datastore/Migration/001_initial_setup.cs | 4 +-
.../Migration/003_add_medium_support.cs | 2 +-
.../004_add_various_qualities_in_profile.cs | 17 +-
...parate_automatic_and_interactive_search.cs | 2 +-
.../008_change_quality_size_mb_to_kb.cs | 2 +-
.../Migration/012_add_release_status.cs | 2 +-
.../013_album_download_notification.cs | 2 +-
.../014_fix_language_metadata_profiles.cs | 16 +-
.../Datastore/Migration/015_remove_fanzub.cs | 2 +-
.../019_add_ape_quality_in_profiles.cs | 4 +-
.../Migration/023_add_release_groups_etc.cs | 127 +++---
...me_quality_profiles_add_upgrade_allowed.cs | 4 +-
.../028_clean_artistmetadata_table.cs | 62 +--
.../029_health_issue_notification.cs | 10 +-
.../030_add_mediafilerepository_mtime.cs | 58 +--
.../031_add_artistmetadataid_constraint.cs | 10 +-
.../Migration/033_download_propers_config.cs | 6 +-
.../Migration/035_multi_disc_naming_format.cs | 2 +-
.../036_add_download_client_priority.cs | 55 ++-
.../039_add_root_folder_add_defaults.cs | 2 +-
.../Migration/042_remove_album_folders.cs | 4 +-
...045_remove_chown_and_folderchmod_config.cs | 4 +-
.../Migration/047_update_notifiarr.cs | 2 +-
.../Migration/049_email_multiple_addresses.cs | 4 +-
.../Migration/051_cdh_per_downloadclient.cs | 26 +-
.../Migration/052_download_history.cs | 2 +-
.../Migration/057_import_list_search.cs | 2 +-
.../058_import_list_monitor_existing.cs | 2 +-
.../Datastore/Migration/059_indexer_tags.cs | 4 +-
.../Framework/MigrationController.cs | 16 +-
.../Datastore/PostgresOptions.cs | 26 ++
src/NzbDrone.Core/Datastore/SqlBuilder.cs | 8 +
src/NzbDrone.Core/Datastore/TableMapper.cs | 37 +-
src/NzbDrone.Core/Datastore/TableMapping.cs | 16 +-
src/NzbDrone.Core/Datastore/WhereBuilder.cs | 389 +-----------------
.../Datastore/WhereBuilderPostgres.cs | 389 ++++++++++++++++++
.../Datastore/WhereBuilderSqlite.cs | 389 ++++++++++++++++++
.../History/EntityHistoryRepository.cs | 6 +-
.../CleanupAbsolutePathMetadataFiles.cs | 34 +-
.../CleanupAdditionalNamingSpecs.cs | 8 +-
.../Housekeepers/CleanupAdditionalUsers.cs | 6 +-
...ownloadClientUnavailablePendingReleases.cs | 33 +-
.../CleanupDuplicateMetadataFiles.cs | 48 +--
.../Housekeepers/CleanupOrphanedAlbums.cs | 12 +-
.../CleanupOrphanedArtistMetadata.cs | 14 +-
.../Housekeepers/CleanupOrphanedBlacklist.cs | 12 +-
.../CleanupOrphanedDownloadClientStatus.cs | 12 +-
.../CleanupOrphanedHistoryItems.cs | 24 +-
.../CleanupOrphanedImportListStatus.cs | 12 +-
.../CleanupOrphanedIndexerStatus.cs | 12 +-
.../CleanupOrphanedMetadataFiles.cs | 60 +--
.../CleanupOrphanedPendingReleases.cs | 12 +-
.../Housekeepers/CleanupOrphanedReleases.cs | 12 +-
.../Housekeepers/CleanupOrphanedTrackFiles.cs | 28 +-
.../Housekeepers/CleanupOrphanedTracks.cs | 12 +-
.../Housekeepers/CleanupUnusedTags.cs | 21 +-
.../FixFutureRunScheduledTasks.cs | 6 +-
.../Instrumentation/DatabaseTarget.cs | 68 ++-
src/NzbDrone.Core/Lidarr.Core.csproj | 2 +
src/NzbDrone.Core/Localization/Core/en.json | 1 +
.../MediaFiles/MediaFileRepository.cs | 8 +-
.../Messaging/Commands/CommandRepository.cs | 2 +-
.../Music/Repositories/AlbumRepository.cs | 97 +++--
.../Music/Repositories/ArtistRepository.cs | 8 +-
.../Music/Repositories/ReleaseRepository.cs | 17 +-
.../Music/Repositories/TrackRepository.cs | 4 +-
.../Music/Services/AlbumService.cs | 4 +-
.../Services/RefreshAlbumReleaseService.cs | 2 +-
.../Music/Services/RefreshAlbumService.cs | 2 +-
.../Music/Services/RefreshArtistService.cs | 2 +-
.../Music/Services/ReleaseService.cs | 4 +-
.../Music/Services/TrackService.cs | 4 +-
.../Update/UpdatePackageProvider.cs | 6 +-
src/NzbDrone.Host.Test/ContainerFixture.cs | 3 +
src/NzbDrone.Host/Bootstrap.cs | 9 +-
.../IntegrationTest.cs | 33 +-
.../Datastore/PostgresDatabase.cs | 69 ++++
.../Datastore/SqliteDatabase.cs | 14 +
src/NzbDrone.Test.Common/NzbDroneRunner.cs | 26 +-
src/postgres.runsettings | 11 +
119 files changed, 2420 insertions(+), 993 deletions(-)
rename src/NzbDrone.Core.Test/Datastore/{WhereBuilderFixture.cs => WhereBuilderPostgresFixture.cs} (86%)
create mode 100644 src/NzbDrone.Core.Test/Datastore/WhereBuilderSqliteFixture.cs
create mode 100644 src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupOrphanedReleasesFixture.cs
create mode 100644 src/NzbDrone.Core/Datastore/PostgresOptions.cs
create mode 100644 src/NzbDrone.Core/Datastore/WhereBuilderPostgres.cs
create mode 100644 src/NzbDrone.Core/Datastore/WhereBuilderSqlite.cs
create mode 100644 src/NzbDrone.Test.Common/Datastore/PostgresDatabase.cs
create mode 100644 src/NzbDrone.Test.Common/Datastore/SqliteDatabase.cs
create mode 100644 src/postgres.runsettings
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index f6c3cd2bd..6fd2d382a 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -534,6 +534,62 @@ stages:
testResultsFiles: '**/TestResult.xml'
testRunTitle: '$(testName) Unit Tests'
failTaskOnFailedTests: true
+
+ - job: Unit_LinuxCore_Postgres
+ displayName: Unit Native LinuxCore with Postgres Database
+ dependsOn: Prepare
+ condition: and(succeeded(), eq(dependencies.Prepare.outputs['setVar.backendNotUpdated'], '0'))
+ variables:
+ pattern: 'Lidarr.*.linux-core-x64.tar.gz'
+ artifactName: linux-x64-tests
+ Lidarr__Postgres__Host: 'localhost'
+ Lidarr__Postgres__Port: '5432'
+ Lidarr__Postgres__User: 'lidarr'
+ Lidarr__Postgres__Password: 'lidarr'
+
+ pool:
+ vmImage: ${{ variables.linuxImage }}
+
+ timeoutInMinutes: 10
+
+ steps:
+ - task: UseDotNet@2
+ displayName: 'Install .net core'
+ inputs:
+ version: $(dotnetVersion)
+ - checkout: none
+ - task: DownloadPipelineArtifact@2
+ displayName: Download Test Artifact
+ inputs:
+ buildType: 'current'
+ artifactName: $(artifactName)
+ targetPath: $(testsFolder)
+ - bash: |
+ chmod a+x _tests/fpcalc
+ displayName: Make fpcalc Executable
+ condition: and(succeeded(), ne(variables['osName'], 'Windows'))
+ - bash: find ${TESTSFOLDER} -name "Lidarr.Test.Dummy" -exec chmod a+x {} \;
+ displayName: Make Test Dummy Executable
+ condition: and(succeeded(), ne(variables['osName'], 'Windows'))
+ - bash: |
+ docker run -d --name=postgres14 \
+ -e POSTGRES_PASSWORD=lidarr \
+ -e POSTGRES_USER=lidarr \
+ -p 5432:5432/tcp \
+ postgres:14
+ displayName: Start postgres
+ - bash: |
+ chmod a+x ${TESTSFOLDER}/test.sh
+ ls -lR ${TESTSFOLDER}
+ ${TESTSFOLDER}/test.sh Linux Unit Test
+ displayName: Run Tests
+ - task: PublishTestResults@2
+ displayName: Publish Test Results
+ inputs:
+ testResultsFormat: 'NUnit'
+ testResultsFiles: '**/TestResult.xml'
+ testRunTitle: 'LinuxCore Postgres Unit Tests'
+ failTaskOnFailedTests: true
- stage: Integration
displayName: Integration
@@ -617,6 +673,67 @@ stages:
failTaskOnFailedTests: true
displayName: Publish Test Results
+ - job: Integration_LinuxCore_Postgres
+ displayName: Integration Native LinuxCore with Postgres Database
+ dependsOn: Prepare
+ condition: and(succeeded(), eq(dependencies.Prepare.outputs['setVar.backendNotUpdated'], '0'))
+ variables:
+ pattern: 'Lidarr.*.linux-core-x64.tar.gz'
+ Lidarr__Postgres__Host: 'localhost'
+ Lidarr__Postgres__Port: '5432'
+ Lidarr__Postgres__User: 'lidarr'
+ Lidarr__Postgres__Password: 'lidarr'
+
+ pool:
+ vmImage: ${{ variables.linuxImage }}
+
+ steps:
+ - task: UseDotNet@2
+ displayName: 'Install .net core'
+ inputs:
+ version: $(dotnetVersion)
+ - checkout: none
+ - task: DownloadPipelineArtifact@2
+ displayName: Download Test Artifact
+ inputs:
+ buildType: 'current'
+ artifactName: 'linux-x64-tests'
+ targetPath: $(testsFolder)
+ - task: DownloadPipelineArtifact@2
+ displayName: Download Build Artifact
+ inputs:
+ buildType: 'current'
+ artifactName: Packages
+ itemPattern: '**/$(pattern)'
+ targetPath: $(Build.ArtifactStagingDirectory)
+ - task: ExtractFiles@1
+ inputs:
+ archiveFilePatterns: '$(Build.ArtifactStagingDirectory)/**/$(pattern)'
+ destinationFolder: '$(Build.ArtifactStagingDirectory)/bin'
+ displayName: Extract Package
+ - bash: |
+ mkdir -p ./bin/
+ cp -r -v ${BUILD_ARTIFACTSTAGINGDIRECTORY}/bin/Lidarr/. ./bin/
+ displayName: Move Package Contents
+ - bash: |
+ docker run -d --name=postgres14 \
+ -e POSTGRES_PASSWORD=lidarr \
+ -e POSTGRES_USER=lidarr \
+ -p 5432:5432/tcp \
+ postgres:14
+ displayName: Start postgres
+ - bash: |
+ chmod a+x ${TESTSFOLDER}/test.sh
+ ${TESTSFOLDER}/test.sh Linux Integration Test
+ displayName: Run Integration Tests
+ - task: PublishTestResults@2
+ inputs:
+ testResultsFormat: 'NUnit'
+ testResultsFiles: '**/TestResult.xml'
+ testRunTitle: 'Integration LinuxCore Postgres Database Integration Tests'
+ failTaskOnFailedTests: true
+ displayName: Publish Test Results
+
- job: Integration_FreeBSD
displayName: Integration Native FreeBSD
dependsOn: Prepare
diff --git a/frontend/src/System/Status/About/About.js b/frontend/src/System/Status/About/About.js
index 1acd44b11..4635fef68 100644
--- a/frontend/src/System/Status/About/About.js
+++ b/frontend/src/System/Status/About/About.js
@@ -23,6 +23,8 @@ class About extends Component {
isDocker,
runtimeVersion,
migrationVersion,
+ databaseVersion,
+ databaseType,
appData,
startupPath,
mode,
@@ -68,6 +70,11 @@ class About extends Component {
data={migrationVersion}
/>
+
+
(new Mock().Object);
+ container.RegisterInstance(new Mock().Object);
+ container.RegisterInstance(new Mock>().Object);
var serviceProvider = container.GetServiceProvider();
diff --git a/src/NzbDrone.Common/Instrumentation/CleanseLogMessage.cs b/src/NzbDrone.Common/Instrumentation/CleanseLogMessage.cs
index df0f49e1e..aa9692fb0 100644
--- a/src/NzbDrone.Common/Instrumentation/CleanseLogMessage.cs
+++ b/src/NzbDrone.Common/Instrumentation/CleanseLogMessage.cs
@@ -18,6 +18,7 @@ namespace NzbDrone.Common.Instrumentation
new Regex(@"iptorrents\.com/[/a-z0-9?&;]*?(?:[?&;](u|tp)=(?[^&=;]+?))+(?= |;|&|$)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new Regex(@"/fetch/[a-z0-9]{32}/(?[a-z0-9]{32})", RegexOptions.Compiled),
new Regex(@"getnzb.*?(?<=\?|&)(r)=(?[^&=]+?)(?= |&|$)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
+ new Regex(@"\b(\w*)?(_?(?[^&=]+?)(?= |&|$|;)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
// Trackers Announce Keys; Designed for Qbit Json; should work for all in theory
new Regex(@"announce(\.php)?(/|%2f|%3fpasskey%3d)(?[a-z0-9]{16,})|(?[a-z0-9]{16,})(/|%2f)announce"),
diff --git a/src/NzbDrone.Common/Processes/ProcessProvider.cs b/src/NzbDrone.Common/Processes/ProcessProvider.cs
index 9336248bc..27fd09299 100644
--- a/src/NzbDrone.Common/Processes/ProcessProvider.cs
+++ b/src/NzbDrone.Common/Processes/ProcessProvider.cs
@@ -127,7 +127,18 @@ namespace NzbDrone.Common.Processes
try
{
_logger.Trace("Setting environment variable '{0}' to '{1}'", environmentVariable.Key, environmentVariable.Value);
- startInfo.EnvironmentVariables.Add(environmentVariable.Key.ToString(), environmentVariable.Value.ToString());
+
+ var key = environmentVariable.Key.ToString();
+ var value = environmentVariable.Value?.ToString();
+
+ if (startInfo.EnvironmentVariables.ContainsKey(key))
+ {
+ startInfo.EnvironmentVariables[key] = value;
+ }
+ else
+ {
+ startInfo.EnvironmentVariables.Add(key, value);
+ }
}
catch (Exception e)
{
diff --git a/src/NzbDrone.Core.Test/Datastore/DatabaseFixture.cs b/src/NzbDrone.Core.Test/Datastore/DatabaseFixture.cs
index 807057837..f6c182660 100644
--- a/src/NzbDrone.Core.Test/Datastore/DatabaseFixture.cs
+++ b/src/NzbDrone.Core.Test/Datastore/DatabaseFixture.cs
@@ -15,7 +15,7 @@ namespace NzbDrone.Core.Test.Datastore
public void SingleOrDefault_should_return_null_on_empty_db()
{
Mocker.Resolve()
- .OpenConnection().Query("SELECT * FROM Artists")
+ .OpenConnection().Query("SELECT * FROM \"Artists\"")
.SingleOrDefault(c => c.CleanName == "SomeTitle")
.Should()
.BeNull();
diff --git a/src/NzbDrone.Core.Test/Datastore/DatabaseRelationshipFixture.cs b/src/NzbDrone.Core.Test/Datastore/DatabaseRelationshipFixture.cs
index ab17a1550..6c073b18f 100644
--- a/src/NzbDrone.Core.Test/Datastore/DatabaseRelationshipFixture.cs
+++ b/src/NzbDrone.Core.Test/Datastore/DatabaseRelationshipFixture.cs
@@ -2,7 +2,9 @@ using System;
using System.Linq;
using FizzWare.NBuilder;
using FluentAssertions;
+using FluentAssertions.Equivalency;
using NUnit.Framework;
+using NzbDrone.Core.Datastore;
using NzbDrone.Core.History;
using NzbDrone.Core.Music;
using NzbDrone.Core.Qualities;
@@ -13,6 +15,17 @@ namespace NzbDrone.Core.Test.Datastore
[TestFixture]
public class DatabaseRelationshipFixture : DbTest
{
+ [SetUp]
+ public void Setup()
+ {
+ AssertionOptions.AssertEquivalencyUsing(options =>
+ {
+ options.Using(ctx => ctx.Subject.Should().BeCloseTo(ctx.Expectation.ToUniversalTime())).WhenTypeIs();
+ options.Using(ctx => ctx.Subject.Should().BeCloseTo(ctx.Expectation.Value.ToUniversalTime())).WhenTypeIs();
+ return options;
+ });
+ }
+
[Test]
public void one_to_one()
{
@@ -33,13 +46,7 @@ namespace NzbDrone.Core.Test.Datastore
var loadedAlbum = Db.Single().Album.Value;
loadedAlbum.Should().NotBeNull();
- loadedAlbum.Should().BeEquivalentTo(album,
- options => options
- .IncludingAllRuntimeProperties()
- .Excluding(c => c.Artist)
- .Excluding(c => c.ArtistId)
- .Excluding(c => c.ArtistMetadata)
- .Excluding(c => c.AlbumReleases));
+ loadedAlbum.Should().BeEquivalentTo(album, AlbumComparerOptions);
}
[Test]
@@ -86,5 +93,9 @@ namespace NzbDrone.Core.Test.Datastore
returnedHistory[0].Quality.Quality.Should().Be(Quality.MP3_320);
}
+
+ private EquivalencyAssertionOptions AlbumComparerOptions(EquivalencyAssertionOptions opts) => opts.ComparingByMembers()
+ .Excluding(ctx => ctx.SelectedMemberInfo.MemberType.IsGenericType && ctx.SelectedMemberInfo.MemberType.GetGenericTypeDefinition() == typeof(LazyLoaded<>))
+ .Excluding(x => x.ArtistId);
}
}
diff --git a/src/NzbDrone.Core.Test/Datastore/LazyLoadingFixture.cs b/src/NzbDrone.Core.Test/Datastore/LazyLoadingFixture.cs
index 5241eefcd..427f62f2b 100644
--- a/src/NzbDrone.Core.Test/Datastore/LazyLoadingFixture.cs
+++ b/src/NzbDrone.Core.Test/Datastore/LazyLoadingFixture.cs
@@ -104,7 +104,7 @@ namespace NzbDrone.Core.Test.Datastore
public void should_lazy_load_artist_for_trackfile()
{
var db = Mocker.Resolve();
- var tracks = db.Query(new SqlBuilder()).ToList();
+ var tracks = db.Query(new SqlBuilder(db.DatabaseType)).ToList();
Assert.IsNotEmpty(tracks);
foreach (var track in tracks)
@@ -120,7 +120,7 @@ namespace NzbDrone.Core.Test.Datastore
public void should_lazy_load_trackfile_if_not_joined()
{
var db = Mocker.Resolve();
- var tracks = db.Query