New: Cross Browser Testing UI

2019-09-16 21:27:15 +01:00 · 2019-09-16 21:27:15 +01:00 · 22d48ab8aa
parent 6338d6d2d0
commit 22d48ab8aa
7 changed files with 168 additions and 65 deletions
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@ -100,40 +100,6 @@ stages:
        artifact: '$(osName)Frontend'
        displayName: Publish Frontend
        condition: and(succeeded(), eq(variables['osName'], 'Windows'))
-        
-  - stage: Installer
-    dependsOn:
-    - Build_Backend
-    - Build_Frontend
-    jobs:
-    - job: Windows_Installer
-      displayName: Create Installer
-      pool:
-        vmImage: 'vs2017-win2016'
-      steps:
-      - checkout: self
-        fetchDepth: 1
-      - task: DownloadPipelineArtifact@2
-        inputs:
-          buildType: 'current'
-          artifactName: WindowsBackend
-          targetPath: _output
-        displayName: Fetch Backend
-      - task: DownloadPipelineArtifact@2
-        inputs:
-          buildType: 'current'
-          artifactName: WindowsFrontend
-          targetPath: _output
-        displayName: Fetch Frontend
-      - bash: ./build.sh --packages
-        displayName: Create Packages
-      - bash: |
-          ./setup/inno/ISCC.exe "./setup/lidarr.iss"
-          cp ./setup/output/Lidarr.*windows.exe ${BUILD_ARTIFACTSTAGINGDIRECTORY}/${WINDOWSINSTALLER}
-        displayName: Create Windows installer
-      - publish: $(Build.ArtifactStagingDirectory)
-        artifact: 'WindowsInstaller'
-        displayName: Publish Installer

  - stage: Packages
    dependsOn:
@ -222,6 +188,40 @@ stages:
        env:
          SENTRY_AUTH_TOKEN: $(sentryAuthToken)
          SENTRY_ORG: $(sentryOrg)
+
+  - stage: Installer
+    dependsOn:
+    - Build_Backend
+    - Build_Frontend
+    jobs:
+    - job: Windows_Installer
+      displayName: Create Installer
+      pool:
+        vmImage: 'vs2017-win2016'
+      steps:
+      - checkout: self
+        fetchDepth: 1
+      - task: DownloadPipelineArtifact@2
+        inputs:
+          buildType: 'current'
+          artifactName: WindowsBackend
+          targetPath: _output
+        displayName: Fetch Backend
+      - task: DownloadPipelineArtifact@2
+        inputs:
+          buildType: 'current'
+          artifactName: WindowsFrontend
+          targetPath: _output
+        displayName: Fetch Frontend
+      - bash: ./build.sh --packages
+        displayName: Create Packages
+      - bash: |
+          ./setup/inno/ISCC.exe "./setup/lidarr.iss"
+          cp ./setup/output/Lidarr.*windows.exe ${BUILD_ARTIFACTSTAGINGDIRECTORY}/${WINDOWSINSTALLER}
+        displayName: Create Windows installer
+      - publish: $(Build.ArtifactStagingDirectory)
+        artifact: 'WindowsInstaller'
+        displayName: Publish Installer
    
  - stage: Unit_Test
    displayName: Unit Tests
@ -448,7 +448,7 @@ stages:

      container: $[ variables['containerImage'] ]

-      timeoutInMinutes: 15
+      timeoutInMinutes: 6
        
      steps:
      - bash: mono --version
@ -499,16 +499,6 @@ stages:
    - job: Automation
      strategy:
        matrix:
-          Linux:
-            osName: 'Linux'
-            imageName: 'ubuntu-16.04'
-            pattern: 'Lidarr.**.linux.tar.gz'
-            failBuild: true
-          Mac:
-            osName: 'Mac'
-            imageName: 'macos-10.13' # Fails due to firefox not being installed on image
-            pattern: 'Lidarr.**.osx.tar.gz'
-            failBuild: false
          Windows:
            osName: 'Windows'
            imageName: 'vs2017-win2016'
@ -542,27 +532,22 @@ stages:
          mkdir -p ./bin/
          cp -r -v ${BUILD_ARTIFACTSTAGINGDIRECTORY}/bin/Lidarr/. ./bin/
        displayName: Move Package Contents
-      - bash: |
-          if [[ $OSNAME == "Mac" ]]; then
-          url=https://github.com/mozilla/geckodriver/releases/download/v0.24.0/geckodriver-v0.24.0-macos.tar.gz
-          elif [[ $OSNAME == "Linux" ]]; then
-          url=https://github.com/mozilla/geckodriver/releases/download/v0.24.0/geckodriver-v0.24.0-linux64.tar.gz
-          else
-          echo "Unhandled OS"
-          exit 1
-          fi
-          curl -s -L "$url" | tar -xz
-          chmod +x geckodriver
-          mv geckodriver _tests
-        displayName: Install Gecko Driver
-        condition: and(succeeded(), ne(variables['osName'], 'Windows'))
-      - bash: ls -lR
+      - script: |
+          call bin\serviceinstall.exe
+        displayName: Start Lidarr Service
      - task: Bash@3
        displayName: Run Automation Tests
        inputs:
          targetType: 'filePath'
          filePath: '$(testsFolder)/test.sh'
          arguments: $(osName) Automation Test
+        env:
+          BROWSERSTACK_USERNAME: $(browserStackUser)
+          BROWSERSTACK_ACCESS_KEY: $(browserStackKey)
+      - script: |
+          call sc stop lidarr
+          call bin\serviceuninstall.exe
+        displayName: Stop and Remove Lidarr Service
      - task: PublishTestResults@2
        inputs:
          testResultsFormat: 'NUnit'
--- a/build.sh
+++ b/build.sh
@ -144,8 +144,7 @@ Build()
 YarnInstall()
 {
    ProgressStart 'yarn install'
-    yarn install
-    #npm-cache install npm || CheckExitCode npm install --no-optional --no-bin-links
+    yarn install --frozen-lockfile
    ProgressEnd 'yarn install'
 }

--- a/src/NzbDrone.Automation.Test/AutomationTest.cs
+++ b/src/NzbDrone.Automation.Test/AutomationTest.cs
@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using FluentAssertions;
@ -7,6 +8,7 @@ using NLog.Targets;
 using NUnit.Framework;
 using NzbDrone.Automation.Test.PageModel;
 using NzbDrone.Common.EnvironmentInfo;
+using NzbDrone.Common.Extensions;
 using NzbDrone.Test.Common;
 using OpenQA.Selenium;
 using OpenQA.Selenium.Firefox;
@ -32,8 +34,16 @@ namespace NzbDrone.Automation.Test
        }

        [OneTimeSetUp]
-        public void SmokeTestSetup()
+        public virtual void SmokeTestSetup()
        {
+            string username = Environment.GetEnvironmentVariable("BROWSERSTACK_USERNAME");
+            string accessKey = Environment.GetEnvironmentVariable("BROWSERSTACK_ACCESS_KEY");
+
+            if (username.IsNotNullOrWhiteSpace() && accessKey.IsNotNullOrWhiteSpace())
+            {
+                Assert.Ignore("BrowserStack Tests Enabled, Don't Run Normal Automation Tests");
+            }
+
            var options = new FirefoxOptions();
            options.AddArguments("--headless");
            driver = new FirefoxDriver(options);
@ -59,7 +69,7 @@ namespace NzbDrone.Automation.Test
        }

        [OneTimeTearDown]
-        public void SmokeTestTearDown()
+        public virtual void SmokeTestTearDown()
        {
            _runner.KillAll();
            driver.Quit();
--- a/src/NzbDrone.Automation.Test/BrowserStackAutomationTest.cs
+++ b/src/NzbDrone.Automation.Test/BrowserStackAutomationTest.cs
@ -0,0 +1,91 @@
+using System;
+using System.Collections.Generic;
+using BrowserStack;
+using FluentAssertions;
+using NUnit.Framework;
+using NzbDrone.Automation.Test.PageModel;
+using NzbDrone.Common.EnvironmentInfo;
+using NzbDrone.Common.Extensions;
+using OpenQA.Selenium.Remote;
+
+namespace NzbDrone.Automation.Test
+{
+    [TestFixture]
+    [AutomationTest]
+    [Parallelizable(ParallelScope.Fixtures)]
+    public abstract class BrowserStackAutomationTest : MainPagesTest
+    {
+        protected string browser;
+        protected string browserVersion;
+        protected string os;
+        protected string osVersion;
+        protected string device;
+        private Local browserStackLocal;
+
+        public BrowserStackAutomationTest(string device, string os, string osVersion, string browser, string browserVersion)
+        {
+            this.device = device;
+            this.browser = browser;
+            this.browserVersion = browserVersion;
+            this.os = os;
+            this.osVersion = osVersion;
+        }
+
+        [OneTimeSetUp]
+        public override void SmokeTestSetup()
+        {
+            string username = Environment.GetEnvironmentVariable("BROWSERSTACK_USERNAME");
+            string accessKey = Environment.GetEnvironmentVariable("BROWSERSTACK_ACCESS_KEY");
+
+            if (username.IsNullOrWhiteSpace() || accessKey.IsNullOrWhiteSpace())
+            {
+                Assert.Ignore("BrowserStack Tests Disabled, No Credentials");
+            }
+
+            string browserstackLocal = "true";
+            string browserstackLocalIdentifier = string.Format("Lidarr_{0}_{1}", DateTime.UtcNow.Ticks, new Random().Next());
+            string buildName = BuildInfo.Version.ToString();
+
+            DesiredCapabilities capabilities = new DesiredCapabilities();
+
+            capabilities.SetCapability("device", device);
+            capabilities.SetCapability("os", os);
+            capabilities.SetCapability("os_version", osVersion);
+            capabilities.SetCapability("browser", browser);
+            capabilities.SetCapability("browser_version", browserVersion);
+            capabilities.SetCapability("browserstack.local", browserstackLocal);
+            capabilities.SetCapability("browserstack.localIdentifier", browserstackLocalIdentifier);
+            capabilities.SetCapability("browserstack.debug", "true");
+            capabilities.SetCapability("name", "Function Tests: " + browser);
+            capabilities.SetCapability("project", "Lidarr");
+            capabilities.SetCapability("build", buildName);
+
+            browserStackLocal = new Local();
+            List<KeyValuePair<string, string>> bsLocalArgs = new List<KeyValuePair<string, string>>();
+            bsLocalArgs.Add(new KeyValuePair<string, string>("key", accessKey));
+            bsLocalArgs.Add(new KeyValuePair<string, string>("localIdentifier", browserstackLocalIdentifier));
+            browserStackLocal.start(bsLocalArgs);
+
+            driver = new RemoteWebDriver(new Uri("https://" + username + ":" + accessKey + "@hub.browserstack.com/wd/hub"), capabilities);
+
+            driver.Url = "http://localhost:8686";
+
+            var page = new PageBase(driver);
+            page.WaitForNoSpinner();
+
+            driver.ExecuteScript("window.Lidarr.NameViews = true;");
+
+            GetPageErrors().Should().BeEmpty();
+        }
+
+        [OneTimeTearDown]
+        public override void SmokeTestTearDown()
+        {
+            driver.Quit();
+            if (browserStackLocal != null)
+            {
+                browserStackLocal.stop();
+            }
+        }
+    }
+}
--- a/src/NzbDrone.Automation.Test/BrowserStackFixture.cs
+++ b/src/NzbDrone.Automation.Test/BrowserStackFixture.cs
@ -0,0 +1,16 @@
+using NUnit.Framework;
+
+namespace NzbDrone.Automation.Test
+{
+    [TestFixture("","Windows","10","Chrome", "63")]
+    [TestFixture("", "Windows","10", "Firefox", "67")]
+    [TestFixture("", "Windows","10", "Edge", "18")]
+    [TestFixture("iPhone X", "", "11", "iPhone", "")]
+    [TestFixture("Samsung Galaxy S9 Plus", "", "9.0", "android", "")]
+    public class BSMainPagesTest : BrowserStackAutomationTest
+    {
+        public BSMainPagesTest(string device, string os, string osVersion, string browser, string browserVersion) : 
+            base(device, os, osVersion, browser, browserVersion) { }
+
+    }
+}
--- a/src/NzbDrone.Automation.Test/Lidarr.Automation.Test.csproj
+++ b/src/NzbDrone.Automation.Test/Lidarr.Automation.Test.csproj
@ -4,6 +4,7 @@
    <Platforms>x86</Platforms>
  </PropertyGroup>
  <ItemGroup>
+    <PackageReference Include="BrowserStackLocal" Version="1.4.0" />
    <PackageReference Include="Selenium.Firefox.WebDriver" Version="0.24.0" />
    <PackageReference Include="Selenium.Support" Version="3.141.0" />
  </ItemGroup>
--- a/test.sh
+++ b/test.sh
@ -25,7 +25,7 @@ export LIDARR_TESTS_LOG_OUTPUT="File"

 NUNIT="$TEST_DIR/NUnit.ConsoleRunner.3.10.0/tools/nunit3-console.exe"
 NUNIT_COMMAND="$NUNIT"
-NUNIT_PARAMS="--workers=1"
+NUNIT_PARAMS=""

 if [ "$PLATFORM" = "Mac" ]; then

@ -53,6 +53,7 @@ if [ "$TYPE" = "Unit" ]; then
  WHERE="$WHERE && cat != IntegrationTest && cat != AutomationTest"
 elif [ "$TYPE" = "Integration" ] || [ "$TYPE" = "int" ] ; then
  WHERE="$WHERE && cat == IntegrationTest"
+  NUNIT_PARAMS="--workers=1"
 elif [ "$TYPE" = "Automation" ] ; then
  WHERE="$WHERE && cat == AutomationTest"
 else