Fix flaky tests (#788)

* New DB for each test, hopefully takes care of race condition when using new DB.
* Centralize timeout setting

.github/workflows/test.yml

@@ -12,14 +12,6 @@ jobs:
       matrix:
         python-version: [3.6, 3.7, 3.8, 3.9]
         os: [ubuntu-latest, macos-latest]
-        exclude:
-          # Only test Python 3.7 on macOS
-          - os: macos-latest
-            python-version: 3.6
-          - os: macos-latest
-            python-version: 3.8
-          - os: macos-latest
-            python-version: 3.9
 
     steps:
     - uses: actions/checkout@v2

tests/conftest.py

@@ -16,12 +16,31 @@ models = [RepoModel, RepoPassword, BackupProfileModel, SourceFileModel,
 
 def pytest_configure(config):
     sys._called_from_test = True
+    pytest._wait_defaults = {'timeout': 20000}
+    os.environ['LANG'] = 'en'  # Ensure we test an English UI
+
+
+@pytest.fixture(scope='session')
+def qapp(tmpdir_factory):
+    # DB is required to init QApplication. New DB used for every test.
+    tmp_db = tmpdir_factory.mktemp('Vorta').join('settings.sqlite')
+    mock_db = peewee.SqliteDatabase(str(tmp_db))
+    vorta.models.init_db(mock_db)
+
+    from vorta.application import VortaApp
+    VortaApp.set_borg_details_action = MagicMock()  # Can't use pytest-mock in session scope
+    VortaApp.scheduler = MagicMock()
+
+    qapp = VortaApp([])  # Only init QApplication once to avoid segfaults while testing.
+
+    yield qapp
 
 
 @pytest.fixture(scope='function', autouse=True)
-def init_db(qapp):
-    vorta.models.db.drop_tables(models)
-    vorta.models.init_db()
+def init_db(qapp, tmpdir_factory):
+    tmp_db = tmpdir_factory.mktemp('Vorta').join('settings.sqlite')
+    mock_db = peewee.SqliteDatabase(str(tmp_db), pragmas={'journal_mode': 'wal', })
+    vorta.models.init_db(mock_db)
 
     new_repo = RepoModel(url='i0fi93@i593.repo.borgbase.com:repo')
     new_repo.encryption = 'none'
@@ -44,15 +63,6 @@ def init_db(qapp):
     qapp.main_window = MainWindow(qapp)  # Re-open main window to apply mock data in UI
 
 
-@pytest.fixture(scope='session', autouse=True)
-def local_en():
-    """
-    Some tests use English strings. So override whatever language the current user
-    has and run the tests with the English UI.
-    """
-    os.environ['LANG'] = 'en'
-
-
 @pytest.fixture(scope='function', autouse=True)
 def cleanup(request, qapp, qtbot):
     """
@@ -64,21 +74,6 @@ def cleanup(request, qapp, qtbot):
     request.addfinalizer(ensure_borg_thread_stopped)
 
 
-@pytest.fixture(scope='session')
-def qapp(tmpdir_factory, local_en):
-    tmp_db = tmpdir_factory.mktemp('Vorta').join('settings.sqlite')
-    mock_db = peewee.SqliteDatabase(str(tmp_db), pragmas={'journal_mode': 'wal', })
-    vorta.models.init_db(mock_db)
-
-    from vorta.application import VortaApp
-    VortaApp.set_borg_details_action = MagicMock()  # Can't use pytest-mock in session scope
-    VortaApp.scheduler = MagicMock()
-
-    qapp = VortaApp([])  # Only init QApplication once to avoid segfaults while testing.
-
-    yield qapp
-
-
 @pytest.fixture
 def choose_file_dialog(*args):
     class MockFileDialog:

tests/test_archives.py

@@ -40,11 +40,11 @@ def test_repo_list(qapp, qtbot, mocker, borg_json_output):
 
     main.tabWidget.setCurrentIndex(3)
     tab.list_action()
-    qtbot.waitUntil(lambda: not tab.checkButton.isEnabled(), timeout=3000)
+    qtbot.waitUntil(lambda: not tab.checkButton.isEnabled(), **pytest._wait_defaults)
 
     assert not tab.checkButton.isEnabled()
 
-    qtbot.waitUntil(lambda: main.progressText.text() == 'Refreshing archives done.', timeout=3000)
+    qtbot.waitUntil(lambda: main.progressText.text() == 'Refreshing archives done.', **pytest._wait_defaults)
     assert ArchiveModel.select().count() == 6
     assert main.progressText.text() == 'Refreshing archives done.'
     assert tab.checkButton.isEnabled()
@@ -61,7 +61,7 @@ def test_repo_prune(qapp, qtbot, mocker, borg_json_output):
 
     qtbot.mouseClick(tab.pruneButton, QtCore.Qt.LeftButton)
 
-    qtbot.waitUntil(lambda: main.progressText.text().startswith('Refreshing archives done.'), timeout=5000)
+    qtbot.waitUntil(lambda: main.progressText.text().startswith('Refreshing archives done.'), **pytest._wait_defaults)
 
 
 def test_check(qapp, mocker, borg_json_output, qtbot):
@@ -76,7 +76,7 @@ def test_check(qapp, mocker, borg_json_output, qtbot):
 
     qtbot.mouseClick(tab.checkButton, QtCore.Qt.LeftButton)
     success_text = 'INFO: Archive consistency check complete'
-    qtbot.waitUntil(lambda: main.logText.text().startswith(success_text), timeout=3000)
+    qtbot.waitUntil(lambda: main.logText.text().startswith(success_text), **pytest._wait_defaults)
 
 
 @pytest.mark.skipif(sys.platform == 'darwin', reason="Test currently broken by Homebrew")
@@ -104,10 +104,10 @@ def test_archive_mount(qapp, qtbot, mocker, borg_json_output, monkeypatch, choos
     )
 
     qtbot.mouseClick(tab.mountButton, QtCore.Qt.LeftButton)
-    qtbot.waitUntil(lambda: tab.mountErrors.text().startswith('Mounted'), timeout=10000)
+    qtbot.waitUntil(lambda: tab.mountErrors.text().startswith('Mounted'), **pytest._wait_defaults)
 
     qtbot.mouseClick(tab.mountButton, QtCore.Qt.LeftButton)
-    qtbot.waitUntil(lambda: tab.mountErrors.text().startswith('Un-mounted successfully.'), timeout=10000)
+    qtbot.waitUntil(lambda: tab.mountErrors.text().startswith('Un-mounted successfully.'), **pytest._wait_defaults)
 
 
 def test_archive_extract(qapp, qtbot, mocker, borg_json_output):
@@ -124,8 +124,8 @@ def test_archive_extract(qapp, qtbot, mocker, borg_json_output):
     mocker.patch.object(vorta.borg.borg_thread, 'Popen', return_value=popen_result)
     qtbot.mouseClick(tab.extractButton, QtCore.Qt.LeftButton)
 
-    qtbot.waitUntil(lambda: hasattr(tab, '_window'), timeout=10000)
-    qtbot.waitUntil(lambda: tab._window == qapp.activeWindow(), timeout=5000)
+    qtbot.waitUntil(lambda: hasattr(tab, '_window'), **pytest._wait_defaults)
+    qtbot.waitUntil(lambda: tab._window == qapp.activeWindow(), **pytest._wait_defaults)
 
     assert tab._window.treeView.model().rootItem.childItems[0].data(0) == 'Users'
     tab._window.treeView.model().rootItem.childItems[0].load_children()
@@ -147,7 +147,7 @@ def test_archive_delete(qapp, qtbot, mocker, borg_json_output):
     mocker.patch.object(vorta.views.archive_tab.ArchiveTab, 'confirm_dialog', lambda x, y, z: True)
     qtbot.mouseClick(tab.deleteButton, QtCore.Qt.LeftButton)
 
-    qtbot.waitUntil(lambda: main.progressText.text() == 'Archive deleted.', timeout=3000)
+    qtbot.waitUntil(lambda: main.progressText.text() == 'Archive deleted.', **pytest._wait_defaults)
     assert ArchiveModel.select().count() == 1
     assert tab.archiveTable.rowCount() == 1
 
@@ -165,14 +165,14 @@ def test_archive_diff(qapp, qtbot, mocker, borg_json_output):
     mocker.patch.object(vorta.borg.borg_thread, 'Popen', return_value=popen_result)
 
     qtbot.mouseClick(tab.diffButton, QtCore.Qt.LeftButton)
-    qtbot.waitUntil(lambda: hasattr(tab, '_window'), timeout=5000)
-    qtbot.waitUntil(lambda: tab._window == qapp.activeWindow(), timeout=5000)
+    qtbot.waitUntil(lambda: hasattr(tab, '_window'), **pytest._wait_defaults)
+    qtbot.waitUntil(lambda: tab._window == qapp.activeWindow(), **pytest._wait_defaults)
 
     tab._window.archiveTable.selectRow(0)
     tab._window.archiveTable.selectRow(1)
     qtbot.mouseClick(tab._window.diffButton, QtCore.Qt.LeftButton)
-    qtbot.waitUntil(lambda: hasattr(tab, '_resultwindow'), timeout=5000)
-    qtbot.waitUntil(lambda: tab._resultwindow == qapp.activeWindow(), timeout=5000)
+    qtbot.waitUntil(lambda: hasattr(tab, '_resultwindow'), **pytest._wait_defaults)
+    qtbot.waitUntil(lambda: tab._resultwindow == qapp.activeWindow(), **pytest._wait_defaults)
 
     assert tab._resultwindow.treeView.model().rootItem.childItems[0].data(0) == 'test'
     tab._resultwindow.treeView.model().rootItem.childItems[0].load_children()

tests/test_borg.py

@@ -1,3 +1,4 @@
+import pytest
 import vorta.borg
 import vorta.models
 from vorta.borg.prune import BorgPruneThread
@@ -11,7 +12,7 @@ def test_borg_prune(qapp, qtbot, mocker, borg_json_output):
     params = BorgPruneThread.prepare(vorta.models.BackupProfileModel.select().first())
     thread = BorgPruneThread(params['cmd'], params, qapp)
 
-    with qtbot.waitSignal(thread.result, timeout=10000) as blocker:
+    with qtbot.waitSignal(thread.result, **pytest._wait_defaults) as blocker:
         blocker.connect(thread.updated)
         thread.run()
 

tests/test_lock.py

@@ -1,3 +1,4 @@
+import pytest
 from PyQt5 import QtCore
 import vorta.borg.borg_thread
 import vorta.application
@@ -13,7 +14,7 @@ def test_create_perm_error(qapp, borg_json_output, mocker, qtbot):
 
     qtbot.mouseClick(main.createStartBtn, QtCore.Qt.LeftButton)
 
-    qtbot.waitUntil(lambda: hasattr(qapp, '_msg'), timeout=10000)
+    qtbot.waitUntil(lambda: hasattr(qapp, '_msg'), **pytest._wait_defaults)
     assert qapp._msg.text().startswith("You do not have permission")
     del qapp._msg
 
@@ -29,7 +30,7 @@ def test_create_lock(qapp, borg_json_output, mocker, qtbot):
 
     qtbot.mouseClick(main.createStartBtn, QtCore.Qt.LeftButton)
 
-    qtbot.waitUntil(lambda: hasattr(qapp, '_msg'), timeout=10000)
+    qtbot.waitUntil(lambda: hasattr(qapp, '_msg'), **pytest._wait_defaults)
     assert "The repository at" in qapp._msg.text()
 
     # Break locked repo
@@ -37,7 +38,7 @@ def test_create_lock(qapp, borg_json_output, mocker, qtbot):
     popen_result = mocker.MagicMock(stdout=stdout, stderr=stderr, returncode=0)
     mocker.patch.object(vorta.borg.borg_thread, 'Popen', return_value=popen_result)
 
-    qtbot.waitUntil(lambda: main.createStartBtn.isEnabled(), timeout=3000)  # Prevent thread collision
+    qtbot.waitUntil(lambda: main.createStartBtn.isEnabled(), **pytest._wait_defaults)  # Prevent thread collision
     qapp._msg.accept()
     exp_message_text = 'Repository lock broken. Please redo your last action.'
-    qtbot.waitUntil(lambda: main.progressText.text() == exp_message_text, timeout=5000)
+    qtbot.waitUntil(lambda: main.progressText.text() == exp_message_text, **pytest._wait_defaults)

tests/test_misc.py

@@ -1,8 +1,9 @@
+import os
+import sys
+import pytest
 from PyQt5 import QtCore
 from PyQt5.QtWidgets import QCheckBox
 from pathlib import Path
-import os
-import sys
 
 
 def test_autostart(qapp, qtbot):
@@ -26,7 +27,7 @@ def test_autostart(qapp, qtbot):
     if sys.platform == 'linux':
         autostart_path = Path(os.environ.get(
             "XDG_CONFIG_HOME", os.path.expanduser("~") + '/.config') + "/autostart") / "vorta.desktop"
-        qtbot.waitUntil(lambda: autostart_path.exists(), timeout=5000)
+        qtbot.waitUntil(lambda: autostart_path.exists(), **pytest._wait_defaults)
 
         with open(autostart_path) as desktop_file:
             desktop_file_text = desktop_file.read()

tests/test_profile.py

@@ -1,3 +1,4 @@
+import pytest
 from PyQt5 import QtCore
 from PyQt5.QtWidgets import QDialogButtonBox
 from vorta.models import BackupProfileModel
@@ -9,7 +10,7 @@ def test_profile_add(qapp, qtbot):
 
     add_profile_window = main.window
     qtbot.addWidget(add_profile_window)
-    qtbot.waitUntil(lambda: add_profile_window == qapp.activeWindow(), timeout=5000)
+    qtbot.waitUntil(lambda: add_profile_window == qapp.activeWindow(), **pytest._wait_defaults)
 
     qtbot.keyClicks(add_profile_window.profileNameField, 'Test Profile')
     qtbot.mouseClick(add_profile_window.buttonBox.button(QDialogButtonBox.Save), QtCore.Qt.LeftButton)
@@ -24,7 +25,7 @@ def test_profile_edit(qapp, qtbot):
 
     edit_profile_window = main.window
     qtbot.addWidget(edit_profile_window)
-    qtbot.waitUntil(lambda: edit_profile_window == qapp.activeWindow(), timeout=5000)
+    qtbot.waitUntil(lambda: edit_profile_window == qapp.activeWindow(), **pytest._wait_defaults)
 
     edit_profile_window.profileNameField.setText("")
     qtbot.keyClicks(edit_profile_window.profileNameField, 'Test Profile')

tests/test_repo.py

@@ -1,5 +1,6 @@
 import os
 import uuid
+import pytest
 from PyQt5 import QtCore
 
 import vorta.borg.borg_thread
@@ -53,7 +54,7 @@ def test_repo_unlink(qapp, qtbot):
 
     main.tabWidget.setCurrentIndex(0)
     qtbot.mouseClick(tab.repoRemoveToolbutton, QtCore.Qt.LeftButton)
-    qtbot.waitUntil(lambda: tab.repoSelector.count() == 4, timeout=5000)
+    qtbot.waitUntil(lambda: tab.repoSelector.count() == 4, **pytest._wait_defaults)
     assert RepoModel.select().count() == 0
 
     qtbot.mouseClick(main.createStartBtn, QtCore.Qt.LeftButton)
@@ -92,7 +93,7 @@ def test_repo_add_success(qapp, qtbot, mocker, borg_json_output):
 
     qtbot.mouseClick(add_repo_window.saveButton, QtCore.Qt.LeftButton)
 
-    with qtbot.waitSignal(add_repo_window.thread.result, timeout=3000) as _:
+    with qtbot.waitSignal(add_repo_window.thread.result, **pytest._wait_defaults) as _:
         pass
 
     assert EventLogModel.select().count() == 2
@@ -135,8 +136,8 @@ def test_create(qapp, borg_json_output, mocker, qtbot):
     mocker.patch.object(vorta.borg.borg_thread, 'Popen', return_value=popen_result)
 
     qtbot.mouseClick(main.createStartBtn, QtCore.Qt.LeftButton)
-    qtbot.waitUntil(lambda: main.progressText.text().startswith('Backup finished.'), timeout=3000)
-    qtbot.waitUntil(lambda: main.createStartBtn.isEnabled(), timeout=3000)
+    qtbot.waitUntil(lambda: main.progressText.text().startswith('Backup finished.'), **pytest._wait_defaults)
+    qtbot.waitUntil(lambda: main.createStartBtn.isEnabled(), **pytest._wait_defaults)
     assert EventLogModel.select().count() == 1
     assert ArchiveModel.select().count() == 3
     assert RepoModel.get(id=1).unique_size == 15520474

tests/test_scheduler.py

@@ -1,3 +1,4 @@
+import pytest
 import vorta.borg
 import vorta.models
 
@@ -9,4 +10,4 @@ def test_scheduler_create_backup(qapp, qtbot, mocker, borg_json_output):
 
     qapp.scheduler.create_backup(1)
 
-    qtbot.waitUntil(lambda: vorta.models.EventLogModel.select().count() == 2, timeout=5000)
+    qtbot.waitUntil(lambda: vorta.models.EventLogModel.select().count() == 2, **pytest._wait_defaults)

tests/test_source.py

@@ -1,4 +1,5 @@
 import os
+import pytest
 import vorta.models
 import vorta.views
 from PyQt5.QtWidgets import QApplication
@@ -14,7 +15,7 @@ def test_add_folder(qapp, qtbot, tmpdir, monkeypatch, choose_file_dialog):
     tab = main.sourceTab
 
     tab.sourceAddFolder.click()
-    qtbot.waitUntil(lambda: tab.sourceFilesWidget.rowCount() == 2, timeout=5000)
+    qtbot.waitUntil(lambda: tab.sourceFilesWidget.rowCount() == 2, **pytest._wait_defaults)
 
     # Test paste button
     QApplication.clipboard().setText(os.path.expanduser('~'))  # Load clipboard