Refactor BorgThread into new class to support different commands.

README.md

@@ -42,7 +42,7 @@ $ vorta
 ## Development
 Install in development mode:
 ```
-$ pip install -e .
+$ python setup.py develop
 ```
 
 Then run via
@@ -64,7 +64,7 @@ $ pyinstaller --clean --noconfirm vorta.spec
 ### Testing (work in progress)
 Tests are in the folder `/tests`. Testing happens at the level of UI components. Calls to `borg` are mocked and can be replaced with some example json-output. To run tests:
 ```
-$ pytest
+$ python setup.py test
 ```
 
 To update and view coverage information

setup.cfg

@@ -38,10 +38,8 @@ install_requires =
   python-dateutil
   keyring
   apscheduler
-  sentry_sdk
-
-[options.extras_require]
-tests =
+  sentry-sdk
+tests_require =
   pytest
   pytest-qt
   pytest-mock
@@ -54,6 +52,9 @@ tests =
 gui_scripts =
   vorta = vorta.__main__:main
 
+[aliases]
+test=pytest
+
 [tool:pytest]
 addopts = -vs
 testpaths = tests
@@ -65,8 +66,11 @@ filterwarnings =
 source = src
 
 [tox:tox]
-envlist = py37
+envlist = py36,py37
 
 [tox:testenv]
 deps=pytest
 commands=pytest
+
+[tox:testenv:py36]
+basepython = python3.4

src/vorta/application.py

@@ -1,13 +1,13 @@
-import sys
-import os
 from PyQt5 import QtCore
 from PyQt5.QtWidgets import QApplication
+from PyQt5.QtGui import QIcon
 
 from .tray_menu import TrayMenu
 from .scheduler import VortaScheduler
-from .models import BackupProfileModel, SnapshotModel, BackupProfileMixin
-from .borg_create import BorgCreateThread
+from .models import BackupProfileMixin
+from .borg.create import BorgCreateThread
 from .views.main_window import MainWindow
+from .utils import get_asset
 
 
 class VortaApp(QApplication, BackupProfileMixin):
@@ -33,11 +33,14 @@ class VortaApp(QApplication, BackupProfileMixin):
         self.main_window = MainWindow(self)
         self.main_window.show()
 
+        self.backup_started_event.connect(self.backup_started_event_response)
+        self.backup_finished_event.connect(self.backup_finished_event_response)
+
     def create_backup_action(self):
         msg = BorgCreateThread.prepare()
         if msg['ok']:
-            self.thread = BorgCreateThread(msg['cmd'], msg['params'], parent=self)
-            self.thread.start()
+            thread = BorgCreateThread(msg['cmd'], msg['params'], parent=self)
+            thread.start()
         else:
             self.backup_log_event.emit(msg['message'])
 
@@ -45,4 +48,10 @@ class VortaApp(QApplication, BackupProfileMixin):
         self.main_window.show()
         self.main_window.raise_()
 
+    def backup_started_event_response(self):
+        icon = QIcon(get_asset('icons/hdd-o-active.png'))
+        self.tray.setIcon(icon)
 
+    def backup_finished_event_response(self):
+        icon = QIcon(get_asset('icons/hdd-o.png'))
+        self.tray.setIcon(icon)

src/vorta/borg/borg_thread.py

@@ -7,7 +7,8 @@ from PyQt5 import QtCore
 from PyQt5.QtWidgets import QApplication
 from subprocess import Popen, PIPE
 
-from .models import EventLogModel
+from vorta.models import SourceDirModel, BackupProfileModel, WifiSettingModel, EventLogModel
+from vorta.utils import get_current_wifi, keyring
 
 mutex = QtCore.QMutex()
 
@@ -33,14 +34,10 @@ class BorgThread(QtCore.QThread):
         super().__init__(parent)
         self.app = QApplication.instance()
         self.app.backup_cancelled_event.connect(self.cancel)
-
-        # Find packaged borg binary. Prefer globally installed.
-        if not shutil.which('borg'):
-            meipass_borg = os.path.join(sys._MEIPASS, 'bin', 'borg')
-            if os.path.isfile(meipass_borg):
-                cmd[0] = meipass_borg
         self.cmd = cmd
 
+        self.cmd[0] = self.prepare_bin()
+
         env = os.environ.copy()
         env['BORG_HOSTNAME_IS_UNIQUE'] = '1'
         if params.get('password') and params['password'] is not None:
@@ -62,6 +59,60 @@ class BorgThread(QtCore.QThread):
         else:
             return True
 
+    @classmethod
+    def prepare(cls):
+        profile = BackupProfileModel.get(id=1)
+        ret = {'ok': False}
+
+        # Do checks to see if running Borg is possible.
+        if cls.is_running():
+            ret['message'] = 'Backup is already in progress.'
+            return ret
+
+        if profile.repo is None:
+            ret['message'] = 'Add a remote backup repository first.'
+            return ret
+
+        n_backup_folders = SourceDirModel.select().count()
+        if n_backup_folders == 0:
+            ret['message'] = 'Add some folders to back up first.'
+            return ret
+
+        current_wifi = get_current_wifi()
+        if current_wifi is not None:
+            wifi_is_disallowed = WifiSettingModel.select().where(
+                (WifiSettingModel.ssid == current_wifi)
+                & (WifiSettingModel.allowed == False)
+                & (WifiSettingModel.profile == profile.id)
+            )
+            if wifi_is_disallowed.count() > 0:
+                ret['message'] = 'Current Wifi is not allowed.'
+                return ret
+
+        if cls.prepare_bin() is None:
+            ret['message'] = 'Borg binary was not found.'
+            return ret
+
+        ret['params'] = {'password': keyring.get_password("vorta-repo", profile.repo.url)}
+
+        return ret
+
+    @classmethod
+    def prepare_bin(cls):
+        """Find packaged borg binary. Prefer globally installed."""
+
+        # Look in current PATH.
+        if shutil.which('borg'):
+            return 'borg'
+        else:
+            # Look in pyinstaller package
+            cwd = getattr(sys, '_MEIPASS', os.getcwd())
+            meipass_borg = os.path.join(cwd, 'bin', 'borg')
+            if os.path.isfile(meipass_borg):
+                return meipass_borg
+            else:
+                return None
+
     def run(self):
         self.started_event()
         mutex.lock()

src/vorta/borg/create.py

@@ -4,8 +4,7 @@ import platform
 from dateutil import parser
 from datetime import datetime as dt
 
-from .models import SourceDirModel, BackupProfileModel, WifiSettingModel, SnapshotModel, BackupProfileMixin
-from .utils import get_current_wifi, keyring
+from vorta.models import SourceDirModel, SnapshotModel, BackupProfileModel, BackupProfileMixin
 from .borg_thread import BorgThread
 
 
@@ -49,34 +48,10 @@ class BorgCreateThread(BorgThread, BackupProfileMixin):
         Centralize it here and return the required arguments to the caller.
         """
         profile = BackupProfileModel.get(id=1)
-
-        ret = {'ok': False}
-
-        if cls.is_running():
-            ret['message'] = 'Backup is already in progress.'
+        ret = super().prepare()
+        if not ret['ok']:
             return ret
 
-        if profile.repo is None:
-            ret['message'] = 'Add a remote backup repository first.'
-            return ret
-
-        n_backup_folders = SourceDirModel.select().count()
-        if n_backup_folders == 0:
-            ret['message'] = 'Add some folders to back up first.'
-            return ret
-
-        current_wifi = get_current_wifi()
-        if current_wifi is not None:
-            wifi_is_disallowed = WifiSettingModel.select().where(
-                (WifiSettingModel.ssid == current_wifi)
-                & (WifiSettingModel.allowed == False)
-                & (WifiSettingModel.profile == profile.id)
-            )
-            if wifi_is_disallowed.count() > 0:
-                ret['message'] = 'Current Wifi is not allowed.'
-                return ret
-
-        params = {'password': keyring.get_password("vorta-repo", profile.repo.url)}
         cmd = ['borg', 'create', '--list', '--info', '--log-json', '--json', '-C', profile.compression]
 
         # Add excludes
@@ -108,6 +83,5 @@ class BorgCreateThread(BorgThread, BackupProfileMixin):
         ret['message'] = 'Starting backup..'
         ret['ok'] = True
         ret['cmd'] = cmd
-        ret['params'] = params
 
         return ret

src/vorta/borg/prune.py

@@ -0,0 +1,34 @@
+from .borg_thread import BorgThread
+
+
+class BorgPruneThread(BorgThread):
+    def process_result(self, result):
+        pass
+
+    def log_event(self, msg):
+        self.app.backup_log_event.emit(msg)
+
+    def started_event(self):
+        self.app.backup_started_event.emit()
+        self.app.backup_log_event.emit('Backup started.')
+
+    def finished_event(self, result):
+        self.app.backup_finished_event.emit(result)
+
+    @classmethod
+    def prepare(cls):
+        ret, params, profile = super().prepare()
+        cmd = ['borg', 'prune', '--list', '--stats', '--info', '--log-json', '--json', ]
+
+        # -H, --keep-hourly	number of hourly archives to keep
+        # -d, --keep-daily	number of daily archives to keep
+        # -w, --keep-weekly	number of weekly archives to keep
+        # -m, --keep-monthly	number of monthly archives to keep
+        # -y, --keep-yearly	number of yearly archives to keep
+
+        ret['message'] = 'Pruning repository..'
+        ret['ok'] = True
+        ret['cmd'] = cmd
+        ret['params'] = params
+
+        return ret

src/vorta/models.py

@@ -163,8 +163,6 @@ def init_db(con):
     else:
         migrator = SqliteMigrator(con)
 
-    if current_schema.version < 3:  # version 2 to 3
-        pass
 
     if current_schema.version < 4:  # version 3 to 4
         _apply_schema_update(

src/vorta/scheduler.py

@@ -1,7 +1,7 @@
 from apscheduler.schedulers.qt import QtScheduler
 from apscheduler.triggers import cron
 
-from .borg_create import BorgCreateThread
+from vorta.borg.create import BorgCreateThread
 from .models import BackupProfileMixin
 
 

src/vorta/tray_menu.py

@@ -1,10 +1,8 @@
-from PyQt5 import QtWidgets, QtGui, QtCore
-from PyQt5.QtWidgets import QMenu, QApplication, QSystemTrayIcon, QMessageBox, QDialog
-from .views.main_window import MainWindow
+from PyQt5.QtWidgets import QMenu, QSystemTrayIcon
 from PyQt5.QtGui import QIcon
 
 from .utils import get_asset
-from .borg_thread import BorgThread
+from vorta.borg.borg_thread import BorgThread
 
 
 class TrayMenu(QSystemTrayIcon):

src/vorta/utils.py

@@ -77,15 +77,16 @@ def get_sorted_wifis():
     """Get SSIDs from OS and merge with settings in DB."""
     app = QApplication.instance()
 
-    plist_file = open('/Library/Preferences/SystemConfiguration/com.apple.airport.preferences.plist', 'rb')
-    wifis = plistlib.load(plist_file)['KnownNetworks']
-    if wifis:
-        for wifi in wifis.values():
-            timestamp = wifi.get('LastConnected', None)
-            ssid = wifi['SSIDString']
-            WifiSettingModel.get_or_create(ssid=ssid, profile=app.profile,
-                                           defaults={'last_connected': timestamp,
-                                                    'allowed': True})
+    if sys.platform == 'darwin':
+        plist_file = open('/Library/Preferences/SystemConfiguration/com.apple.airport.preferences.plist', 'rb')
+        wifis = plistlib.load(plist_file)['KnownNetworks']
+        if wifis:
+            for wifi in wifis.values():
+                timestamp = wifi.get('LastConnected', None)
+                ssid = wifi['SSIDString']
+                WifiSettingModel.get_or_create(ssid=ssid, profile=app.profile,
+                                               defaults={'last_connected': timestamp,
+                                                        'allowed': True})
 
     return WifiSettingModel.select().order_by(-WifiSettingModel.last_connected)
 
@@ -96,11 +97,13 @@ def get_current_wifi():
 
     From https://gist.github.com/keithweaver/00edf356e8194b89ed8d3b7bbead000c
     """
-    cmd = ['/System/Library/PrivateFrameworks/Apple80211.framework/Versions/Current/Resources/airport','-I']
-    process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
-    out, err = process.communicate()
-    process.wait()
-    for line in out.decode("utf-8").split('\n'):
-        split_line = line.strip().split(':')
-        if split_line[0] == 'SSID':
-            return split_line[1].strip()
+
+    if sys.platform == 'darwin':
+        cmd = ['/System/Library/PrivateFrameworks/Apple80211.framework/Versions/Current/Resources/airport','-I']
+        process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+        out, err = process.communicate()
+        process.wait()
+        for line in out.decode("utf-8").split('\n'):
+            split_line = line.strip().split(':')
+            if split_line[0] == 'SSID':
+                return split_line[1].strip()

src/vorta/views/main_window.py

@@ -7,7 +7,7 @@ from .source_tab import SourceTab
 from .snapshots_tab import SnapshotTab
 from .schedule_tab import ScheduleTab
 from ..utils import get_asset
-from ..borg_thread import BorgThread
+from vorta.borg.borg_thread import BorgThread
 
 
 uifile = get_asset('UI/mainwindow.ui')
@@ -55,24 +55,24 @@ class MainWindow(MainWindowBase, MainWindowUI):
             self.createProgress.setRange(0, progress_max)
         self.createProgressText.repaint()
 
+    def _toggle_buttons(self, create_enabled=True):
+        self.createStartBtn.setEnabled(create_enabled)
+        self.createStartBtn.repaint()
+        self.cancelButton.setEnabled(not create_enabled)
+        self.cancelButton.repaint()
+
     def backup_started_event(self):
             self.set_status(progress_max=0)
-            self.createStartBtn.setEnabled(False)
-            self.createStartBtn.repaint()
-            self.cancelButton.setEnabled(True)
-            self.cancelButton.repaint()
+            self._toggle_buttons(create_enabled=False)
 
     def backup_finished_event(self):
-        self.createStartBtn.setEnabled(True)
-        self.createStartBtn.repaint()
         self.set_status(progress_max=100)
+        self._toggle_buttons(create_enabled=True)
         self.snapshotTab.populate()
+        self.repoTab.init_repo_stats()
 
     def backup_cancelled_event(self):
-        self.createStartBtn.setEnabled(True)
-        self.createStartBtn.repaint()
-        self.cancelButton.setEnabled(False)
-        self.cancelButton.repaint()
+        self._toggle_buttons(create_enabled=True)
         self.set_status(progress_max=100)
         self.set_status('Backup cancelled')
 

src/vorta/views/repo_add.py

@@ -1,6 +1,6 @@
-from PyQt5 import uic, QtCore
+from PyQt5 import uic
 from ..utils import get_private_keys, get_asset
-from ..borg_thread import BorgThread
+from vorta.borg.borg_thread import BorgThread
 
 uifile = get_asset('UI/repoadd.ui')
 AddRepoUI, AddRepoBase = uic.loadUiType(uifile)
@@ -8,7 +8,7 @@ AddRepoUI, AddRepoBase = uic.loadUiType(uifile)
 
 class AddRepoWindow(AddRepoBase, AddRepoUI):
     connection_message = 'Setting up new repo...'
-    cmd = ["borg", "init", "--log-json"]
+    cmd = ["borg", "init", "--info", "--json", "--log-json"]
 
     def __init__(self, parent=None):
         super().__init__(parent)
@@ -33,20 +33,24 @@ class AddRepoWindow(AddRepoBase, AddRepoUI):
         return out
 
     def run(self):
+        self.saveButton.setEnabled(False)
         if self.validate():
-            self.set_status(self.connection_message)
+            self._set_status(self.connection_message)
             cmd = self.cmd + [self.values['repo_url']]
             thread = BorgThread(cmd, self.values, parent=self)
-            thread.updated.connect(self.set_status)
+            thread.updated.connect(self._set_status)
             thread.result.connect(self.run_result)
-            self.thread = thread
+            self.thread = thread  # Needs to be connected for tests to work.
             self.thread.start()
+        else:
+            self.saveButton.setEnabled(True)
 
-    def set_status(self, text):
+    def _set_status(self, text):
         self.errorText.setText(text)
         self.errorText.repaint()
 
     def run_result(self, result):
+        self.saveButton.setEnabled(True)
         if result['returncode'] == 0:
             self.result = result
             self.accept()
@@ -64,13 +68,20 @@ class AddRepoWindow(AddRepoBase, AddRepoUI):
             self.sshComboBox.addItem(f'{key["filename"]} ({key["format"]}:{key["fingerprint"]})', key['filename'])
 
     def validate(self):
+        """Pre-flight check for valid input and borg binary."""
+
+        # TODO: valid repo is xx.xx:xx. add rex
         if len(self.values['repo_url']) < 5 or ':' not in self.values['repo_url']:
-            self.set_status('Please enter a valid repo URL including hostname and path.')
+            self._set_status('Please enter a valid repo URL including hostname and path.')
+            return False
+
+        if BorgThread.prepare_bin() is None:
+            self._set_status('Borg binary was not found.')
             return False
 
         if self.__class__ == AddRepoWindow:
             if self.values['encryption'] != 'none' and len(self.values['password']) < 8:
-                self.set_status('Please use a longer password.')
+                self._set_status('Please use a longer password.')
                 return False
 
             self.cmd.append(f"--encryption={self.values['encryption']}")
@@ -80,7 +91,7 @@ class AddRepoWindow(AddRepoBase, AddRepoUI):
 
 class ExistingRepoWindow(AddRepoWindow):
     connection_message = 'Validating existing repo...'
-    cmd = ["borg", "list", "--json"]
+    cmd = ["borg", "list", "--info", "--json", "--log-json"]
 
     def __init__(self):
         super().__init__()

src/vorta/views/repo_tab.py

@@ -156,6 +156,7 @@ class RepoTab(RepoBase, RepoUI, BackupProfileMixin):
             self.repoSelector.addItem(new_repo.url, new_repo.id)
             self.repoSelector.setCurrentIndex(self.repoSelector.count()-1)
             self.repo_changed.emit()
+            self.init_repo_stats()
 
     def repo_unlink_action(self):
         profile = self.profile
@@ -177,5 +178,6 @@ class RepoTab(RepoBase, RepoUI, BackupProfileMixin):
             msg.exec_()
 
             self.repo_changed.emit()
+            self.init_repo_stats()
 
 

src/vorta/views/snapshots_tab.py

@@ -2,7 +2,7 @@ from datetime import timedelta
 from PyQt5 import uic
 from PyQt5.QtWidgets import QFileDialog, QTableWidgetItem, QTableView, QHeaderView
 
-from ..borg_thread import BorgThread
+from vorta.borg.borg_thread import BorgThread
 from ..utils import get_asset, keyring, pretty_bytes
 from ..models import BackupProfileMixin
 

tests/fixtures.py

@@ -11,9 +11,3 @@ def app(tmpdir):
     mock_db = peewee.SqliteDatabase(str(tmp_db))
     vorta.models.init_db(mock_db)
     return VortaApp([])
-
-
-@pytest.fixture()
-def main(app):
-    main = app.main_window
-    return main

tests/test_repo.py

@@ -1,25 +1,23 @@
-
-import pytest
 import io
 from PyQt5 import QtCore
 
-import vorta.borg_thread
+import vorta.borg.borg_thread
 import vorta.models
 from vorta.views.repo_add import AddRepoWindow
 from vorta.models import EventLogModel, RepoModel
 
 from .fixtures import *
 
-def test_repo_tab(main, qtbot):
+def test_repo_tab(app, qtbot):
+    main = app.main_window
     qtbot.mouseClick(main.createStartBtn, QtCore.Qt.LeftButton)
     assert main.createProgressText.text() == 'Add a remote backup repository first.'
 
 
-def test_repo_add(main, qtbot, mocker):
+def test_repo_add(app, qtbot, mocker):
     # Add new repo window
-    add_repo_window = AddRepoWindow(main)
-    qtbot.addWidget(add_repo_window)
-    add_repo_window.show()
+    main = app.main_window
+    add_repo_window = AddRepoWindow(main.repoTab)
     qtbot.keyClicks(add_repo_window.repoURL, 'aaa')
     qtbot.mouseClick(add_repo_window.saveButton, QtCore.Qt.LeftButton)
     assert add_repo_window.errorText.text() == 'Please enter a valid repo URL including hostname and path.'
@@ -33,7 +31,7 @@ def test_repo_add(main, qtbot, mocker):
     popen_result =mocker.MagicMock(stdout=io.StringIO("some initial binary data"),
                               stderr=io.StringIO("some initial binary data"),
                               returncode=0)
-    mocker.patch.object(vorta.borg_thread, 'Popen', return_value=popen_result)
+    mocker.patch.object(vorta.borg.borg_thread, 'Popen', return_value=popen_result)
 
     qtbot.mouseClick(add_repo_window.saveButton, QtCore.Qt.LeftButton)
 
@@ -44,3 +42,4 @@ def test_repo_add(main, qtbot, mocker):
 
     assert EventLogModel.select().count() == 1
     assert RepoModel.get(id=1).url == 'aaabbb.com:repo'
+

tests/test_schedule.py

@@ -6,9 +6,8 @@ from vorta.views.schedule_tab import ScheduleTab
 
 from .fixtures import *
 
-def test_schedule_tab(main, qtbot):
-    tab = ScheduleTab(main.scheduleTabSlot)
-    # qtbot.addWidget(tab)
+def test_schedule_tab(app, qtbot):
+    tab = ScheduleTab(app.main_window.scheduleTabSlot)
     qtbot.mouseClick(tab.scheduleApplyButton, QtCore.Qt.LeftButton)
     assert tab.nextBackupDateTimeLabel.text() == 'Manual Backups'