From 1d0a6b5f454bb0a75723900db1fa6b1d29b505c8 Mon Sep 17 00:00:00 2001 From: Manu Date: Sat, 27 Oct 2018 18:12:30 +0800 Subject: [PATCH] Working version with snapshots and ssh creation. --- README.md | 9 ++- environment.yml | 1 + vorta/UI/mainwindow.ui | 135 ++++++++++++++++++------------- vorta/borg_runner.py | 4 +- vorta/main.py | 177 +++++++++++++++++++++++++++++++---------- vorta/models.py | 15 +++- vorta/repo_init.py | 1 + vorta/utils.py | 13 +++ 8 files changed, 249 insertions(+), 106 deletions(-) create mode 100644 vorta/utils.py diff --git a/README.md b/README.md index eacc3e5b..e9666cfe 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,10 @@ - Select and manage SSH keys - Initialize new remote Borg repositories - Create new Borg snapshots (backups) from local folders -- Display existing snapshots and repository details. +- Mount existing snapshots with FUSE - Settings stored in sqlite -- Borg binary integrated -Planned features: +Missing features: - Scheduling for background backups. - Rule-based scheduling by time, Wifi SSID, etc. @@ -18,6 +17,7 @@ Planned features: - Securely save repo password in Keychain instead of database. - Handle encrypted SSH keys - Check for duplicate source dirs +- Exclude options/patterns. ## Development Conda is used for dependency management. Create a new virtual env using: @@ -36,6 +36,9 @@ To build a binary package: $ pyinstaller --clean --noconfirm vorta.spec ``` +## Why the Name? +[Vorta](http://memory-alpha.wikia.com/wiki/Vorta) are a race referenced in Star Trek. They serve the Dominion and are replaced by their clones if they die. Just like our backups. + ## Author (C) 2018 Manuel Riel for [BorgBase.com](https://www.borgbase.com) diff --git a/environment.yml b/environment.yml index 28c158af..f875f678 100644 --- a/environment.yml +++ b/environment.yml @@ -12,6 +12,7 @@ dependencies: - paramiko - pyqt5 - peewee + - python-dateutil dev-dependencies: - pyinstaller diff --git a/vorta/UI/mainwindow.ui b/vorta/UI/mainwindow.ui index f8580d69..8d41186b 100644 --- a/vorta/UI/mainwindow.ui +++ b/vorta/UI/mainwindow.ui @@ -39,7 +39,7 @@ - 2 + 0 @@ -81,104 +81,82 @@ 25 - + - Repository ID: + Encryption: - - - - 0 - 0 - - - - - - - - - - - Location: - - - - - + - - - Encrypted: + + + + 0 + 0 + + + + Qt::Horizontal - - - - - - - - - - Qt::Horizontal - - - - + + + 0 + 0 + + Qt::Horizontal - + Original Size: - - + + - Compressed Size: + - + Deduplicated Size: - - + + - + Compressed Size: - + - - + + @@ -385,7 +363,10 @@ - + + + true + <html><head/><body><p>To mount snapshots, first install &quot;FUSE for macOS&quot; from <a href="https://osxfuse.github.io/"><span style=" text-decoration: underline; color:#0000ff;">here</span></a>.</p></body></html> @@ -394,7 +375,7 @@ - + 0 @@ -407,7 +388,7 @@ - + false @@ -423,7 +404,10 @@ - + + + false + 0 @@ -449,6 +433,43 @@ true + + + + + + + 255 + 151 + 41 + + + + + + + + + 255 + 151 + 41 + + + + + + + + + 220 + 220 + 220 + + + + + + 0 @@ -497,7 +518,7 @@ - Backup Configuration + Backup Profile diff --git a/vorta/borg_runner.py b/vorta/borg_runner.py index e0e93c77..44279e03 100644 --- a/vorta/borg_runner.py +++ b/vorta/borg_runner.py @@ -43,7 +43,9 @@ class BorgThread(QtCore.QThread): 'params': self.params, 'returncode': p.returncode, } - if stdout.strip(): + try: result['data'] = json.loads(stdout) + except: + result['data'] = {} self.result.emit(result) diff --git a/vorta/main.py b/vorta/main.py index 83746b49..a06dc54a 100644 --- a/vorta/main.py +++ b/vorta/main.py @@ -1,17 +1,17 @@ -import sys import os import platform from datetime import datetime as dt -from PyQt5.QtWidgets import QApplication, QFileDialog, QTableWidgetItem, QTableView -from PyQt5 import uic, QtCore, QtGui +from dateutil import parser +from PyQt5.QtWidgets import QApplication, QFileDialog, QTableWidgetItem, QTableView, QTableWidget +from PyQt5 import uic, QtCore, QtGui, QtWidgets from .repo_add import AddRepoWindow, ExistingRepoWindow -from .repo_init import InitRepoWindow from .ssh_add import SSHAddWindow from .config import APP_NAME, reset_app from .models import RepoModel, SourceDirModel, SnapshotModel, BackupProfileModel from .ssh_keys import get_private_keys from .borg_runner import BorgThread +from .utils import prettyByes uifile = os.path.join(os.path.dirname(__file__), 'UI/mainwindow.ui') @@ -33,44 +33,59 @@ class MainWindow(MainWindowBase, MainWindowUI): self.createStartBtn.clicked.connect(self.create_action) self.actionResetApp.triggered.connect(self.menu_reset) + def set_status(self, text=None, progress_max=None): + if text: + self.createProgressText.setText(text) + if progress_max is not None: + self.createProgress.setRange(0, progress_max) + self.createProgressText.repaint() + def create_action(self): + n_backup_folders = self.sourceDirectoriesWidget.count() + if n_backup_folders == 0: + self.set_status('Add some folders to back up first.') + return + self.set_status('Starting Backup.', progress_max=0) + self.createStartBtn.setEnabled(False) + self.createStartBtn.repaint() + repo_id = self.repoSelector.currentData() repo = RepoModel.get(id=repo_id) - cmd = ['borg', 'create', '--log-json', '--json', + cmd = ['borg', 'create', '--log-json', '--json', '-C', self.profile.compression, f'{repo.url}::{platform.node()}-{dt.now().isoformat()}' ] - for i in range(self.sourceDirectoriesWidget.count()): + for i in range(n_backup_folders): cmd.append(self.sourceDirectoriesWidget.item(i).text()) - self.createProgress.setRange(0, 0) + thread = BorgThread(self, cmd, {}) thread.updated.connect(self.create_update_log) thread.result.connect(self.create_get_result) thread.start() def create_update_log(self, text): - self.createProgressText.setText(text) + self.set_status(text) def create_get_result(self, result): - self.createProgress.setRange(0, 100) - print(result) + self.createStartBtn.setEnabled(True) + self.createStartBtn.repaint() if result['returncode'] == 0: - self.createProgressText.setText('Finished backup.') + self.set_status('Finished backup.', 100) new_snapshot = SnapshotModel( + snapshot_id=result['data']['archive']['id'], name=result['data']['archive']['name'], - time=result['data']['archive']['start'], + time=parser.parse(result['data']['archive']['start']), repo=self.repoSelector.currentData() ) - - def init_repo(self): - self.repoSelector.model().item(0).setEnabled(False) - self.repoSelector.addItem('Initialize New Repository', 'init') - self.repoSelector.addItem('Add Existing Repository', 'existing') - for repo in RepoModel.select(): - self.repoSelector.addItem(repo.url, repo.id) - - if self.profile.repo: - self.repoSelector.setCurrentIndex(self.repoSelector.findData(self.profile.repo.id)) - self.repoSelector.currentIndexChanged.connect(self.repo_select_action) + new_snapshot.save() + if 'cache' in result['data']: + stats = result['data']['cache']['stats'] + repo = self.profile.repo + repo.total_size = stats['total_size'] + repo.unique_csize = stats['unique_csize'] + repo.unique_size = stats['unique_size'] + repo.total_unique_chunks = stats['total_unique_chunks'] + repo.save() + self.init_snapshots() def init_source(self): self.sourceAdd.clicked.connect(self.source_add) @@ -96,24 +111,90 @@ class MainWindow(MainWindowBase, MainWindowUI): self.profile.save() print('set ssh key to', self.profile.ssh_key) - def init_snapshots(self): - snapshots = [s for s in SnapshotModel.select()] - self.snapshotTable.setRowCount(len(snapshots)) + if self.profile.repo: + snapshots = [s for s in self.profile.repo.snapshots.select()] + + for row, snapshot in enumerate(snapshots): + self.snapshotTable.insertRow(row) + self.snapshotTable.setItem(row, 0, QTableWidgetItem(snapshot.name)) + formatted_time = snapshot.time.strftime('%Y-%m-%d %H:%M') + self.snapshotTable.setItem(row, 1, QTableWidgetItem(formatted_time)) + + self.sizeCompressed.setText(prettyByes(self.profile.repo.unique_csize)) + self.sizeDeduplicated.setText(prettyByes(self.profile.repo.unique_size)) + self.sizeOriginal.setText(prettyByes(self.profile.repo.total_size)) + self.repoEncryption.setText(str(self.profile.repo.encryption)) + self.snapshotTable.setRowCount(len(snapshots)) + + header = self.snapshotTable.horizontalHeader() + header.setVisible(True) + header.setSectionResizeMode(0, QtWidgets.QHeaderView.Stretch) + header.setSectionResizeMode(1, QtWidgets.QHeaderView.ResizeToContents) - for row, snapshot in enumerate(snapshots): - self.snapshotTable.insertRow(row) - self.snapshotTable.setItem(row, 0, QTableWidgetItem(snapshot.name)) - self.snapshotTable.setItem(row, 1, QTableWidgetItem(snapshot.time)) - self.snapshotTable.resizeColumnsToContents() - self.snapshotTable.horizontalHeader().setVisible(True) self.snapshotTable.setSelectionBehavior(QTableView.SelectRows) self.snapshotTable.setEditTriggers(QTableView.NoEditTriggers) + self.snapshotMountButton.clicked.connect(self.snapshot_mount) + self.snapshotDeleteButton.clicked.connect(self.snapshot_mount) + self.snapshotRefreshButton.clicked.connect(self.snapshot_mount) + + def snapshot_mount(self): + cmd = ['borg', 'mount', '--log-json'] + row_selected = self.snapshotTable.selectionModel().selectedRows() + if row_selected: + snapshot_cell = self.snapshotTable.item(row_selected[0].row(), 0) + if snapshot_cell: + snapshot_name = snapshot_cell.text() + cmd.append(f'{self.profile.repo.url}::{snapshot_name}') + else: + cmd.append(f'{self.profile.repo.url}') + else: + cmd.append(f'{self.profile.repo.url}') + + options = QFileDialog.Options() + options |= QFileDialog.ShowDirsOnly + options |= QFileDialog.DontUseNativeDialog + mountPoint = QFileDialog.getExistingDirectory( + self, "Choose Mount Point", "", options=options) + if mountPoint: + cmd.append(mountPoint) + + self.set_status('Mounting snapshot into folder', 0) + thread = BorgThread(self, cmd, {}) + thread.updated.connect(self.mount_update_log) + thread.result.connect(self.mount_get_result) + thread.start() + + def mount_update_log(self, text): + self.mountErrors.setText(text) + + def mount_get_result(self, result): + self.set_status(progress_max=100) + if result['returncode'] == 0: + self.set_status('Mounted successfully.') + def init_compression(self): self.repoCompression.addItem('LZ4 (default)', 'lz4') self.repoCompression.addItem('Zstandard (medium)', 'zstd') self.repoCompression.addItem('LZMA (high)', 'lzma,6') + self.repoCompression.setCurrentIndex(self.repoCompression.findData(self.profile.compression)) + self.repoCompression.currentIndexChanged.connect(self.compression_select_action) + + def compression_select_action(self, index): + self.profile.compression = self.repoCompression.currentData() + self.profile.save() + + def init_repo(self): + self.repoSelector.model().item(0).setEnabled(False) + self.repoSelector.addItem('Initialize New Repository', 'init') + self.repoSelector.addItem('Add Existing Repository', 'existing') + for repo in RepoModel.select(): + self.repoSelector.addItem(repo.url, repo.id) + + if self.profile.repo: + self.repoSelector.setCurrentIndex(self.repoSelector.findData(self.profile.repo.id)) + self.repoSelector.currentIndexChanged.connect(self.repo_select_action) def repo_select_action(self, index): if index <= 2: @@ -132,40 +213,54 @@ class MainWindow(MainWindowBase, MainWindowUI): else: cmd = ["borg", "list", "--json", params['repo_url']] - init_window = InitRepoWindow(cmd, params) - init_window.setParent(self, QtCore.Qt.Sheet) - init_window.show() - init_window._thread.start() - init_window._thread.result.connect(self.repo_add_result) + self.set_status('Connecting to repo...', 0) + thread = BorgThread(self, cmd, params) + thread.updated.connect(self.repo_add_update_log) + thread.result.connect(self.repo_add_result) + thread.start() else: self.profile.repo = self.repoSelector.currentData() self.profile.save() + self.init_snapshots() + + def repo_add_update_log(self, text): + self.set_status(text) def repo_add_result(self, result): - print(result) if result['returncode'] == 0: + self.set_status('Successfully connected to repo.', 100) new_repo, _ = RepoModel.get_or_create( url=result['params']['repo_url'], defaults={ 'password': result['params']['password'], - 'encryption': 'na' + # 'encryption': result['params'].get('encryption', '') } ) + if 'cache' in result['data']: + stats = result['data']['cache']['stats'] + new_repo.total_size = stats['total_size'] + new_repo.unique_csize = stats['unique_csize'] + new_repo.unique_size = stats['unique_size'] + new_repo.total_unique_chunks = stats['total_unique_chunks'] + new_repo.encryption = result['data']['encryption']['mode'] new_repo.save() + self.profile.repo = new_repo.id + self.profile.save() if 'archives' in result['data'].keys(): for snapshot in result['data']['archives']: new_snapshot, _ = SnapshotModel.get_or_create( - id=snapshot['id'], + snapshot_id=snapshot['id'], defaults={ 'repo': new_repo.id, 'name': snapshot['name'], - 'time': snapshot['time'] + 'time': parser.parse(snapshot['time']) } ) new_snapshot.save() self.repoSelector.addItem(new_repo.url, new_repo.id) self.repoSelector.setCurrentIndex(self.repoSelector.count()-1) + self.init_snapshots() def source_add(self): options = QFileDialog.Options() diff --git a/vorta/models.py b/vorta/models.py index 5fdaee09..c51ae786 100644 --- a/vorta/models.py +++ b/vorta/models.py @@ -11,7 +11,11 @@ class RepoModel(peewee.Model): url = peewee.CharField(unique=True) password = peewee.CharField() added_at = peewee.DateTimeField(default=datetime.utcnow) - encryption = peewee.CharField() + encryption = peewee.CharField(null=True) + unique_size = peewee.IntegerField(null=True) + unique_csize = peewee.IntegerField(null=True) + total_size = peewee.IntegerField(null=True) + total_unique_chunks = peewee.IntegerField(null=True) class Meta: database = db @@ -23,7 +27,7 @@ class BackupProfileModel(peewee.Model): added_at = peewee.DateTimeField(default=datetime.utcnow) repo = peewee.ForeignKeyField(RepoModel, default=None, null=True) ssh_key = peewee.CharField(default=None, null=True) - + compression = peewee.CharField(default='lz4') class Meta: database = db @@ -40,11 +44,14 @@ class SourceDirModel(peewee.Model): class SnapshotModel(peewee.Model): """A snapshot to a specific remote repository.""" - id = peewee.CharField(primary_key=True) + snapshot_id = peewee.CharField(unique=True) name = peewee.CharField() - repo = peewee.ForeignKeyField(RepoModel) + repo = peewee.ForeignKeyField(RepoModel, backref='snapshots') time = peewee.DateTimeField() + def formatted_time(self): + return + class Meta: database = db diff --git a/vorta/repo_init.py b/vorta/repo_init.py index d7256054..b92d2eae 100644 --- a/vorta/repo_init.py +++ b/vorta/repo_init.py @@ -12,6 +12,7 @@ class InitRepoWindow(InitRepoBase, InitRepoUI): self.setupUi(self) self.closeButton.clicked.connect(self.close) + self.logText.appendPlainText('Connecting to repo...') self._thread = BorgThread(self, cmd, env) self._thread.updated.connect(self.update_log) self._thread.result.connect(self.get_result) diff --git a/vorta/utils.py b/vorta/utils.py new file mode 100644 index 00000000..bd5d6011 --- /dev/null +++ b/vorta/utils.py @@ -0,0 +1,13 @@ + +def prettyByes(size): + """from https://stackoverflow.com/questions/12523586/ + python-format-size-application-converting-b-to-kb-mb-gb-tb/37423778""" + if type(size) != int: + return '' + power = 2**10 + n = 0 + Dic_powerN = {0: '', 1: 'K', 2: 'M', 3: 'G', 4: 'T'} + while size > power: + size /= power + n += 1 + return str(round(size))+Dic_powerN[n]+'B'