mirror
/
vorta
mirror of https://github.com/borgbase/vorta
1
0
Fork 0
Code Issues Releases Wiki Activity

Display number of files and sizes in source tab. By @XXXBold (#617)

This commit is contained in:
XXXBold 2020-10-30 08:30:52 +00:00 committed by GitHub
parent 5df80610e2
commit 461ea05210
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 285 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
src/vorta/assets/UI/sourcetab.ui
View File

@ -24,7 +24,10 @@
<item>
<layout class="QHBoxLayout" name="horizontalLayout" stretch="0,0">
<item>
<widget class="QListWidget" name="sourceFilesWidget">
<widget class="QTableWidget" name="sourceFilesWidget">
<property name="enabled">
<bool>true</bool>
</property>
<property name="sizePolicy">
<sizepolicy hsizetype="Expanding" vsizetype="Expanding">
<horstretch>0</horstretch>
@ -37,12 +40,50 @@
<height>0</height>
</size>
</property>
<property name="editTriggers">
<set>QAbstractItemView::NoEditTriggers</set>
</property>
<property name="selectionMode">
<enum>QAbstractItemView::ExtendedSelection</enum>
</property>
<property name="sortingEnabled">
<bool>true</bool>
<property name="selectionBehavior">
<enum>QAbstractItemView::SelectRows</enum>
</property>
<property name="showGrid">
<bool>false</bool>
</property>
<attribute name="horizontalHeaderVisible">
<bool>false</bool>
</attribute>
<attribute name="horizontalHeaderHighlightSections">
<bool>false</bool>
</attribute>
<attribute name="verticalHeaderVisible">
<bool>false</bool>
</attribute>
<attribute name="verticalHeaderHighlightSections">
<bool>false</bool>
</attribute>
<column>
<property name="text">
<string>Path</string>
</property>
</column>
<column>
<property name="text">
<string>Type</string>
</property>
</column>
<column>
<property name="text">
<string>Size</string>
</property>
</column>
<column>
<property name="text">
<string>File Count</string>
</property>
</column>
</widget>
</item>
<item>
@ -61,6 +102,13 @@
</property>
</widget>
</item>
<item>
<widget class="QPushButton" name="sourceRemove">
<property name="text">
<string>Remove</string>
</property>
</widget>
</item>
<item>
<widget class="QPushButton" name="paste">
<property name="toolTip">
@ -71,13 +119,26 @@
</property>
</widget>
</item>
<item alignment="Qt::AlignTop">
<widget class="QPushButton" name="sourceRemove">
<item>
<widget class="QPushButton" name="sourcesUpdate">
<property name="text">
<string>Remove</string>
<string>Recalculate sizes</string>
</property>
</widget>
</item>
<item>
<spacer name="verticalSpacer">
<property name="orientation">
<enum>Qt::Vertical</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>20</width>
<height>40</height>
</size>
</property>
</spacer>
</item>
</layout>
</item>
</layout>

21
src/vorta/models.py
View File

@ -16,7 +16,7 @@ from playhouse.migrate import SqliteMigrator, migrate
from vorta.i18n import trans_late
from vorta.utils import slugify
SCHEMA_VERSION = 15
SCHEMA_VERSION = 16
db = pw.Proxy()
@ -106,6 +106,9 @@ class BackupProfileModel(pw.Model):
class SourceFileModel(pw.Model):
"""A folder to be backed up, related to a Backup Configuration."""
dir = pw.CharField()
dir_size = pw.BigIntegerField()
dir_files_count = pw.BigIntegerField()
path_isdir = pw.BooleanField()
profile = pw.ForeignKeyField(BackupProfileModel, default=1)
added_at = pw.DateTimeField(default=datetime.utcnow)
@ -215,6 +218,11 @@ def get_misc_settings():
'label': trans_late('settings',
'Open main window on startup')
},
{
'key': 'get_srcpath_datasize', 'value': True, 'type': 'checkbox',
'label': trans_late('settings',
'Get statistics of file/folder when added')
},
{
'key': 'previous_profile_id', 'str_value': '1', 'type': 'internal',
'label': 'Previously selected profile'
@ -362,6 +370,17 @@ def init_db(con=None):
'dont_run_on_metered_networks', pw.BooleanField(default=True))
)
if current_schema.version < 16:
_apply_schema_update(
current_schema, 16,
migrator.add_column(SourceFileModel._meta.table_name,
'dir_size', pw.BigIntegerField(default=-1)),
migrator.add_column(SourceFileModel._meta.table_name,
'dir_files_count', pw.BigIntegerField(default=-1)),
migrator.add_column(SourceFileModel._meta.table_name,
'path_isdir', pw.BooleanField(default=False))
)
# Create missing settings and update labels. Leave setting values untouched.
for setting in get_misc_settings():
s, created = SettingsModel.get_or_create(key=setting['key'], defaults=setting)

76
src/vorta/utils.py
View File

@ -17,6 +17,7 @@ from paramiko.ecdsakey import ECDSAKey
from paramiko.ed25519key import Ed25519Key
from paramiko.rsakey import RSAKey
from PyQt5 import QtCore
from PyQt5.QtCore import QFileInfo, QThread, pyqtSignal
from PyQt5.QtWidgets import QApplication, QFileDialog, QSystemTrayIcon
from vorta.borg._compatibility import BorgCompatibility
@ -27,11 +28,50 @@ QApplication.setAttribute(QtCore.Qt.AA_EnableHighDpiScaling, True) # enable hig
QApplication.setAttribute(QtCore.Qt.AA_UseHighDpiPixmaps, True) # use highdpi icons
borg_compat = BorgCompatibility()
_network_status_monitor = None
class FilePathInfoAsync(QThread):
signal = pyqtSignal(str, str, str)
def __init__(self, path):
self.path = path
QThread.__init__(self)
self.exiting = False
def run(self):
# logger.info("running thread to get path=%s...", self.path)
self.files_count = 0
self.size, self.files_count = get_path_datasize(self.path)
self.signal.emit(self.path, str(self.size), str(self.files_count))
def get_directory_size(dir_path):
''' Get number of files only and total size in bytes from a path.
Based off https://stackoverflow.com/a/17936789 '''
data_size = 0
seen = set()
for curr_path, _, file_names in os.walk(dir_path):
for file_name in file_names:
file_path = os.path.join(curr_path, file_name)
# Ignore symbolic links, since borg doesn't follow them
if os.path.islink(file_path):
continue
stat = os.stat(file_path)
# Visit each file once
if stat.st_ino not in seen:
seen.add(stat.st_ino)
data_size += stat.st_size
files_count = len(seen)
return data_size, files_count
def get_network_status_monitor():
global _network_status_monitor
if _network_status_monitor is None:
@ -40,6 +80,22 @@ def get_network_status_monitor():
return _network_status_monitor
def get_path_datasize(path):
file_info = QFileInfo(path)
data_size = 0
if file_info.isDir():
data_size, files_count = get_directory_size(file_info.absoluteFilePath())
# logger.info("path (folder) %s %u elements size now=%u (%s)",
# file_info.absoluteFilePath(), files_count, data_size, pretty_bytes(data_size))
else:
# logger.info("path (file) %s size=%u", file_info.path(), file_info.size())
data_size = file_info.size()
files_count = 1
return data_size, files_count
def nested_dict():
"""
Combination of two idioms to quickly build dicts from lists of keys:
@ -98,10 +154,24 @@ def get_private_keys():
return available_private_keys
def sort_sizes(size_list):
""" Sorts sizes with extensions. Assumes that size is already in largest unit possible """
final_list = []
for suffix in [" B", " KB", " MB", " GB", " TB"]:
sub_list = [float(size[:-len(suffix)])
for size in size_list if size.endswith(suffix) and size[:-len(suffix)][-1].isnumeric()]
sub_list.sort()
final_list += [(str(size) + suffix) for size in sub_list]
# Skip additional loops
if len(final_list) == len(size_list):
break
return final_list
def pretty_bytes(size):
"""from https://stackoverflow.com/questions/12523586/
python-format-size-application-converting-b-to-kb-mb-gb-tb/37423778"""
if type(size) != int:
if not isinstance(size, int):
return ''
power = 1000 # GiB is base 2**10, GB is base 10**3.
n = 0

134
src/vorta/views/source_tab.py
View File

@ -1,49 +1,161 @@
from PyQt5 import uic
from ..models import SourceFileModel, BackupProfileMixin
from ..utils import get_asset, choose_file_dialog
from PyQt5.QtWidgets import QApplication, QMessageBox
from ..models import SourceFileModel, BackupProfileMixin, SettingsModel
from ..utils import get_asset, choose_file_dialog, pretty_bytes, sort_sizes, FilePathInfoAsync
from PyQt5 import QtCore
from PyQt5.QtCore import QFileInfo
from PyQt5.QtWidgets import QApplication, QMessageBox, QTableWidgetItem, QHeaderView
import os
uifile = get_asset('UI/sourcetab.ui')
SourceUI, SourceBase = uic.loadUiType(uifile)
class SourceColumn:
Path = 0
Type = 1
Size = 2
FilesCount = 3
class SizeItem(QTableWidgetItem):
def __lt__(self, other):
return sort_sizes([self.text(), other.text()]) == [self.text(), other.text()]
class FilesCount(QTableWidgetItem):
def __lt__(self, other):
# Verify that conversion is only performed on valid integers
# If one of the 2 elements is no number, put these elements at the end
# This is important if the text is "Calculating..." or ""
if self.text().isdigit() and other.text().isdigit():
return int(self.text()) < int(other.text()) # Compare & return result
else:
if not self.text().isdigit():
return 1 # Move one down if current item has no valid count
if not other.text().isdigit():
return 0
class SourceTab(SourceBase, SourceUI, BackupProfileMixin):
updateThreads = []
def __init__(self, parent=None):
super().__init__(parent)
self.setupUi(parent)
header = self.sourceFilesWidget.horizontalHeader()
header.setVisible(True)
header.setSortIndicatorShown(1)
header.setSectionResizeMode(SourceColumn.Path, QHeaderView.Stretch)
header.setSectionResizeMode(SourceColumn.Type, QHeaderView.ResizeToContents)
header.setSectionResizeMode(SourceColumn.Size, QHeaderView.ResizeToContents)
header.setSectionResizeMode(SourceColumn.FilesCount, QHeaderView.ResizeToContents)
self.sourceFilesWidget.setSortingEnabled(True)
self.sourceAddFolder.clicked.connect(lambda: self.source_add(want_folder=True))
self.sourceAddFile.clicked.connect(lambda: self.source_add(want_folder=False))
self.sourceRemove.clicked.connect(self.source_remove)
self.sourcesUpdate.clicked.connect(self.sources_update)
self.paste.clicked.connect(self.paste_text)
self.excludePatternsField.textChanged.connect(self.save_exclude_patterns)
self.excludeIfPresentField.textChanged.connect(self.save_exclude_if_present)
self.populate_from_profile()
def set_path_info(self, path, data_size, files_count):
items = self.sourceFilesWidget.findItems(path, QtCore.Qt.MatchExactly)
# Conversion int->str->int needed because QT limits int to 32-bit
data_size = int(data_size)
files_count = int(files_count)
for item in items:
db_item = SourceFileModel.get(dir=path)
if QFileInfo(path).isDir():
self.sourceFilesWidget.item(item.row(), SourceColumn.Type).setText(self.tr("Folder"))
self.sourceFilesWidget.item(item.row(), SourceColumn.FilesCount).setText(format(files_count))
db_item.path_isdir = True
else:
self.sourceFilesWidget.item(item.row(), SourceColumn.Type).setText(self.tr("File"))
# No files count, if entry itself is a file
self.sourceFilesWidget.item(item.row(), SourceColumn.FilesCount).setText("")
db_item.path_isdir = False
self.sourceFilesWidget.item(item.row(), SourceColumn.Size).setText(pretty_bytes(data_size))
db_item.dir_size = data_size
db_item.dir_files_count = files_count
db_item.save()
# Remove thread from list when it's done
for thrd in self.updateThreads:
if thrd.objectName() == path:
self.updateThreads.remove(thrd)
def update_path_info(self, index_row):
path = self.sourceFilesWidget.item(index_row, SourceColumn.Path).text()
self.sourceFilesWidget.item(index_row, SourceColumn.Type).setText(self.tr("Calculating..."))
self.sourceFilesWidget.item(index_row, SourceColumn.Size).setText(self.tr("Calculating..."))
self.sourceFilesWidget.item(index_row, SourceColumn.FilesCount).setText(self.tr("Calculating..."))
getDir = FilePathInfoAsync(path)
getDir.signal.connect(self.set_path_info)
getDir.setObjectName(path)
self.updateThreads.append(getDir) # this is ugly, is there a better way to keep the thread object?
getDir.start()
def add_source_to_table(self, source, update_data):
index_row = self.sourceFilesWidget.rowCount()
self.sourceFilesWidget.insertRow(index_row)
# Insert all items on current row
self.sourceFilesWidget.setItem(index_row, SourceColumn.Path, QTableWidgetItem(source.dir))
self.sourceFilesWidget.setItem(index_row, SourceColumn.Type, QTableWidgetItem(""))
self.sourceFilesWidget.setItem(index_row, SourceColumn.Size, SizeItem(""))
self.sourceFilesWidget.setItem(index_row, SourceColumn.FilesCount, FilesCount(""))
if update_data:
self.update_path_info(index_row)
else: # Use cached data from DB
if source.dir_size > -1:
self.sourceFilesWidget.item(index_row, SourceColumn.Size).setText(pretty_bytes(source.dir_size))
if source.path_isdir:
self.sourceFilesWidget.item(index_row, SourceColumn.Type).setText(self.tr("Folder"))
self.sourceFilesWidget.item(index_row,
SourceColumn.FilesCount).setText(format(source.dir_files_count))
else:
self.sourceFilesWidget.item(index_row, SourceColumn.Type).setText(self.tr("File"))
def populate_from_profile(self):
profile = self.profile()
self.excludePatternsField.textChanged.disconnect()
self.excludeIfPresentField.textChanged.disconnect()
self.sourceFilesWidget.clear()
self.sourceFilesWidget.setRowCount(0) # Clear rows
self.excludePatternsField.clear()
self.excludeIfPresentField.clear()
for source in SourceFileModel.select().where(SourceFileModel.profile == profile):
self.sourceFilesWidget.addItem(source.dir)
self.add_source_to_table(source, False)
# Initially, sort entries by path name in ascending order
self.sourceFilesWidget.model().sort(SourceColumn.Path, QtCore.Qt.AscendingOrder)
self.excludePatternsField.appendPlainText(profile.exclude_patterns)
self.excludeIfPresentField.appendPlainText(profile.exclude_if_present)
self.excludePatternsField.textChanged.connect(self.save_exclude_patterns)
self.excludeIfPresentField.textChanged.connect(self.save_exclude_if_present)
def sources_update(self):
for row in range(0, self.sourceFilesWidget.rowCount()):
self.update_path_info(row) # Update data for each entry
def source_add(self, want_folder):
def receive():
dirs = dialog.selectedFiles()
for dir in dirs:
new_source, created = SourceFileModel.get_or_create(dir=dir, profile=self.profile())
new_source, created = SourceFileModel.get_or_create(dir=dir,
dir_size=-1,
dir_files_count=-1,
path_isdir=False,
profile=self.profile())
if created:
self.sourceFilesWidget.addItem(dir)
self.add_source_to_table(new_source, SettingsModel.get(key="get_srcpath_datasize").value)
new_source.save()
msg = self.tr("Choose directory to back up") if want_folder else self.tr("Choose file(s) to back up")
@ -51,14 +163,14 @@ class SourceTab(SourceBase, SourceUI, BackupProfileMixin):
dialog.open(receive)
def source_remove(self):
indexes = self.sourceFilesWidget.selectionModel().selectedIndexes()
indexes = self.sourceFilesWidget.selectionModel().selectedRows()
# sort indexes, starting with lowest
indexes.sort()
# remove each selected entry, starting with highest index (otherways, higher indexes become invalid)
# remove each selected row, starting with highest index (otherways, higher indexes become invalid)
for index in reversed(indexes):
item = self.sourceFilesWidget.takeItem(index.row())
db_item = SourceFileModel.get(dir=item.text())
db_item = SourceFileModel.get(dir=self.sourceFilesWidget.item(index.row(), SourceColumn.Path).text())
db_item.delete_instance()
self.sourceFilesWidget.removeRow(index.row())
def save_exclude_patterns(self):
profile = self.profile()

2
tests/conftest.py
View File

@ -37,7 +37,7 @@ def init_db(qapp):
test_archive1 = ArchiveModel(snapshot_id='99998', name='test-archive1', time=dt(2000, 1, 1, 0, 0), repo=1)
test_archive1.save()
source_dir = SourceFileModel(dir='/tmp/another', repo=new_repo)
source_dir = SourceFileModel(dir='/tmp/another', repo=new_repo, dir_size=100, dir_files_count=18, path_isdir=True)
source_dir.save()
qapp.main_window = MainWindow(qapp) # Re-open main window to apply mock data in UI

2
tests/test_source.py
View File

@ -11,4 +11,4 @@ def test_add_folder(qapp, qtbot, tmpdir, monkeypatch, choose_file_dialog):
tab = main.sourceTab
tab.sourceAddFolder.click()
qtbot.waitUntil(lambda: tab.sourceFilesWidget.count() == 2, timeout=5000)
qtbot.waitUntil(lambda: tab.sourceFilesWidget.rowCount() == 2, timeout=5000)