mirror of https://github.com/borgbase/vorta
Move slow code to async QThread workers (WIP)
This commit is contained in:
Manu
Manu
parent
d721011c90
commit
61fe1f3a63
|
|
@ -12,8 +12,8 @@ from enum import Enum
|
|||
import peewee as pw
|
||||
from playhouse import signals
|
||||
|
||||
from vorta.utils import slugify
|
||||
from vorta.views.utils import get_exclusion_presets
|
||||
from vorta.store.utils import slugify
|
||||
# from vorta.views.utils import get_exclusion_presets
|
||||
|
||||
DB = pw.Proxy()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
import unicodedata
|
||||
import re
|
||||
|
||||
def slugify(value):
|
||||
"""
|
||||
Converts to lowercase, removes non-word characters (alphanumerics and
|
||||
underscores) and converts spaces to hyphens. Also strips leading and
|
||||
trailing whitespace.
|
||||
|
||||
Copied from Django.
|
||||
"""
|
||||
value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii')
|
||||
value = re.sub(r'[^\w\s-]', '', value).strip().lower()
|
||||
return re.sub(r'[-\s]+', '-', value)
|
||||
|
|
@ -1,155 +1,33 @@
|
|||
import argparse
|
||||
import errno
|
||||
import fnmatch
|
||||
import getpass
|
||||
import math
|
||||
import getpass
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import sys
|
||||
import unicodedata
|
||||
from datetime import datetime as dt
|
||||
from datetime import datetime as dt, timedelta
|
||||
from functools import reduce
|
||||
from typing import Any, Callable, Iterable, List, Optional, Tuple, TypeVar
|
||||
|
||||
import psutil
|
||||
from PyQt6 import QtCore
|
||||
from PyQt6.QtCore import QFileInfo, QThread, pyqtSignal
|
||||
from PyQt6.QtWidgets import QApplication, QFileDialog, QSystemTrayIcon
|
||||
from PyQt6.QtCore import QFileInfo, QThread, pyqtSignal, Qt
|
||||
from PyQt6.QtWidgets import (QApplication, QFileDialog, QSystemTrayIcon,
|
||||
QListWidgetItem, QTableWidgetItem)
|
||||
|
||||
from vorta.network_status.abc import NetworkStatusMonitor
|
||||
from vorta.borg._compatibility import BorgCompatibility
|
||||
from vorta.log import logger
|
||||
from vorta.network_status.abc import NetworkStatusMonitor
|
||||
|
||||
|
||||
# Used to store whether a user wanted to override the
|
||||
# default directory for the --development flag
|
||||
DEFAULT_DIR_FLAG = object()
|
||||
METRIC_UNITS = ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']
|
||||
NONMETRIC_UNITS = ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi']
|
||||
|
||||
borg_compat = BorgCompatibility()
|
||||
_network_status_monitor = None
|
||||
|
||||
|
||||
class FilePathInfoAsync(QThread):
|
||||
signal = pyqtSignal(str, str, str)
|
||||
|
||||
def __init__(self, path, exclude_patterns_str):
|
||||
self.path = path
|
||||
QThread.__init__(self)
|
||||
self.exiting = False
|
||||
self.exclude_patterns = []
|
||||
for _line in (exclude_patterns_str or '').splitlines():
|
||||
line = _line.strip()
|
||||
if line != '':
|
||||
self.exclude_patterns.append(line)
|
||||
|
||||
def run(self):
|
||||
# logger.info("running thread to get path=%s...", self.path)
|
||||
self.size, self.files_count = get_path_datasize(self.path, self.exclude_patterns)
|
||||
self.signal.emit(self.path, str(self.size), str(self.files_count))
|
||||
|
||||
|
||||
def normalize_path(path):
|
||||
"""normalize paths for MacOS (but do nothing on other platforms)"""
|
||||
# HFS+ converts paths to a canonical form, so users shouldn't be required to enter an exact match.
|
||||
# Windows and Unix filesystems allow different forms, so users always have to enter an exact match.
|
||||
return unicodedata.normalize('NFD', path) if sys.platform == 'darwin' else path
|
||||
|
||||
|
||||
# prepare patterns as borg does
|
||||
# see `FnmatchPattern._prepare` at
|
||||
# https://github.com/borgbackup/borg/blob/master//src/borg/patterns.py
|
||||
def prepare_pattern(pattern):
|
||||
"""Prepare and process fnmatch patterns as borg does"""
|
||||
if pattern.endswith(os.path.sep):
|
||||
# trailing sep indicates that the contents should be excluded
|
||||
# but not the directory it self.
|
||||
pattern = os.path.normpath(pattern).rstrip(os.path.sep)
|
||||
pattern += os.path.sep + '*' + os.path.sep
|
||||
else:
|
||||
pattern = os.path.normpath(pattern) + os.path.sep + '*'
|
||||
|
||||
pattern = pattern.lstrip(os.path.sep) # sep at beginning is removed
|
||||
return re.compile(fnmatch.translate(pattern))
|
||||
|
||||
|
||||
def match(pattern: re.Pattern, path: str):
|
||||
"""Check whether a path matches the given pattern."""
|
||||
path = path.lstrip(os.path.sep) + os.path.sep
|
||||
return pattern.match(path) is not None
|
||||
|
||||
|
||||
def get_directory_size(dir_path, exclude_patterns):
|
||||
'''Get number of files only and total size in bytes from a path.
|
||||
Based off https://stackoverflow.com/a/17936789'''
|
||||
exclude_patterns = [prepare_pattern(p) for p in exclude_patterns]
|
||||
|
||||
data_size_filtered = 0
|
||||
seen = set()
|
||||
seen_filtered = set()
|
||||
|
||||
for dir_path, subdirectories, file_names in os.walk(dir_path, topdown=True):
|
||||
is_excluded = False
|
||||
for pattern in exclude_patterns:
|
||||
if match(pattern, dir_path):
|
||||
is_excluded = True
|
||||
break
|
||||
|
||||
if is_excluded:
|
||||
subdirectories.clear() # so that os.walk won't walk them
|
||||
continue
|
||||
|
||||
for file_name in file_names:
|
||||
file_path = os.path.join(dir_path, file_name)
|
||||
|
||||
# Ignore symbolic links, since borg doesn't follow them
|
||||
if os.path.islink(file_path):
|
||||
continue
|
||||
|
||||
is_excluded = False
|
||||
for pattern in exclude_patterns:
|
||||
if match(pattern, file_path):
|
||||
is_excluded = True
|
||||
break
|
||||
|
||||
try:
|
||||
stat = os.stat(file_path)
|
||||
if stat.st_ino not in seen: # Visit each file only once
|
||||
# this won't add the size of a hardlinked file
|
||||
seen.add(stat.st_ino)
|
||||
if not is_excluded:
|
||||
data_size_filtered += stat.st_size
|
||||
seen_filtered.add(stat.st_ino)
|
||||
except (FileNotFoundError, PermissionError):
|
||||
continue
|
||||
|
||||
files_count_filtered = len(seen_filtered)
|
||||
|
||||
return data_size_filtered, files_count_filtered
|
||||
|
||||
|
||||
def get_network_status_monitor():
|
||||
global _network_status_monitor
|
||||
if _network_status_monitor is None:
|
||||
_network_status_monitor = NetworkStatusMonitor.get_network_status_monitor()
|
||||
logger.info(
|
||||
'Using %s NetworkStatusMonitor implementation.',
|
||||
_network_status_monitor.__class__.__name__,
|
||||
)
|
||||
return _network_status_monitor
|
||||
|
||||
|
||||
def get_path_datasize(path, exclude_patterns):
|
||||
file_info = QFileInfo(path)
|
||||
|
||||
if file_info.isDir():
|
||||
data_size, files_count = get_directory_size(file_info.absoluteFilePath(), exclude_patterns)
|
||||
else:
|
||||
data_size = file_info.size()
|
||||
files_count = 1
|
||||
|
||||
return data_size, files_count
|
||||
borg_compat = BorgCompatibility()
|
||||
|
||||
|
||||
def nested_dict():
|
||||
|
|
@ -220,22 +98,6 @@ def get_private_keys() -> List[str]:
|
|||
return available_private_keys
|
||||
|
||||
|
||||
def sort_sizes(size_list):
|
||||
"""Sorts sizes with extensions. Assumes that size is already in largest unit possible"""
|
||||
final_list = []
|
||||
for suffix in [" B", " KB", " MB", " GB", " TB", " PB", " EB", " ZB", " YB"]:
|
||||
sub_list = [
|
||||
float(size[: -len(suffix)])
|
||||
for size in size_list
|
||||
if size.endswith(suffix) and size[: -len(suffix)][-1].isnumeric()
|
||||
]
|
||||
sub_list.sort()
|
||||
final_list += [(str(size) + suffix) for size in sub_list]
|
||||
# Skip additional loops
|
||||
if len(final_list) == len(size_list):
|
||||
break
|
||||
return final_list
|
||||
|
||||
|
||||
Number = TypeVar("Number", int, float)
|
||||
|
||||
|
|
@ -244,6 +106,16 @@ def clamp(n: Number, min_: Number, max_: Number) -> Number:
|
|||
"""Restrict the number n inside a range"""
|
||||
return min(max_, max(n, min_))
|
||||
|
||||
def get_network_status_monitor():
|
||||
global _network_status_monitor
|
||||
if _network_status_monitor is None:
|
||||
_network_status_monitor = NetworkStatusMonitor.get_network_status_monitor()
|
||||
logger.info(
|
||||
'Using %s NetworkStatusMonitor implementation.',
|
||||
_network_status_monitor.__class__.__name__,
|
||||
)
|
||||
return _network_status_monitor
|
||||
|
||||
|
||||
def find_best_unit_for_sizes(sizes: Iterable[int], metric: bool = True, precision: int = 1) -> int:
|
||||
"""
|
||||
|
|
@ -303,37 +175,6 @@ def get_asset(path):
|
|||
return os.path.join(bundle_dir, path)
|
||||
|
||||
|
||||
def get_sorted_wifis(profile):
|
||||
"""
|
||||
Get Wifi networks known to the OS (only current one on macOS) and
|
||||
merge with networks from other profiles. Update last connected time.
|
||||
"""
|
||||
|
||||
from vorta.store.models import WifiSettingModel
|
||||
|
||||
# Pull networks known to OS and all other backup profiles
|
||||
system_wifis = get_network_status_monitor().get_known_wifis()
|
||||
from_other_profiles = WifiSettingModel.select().where(WifiSettingModel.profile != profile.id).execute()
|
||||
|
||||
for wifi in list(from_other_profiles) + system_wifis:
|
||||
db_wifi, created = WifiSettingModel.get_or_create(
|
||||
ssid=wifi.ssid,
|
||||
profile=profile.id,
|
||||
defaults={'last_connected': wifi.last_connected, 'allowed': True},
|
||||
)
|
||||
|
||||
# Update last connected time
|
||||
if not created and db_wifi.last_connected != wifi.last_connected:
|
||||
db_wifi.last_connected = wifi.last_connected
|
||||
db_wifi.save()
|
||||
|
||||
# Finally return list of networks and settings for that profile
|
||||
return (
|
||||
WifiSettingModel.select()
|
||||
.where(WifiSettingModel.profile == profile.id)
|
||||
.order_by(-WifiSettingModel.last_connected)
|
||||
)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description='Vorta Backup GUI for Borg.')
|
||||
|
|
@ -368,19 +209,6 @@ def parse_args():
|
|||
return parser.parse_known_args()[0]
|
||||
|
||||
|
||||
def slugify(value):
|
||||
"""
|
||||
Converts to lowercase, removes non-word characters (alphanumerics and
|
||||
underscores) and converts spaces to hyphens. Also strips leading and
|
||||
trailing whitespace.
|
||||
|
||||
Copied from Django.
|
||||
"""
|
||||
value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii')
|
||||
value = re.sub(r'[^\w\s-]', '', value).strip().lower()
|
||||
return re.sub(r'[-\s]+', '-', value)
|
||||
|
||||
|
||||
def uses_dark_mode():
|
||||
"""
|
||||
This function detects if we are running in dark mode (e.g. macOS dark mode).
|
||||
|
|
@ -431,60 +259,6 @@ def format_archive_name(profile, archive_name_tpl):
|
|||
SHELL_PATTERN_ELEMENT = re.compile(r'([?\[\]*])')
|
||||
|
||||
|
||||
def get_mount_points(repo_url):
|
||||
mount_points = {}
|
||||
repo_mounts = []
|
||||
for proc in psutil.process_iter():
|
||||
try:
|
||||
name = proc.name()
|
||||
if name == 'borg' or name.startswith('python'):
|
||||
if 'mount' not in proc.cmdline():
|
||||
continue
|
||||
|
||||
if borg_compat.check('V2'):
|
||||
# command line syntax:
|
||||
# `borg mount -r <repo> <mountpoint> <path> (-a <archive_pattern>)`
|
||||
cmd = proc.cmdline()
|
||||
if repo_url in cmd:
|
||||
i = cmd.index(repo_url)
|
||||
if len(cmd) > i + 1:
|
||||
mount_point = cmd[i + 1]
|
||||
|
||||
# Archive mount?
|
||||
ao = '-a' in cmd
|
||||
if ao or '--match-archives' in cmd:
|
||||
i = cmd.index('-a' if ao else '--match-archives')
|
||||
if len(cmd) >= i + 1 and not SHELL_PATTERN_ELEMENT.search(cmd[i + 1]):
|
||||
mount_points[mount_point] = cmd[i + 1]
|
||||
else:
|
||||
repo_mounts.append(mount_point)
|
||||
else:
|
||||
for idx, parameter in enumerate(proc.cmdline()):
|
||||
if parameter.startswith(repo_url):
|
||||
# mount from this repo
|
||||
|
||||
# The borg mount command specifies that the mount_point
|
||||
# parameter comes after the archive name
|
||||
if len(proc.cmdline()) > idx + 1:
|
||||
mount_point = proc.cmdline()[idx + 1]
|
||||
|
||||
# archive or full mount?
|
||||
if parameter[len(repo_url) :].startswith('::'):
|
||||
archive_name = parameter[len(repo_url) + 2 :]
|
||||
mount_points[archive_name] = mount_point
|
||||
break
|
||||
else:
|
||||
# repo mount point
|
||||
repo_mounts.append(mount_point)
|
||||
|
||||
except (psutil.ZombieProcess, psutil.AccessDenied, psutil.NoSuchProcess):
|
||||
# Getting process details may fail (e.g. zombie process on macOS)
|
||||
# or because the process is owned by another user.
|
||||
# Also see https://github.com/giampaolo/psutil/issues/783
|
||||
continue
|
||||
|
||||
return mount_points, repo_mounts
|
||||
|
||||
|
||||
def is_system_tray_available():
|
||||
app = QApplication.instance()
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import logging
|
||||
import sys
|
||||
from datetime import timedelta
|
||||
from typing import Dict, Optional
|
||||
|
||||
from PyQt6 import QtCore, uic
|
||||
|
|
@ -36,17 +35,14 @@ from vorta.store.models import ArchiveModel, BackupProfileMixin, SettingsModel
|
|||
from vorta.utils import (
|
||||
borg_compat,
|
||||
choose_file_dialog,
|
||||
find_best_unit_for_sizes,
|
||||
format_archive_name,
|
||||
get_asset,
|
||||
get_mount_points,
|
||||
pretty_bytes,
|
||||
get_asset
|
||||
)
|
||||
from vorta.views import diff_result, extract_dialog
|
||||
from vorta.views.diff_result import DiffResultDialog, DiffTree
|
||||
from vorta.views.extract_dialog import ExtractDialog, ExtractTree
|
||||
from vorta.views.source_tab import SizeItem
|
||||
from vorta.views.utils import get_colored_icon
|
||||
from vorta.views.workers.archive_table_worker import PopulateArchiveTableAsync
|
||||
|
||||
uifile = get_asset('UI/archivetab.ui')
|
||||
ArchiveTabUI, ArchiveTabBase = uic.loadUiType(uifile)
|
||||
|
|
@ -73,6 +69,7 @@ class ArchiveTab(ArchiveTabBase, ArchiveTabUI, BackupProfileMixin):
|
|||
super().__init__(parent)
|
||||
self.setupUi(parent)
|
||||
self.mount_points = {} # mapping of archive name to mount point
|
||||
self.workers = []
|
||||
self.repo_mount_point: Optional[str] = None # mount point of whole repo
|
||||
self.menu = None
|
||||
self.app = app
|
||||
|
|
@ -241,10 +238,6 @@ class ArchiveTab(ArchiveTabBase, ArchiveTabUI, BackupProfileMixin):
|
|||
"""Populate archive list and prune settings from profile."""
|
||||
profile = self.profile()
|
||||
if profile.repo is not None:
|
||||
# get mount points
|
||||
self.mount_points, repo_mount_points = get_mount_points(profile.repo.url)
|
||||
if repo_mount_points:
|
||||
self.repo_mount_point = repo_mount_points[0]
|
||||
|
||||
if profile.repo.name:
|
||||
repo_name = f"{profile.repo.name} ({profile.repo.url})"
|
||||
|
|
@ -252,58 +245,10 @@ class ArchiveTab(ArchiveTabBase, ArchiveTabUI, BackupProfileMixin):
|
|||
repo_name = profile.repo.url
|
||||
self.toolBox.setItemText(0, self.tr('Archives for {}').format(repo_name))
|
||||
|
||||
archives = [s for s in profile.repo.archives.select().order_by(ArchiveModel.time.desc())]
|
||||
populateArchiveTableWorker = PopulateArchiveTableAsync(profile, self.mount_points, self.archiveTable)
|
||||
self.workers.append(populateArchiveTableWorker) # preserve worker reference
|
||||
populateArchiveTableWorker.start()
|
||||
|
||||
# if no archive's name can be found in self.mount_points, then hide the mount point column
|
||||
if not any(a.name in self.mount_points for a in archives):
|
||||
self.archiveTable.hideColumn(3)
|
||||
else:
|
||||
self.archiveTable.showColumn(3)
|
||||
|
||||
sorting = self.archiveTable.isSortingEnabled()
|
||||
self.archiveTable.setSortingEnabled(False)
|
||||
best_unit = find_best_unit_for_sizes((a.size for a in archives), precision=SIZE_DECIMAL_DIGITS)
|
||||
for row, archive in enumerate(archives):
|
||||
self.archiveTable.insertRow(row)
|
||||
|
||||
formatted_time = archive.time.strftime('%Y-%m-%d %H:%M')
|
||||
self.archiveTable.setItem(row, 0, QTableWidgetItem(formatted_time))
|
||||
|
||||
# format units based on user settings for 'dynamic' or 'fixed' units
|
||||
fixed_unit = best_unit if SettingsModel.get(key='enable_fixed_units').value else None
|
||||
size = pretty_bytes(archive.size, fixed_unit=fixed_unit, precision=SIZE_DECIMAL_DIGITS)
|
||||
self.archiveTable.setItem(row, 1, SizeItem(size))
|
||||
|
||||
if archive.duration is not None:
|
||||
formatted_duration = str(timedelta(seconds=round(archive.duration)))
|
||||
else:
|
||||
formatted_duration = ''
|
||||
|
||||
self.archiveTable.setItem(row, 2, QTableWidgetItem(formatted_duration))
|
||||
|
||||
mount_point = self.mount_points.get(archive.name)
|
||||
if mount_point is not None:
|
||||
item = QTableWidgetItem(mount_point)
|
||||
self.archiveTable.setItem(row, 3, item)
|
||||
|
||||
self.archiveTable.setItem(row, 4, QTableWidgetItem(archive.name))
|
||||
|
||||
if archive.trigger == 'scheduled':
|
||||
item = QTableWidgetItem(get_colored_icon('clock-o'), '')
|
||||
item.setToolTip(self.tr('Scheduled'))
|
||||
self.archiveTable.setItem(row, 5, item)
|
||||
elif archive.trigger == 'user':
|
||||
item = QTableWidgetItem(get_colored_icon('user'), '')
|
||||
item.setToolTip(self.tr('User initiated'))
|
||||
item.setTextAlignment(Qt.AlignmentFlag.AlignRight)
|
||||
self.archiveTable.setItem(row, 5, item)
|
||||
|
||||
self.archiveTable.setRowCount(len(archives))
|
||||
self.archiveTable.setSortingEnabled(sorting)
|
||||
item = self.archiveTable.item(0, 0)
|
||||
self.archiveTable.scrollToItem(item)
|
||||
|
||||
self.archiveTable.selectionModel().clearSelection()
|
||||
if self.remaining_refresh_archives == 0:
|
||||
self._toggle_all_buttons(enabled=True)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -180,10 +180,15 @@ class MainWindow(MainWindowBase, MainWindowUI):
|
|||
if not backup_profile_id:
|
||||
return
|
||||
self.current_profile = BackupProfileModel.get(id=backup_profile_id)
|
||||
logger.info('step 1')
|
||||
self.archiveTab.populate_from_profile()
|
||||
self.repoTab.populate_from_profile()
|
||||
logger.info('step 2')
|
||||
self.repoTab.populate_from_profile() # 1s
|
||||
logger.info('step 3')
|
||||
self.sourceTab.populate_from_profile()
|
||||
self.scheduleTab.populate_from_profile()
|
||||
logger.info('step 4')
|
||||
self.scheduleTab.populate_from_profile() #1s
|
||||
logger.info('step 5')
|
||||
SettingsModel.update({SettingsModel.str_value: self.current_profile.id}).where(
|
||||
SettingsModel.key == 'previous_profile_id'
|
||||
).execute()
|
||||
|
|
|
|||
|
|
@ -173,8 +173,6 @@ class RepoTab(RepoBase, RepoUI, BackupProfileMixin):
|
|||
self.repoEncryption.setText(na)
|
||||
self.repoEncryption.setToolTip(no_repo_selected)
|
||||
|
||||
self.repo_changed.emit()
|
||||
|
||||
def init_ssh(self):
|
||||
keys = get_private_keys()
|
||||
self.sshComboBox.clear()
|
||||
|
|
@ -264,6 +262,7 @@ class RepoTab(RepoBase, RepoUI, BackupProfileMixin):
|
|||
profile.repo = self.repoSelector.currentData()
|
||||
profile.save()
|
||||
self.init_repo_stats()
|
||||
self.repo_changed.emit()
|
||||
|
||||
def process_new_repo(self, result):
|
||||
if result['returncode'] == 0:
|
||||
|
|
@ -276,10 +275,12 @@ class RepoTab(RepoBase, RepoUI, BackupProfileMixin):
|
|||
self.repoSelector.setCurrentIndex(self.repoSelector.count() - 1)
|
||||
self.repo_added.emit()
|
||||
self.init_repo_stats()
|
||||
self.repo_changed.emit()
|
||||
|
||||
def repo_unlink_action(self):
|
||||
profile = self.profile()
|
||||
self.init_repo_stats()
|
||||
self.repo_changed.emit()
|
||||
|
||||
msg = QMessageBox()
|
||||
msg.setStandardButtons(QMessageBox.StandardButton.Ok)
|
||||
|
|
|
|||
|
|
@ -12,7 +12,8 @@ from vorta import application, config
|
|||
from vorta.i18n import get_locale
|
||||
from vorta.scheduler import ScheduleStatusType
|
||||
from vorta.store.models import BackupProfileMixin, EventLogModel, WifiSettingModel
|
||||
from vorta.utils import get_asset, get_sorted_wifis
|
||||
from vorta.utils import get_asset
|
||||
from vorta.views.workers.wifi_list_worker import PopulateWifiAsync
|
||||
from vorta.views.utils import get_colored_icon
|
||||
|
||||
uifile = get_asset('UI/scheduletab.ui')
|
||||
|
|
@ -33,6 +34,7 @@ class ScheduleTab(ScheduleBase, ScheduleUI, BackupProfileMixin):
|
|||
self.setupUi(parent)
|
||||
self.app: application.VortaApp = QApplication.instance()
|
||||
self.toolBox.setCurrentIndex(0)
|
||||
self.workers = []
|
||||
|
||||
self.schedulerRadioMapping = {
|
||||
'off': self.scheduleOffRadio,
|
||||
|
|
@ -171,7 +173,9 @@ class ScheduleTab(ScheduleBase, ScheduleUI, BackupProfileMixin):
|
|||
else:
|
||||
self.createCmdLineEdit.setEnabled(False)
|
||||
|
||||
self.populate_wifi()
|
||||
populateWifiWorker = PopulateWifiAsync(profile, self.wifiListWidget)
|
||||
self.workers.append(populateWifiWorker) # preserve reference
|
||||
populateWifiWorker.start()
|
||||
self.populate_logs()
|
||||
self.draw_next_scheduled_backup()
|
||||
|
||||
|
|
@ -191,18 +195,18 @@ class ScheduleTab(ScheduleBase, ScheduleUI, BackupProfileMixin):
|
|||
self.nextBackupDateTimeLabel.setText(text)
|
||||
self.nextBackupDateTimeLabel.repaint()
|
||||
|
||||
def populate_wifi(self):
|
||||
self.wifiListWidget.clear()
|
||||
for wifi in get_sorted_wifis(self.profile()):
|
||||
item = QListWidgetItem()
|
||||
item.setText(wifi.ssid)
|
||||
item.setFlags(item.flags() | QtCore.Qt.ItemFlag.ItemIsUserCheckable)
|
||||
if wifi.allowed:
|
||||
item.setCheckState(QtCore.Qt.CheckState.Checked)
|
||||
else:
|
||||
item.setCheckState(QtCore.Qt.CheckState.Unchecked)
|
||||
self.wifiListWidget.addItem(item)
|
||||
self.wifiListWidget.itemChanged.connect(self.save_wifi_item)
|
||||
# def populate_wifi(self):
|
||||
# self.wifiListWidget.clear()
|
||||
# for wifi in get_sorted_wifis(self.profile()):
|
||||
# item = QListWidgetItem()
|
||||
# item.setText(wifi.ssid)
|
||||
# item.setFlags(item.flags() | QtCore.Qt.ItemFlag.ItemIsUserCheckable)
|
||||
# if wifi.allowed:
|
||||
# item.setCheckState(QtCore.Qt.CheckState.Checked)
|
||||
# else:
|
||||
# item.setCheckState(QtCore.Qt.CheckState.Unchecked)
|
||||
# self.wifiListWidget.addItem(item)
|
||||
# self.wifiListWidget.itemChanged.connect(self.save_wifi_item)
|
||||
|
||||
def save_wifi_item(self, item):
|
||||
db_item = WifiSettingModel.get(ssid=item.text(), profile=self.profile().id)
|
||||
|
|
|
|||
|
|
@ -14,15 +14,14 @@ from PyQt6.QtWidgets import (
|
|||
)
|
||||
|
||||
from vorta.store.models import BackupProfileMixin, SettingsModel, SourceFileModel
|
||||
from vorta.views.workers.file_path_info_worker import FilePathInfoAsync
|
||||
from vorta.utils import (
|
||||
FilePathInfoAsync,
|
||||
choose_file_dialog,
|
||||
get_asset,
|
||||
pretty_bytes,
|
||||
sort_sizes,
|
||||
pretty_bytes
|
||||
)
|
||||
from vorta.views.exclude_dialog import ExcludeDialog
|
||||
from vorta.views.utils import get_colored_icon
|
||||
from vorta.views.utils import get_colored_icon, SizeItem
|
||||
|
||||
uifile = get_asset('UI/sourcetab.ui')
|
||||
SourceUI, SourceBase = uic.loadUiType(uifile)
|
||||
|
|
@ -36,23 +35,6 @@ class SourceColumn:
|
|||
FilesCount = 2
|
||||
|
||||
|
||||
class SizeItem(QTableWidgetItem):
|
||||
def __init__(self, s):
|
||||
super().__init__(s)
|
||||
self.setTextAlignment(Qt.AlignmentFlag.AlignVCenter + Qt.AlignmentFlag.AlignRight)
|
||||
|
||||
def __lt__(self, other):
|
||||
if other.text() == '':
|
||||
return False
|
||||
elif self.text() == '':
|
||||
return True
|
||||
else:
|
||||
return sort_sizes([self.text(), other.text()]) == [
|
||||
self.text(),
|
||||
other.text(),
|
||||
]
|
||||
|
||||
|
||||
class FilesCount(QTableWidgetItem):
|
||||
def __lt__(self, other):
|
||||
# Verify that conversion is only performed on valid integers
|
||||
|
|
|
|||
|
|
@ -3,10 +3,46 @@ import os
|
|||
import sys
|
||||
|
||||
from PyQt6.QtGui import QIcon, QImage, QPixmap
|
||||
from PyQt6.QtCore import Qt
|
||||
from PyQt6.QtWidgets import QTableWidgetItem
|
||||
|
||||
from vorta.utils import get_asset, uses_dark_mode
|
||||
|
||||
|
||||
class SizeItem(QTableWidgetItem):
|
||||
def __init__(self, s):
|
||||
super().__init__(s)
|
||||
self.setTextAlignment(Qt.AlignmentFlag.AlignVCenter + Qt.AlignmentFlag.AlignRight)
|
||||
|
||||
def __lt__(self, other):
|
||||
if other.text() == '':
|
||||
return False
|
||||
elif self.text() == '':
|
||||
return True
|
||||
else:
|
||||
return sort_sizes([self.text(), other.text()]) == [
|
||||
self.text(),
|
||||
other.text(),
|
||||
]
|
||||
|
||||
|
||||
def sort_sizes(size_list):
|
||||
"""Sorts sizes with extensions. Assumes that size is already in largest unit possible"""
|
||||
final_list = []
|
||||
for suffix in [" B", " KB", " MB", " GB", " TB", " PB", " EB", " ZB", " YB"]:
|
||||
sub_list = [
|
||||
float(size[: -len(suffix)])
|
||||
for size in size_list
|
||||
if size.endswith(suffix) and size[: -len(suffix)][-1].isnumeric()
|
||||
]
|
||||
sub_list.sort()
|
||||
final_list += [(str(size) + suffix) for size in sub_list]
|
||||
# Skip additional loops
|
||||
if len(final_list) == len(size_list):
|
||||
break
|
||||
return final_list
|
||||
|
||||
|
||||
def get_colored_icon(icon_name, scaled_height=128, return_qpixmap=False):
|
||||
"""
|
||||
Return SVG icon in the correct color.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,134 @@
|
|||
|
||||
from datetime import timedelta
|
||||
|
||||
import psutil
|
||||
from PyQt6.QtCore import QThread, Qt
|
||||
from PyQt6.QtWidgets import QTableWidgetItem
|
||||
|
||||
from vorta.store.models import ArchiveModel, SettingsModel
|
||||
from vorta.views.utils import get_colored_icon, SizeItem
|
||||
from vorta.utils import borg_compat, pretty_bytes, find_best_unit_for_sizes, SHELL_PATTERN_ELEMENT
|
||||
|
||||
SIZE_DECIMAL_DIGITS = 1
|
||||
|
||||
|
||||
class PopulateArchiveTableAsync(QThread):
|
||||
def __init__(self, profile, mount_points, archiveTable):
|
||||
QThread.__init__(self)
|
||||
self.profile = profile
|
||||
self.mount_points = mount_points
|
||||
self.archiveTable = archiveTable
|
||||
|
||||
def run(self):
|
||||
# get mount points
|
||||
self.mount_points, repo_mount_points = get_mount_points(self.profile.repo.url)
|
||||
if repo_mount_points:
|
||||
self.repo_mount_point = repo_mount_points[0]
|
||||
|
||||
archives = [s for s in self.profile.repo.archives.select().order_by(ArchiveModel.time.desc())]
|
||||
|
||||
# if no archive's name can be found in self.mount_points, then hide the mount point column
|
||||
if not any(a.name in self.mount_points for a in archives):
|
||||
self.archiveTable.hideColumn(3)
|
||||
else:
|
||||
self.archiveTable.showColumn(3)
|
||||
|
||||
sorting = self.archiveTable.isSortingEnabled()
|
||||
self.archiveTable.setSortingEnabled(False)
|
||||
best_unit = find_best_unit_for_sizes((a.size for a in archives), precision=SIZE_DECIMAL_DIGITS)
|
||||
for row, archive in enumerate(archives):
|
||||
self.archiveTable.insertRow(row)
|
||||
|
||||
formatted_time = archive.time.strftime('%Y-%m-%d %H:%M')
|
||||
self.archiveTable.setItem(row, 0, QTableWidgetItem(formatted_time))
|
||||
|
||||
# format units based on user settings for 'dynamic' or 'fixed' units
|
||||
fixed_unit = best_unit if SettingsModel.get(key='enable_fixed_units').value else None
|
||||
size = pretty_bytes(archive.size, fixed_unit=fixed_unit, precision=SIZE_DECIMAL_DIGITS)
|
||||
self.archiveTable.setItem(row, 1, SizeItem(size))
|
||||
|
||||
if archive.duration is not None:
|
||||
formatted_duration = str(timedelta(seconds=round(archive.duration)))
|
||||
else:
|
||||
formatted_duration = ''
|
||||
|
||||
self.archiveTable.setItem(row, 2, QTableWidgetItem(formatted_duration))
|
||||
|
||||
mount_point = self.mount_points.get(archive.name)
|
||||
if mount_point is not None:
|
||||
item = QTableWidgetItem(mount_point)
|
||||
self.archiveTable.setItem(row, 3, item)
|
||||
|
||||
self.archiveTable.setItem(row, 4, QTableWidgetItem(archive.name))
|
||||
|
||||
if archive.trigger == 'scheduled':
|
||||
item = QTableWidgetItem(get_colored_icon('clock-o'), '')
|
||||
item.setToolTip(self.tr('Scheduled'))
|
||||
self.archiveTable.setItem(row, 5, item)
|
||||
elif archive.trigger == 'user':
|
||||
item = QTableWidgetItem(get_colored_icon('user'), '')
|
||||
item.setToolTip(self.tr('User initiated'))
|
||||
item.setTextAlignment(Qt.AlignmentFlag.AlignRight)
|
||||
self.archiveTable.setItem(row, 5, item)
|
||||
|
||||
self.archiveTable.setRowCount(len(archives))
|
||||
self.archiveTable.setSortingEnabled(sorting)
|
||||
item = self.archiveTable.item(0, 0)
|
||||
self.archiveTable.scrollToItem(item)
|
||||
|
||||
self.archiveTable.selectionModel().clearSelection()
|
||||
|
||||
|
||||
def get_mount_points(repo_url):
|
||||
mount_points = {}
|
||||
repo_mounts = []
|
||||
for proc in psutil.process_iter():
|
||||
try:
|
||||
name = proc.name()
|
||||
if name == 'borg' or name.startswith('python'):
|
||||
if 'mount' not in proc.cmdline():
|
||||
continue
|
||||
|
||||
if borg_compat.check('V2'):
|
||||
# command line syntax:
|
||||
# `borg mount -r <repo> <mountpoint> <path> (-a <archive_pattern>)`
|
||||
cmd = proc.cmdline()
|
||||
if repo_url in cmd:
|
||||
i = cmd.index(repo_url)
|
||||
if len(cmd) > i + 1:
|
||||
mount_point = cmd[i + 1]
|
||||
|
||||
# Archive mount?
|
||||
ao = '-a' in cmd
|
||||
if ao or '--match-archives' in cmd:
|
||||
i = cmd.index('-a' if ao else '--match-archives')
|
||||
if len(cmd) >= i + 1 and not SHELL_PATTERN_ELEMENT.search(cmd[i + 1]):
|
||||
mount_points[mount_point] = cmd[i + 1]
|
||||
else:
|
||||
repo_mounts.append(mount_point)
|
||||
else:
|
||||
for idx, parameter in enumerate(proc.cmdline()):
|
||||
if parameter.startswith(repo_url):
|
||||
# mount from this repo
|
||||
|
||||
# The borg mount command specifies that the mount_point
|
||||
# parameter comes after the archive name
|
||||
if len(proc.cmdline()) > idx + 1:
|
||||
mount_point = proc.cmdline()[idx + 1]
|
||||
|
||||
# archive or full mount?
|
||||
if parameter[len(repo_url) :].startswith('::'):
|
||||
archive_name = parameter[len(repo_url) + 2 :]
|
||||
mount_points[archive_name] = mount_point
|
||||
break
|
||||
else:
|
||||
# repo mount point
|
||||
repo_mounts.append(mount_point)
|
||||
|
||||
except (psutil.ZombieProcess, psutil.AccessDenied, psutil.NoSuchProcess):
|
||||
# Getting process details may fail (e.g. zombie process on macOS)
|
||||
# or because the process is owned by another user.
|
||||
# Also see https://github.com/giampaolo/psutil/issues/783
|
||||
continue
|
||||
|
||||
return mount_points, repo_mounts
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
|
||||
from datetime import datetime as dt, timedelta
|
||||
import unicodedata
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
import fnmatch
|
||||
import psutil
|
||||
|
||||
from PyQt6 import QtCore
|
||||
from PyQt6.QtCore import QFileInfo, QThread, pyqtSignal, Qt
|
||||
from PyQt6.QtWidgets import (QApplication, QFileDialog, QSystemTrayIcon,
|
||||
QListWidgetItem, QTableWidgetItem)
|
||||
|
||||
|
||||
class FilePathInfoAsync(QThread):
|
||||
signal = pyqtSignal(str, str, str)
|
||||
|
||||
def __init__(self, path, exclude_patterns_str):
|
||||
self.path = path
|
||||
QThread.__init__(self)
|
||||
self.exiting = False
|
||||
self.exclude_patterns = []
|
||||
for _line in (exclude_patterns_str or '').splitlines():
|
||||
line = _line.strip()
|
||||
if line != '':
|
||||
self.exclude_patterns.append(line)
|
||||
|
||||
def run(self):
|
||||
# logger.info("running thread to get path=%s...", self.path)
|
||||
self.size, self.files_count = get_path_datasize(self.path, self.exclude_patterns)
|
||||
self.signal.emit(self.path, str(self.size), str(self.files_count))
|
||||
|
||||
|
||||
def normalize_path(path):
|
||||
"""normalize paths for MacOS (but do nothing on other platforms)"""
|
||||
# HFS+ converts paths to a canonical form, so users shouldn't be required to enter an exact match.
|
||||
# Windows and Unix filesystems allow different forms, so users always have to enter an exact match.
|
||||
return unicodedata.normalize('NFD', path) if sys.platform == 'darwin' else path
|
||||
|
||||
def get_path_datasize(path, exclude_patterns):
|
||||
file_info = QFileInfo(path)
|
||||
|
||||
if file_info.isDir():
|
||||
data_size, files_count = get_directory_size(file_info.absoluteFilePath(), exclude_patterns)
|
||||
else:
|
||||
data_size = file_info.size()
|
||||
files_count = 1
|
||||
|
||||
return data_size, files_count
|
||||
|
||||
|
||||
# prepare patterns as borg does
|
||||
# see `FnmatchPattern._prepare` at
|
||||
# https://github.com/borgbackup/borg/blob/master//src/borg/patterns.py
|
||||
def prepare_pattern(pattern):
|
||||
"""Prepare and process fnmatch patterns as borg does"""
|
||||
if pattern.endswith(os.path.sep):
|
||||
# trailing sep indicates that the contents should be excluded
|
||||
# but not the directory it self.
|
||||
pattern = os.path.normpath(pattern).rstrip(os.path.sep)
|
||||
pattern += os.path.sep + '*' + os.path.sep
|
||||
else:
|
||||
pattern = os.path.normpath(pattern) + os.path.sep + '*'
|
||||
|
||||
pattern = pattern.lstrip(os.path.sep) # sep at beginning is removed
|
||||
return re.compile(fnmatch.translate(pattern))
|
||||
|
||||
|
||||
def match(pattern: re.Pattern, path: str):
|
||||
"""Check whether a path matches the given pattern."""
|
||||
path = path.lstrip(os.path.sep) + os.path.sep
|
||||
return pattern.match(path) is not None
|
||||
|
||||
|
||||
def get_directory_size(dir_path, exclude_patterns):
|
||||
'''Get number of files only and total size in bytes from a path.
|
||||
Based off https://stackoverflow.com/a/17936789'''
|
||||
exclude_patterns = [prepare_pattern(p) for p in exclude_patterns]
|
||||
|
||||
data_size_filtered = 0
|
||||
seen = set()
|
||||
seen_filtered = set()
|
||||
|
||||
for dir_path, subdirectories, file_names in os.walk(dir_path, topdown=True):
|
||||
is_excluded = False
|
||||
for pattern in exclude_patterns:
|
||||
if match(pattern, dir_path):
|
||||
is_excluded = True
|
||||
break
|
||||
|
||||
if is_excluded:
|
||||
subdirectories.clear() # so that os.walk won't walk them
|
||||
continue
|
||||
|
||||
for file_name in file_names:
|
||||
file_path = os.path.join(dir_path, file_name)
|
||||
|
||||
# Ignore symbolic links, since borg doesn't follow them
|
||||
if os.path.islink(file_path):
|
||||
continue
|
||||
|
||||
is_excluded = False
|
||||
for pattern in exclude_patterns:
|
||||
if match(pattern, file_path):
|
||||
is_excluded = True
|
||||
break
|
||||
|
||||
try:
|
||||
stat = os.stat(file_path)
|
||||
if stat.st_ino not in seen: # Visit each file only once
|
||||
# this won't add the size of a hardlinked file
|
||||
seen.add(stat.st_ino)
|
||||
if not is_excluded:
|
||||
data_size_filtered += stat.st_size
|
||||
seen_filtered.add(stat.st_ino)
|
||||
except (FileNotFoundError, PermissionError):
|
||||
continue
|
||||
|
||||
files_count_filtered = len(seen_filtered)
|
||||
|
||||
return data_size_filtered, files_count_filtered
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
|
||||
import logging
|
||||
from PyQt6 import QtCore
|
||||
from PyQt6.QtCore import QThread, pyqtSignal, Qt
|
||||
from PyQt6.QtWidgets import QListWidgetItem
|
||||
|
||||
from vorta.utils import get_network_status_monitor
|
||||
from vorta.store.models import WifiSettingModel
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class PopulateWifiAsync(QThread):
|
||||
def __init__(self, profile, wifiListWidget):
|
||||
QThread.__init__(self)
|
||||
self.profile = profile
|
||||
self.wifiListWidget = wifiListWidget
|
||||
|
||||
def run(self):
|
||||
self.wifiListWidget.clear()
|
||||
for wifi in get_sorted_wifis(self.profile):
|
||||
item = QListWidgetItem()
|
||||
item.setText(wifi.ssid)
|
||||
item.setFlags(item.flags() | QtCore.Qt.ItemFlag.ItemIsUserCheckable)
|
||||
if wifi.allowed:
|
||||
item.setCheckState(QtCore.Qt.CheckState.Checked)
|
||||
else:
|
||||
item.setCheckState(QtCore.Qt.CheckState.Unchecked)
|
||||
self.wifiListWidget.addItem(item)
|
||||
|
||||
|
||||
def get_sorted_wifis(profile):
|
||||
"""
|
||||
Get Wifi networks known to the OS (only current one on macOS) and
|
||||
merge with networks from other profiles. Update last connected time.
|
||||
"""
|
||||
|
||||
|
||||
# Pull networks known to OS and all other backup profiles
|
||||
system_wifis = get_network_status_monitor().get_known_wifis()
|
||||
from_other_profiles = WifiSettingModel.select().where(WifiSettingModel.profile != profile.id).execute()
|
||||
|
||||
for wifi in list(from_other_profiles) + system_wifis:
|
||||
db_wifi, created = WifiSettingModel.get_or_create(
|
||||
ssid=wifi.ssid,
|
||||
profile=profile.id,
|
||||
defaults={'last_connected': wifi.last_connected, 'allowed': True},
|
||||
)
|
||||
|
||||
# Update last connected time
|
||||
if not created and db_wifi.last_connected != wifi.last_connected:
|
||||
db_wifi.last_connected = wifi.last_connected
|
||||
db_wifi.save()
|
||||
|
||||
# Finally return list of networks and settings for that profile
|
||||
return (
|
||||
WifiSettingModel.select()
|
||||
.where(WifiSettingModel.profile == profile.id)
|
||||
.order_by(-WifiSettingModel.last_connected)
|
||||
)
|
||||
Loading…
Reference in New Issue