Wait on network for remote repos

Don't try and start jobs with remote repos until the network is up.
This should prevent job failures when, for instance, waking from sleep.

Mac implementation is a stub currently.

Author: rzeigler

src/vorta/network_status/abc.py

@@ -26,6 +26,13 @@ def is_network_status_available(self):
         """Is the network status really available, and not just a dummy implementation?"""
         return type(self) is not NetworkStatusMonitor
 
+    def is_network_active(self) -> bool:
+        """Is there an active network connection.
+
+        True signals that the network is up. The internet may still not be reachable though.
+        """
+        raise NotImplementedError()
+
     def is_network_metered(self) -> bool:
         """Is the currently connected network a metered connection?"""
         raise NotImplementedError()

src/vorta/network_status/darwin.py

@@ -25,6 +25,10 @@ def is_network_metered(self) -> bool:
 
         return is_ios_hotspot or any(is_network_metered_with_android(d) for d in get_network_devices())
 
+    def is_network_active(self):
+        # Not yet implemented
+        return True
+
     def get_current_wifi(self) -> Optional[str]:
         """
         Get current SSID or None if Wi-Fi is off.

src/vorta/network_status/network_manager.py

@@ -25,6 +25,13 @@ def is_network_metered(self) -> bool:
             logger.exception("Failed to check if network is metered, assuming it isn't")
             return False
 
+    def is_network_active(self):
+        try:
+            return self._nm.get_connectivity_state() is not NMConnectivityState.NONE
+        except DBusException:
+            logger.exception("Failed to check connectivity state. Assuming connected")
+            return True
+
     def get_current_wifi(self) -> Optional[str]:
         # Only check the primary connection. VPN over WiFi will still show the WiFi as Primary Connection.
         # We don't check all active connections, as NM won't disable WiFi when connecting a cable.
@@ -126,6 +133,9 @@ def isValid(self):
             return False
         return True
 
+    def get_connectivity_state(self) -> 'NMConnectivityState':
+        return NMConnectivityState(read_dbus_property(self._nm, 'Connectivity'))
+
     def get_primary_connection_path(self) -> Optional[str]:
         return read_dbus_property(self._nm, 'PrimaryConnection')
 
@@ -186,3 +196,13 @@ class NMDeviceType(Enum):
     # Only the types we care about
     UNKNOWN = 0
     WIFI = 2
+
+
+class NMConnectivityState(Enum):
+    """https://www.networkmanager.dev/docs/api/latest/nm-dbus-types.html#NMConnectivityState"""
+
+    UNKNOWN = 0
+    NONE = 1
+    PORTAL = 2
+    LIMITED = 3
+    FULL = 4

src/vorta/scheduler.py

@@ -3,7 +3,7 @@
 import threading
 from datetime import datetime as dt
 from datetime import timedelta
-from typing import Dict, NamedTuple, Optional, Tuple, Union
+from typing import Dict, List, NamedTuple, Optional, Tuple, Union
 
 from packaging import version
 from PyQt6 import QtCore, QtDBus
@@ -19,7 +19,7 @@
 from vorta.i18n import translate
 from vorta.notifications import VortaNotifications
 from vorta.store.models import BackupProfileModel, EventLogModel
-from vorta.utils import borg_compat
+from vorta.utils import borg_compat, get_network_status_monitor
 
 logger = logging.getLogger(__name__)
 
@@ -59,6 +59,13 @@ def __init__(self):
         self.qt_timer.setInterval(15 * 60 * 1000)
         self.qt_timer.start()
 
+        # Network backup profiles that are waiting on the network
+        self.network_deferred_timer = QTimer()
+        self.network_deferred_timer.timeout.connect(self.create_backup_if_net_up)
+        self.network_deferred_timer.setInterval(5 * 1000)  # Short interval for the network to come up
+        # Don't start until its actually needed
+        self.network_deferred_profiles: List[str] = []
+
         # connect signals
         self.app.backup_finished_event.connect(lambda res: self.set_timer_for_profile(res['params']['profile_id']))
 
@@ -81,6 +88,27 @@ def loginSuspendNotify(self, suspend: bool):
             logger.debug("Got login suspend/resume notification")
             self.reload_all_timers()
 
+    def create_backup_if_net_up(self):
+        nm = get_network_status_monitor()
+        if nm.is_network_active():
+            # Cancel the timer
+            self.network_deferred_timer.stop()
+            logger.info("the network is active, dispatching waiting jobs")
+            # create_backup will add to waiting_network if the network goes down again
+            # flip ahead of time here in case that happens
+            waiting = self.network_deferred_profiles
+            self.network_deferred_profiles = []
+            for profile_id in waiting:
+                self.create_backup(profile_id)
+        else:
+            logger.debug("there are jobs waiting on the network, but it is not yet up")
+
+    def defer_backup(self, profile_id):
+        if not self.network_deferred_profiles:
+            # Nothing is currently waiting so start the timer
+            self.network_deferred_timer.start()
+        self.network_deferred_profiles.append(profile_id)
+
     def tr(self, *args, **kwargs):
         scope = self.__class__.__name__
         return translate(scope, *args, **kwargs)
@@ -397,6 +425,15 @@ def create_backup(self, profile_id):
             logger.info('Profile not found. Maybe deleted?')
             return
 
+        if profile.repo.is_remote_repo() and not get_network_status_monitor().is_network_active():
+            logger.info(
+                'repo %s is remote and there is no active network connection, deferring backup for %s',
+                profile.repo.name,
+                profile.name,
+            )
+            self.defer_backup(profile_id)
+            return
+
         # Skip if a job for this profile (repo) is already in progress
         if self.app.jobs_manager.is_worker_running(site=profile.repo.id):
             logger.debug('A job for repo %s is already active.', profile.repo.id)
@@ -521,6 +558,12 @@ def post_backup_tasks(self, profile_id):
         )
 
     def remove_job(self, profile_id):
+        if profile_id in self.network_deferred_profiles:
+            self.network_deferred_profiles.remove(profile_id)
+            # If nothing is waiting cancel the timer
+            if not self.network_deferred_profiles:
+                self.network_deferred_timer.stop()
+
         if profile_id in self.timers:
             qtimer = self.timers[profile_id].get('qtt')
             if qtimer is not None: