@@ -118,13 +118,15 @@ class BackupService {
118118 private var isObserving = false
119119 private var isRestoring = false
120120 private var isWiping = false
121- private var lastNotificationTime : UInt64 = 0
121+ private var backupFailureNotificationGate = BackupFailureNotificationGate ( )
122+ private var foregroundObservationGeneration : UInt64 = 0
122123
123124 private let defaults : UserDefaults
124125 private let backupData : ( ( BackupCategory ) async throws -> Data ) ?
125126 private let uploadBackup : ( String , Data ) async throws -> Void
126127 private let waitForBackupDelay : ( ) async throws -> Void
127128 private let waitForTemporaryFailureDelay : ( UInt64 ) async throws -> Void
129+ private let waitForForegroundRecoveryTimeout : ( ) async throws -> Void
128130 private let backupStatusesKey = " backupStatuses "
129131 private var backupVersions : [ BackupCategory : UInt64 ] = [ : ]
130132
@@ -159,6 +161,9 @@ class BackupService {
159161 } ,
160162 waitForTemporaryFailureDelay : @escaping ( UInt64 ) async throws -> Void = {
161163 try await Task . sleep ( nanoseconds : $0 )
164+ } ,
165+ waitForForegroundRecoveryTimeout : @escaping ( ) async throws -> Void = {
166+ try await Task . sleep ( for : . seconds ( BackupService . backupFailureCheckInterval ) )
162167 }
163168 ) {
164169 self . defaults = defaults
@@ -167,6 +172,7 @@ class BackupService {
167172 self . uploadBackup = uploadBackup
168173 self . waitForBackupDelay = waitForBackupDelay
169174 self . waitForTemporaryFailureDelay = waitForTemporaryFailureDelay
175+ self . waitForForegroundRecoveryTimeout = waitForForegroundRecoveryTimeout
170176 let statuses = getAllBackupStatuses ( )
171177 var clearedStatuses = statuses
172178 for category in BackupCategory . allCases {
@@ -200,10 +206,84 @@ class BackupService {
200206 }
201207 }
202208
203- func startObservingBackups ( ) {
204- guard ! shouldSkipBackup ( ) else { return }
209+ @ discardableResult
210+ func setAppActive ( _ isActive : Bool ) -> Task < Void , Never > ? {
211+ let ( shouldRecover , generation ) = stateQueue . sync {
212+ backupFailureNotificationGate . setActive ( isActive )
213+ return ( isActive && backupFailureNotificationGate . beginForegroundRecovery ( ) , foregroundObservationGeneration )
214+ }
215+
216+ guard shouldRecover else { return nil }
217+ return Task {
218+ await waitForForegroundRetries ( generation : generation )
219+ self . stateQueue . sync {
220+ self . backupFailureNotificationGate . endForegroundRecovery ( )
221+ }
222+ let isObserving = try? await ServiceQueue . background ( . backup ) {
223+ self . isObserving
224+ }
225+ let shouldCheck = self . stateQueue . sync {
226+ self . foregroundObservationGeneration == generation && self . backupFailureNotificationGate . shouldCheckForFailedBackups (
227+ isObserving : isObserving == true
228+ )
229+ }
230+
231+ guard shouldCheck else { return }
232+ self . checkForFailedBackups ( )
233+ }
234+ }
205235
236+ private func waitForForegroundRetries ( generation : UInt64 ) async {
237+ let ( settled , finish ) = AsyncStream < Void > . makeStream ( )
238+ // Bound alert suppression, not the uploads: an FFI write may ignore cancellation.
206239 Task {
240+ await self . retryPendingBackups ( generation : generation )
241+ finish . finish ( )
242+ }
243+ let timeout = Task {
244+ do {
245+ try await self . waitForForegroundRecoveryTimeout ( )
246+ try Task . checkCancellation ( )
247+ finish . finish ( )
248+ } catch { }
249+ }
250+ for await _ in settled { }
251+ timeout . cancel ( )
252+ }
253+
254+ private func retryPendingBackups ( generation : UInt64 ) async {
255+ let tasks = try? await ServiceQueue . background ( . backup ) { [ self ] in
256+ guard isObserving , stateQueue . sync ( execute : { foregroundObservationGeneration == generation } ) else {
257+ return [ Task < BackupOutcome , Never > ] ( )
258+ }
259+ return BackupCategory . allCases . compactMap { category -> Task < BackupOutcome , Never > ? in
260+ guard category != . lightningConnections , ! shouldSkipBackup ( category : category ) else { return nil }
261+ let status = getBackupStatus ( category : category )
262+ guard status . isRequired , ! status . running ,
263+ backupJobs [ category ] == nil , runningBackupTasks [ category ] == nil
264+ else { return nil }
265+
266+ let task = Task {
267+ let outcome = await self . runBackup ( category : category , isScheduled : true )
268+ if outcome . canRetryPendingBackup , ! Task . isCancelled , self . getBackupStatus ( category : category ) . isRequired {
269+ await self . scheduleBackup ( category : category , initialOutcome : outcome , foregroundGeneration : generation )
270+ }
271+ return outcome
272+ }
273+ runningBackupTasks [ category ] = task
274+ return task
275+ }
276+ }
277+ for task in tasks ?? [ ] {
278+ _ = await task . value
279+ }
280+ }
281+
282+ @ discardableResult
283+ func startObservingBackups ( ) -> Task < Void , Never > ? {
284+ guard ! shouldSkipBackup ( ) else { return nil }
285+
286+ return Task {
207287 let shouldStart = try? await ServiceQueue . background ( . backup ) {
208288 guard ! self . isObserving else { return false }
209289 self . isObserving = true
@@ -223,6 +303,7 @@ class BackupService {
223303 let shouldStop = try? await ServiceQueue . background ( . backup ) {
224304 let wasObserving = self . isObserving
225305 self . isObserving = false
306+ self . stateQueue . sync { self . foregroundObservationGeneration &+= 1 }
226307
227308 self . backupJobs . values . forEach { $0 . cancel ( ) }
228309 self . backupJobs . removeAll ( )
@@ -266,13 +347,19 @@ class BackupService {
266347 var outcome = BackupOutcome . failed
267348 var requiredAtStart : UInt64 = 0
268349 var versionAtStart : UInt64 = 0
350+ let backupTime = UInt64 ( Date ( ) . timeIntervalSince1970 )
269351 updateBackupStatus ( category : category ) { status in
270- requiredAtStart = max ( status . required , status . synced + 1 , UInt64 ( Date ( ) . timeIntervalSince1970 ) )
352+ // Retrying must not restart the age of a backup that is already overdue.
353+ requiredAtStart = status . isRequired ? status . required : max ( status . required , status . synced + 1 , backupTime )
271354 versionAtStart = self . backupVersions [ category , default : 0 ]
272355 return BackupItemStatus ( synced : status . synced , required : requiredAtStart , running : true )
273356 }
274357
275358 do {
359+ try Task . checkCancellation ( )
360+ if isScheduled , shouldSkipBackup ( category : category ) {
361+ throw CancellationError ( )
362+ }
276363 if category == . usdt , backupData == nil {
277364 try await backupUsdtRecoveryIfPresent ( )
278365 } else if category == . wallet {
@@ -284,12 +371,16 @@ class BackupService {
284371 }
285372 } else {
286373 let data = try await getBackupDataBytes ( category : category )
374+ try Task . checkCancellation ( )
375+ if isScheduled , shouldSkipBackup ( category : category ) {
376+ throw CancellationError ( )
377+ }
287378 try await uploadBackup ( category . rawValue , data )
288379 }
289380 updateBackupStatus ( category : category ) { status in
290381 let changed = self . backupVersions [ category , default : 0 ] != versionAtStart
291382 return BackupItemStatus (
292- synced : changed ? min ( requiredAtStart , status . required - 1 ) : requiredAtStart ,
383+ synced : changed ? min ( requiredAtStart , status . required - 1 ) : max ( requiredAtStart , backupTime ) ,
293384 required : status . required ,
294385 running : false
295386 )
@@ -300,7 +391,7 @@ class BackupService {
300391 updateBackupStatus ( category : category ) { status in
301392 BackupItemStatus (
302393 synced : status . synced ,
303- required : category == . wallet || category == . usdt ? status . required : status . synced ,
394+ required : status . required ,
304395 running : false
305396 )
306397 }
@@ -316,6 +407,7 @@ class BackupService {
316407 Logger . warn ( " Backup temporarily unavailable for: ' \( category . rawValue ) ' " , context : " BackupService " )
317408 } else {
318409 Logger . error ( " Backup failed for: ' \( category . rawValue ) ': \( error ) " , context : " BackupService " )
410+ checkForFailedBackups ( )
319411 }
320412 }
321413 _ = try? await ServiceQueue . background ( . backup ) { self . runningBackupTasks . removeValue ( forKey : category ) }
@@ -809,8 +901,13 @@ class BackupService {
809901 }
810902 }
811903
812- private func scheduleBackup ( category : BackupCategory ) async {
904+ private func scheduleBackup ( category : BackupCategory , initialOutcome : BackupOutcome = . failed , foregroundGeneration : UInt64 ? = nil ) async {
813905 try? await ServiceQueue . background ( . backup ) { [ self ] in
906+ if let foregroundGeneration {
907+ guard isObserving , ! shouldSkipBackup ( category : category ) ,
908+ stateQueue . sync ( execute : { foregroundObservationGeneration == foregroundGeneration } )
909+ else { return }
910+ }
814911 guard backupJobs [ category ] == nil else { return }
815912 updateBackupStatus ( category : category ) { status in
816913 BackupItemStatus (
@@ -819,12 +916,12 @@ class BackupService {
819916 running : true
820917 )
821918 }
822- backupJobs [ category ] = Task { await self . runScheduledBackup ( category : category ) }
919+ backupJobs [ category ] = Task { await self . runScheduledBackup ( category : category , initialOutcome : initialOutcome ) }
823920 }
824921 }
825922
826- private func runScheduledBackup ( category : BackupCategory ) async {
827- var outcome = BackupOutcome . failed
923+ private func runScheduledBackup ( category : BackupCategory , initialOutcome : BackupOutcome ) async {
924+ var outcome = initialOutcome
828925 var retryDelay : UInt64 = 10_000_000_000
829926 while ! Task . isCancelled {
830927 do {
@@ -858,27 +955,36 @@ class BackupService {
858955
859956 private func checkForFailedBackups ( ) {
860957 let currentTime = UInt64 ( Date ( ) . timeIntervalSince1970 )
861-
862- let hasFailedBackups = BackupCategory . allCases . contains { category in
863- let status = getBackupStatus ( category : category )
864- return status . isRequired &&
865- currentTime > status . required &&
866- ( currentTime - status . required ) > Self . failedBackupCheckTime
958+ if hasFailedBackups ( at : currentTime ) {
959+ showBackupFailureNotification ( )
867960 }
961+ }
868962
869- if hasFailedBackups {
870- showBackupFailureNotification ( currentTime : currentTime )
963+ private func hasFailedBackups ( at currentTime : UInt64 ) -> Bool {
964+ BackupCategory . allCases . contains { category in
965+ let status = getBackupStatus ( category : category )
966+ return BackupFailureNotificationGate . hasFailedBackup (
967+ status ,
968+ at : currentTime ,
969+ failureAge : Self . failedBackupCheckTime
970+ )
871971 }
872972 }
873973
874- private func showBackupFailureNotification ( currentTime : UInt64 ) {
974+ private func showBackupFailureNotification ( ) {
875975 Task {
876976 try? await ServiceQueue . background ( . backup ) {
877- if currentTime - self . lastNotificationTime < Self . failedBackupNotificationInterval {
878- return
977+ guard self . isObserving , ! self . shouldSkipBackup ( ) else { return }
978+ let currentTime = UInt64 ( Date ( ) . timeIntervalSince1970 )
979+ guard self . statusUpdateQueue . sync ( execute : { self . hasFailedBackups ( at : currentTime ) } ) else { return }
980+
981+ let shouldNotify = self . stateQueue . sync {
982+ self . backupFailureNotificationGate . shouldNotify (
983+ at : currentTime ,
984+ notificationInterval : Self . failedBackupNotificationInterval
985+ )
879986 }
880-
881- self . lastNotificationTime = currentTime
987+ guard shouldNotify else { return }
882988
883989 let backupCheckIntervalMinutes = Int ( Self . backupFailureCheckInterval / 60 )
884990 self . backupFailureSubject . send ( backupCheckIntervalMinutes )
@@ -1281,3 +1387,54 @@ actor WalletBackupWriter {
12811387 }
12821388 }
12831389}
1390+
1391+ struct BackupFailureNotificationGate {
1392+ private var isActive = false
1393+ private var isRecovering = false
1394+ private var lastNotificationTime : UInt64 ?
1395+
1396+ mutating func setActive ( _ isActive : Bool ) {
1397+ self . isActive = isActive
1398+ }
1399+
1400+ mutating func beginForegroundRecovery ( ) -> Bool {
1401+ guard ! isRecovering else { return false }
1402+ isRecovering = true
1403+ return true
1404+ }
1405+
1406+ mutating func endForegroundRecovery ( ) {
1407+ isRecovering = false
1408+ }
1409+
1410+ func shouldCheckForFailedBackups ( isObserving : Bool ) -> Bool {
1411+ isActive && isObserving
1412+ }
1413+
1414+ static func hasFailedBackup (
1415+ _ status : BackupItemStatus ,
1416+ at currentTime : UInt64 ,
1417+ failureAge : UInt64
1418+ ) -> Bool {
1419+ status . isRequired && ! status . running && currentTime >= status . required &&
1420+ currentTime - status . required > failureAge
1421+ }
1422+
1423+ mutating func shouldNotify (
1424+ at currentTime : UInt64 ,
1425+ notificationInterval : UInt64
1426+ ) -> Bool {
1427+ guard isActive , ! isRecovering else { return false }
1428+
1429+ if let lastNotificationTime {
1430+ guard currentTime >= lastNotificationTime ,
1431+ currentTime - lastNotificationTime >= notificationInterval
1432+ else {
1433+ return false
1434+ }
1435+ }
1436+
1437+ lastNotificationTime = currentTime
1438+ return true
1439+ }
1440+ }
0 commit comments