Skip to content

Commit e5aa577

Browse files
committed
Merge foreground backup recovery from master
2 parents 256a7ae + 543d938 commit e5aa577

5 files changed

Lines changed: 785 additions & 23 deletions

File tree

‎Bitkit/AppScene.swift‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1288,6 +1288,7 @@ struct AppScene: View {
12881288
isContactRefreshForeground = newPhase == .active
12891289
}
12901290
Logger.info("Scene phase changed: \(newPhase)", context: "AppScene")
1291+
BackupService.shared.setAppActive(newPhase == .active)
12911292

12921293
let returnedFromBackground = foregroundReturnTracker.scenePhaseChanged(to: newPhase)
12931294

‎Bitkit/Services/BackupService.swift‎

Lines changed: 180 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -118,13 +118,15 @@ class BackupService {
118118
private var isObserving = false
119119
private var isRestoring = false
120120
private var isWiping = false
121-
private var lastNotificationTime: UInt64 = 0
121+
private var backupFailureNotificationGate = BackupFailureNotificationGate()
122+
private var foregroundObservationGeneration: UInt64 = 0
122123

123124
private let defaults: UserDefaults
124125
private let backupData: ((BackupCategory) async throws -> Data)?
125126
private let uploadBackup: (String, Data) async throws -> Void
126127
private let waitForBackupDelay: () async throws -> Void
127128
private let waitForTemporaryFailureDelay: (UInt64) async throws -> Void
129+
private let waitForForegroundRecoveryTimeout: () async throws -> Void
128130
private let backupStatusesKey = "backupStatuses"
129131
private var backupVersions: [BackupCategory: UInt64] = [:]
130132

@@ -159,6 +161,9 @@ class BackupService {
159161
},
160162
waitForTemporaryFailureDelay: @escaping (UInt64) async throws -> Void = {
161163
try await Task.sleep(nanoseconds: $0)
164+
},
165+
waitForForegroundRecoveryTimeout: @escaping () async throws -> Void = {
166+
try await Task.sleep(for: .seconds(BackupService.backupFailureCheckInterval))
162167
}
163168
) {
164169
self.defaults = defaults
@@ -167,6 +172,7 @@ class BackupService {
167172
self.uploadBackup = uploadBackup
168173
self.waitForBackupDelay = waitForBackupDelay
169174
self.waitForTemporaryFailureDelay = waitForTemporaryFailureDelay
175+
self.waitForForegroundRecoveryTimeout = waitForForegroundRecoveryTimeout
170176
let statuses = getAllBackupStatuses()
171177
var clearedStatuses = statuses
172178
for category in BackupCategory.allCases {
@@ -200,10 +206,84 @@ class BackupService {
200206
}
201207
}
202208

203-
func startObservingBackups() {
204-
guard !shouldSkipBackup() else { return }
209+
@discardableResult
210+
func setAppActive(_ isActive: Bool) -> Task<Void, Never>? {
211+
let (shouldRecover, generation) = stateQueue.sync {
212+
backupFailureNotificationGate.setActive(isActive)
213+
return (isActive && backupFailureNotificationGate.beginForegroundRecovery(), foregroundObservationGeneration)
214+
}
215+
216+
guard shouldRecover else { return nil }
217+
return Task {
218+
await waitForForegroundRetries(generation: generation)
219+
self.stateQueue.sync {
220+
self.backupFailureNotificationGate.endForegroundRecovery()
221+
}
222+
let isObserving = try? await ServiceQueue.background(.backup) {
223+
self.isObserving
224+
}
225+
let shouldCheck = self.stateQueue.sync {
226+
self.foregroundObservationGeneration == generation && self.backupFailureNotificationGate.shouldCheckForFailedBackups(
227+
isObserving: isObserving == true
228+
)
229+
}
230+
231+
guard shouldCheck else { return }
232+
self.checkForFailedBackups()
233+
}
234+
}
205235

236+
private func waitForForegroundRetries(generation: UInt64) async {
237+
let (settled, finish) = AsyncStream<Void>.makeStream()
238+
// Bound alert suppression, not the uploads: an FFI write may ignore cancellation.
206239
Task {
240+
await self.retryPendingBackups(generation: generation)
241+
finish.finish()
242+
}
243+
let timeout = Task {
244+
do {
245+
try await self.waitForForegroundRecoveryTimeout()
246+
try Task.checkCancellation()
247+
finish.finish()
248+
} catch {}
249+
}
250+
for await _ in settled {}
251+
timeout.cancel()
252+
}
253+
254+
private func retryPendingBackups(generation: UInt64) async {
255+
let tasks = try? await ServiceQueue.background(.backup) { [self] in
256+
guard isObserving, stateQueue.sync(execute: { foregroundObservationGeneration == generation }) else {
257+
return [Task<BackupOutcome, Never>]()
258+
}
259+
return BackupCategory.allCases.compactMap { category -> Task<BackupOutcome, Never>? in
260+
guard category != .lightningConnections, !shouldSkipBackup(category: category) else { return nil }
261+
let status = getBackupStatus(category: category)
262+
guard status.isRequired, !status.running,
263+
backupJobs[category] == nil, runningBackupTasks[category] == nil
264+
else { return nil }
265+
266+
let task = Task {
267+
let outcome = await self.runBackup(category: category, isScheduled: true)
268+
if outcome.canRetryPendingBackup, !Task.isCancelled, self.getBackupStatus(category: category).isRequired {
269+
await self.scheduleBackup(category: category, initialOutcome: outcome, foregroundGeneration: generation)
270+
}
271+
return outcome
272+
}
273+
runningBackupTasks[category] = task
274+
return task
275+
}
276+
}
277+
for task in tasks ?? [] {
278+
_ = await task.value
279+
}
280+
}
281+
282+
@discardableResult
283+
func startObservingBackups() -> Task<Void, Never>? {
284+
guard !shouldSkipBackup() else { return nil }
285+
286+
return Task {
207287
let shouldStart = try? await ServiceQueue.background(.backup) {
208288
guard !self.isObserving else { return false }
209289
self.isObserving = true
@@ -223,6 +303,7 @@ class BackupService {
223303
let shouldStop = try? await ServiceQueue.background(.backup) {
224304
let wasObserving = self.isObserving
225305
self.isObserving = false
306+
self.stateQueue.sync { self.foregroundObservationGeneration &+= 1 }
226307

227308
self.backupJobs.values.forEach { $0.cancel() }
228309
self.backupJobs.removeAll()
@@ -266,13 +347,19 @@ class BackupService {
266347
var outcome = BackupOutcome.failed
267348
var requiredAtStart: UInt64 = 0
268349
var versionAtStart: UInt64 = 0
350+
let backupTime = UInt64(Date().timeIntervalSince1970)
269351
updateBackupStatus(category: category) { status in
270-
requiredAtStart = max(status.required, status.synced + 1, UInt64(Date().timeIntervalSince1970))
352+
// Retrying must not restart the age of a backup that is already overdue.
353+
requiredAtStart = status.isRequired ? status.required : max(status.required, status.synced + 1, backupTime)
271354
versionAtStart = self.backupVersions[category, default: 0]
272355
return BackupItemStatus(synced: status.synced, required: requiredAtStart, running: true)
273356
}
274357

275358
do {
359+
try Task.checkCancellation()
360+
if isScheduled, shouldSkipBackup(category: category) {
361+
throw CancellationError()
362+
}
276363
if category == .usdt, backupData == nil {
277364
try await backupUsdtRecoveryIfPresent()
278365
} else if category == .wallet {
@@ -284,12 +371,16 @@ class BackupService {
284371
}
285372
} else {
286373
let data = try await getBackupDataBytes(category: category)
374+
try Task.checkCancellation()
375+
if isScheduled, shouldSkipBackup(category: category) {
376+
throw CancellationError()
377+
}
287378
try await uploadBackup(category.rawValue, data)
288379
}
289380
updateBackupStatus(category: category) { status in
290381
let changed = self.backupVersions[category, default: 0] != versionAtStart
291382
return BackupItemStatus(
292-
synced: changed ? min(requiredAtStart, status.required - 1) : requiredAtStart,
383+
synced: changed ? min(requiredAtStart, status.required - 1) : max(requiredAtStart, backupTime),
293384
required: status.required,
294385
running: false
295386
)
@@ -300,7 +391,7 @@ class BackupService {
300391
updateBackupStatus(category: category) { status in
301392
BackupItemStatus(
302393
synced: status.synced,
303-
required: category == .wallet || category == .usdt ? status.required : status.synced,
394+
required: status.required,
304395
running: false
305396
)
306397
}
@@ -316,6 +407,7 @@ class BackupService {
316407
Logger.warn("Backup temporarily unavailable for: '\(category.rawValue)'", context: "BackupService")
317408
} else {
318409
Logger.error("Backup failed for: '\(category.rawValue)': \(error)", context: "BackupService")
410+
checkForFailedBackups()
319411
}
320412
}
321413
_ = try? await ServiceQueue.background(.backup) { self.runningBackupTasks.removeValue(forKey: category) }
@@ -809,8 +901,13 @@ class BackupService {
809901
}
810902
}
811903

812-
private func scheduleBackup(category: BackupCategory) async {
904+
private func scheduleBackup(category: BackupCategory, initialOutcome: BackupOutcome = .failed, foregroundGeneration: UInt64? = nil) async {
813905
try? await ServiceQueue.background(.backup) { [self] in
906+
if let foregroundGeneration {
907+
guard isObserving, !shouldSkipBackup(category: category),
908+
stateQueue.sync(execute: { foregroundObservationGeneration == foregroundGeneration })
909+
else { return }
910+
}
814911
guard backupJobs[category] == nil else { return }
815912
updateBackupStatus(category: category) { status in
816913
BackupItemStatus(
@@ -819,12 +916,12 @@ class BackupService {
819916
running: true
820917
)
821918
}
822-
backupJobs[category] = Task { await self.runScheduledBackup(category: category) }
919+
backupJobs[category] = Task { await self.runScheduledBackup(category: category, initialOutcome: initialOutcome) }
823920
}
824921
}
825922

826-
private func runScheduledBackup(category: BackupCategory) async {
827-
var outcome = BackupOutcome.failed
923+
private func runScheduledBackup(category: BackupCategory, initialOutcome: BackupOutcome) async {
924+
var outcome = initialOutcome
828925
var retryDelay: UInt64 = 10_000_000_000
829926
while !Task.isCancelled {
830927
do {
@@ -858,27 +955,36 @@ class BackupService {
858955

859956
private func checkForFailedBackups() {
860957
let currentTime = UInt64(Date().timeIntervalSince1970)
861-
862-
let hasFailedBackups = BackupCategory.allCases.contains { category in
863-
let status = getBackupStatus(category: category)
864-
return status.isRequired &&
865-
currentTime > status.required &&
866-
(currentTime - status.required) > Self.failedBackupCheckTime
958+
if hasFailedBackups(at: currentTime) {
959+
showBackupFailureNotification()
867960
}
961+
}
868962

869-
if hasFailedBackups {
870-
showBackupFailureNotification(currentTime: currentTime)
963+
private func hasFailedBackups(at currentTime: UInt64) -> Bool {
964+
BackupCategory.allCases.contains { category in
965+
let status = getBackupStatus(category: category)
966+
return BackupFailureNotificationGate.hasFailedBackup(
967+
status,
968+
at: currentTime,
969+
failureAge: Self.failedBackupCheckTime
970+
)
871971
}
872972
}
873973

874-
private func showBackupFailureNotification(currentTime: UInt64) {
974+
private func showBackupFailureNotification() {
875975
Task {
876976
try? await ServiceQueue.background(.backup) {
877-
if currentTime - self.lastNotificationTime < Self.failedBackupNotificationInterval {
878-
return
977+
guard self.isObserving, !self.shouldSkipBackup() else { return }
978+
let currentTime = UInt64(Date().timeIntervalSince1970)
979+
guard self.statusUpdateQueue.sync(execute: { self.hasFailedBackups(at: currentTime) }) else { return }
980+
981+
let shouldNotify = self.stateQueue.sync {
982+
self.backupFailureNotificationGate.shouldNotify(
983+
at: currentTime,
984+
notificationInterval: Self.failedBackupNotificationInterval
985+
)
879986
}
880-
881-
self.lastNotificationTime = currentTime
987+
guard shouldNotify else { return }
882988

883989
let backupCheckIntervalMinutes = Int(Self.backupFailureCheckInterval / 60)
884990
self.backupFailureSubject.send(backupCheckIntervalMinutes)
@@ -1281,3 +1387,54 @@ actor WalletBackupWriter {
12811387
}
12821388
}
12831389
}
1390+
1391+
struct BackupFailureNotificationGate {
1392+
private var isActive = false
1393+
private var isRecovering = false
1394+
private var lastNotificationTime: UInt64?
1395+
1396+
mutating func setActive(_ isActive: Bool) {
1397+
self.isActive = isActive
1398+
}
1399+
1400+
mutating func beginForegroundRecovery() -> Bool {
1401+
guard !isRecovering else { return false }
1402+
isRecovering = true
1403+
return true
1404+
}
1405+
1406+
mutating func endForegroundRecovery() {
1407+
isRecovering = false
1408+
}
1409+
1410+
func shouldCheckForFailedBackups(isObserving: Bool) -> Bool {
1411+
isActive && isObserving
1412+
}
1413+
1414+
static func hasFailedBackup(
1415+
_ status: BackupItemStatus,
1416+
at currentTime: UInt64,
1417+
failureAge: UInt64
1418+
) -> Bool {
1419+
status.isRequired && !status.running && currentTime >= status.required &&
1420+
currentTime - status.required > failureAge
1421+
}
1422+
1423+
mutating func shouldNotify(
1424+
at currentTime: UInt64,
1425+
notificationInterval: UInt64
1426+
) -> Bool {
1427+
guard isActive, !isRecovering else { return false }
1428+
1429+
if let lastNotificationTime {
1430+
guard currentTime >= lastNotificationTime,
1431+
currentTime - lastNotificationTime >= notificationInterval
1432+
else {
1433+
return false
1434+
}
1435+
}
1436+
1437+
lastNotificationTime = currentTime
1438+
return true
1439+
}
1440+
}

0 commit comments

Comments
 (0)