Retry backup failures.
This commit is contained in:
parent
f164d5e94b
commit
05db8e3f7f
|
@ -92,7 +92,8 @@
|
|||
[self updateTableContents];
|
||||
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
[self showBackup];
|
||||
// [self showBackup];
|
||||
[self showDebugUI];
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
@ -251,15 +251,11 @@ NS_ASSUME_NONNULL_BEGIN
|
|||
NSDate *_Nullable lastExportFailureDate = self.lastExportFailureDate;
|
||||
// Wait N hours before retrying after a success.
|
||||
const NSTimeInterval kRetryAfterSuccess = 24 * kHourInterval;
|
||||
// TODO: Remove.
|
||||
// const NSTimeInterval kRetryAfterSuccess = 0;
|
||||
if (lastExportSuccessDate && fabs(lastExportSuccessDate.timeIntervalSinceNow) < kRetryAfterSuccess) {
|
||||
return NO;
|
||||
}
|
||||
// Wait N hours before retrying after a failure.
|
||||
const NSTimeInterval kRetryAfterFailure = 6 * kHourInterval;
|
||||
// TODO: Remove.
|
||||
// const NSTimeInterval kRetryAfterFailure = 0;
|
||||
if (lastExportFailureDate && fabs(lastExportFailureDate.timeIntervalSinceNow) < kRetryAfterFailure) {
|
||||
return NO;
|
||||
}
|
||||
|
|
|
@ -13,16 +13,24 @@ import CloudKit
|
|||
static let signalBackupRecordType = "signalBackup"
|
||||
static let manifestRecordName = "manifest"
|
||||
static let payloadKey = "payload"
|
||||
static let maxImmediateRetries = 5
|
||||
|
||||
@objc
|
||||
public class func recordIdForTest() -> String {
|
||||
private class func recordIdForTest() -> String {
|
||||
return "test-\(NSUUID().uuidString)"
|
||||
}
|
||||
|
||||
private class func database() -> CKDatabase {
|
||||
let myContainer = CKContainer.default()
|
||||
let privateDatabase = myContainer.privateCloudDatabase
|
||||
return privateDatabase
|
||||
}
|
||||
|
||||
// MARK: - Upload
|
||||
|
||||
@objc
|
||||
public class func saveTestFileToCloud(fileUrl: URL,
|
||||
success: @escaping (String) -> Void,
|
||||
failure: @escaping (Error) -> Void) {
|
||||
success: @escaping (String) -> Swift.Void,
|
||||
failure: @escaping (Error) -> Swift.Void) {
|
||||
saveFileToCloud(fileUrl: fileUrl,
|
||||
recordName: NSUUID().uuidString,
|
||||
recordType: signalBackupRecordType,
|
||||
|
@ -36,13 +44,13 @@ import CloudKit
|
|||
// complete.
|
||||
@objc
|
||||
public class func saveEphemeralDatabaseFileToCloud(fileUrl: URL,
|
||||
success: @escaping (String) -> Void,
|
||||
failure: @escaping (Error) -> Void) {
|
||||
success: @escaping (String) -> Swift.Void,
|
||||
failure: @escaping (Error) -> Swift.Void) {
|
||||
saveFileToCloud(fileUrl: fileUrl,
|
||||
recordName: "ephemeralFile-\(NSUUID().uuidString)",
|
||||
recordType: signalBackupRecordType,
|
||||
success: success,
|
||||
failure: failure)
|
||||
recordType: signalBackupRecordType,
|
||||
success: success,
|
||||
failure: failure)
|
||||
}
|
||||
|
||||
// "Persistent" files may be shared between backup export; they should only be saved
|
||||
|
@ -50,9 +58,9 @@ import CloudKit
|
|||
// backups can reuse the same record.
|
||||
@objc
|
||||
public class func savePersistentFileOnceToCloud(fileId: String,
|
||||
fileUrlBlock: @escaping (()) -> URL?,
|
||||
success: @escaping (String) -> Void,
|
||||
failure: @escaping (Error) -> Void) {
|
||||
fileUrlBlock: @escaping (Swift.Void) -> URL?,
|
||||
success: @escaping (String) -> Swift.Void,
|
||||
failure: @escaping (Error) -> Swift.Void) {
|
||||
saveFileOnceToCloud(recordName: "persistentFile-\(fileId)",
|
||||
recordType: signalBackupRecordType,
|
||||
fileUrlBlock: fileUrlBlock,
|
||||
|
@ -62,8 +70,8 @@ import CloudKit
|
|||
|
||||
@objc
|
||||
public class func upsertManifestFileToCloud(fileUrl: URL,
|
||||
success: @escaping (String) -> Void,
|
||||
failure: @escaping (Error) -> Void) {
|
||||
success: @escaping (String) -> Swift.Void,
|
||||
failure: @escaping (Error) -> Swift.Void) {
|
||||
// We want to use a well-known record id and type for manifest files.
|
||||
upsertFileToCloud(fileUrl: fileUrl,
|
||||
recordName: manifestRecordName,
|
||||
|
@ -76,8 +84,8 @@ import CloudKit
|
|||
public class func saveFileToCloud(fileUrl: URL,
|
||||
recordName: String,
|
||||
recordType: String,
|
||||
success: @escaping (String) -> Void,
|
||||
failure: @escaping (Error) -> Void) {
|
||||
success: @escaping (String) -> Swift.Void,
|
||||
failure: @escaping (Error) -> Swift.Void) {
|
||||
let recordID = CKRecordID(recordName: recordName)
|
||||
let record = CKRecord(recordType: recordType, recordID: recordID)
|
||||
let asset = CKAsset(fileURL: fileUrl)
|
||||
|
@ -90,49 +98,45 @@ import CloudKit
|
|||
|
||||
@objc
|
||||
public class func saveRecordToCloud(record: CKRecord,
|
||||
success: @escaping (String) -> Void,
|
||||
failure: @escaping (Error) -> Void) {
|
||||
|
||||
let myContainer = CKContainer.default()
|
||||
let privateDatabase = myContainer.privateCloudDatabase
|
||||
privateDatabase.save(record) {
|
||||
(record, error) in
|
||||
|
||||
if let error = error {
|
||||
Logger.error("\(self.logTag) error saving record: \(error)")
|
||||
failure(error)
|
||||
} else {
|
||||
guard let recordName = record?.recordID.recordName else {
|
||||
Logger.error("\(self.logTag) error retrieving saved record's name.")
|
||||
failure(OWSErrorWithCodeDescription(.exportBackupError,
|
||||
NSLocalizedString("BACKUP_EXPORT_ERROR_SAVE_FILE_TO_CLOUD_FAILED",
|
||||
comment: "Error indicating the a backup export failed to save a file to the cloud.")))
|
||||
return
|
||||
}
|
||||
Logger.info("\(self.logTag) saved record.")
|
||||
success(recordName)
|
||||
}
|
||||
}
|
||||
success: @escaping (String) -> Swift.Void,
|
||||
failure: @escaping (Error) -> Swift.Void) {
|
||||
saveRecordToCloud(record: record,
|
||||
remainingRetries: maxImmediateRetries,
|
||||
success: success,
|
||||
failure: failure)
|
||||
}
|
||||
|
||||
@objc
|
||||
public class func deleteRecordFromCloud(recordName: String,
|
||||
success: @escaping (()) -> Void,
|
||||
failure: @escaping (Error) -> Void) {
|
||||
private class func saveRecordToCloud(record: CKRecord,
|
||||
remainingRetries: Int,
|
||||
success: @escaping (String) -> Swift.Void,
|
||||
failure: @escaping (Error) -> Swift.Void) {
|
||||
|
||||
let recordID = CKRecordID(recordName: recordName)
|
||||
database().save(record) {
|
||||
(_, error) in
|
||||
|
||||
let myContainer = CKContainer.default()
|
||||
let privateDatabase = myContainer.privateCloudDatabase
|
||||
privateDatabase.delete(withRecordID: recordID) {
|
||||
(record, error) in
|
||||
|
||||
if let error = error {
|
||||
Logger.error("\(self.logTag) error deleting record: \(error)")
|
||||
failure(error)
|
||||
} else {
|
||||
Logger.info("\(self.logTag) deleted record.")
|
||||
success()
|
||||
let response = responseForCloudKitError(error: error,
|
||||
remainingRetries: remainingRetries,
|
||||
label: "Save Record")
|
||||
switch response {
|
||||
case .success:
|
||||
let recordName = record.recordID.recordName
|
||||
success(recordName)
|
||||
case .failureDoNotRetry(let responseError):
|
||||
failure(responseError)
|
||||
case .failureRetryAfterDelay(let retryDelay):
|
||||
DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + retryDelay, execute: {
|
||||
saveRecordToCloud(record: record,
|
||||
remainingRetries: remainingRetries - 1,
|
||||
success: success,
|
||||
failure: failure)
|
||||
})
|
||||
case .failureRetryWithoutDelay:
|
||||
DispatchQueue.global().async {
|
||||
saveRecordToCloud(record: record,
|
||||
remainingRetries: remainingRetries - 1,
|
||||
success: success,
|
||||
failure: failure)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -146,8 +150,8 @@ import CloudKit
|
|||
public class func upsertFileToCloud(fileUrl: URL,
|
||||
recordName: String,
|
||||
recordType: String,
|
||||
success: @escaping (String) -> Void,
|
||||
failure: @escaping (Error) -> Void) {
|
||||
success: @escaping (String) -> Swift.Void,
|
||||
failure: @escaping (Error) -> Swift.Void) {
|
||||
|
||||
checkForFileInCloud(recordName: recordName,
|
||||
success: { (record) in
|
||||
|
@ -178,9 +182,9 @@ import CloudKit
|
|||
@objc
|
||||
public class func saveFileOnceToCloud(recordName: String,
|
||||
recordType: String,
|
||||
fileUrlBlock: @escaping (()) -> URL?,
|
||||
success: @escaping (String) -> Void,
|
||||
failure: @escaping (Error) -> Void) {
|
||||
fileUrlBlock: @escaping (Swift.Void) -> URL?,
|
||||
success: @escaping (String) -> Swift.Void,
|
||||
failure: @escaping (Error) -> Swift.Void) {
|
||||
|
||||
checkForFileInCloud(recordName: recordName,
|
||||
success: { (record) in
|
||||
|
@ -207,9 +211,59 @@ import CloudKit
|
|||
failure: failure)
|
||||
}
|
||||
|
||||
// MARK: - Delete
|
||||
|
||||
@objc
|
||||
public class func deleteRecordFromCloud(recordName: String,
|
||||
success: @escaping (Swift.Void) -> Swift.Void,
|
||||
failure: @escaping (Error) -> Swift.Void) {
|
||||
deleteRecordFromCloud(recordName: recordName,
|
||||
remainingRetries: maxImmediateRetries,
|
||||
success: success,
|
||||
failure: failure)
|
||||
}
|
||||
|
||||
private class func deleteRecordFromCloud(recordName: String,
|
||||
remainingRetries: Int,
|
||||
success: @escaping (Swift.Void) -> Swift.Void,
|
||||
failure: @escaping (Error) -> Swift.Void) {
|
||||
|
||||
let recordID = CKRecordID(recordName: recordName)
|
||||
|
||||
database().delete(withRecordID: recordID) {
|
||||
(record, error) in
|
||||
|
||||
let response = responseForCloudKitError(error: error,
|
||||
remainingRetries: remainingRetries,
|
||||
label: "Delete Record")
|
||||
switch response {
|
||||
case .success:
|
||||
success()
|
||||
case .failureDoNotRetry(let responseError):
|
||||
failure(responseError)
|
||||
case .failureRetryAfterDelay(let retryDelay):
|
||||
DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + retryDelay, execute: {
|
||||
deleteRecordFromCloud(recordName: recordName,
|
||||
remainingRetries: remainingRetries - 1,
|
||||
success: success,
|
||||
failure: failure)
|
||||
})
|
||||
case .failureRetryWithoutDelay:
|
||||
DispatchQueue.global().async {
|
||||
deleteRecordFromCloud(recordName: recordName,
|
||||
remainingRetries: remainingRetries - 1,
|
||||
success: success,
|
||||
failure: failure)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Exists?
|
||||
|
||||
private class func checkForFileInCloud(recordName: String,
|
||||
success: @escaping (CKRecord?) -> Void,
|
||||
failure: @escaping (Error) -> Void) {
|
||||
success: @escaping (CKRecord?) -> Swift.Void,
|
||||
failure: @escaping (Error) -> Swift.Void) {
|
||||
let recordId = CKRecordID(recordName: recordName)
|
||||
let fetchOperation = CKFetchRecordsOperation(recordIDs: [recordId ])
|
||||
// Don't download the file; we're just using the fetch to check whether or
|
||||
|
@ -240,14 +294,12 @@ import CloudKit
|
|||
// Record found.
|
||||
success(record)
|
||||
}
|
||||
let myContainer = CKContainer.default()
|
||||
let privateDatabase = myContainer.privateCloudDatabase
|
||||
privateDatabase.add(fetchOperation)
|
||||
database().add(fetchOperation)
|
||||
}
|
||||
|
||||
@objc
|
||||
public class func checkForManifestInCloud(success: @escaping (Bool) -> Void,
|
||||
failure: @escaping (Error) -> Void) {
|
||||
public class func checkForManifestInCloud(success: @escaping (Bool) -> Swift.Void,
|
||||
failure: @escaping (Error) -> Swift.Void) {
|
||||
|
||||
checkForFileInCloud(recordName: manifestRecordName,
|
||||
success: { (record) in
|
||||
|
@ -257,8 +309,8 @@ import CloudKit
|
|||
}
|
||||
|
||||
@objc
|
||||
public class func fetchAllRecordNames(success: @escaping ([String]) -> Void,
|
||||
failure: @escaping (Error) -> Void) {
|
||||
public class func fetchAllRecordNames(success: @escaping ([String]) -> Swift.Void,
|
||||
failure: @escaping (Error) -> Swift.Void) {
|
||||
|
||||
let query = CKQuery(recordType: signalBackupRecordType, predicate: NSPredicate(value: true))
|
||||
// Fetch the first page of results for this query.
|
||||
|
@ -272,12 +324,12 @@ import CloudKit
|
|||
private class func fetchAllRecordNamesStep(query: CKQuery,
|
||||
previousRecordNames: [String],
|
||||
cursor: CKQueryCursor?,
|
||||
success: @escaping ([String]) -> Void,
|
||||
failure: @escaping (Error) -> Void) {
|
||||
success: @escaping ([String]) -> Swift.Void,
|
||||
failure: @escaping (Error) -> Swift.Void) {
|
||||
|
||||
var allRecordNames = previousRecordNames
|
||||
|
||||
let queryOperation = CKQueryOperation(query: query)
|
||||
let queryOperation = CKQueryOperation(query: query)
|
||||
// If this isn't the first page of results for this query, resume
|
||||
// where we left off.
|
||||
queryOperation.cursor = cursor
|
||||
|
@ -306,25 +358,24 @@ import CloudKit
|
|||
Logger.info("\(self.logTag) fetched \(allRecordNames.count) record names.")
|
||||
success(allRecordNames)
|
||||
}
|
||||
|
||||
let myContainer = CKContainer.default()
|
||||
let privateDatabase = myContainer.privateCloudDatabase
|
||||
privateDatabase.add(queryOperation)
|
||||
database().add(queryOperation)
|
||||
}
|
||||
|
||||
// MARK: - Download
|
||||
|
||||
@objc
|
||||
public class func downloadManifestFromCloud(
|
||||
success: @escaping (Data) -> Void,
|
||||
failure: @escaping (Error) -> Void) {
|
||||
success: @escaping (Data) -> Swift.Void,
|
||||
failure: @escaping (Error) -> Swift.Void) {
|
||||
downloadDataFromCloud(recordName: manifestRecordName,
|
||||
success: success,
|
||||
failure: failure)
|
||||
success: success,
|
||||
failure: failure)
|
||||
}
|
||||
|
||||
@objc
|
||||
public class func downloadDataFromCloud(recordName: String,
|
||||
success: @escaping (Data) -> Void,
|
||||
failure: @escaping (Error) -> Void) {
|
||||
success: @escaping (Data) -> Swift.Void,
|
||||
failure: @escaping (Error) -> Swift.Void) {
|
||||
|
||||
downloadFromCloud(recordName: recordName,
|
||||
success: { (asset) in
|
||||
|
@ -346,8 +397,8 @@ import CloudKit
|
|||
@objc
|
||||
public class func downloadFileFromCloud(recordName: String,
|
||||
toFileUrl: URL,
|
||||
success: @escaping (()) -> Void,
|
||||
failure: @escaping (Error) -> Void) {
|
||||
success: @escaping (Swift.Void) -> Swift.Void,
|
||||
failure: @escaping (Error) -> Swift.Void) {
|
||||
|
||||
downloadFromCloud(recordName: recordName,
|
||||
success: { (asset) in
|
||||
|
@ -367,8 +418,8 @@ import CloudKit
|
|||
}
|
||||
|
||||
private class func downloadFromCloud(recordName: String,
|
||||
success: @escaping (CKAsset) -> Void,
|
||||
failure: @escaping (Error) -> Void) {
|
||||
success: @escaping (CKAsset) -> Swift.Void,
|
||||
failure: @escaping (Error) -> Swift.Void) {
|
||||
|
||||
let recordId = CKRecordID(recordName: recordName)
|
||||
let fetchOperation = CKFetchRecordsOperation(recordIDs: [recordId ])
|
||||
|
@ -394,13 +445,13 @@ import CloudKit
|
|||
}
|
||||
success(asset)
|
||||
}
|
||||
let myContainer = CKContainer.default()
|
||||
let privateDatabase = myContainer.privateCloudDatabase
|
||||
privateDatabase.add(fetchOperation)
|
||||
database().add(fetchOperation)
|
||||
}
|
||||
|
||||
// MARK: - Access
|
||||
|
||||
@objc
|
||||
public class func checkCloudKitAccess(completion: @escaping (Bool) -> Void) {
|
||||
public class func checkCloudKitAccess(completion: @escaping (Bool) -> Swift.Void) {
|
||||
CKContainer.default().accountStatus(completionHandler: { (accountStatus, error) in
|
||||
DispatchQueue.main.async {
|
||||
switch accountStatus {
|
||||
|
@ -422,4 +473,56 @@ import CloudKit
|
|||
}
|
||||
})
|
||||
}
|
||||
|
||||
// MARK: - Retry
|
||||
|
||||
private enum CKErrorResponse {
|
||||
case success
|
||||
case failureDoNotRetry(error:Error)
|
||||
case failureRetryAfterDelay(retryDelay: Double)
|
||||
case failureRetryWithoutDelay
|
||||
}
|
||||
|
||||
private class func responseForCloudKitError(error: Error?,
|
||||
remainingRetries: Int,
|
||||
label: String) -> CKErrorResponse {
|
||||
if let error = error as? CKError {
|
||||
Logger.error("\(self.logTag) \(label) failed: \(error)")
|
||||
if remainingRetries < 1 {
|
||||
Logger.verbose("\(self.logTag) \(label) no more retries.")
|
||||
return .failureDoNotRetry(error:error)
|
||||
}
|
||||
|
||||
if #available(iOS 11, *) {
|
||||
if error.code == CKError.serverResponseLost {
|
||||
Logger.verbose("\(self.logTag) \(label) retry without delay.")
|
||||
return .failureRetryWithoutDelay
|
||||
}
|
||||
}
|
||||
|
||||
switch error {
|
||||
case CKError.requestRateLimited, CKError.serviceUnavailable, CKError.zoneBusy:
|
||||
let retryDelay = error.retryAfterSeconds ?? 3.0
|
||||
Logger.verbose("\(self.logTag) \(label) retry with delay: \(retryDelay).")
|
||||
return .failureRetryAfterDelay(retryDelay:retryDelay)
|
||||
case CKError.networkFailure:
|
||||
Logger.verbose("\(self.logTag) \(label) retry without delay.")
|
||||
return .failureRetryWithoutDelay
|
||||
default:
|
||||
Logger.verbose("\(self.logTag) \(label) unknown CKError.")
|
||||
return .failureDoNotRetry(error:error)
|
||||
}
|
||||
} else if let error = error {
|
||||
Logger.error("\(self.logTag) \(label) failed: \(error)")
|
||||
if remainingRetries < 1 {
|
||||
Logger.verbose("\(self.logTag) \(label) no more retries.")
|
||||
return .failureDoNotRetry(error:error)
|
||||
}
|
||||
Logger.verbose("\(self.logTag) \(label) unknown error.")
|
||||
return .failureDoNotRetry(error:error)
|
||||
} else {
|
||||
Logger.info("\(self.logTag) \(label) succeeded.")
|
||||
return .success
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue