Retry backup failures.

This commit is contained in:
Matthew Chen 2018-03-13 14:05:51 -03:00
parent f164d5e94b
commit 05db8e3f7f
3 changed files with 195 additions and 95 deletions

View File

@ -92,7 +92,8 @@
[self updateTableContents];
dispatch_async(dispatch_get_main_queue(), ^{
[self showBackup];
// [self showBackup];
[self showDebugUI];
});
}

View File

@ -251,15 +251,11 @@ NS_ASSUME_NONNULL_BEGIN
NSDate *_Nullable lastExportFailureDate = self.lastExportFailureDate;
// Wait N hours before retrying after a success.
const NSTimeInterval kRetryAfterSuccess = 24 * kHourInterval;
// TODO: Remove.
// const NSTimeInterval kRetryAfterSuccess = 0;
if (lastExportSuccessDate && fabs(lastExportSuccessDate.timeIntervalSinceNow) < kRetryAfterSuccess) {
return NO;
}
// Wait N hours before retrying after a failure.
const NSTimeInterval kRetryAfterFailure = 6 * kHourInterval;
// TODO: Remove.
// const NSTimeInterval kRetryAfterFailure = 0;
if (lastExportFailureDate && fabs(lastExportFailureDate.timeIntervalSinceNow) < kRetryAfterFailure) {
return NO;
}

View File

@ -13,16 +13,24 @@ import CloudKit
static let signalBackupRecordType = "signalBackup"
static let manifestRecordName = "manifest"
static let payloadKey = "payload"
static let maxImmediateRetries = 5
@objc
public class func recordIdForTest() -> String {
private class func recordIdForTest() -> String {
return "test-\(NSUUID().uuidString)"
}
private class func database() -> CKDatabase {
let myContainer = CKContainer.default()
let privateDatabase = myContainer.privateCloudDatabase
return privateDatabase
}
// MARK: - Upload
@objc
public class func saveTestFileToCloud(fileUrl: URL,
success: @escaping (String) -> Void,
failure: @escaping (Error) -> Void) {
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
saveFileToCloud(fileUrl: fileUrl,
recordName: NSUUID().uuidString,
recordType: signalBackupRecordType,
@ -36,13 +44,13 @@ import CloudKit
// complete.
@objc
public class func saveEphemeralDatabaseFileToCloud(fileUrl: URL,
success: @escaping (String) -> Void,
failure: @escaping (Error) -> Void) {
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
saveFileToCloud(fileUrl: fileUrl,
recordName: "ephemeralFile-\(NSUUID().uuidString)",
recordType: signalBackupRecordType,
success: success,
failure: failure)
recordType: signalBackupRecordType,
success: success,
failure: failure)
}
// "Persistent" files may be shared between backup export; they should only be saved
@ -50,9 +58,9 @@ import CloudKit
// backups can reuse the same record.
@objc
public class func savePersistentFileOnceToCloud(fileId: String,
fileUrlBlock: @escaping (()) -> URL?,
success: @escaping (String) -> Void,
failure: @escaping (Error) -> Void) {
fileUrlBlock: @escaping (Swift.Void) -> URL?,
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
saveFileOnceToCloud(recordName: "persistentFile-\(fileId)",
recordType: signalBackupRecordType,
fileUrlBlock: fileUrlBlock,
@ -62,8 +70,8 @@ import CloudKit
@objc
public class func upsertManifestFileToCloud(fileUrl: URL,
success: @escaping (String) -> Void,
failure: @escaping (Error) -> Void) {
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
// We want to use a well-known record id and type for manifest files.
upsertFileToCloud(fileUrl: fileUrl,
recordName: manifestRecordName,
@ -76,8 +84,8 @@ import CloudKit
public class func saveFileToCloud(fileUrl: URL,
recordName: String,
recordType: String,
success: @escaping (String) -> Void,
failure: @escaping (Error) -> Void) {
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
let recordID = CKRecordID(recordName: recordName)
let record = CKRecord(recordType: recordType, recordID: recordID)
let asset = CKAsset(fileURL: fileUrl)
@ -90,49 +98,45 @@ import CloudKit
@objc
public class func saveRecordToCloud(record: CKRecord,
success: @escaping (String) -> Void,
failure: @escaping (Error) -> Void) {
let myContainer = CKContainer.default()
let privateDatabase = myContainer.privateCloudDatabase
privateDatabase.save(record) {
(record, error) in
if let error = error {
Logger.error("\(self.logTag) error saving record: \(error)")
failure(error)
} else {
guard let recordName = record?.recordID.recordName else {
Logger.error("\(self.logTag) error retrieving saved record's name.")
failure(OWSErrorWithCodeDescription(.exportBackupError,
NSLocalizedString("BACKUP_EXPORT_ERROR_SAVE_FILE_TO_CLOUD_FAILED",
comment: "Error indicating the a backup export failed to save a file to the cloud.")))
return
}
Logger.info("\(self.logTag) saved record.")
success(recordName)
}
}
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
saveRecordToCloud(record: record,
remainingRetries: maxImmediateRetries,
success: success,
failure: failure)
}
@objc
public class func deleteRecordFromCloud(recordName: String,
success: @escaping (()) -> Void,
failure: @escaping (Error) -> Void) {
private class func saveRecordToCloud(record: CKRecord,
remainingRetries: Int,
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
let recordID = CKRecordID(recordName: recordName)
database().save(record) {
(_, error) in
let myContainer = CKContainer.default()
let privateDatabase = myContainer.privateCloudDatabase
privateDatabase.delete(withRecordID: recordID) {
(record, error) in
if let error = error {
Logger.error("\(self.logTag) error deleting record: \(error)")
failure(error)
} else {
Logger.info("\(self.logTag) deleted record.")
success()
let response = responseForCloudKitError(error: error,
remainingRetries: remainingRetries,
label: "Save Record")
switch response {
case .success:
let recordName = record.recordID.recordName
success(recordName)
case .failureDoNotRetry(let responseError):
failure(responseError)
case .failureRetryAfterDelay(let retryDelay):
DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + retryDelay, execute: {
saveRecordToCloud(record: record,
remainingRetries: remainingRetries - 1,
success: success,
failure: failure)
})
case .failureRetryWithoutDelay:
DispatchQueue.global().async {
saveRecordToCloud(record: record,
remainingRetries: remainingRetries - 1,
success: success,
failure: failure)
}
}
}
}
@ -146,8 +150,8 @@ import CloudKit
public class func upsertFileToCloud(fileUrl: URL,
recordName: String,
recordType: String,
success: @escaping (String) -> Void,
failure: @escaping (Error) -> Void) {
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
checkForFileInCloud(recordName: recordName,
success: { (record) in
@ -178,9 +182,9 @@ import CloudKit
@objc
public class func saveFileOnceToCloud(recordName: String,
recordType: String,
fileUrlBlock: @escaping (()) -> URL?,
success: @escaping (String) -> Void,
failure: @escaping (Error) -> Void) {
fileUrlBlock: @escaping (Swift.Void) -> URL?,
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
checkForFileInCloud(recordName: recordName,
success: { (record) in
@ -207,9 +211,59 @@ import CloudKit
failure: failure)
}
// MARK: - Delete
@objc
public class func deleteRecordFromCloud(recordName: String,
success: @escaping (Swift.Void) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
deleteRecordFromCloud(recordName: recordName,
remainingRetries: maxImmediateRetries,
success: success,
failure: failure)
}
private class func deleteRecordFromCloud(recordName: String,
remainingRetries: Int,
success: @escaping (Swift.Void) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
let recordID = CKRecordID(recordName: recordName)
database().delete(withRecordID: recordID) {
(record, error) in
let response = responseForCloudKitError(error: error,
remainingRetries: remainingRetries,
label: "Delete Record")
switch response {
case .success:
success()
case .failureDoNotRetry(let responseError):
failure(responseError)
case .failureRetryAfterDelay(let retryDelay):
DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + retryDelay, execute: {
deleteRecordFromCloud(recordName: recordName,
remainingRetries: remainingRetries - 1,
success: success,
failure: failure)
})
case .failureRetryWithoutDelay:
DispatchQueue.global().async {
deleteRecordFromCloud(recordName: recordName,
remainingRetries: remainingRetries - 1,
success: success,
failure: failure)
}
}
}
}
// MARK: - Exists?
private class func checkForFileInCloud(recordName: String,
success: @escaping (CKRecord?) -> Void,
failure: @escaping (Error) -> Void) {
success: @escaping (CKRecord?) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
let recordId = CKRecordID(recordName: recordName)
let fetchOperation = CKFetchRecordsOperation(recordIDs: [recordId ])
// Don't download the file; we're just using the fetch to check whether or
@ -240,14 +294,12 @@ import CloudKit
// Record found.
success(record)
}
let myContainer = CKContainer.default()
let privateDatabase = myContainer.privateCloudDatabase
privateDatabase.add(fetchOperation)
database().add(fetchOperation)
}
@objc
public class func checkForManifestInCloud(success: @escaping (Bool) -> Void,
failure: @escaping (Error) -> Void) {
public class func checkForManifestInCloud(success: @escaping (Bool) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
checkForFileInCloud(recordName: manifestRecordName,
success: { (record) in
@ -257,8 +309,8 @@ import CloudKit
}
@objc
public class func fetchAllRecordNames(success: @escaping ([String]) -> Void,
failure: @escaping (Error) -> Void) {
public class func fetchAllRecordNames(success: @escaping ([String]) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
let query = CKQuery(recordType: signalBackupRecordType, predicate: NSPredicate(value: true))
// Fetch the first page of results for this query.
@ -272,12 +324,12 @@ import CloudKit
private class func fetchAllRecordNamesStep(query: CKQuery,
previousRecordNames: [String],
cursor: CKQueryCursor?,
success: @escaping ([String]) -> Void,
failure: @escaping (Error) -> Void) {
success: @escaping ([String]) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
var allRecordNames = previousRecordNames
let queryOperation = CKQueryOperation(query: query)
let queryOperation = CKQueryOperation(query: query)
// If this isn't the first page of results for this query, resume
// where we left off.
queryOperation.cursor = cursor
@ -306,25 +358,24 @@ import CloudKit
Logger.info("\(self.logTag) fetched \(allRecordNames.count) record names.")
success(allRecordNames)
}
let myContainer = CKContainer.default()
let privateDatabase = myContainer.privateCloudDatabase
privateDatabase.add(queryOperation)
database().add(queryOperation)
}
// MARK: - Download
@objc
public class func downloadManifestFromCloud(
success: @escaping (Data) -> Void,
failure: @escaping (Error) -> Void) {
success: @escaping (Data) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
downloadDataFromCloud(recordName: manifestRecordName,
success: success,
failure: failure)
success: success,
failure: failure)
}
@objc
public class func downloadDataFromCloud(recordName: String,
success: @escaping (Data) -> Void,
failure: @escaping (Error) -> Void) {
success: @escaping (Data) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
downloadFromCloud(recordName: recordName,
success: { (asset) in
@ -346,8 +397,8 @@ import CloudKit
@objc
public class func downloadFileFromCloud(recordName: String,
toFileUrl: URL,
success: @escaping (()) -> Void,
failure: @escaping (Error) -> Void) {
success: @escaping (Swift.Void) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
downloadFromCloud(recordName: recordName,
success: { (asset) in
@ -367,8 +418,8 @@ import CloudKit
}
private class func downloadFromCloud(recordName: String,
success: @escaping (CKAsset) -> Void,
failure: @escaping (Error) -> Void) {
success: @escaping (CKAsset) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
let recordId = CKRecordID(recordName: recordName)
let fetchOperation = CKFetchRecordsOperation(recordIDs: [recordId ])
@ -394,13 +445,13 @@ import CloudKit
}
success(asset)
}
let myContainer = CKContainer.default()
let privateDatabase = myContainer.privateCloudDatabase
privateDatabase.add(fetchOperation)
database().add(fetchOperation)
}
// MARK: - Access
@objc
public class func checkCloudKitAccess(completion: @escaping (Bool) -> Void) {
public class func checkCloudKitAccess(completion: @escaping (Bool) -> Swift.Void) {
CKContainer.default().accountStatus(completionHandler: { (accountStatus, error) in
DispatchQueue.main.async {
switch accountStatus {
@ -422,4 +473,56 @@ import CloudKit
}
})
}
// MARK: - Retry
private enum CKErrorResponse {
case success
case failureDoNotRetry(error:Error)
case failureRetryAfterDelay(retryDelay: Double)
case failureRetryWithoutDelay
}
private class func responseForCloudKitError(error: Error?,
remainingRetries: Int,
label: String) -> CKErrorResponse {
if let error = error as? CKError {
Logger.error("\(self.logTag) \(label) failed: \(error)")
if remainingRetries < 1 {
Logger.verbose("\(self.logTag) \(label) no more retries.")
return .failureDoNotRetry(error:error)
}
if #available(iOS 11, *) {
if error.code == CKError.serverResponseLost {
Logger.verbose("\(self.logTag) \(label) retry without delay.")
return .failureRetryWithoutDelay
}
}
switch error {
case CKError.requestRateLimited, CKError.serviceUnavailable, CKError.zoneBusy:
let retryDelay = error.retryAfterSeconds ?? 3.0
Logger.verbose("\(self.logTag) \(label) retry with delay: \(retryDelay).")
return .failureRetryAfterDelay(retryDelay:retryDelay)
case CKError.networkFailure:
Logger.verbose("\(self.logTag) \(label) retry without delay.")
return .failureRetryWithoutDelay
default:
Logger.verbose("\(self.logTag) \(label) unknown CKError.")
return .failureDoNotRetry(error:error)
}
} else if let error = error {
Logger.error("\(self.logTag) \(label) failed: \(error)")
if remainingRetries < 1 {
Logger.verbose("\(self.logTag) \(label) no more retries.")
return .failureDoNotRetry(error:error)
}
Logger.verbose("\(self.logTag) \(label) unknown error.")
return .failureDoNotRetry(error:error)
} else {
Logger.info("\(self.logTag) \(label) succeeded.")
return .success
}
}
}