Retry backup failures.

This commit is contained in:
Matthew Chen 2018-03-13 14:39:00 -03:00
parent 05db8e3f7f
commit cf13a780e9
2 changed files with 151 additions and 64 deletions

View File

@ -13,7 +13,7 @@ import CloudKit
static let signalBackupRecordType = "signalBackup"
static let manifestRecordName = "manifest"
static let payloadKey = "payload"
static let maxImmediateRetries = 5
static let maxRetries = 5
private class func recordIdForTest() -> String {
return "test-\(NSUUID().uuidString)"
@ -25,6 +25,12 @@ import CloudKit
return privateDatabase
}
private class func invalidServiceResponseError() -> Error {
return OWSErrorWithCodeDescription(.backupFailure,
NSLocalizedString("BACKUP_EXPORT_ERROR_INVALID_CLOUDKIT_RESPONSE",
comment: "Error indicating that the app received an invalid response from CloudKit."))
}
// MARK: - Upload
@objc
@ -101,7 +107,7 @@ import CloudKit
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
saveRecordToCloud(record: record,
remainingRetries: maxImmediateRetries,
remainingRetries: maxRetries,
success: success,
failure: failure)
}
@ -137,6 +143,9 @@ import CloudKit
success: success,
failure: failure)
}
case .unknownItem:
owsFail("\(self.logTag) unexpected CloudKit response.")
failure(invalidServiceResponseError())
}
}
}
@ -154,6 +163,7 @@ import CloudKit
failure: @escaping (Error) -> Swift.Void) {
checkForFileInCloud(recordName: recordName,
remainingRetries: maxRetries,
success: { (record) in
if let record = record {
// Record found, updating existing record.
@ -187,6 +197,7 @@ import CloudKit
failure: @escaping (Error) -> Swift.Void) {
checkForFileInCloud(recordName: recordName,
remainingRetries: maxRetries,
success: { (record) in
if record != nil {
// Record found, skipping save.
@ -218,7 +229,7 @@ import CloudKit
success: @escaping (Swift.Void) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
deleteRecordFromCloud(recordName: recordName,
remainingRetries: maxImmediateRetries,
remainingRetries: maxRetries,
success: success,
failure: failure)
}
@ -231,7 +242,7 @@ import CloudKit
let recordID = CKRecordID(recordName: recordName)
database().delete(withRecordID: recordID) {
(record, error) in
(_, error) in
let response = responseForCloudKitError(error: error,
remainingRetries: remainingRetries,
@ -255,6 +266,9 @@ import CloudKit
success: success,
failure: failure)
}
case .unknownItem:
owsFail("\(self.logTag) unexpected CloudKit response.")
failure(invalidServiceResponseError())
}
}
}
@ -262,6 +276,7 @@ import CloudKit
// MARK: - Exists?
private class func checkForFileInCloud(recordName: String,
remainingRetries: Int,
success: @escaping (CKRecord?) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
let recordId = CKRecordID(recordName: recordName)
@ -270,29 +285,39 @@ import CloudKit
// not this record already exists.
fetchOperation.desiredKeys = []
fetchOperation.perRecordCompletionBlock = { (record, recordId, error) in
if let error = error {
if let ckerror = error as? CKError {
if ckerror.code == .unknownItem {
// Record not found.
success(nil)
return
}
Logger.error("\(self.logTag) error fetching record: \(error) \(ckerror.code).")
} else {
Logger.error("\(self.logTag) error fetching record: \(error).")
let response = responseForCloudKitError(error: error,
remainingRetries: remainingRetries,
label: "Check for Record")
switch response {
case .success:
guard let record = record else {
owsFail("\(self.logTag) missing fetching record.")
failure(invalidServiceResponseError())
return
}
failure(error)
return
// Record found.
success(record)
case .failureDoNotRetry(let responseError):
failure(responseError)
case .failureRetryAfterDelay(let retryDelay):
DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + retryDelay, execute: {
checkForFileInCloud(recordName: recordName,
remainingRetries: remainingRetries - 1,
success: success,
failure: failure)
})
case .failureRetryWithoutDelay:
DispatchQueue.global().async {
checkForFileInCloud(recordName: recordName,
remainingRetries: remainingRetries - 1,
success: success,
failure: failure)
}
case .unknownItem:
// Record not found.
success(nil)
}
guard let record = record else {
Logger.error("\(self.logTag) missing fetching record.")
failure(OWSErrorWithCodeDescription(.exportBackupError,
NSLocalizedString("BACKUP_EXPORT_ERROR_SAVE_FILE_TO_CLOUD_FAILED",
comment: "Error indicating the a backup export failed to save a file to the cloud.")))
return
}
// Record found.
success(record)
}
database().add(fetchOperation)
}
@ -302,6 +327,7 @@ import CloudKit
failure: @escaping (Error) -> Swift.Void) {
checkForFileInCloud(recordName: manifestRecordName,
remainingRetries: maxRetries,
success: { (record) in
success(record != nil)
},
@ -317,6 +343,7 @@ import CloudKit
fetchAllRecordNamesStep(query: query,
previousRecordNames: [String](),
cursor: nil,
remainingRetries: maxRetries,
success: success,
failure: failure)
}
@ -324,6 +351,7 @@ import CloudKit
private class func fetchAllRecordNamesStep(query: CKQuery,
previousRecordNames: [String],
cursor: CKQueryCursor?,
remainingRetries: Int,
success: @escaping ([String]) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
@ -340,23 +368,49 @@ import CloudKit
allRecordNames.append(record.recordID.recordName)
}
queryOperation.queryCompletionBlock = { (cursor, error) in
if let error = error {
Logger.error("\(self.logTag) error fetching all record names: \(error).")
failure(error)
return
let response = responseForCloudKitError(error: error,
remainingRetries: remainingRetries,
label: "Fetch All Records")
switch response {
case .success:
if let cursor = cursor {
Logger.verbose("\(self.logTag) fetching more record names \(allRecordNames.count).")
// There are more pages of results, continue fetching.
fetchAllRecordNamesStep(query: query,
previousRecordNames: allRecordNames,
cursor: cursor,
remainingRetries: maxRetries,
success: success,
failure: failure)
return
}
Logger.info("\(self.logTag) fetched \(allRecordNames.count) record names.")
success(allRecordNames)
case .failureDoNotRetry(let responseError):
failure(responseError)
case .failureRetryAfterDelay(let retryDelay):
DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + retryDelay, execute: {
fetchAllRecordNamesStep(query: query,
previousRecordNames: allRecordNames,
cursor: cursor,
remainingRetries: remainingRetries - 1,
success: success,
failure: failure)
})
case .failureRetryWithoutDelay:
DispatchQueue.global().async {
fetchAllRecordNamesStep(query: query,
previousRecordNames: allRecordNames,
cursor: cursor,
remainingRetries: remainingRetries - 1,
success: success,
failure: failure)
}
case .unknownItem:
owsFail("\(self.logTag) unexpected CloudKit response.")
failure(invalidServiceResponseError())
}
if let cursor = cursor {
Logger.verbose("\(self.logTag) fetching more record names \(allRecordNames.count).")
// There are more pages of results, continue fetching.
fetchAllRecordNamesStep(query: query,
previousRecordNames: allRecordNames,
cursor: cursor,
success: success,
failure: failure)
return
}
Logger.info("\(self.logTag) fetched \(allRecordNames.count) record names.")
success(allRecordNames)
}
database().add(queryOperation)
}
@ -378,6 +432,7 @@ import CloudKit
failure: @escaping (Error) -> Swift.Void) {
downloadFromCloud(recordName: recordName,
remainingRetries: maxRetries,
success: { (asset) in
DispatchQueue.global().async {
do {
@ -385,9 +440,7 @@ import CloudKit
success(data)
} catch {
Logger.error("\(self.logTag) couldn't load asset file: \(error).")
failure(OWSErrorWithCodeDescription(.exportBackupError,
NSLocalizedString("BACKUP_IMPORT_ERROR_DOWNLOAD_FILE_FROM_CLOUD_FAILED",
comment: "Error indicating the a backup import failed to download a file from the cloud.")))
failure(invalidServiceResponseError())
}
}
},
@ -401,6 +454,7 @@ import CloudKit
failure: @escaping (Error) -> Swift.Void) {
downloadFromCloud(recordName: recordName,
remainingRetries: maxRetries,
success: { (asset) in
DispatchQueue.global().async {
do {
@ -408,16 +462,20 @@ import CloudKit
success()
} catch {
Logger.error("\(self.logTag) couldn't copy asset file: \(error).")
failure(OWSErrorWithCodeDescription(.exportBackupError,
NSLocalizedString("BACKUP_IMPORT_ERROR_DOWNLOAD_FILE_FROM_CLOUD_FAILED",
comment: "Error indicating the a backup import failed to download a file from the cloud.")))
failure(invalidServiceResponseError())
}
}
},
failure: failure)
}
// We return the CKAsset and not its fileUrl because
// CloudKit offers no guarantees around how long it'll
// keep around the underlying file. Presumably we can
// defer cleanup by maintaining a strong reference to
// the asset.
private class func downloadFromCloud(recordName: String,
remainingRetries: Int,
success: @escaping (CKAsset) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
@ -425,25 +483,43 @@ import CloudKit
let fetchOperation = CKFetchRecordsOperation(recordIDs: [recordId ])
// Download all keys for this record.
fetchOperation.perRecordCompletionBlock = { (record, recordId, error) in
if let error = error {
failure(error)
return
}
guard let record = record else {
let response = responseForCloudKitError(error: error,
remainingRetries: remainingRetries,
label: "Download Record")
switch response {
case .success:
guard let record = record else {
Logger.error("\(self.logTag) missing fetching record.")
failure(invalidServiceResponseError())
return
}
guard let asset = record[payloadKey] as? CKAsset else {
Logger.error("\(self.logTag) record missing payload.")
failure(invalidServiceResponseError())
return
}
success(asset)
case .failureDoNotRetry(let responseError):
failure(responseError)
case .failureRetryAfterDelay(let retryDelay):
DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + retryDelay, execute: {
downloadFromCloud(recordName: recordName,
remainingRetries: remainingRetries - 1,
success: success,
failure: failure)
})
case .failureRetryWithoutDelay:
DispatchQueue.global().async {
downloadFromCloud(recordName: recordName,
remainingRetries: remainingRetries - 1,
success: success,
failure: failure)
}
case .unknownItem:
Logger.error("\(self.logTag) missing fetching record.")
failure(OWSErrorWithCodeDescription(.exportBackupError,
NSLocalizedString("BACKUP_IMPORT_ERROR_DOWNLOAD_FILE_FROM_CLOUD_FAILED",
comment: "Error indicating the a backup import failed to download a file from the cloud.")))
return
failure(invalidServiceResponseError())
}
guard let asset = record[payloadKey] as? CKAsset else {
Logger.error("\(self.logTag) record missing payload.")
failure(OWSErrorWithCodeDescription(.exportBackupError,
NSLocalizedString("BACKUP_IMPORT_ERROR_DOWNLOAD_FILE_FROM_CLOUD_FAILED",
comment: "Error indicating the a backup import failed to download a file from the cloud.")))
return
}
success(asset)
}
database().add(fetchOperation)
}
@ -481,13 +557,22 @@ import CloudKit
case failureDoNotRetry(error:Error)
case failureRetryAfterDelay(retryDelay: Double)
case failureRetryWithoutDelay
// This only applies to fetches.
case unknownItem
}
private class func responseForCloudKitError(error: Error?,
remainingRetries: Int,
label: String) -> CKErrorResponse {
if let error = error as? CKError {
if error.code == CKError.unknownItem {
// This is not always an error for our purposes.
Logger.verbose("\(self.logTag) \(label) unknown item.")
return .unknownItem
}
Logger.error("\(self.logTag) \(label) failed: \(error)")
if remainingRetries < 1 {
Logger.verbose("\(self.logTag) \(label) no more retries.")
return .failureDoNotRetry(error:error)

View File

@ -39,6 +39,8 @@ typedef NS_ENUM(NSInteger, OWSErrorCode) {
OWSErrorCodeImportBackupFailed = 777417,
// A possibly recoverable error occured while importing a backup.
OWSErrorCodeImportBackupError = 777418,
// A non-recoverable while importing or exporting a backup.
OWSErrorCodeBackupFailure = 777419,
};
extern NSString *const OWSErrorRecipientIdentifierKey;