oxen-io
/
session-ios
mirror of https://github.com/oxen-io/session-ios.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Clean up cloud after successful backup export.

This commit is contained in:
Matthew Chen 2018-03-08 10:14:35 -03:00 committed by Matthew Chen
parent aa546a02df
commit 90c8f5483b
3 changed files with 249 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
Signal/src/util/OWSBackup.m
View File

@ -139,6 +139,14 @@ NS_ASSUME_NONNULL_BEGIN
[self.dbConnection setBool:value
forKey:OWSBackup_IsBackupEnabledKey
inCollection:OWSPrimaryStorage_OWSBackupCollection];
OWSAssert(self.isBackupEnabled);
if (!value) {
[self.dbConnection removeObjectForKey:OWSBackup_LastExportSuccessDateKey
inCollection:OWSPrimaryStorage_OWSBackupCollection];
[self.dbConnection removeObjectForKey:OWSBackup_LastExportFailureDateKey
inCollection:OWSPrimaryStorage_OWSBackupCollection];
}
[[NSNotificationCenter defaultCenter] postNotificationNameAsync:NSNotificationNameBackupStateDidChange
object:nil
@ -162,12 +170,18 @@ NS_ASSUME_NONNULL_BEGIN
NSDate *_Nullable lastExportSuccessDate = self.lastExportSuccessDate;
NSDate *_Nullable lastExportFailureDate = self.lastExportFailureDate;
// Wait N hours before retrying after a success.
const NSTimeInterval kRetryAfterSuccess = 24 * kHourInterval;
//
// TODO: Use actual values in production.
// const NSTimeInterval kRetryAfterSuccess = 24 * kHourInterval;
const NSTimeInterval kRetryAfterSuccess = 0;
if (lastExportSuccessDate && fabs(lastExportSuccessDate.timeIntervalSinceNow) < kRetryAfterSuccess) {
return NO;
}
// Wait N hours before retrying after a failure.
const NSTimeInterval kRetryAfterFailure = 6 * kHourInterval;
//
// TODO: Use actual values in production.
// const NSTimeInterval kRetryAfterFailure = 6 * kHourInterval;
const NSTimeInterval kRetryAfterFailure = 0;
if (lastExportFailureDate && fabs(lastExportFailureDate.timeIntervalSinceNow) < kRetryAfterFailure) {
return NO;
}

148
Signal/src/util/OWSBackupAPI.swift
View File

@ -7,6 +7,13 @@ import SignalServiceKit
import CloudKit
@objc public class OWSBackupAPI: NSObject {
// If we change the record types, we need to ensure indices
// are configured properly in the CloudKit dashboard.
static let signalBackupRecordType = "signalBackup"
static let manifestRecordName = "manifest"
static let payloadKey = "payload"
@objc
public class func recordIdForTest() -> String {
return "test-\(NSUUID().uuidString)"
@ -18,7 +25,7 @@ import CloudKit
failure: @escaping (Error) -> Swift.Void) {
saveFileToCloud(fileUrl: fileUrl,
recordName: NSUUID().uuidString,
recordType: "test",
recordType: signalBackupRecordType,
success: success,
failure: failure)
}
@ -29,46 +36,41 @@ import CloudKit
// complete.
@objc
public class func saveEphemeralDatabaseFileToCloud(fileUrl: URL,
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
saveFileToCloud(fileUrl: fileUrl,
recordName: NSUUID().uuidString,
recordType: "ephemeralFile",
recordType: signalBackupRecordType,
success: success,
failure: failure)
}
// "Persistent" files may be shared between backup export; they should only be saved
// once. For example, attachment files should only be uploaded once. Subsequent
//
// backups can reuse the same record.
@objc
public class func savePersistentFileOnceToCloud(fileId: String,
fileUrlBlock: @escaping (Swift.Void) -> URL?,
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
fileUrlBlock: @escaping (Swift.Void) -> URL?,
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
saveFileOnceToCloud(recordName: "persistentFile-\(fileId)",
recordType: "persistentFile",
fileUrlBlock: fileUrlBlock,
success: success,
failure: failure)
recordType: signalBackupRecordType,
fileUrlBlock: fileUrlBlock,
success: success,
failure: failure)
}
// TODO:
static let manifestRecordName = "manifest_"
static let manifestRecordType = "manifest"
static let payloadKey = "payload"
@objc
public class func upsertAttachmentToCloud(fileUrl: URL,
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
// We want to use a well-known record id and type for manifest files.
upsertFileToCloud(fileUrl: fileUrl,
recordName: manifestRecordName,
recordType: manifestRecordType,
success: success,
failure: failure)
}
// @objc
// public class func upsertAttachmentToCloud(fileUrl: URL,
// success: @escaping (String) -> Swift.Void,
// failure: @escaping (Error) -> Swift.Void) {
// // We want to use a well-known record id and type for manifest files.
// upsertFileToCloud(fileUrl: fileUrl,
// recordName: manifestRecordName,
// recordType: signalBackupRecordType,
// success: success,
// failure: failure)
// }
@objc
public class func upsertManifestFileToCloud(fileUrl: URL,
@ -77,7 +79,7 @@ import CloudKit
// We want to use a well-known record id and type for manifest files.
upsertFileToCloud(fileUrl: fileUrl,
recordName: manifestRecordName,
recordType: manifestRecordType,
recordType: signalBackupRecordType,
success: success,
failure: failure)
}
@ -125,6 +127,28 @@ import CloudKit
}
}
@objc
public class func deleteRecordFromCloud(recordName: String,
success: @escaping (Swift.Void) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
let recordID = CKRecordID(recordName: recordName)
let myContainer = CKContainer.default()
let privateDatabase = myContainer.privateCloudDatabase
privateDatabase.delete(withRecordID: recordID) {
(record, error) in
if let error = error {
Logger.error("\(self.logTag) error deleting record: \(error)")
failure(error)
} else {
Logger.info("\(self.logTag) deleted record.")
success()
}
}
}
@objc
public class func upsertFileToCloud(fileUrl: URL,
recordName: String,
@ -177,10 +201,10 @@ import CloudKit
@objc
public class func saveFileOnceToCloud(recordName: String,
recordType: String,
fileUrlBlock: @escaping (Swift.Void) -> URL?,
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
recordType: String,
fileUrlBlock: @escaping (Swift.Void) -> URL?,
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
let recordId = CKRecordID(recordName: recordName)
let fetchOperation = CKFetchRecordsOperation(recordIDs: [recordId ])
// Don't download the file; we're just using the fetch to check whether or
@ -193,7 +217,7 @@ import CloudKit
// No record found to update, saving new record.
guard let fileUrl = fileUrlBlock() else {
Logger.error("\(self.logTag) error preparing file for upload: \(error).")
Logger.error("\(self.logTag) error preparing file for upload.")
return
}
@ -219,6 +243,62 @@ import CloudKit
privateDatabase.add(fetchOperation)
}
@objc
public class func fetchAllRecordNames(success: @escaping ([String]) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
let query = CKQuery(recordType: signalBackupRecordType, predicate: NSPredicate(value: true))
// Fetch the first page of results for this query.
fetchAllRecordNamesStep(query: query,
previousRecordNames: [String](),
cursor: nil,
success: success,
failure: failure)
}
private class func fetchAllRecordNamesStep(query: CKQuery,
previousRecordNames: [String],
cursor: CKQueryCursor?,
success: @escaping ([String]) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
var allRecordNames = previousRecordNames
let queryOperation = CKQueryOperation(query: query)
// If this isn't the first page of results for this query, resume
// where we left off.
queryOperation.cursor = cursor
// Don't download the file; we're just using the query to get a list of record names.
queryOperation.desiredKeys = []
queryOperation.recordFetchedBlock = { (record) in
assert(record.recordID.recordName.count > 0)
allRecordNames.append(record.recordID.recordName)
}
queryOperation.queryCompletionBlock = { (cursor, error) in
if let error = error {
Logger.error("\(self.logTag) error fetching all record names: \(error).")
failure(error)
return
}
if let cursor = cursor {
Logger.verbose("\(self.logTag) fetching more record names \(allRecordNames.count).")
// There are more pages of results, continue fetching.
fetchAllRecordNamesStep(query: query,
previousRecordNames: allRecordNames,
cursor: cursor,
success: success,
failure: failure)
return
}
Logger.info("\(self.logTag) fetched \(allRecordNames.count) record names.")
success(allRecordNames)
}
let myContainer = CKContainer.default()
let privateDatabase = myContainer.privateCloudDatabase
privateDatabase.add(queryOperation)
}
@objc
public class func checkCloudKitAccess(completion: @escaping (Bool) -> Swift.Void) {
CKContainer.default().accountStatus(completionHandler: { (accountStatus, error) in

146
Signal/src/util/OWSBackupExport.m
View File

@ -7,6 +7,7 @@
#import "zlib.h"
#import <Curve25519Kit/Randomness.h>
#import <SSZipArchive/SSZipArchive.h>
#import <SignalServiceKit/NSData+Base64.h>
#import <SignalServiceKit/NSDate+OWS.h>
#import <SignalServiceKit/OWSBackgroundTask.h>
#import <SignalServiceKit/OWSBackupStorage.h>
@ -78,6 +79,7 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
OWSFail(@"%@ attachment could not be encrypted: %@", self.logTag, self.attachmentFilePath);
return;
}
self.tempFilePath = tempFilePath;
}
- (nullable NSString *)encryptAsTempFile:(NSString *)srcFilePath
@ -114,7 +116,7 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
@property (nonatomic, nullable) OWSBackupStorage *backupStorage;
@property (nonatomic, nullable) NSData *databaseSalt;
@property (nonatomic, nullable) NSData *databaseKeySpec;
@property (nonatomic, nullable) OWSBackgroundTask *backgroundTask;
@ -209,24 +211,32 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
if (self.isComplete) {
return;
}
[self saveToCloud:^(NSError *_Nullable error) {
if (error) {
[weakSelf failWithError:error];
} else {
[weakSelf succeed];
[self saveToCloud:^(NSError *_Nullable saveError) {
if (saveError) {
[weakSelf failWithError:saveError];
return;
}
[self cleanUpCloud:^(NSError *_Nullable cleanUpError) {
if (cleanUpError) {
[weakSelf failWithError:cleanUpError];
return;
}
[weakSelf succeed];
}];
}];
}];
}
- (void)configureExport:(OWSBackupExportBoolCompletion)completion
{
OWSAssert(completion);
DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
NSString *temporaryDirectory = NSTemporaryDirectory();
self.exportDirPath = [temporaryDirectory stringByAppendingString:[NSUUID UUID].UUIDString];
NSString *exportDatabaseDirPath = [self.exportDirPath stringByAppendingPathComponent:@"Database"];
self.databaseSalt = [Randomness generateRandomBytes:(int)kSQLCipherSaltLength];
self.databaseKeySpec = [Randomness generateRandomBytes:(int)kSQLCipherKeySpecLength];
if (![OWSFileSystem ensureDirectoryExists:self.exportDirPath]) {
OWSProdLogAndFail(@"%@ Could not create exportDirPath.", self.logTag);
@ -236,8 +246,8 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
OWSProdLogAndFail(@"%@ Could not create exportDatabaseDirPath.", self.logTag);
return completion(NO);
}
if (!self.databaseSalt) {
OWSProdLogAndFail(@"%@ Could not create databaseSalt.", self.logTag);
if (!self.databaseKeySpec) {
OWSProdLogAndFail(@"%@ Could not create databaseKeySpec.", self.logTag);
return completion(NO);
}
__weak OWSBackupExport *weakSelf = self;
@ -246,14 +256,7 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
if (!backupKey) {
return (NSData *)nil;
}
NSData *_Nullable databaseSalt = weakSelf.databaseSalt;
if (!databaseSalt) {
return (NSData *)nil;
}
OWSCAssert(backupKey.length > 0);
NSData *_Nullable keySpec =
[YapDatabaseCryptoUtils deriveDatabaseKeySpecForPassword:backupKey saltData:databaseSalt];
return keySpec;
return weakSelf.databaseKeySpec;
};
self.backupStorage =
[[OWSBackupStorage alloc] initBackupStorageWithDatabaseDirPath:exportDatabaseDirPath keySpecBlock:keySpecBlock];
@ -394,6 +397,8 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
- (void)saveToCloud:(OWSBackupExportCompletion)completion
{
OWSAssert(completion);
DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
self.databaseRecordMap = [NSMutableDictionary new];
@ -404,6 +409,8 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
- (void)saveNextFileToCloud:(OWSBackupExportCompletion)completion
{
OWSAssert(completion);
if (self.isComplete) {
return;
}
@ -417,8 +424,7 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
// TODO: Security review.
[OWSBackupAPI saveEphemeralDatabaseFileToCloudWithFileUrl:[NSURL fileURLWithPath:filePath]
success:^(NSString *recordName) {
// Ensure that we continue to perform the backup export
// off the main thread.
// Ensure that we continue to work off the main thread.
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
OWSBackupExport *strongSelf = weakSelf;
if (!strongSelf) {
@ -450,14 +456,18 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
[OWSBackupAPI savePersistentFileOnceToCloudWithFileId:attachmentId
fileUrlBlock:^{
[attachmentExport prepareForUpload];
if (attachmentExport.tempFilePath.length < 1 || attachmentExport.relativeFilePath.length < 1) {
if (attachmentExport.tempFilePath.length < 1) {
DDLogError(@"%@ attachment export missing temp file path", self.logTag);
return (NSURL *)nil;
}
if (attachmentExport.relativeFilePath.length < 1) {
DDLogError(@"%@ attachment export missing relative file path", self.logTag);
return (NSURL *)nil;
}
return [NSURL fileURLWithPath:attachmentExport.tempFilePath];
}
success:^(NSString *recordName) {
// Ensure that we continue to perform the backup export
// off the main thread.
// Ensure that we continue to work off the main thread.
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
OWSBackupExport *strongSelf = weakSelf;
if (!strongSelf) {
@ -468,8 +478,7 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
});
}
failure:^(NSError *error) {
// Ensure that we continue to perform the backup export
// off the main thread.
// Ensure that we continue to work off the main thread.
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
// Attachment files are non-critical so any error uploading them is recoverable.
[weakSelf saveNextFileToCloud:completion];
@ -489,8 +498,7 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
[OWSBackupAPI upsertManifestFileToCloudWithFileUrl:[NSURL fileURLWithPath:self.manifestFilePath]
success:^(NSString *recordName) {
// Ensure that we continue to perform the backup export
// off the main thread.
// Ensure that we continue to work off the main thread.
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
OWSBackupExport *strongSelf = weakSelf;
if (!strongSelf) {
@ -533,10 +541,13 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
OWSAssert(self.databaseRecordMap.count > 0);
OWSAssert(self.attachmentRecordMap);
OWSAssert(self.exportDirPath.length > 0);
OWSAssert(self.databaseKeySpec.length > 0);
NSDictionary *json = @{
@"database_files" : self.databaseRecordMap,
@"attachment_files" : self.attachmentRecordMap,
// JSON doesn't support byte arrays.
@"database_key_spec" : self.databaseKeySpec.base64EncodedString,
};
NSError *error;
NSData *_Nullable jsonData =
@ -554,11 +565,92 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
return YES;
}
- (void)cleanUpCloud:(OWSBackupExportCompletion)completion
{
OWSAssert(completion);
DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
// Now that our backup export has successfully completed,
// we try to clean up the cloud. We can safely delete any
// records not involved in this backup export.
NSMutableSet<NSString *> *activeRecordNames = [NSMutableSet new];
OWSAssert(self.databaseRecordMap.count > 0);
[activeRecordNames addObjectsFromArray:self.databaseRecordMap.allKeys];
OWSAssert(self.attachmentRecordMap);
[activeRecordNames addObjectsFromArray:self.attachmentRecordMap.allKeys];
OWSAssert(self.manifestRecordName.length > 0);
[activeRecordNames addObject:self.manifestRecordName];
__weak OWSBackupExport *weakSelf = self;
[OWSBackupAPI fetchAllRecordNamesWithSuccess:^(NSArray<NSString *> *recordNames) {
// Ensure that we continue to work off the main thread.
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
NSMutableSet<NSString *> *obsoleteRecordNames = [NSMutableSet new];
[obsoleteRecordNames addObjectsFromArray:recordNames];
[obsoleteRecordNames minusSet:activeRecordNames];
DDLogVerbose(@"%@ recordNames: %zd - activeRecordNames: %zd = obsoleteRecordNames: %zd",
self.logTag,
recordNames.count,
activeRecordNames.count,
obsoleteRecordNames.count);
[weakSelf deleteRecordsFromCloud:[obsoleteRecordNames.allObjects mutableCopy] completion:completion];
});
}
failure:^(NSError *error) {
// Ensure that we continue to work off the main thread.
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
// Cloud cleanup is non-critical so any error is recoverable.
completion(nil);
});
}];
}
- (void)deleteRecordsFromCloud:(NSMutableArray<NSString *> *)obsoleteRecordNames
completion:(OWSBackupExportCompletion)completion
{
OWSAssert(obsoleteRecordNames);
OWSAssert(completion);
DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
if (obsoleteRecordNames.count < 1) {
// No more records to delete; cleanup is complete.
completion(nil);
return;
}
NSString *recordName = obsoleteRecordNames.lastObject;
[obsoleteRecordNames removeLastObject];
__weak OWSBackupExport *weakSelf = self;
[OWSBackupAPI deleteRecordFromCloudWithRecordName:recordName
success:^{
// Ensure that we continue to work off the main thread.
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
[weakSelf deleteRecordsFromCloud:obsoleteRecordNames completion:completion];
});
}
failure:^(NSError *error) {
// Ensure that we continue to work off the main thread.
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
// Cloud cleanup is non-critical so any error is recoverable.
[weakSelf deleteRecordsFromCloud:obsoleteRecordNames completion:completion];
});
}];
}
#pragma mark -
- (void)cancel
{
OWSAssertIsOnMainThread();
// TODO:
self.isComplete = YES;
}