Export database for backup.

This commit is contained in:
Matthew Chen 2018-03-06 12:10:22 -03:00
parent b603a8dcbe
commit c84bf81cf3
12 changed files with 371 additions and 68 deletions

View File

@ -10,6 +10,7 @@
#import "MainAppContext.h"
#import "NotificationsManager.h"
#import "OWS2FASettingsViewController.h"
#import "OWSBackup.h"
#import "OWSNavigationController.h"
#import "Pastelog.h"
#import "PushManager.h"
@ -1165,6 +1166,8 @@ static NSString *const kURLHostVerifyPrefix = @"verify";
[OWSPreferences setIsReadyForAppExtensions];
[self ensureRootViewController];
[OWSBackup.sharedManager setup];
}
- (void)registrationStateDidChange

View File

@ -92,8 +92,8 @@
[self updateTableContents];
dispatch_async(dispatch_get_main_queue(), ^{
// [self showBackup];
[self showDebugUI];
[self showBackup];
// [self showDebugUI];
});
}

View File

@ -49,6 +49,8 @@ typedef NS_ENUM(NSUInteger, OWSBackupState) {
- (BOOL)isBackupEnabled;
- (void)setIsBackupEnabled:(BOOL)value;
- (void)setup;
//- (void)exportBackup:(nullable TSThread *)currentThread skipPassword:(BOOL)skipPassword;
//
//- (void)importBackup:(NSString *)backupZipPath password:(NSString *_Nullable)password;

View File

@ -49,7 +49,7 @@ NS_ASSUME_NONNULL_BEGIN
// NSString *const Keychain_ImportBackupService = @"OWSKeychainService";
// NSString *const Keychain_ImportBackupKey = @"ImportBackupKey";
@interface OWSBackup ()
@interface OWSBackup () <OWSBackupExportDelegate>
@property (nonatomic, readonly) YapDatabaseConnection *dbConnection;
@ -112,6 +112,16 @@ NS_ASSUME_NONNULL_BEGIN
OWSSingletonAssert();
return self;
}
- (void)dealloc
{
[[NSNotificationCenter defaultCenter] removeObserver:self];
}
- (void)setup
{
[[NSNotificationCenter defaultCenter] addObserver:self
selector:@selector(applicationDidBecomeActive:)
name:OWSApplicationDidBecomeActiveNotification
@ -121,12 +131,14 @@ NS_ASSUME_NONNULL_BEGIN
name:RegistrationStateDidChangeNotification
object:nil];
return self;
}
- (void)dealloc
{
[[NSNotificationCenter defaultCenter] removeObserver:self];
// We want to start a backup if necessary on app launch, but app launch is a
// busy time and it's important to remain responsive, so wait a few seconds before
// starting the backup.
//
// TODO: Make this period longer.
dispatch_after(dispatch_time(DISPATCH_TIME_NOW, 2 * NSEC_PER_SEC), dispatch_get_main_queue(), ^{
[self ensureBackupExportState];
});
}
//- (void)observeNotifications
@ -168,6 +180,8 @@ NS_ASSUME_NONNULL_BEGIN
[[NSNotificationCenter defaultCenter] postNotificationNameAsync:NSNotificationNameBackupStateDidChange
object:nil
userInfo:nil];
[self ensureBackupExportState];
}
- (BOOL)shouldHaveBackupExport
@ -183,7 +197,9 @@ NS_ASSUME_NONNULL_BEGIN
return NO;
}
// TODO: There's probably other conditions that affect this decision.
// TODO: There's other conditions that affect this decision,
// e.g. we want to throttle on time and only _try_ every N days
// and _succeed_ every N days. And wifi v. cellular may play into it.
return YES;
}
@ -197,7 +213,7 @@ NS_ASSUME_NONNULL_BEGIN
} else if (self.shouldHaveBackupExport && !self.backupExport) {
self.backupExport =
[[OWSBackupExport alloc] initWithDelegate:self primaryStorage:[OWSPrimaryStorage sharedManager]];
[self.backupExport start];
[self.backupExport startAsync];
}
// BOOL shouldHaveBackupExport
@ -220,6 +236,42 @@ NS_ASSUME_NONNULL_BEGIN
[self ensureBackupExportState];
}
#pragma mark - OWSBackupExportDelegate
// TODO: This should eventually be the backup key stored in the Signal Service
// and retrieved with the backup PIN.
- (nullable NSData *)backupKey
{
// We use a delegate method to avoid storing this key in memory.
// It will eventually be stored in the keychain.
return [@"test backup key" dataUsingEncoding:NSUTF8StringEncoding];
}
- (void)backupExportDidSucceed:(OWSBackupExport *)backupExport
{
if (self.backupExport != backupExport) {
return;
}
DDLogInfo(@"%@ %s.", self.logTag, __PRETTY_FUNCTION__);
// TODO:
self.backupExport = nil;
}
- (void)backupExportDidFail:(OWSBackupExport *)backupExport error:(NSError *)error
{
if (self.backupExport != backupExport) {
return;
}
DDLogInfo(@"%@ %s: %@", self.logTag, __PRETTY_FUNCTION__, error);
// TODO:
self.backupExport = nil;
}
//- (void)setBackupProgress:(CGFloat)backupProgress
//{
// _backupProgress = backupProgress;

View File

@ -8,11 +8,17 @@ NS_ASSUME_NONNULL_BEGIN
// extern NSString *const NSNotificationNameBackupStateDidChange;
@class OWSBackupExport;
@protocol OWSBackupExportDelegate <NSObject>
- (void)backupExportDidSucceed;
// TODO: This should eventually be the backup key stored in the Signal Service
// and retrieved with the backup PIN.
- (nullable NSData *)backupKey;
- (void)backupExportDidFailWithError:(NSError *)error;
- (void)backupExportDidSucceed:(OWSBackupExport *)backupExport;
- (void)backupExportDidFail:(OWSBackupExport *)backupExport error:(NSError *)error;
@end
@ -34,7 +40,7 @@ NS_ASSUME_NONNULL_BEGIN
- (instancetype)initWithDelegate:(id<OWSBackupExportDelegate>)delegate
primaryStorage:(OWSPrimaryStorage *)primaryStorage;
- (void)start;
- (void)startAsync;
- (void)cancel;

View File

@ -17,8 +17,17 @@
//#import <SignalServiceKit/OWSPrimaryStorage.h>
//#import "NSNotificationCenter+OWS.h"
#import <Curve25519Kit/Randomness.h>
#import <SignalServiceKit/NSDate+OWS.h>
#import <SignalServiceKit/OWSBackgroundTask.h>
#import <SignalServiceKit/OWSBackupStorage.h>
#import <SignalServiceKit/OWSError.h>
#import <SignalServiceKit/TSMessage.h>
#import <SignalServiceKit/TSThread.h>
#import <SignalServiceKit/Threading.h>
#import <SignalServiceKit/YapDatabaseConnection+OWS.h>
#import <YapDatabase/YapDatabase.h>
#import <YapDatabase/YapDatabaseCryptoUtils.h>
// NSString *const NSNotificationNameBackupStateDidChange = @"NSNotificationNameBackupStateDidChange";
//
@ -27,6 +36,9 @@
NS_ASSUME_NONNULL_BEGIN
typedef void (^OWSBackupExportBoolCompletion)(BOOL success);
typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
//// Hide the "import" directories from exports, etc. by prefixing their name with a period.
////
//// OWSBackup backs up files and directories in the "app documents" and "shared data container",
@ -50,11 +62,19 @@ NS_ASSUME_NONNULL_BEGIN
@property (nonatomic, weak) id<OWSBackupExportDelegate> delegate;
@property (nonatomic, readonly) YapDatabaseConnection *dbConnection;
@property (nonatomic, readonly) YapDatabaseConnection *srcDBConnection;
@property (nonatomic, readonly) YapDatabaseConnection *dstDBConnection;
// Indicates that the backup succeeded, failed or was cancelled.
@property (atomic) BOOL isComplete;
@property (atomic, nullable) OWSBackupStorage *backupStorage;
@property (atomic, nullable) NSData *databaseSalt;
@property (atomic, nullable) OWSBackgroundTask *backgroundTask;
//- (NSData *)databasePassword;
//
//+ (void)storeDatabasePassword:(NSString *)password;
@ -84,7 +104,7 @@ NS_ASSUME_NONNULL_BEGIN
@implementation OWSBackupExport
@synthesize dbConnection = _dbConnection;
//@synthesize dbConnection = _dbConnection;
//+ (instancetype)sharedManager
//{
@ -113,29 +133,217 @@ NS_ASSUME_NONNULL_BEGIN
}
OWSAssert(primaryStorage);
OWSAssert([OWSStorage isStorageReady]);
self.delegate = delegate;
_dbConnection = primaryStorage.newDatabaseConnection;
_srcDBConnection = primaryStorage.newDatabaseConnection;
// _backupExportState = OWSBackupState_AtRest;
OWSSingletonAssert();
// TODO: Remove.
DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
return self;
}
- (void)dealloc
{
// Surface memory leaks by logging the deallocation.
DDLogVerbose(@"Dealloc: %@", self.class);
[[NSNotificationCenter defaultCenter] removeObserver:self];
}
- (void)startAsync
{
DDLogInfo(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
[self start];
});
}
- (void)start
{
// TODO:
self.backgroundTask = [OWSBackgroundTask backgroundTaskWithLabelStr:__PRETTY_FUNCTION__];
[self configureExport:^(BOOL success) {
if (!success) {
[self failWithErrorDescription:
NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT",
@"Error indicating the a backup export could not export the user's data.")];
;
return;
}
if (self.isComplete) {
return;
}
if (![self exportDatabase]) {
[self failWithErrorDescription:
NSLocalizedString(@"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT",
@"Error indicating the a backup export could not export the user's data.")];
;
return;
}
if (self.isComplete) {
return;
}
// TODO:
[self succeed];
}];
}
- (void)configureExport:(OWSBackupExportBoolCompletion)completion
{
DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
NSString *temporaryDirectory = NSTemporaryDirectory();
NSString *exportDirPath = [temporaryDirectory stringByAppendingString:[NSUUID UUID].UUIDString];
NSString *exportDatabaseDirPath = [exportDirPath stringByAppendingPathComponent:@"Database"];
self.databaseSalt = [Randomness generateRandomBytes:(int)kSQLCipherSaltLength];
if (![OWSFileSystem ensureDirectoryExists:exportDirPath]) {
DDLogError(@"%@ Could not create exportDirPath.", self.logTag);
return completion(NO);
}
if (![OWSFileSystem ensureDirectoryExists:exportDatabaseDirPath]) {
DDLogError(@"%@ Could not create exportDatabaseDirPath.", self.logTag);
return completion(NO);
}
if (!self.databaseSalt) {
DDLogError(@"%@ Could not create databaseSalt.", self.logTag);
return completion(NO);
}
__weak OWSBackupExport *weakSelf = self;
BackupStorageKeySpecBlock keySpecBlock = ^{
NSData *_Nullable backupKey = [weakSelf.delegate backupKey];
if (!backupKey) {
return (NSData *)nil;
}
NSData *_Nullable databaseSalt = weakSelf.databaseSalt;
if (!databaseSalt) {
return (NSData *)nil;
}
OWSCAssert(backupKey.length > 0);
NSData *_Nullable keySpec =
[YapDatabaseCryptoUtils deriveDatabaseKeySpecForPassword:backupKey saltData:databaseSalt];
return keySpec;
};
self.backupStorage =
[[OWSBackupStorage alloc] initBackupStorageWithDatabaseDirPath:exportDatabaseDirPath keySpecBlock:keySpecBlock];
if (!self.backupStorage) {
DDLogError(@"%@ Could not create backupStorage.", self.logTag);
return completion(NO);
}
_dstDBConnection = self.backupStorage.newDatabaseConnection;
if (!self.dstDBConnection) {
DDLogError(@"%@ Could not create dstDBConnection.", self.logTag);
return completion(NO);
}
// TODO: Do we really need to run these registrations on the main thread?
dispatch_async(dispatch_get_main_queue(), ^{
[self.backupStorage runSyncRegistrations];
[self.backupStorage runAsyncRegistrationsWithCompletion:^{
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
completion(YES);
});
}];
});
// // The backup storage is empty and therefore async registrations should
// // complete very quickly. We'll wait a short period for them to complete.
// NSDate *readyStart = [NSDate new];
// while (YES) {
// if (self.backupStorage.areAllRegistrationsComplete) {
// return YES;
// }
// DDLogVerbose(@"%@ waiting for backup storage: %f %f", self.logTag, readyStart.timeIntervalSinceNow,
// kMinuteInterval); if (fabs(readyStart.timeIntervalSinceNow) > kMinuteInterval) {
// return NO;
// }
// // Sleep 100 ms.
// usleep(100 * 1000);
// }
}
- (BOOL)exportDatabase
{
DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
__block unsigned long long copiedThreads = 0;
__block unsigned long long copiedInteractions = 0;
__block unsigned long long copiedEntities = 0;
[self.srcDBConnection readWithBlock:^(YapDatabaseReadTransaction *srcTransaction) {
[self.dstDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *dstTransaction) {
// Copy threads.
[srcTransaction
enumerateKeysAndObjectsInCollection:[TSThread collection]
usingBlock:^(NSString *key, id object, BOOL *stop) {
if (self.isComplete) {
*stop = YES;
return;
}
if (![object isKindOfClass:[TSThread class]]) {
DDLogError(@"%@ unexpected class: %@", self.logTag, [object class]);
return;
}
TSThread *thread = object;
[thread saveWithTransaction:dstTransaction];
copiedThreads++;
copiedEntities++;
}];
// Copy interactions.
[srcTransaction
enumerateKeysAndObjectsInCollection:[TSInteraction collection]
usingBlock:^(NSString *key, id object, BOOL *stop) {
if (self.isComplete) {
*stop = YES;
return;
}
if (![object isKindOfClass:[TSInteraction class]]) {
DDLogError(@"%@ unexpected class: %@", self.logTag, [object class]);
return;
}
// Ignore disappearing messages.
if ([object isKindOfClass:[TSMessage class]]) {
TSMessage *message = object;
if (message.isExpiringMessage) {
return;
}
}
TSInteraction *interaction = object;
// Ignore dynamic interactions.
if (interaction.isDynamicInteraction) {
return;
}
[interaction saveWithTransaction:dstTransaction];
copiedInteractions++;
copiedEntities++;
}];
// TODO: Copy attachments.
}];
}];
// TODO: Should we do a database checkpoint?
DDLogInfo(@"%@ copiedThreads: %llu", self.logTag, copiedThreads);
DDLogInfo(@"%@ copiedMessages: %llu", self.logTag, copiedInteractions);
DDLogInfo(@"%@ copiedEntities: %llu", self.logTag, copiedEntities);
[self.backupStorage logFileSizes];
return YES;
}
- (void)cancel
{
OWSAssertIsOnMainThread();
// TODO:
self.isComplete = YES;
}
@ -144,24 +352,33 @@ NS_ASSUME_NONNULL_BEGIN
{
DDLogInfo(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
self.isComplete = YES;
dispatch_async(dispatch_get_main_queue(), ^{
[self.delegate backupExportDidSucceed];
if (self.isComplete) {
return;
}
self.isComplete = YES;
[self.delegate backupExportDidSucceed:self];
});
// TODO:
}
- (void)failWithErrorDescription:(NSString *)description
{
[self failWithError:OWSErrorWithCodeDescription(OWSErrorCodeExportBackupFailed, description)];
}
- (void)failWithError:(NSError *)error
{
DDLogError(@"%@ %s %@", self.logTag, __PRETTY_FUNCTION__, error);
self.isComplete = YES;
// TODO:
dispatch_async(dispatch_get_main_queue(), ^{
[self.delegate backupExportDidFailWithError:error];
if (self.isComplete) {
return;
}
self.isComplete = YES;
[self.delegate backupExportDidFail:self error:error];
});
}

View File

@ -10,17 +10,25 @@
NS_ASSUME_NONNULL_BEGIN
typedef NSData *_Nullable (^BackupStorageKeySpecBlock)(void);
@interface OWSBackupStorage : OWSStorage
- (instancetype)init NS_UNAVAILABLE;
- (instancetype)initStorage NS_UNAVAILABLE;
- (instancetype)initBackupStorageWithdatabaseDirPath:(NSString *)databaseDirPath
databaseKeySpec:(NSData *)databaseKeySpec NS_DESIGNATED_INITIALIZER;
- (instancetype)initBackupStorageWithDatabaseDirPath:(NSString *)databaseDirPath
keySpecBlock:(BackupStorageKeySpecBlock)keySpecBlock NS_DESIGNATED_INITIALIZER;
- (YapDatabaseConnection *)dbConnection;
- (void)logFileSizes;
- (void)runSyncRegistrations;
- (void)runAsyncRegistrationsWithCompletion:(void (^_Nonnull)(void))completion;
- (BOOL)areAllRegistrationsComplete;
@end
NS_ASSUME_NONNULL_END

View File

@ -10,13 +10,11 @@ NS_ASSUME_NONNULL_BEGIN
@interface OWSBackupStorage ()
@property (nonatomic, readonly, nullable) YapDatabaseConnection *dbConnection;
@property (atomic) BOOL areAsyncRegistrationsComplete;
@property (atomic) BOOL areSyncRegistrationsComplete;
@property (nonatomic, readonly) NSString *databaseDirPath;
@property (nonatomic, readonly) NSData *databaseKeySpec;
@property (nonatomic, readonly) BackupStorageKeySpecBlock keySpecBlock;
@end
@ -24,32 +22,34 @@ NS_ASSUME_NONNULL_BEGIN
@implementation OWSBackupStorage
@synthesize databaseKeySpec = _databaseKeySpec;
- (instancetype)initBackupStorageWithdatabaseDirPath:(NSString *)databaseDirPath
databaseKeySpec:(NSData *)databaseKeySpec
- (instancetype)initBackupStorageWithDatabaseDirPath:(NSString *)databaseDirPath
keySpecBlock:(BackupStorageKeySpecBlock)keySpecBlock
{
OWSAssert(databaseDirPath.length > 0);
OWSAssert(databaseKeySpec.length > 0);
OWSAssert(keySpecBlock);
OWSAssert([OWSFileSystem ensureDirectoryExists:databaseDirPath]);
self = [super initStorage];
if (self) {
[self protectFiles];
_dbConnection = self.newDatabaseConnection;
_databaseDirPath = databaseDirPath;
_databaseKeySpec = databaseKeySpec;
_keySpecBlock = keySpecBlock;
[self loadDatabase];
}
return self;
}
- (void)loadDatabase
{
[super loadDatabase];
[self protectFiles];
}
- (void)resetStorage
{
_dbConnection = nil;
[super resetStorage];
}
@ -93,11 +93,16 @@ NS_ASSUME_NONNULL_BEGIN
}];
}
- (void)protectFiles
- (void)logFileSizes
{
DDLogInfo(@"%@ Database file size: %@", self.logTag, [OWSFileSystem fileSizeOfPath:self.databaseFilePath]);
DDLogInfo(@"%@ \t SHM file size: %@", self.logTag, [OWSFileSystem fileSizeOfPath:self.databaseFilePath_SHM]);
DDLogInfo(@"%@ \t WAL file size: %@", self.logTag, [OWSFileSystem fileSizeOfPath:self.databaseFilePath_WAL]);
}
- (void)protectFiles
{
[self logFileSizes];
// Protect the entire new database directory.
[OWSFileSystem protectFileOrFolderAtPath:self.databaseDirPath];
@ -140,7 +145,9 @@ NS_ASSUME_NONNULL_BEGIN
- (NSData *)databaseKeySpec
{
return self.databaseKeySpec;
OWSAssert(self.keySpecBlock);
return self.keySpecBlock();
}
- (void)ensureDatabaseKeySpecExists

View File

@ -92,6 +92,8 @@ void runAsyncRegistrationsForStorage(OWSStorage *storage)
self = [super initStorage];
if (self) {
[self loadDatabase];
_dbReadConnection = self.newDatabaseConnection;
_dbReadWriteConnection = self.newDatabaseConnection;

View File

@ -1,5 +1,5 @@
//
// Copyright (c) 2017 Open Whisper Systems. All rights reserved.
// Copyright (c) 2018 Open Whisper Systems. All rights reserved.
//
#import "OWSStorage.h"
@ -8,6 +8,8 @@ NS_ASSUME_NONNULL_BEGIN
@interface OWSStorage (Subclass)
- (void)loadDatabase;
- (void)runSyncRegistrations;
- (void)runAsyncRegistrationsWithCompletion:(void (^_Nonnull)(void))completion;

View File

@ -267,30 +267,6 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void);
self = [super init];
if (self) {
if (![self tryToLoadDatabase]) {
// Failing to load the database is catastrophic.
//
// The best we can try to do is to discard the current database
// and behave like a clean install.
OWSFail(@"%@ Could not load database", self.logTag);
OWSProdCritical([OWSAnalyticsEvents storageErrorCouldNotLoadDatabase]);
// Try to reset app by deleting all databases.
//
// TODO: Possibly clean up all app files.
// [OWSStorage deleteDatabaseFiles];
if (![self tryToLoadDatabase]) {
OWSFail(@"%@ Could not load database (second try)", self.logTag);
OWSProdCritical([OWSAnalyticsEvents storageErrorCouldNotLoadDatabaseSecondAttempt]);
// Sleep to give analytics events time to be delivered.
[NSThread sleepForTimeInterval:15.0f];
OWSRaiseException(OWSStorageExceptionName_NoDatabase, @"Failed to initialize database.");
}
}
[[NSNotificationCenter defaultCenter] addObserver:self
selector:@selector(resetStorage)
name:OWSResetStorageNotification
@ -305,6 +281,33 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void);
[[NSNotificationCenter defaultCenter] removeObserver:self];
}
- (void)loadDatabase
{
if (![self tryToLoadDatabase]) {
// Failing to load the database is catastrophic.
//
// The best we can try to do is to discard the current database
// and behave like a clean install.
OWSFail(@"%@ Could not load database", self.logTag);
OWSProdCritical([OWSAnalyticsEvents storageErrorCouldNotLoadDatabase]);
// Try to reset app by deleting all databases.
//
// TODO: Possibly clean up all app files.
// [OWSStorage deleteDatabaseFiles];
if (![self tryToLoadDatabase]) {
OWSFail(@"%@ Could not load database (second try)", self.logTag);
OWSProdCritical([OWSAnalyticsEvents storageErrorCouldNotLoadDatabaseSecondAttempt]);
// Sleep to give analytics events time to be delivered.
[NSThread sleepForTimeInterval:15.0f];
OWSRaiseException(OWSStorageExceptionName_NoDatabase, @"Failed to initialize database.");
}
}
}
- (nullable id)dbNotificationObject
{
OWSAssert(self.database);

View File

@ -31,6 +31,7 @@ typedef NS_ENUM(NSInteger, OWSErrorCode) {
OWSErrorCodeMoveFileToSharedDataContainerError = 777412,
OWSErrorCodeRegistrationMissing2FAPIN = 777413,
OWSErrorCodeDebugLogUploadFailed = 777414,
OWSErrorCodeExportBackupFailed = 777415,
};
extern NSString *const OWSErrorRecipientIdentifierKey;