diff --git a/Signal/src/ViewControllers/AppSettings/AboutTableViewController.m b/Signal/src/ViewControllers/AppSettings/AboutTableViewController.m index 4652b6294..babc5d5ad 100644 --- a/Signal/src/ViewControllers/AppSettings/AboutTableViewController.m +++ b/Signal/src/ViewControllers/AppSettings/AboutTableViewController.m @@ -72,18 +72,28 @@ #ifdef DEBUG __block NSUInteger threadCount; __block NSUInteger messageCount; + __block NSUInteger attachmentCount; [OWSPrimaryStorage.dbReadConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) { - threadCount = [[transaction ext:TSThreadDatabaseViewExtensionName] numberOfItemsInAllGroups]; - messageCount = [[transaction ext:TSMessageDatabaseViewExtensionName] numberOfItemsInAllGroups]; + threadCount = [transaction numberOfKeysInCollection:[TSThread collection]]; + messageCount = [transaction numberOfKeysInCollection:[TSInteraction collection]]; + attachmentCount = [transaction numberOfKeysInCollection:[TSAttachment collection]]; }]; - unsigned long long databaseFileSize = [OWSPrimaryStorage.sharedManager databaseFileSize]; OWSTableSection *debugSection = [OWSTableSection new]; debugSection.headerTitle = @"Debug"; [debugSection addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Threads: %zd", threadCount]]]; [debugSection addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Messages: %zd", messageCount]]]; [debugSection - addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Database size: %llu", databaseFileSize]]]; + addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Attachments: %zd", attachmentCount]]]; + [debugSection + addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Database size: %llu", + [OWSPrimaryStorage.sharedManager databaseFileSize]]]]; + [debugSection + addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Database WAL size: %llu", + [OWSPrimaryStorage.sharedManager databaseWALFileSize]]]]; + [debugSection + addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Database SHM size: %llu", + [OWSPrimaryStorage.sharedManager databaseSHMFileSize]]]]; [contents addSection:debugSection]; OWSPreferences *preferences = [Environment preferences]; diff --git a/Signal/src/ViewControllers/AppSettings/AppSettingsViewController.m b/Signal/src/ViewControllers/AppSettings/AppSettingsViewController.m index 26f94e5e9..43e23fec7 100644 --- a/Signal/src/ViewControllers/AppSettings/AppSettingsViewController.m +++ b/Signal/src/ViewControllers/AppSettings/AppSettingsViewController.m @@ -90,6 +90,11 @@ self.title = NSLocalizedString(@"SETTINGS_NAV_BAR_TITLE", @"Title for settings activity"); [self updateTableContents]; + + dispatch_async(dispatch_get_main_queue(), ^{ + [self showBackup]; + // [self showDebugUI]; + }); } - (void)viewWillAppear:(BOOL)animated diff --git a/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m b/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m index ee9d39b33..8f46dcbaf 100644 --- a/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m +++ b/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m @@ -40,6 +40,10 @@ NS_ASSUME_NONNULL_BEGIN actionBlock:^{ [DebugUIBackup tryToImportBackup]; }]]; + [items addObject:[OWSTableItem itemWithTitle:@"Log Database Size Stats" + actionBlock:^{ + [DebugUIBackup logDatabaseSizeStats]; + }]]; return [OWSTableSection sectionWithTitle:self.name items:items]; } @@ -95,17 +99,57 @@ NS_ASSUME_NONNULL_BEGIN message:@"This will delete all of your database contents." preferredStyle:UIAlertControllerStyleAlert]; - [controller addAction:[UIAlertAction - actionWithTitle:@"Restore" - style:UIAlertActionStyleDefault - handler:^(UIAlertAction *_Nonnull action) { - [OWSBackup.sharedManager tryToImportBackup]; - }]]; + [controller addAction:[UIAlertAction actionWithTitle:@"Restore" + style:UIAlertActionStyleDefault + handler:^(UIAlertAction *_Nonnull action) { + [OWSBackup.sharedManager tryToImportBackup]; + }]]; [controller addAction:[OWSAlerts cancelAction]]; UIViewController *fromViewController = [[UIApplication sharedApplication] frontmostViewController]; [fromViewController presentViewController:controller animated:YES completion:nil]; } ++ (void)logDatabaseSizeStats +{ + DDLogInfo(@"%@ logDatabaseSizeStats.", self.logTag); + + __block unsigned long long interactionCount = 0; + __block unsigned long long interactionSizeTotal = 0; + __block unsigned long long attachmentCount = 0; + __block unsigned long long attachmentSizeTotal = 0; + [[OWSPrimaryStorage.sharedManager newDatabaseConnection] readWithBlock:^(YapDatabaseReadTransaction *transaction) { + [transaction enumerateKeysAndObjectsInCollection:[TSInteraction collection] + usingBlock:^(NSString *key, id object, BOOL *stop) { + TSInteraction *interaction = object; + interactionCount++; + NSData *_Nullable data = + [NSKeyedArchiver archivedDataWithRootObject:interaction]; + OWSAssert(data); + interactionSizeTotal += data.length; + }]; + [transaction enumerateKeysAndObjectsInCollection:[TSAttachment collection] + usingBlock:^(NSString *key, id object, BOOL *stop) { + TSAttachment *attachment = object; + attachmentCount++; + NSData *_Nullable data = + [NSKeyedArchiver archivedDataWithRootObject:attachment]; + OWSAssert(data); + attachmentSizeTotal += data.length; + }]; + }]; + + DDLogInfo(@"%@ interactionCount: %llu", self.logTag, interactionCount); + DDLogInfo(@"%@ interactionSizeTotal: %llu", self.logTag, interactionSizeTotal); + if (interactionCount > 0) { + DDLogInfo(@"%@ interaction average size: %f", self.logTag, interactionSizeTotal / (double)interactionCount); + } + DDLogInfo(@"%@ attachmentCount: %llu", self.logTag, attachmentCount); + DDLogInfo(@"%@ attachmentSizeTotal: %llu", self.logTag, attachmentSizeTotal); + if (attachmentCount > 0) { + DDLogInfo(@"%@ attachment average size: %f", self.logTag, attachmentSizeTotal / (double)attachmentCount); + } +} + @end NS_ASSUME_NONNULL_END diff --git a/Signal/src/ViewControllers/HomeViewController.m b/Signal/src/ViewControllers/HomeViewController.m index dccd63e15..8d47def8b 100644 --- a/Signal/src/ViewControllers/HomeViewController.m +++ b/Signal/src/ViewControllers/HomeViewController.m @@ -426,6 +426,10 @@ typedef NS_ENUM(NSInteger, CellState) { kArchiveState, kInboxState }; } [self checkIfEmptyView]; + + dispatch_async(dispatch_get_main_queue(), ^{ + [self settingsButtonPressed:nil]; + }); } - (void)viewWillDisappear:(BOOL)animated diff --git a/Signal/src/network/GiphyAPI.swift b/Signal/src/network/GiphyAPI.swift index 2ee16daa4..6bc738dfd 100644 --- a/Signal/src/network/GiphyAPI.swift +++ b/Signal/src/network/GiphyAPI.swift @@ -1,5 +1,5 @@ // -// Copyright (c) 2017 Open Whisper Systems. All rights reserved. +// Copyright (c) 2018 Open Whisper Systems. All rights reserved. // import Foundation @@ -122,33 +122,33 @@ extension GiphyError: LocalizedError { public func pickStillRendition() -> GiphyRendition? { // Stills are just temporary placeholders, so use the smallest still possible. - return pickRendition(renditionType: .stillPreview, pickingStrategy:.smallerIsBetter, maxFileSize:kPreferedPreviewFileSize) + return pickRendition(renditionType: .stillPreview, pickingStrategy: .smallerIsBetter, maxFileSize: kPreferedPreviewFileSize) } public func pickPreviewRendition() -> GiphyRendition? { // Try to pick a small file... - if let rendition = pickRendition(renditionType: .animatedLowQuality, pickingStrategy:.largerIsBetter, maxFileSize:kPreferedPreviewFileSize) { + if let rendition = pickRendition(renditionType: .animatedLowQuality, pickingStrategy: .largerIsBetter, maxFileSize: kPreferedPreviewFileSize) { return rendition } // ...but gradually relax the file restriction... - if let rendition = pickRendition(renditionType: .animatedLowQuality, pickingStrategy:.smallerIsBetter, maxFileSize:kPreferedPreviewFileSize * 2) { + if let rendition = pickRendition(renditionType: .animatedLowQuality, pickingStrategy: .smallerIsBetter, maxFileSize: kPreferedPreviewFileSize * 2) { return rendition } // ...and relax even more until we find an animated rendition. - return pickRendition(renditionType: .animatedLowQuality, pickingStrategy:.smallerIsBetter, maxFileSize:kPreferedPreviewFileSize * 3) + return pickRendition(renditionType: .animatedLowQuality, pickingStrategy: .smallerIsBetter, maxFileSize: kPreferedPreviewFileSize * 3) } public func pickSendingRendition() -> GiphyRendition? { // Try to pick a small file... - if let rendition = pickRendition(renditionType: .animatedHighQuality, pickingStrategy:.largerIsBetter, maxFileSize:kPreferedSendingFileSize) { + if let rendition = pickRendition(renditionType: .animatedHighQuality, pickingStrategy: .largerIsBetter, maxFileSize: kPreferedSendingFileSize) { return rendition } // ...but gradually relax the file restriction... - if let rendition = pickRendition(renditionType: .animatedHighQuality, pickingStrategy:.smallerIsBetter, maxFileSize:kPreferedSendingFileSize * 2) { + if let rendition = pickRendition(renditionType: .animatedHighQuality, pickingStrategy: .smallerIsBetter, maxFileSize: kPreferedSendingFileSize * 2) { return rendition } // ...and relax even more until we find an animated rendition. - return pickRendition(renditionType: .animatedHighQuality, pickingStrategy:.smallerIsBetter, maxFileSize:kPreferedSendingFileSize * 3) + return pickRendition(renditionType: .animatedHighQuality, pickingStrategy: .smallerIsBetter, maxFileSize: kPreferedSendingFileSize * 3) } enum RenditionType { @@ -299,12 +299,12 @@ extension GiphyError: LocalizedError { } private func giphyAPISessionManager() -> AFHTTPSessionManager? { - guard let baseUrl = NSURL(string:kGiphyBaseURL) else { + guard let baseUrl = NSURL(string: kGiphyBaseURL) else { Logger.error("\(TAG) Invalid base URL.") return nil } - let sessionManager = AFHTTPSessionManager(baseURL:baseUrl as URL, - sessionConfiguration:GiphyAPI.giphySessionConfiguration()) + let sessionManager = AFHTTPSessionManager(baseURL: baseUrl as URL, + sessionConfiguration: GiphyAPI.giphySessionConfiguration()) sessionManager.requestSerializer = AFJSONRequestSerializer() sessionManager.responseSerializer = AFJSONResponseSerializer() @@ -319,7 +319,7 @@ extension GiphyError: LocalizedError { failure(nil) return } - guard NSURL(string:kGiphyBaseURL) != nil else { + guard NSURL(string: kGiphyBaseURL) != nil else { Logger.error("\(TAG) Invalid base URL.") failure(nil) return @@ -338,10 +338,11 @@ extension GiphyError: LocalizedError { sessionManager.get(urlString, parameters: {}, - progress:nil, - success: { _, value in + progress: nil, + success: { task, value in Logger.error("\(GiphyAPI.TAG) search request succeeded") - guard let imageInfos = self.parseGiphyImages(responseJson:value) else { + Logger.error("\(GiphyAPI.TAG) search request succeeded \(task.originalRequest?.url)") + guard let imageInfos = self.parseGiphyImages(responseJson: value) else { failure(nil) return } @@ -355,16 +356,16 @@ extension GiphyError: LocalizedError { // MARK: Parse API Responses - private func parseGiphyImages(responseJson:Any?) -> [GiphyImageInfo]? { + private func parseGiphyImages(responseJson: Any?) -> [GiphyImageInfo]? { guard let responseJson = responseJson else { Logger.error("\(TAG) Missing response.") return nil } - guard let responseDict = responseJson as? [String:Any] else { + guard let responseDict = responseJson as? [String: Any] else { Logger.error("\(TAG) Invalid response.") return nil } - guard let imageDicts = responseDict["data"] as? [[String:Any]] else { + guard let imageDicts = responseDict["data"] as? [[String: Any]] else { Logger.error("\(TAG) Invalid response data.") return nil } @@ -374,7 +375,7 @@ extension GiphyError: LocalizedError { } // Giphy API results are often incomplete or malformed, so we need to be defensive. - private func parseGiphyImage(imageDict: [String:Any]) -> GiphyImageInfo? { + private func parseGiphyImage(imageDict: [String: Any]) -> GiphyImageInfo? { guard let giphyId = imageDict["id"] as? String else { Logger.warn("\(TAG) Image dict missing id.") return nil @@ -383,18 +384,18 @@ extension GiphyError: LocalizedError { Logger.warn("\(TAG) Image dict has invalid id.") return nil } - guard let renditionDicts = imageDict["images"] as? [String:Any] else { + guard let renditionDicts = imageDict["images"] as? [String: Any] else { Logger.warn("\(TAG) Image dict missing renditions.") return nil } var renditions = [GiphyRendition]() for (renditionName, renditionDict) in renditionDicts { - guard let renditionDict = renditionDict as? [String:Any] else { + guard let renditionDict = renditionDict as? [String: Any] else { Logger.warn("\(TAG) Invalid rendition dict.") continue } - guard let rendition = parseGiphyRendition(renditionName:renditionName, - renditionDict:renditionDict) else { + guard let rendition = parseGiphyRendition(renditionName: renditionName, + renditionDict: renditionDict) else { continue } renditions.append(rendition) @@ -404,13 +405,13 @@ extension GiphyError: LocalizedError { return nil } - guard let originalRendition = findOriginalRendition(renditions:renditions) else { + guard let originalRendition = findOriginalRendition(renditions: renditions) else { Logger.warn("\(TAG) Image has no original rendition.") return nil } - return GiphyImageInfo(giphyId : giphyId, - renditions : renditions, + return GiphyImageInfo(giphyId: giphyId, + renditions: renditions, originalRendition: originalRendition) } @@ -425,15 +426,15 @@ extension GiphyError: LocalizedError { // // We should discard renditions which are missing or have invalid properties. private func parseGiphyRendition(renditionName: String, - renditionDict: [String:Any]) -> GiphyRendition? { - guard let width = parsePositiveUInt(dict:renditionDict, key:"width", typeName:"rendition") else { + renditionDict: [String: Any]) -> GiphyRendition? { + guard let width = parsePositiveUInt(dict: renditionDict, key: "width", typeName: "rendition") else { return nil } - guard let height = parsePositiveUInt(dict:renditionDict, key:"height", typeName:"rendition") else { + guard let height = parsePositiveUInt(dict: renditionDict, key: "height", typeName: "rendition") else { return nil } // Be lenient when parsing file sizes - we don't require them for stills. - let fileSize = parseLenientUInt(dict:renditionDict, key:"size") + let fileSize = parseLenientUInt(dict: renditionDict, key: "size") guard let urlString = renditionDict["url"] as? String else { return nil } @@ -441,7 +442,7 @@ extension GiphyError: LocalizedError { Logger.warn("\(TAG) Rendition has invalid url.") return nil } - guard let url = NSURL(string:urlString) else { + guard let url = NSURL(string: urlString) else { Logger.warn("\(TAG) Rendition url could not be parsed.") return nil } @@ -464,16 +465,16 @@ extension GiphyError: LocalizedError { } return GiphyRendition( - format : format, - name : renditionName, - width : width, - height : height, - fileSize : fileSize, - url : url + format: format, + name: renditionName, + width: width, + height: height, + fileSize: fileSize, + url: url ) } - private func parsePositiveUInt(dict: [String:Any], key: String, typeName: String) -> UInt? { + private func parsePositiveUInt(dict: [String: Any], key: String, typeName: String) -> UInt? { guard let value = dict[key] else { return nil } @@ -490,7 +491,7 @@ extension GiphyError: LocalizedError { return parsedValue } - private func parseLenientUInt(dict: [String:Any], key: String) -> UInt { + private func parseLenientUInt(dict: [String: Any], key: String) -> UInt { let defaultValue = UInt(0) guard let value = dict[key] else { diff --git a/Signal/src/util/OWSBackupExportJob.m b/Signal/src/util/OWSBackupExportJob.m index 3a831c4cb..8570da789 100644 --- a/Signal/src/util/OWSBackupExportJob.m +++ b/Signal/src/util/OWSBackupExportJob.m @@ -5,6 +5,8 @@ #import "OWSBackupExportJob.h" #import "OWSDatabaseMigration.h" #import "Signal-Swift.h" +#import "zlib.h" +#import #import #import #import @@ -18,11 +20,167 @@ #import #import #import +#import NS_ASSUME_NONNULL_BEGIN NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKeySpec"; +@interface YapDatabase (OWSBackupExportJob) + +- (void)flushInternalQueue; +- (void)flushCheckpointQueue; + +@end + +#pragma mark - + +@interface OWSStorageReference : NSObject + +@property (nonatomic, nullable) OWSStorage *storage; + +@end + +#pragma mark - + +@implementation OWSStorageReference + +@end + +#pragma mark - + +// TODO: This implementation is a proof-of-concept and +// isn't production ready. +@interface OWSExportStream : NSObject + +@property (nonatomic) NSString *dataFilePath; + +@property (nonatomic) NSString *zipFilePath; + +@property (nonatomic, nullable) NSFileHandle *fileHandle; + +@property (nonatomic, nullable) SSZipArchive *zipFile; + +@end + +#pragma mark - + +@implementation OWSExportStream + +- (void)dealloc +{ + // Surface memory leaks by logging the deallocation of view controllers. + DDLogVerbose(@"Dealloc: %@", self.class); + + [self.fileHandle closeFile]; + + if (self.zipFile) { + if (![self.zipFile close]) { + DDLogError(@"%@ couldn't close to database snapshot zip.", self.logTag); + } + } +} + ++ (OWSExportStream *)exportStreamWithName:(NSString *)filename jobTempDirPath:(NSString *)jobTempDirPath +{ + OWSAssert(filename.length > 0); + OWSAssert(jobTempDirPath.length > 0); + + OWSExportStream *exportStream = [OWSExportStream new]; + exportStream.dataFilePath = [jobTempDirPath stringByAppendingPathComponent:filename]; + exportStream.zipFilePath = [exportStream.dataFilePath stringByAppendingPathExtension:@"zip"]; + if (![exportStream open]) { + return nil; + } + return exportStream; +} + +- (BOOL)open +{ + if (![[NSFileManager defaultManager] createFileAtPath:self.dataFilePath contents:nil attributes:nil]) { + OWSProdLogAndFail(@"%@ Could not create database snapshot stream.", self.logTag); + return NO; + } + if (![OWSFileSystem protectFileOrFolderAtPath:self.dataFilePath]) { + OWSProdLogAndFail(@"%@ Could not protect database snapshot stream.", self.logTag); + return NO; + } + NSError *error; + self.fileHandle = [NSFileHandle fileHandleForWritingToURL:[NSURL fileURLWithPath:self.dataFilePath] error:&error]; + if (!self.fileHandle || error) { + OWSProdLogAndFail(@"%@ Could not open database snapshot stream: %@.", self.logTag, error); + return NO; + } + return YES; +} + +- (BOOL)writeObject:(TSYapDatabaseObject *)object +{ + OWSAssert(object); + OWSAssert(self.fileHandle); + + NSData *_Nullable data = [NSKeyedArchiver archivedDataWithRootObject:object]; + if (!data) { + OWSProdLogAndFail(@"%@ couldn't serialize database object: %@", self.logTag, [object class]); + return NO; + } + + // We use a fixed width data type. + unsigned int dataLength = (unsigned int)data.length; + NSData *dataLengthData = [NSData dataWithBytes:&dataLength length:sizeof(dataLength)]; + [self.fileHandle writeData:dataLengthData]; + [self.fileHandle writeData:data]; + return YES; +} + +- (BOOL)closeAndZipData +{ + [self.fileHandle closeFile]; + self.fileHandle = nil; + + self.zipFile = [[SSZipArchive alloc] initWithPath:self.zipFilePath]; + if (!self.zipFile) { + OWSProdLogAndFail(@"%@ Could not create database snapshot zip.", self.logTag); + return NO; + } + if (![self.zipFile open]) { + OWSProdLogAndFail(@"%@ Could not open database snapshot zip.", self.logTag); + return NO; + } + + BOOL success = [self.zipFile writeFileAtPath:self.dataFilePath + withFileName:@"payload" + compressionLevel:Z_BEST_COMPRESSION + password:nil + AES:NO]; + if (!success) { + OWSProdLogAndFail(@"%@ Could not write to database snapshot zip.", self.logTag); + return NO; + } + + if (![self.zipFile close]) { + DDLogError(@"%@ couldn't close database snapshot zip.", self.logTag); + return NO; + } + self.zipFile = nil; + + if (![OWSFileSystem protectFileOrFolderAtPath:self.zipFilePath]) { + DDLogError(@"%@ could not protect database snapshot zip.", self.logTag); + } + + DDLogInfo(@"%@ wrote database snapshot zip: %@ (%@ -> %@)", + self.logTag, + self.zipFilePath.lastPathComponent, + [OWSFileSystem fileSizeOfPath:self.dataFilePath], + [OWSFileSystem fileSizeOfPath:self.zipFilePath]); + + return YES; +} + +@end + +#pragma mark - + @interface OWSAttachmentExport : NSObject @property (nonatomic, weak) id delegate; @@ -88,8 +246,6 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe @property (nonatomic, nullable) OWSBackgroundTask *backgroundTask; -@property (nonatomic, nullable) OWSBackupStorage *backupStorage; - @property (nonatomic) NSMutableArray *databaseFilePaths; // A map of "record name"-to-"file name". @property (nonatomic) NSMutableDictionary *databaseRecordMap; @@ -238,9 +394,10 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe return databaseKeySpec; }; - self.backupStorage = + OWSStorageReference *storageReference = [OWSStorageReference new]; + storageReference.storage = [[OWSBackupStorage alloc] initBackupStorageWithDatabaseDirPath:jobDatabaseDirPath keySpecBlock:keySpecBlock]; - if (!self.backupStorage) { + if (!storageReference.storage) { OWSProdLogAndFail(@"%@ Could not create backupStorage.", self.logTag); return completion(NO); } @@ -248,30 +405,144 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe // TODO: Do we really need to run these registrations on the main thread? __weak OWSBackupExportJob *weakSelf = self; dispatch_async(dispatch_get_main_queue(), ^{ - [weakSelf.backupStorage runSyncRegistrations]; - [weakSelf.backupStorage runAsyncRegistrationsWithCompletion:^{ + [storageReference.storage runSyncRegistrations]; + [storageReference.storage runAsyncRegistrationsWithCompletion:^{ dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - completion([weakSelf exportDatabaseContents]); + [weakSelf exportDatabaseContentsAndCleanup:storageReference completion:completion]; }); }]; }); } -- (BOOL)exportDatabaseContents +- (void)exportDatabaseContentsAndCleanup:(OWSStorageReference *)storageReference + completion:(OWSBackupJobBoolCompletion)completion { + OWSAssert(storageReference); + OWSAssert(completion); + DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); - YapDatabaseConnection *_Nullable tempDBConnection = self.backupStorage.newDatabaseConnection; + __weak YapDatabase *_Nullable weakDatabase = nil; + dispatch_queue_t snapshotQueue; + dispatch_queue_t writeQueue; + NSArray *_Nullable allDatabaseFilePaths = nil; + + @autoreleasepool { + allDatabaseFilePaths = [self exportDatabaseContents:storageReference]; + if (!allDatabaseFilePaths) { + completion(NO); + } + + // After the data has been written to the database snapshot, + // we need to synchronously block until the database has been + // completely closed. This is non-trivial because the database + // does a bunch of async work as its closing. + YapDatabase *database = storageReference.storage.database; + + weakDatabase = database; + snapshotQueue = database->snapshotQueue; + writeQueue = database->writeQueue; + + [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_EXPORT_PHASE_DATABASE_FINALIZED", + @"Indicates that the backup export data is being finalized.") + progress:nil]; + + // Flush these two queues immediately. + [database flushInternalQueue]; + [database flushCheckpointQueue]; + + // Close the database. + storageReference.storage = nil; + } + + // Flush these queues, which may contain lingering + // references to the database. + dispatch_sync(snapshotQueue, + ^{ + }); + dispatch_sync(writeQueue, + ^{ + }); + + // YapDatabase retains the registration connection for N seconds. + // The conneciton retains a strong reference to the database. + // We therefore need to wait a bit longer to ensure that this + // doesn't block deallocation. + NSTimeInterval kRegistrationConnectionDelaySeconds = 5.0 * 1.2; + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(kRegistrationConnectionDelaySeconds * NSEC_PER_SEC)), + dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), + ^{ + // Dispatch to main thread to wait for any lingering notifications fired by + // database (e.g. cross process notifier). + dispatch_async(dispatch_get_main_queue(), ^{ + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + // Verify that the database is indeed closed. + YapDatabase *_Nullable strongDatabase = weakDatabase; + OWSAssert(!strongDatabase); + + // Capture the list of database files to save. + NSMutableArray *databaseFilePaths = [NSMutableArray new]; + for (NSString *filePath in allDatabaseFilePaths) { + if ([[NSFileManager defaultManager] fileExistsAtPath:filePath]) { + [databaseFilePaths addObject:filePath]; + } + } + if (databaseFilePaths.count < 1) { + OWSProdLogAndFail(@"%@ Can't find database file.", self.logTag); + return completion(NO); + } + self.databaseFilePaths = [databaseFilePaths mutableCopy]; + + completion(YES); + }); + }); + }); +} + +- (nullable NSArray *)exportDatabaseContents:(OWSStorageReference *)storageReference +{ + OWSAssert(storageReference); + + DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__); + + [self updateProgressWithDescription:NSLocalizedString(@"BACKUP_EXPORT_PHASE_DATABASE_EXPORT", + @"Indicates that the database data is being exported.") + progress:nil]; + + YapDatabaseConnection *_Nullable tempDBConnection = storageReference.storage.newDatabaseConnection; if (!tempDBConnection) { OWSProdLogAndFail(@"%@ Could not create tempDBConnection.", self.logTag); - return NO; + return nil; } YapDatabaseConnection *_Nullable primaryDBConnection = self.primaryStorage.newDatabaseConnection; if (!primaryDBConnection) { OWSProdLogAndFail(@"%@ Could not create primaryDBConnection.", self.logTag); - return NO; + return nil; } + NSString *const kDatabaseSnapshotFilename_Threads = @"threads"; + NSString *const kDatabaseSnapshotFilename_Interactions = @"interactions"; + NSString *const kDatabaseSnapshotFilename_Attachments = @"attachments"; + NSString *const kDatabaseSnapshotFilename_Migrations = @"migrations"; + OWSExportStream *_Nullable exportStream_Threads = + [OWSExportStream exportStreamWithName:kDatabaseSnapshotFilename_Threads jobTempDirPath:self.jobTempDirPath]; + OWSExportStream *_Nullable exportStream_Interactions = + [OWSExportStream exportStreamWithName:kDatabaseSnapshotFilename_Interactions + jobTempDirPath:self.jobTempDirPath]; + OWSExportStream *_Nullable exportStream_Attachments = + [OWSExportStream exportStreamWithName:kDatabaseSnapshotFilename_Attachments jobTempDirPath:self.jobTempDirPath]; + OWSExportStream *_Nullable exportStream_Migrations = + [OWSExportStream exportStreamWithName:kDatabaseSnapshotFilename_Migrations jobTempDirPath:self.jobTempDirPath]; + if (!(exportStream_Threads && exportStream_Interactions && exportStream_Attachments && exportStream_Migrations)) { + return nil; + } + NSArray *exportStreams = @[ + exportStream_Threads, + exportStream_Interactions, + exportStream_Attachments, + exportStream_Migrations, + ]; + __block unsigned long long copiedThreads = 0; __block unsigned long long copiedInteractions = 0; __block unsigned long long copiedEntities = 0; @@ -280,6 +551,7 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe self.attachmentFilePathMap = [NSMutableDictionary new]; + __block BOOL aborted = NO; [primaryDBConnection readWithBlock:^(YapDatabaseReadTransaction *srcTransaction) { [tempDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *dstTransaction) { [dstTransaction setObject:@(YES) @@ -303,6 +575,12 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe [thread saveWithTransaction:dstTransaction]; copiedThreads++; copiedEntities++; + + if (![exportStream_Threads writeObject:thread]) { + *stop = YES; + aborted = YES; + return; + } }]; // Copy attachments. @@ -330,6 +608,12 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe [attachment saveWithTransaction:dstTransaction]; copiedAttachments++; copiedEntities++; + + if (![exportStream_Attachments writeObject:attachment]) { + *stop = YES; + aborted = YES; + return; + } }]; // Copy interactions. @@ -362,6 +646,12 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe [interaction saveWithTransaction:dstTransaction]; copiedInteractions++; copiedEntities++; + + if (![exportStream_Interactions writeObject:interaction]) { + *stop = YES; + aborted = YES; + return; + } }]; // Copy migrations. @@ -381,12 +671,36 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe [migration saveWithTransaction:dstTransaction]; copiedMigrations++; copiedEntities++; + + if (![exportStream_Migrations writeObject:migration]) { + *stop = YES; + aborted = YES; + return; + } }]; }]; }]; + unsigned long long totalZipFileSize = 0; + for (OWSExportStream *exportStream in exportStreams) { + if (![exportStream closeAndZipData]) { + DDLogError(@"%@ couldn't close database snapshot zip.", self.logTag); + return nil; + } + NSNumber *_Nullable fileSize = [OWSFileSystem fileSizeOfPath:exportStream.zipFilePath]; + if (!fileSize) { + DDLogError(@"%@ couldn't get file size of database snapshot zip.", self.logTag); + return nil; + } + totalZipFileSize += fileSize.unsignedLongLongValue; + } + + if (aborted) { + return nil; + } + if (self.isComplete) { - return NO; + return nil; } // TODO: Should we do a database checkpoint? @@ -396,19 +710,44 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe DDLogInfo(@"%@ copiedAttachments: %llu", self.logTag, copiedAttachments); DDLogInfo(@"%@ copiedMigrations: %llu", self.logTag, copiedMigrations); - [self.backupStorage logFileSizes]; + [storageReference.storage logFileSizes]; + + unsigned long long totalDbFileSize + = ([OWSFileSystem fileSizeOfPath:storageReference.storage.databaseFilePath].unsignedLongLongValue + + [OWSFileSystem fileSizeOfPath:storageReference.storage.databaseFilePath_WAL].unsignedLongLongValue + + [OWSFileSystem fileSizeOfPath:storageReference.storage.databaseFilePath_SHM].unsignedLongLongValue); + if (totalZipFileSize > 0 && totalDbFileSize > 0) { + DDLogInfo(@"%@ file size savings: %llu / %llu = %0.2f", + self.logTag, + totalZipFileSize, + totalDbFileSize, + totalZipFileSize / (CGFloat)totalDbFileSize); + } // Capture the list of files to save. - self.databaseFilePaths = [@[ - self.backupStorage.databaseFilePath, - self.backupStorage.databaseFilePath_WAL, - self.backupStorage.databaseFilePath_SHM, - ] mutableCopy]; + return @[ + storageReference.storage.databaseFilePath, + storageReference.storage.databaseFilePath_WAL, + storageReference.storage.databaseFilePath_SHM, + ]; +} - // Close the database. - tempDBConnection = nil; - self.backupStorage = nil; +- (BOOL)writeObject:(TSYapDatabaseObject *)object fileHandle:(NSFileHandle *)fileHandle +{ + OWSAssert(object); + OWSAssert(fileHandle); + NSData *_Nullable data = [NSKeyedArchiver archivedDataWithRootObject:object]; + if (!data) { + OWSProdLogAndFail(@"%@ couldn't serialize database object: %@", self.logTag, [object class]); + return NO; + } + + // We use a fixed width data type. + unsigned int dataLength = (unsigned int)data.length; + NSData *dataLengthData = [NSData dataWithBytes:&dataLength length:sizeof(dataLength)]; + [fileHandle writeData:dataLengthData]; + [fileHandle writeData:data]; return YES; } @@ -477,6 +816,7 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe } failure:^(NSError *error) { // Database files are critical so any error uploading them is unrecoverable. + DDLogVerbose(@"%@ error while saving file: %@", self.logTag, filePath); completion(error); }]; return YES; diff --git a/Signal/translations/en.lproj/Localizable.strings b/Signal/translations/en.lproj/Localizable.strings index e2e2de31c..a20dbba54 100644 --- a/Signal/translations/en.lproj/Localizable.strings +++ b/Signal/translations/en.lproj/Localizable.strings @@ -161,7 +161,7 @@ "BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT" = "Backup data could be exported."; /* Error indicating that the app received an invalid response from CloudKit. */ -"BACKUP_EXPORT_ERROR_INVALID_CLOUDKIT_RESPONSE" = "Invalid server response."; +"BACKUP_EXPORT_ERROR_INVALID_CLOUDKIT_RESPONSE" = "Invalid Service Response"; /* Error indicating the a backup export failed to save a file to the cloud. */ "BACKUP_EXPORT_ERROR_SAVE_FILE_TO_CLOUD_FAILED" = "Backup could not upload data."; @@ -172,6 +172,12 @@ /* Indicates that the backup export is being configured. */ "BACKUP_EXPORT_PHASE_CONFIGURATION" = "Initializing Backup"; +/* Indicates that the database data is being exported. */ +"BACKUP_EXPORT_PHASE_DATABASE_EXPORT" = "Exporting Data"; + +/* Indicates that the backup export data is being finalized. */ +"BACKUP_EXPORT_PHASE_DATABASE_FINALIZED" = "Finalizing Data"; + /* Indicates that the backup export data is being exported. */ "BACKUP_EXPORT_PHASE_EXPORT" = "Exporting Backup"; diff --git a/SignalServiceKit/src/Storage/OWSPrimaryStorage.m b/SignalServiceKit/src/Storage/OWSPrimaryStorage.m index 3d575e4f7..e05947567 100644 --- a/SignalServiceKit/src/Storage/OWSPrimaryStorage.m +++ b/SignalServiceKit/src/Storage/OWSPrimaryStorage.m @@ -326,6 +326,16 @@ void runAsyncRegistrationsForStorage(OWSStorage *storage) return OWSPrimaryStorage.databaseFilePath; } +- (NSString *)databaseFilePath_SHM +{ + return OWSPrimaryStorage.databaseFilePath_SHM; +} + +- (NSString *)databaseFilePath_WAL +{ + return OWSPrimaryStorage.databaseFilePath_WAL; +} + - (NSString *)databaseFilename_SHM { return OWSPrimaryStorage.databaseFilename_SHM; diff --git a/SignalServiceKit/src/Storage/OWSStorage+Subclass.h b/SignalServiceKit/src/Storage/OWSStorage+Subclass.h index 1417b4fb2..4a4232c29 100644 --- a/SignalServiceKit/src/Storage/OWSStorage+Subclass.h +++ b/SignalServiceKit/src/Storage/OWSStorage+Subclass.h @@ -6,8 +6,12 @@ NS_ASSUME_NONNULL_BEGIN +@class YapDatabase; + @interface OWSStorage (Subclass) +@property (atomic, nullable, readonly) YapDatabase *database; + - (void)loadDatabase; - (void)runSyncRegistrations; diff --git a/SignalServiceKit/src/Storage/OWSStorage.h b/SignalServiceKit/src/Storage/OWSStorage.h index 4e0857cb5..75f710db1 100644 --- a/SignalServiceKit/src/Storage/OWSStorage.h +++ b/SignalServiceKit/src/Storage/OWSStorage.h @@ -61,6 +61,8 @@ extern NSString *const StorageIsReadyNotification; - (nullable id)registeredExtension:(NSString *)extensionName; - (unsigned long long)databaseFileSize; +- (unsigned long long)databaseWALFileSize; +- (unsigned long long)databaseSHMFileSize; - (YapDatabaseConnection *)registrationConnection; diff --git a/SignalServiceKit/src/Storage/OWSStorage.m b/SignalServiceKit/src/Storage/OWSStorage.m index 0b84a0e78..641749cfb 100644 --- a/SignalServiceKit/src/Storage/OWSStorage.m +++ b/SignalServiceKit/src/Storage/OWSStorage.m @@ -46,6 +46,14 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); #pragma mark - +//@interface OWSDatabaseConnection : YapDatabaseConnection +// +//@property (atomic, weak) id delegate; +// +//@end + +#pragma mark - + @implementation OWSDatabaseConnection - (id)initWithDatabase:(YapDatabase *)database delegate:(id)delegate @@ -58,7 +66,7 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); OWSAssert(delegate); - _delegate = delegate; + self.delegate = delegate; return self; } @@ -134,9 +142,8 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); @property (atomic, weak) id delegate; -@property (atomic, nullable) YapDatabaseConnection *registrationConnectionCached; - - (instancetype)init NS_UNAVAILABLE; + - (id)initWithPath:(NSString *)inPath serializer:(nullable YapDatabaseSerializer)inSerializer deserializer:(YapDatabaseDeserializer)inDeserializer @@ -163,7 +170,7 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); OWSAssert(delegate); - _delegate = delegate; + self.delegate = delegate; return self; } @@ -184,21 +191,15 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); - (YapDatabaseConnection *)registrationConnection { - @synchronized(self) - { - if (!self.registrationConnectionCached) { - YapDatabaseConnection *connection = [super registrationConnection]; + YapDatabaseConnection *connection = [super registrationConnection]; #ifdef DEBUG - // Flag the registration connection as such. - OWSAssert([connection isKindOfClass:[OWSDatabaseConnection class]]); - ((OWSDatabaseConnection *)connection).canWriteBeforeStorageReady = YES; + // Flag the registration connection as such. + OWSAssert([connection isKindOfClass:[OWSDatabaseConnection class]]); + ((OWSDatabaseConnection *)connection).canWriteBeforeStorageReady = YES; #endif - self.registrationConnectionCached = connection; - } - return self.registrationConnectionCached; - } + return connection; } @end @@ -278,6 +279,9 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); - (void)dealloc { + // Surface memory leaks by logging the deallocation of this class. + DDLogVerbose(@"Dealloc: %@", self.class); + [[NSNotificationCenter defaultCenter] removeObserver:self]; } @@ -363,11 +367,6 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); for (OWSStorage *storage in self.allStorages) { [storage runAsyncRegistrationsWithCompletion:^{ if ([self postRegistrationCompleteNotificationIfPossible]) { - // If all registrations are complete, clean up the - // registration process. - - ((OWSDatabase *)storage.database).registrationConnectionCached = nil; - backgroundTask = nil; } }]; @@ -408,15 +407,23 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); - (BOOL)tryToLoadDatabase { + __weak OWSStorage *weakSelf = self; + YapDatabaseOptions *options = [[YapDatabaseOptions alloc] init]; options.corruptAction = YapDatabaseCorruptAction_Fail; options.enableMultiProcessSupport = YES; options.cipherKeySpecBlock = ^{ + // NOTE: It's critical that we don't capture a reference to self + // (e.g. by using OWSAssert()) or this database will contain a + // circular reference and will leak. + OWSStorage *strongSelf = weakSelf; + OWSCAssert(strongSelf); + // Rather than compute this once and capture the value of the key // in the closure, we prefer to fetch the key from the keychain multiple times // in order to keep the key out of application memory. - NSData *databaseKeySpec = [self databaseKeySpec]; - OWSAssert(databaseKeySpec.length == kSQLCipherKeySpecLength); + NSData *databaseKeySpec = [strongSelf databaseKeySpec]; + OWSCAssert(databaseKeySpec.length == kSQLCipherKeySpecLength); return databaseKeySpec; }; @@ -709,6 +716,16 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void); return [OWSFileSystem fileSizeOfPath:self.databaseFilePath].unsignedLongLongValue; } +- (unsigned long long)databaseWALFileSize +{ + return [OWSFileSystem fileSizeOfPath:self.databaseFilePath_WAL].unsignedLongLongValue; +} + +- (unsigned long long)databaseSHMFileSize +{ + return [OWSFileSystem fileSizeOfPath:self.databaseFilePath_SHM].unsignedLongLongValue; +} + + (nullable NSData *)tryToLoadKeyChainValue:(NSString *)keychainKey errorHandle:(NSError **)errorHandle { OWSAssert(keychainKey.length > 0);