oxen-io
/
session-ios
mirror of https://github.com/oxen-io/session-ios.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Improve perf of database snapshots.

This commit is contained in:
Matthew Chen 2018-03-16 08:48:21 -03:00
parent 267e85915a
commit 1bbd41f725
11 changed files with 536 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
Signal/src/ViewControllers/AppSettings/AboutTableViewController.m
View File

@ -72,18 +72,28 @@
#ifdef DEBUG
__block NSUInteger threadCount;
__block NSUInteger messageCount;
__block NSUInteger attachmentCount;
[OWSPrimaryStorage.dbReadConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) {
threadCount = [[transaction ext:TSThreadDatabaseViewExtensionName] numberOfItemsInAllGroups];
messageCount = [[transaction ext:TSMessageDatabaseViewExtensionName] numberOfItemsInAllGroups];
threadCount = [transaction numberOfKeysInCollection:[TSThread collection]];
messageCount = [transaction numberOfKeysInCollection:[TSInteraction collection]];
attachmentCount = [transaction numberOfKeysInCollection:[TSAttachment collection]];
}];
unsigned long long databaseFileSize = [OWSPrimaryStorage.sharedManager databaseFileSize];
OWSTableSection *debugSection = [OWSTableSection new];
debugSection.headerTitle = @"Debug";
[debugSection addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Threads: %zd", threadCount]]];
[debugSection addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Messages: %zd", messageCount]]];
[debugSection
addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Database size: %llu", databaseFileSize]]];
addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Attachments: %zd", attachmentCount]]];
[debugSection
addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Database size: %llu",
[OWSPrimaryStorage.sharedManager databaseFileSize]]]];
[debugSection
addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Database WAL size: %llu",
[OWSPrimaryStorage.sharedManager databaseWALFileSize]]]];
[debugSection
addItem:[OWSTableItem labelItemWithText:[NSString stringWithFormat:@"Database SHM size: %llu",
[OWSPrimaryStorage.sharedManager databaseSHMFileSize]]]];
[contents addSection:debugSection];
OWSPreferences *preferences = [Environment preferences];

5
Signal/src/ViewControllers/AppSettings/AppSettingsViewController.m
View File

@ -90,6 +90,11 @@
self.title = NSLocalizedString(@"SETTINGS_NAV_BAR_TITLE", @"Title for settings activity");
[self updateTableContents];
dispatch_async(dispatch_get_main_queue(), ^{
[self showBackup];
// [self showDebugUI];
});
}
- (void)viewWillAppear:(BOOL)animated

56
Signal/src/ViewControllers/DebugUI/DebugUIBackup.m
View File

@ -40,6 +40,10 @@ NS_ASSUME_NONNULL_BEGIN
actionBlock:^{
[DebugUIBackup tryToImportBackup];
}]];
[items addObject:[OWSTableItem itemWithTitle:@"Log Database Size Stats"
actionBlock:^{
[DebugUIBackup logDatabaseSizeStats];
}]];
return [OWSTableSection sectionWithTitle:self.name items:items];
}
@ -95,17 +99,57 @@ NS_ASSUME_NONNULL_BEGIN
message:@"This will delete all of your database contents."
preferredStyle:UIAlertControllerStyleAlert];
[controller addAction:[UIAlertAction
actionWithTitle:@"Restore"
style:UIAlertActionStyleDefault
handler:^(UIAlertAction *_Nonnull action) {
[OWSBackup.sharedManager tryToImportBackup];
}]];
[controller addAction:[UIAlertAction actionWithTitle:@"Restore"
style:UIAlertActionStyleDefault
handler:^(UIAlertAction *_Nonnull action) {
[OWSBackup.sharedManager tryToImportBackup];
}]];
[controller addAction:[OWSAlerts cancelAction]];
UIViewController *fromViewController = [[UIApplication sharedApplication] frontmostViewController];
[fromViewController presentViewController:controller animated:YES completion:nil];
}
+ (void)logDatabaseSizeStats
{
DDLogInfo(@"%@ logDatabaseSizeStats.", self.logTag);
__block unsigned long long interactionCount = 0;
__block unsigned long long interactionSizeTotal = 0;
__block unsigned long long attachmentCount = 0;
__block unsigned long long attachmentSizeTotal = 0;
[[OWSPrimaryStorage.sharedManager newDatabaseConnection] readWithBlock:^(YapDatabaseReadTransaction *transaction) {
[transaction enumerateKeysAndObjectsInCollection:[TSInteraction collection]
usingBlock:^(NSString *key, id object, BOOL *stop) {
TSInteraction *interaction = object;
interactionCount++;
NSData *_Nullable data =
[NSKeyedArchiver archivedDataWithRootObject:interaction];
OWSAssert(data);
interactionSizeTotal += data.length;
}];
[transaction enumerateKeysAndObjectsInCollection:[TSAttachment collection]
usingBlock:^(NSString *key, id object, BOOL *stop) {
TSAttachment *attachment = object;
attachmentCount++;
NSData *_Nullable data =
[NSKeyedArchiver archivedDataWithRootObject:attachment];
OWSAssert(data);
attachmentSizeTotal += data.length;
}];
}];
DDLogInfo(@"%@ interactionCount: %llu", self.logTag, interactionCount);
DDLogInfo(@"%@ interactionSizeTotal: %llu", self.logTag, interactionSizeTotal);
if (interactionCount > 0) {
DDLogInfo(@"%@ interaction average size: %f", self.logTag, interactionSizeTotal / (double)interactionCount);
}
DDLogInfo(@"%@ attachmentCount: %llu", self.logTag, attachmentCount);
DDLogInfo(@"%@ attachmentSizeTotal: %llu", self.logTag, attachmentSizeTotal);
if (attachmentCount > 0) {
DDLogInfo(@"%@ attachment average size: %f", self.logTag, attachmentSizeTotal / (double)attachmentCount);
}
}
@end
NS_ASSUME_NONNULL_END

4
Signal/src/ViewControllers/HomeViewController.m
View File

@ -426,6 +426,10 @@ typedef NS_ENUM(NSInteger, CellState) { kArchiveState, kInboxState };
}
[self checkIfEmptyView];
dispatch_async(dispatch_get_main_queue(), ^{
[self settingsButtonPressed:nil];
});
}
- (void)viewWillDisappear:(BOOL)animated

79
Signal/src/network/GiphyAPI.swift
View File

@ -1,5 +1,5 @@
//
// Copyright (c) 2017 Open Whisper Systems. All rights reserved.
// Copyright (c) 2018 Open Whisper Systems. All rights reserved.
//
import Foundation
@ -122,33 +122,33 @@ extension GiphyError: LocalizedError {
public func pickStillRendition() -> GiphyRendition? {
// Stills are just temporary placeholders, so use the smallest still possible.
return pickRendition(renditionType: .stillPreview, pickingStrategy:.smallerIsBetter, maxFileSize:kPreferedPreviewFileSize)
return pickRendition(renditionType: .stillPreview, pickingStrategy: .smallerIsBetter, maxFileSize: kPreferedPreviewFileSize)
}
public func pickPreviewRendition() -> GiphyRendition? {
// Try to pick a small file...
if let rendition = pickRendition(renditionType: .animatedLowQuality, pickingStrategy:.largerIsBetter, maxFileSize:kPreferedPreviewFileSize) {
if let rendition = pickRendition(renditionType: .animatedLowQuality, pickingStrategy: .largerIsBetter, maxFileSize: kPreferedPreviewFileSize) {
return rendition
}
// ...but gradually relax the file restriction...
if let rendition = pickRendition(renditionType: .animatedLowQuality, pickingStrategy:.smallerIsBetter, maxFileSize:kPreferedPreviewFileSize * 2) {
if let rendition = pickRendition(renditionType: .animatedLowQuality, pickingStrategy: .smallerIsBetter, maxFileSize: kPreferedPreviewFileSize * 2) {
return rendition
}
// ...and relax even more until we find an animated rendition.
return pickRendition(renditionType: .animatedLowQuality, pickingStrategy:.smallerIsBetter, maxFileSize:kPreferedPreviewFileSize * 3)
return pickRendition(renditionType: .animatedLowQuality, pickingStrategy: .smallerIsBetter, maxFileSize: kPreferedPreviewFileSize * 3)
}
public func pickSendingRendition() -> GiphyRendition? {
// Try to pick a small file...
if let rendition = pickRendition(renditionType: .animatedHighQuality, pickingStrategy:.largerIsBetter, maxFileSize:kPreferedSendingFileSize) {
if let rendition = pickRendition(renditionType: .animatedHighQuality, pickingStrategy: .largerIsBetter, maxFileSize: kPreferedSendingFileSize) {
return rendition
}
// ...but gradually relax the file restriction...
if let rendition = pickRendition(renditionType: .animatedHighQuality, pickingStrategy:.smallerIsBetter, maxFileSize:kPreferedSendingFileSize * 2) {
if let rendition = pickRendition(renditionType: .animatedHighQuality, pickingStrategy: .smallerIsBetter, maxFileSize: kPreferedSendingFileSize * 2) {
return rendition
}
// ...and relax even more until we find an animated rendition.
return pickRendition(renditionType: .animatedHighQuality, pickingStrategy:.smallerIsBetter, maxFileSize:kPreferedSendingFileSize * 3)
return pickRendition(renditionType: .animatedHighQuality, pickingStrategy: .smallerIsBetter, maxFileSize: kPreferedSendingFileSize * 3)
}
enum RenditionType {
@ -299,12 +299,12 @@ extension GiphyError: LocalizedError {
}
private func giphyAPISessionManager() -> AFHTTPSessionManager? {
guard let baseUrl = NSURL(string:kGiphyBaseURL) else {
guard let baseUrl = NSURL(string: kGiphyBaseURL) else {
Logger.error("\(TAG) Invalid base URL.")
return nil
}
let sessionManager = AFHTTPSessionManager(baseURL:baseUrl as URL,
sessionConfiguration:GiphyAPI.giphySessionConfiguration())
let sessionManager = AFHTTPSessionManager(baseURL: baseUrl as URL,
sessionConfiguration: GiphyAPI.giphySessionConfiguration())
sessionManager.requestSerializer = AFJSONRequestSerializer()
sessionManager.responseSerializer = AFJSONResponseSerializer()
@ -319,7 +319,7 @@ extension GiphyError: LocalizedError {
failure(nil)
return
}
guard NSURL(string:kGiphyBaseURL) != nil else {
guard NSURL(string: kGiphyBaseURL) != nil else {
Logger.error("\(TAG) Invalid base URL.")
failure(nil)
return
@ -338,10 +338,11 @@ extension GiphyError: LocalizedError {
sessionManager.get(urlString,
parameters: {},
progress:nil,
success: { _, value in
progress: nil,
success: { task, value in
Logger.error("\(GiphyAPI.TAG) search request succeeded")
guard let imageInfos = self.parseGiphyImages(responseJson:value) else {
Logger.error("\(GiphyAPI.TAG) search request succeeded \(task.originalRequest?.url)")
guard let imageInfos = self.parseGiphyImages(responseJson: value) else {
failure(nil)
return
}
@ -355,16 +356,16 @@ extension GiphyError: LocalizedError {
// MARK: Parse API Responses
private func parseGiphyImages(responseJson:Any?) -> [GiphyImageInfo]? {
private func parseGiphyImages(responseJson: Any?) -> [GiphyImageInfo]? {
guard let responseJson = responseJson else {
Logger.error("\(TAG) Missing response.")
return nil
}
guard let responseDict = responseJson as? [String:Any] else {
guard let responseDict = responseJson as? [String: Any] else {
Logger.error("\(TAG) Invalid response.")
return nil
}
guard let imageDicts = responseDict["data"] as? [[String:Any]] else {
guard let imageDicts = responseDict["data"] as? [[String: Any]] else {
Logger.error("\(TAG) Invalid response data.")
return nil
}
@ -374,7 +375,7 @@ extension GiphyError: LocalizedError {
}
// Giphy API results are often incomplete or malformed, so we need to be defensive.
private func parseGiphyImage(imageDict: [String:Any]) -> GiphyImageInfo? {
private func parseGiphyImage(imageDict: [String: Any]) -> GiphyImageInfo? {
guard let giphyId = imageDict["id"] as? String else {
Logger.warn("\(TAG) Image dict missing id.")
return nil
@ -383,18 +384,18 @@ extension GiphyError: LocalizedError {
Logger.warn("\(TAG) Image dict has invalid id.")
return nil
}
guard let renditionDicts = imageDict["images"] as? [String:Any] else {
guard let renditionDicts = imageDict["images"] as? [String: Any] else {
Logger.warn("\(TAG) Image dict missing renditions.")
return nil
}
var renditions = [GiphyRendition]()
for (renditionName, renditionDict) in renditionDicts {
guard let renditionDict = renditionDict as? [String:Any] else {
guard let renditionDict = renditionDict as? [String: Any] else {
Logger.warn("\(TAG) Invalid rendition dict.")
continue
}
guard let rendition = parseGiphyRendition(renditionName:renditionName,
renditionDict:renditionDict) else {
guard let rendition = parseGiphyRendition(renditionName: renditionName,
renditionDict: renditionDict) else {
continue
}
renditions.append(rendition)
@ -404,13 +405,13 @@ extension GiphyError: LocalizedError {
return nil
}
guard let originalRendition = findOriginalRendition(renditions:renditions) else {
guard let originalRendition = findOriginalRendition(renditions: renditions) else {
Logger.warn("\(TAG) Image has no original rendition.")
return nil
}
return GiphyImageInfo(giphyId : giphyId,
renditions : renditions,
return GiphyImageInfo(giphyId: giphyId,
renditions: renditions,
originalRendition: originalRendition)
}
@ -425,15 +426,15 @@ extension GiphyError: LocalizedError {
//
// We should discard renditions which are missing or have invalid properties.
private func parseGiphyRendition(renditionName: String,
renditionDict: [String:Any]) -> GiphyRendition? {
guard let width = parsePositiveUInt(dict:renditionDict, key:"width", typeName:"rendition") else {
renditionDict: [String: Any]) -> GiphyRendition? {
guard let width = parsePositiveUInt(dict: renditionDict, key: "width", typeName: "rendition") else {
return nil
}
guard let height = parsePositiveUInt(dict:renditionDict, key:"height", typeName:"rendition") else {
guard let height = parsePositiveUInt(dict: renditionDict, key: "height", typeName: "rendition") else {
return nil
}
// Be lenient when parsing file sizes - we don't require them for stills.
let fileSize = parseLenientUInt(dict:renditionDict, key:"size")
let fileSize = parseLenientUInt(dict: renditionDict, key: "size")
guard let urlString = renditionDict["url"] as? String else {
return nil
}
@ -441,7 +442,7 @@ extension GiphyError: LocalizedError {
Logger.warn("\(TAG) Rendition has invalid url.")
return nil
}
guard let url = NSURL(string:urlString) else {
guard let url = NSURL(string: urlString) else {
Logger.warn("\(TAG) Rendition url could not be parsed.")
return nil
}
@ -464,16 +465,16 @@ extension GiphyError: LocalizedError {
}
return GiphyRendition(
format : format,
name : renditionName,
width : width,
height : height,
fileSize : fileSize,
url : url
format: format,
name: renditionName,
width: width,
height: height,
fileSize: fileSize,
url: url
)
}
private func parsePositiveUInt(dict: [String:Any], key: String, typeName: String) -> UInt? {
private func parsePositiveUInt(dict: [String: Any], key: String, typeName: String) -> UInt? {
guard let value = dict[key] else {
return nil
}
@ -490,7 +491,7 @@ extension GiphyError: LocalizedError {
return parsedValue
}
private func parseLenientUInt(dict: [String:Any], key: String) -> UInt {
private func parseLenientUInt(dict: [String: Any], key: String) -> UInt {
let defaultValue = UInt(0)
guard let value = dict[key] else {

382
Signal/src/util/OWSBackupExportJob.m
View File

@ -5,6 +5,8 @@
#import "OWSBackupExportJob.h"
#import "OWSDatabaseMigration.h"
#import "Signal-Swift.h"
#import "zlib.h"
#import <SSZipArchive/SSZipArchive.h>
#import <SignalServiceKit/NSData+Base64.h>
#import <SignalServiceKit/NSDate+OWS.h>
#import <SignalServiceKit/OWSBackgroundTask.h>
@ -18,11 +20,167 @@
#import <SignalServiceKit/Threading.h>
#import <SignalServiceKit/YapDatabaseConnection+OWS.h>
#import <YapDatabase/YapDatabase.h>
#import <YapDatabase/YapDatabasePrivate.h>
NS_ASSUME_NONNULL_BEGIN
NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKeySpec";
@interface YapDatabase (OWSBackupExportJob)
- (void)flushInternalQueue;
- (void)flushCheckpointQueue;
@end
#pragma mark -
@interface OWSStorageReference : NSObject
@property (nonatomic, nullable) OWSStorage *storage;
@end
#pragma mark -
@implementation OWSStorageReference
@end
#pragma mark -
// TODO: This implementation is a proof-of-concept and
// isn't production ready.
@interface OWSExportStream : NSObject
@property (nonatomic) NSString *dataFilePath;
@property (nonatomic) NSString *zipFilePath;
@property (nonatomic, nullable) NSFileHandle *fileHandle;
@property (nonatomic, nullable) SSZipArchive *zipFile;
@end
#pragma mark -
@implementation OWSExportStream
- (void)dealloc
{
// Surface memory leaks by logging the deallocation of view controllers.
DDLogVerbose(@"Dealloc: %@", self.class);
[self.fileHandle closeFile];
if (self.zipFile) {
if (![self.zipFile close]) {
DDLogError(@"%@ couldn't close to database snapshot zip.", self.logTag);
}
}
}
+ (OWSExportStream *)exportStreamWithName:(NSString *)filename jobTempDirPath:(NSString *)jobTempDirPath
{
OWSAssert(filename.length > 0);
OWSAssert(jobTempDirPath.length > 0);
OWSExportStream *exportStream = [OWSExportStream new];
exportStream.dataFilePath = [jobTempDirPath stringByAppendingPathComponent:filename];
exportStream.zipFilePath = [exportStream.dataFilePath stringByAppendingPathExtension:@"zip"];
if (![exportStream open]) {
return nil;
}
return exportStream;
}
- (BOOL)open
{
if (![[NSFileManager defaultManager] createFileAtPath:self.dataFilePath contents:nil attributes:nil]) {
OWSProdLogAndFail(@"%@ Could not create database snapshot stream.", self.logTag);
return NO;
}
if (![OWSFileSystem protectFileOrFolderAtPath:self.dataFilePath]) {
OWSProdLogAndFail(@"%@ Could not protect database snapshot stream.", self.logTag);
return NO;
}
NSError *error;
self.fileHandle = [NSFileHandle fileHandleForWritingToURL:[NSURL fileURLWithPath:self.dataFilePath] error:&error];
if (!self.fileHandle || error) {
OWSProdLogAndFail(@"%@ Could not open database snapshot stream: %@.", self.logTag, error);
return NO;
}
return YES;
}
- (BOOL)writeObject:(TSYapDatabaseObject *)object
{
OWSAssert(object);
OWSAssert(self.fileHandle);
NSData *_Nullable data = [NSKeyedArchiver archivedDataWithRootObject:object];
if (!data) {
OWSProdLogAndFail(@"%@ couldn't serialize database object: %@", self.logTag, [object class]);
return NO;
}
// We use a fixed width data type.
unsigned int dataLength = (unsigned int)data.length;
NSData *dataLengthData = [NSData dataWithBytes:&dataLength length:sizeof(dataLength)];
[self.fileHandle writeData:dataLengthData];
[self.fileHandle writeData:data];
return YES;
}
- (BOOL)closeAndZipData
{
[self.fileHandle closeFile];
self.fileHandle = nil;
self.zipFile = [[SSZipArchive alloc] initWithPath:self.zipFilePath];
if (!self.zipFile) {
OWSProdLogAndFail(@"%@ Could not create database snapshot zip.", self.logTag);
return NO;
}
if (![self.zipFile open]) {
OWSProdLogAndFail(@"%@ Could not open database snapshot zip.", self.logTag);
return NO;
}
BOOL success = [self.zipFile writeFileAtPath:self.dataFilePath
withFileName:@"payload"
compressionLevel:Z_BEST_COMPRESSION
password:nil
AES:NO];
if (!success) {
OWSProdLogAndFail(@"%@ Could not write to database snapshot zip.", self.logTag);
return NO;
}
if (![self.zipFile close]) {
DDLogError(@"%@ couldn't close database snapshot zip.", self.logTag);
return NO;
}
self.zipFile = nil;
if (![OWSFileSystem protectFileOrFolderAtPath:self.zipFilePath]) {
DDLogError(@"%@ could not protect database snapshot zip.", self.logTag);
}
DDLogInfo(@"%@ wrote database snapshot zip: %@ (%@ -> %@)",
self.logTag,
self.zipFilePath.lastPathComponent,
[OWSFileSystem fileSizeOfPath:self.dataFilePath],
[OWSFileSystem fileSizeOfPath:self.zipFilePath]);
return YES;
}
@end
#pragma mark -
@interface OWSAttachmentExport : NSObject
@property (nonatomic, weak) id<OWSBackupJobDelegate> delegate;
@ -88,8 +246,6 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe
@property (nonatomic, nullable) OWSBackgroundTask *backgroundTask;
@property (nonatomic, nullable) OWSBackupStorage *backupStorage;
@property (nonatomic) NSMutableArray<NSString *> *databaseFilePaths;
// A map of "record name"-to-"file name".
@property (nonatomic) NSMutableDictionary<NSString *, NSString *> *databaseRecordMap;
@ -238,9 +394,10 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe
return databaseKeySpec;
};
self.backupStorage =
OWSStorageReference *storageReference = [OWSStorageReference new];
storageReference.storage =
[[OWSBackupStorage alloc] initBackupStorageWithDatabaseDirPath:jobDatabaseDirPath keySpecBlock:keySpecBlock];
if (!self.backupStorage) {
if (!storageReference.storage) {
OWSProdLogAndFail(@"%@ Could not create backupStorage.", self.logTag);
return completion(NO);
}
@ -248,30 +405,144 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe
// TODO: Do we really need to run these registrations on the main thread?
__weak OWSBackupExportJob *weakSelf = self;
dispatch_async(dispatch_get_main_queue(), ^{
[weakSelf.backupStorage runSyncRegistrations];
[weakSelf.backupStorage runAsyncRegistrationsWithCompletion:^{
[storageReference.storage runSyncRegistrations];
[storageReference.storage runAsyncRegistrationsWithCompletion:^{
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
completion([weakSelf exportDatabaseContents]);
[weakSelf exportDatabaseContentsAndCleanup:storageReference completion:completion];
});
}];
});
}
- (BOOL)exportDatabaseContents
- (void)exportDatabaseContentsAndCleanup:(OWSStorageReference *)storageReference
completion:(OWSBackupJobBoolCompletion)completion
{
OWSAssert(storageReference);
OWSAssert(completion);
DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
YapDatabaseConnection *_Nullable tempDBConnection = self.backupStorage.newDatabaseConnection;
__weak YapDatabase *_Nullable weakDatabase = nil;
dispatch_queue_t snapshotQueue;
dispatch_queue_t writeQueue;
NSArray<NSString *> *_Nullable allDatabaseFilePaths = nil;
@autoreleasepool {
allDatabaseFilePaths = [self exportDatabaseContents:storageReference];
if (!allDatabaseFilePaths) {
completion(NO);
}
// After the data has been written to the database snapshot,
// we need to synchronously block until the database has been
// completely closed. This is non-trivial because the database
// does a bunch of async work as its closing.
YapDatabase *database = storageReference.storage.database;
weakDatabase = database;
snapshotQueue = database->snapshotQueue;
writeQueue = database->writeQueue;
[self updateProgressWithDescription:NSLocalizedString(@"BACKUP_EXPORT_PHASE_DATABASE_FINALIZED",
@"Indicates that the backup export data is being finalized.")
progress:nil];
// Flush these two queues immediately.
[database flushInternalQueue];
[database flushCheckpointQueue];
// Close the database.
storageReference.storage = nil;
}
// Flush these queues, which may contain lingering
// references to the database.
dispatch_sync(snapshotQueue,
^{
});
dispatch_sync(writeQueue,
^{
});
// YapDatabase retains the registration connection for N seconds.
// The conneciton retains a strong reference to the database.
// We therefore need to wait a bit longer to ensure that this
// doesn't block deallocation.
NSTimeInterval kRegistrationConnectionDelaySeconds = 5.0 * 1.2;
dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(kRegistrationConnectionDelaySeconds * NSEC_PER_SEC)),
dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0),
^{
// Dispatch to main thread to wait for any lingering notifications fired by
// database (e.g. cross process notifier).
dispatch_async(dispatch_get_main_queue(), ^{
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
// Verify that the database is indeed closed.
YapDatabase *_Nullable strongDatabase = weakDatabase;
OWSAssert(!strongDatabase);
// Capture the list of database files to save.
NSMutableArray<NSString *> *databaseFilePaths = [NSMutableArray new];
for (NSString *filePath in allDatabaseFilePaths) {
if ([[NSFileManager defaultManager] fileExistsAtPath:filePath]) {
[databaseFilePaths addObject:filePath];
}
}
if (databaseFilePaths.count < 1) {
OWSProdLogAndFail(@"%@ Can't find database file.", self.logTag);
return completion(NO);
}
self.databaseFilePaths = [databaseFilePaths mutableCopy];
completion(YES);
});
});
});
}
- (nullable NSArray<NSString *> *)exportDatabaseContents:(OWSStorageReference *)storageReference
{
OWSAssert(storageReference);
DDLogVerbose(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
[self updateProgressWithDescription:NSLocalizedString(@"BACKUP_EXPORT_PHASE_DATABASE_EXPORT",
@"Indicates that the database data is being exported.")
progress:nil];
YapDatabaseConnection *_Nullable tempDBConnection = storageReference.storage.newDatabaseConnection;
if (!tempDBConnection) {
OWSProdLogAndFail(@"%@ Could not create tempDBConnection.", self.logTag);
return NO;
return nil;
}
YapDatabaseConnection *_Nullable primaryDBConnection = self.primaryStorage.newDatabaseConnection;
if (!primaryDBConnection) {
OWSProdLogAndFail(@"%@ Could not create primaryDBConnection.", self.logTag);
return NO;
return nil;
}
NSString *const kDatabaseSnapshotFilename_Threads = @"threads";
NSString *const kDatabaseSnapshotFilename_Interactions = @"interactions";
NSString *const kDatabaseSnapshotFilename_Attachments = @"attachments";
NSString *const kDatabaseSnapshotFilename_Migrations = @"migrations";
OWSExportStream *_Nullable exportStream_Threads =
[OWSExportStream exportStreamWithName:kDatabaseSnapshotFilename_Threads jobTempDirPath:self.jobTempDirPath];
OWSExportStream *_Nullable exportStream_Interactions =
[OWSExportStream exportStreamWithName:kDatabaseSnapshotFilename_Interactions
jobTempDirPath:self.jobTempDirPath];
OWSExportStream *_Nullable exportStream_Attachments =
[OWSExportStream exportStreamWithName:kDatabaseSnapshotFilename_Attachments jobTempDirPath:self.jobTempDirPath];
OWSExportStream *_Nullable exportStream_Migrations =
[OWSExportStream exportStreamWithName:kDatabaseSnapshotFilename_Migrations jobTempDirPath:self.jobTempDirPath];
if (!(exportStream_Threads && exportStream_Interactions && exportStream_Attachments && exportStream_Migrations)) {
return nil;
}
NSArray<OWSExportStream *> *exportStreams = @[
exportStream_Threads,
exportStream_Interactions,
exportStream_Attachments,
exportStream_Migrations,
];
__block unsigned long long copiedThreads = 0;
__block unsigned long long copiedInteractions = 0;
__block unsigned long long copiedEntities = 0;
@ -280,6 +551,7 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe
self.attachmentFilePathMap = [NSMutableDictionary new];
__block BOOL aborted = NO;
[primaryDBConnection readWithBlock:^(YapDatabaseReadTransaction *srcTransaction) {
[tempDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *dstTransaction) {
[dstTransaction setObject:@(YES)
@ -303,6 +575,12 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe
[thread saveWithTransaction:dstTransaction];
copiedThreads++;
copiedEntities++;
if (![exportStream_Threads writeObject:thread]) {
*stop = YES;
aborted = YES;
return;
}
}];
// Copy attachments.
@ -330,6 +608,12 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe
[attachment saveWithTransaction:dstTransaction];
copiedAttachments++;
copiedEntities++;
if (![exportStream_Attachments writeObject:attachment]) {
*stop = YES;
aborted = YES;
return;
}
}];
// Copy interactions.
@ -362,6 +646,12 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe
[interaction saveWithTransaction:dstTransaction];
copiedInteractions++;
copiedEntities++;
if (![exportStream_Interactions writeObject:interaction]) {
*stop = YES;
aborted = YES;
return;
}
}];
// Copy migrations.
@ -381,12 +671,36 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe
[migration saveWithTransaction:dstTransaction];
copiedMigrations++;
copiedEntities++;
if (![exportStream_Migrations writeObject:migration]) {
*stop = YES;
aborted = YES;
return;
}
}];
}];
}];
unsigned long long totalZipFileSize = 0;
for (OWSExportStream *exportStream in exportStreams) {
if (![exportStream closeAndZipData]) {
DDLogError(@"%@ couldn't close database snapshot zip.", self.logTag);
return nil;
}
NSNumber *_Nullable fileSize = [OWSFileSystem fileSizeOfPath:exportStream.zipFilePath];
if (!fileSize) {
DDLogError(@"%@ couldn't get file size of database snapshot zip.", self.logTag);
return nil;
}
totalZipFileSize += fileSize.unsignedLongLongValue;
}
if (aborted) {
return nil;
}
if (self.isComplete) {
return NO;
return nil;
}
// TODO: Should we do a database checkpoint?
@ -396,19 +710,44 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe
DDLogInfo(@"%@ copiedAttachments: %llu", self.logTag, copiedAttachments);
DDLogInfo(@"%@ copiedMigrations: %llu", self.logTag, copiedMigrations);
[self.backupStorage logFileSizes];
[storageReference.storage logFileSizes];
unsigned long long totalDbFileSize
= ([OWSFileSystem fileSizeOfPath:storageReference.storage.databaseFilePath].unsignedLongLongValue +
[OWSFileSystem fileSizeOfPath:storageReference.storage.databaseFilePath_WAL].unsignedLongLongValue +
[OWSFileSystem fileSizeOfPath:storageReference.storage.databaseFilePath_SHM].unsignedLongLongValue);
if (totalZipFileSize > 0 && totalDbFileSize > 0) {
DDLogInfo(@"%@ file size savings: %llu / %llu = %0.2f",
self.logTag,
totalZipFileSize,
totalDbFileSize,
totalZipFileSize / (CGFloat)totalDbFileSize);
}
// Capture the list of files to save.
self.databaseFilePaths = [@[
self.backupStorage.databaseFilePath,
self.backupStorage.databaseFilePath_WAL,
self.backupStorage.databaseFilePath_SHM,
] mutableCopy];
return @[
storageReference.storage.databaseFilePath,
storageReference.storage.databaseFilePath_WAL,
storageReference.storage.databaseFilePath_SHM,
];
}
// Close the database.
tempDBConnection = nil;
self.backupStorage = nil;
- (BOOL)writeObject:(TSYapDatabaseObject *)object fileHandle:(NSFileHandle *)fileHandle
{
OWSAssert(object);
OWSAssert(fileHandle);
NSData *_Nullable data = [NSKeyedArchiver archivedDataWithRootObject:object];
if (!data) {
OWSProdLogAndFail(@"%@ couldn't serialize database object: %@", self.logTag, [object class]);
return NO;
}
// We use a fixed width data type.
unsigned int dataLength = (unsigned int)data.length;
NSData *dataLengthData = [NSData dataWithBytes:&dataLength length:sizeof(dataLength)];
[fileHandle writeData:dataLengthData];
[fileHandle writeData:data];
return YES;
}
@ -477,6 +816,7 @@ NSString *const kOWSBackup_ExportDatabaseKeySpec = @"kOWSBackup_ExportDatabaseKe
}
failure:^(NSError *error) {
// Database files are critical so any error uploading them is unrecoverable.
DDLogVerbose(@"%@ error while saving file: %@", self.logTag, filePath);
completion(error);
}];
return YES;

8
Signal/translations/en.lproj/Localizable.strings
View File

@ -161,7 +161,7 @@
"BACKUP_EXPORT_ERROR_COULD_NOT_EXPORT" = "Backup data could be exported.";
/* Error indicating that the app received an invalid response from CloudKit. */
"BACKUP_EXPORT_ERROR_INVALID_CLOUDKIT_RESPONSE" = "Invalid server response.";
"BACKUP_EXPORT_ERROR_INVALID_CLOUDKIT_RESPONSE" = "Invalid Service Response";
/* Error indicating the a backup export failed to save a file to the cloud. */
"BACKUP_EXPORT_ERROR_SAVE_FILE_TO_CLOUD_FAILED" = "Backup could not upload data.";
@ -172,6 +172,12 @@
/* Indicates that the backup export is being configured. */
"BACKUP_EXPORT_PHASE_CONFIGURATION" = "Initializing Backup";
/* Indicates that the database data is being exported. */
"BACKUP_EXPORT_PHASE_DATABASE_EXPORT" = "Exporting Data";
/* Indicates that the backup export data is being finalized. */
"BACKUP_EXPORT_PHASE_DATABASE_FINALIZED" = "Finalizing Data";
/* Indicates that the backup export data is being exported. */
"BACKUP_EXPORT_PHASE_EXPORT" = "Exporting Backup";

10
SignalServiceKit/src/Storage/OWSPrimaryStorage.m
View File

@ -326,6 +326,16 @@ void runAsyncRegistrationsForStorage(OWSStorage *storage)
return OWSPrimaryStorage.databaseFilePath;
}
- (NSString *)databaseFilePath_SHM
{
return OWSPrimaryStorage.databaseFilePath_SHM;
}
- (NSString *)databaseFilePath_WAL
{
return OWSPrimaryStorage.databaseFilePath_WAL;
}
- (NSString *)databaseFilename_SHM
{
return OWSPrimaryStorage.databaseFilename_SHM;

4
SignalServiceKit/src/Storage/OWSStorage+Subclass.h
View File

@ -6,8 +6,12 @@
NS_ASSUME_NONNULL_BEGIN
@class YapDatabase;
@interface OWSStorage (Subclass)
@property (atomic, nullable, readonly) YapDatabase *database;
- (void)loadDatabase;
- (void)runSyncRegistrations;

2
SignalServiceKit/src/Storage/OWSStorage.h
View File

@ -61,6 +61,8 @@ extern NSString *const StorageIsReadyNotification;
- (nullable id)registeredExtension:(NSString *)extensionName;
- (unsigned long long)databaseFileSize;
- (unsigned long long)databaseWALFileSize;
- (unsigned long long)databaseSHMFileSize;
- (YapDatabaseConnection *)registrationConnection;

61
SignalServiceKit/src/Storage/OWSStorage.m
View File

@ -46,6 +46,14 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void);
#pragma mark -
//@interface OWSDatabaseConnection : YapDatabaseConnection
//
//@property (atomic, weak) id<OWSDatabaseConnectionDelegate> delegate;
//
//@end
#pragma mark -
@implementation OWSDatabaseConnection
- (id)initWithDatabase:(YapDatabase *)database delegate:(id<OWSDatabaseConnectionDelegate>)delegate
@ -58,7 +66,7 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void);
OWSAssert(delegate);
_delegate = delegate;
self.delegate = delegate;
return self;
}
@ -134,9 +142,8 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void);
@property (atomic, weak) id<OWSDatabaseConnectionDelegate> delegate;
@property (atomic, nullable) YapDatabaseConnection *registrationConnectionCached;
- (instancetype)init NS_UNAVAILABLE;
- (id)initWithPath:(NSString *)inPath
serializer:(nullable YapDatabaseSerializer)inSerializer
deserializer:(YapDatabaseDeserializer)inDeserializer
@ -163,7 +170,7 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void);
OWSAssert(delegate);
_delegate = delegate;
self.delegate = delegate;
return self;
}
@ -184,21 +191,15 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void);
- (YapDatabaseConnection *)registrationConnection
{
@synchronized(self)
{
if (!self.registrationConnectionCached) {
YapDatabaseConnection *connection = [super registrationConnection];
YapDatabaseConnection *connection = [super registrationConnection];
#ifdef DEBUG
// Flag the registration connection as such.
OWSAssert([connection isKindOfClass:[OWSDatabaseConnection class]]);
((OWSDatabaseConnection *)connection).canWriteBeforeStorageReady = YES;
// Flag the registration connection as such.
OWSAssert([connection isKindOfClass:[OWSDatabaseConnection class]]);
((OWSDatabaseConnection *)connection).canWriteBeforeStorageReady = YES;
#endif
self.registrationConnectionCached = connection;
}
return self.registrationConnectionCached;
}
return connection;
}
@end
@ -278,6 +279,9 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void);
- (void)dealloc
{
// Surface memory leaks by logging the deallocation of this class.
DDLogVerbose(@"Dealloc: %@", self.class);
[[NSNotificationCenter defaultCenter] removeObserver:self];
}
@ -363,11 +367,6 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void);
for (OWSStorage *storage in self.allStorages) {
[storage runAsyncRegistrationsWithCompletion:^{
if ([self postRegistrationCompleteNotificationIfPossible]) {
// If all registrations are complete, clean up the
// registration process.
((OWSDatabase *)storage.database).registrationConnectionCached = nil;
backgroundTask = nil;
}
}];
@ -408,15 +407,23 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void);
- (BOOL)tryToLoadDatabase
{
__weak OWSStorage *weakSelf = self;
YapDatabaseOptions *options = [[YapDatabaseOptions alloc] init];
options.corruptAction = YapDatabaseCorruptAction_Fail;
options.enableMultiProcessSupport = YES;
options.cipherKeySpecBlock = ^{
// NOTE: It's critical that we don't capture a reference to self
// (e.g. by using OWSAssert()) or this database will contain a
// circular reference and will leak.
OWSStorage *strongSelf = weakSelf;
OWSCAssert(strongSelf);
// Rather than compute this once and capture the value of the key
// in the closure, we prefer to fetch the key from the keychain multiple times
// in order to keep the key out of application memory.
NSData *databaseKeySpec = [self databaseKeySpec];
OWSAssert(databaseKeySpec.length == kSQLCipherKeySpecLength);
NSData *databaseKeySpec = [strongSelf databaseKeySpec];
OWSCAssert(databaseKeySpec.length == kSQLCipherKeySpecLength);
return databaseKeySpec;
};
@ -709,6 +716,16 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void);
return [OWSFileSystem fileSizeOfPath:self.databaseFilePath].unsignedLongLongValue;
}
- (unsigned long long)databaseWALFileSize
{
return [OWSFileSystem fileSizeOfPath:self.databaseFilePath_WAL].unsignedLongLongValue;
}
- (unsigned long long)databaseSHMFileSize
{
return [OWSFileSystem fileSizeOfPath:self.databaseFilePath_SHM].unsignedLongLongValue;
}
+ (nullable NSData *)tryToLoadKeyChainValue:(NSString *)keychainKey errorHandle:(NSError **)errorHandle
{
OWSAssert(keychainKey.length > 0);