From 5ba5b763e44b5265353f234a91e2996a7702afce Mon Sep 17 00:00:00 2001 From: Matthew Chen Date: Thu, 18 Jan 2018 17:32:37 -0500 Subject: [PATCH] Add tests around database conversion. --- Signal.xcodeproj/project.pbxproj | 12 +- Signal/test/util/OWSDatabaseConverterTest.h | 13 ++ Signal/test/util/OWSDatabaseConverterTest.m | 80 +++++++++ SignalMessaging/utils/OWSDatabaseConverter.h | 1 + SignalMessaging/utils/OWSDatabaseConverter.m | 167 ++++++++++++++++++- 5 files changed, 267 insertions(+), 6 deletions(-) create mode 100644 Signal/test/util/OWSDatabaseConverterTest.h create mode 100644 Signal/test/util/OWSDatabaseConverterTest.m diff --git a/Signal.xcodeproj/project.pbxproj b/Signal.xcodeproj/project.pbxproj index 6e29a065d..e551cd010 100644 --- a/Signal.xcodeproj/project.pbxproj +++ b/Signal.xcodeproj/project.pbxproj @@ -186,6 +186,7 @@ 34D8C0281ED3673300188D7C /* DebugUITableViewController.m in Sources */ = {isa = PBXBuildFile; fileRef = 34D8C0261ED3673300188D7C /* DebugUITableViewController.m */; }; 34D8C02B1ED3685800188D7C /* DebugUIContacts.m in Sources */ = {isa = PBXBuildFile; fileRef = 34D8C02A1ED3685800188D7C /* DebugUIContacts.m */; }; 34D99C931F2937CC00D284D6 /* OWSAnalytics.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34D99C911F2937CC00D284D6 /* OWSAnalytics.swift */; }; + 34DB0BED2011548B007B313F /* OWSDatabaseConverterTest.m in Sources */ = {isa = PBXBuildFile; fileRef = 34DB0BEC2011548B007B313F /* OWSDatabaseConverterTest.m */; }; 34DFCB851E8E04B500053165 /* AddToBlockListViewController.m in Sources */ = {isa = PBXBuildFile; fileRef = 34DFCB841E8E04B500053165 /* AddToBlockListViewController.m */; }; 34E3E5681EC4B19400495BAC /* AudioProgressView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34E3E5671EC4B19400495BAC /* AudioProgressView.swift */; }; 34E3EF0D1EFC235B007F6822 /* DebugUIDiskUsage.m in Sources */ = {isa = PBXBuildFile; fileRef = 34E3EF0C1EFC235B007F6822 /* DebugUIDiskUsage.m */; }; @@ -740,6 +741,8 @@ 34D99C8A1F27B13B00D284D6 /* OWSViewController.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OWSViewController.h; sourceTree = ""; }; 34D99C8B1F27B13B00D284D6 /* OWSViewController.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = OWSViewController.m; sourceTree = ""; }; 34D99C911F2937CC00D284D6 /* OWSAnalytics.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OWSAnalytics.swift; sourceTree = ""; }; + 34DB0BEB2011548A007B313F /* OWSDatabaseConverterTest.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OWSDatabaseConverterTest.h; sourceTree = ""; }; + 34DB0BEC2011548B007B313F /* OWSDatabaseConverterTest.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = OWSDatabaseConverterTest.m; sourceTree = ""; }; 34DFCB831E8E04B400053165 /* AddToBlockListViewController.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AddToBlockListViewController.h; sourceTree = ""; }; 34DFCB841E8E04B500053165 /* AddToBlockListViewController.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = AddToBlockListViewController.m; sourceTree = ""; }; 34E3E5671EC4B19400495BAC /* AudioProgressView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AudioProgressView.swift; sourceTree = ""; }; @@ -1895,11 +1898,13 @@ B660F6AB1C29868000687D6E /* ExceptionsTest.m */, B660F6AC1C29868000687D6E /* FunctionalUtilTest.h */, B660F6AD1C29868000687D6E /* FunctionalUtilTest.m */, + 455AC69D1F4F8B0300134004 /* ImageCacheTest.swift */, + 34DB0BEB2011548A007B313F /* OWSDatabaseConverterTest.h */, + 34DB0BEC2011548B007B313F /* OWSDatabaseConverterTest.m */, + 45666F571D9B2880008FE134 /* OWSScrubbingLogFormatterTest.m */, + 45360B8F1F9527DA00FA666C /* SearcherTest.swift */, B660F6B31C29868000687D6E /* UtilTest.h */, B660F6B41C29868000687D6E /* UtilTest.m */, - 45666F571D9B2880008FE134 /* OWSScrubbingLogFormatterTest.m */, - 455AC69D1F4F8B0300134004 /* ImageCacheTest.swift */, - 45360B8F1F9527DA00FA666C /* SearcherTest.swift */, ); path = util; sourceTree = ""; @@ -2998,6 +3003,7 @@ 452D1EE81DCA90D100A57EC4 /* MesssagesBubblesSizeCalculatorTest.swift in Sources */, 45360B901F9527DA00FA666C /* SearcherTest.swift in Sources */, B660F7561C29988E00687D6E /* PushManager.m in Sources */, + 34DB0BED2011548B007B313F /* OWSDatabaseConverterTest.m in Sources */, 45360B911F952AA900FA666C /* MarqueeLabel.swift in Sources */, 454EBAB41F2BE14C00ACE0BB /* OWSAnalytics.swift in Sources */, B660F7721C29988E00687D6E /* AppStoreRating.m in Sources */, diff --git a/Signal/test/util/OWSDatabaseConverterTest.h b/Signal/test/util/OWSDatabaseConverterTest.h new file mode 100644 index 000000000..f83de9fd0 --- /dev/null +++ b/Signal/test/util/OWSDatabaseConverterTest.h @@ -0,0 +1,13 @@ +// +// Copyright (c) 2018 Open Whisper Systems. All rights reserved. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface OWSDatabaseConverterTest : XCTestCase + +@end + +NS_ASSUME_NONNULL_END diff --git a/Signal/test/util/OWSDatabaseConverterTest.m b/Signal/test/util/OWSDatabaseConverterTest.m new file mode 100644 index 000000000..ff51059f5 --- /dev/null +++ b/Signal/test/util/OWSDatabaseConverterTest.m @@ -0,0 +1,80 @@ +// +// Copyright (c) 2018 Open Whisper Systems. All rights reserved. +// + +#import "OWSDatabaseConverterTest.h" +#import "OWSDatabaseConverter.h" +#import +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface OWSStorage (OWSDatabaseConverterTest) + ++ (YapDatabaseDeserializer)logOnFailureDeserializer; + +@end + +#pragma mark - + +@interface OWSDatabaseConverter (OWSDatabaseConverterTest) + ++ (BOOL)doesDatabaseNeedToBeConverted:(NSString *)databaseFilePath; + +@end + +#pragma mark - + +@implementation OWSDatabaseConverterTest + +- (NSData *)randomDatabasePassword +{ + return [Randomness generateRandomBytes:30]; +} + +- (nullable NSString *)createUnconvertedDatabase:(NSData *)passwordData +{ + NSString *temporaryDirectory = NSTemporaryDirectory(); + NSString *filename = [NSUUID UUID].UUIDString; + NSString *databaseFilePath = [temporaryDirectory stringByAppendingPathComponent:filename]; + + YapDatabaseOptions *options = [[YapDatabaseOptions alloc] init]; + options.corruptAction = YapDatabaseCorruptAction_Fail; + options.cipherKeyBlock = ^{ + return passwordData; + }; + options.enableMultiProcessSupport = YES; + + YapDatabase *database = [[YapDatabase alloc] initWithPath:databaseFilePath + serializer:nil + deserializer:[OWSStorage logOnFailureDeserializer] + options:options]; + OWSAssert(database); + return database ? databaseFilePath : nil; +} + +- (void)testDoesDatabaseNeedToBeConverted_Unconverted +{ + NSData *passwordData = [self randomDatabasePassword]; + NSString *_Nullable databaseFilePath = [self createUnconvertedDatabase:passwordData]; + XCTAssertTrue([OWSDatabaseConverter doesDatabaseNeedToBeConverted:databaseFilePath]); +} + +- (void)testDoesDatabaseNeedToBeConverted_Converted +{ + // TODO: When we can create converted databases. +} + +- (void)testDatabaseConversion +{ + NSData *passwordData = [self randomDatabasePassword]; + NSString *_Nullable databaseFilePath = [self createUnconvertedDatabase:passwordData]; + XCTAssertTrue([OWSDatabaseConverter doesDatabaseNeedToBeConverted:databaseFilePath]); + [OWSDatabaseConverter convertDatabaseIfNecessary]; + XCTAssertFalse([OWSDatabaseConverter doesDatabaseNeedToBeConverted:databaseFilePath]); +} + +@end + +NS_ASSUME_NONNULL_END diff --git a/SignalMessaging/utils/OWSDatabaseConverter.h b/SignalMessaging/utils/OWSDatabaseConverter.h index 012e955ac..cafbc740c 100644 --- a/SignalMessaging/utils/OWSDatabaseConverter.h +++ b/SignalMessaging/utils/OWSDatabaseConverter.h @@ -13,6 +13,7 @@ NS_ASSUME_NONNULL_BEGIN - (instancetype)init NS_UNAVAILABLE; + (void)convertDatabaseIfNecessary; ++ (void)convertDatabaseIfNecessary:(NSString *)databaseFilePath; @end diff --git a/SignalMessaging/utils/OWSDatabaseConverter.m b/SignalMessaging/utils/OWSDatabaseConverter.m index 491bf27da..7b31503c5 100644 --- a/SignalMessaging/utils/OWSDatabaseConverter.m +++ b/SignalMessaging/utils/OWSDatabaseConverter.m @@ -10,9 +10,8 @@ NS_ASSUME_NONNULL_BEGIN @implementation OWSDatabaseConverter -+ (BOOL)doesDatabaseNeedToBeConverted ++ (BOOL)doesDatabaseNeedToBeConverted:(NSString *)databaseFilePath { - NSString *databaseFilePath = [TSStorageManager legacyDatabaseFilePath]; if (![[NSFileManager defaultManager] fileExistsAtPath:databaseFilePath]) { DDLogVerbose(@"%@ Skipping database conversion; no legacy database found.", self.logTag); return NO; @@ -50,7 +49,13 @@ NS_ASSUME_NONNULL_BEGIN + (void)convertDatabaseIfNecessary { - if (![self doesDatabaseNeedToBeConverted]) { + NSString *databaseFilePath = [TSStorageManager legacyDatabaseFilePath]; + [self convertDatabaseIfNecessary:databaseFilePath]; +} + ++ (void)convertDatabaseIfNecessary:(NSString *)databaseFilePath +{ + if (![self doesDatabaseNeedToBeConverted:databaseFilePath]) { return; } @@ -60,6 +65,162 @@ NS_ASSUME_NONNULL_BEGIN + (void)convertDatabase { // TODO: + + // Hello Matthew, + // + // I hope you're doing well. We've just pushed some changes out to the SQLCipher prerelease branch on GitHub that + // implement the functionality we talked about that add a few new options: + // + // 1. PRAGMA cipher_plaintext_header_size - set or query the number of bytes to be left unencrypted on the start + // of the first page. This pragma would be called after keying the database, but before use. In our testing 32 + // works for iOS + // 2. PRAGMA cipher_default_plaintext_header_size - set the "global" default to be used when opening database + // connections + // 3. PRAGMA cipher_salt - set or query the salt for the database + // + // When working with the SQLCipherVsSharedData application, there are two changes required. First, modify + // the Podfile to reference SQLCipher with these changes: + // + // pod 'SQLCipher', :git => 'https://github.com/sqlcipher/sqlcipher.git', :commit => 'd5c2bec' + // + // Next, set the plaintext header size immediately after the key is provided: + // + // int status = sqlite3_exec(db, "PRAGMA cipher_plaintext_header_size = 32;", NULL, NULL, NULL); + // + // + // This should allow the demo app to background correctly. + // + // In practice, for a real application, the other changes we talked about on the phone need occur, i.e. to + // provide the salt to the application explicitly. The application can use a raw key spec, where the 96 hex are + // provide (i.e. 64 hex for the 256 bit key, followed by 32 hex for the 128 bit salt) using explicit BLOB syntax, + // e.g. + // + // x'98483C6EB40B6C31A448C22A66DED3B5E5E8D5119CAC8327B655C8B5C483648101010101010101010101010101010101' + // + // Alternately, the application can use the new cipher_salt PRAGMA to provide 32 hex to use as salt in + // conjunction with a standard derived key, e.g. + // + // PRAGMA cipher_salt = "x'01010101010101010101010101010101'"; + // + // Since you mentioned the Signal application is using a derived key, the second option might be easiest. You + // could load the first 16 bytes of the existing file, or query the database using cipher_salt, and then store + // that along side the key in the keychain. Then following migration you can provide both the key and the salt + // explicitly. + // + // With respect to migrating existing databases, it is possible to open a database, set the pragma, modify the + // first page, then checkpoint to ensure that all WAL frames are written back to the main database. This allows + // you to "decrypt" the first part of the header almost instantaneously, without having to re-encrypt all of the + // content. Keep in mind that you'll need to record the salt separately in this case. There are a few examples of + // this in the test cases we wrote up for this new functionality, starting here: + // + // https://github.com/sqlcipher/sqlcipher/blob/d5c2bec7688cef298292906c029d26b2c043219d/test/crypto.test#L2669 + // + // I was hoping you could take a look at this new functionality, provide feedback, and perform some initial + // testing on your side. Please let us know if you have any questions, or would like to discuss the specifics of + // implementation further. Thanks! + // + // Cheers, + // Stephen + + // - (BOOL)openDatabase + // { + // // Open the database connection. + // // + // // We use SQLITE_OPEN_NOMUTEX to use the multi-thread threading mode, + // // as we will be serializing access to the connection externally. + // + // int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_PRIVATECACHE; + // + // int status = sqlite3_open_v2([databasePath UTF8String], &db, flags, NULL); + // if (status != SQLITE_OK) + // { + // // There are a few reasons why the database might not open. + // // One possibility is if the database file has become corrupt. + // + // // Sometimes the open function returns a db to allow us to query it for the error message. + // // The openConfigCreate block will close it for us. + // if (db) { + // YDBLogError(@"Error opening database: %d %s", status, sqlite3_errmsg(db)); + // } + // else { + // YDBLogError(@"Error opening database: %d", status); + // } + // + // return NO; + // } + // // Add a busy handler if we are in multiprocess mode + // if (options.enableMultiProcessSupport) { + // sqlite3_busy_handler(db, connectionBusyHandler, (__bridge void *)(self)); + // } + // + // return YES; + // } + // + // + // + //#ifdef SQLITE_HAS_CODEC + // /** + // * Configures database encryption via SQLCipher. + // **/ + // - (BOOL)configureEncryptionForDatabase:(sqlite3 *)sqlite + // { + // if (options.cipherKeyBlock) + // { + // NSData *keyData = options.cipherKeyBlock(); + // + // if (keyData == nil) + // { + // NSAssert(NO, @"YapDatabaseOptions.cipherKeyBlock cannot return nil!"); + // return NO; + // } + // + // //Setting the PBKDF2 default iteration number (this will have effect next time database is opened) + // if (options.cipherDefaultkdfIterNumber > 0) { + // char *errorMsg; + // NSString *pragmaCommand = [NSString stringWithFormat:@"PRAGMA cipher_default_kdf_iter = %lu", + // (unsigned long)options.cipherDefaultkdfIterNumber]; if (sqlite3_exec(sqlite, [pragmaCommand + // UTF8String], NULL, NULL, &errorMsg) != SQLITE_OK) + // { + // YDBLogError(@"failed to set database cipher_default_kdf_iter: %s", errorMsg); + // return NO; + // } + // } + // + // //Setting the PBKDF2 iteration number + // if (options.kdfIterNumber > 0) { + // char *errorMsg; + // NSString *pragmaCommand = [NSString stringWithFormat:@"PRAGMA kdf_iter = %lu", (unsigned + // long)options.kdfIterNumber]; if (sqlite3_exec(sqlite, [pragmaCommand UTF8String], NULL, NULL, + // &errorMsg) != SQLITE_OK) + // { + // YDBLogError(@"failed to set database kdf_iter: %s", errorMsg); + // return NO; + // } + // } + // + // //Setting the encrypted database page size + // if (options.cipherPageSize > 0) { + // char *errorMsg; + // NSString *pragmaCommand = [NSString stringWithFormat:@"PRAGMA cipher_page_size = %lu", (unsigned + // long)options.cipherPageSize]; if (sqlite3_exec(sqlite, [pragmaCommand UTF8String], NULL, NULL, + // &errorMsg) != SQLITE_OK) + // { + // YDBLogError(@"failed to set database cipher_page_size: %s", errorMsg); + // return NO; + // } + // } + // + // int status = sqlite3_key(sqlite, [keyData bytes], (int)[keyData length]); + // if (status != SQLITE_OK) + // { + // YDBLogError(@"Error setting SQLCipher key: %d %s", status, sqlite3_errmsg(sqlite)); + // return NO; + // } + // } + // + // return YES; + // } + //#endif } @end