// Copyright © 2022 Rangeproof Pty Ltd. All rights reserved. // // stringlint:disable import Foundation import AVKit import GRDB import YapDatabase import SessionUtilitiesKit import SessionSnodeKit // Note: Looks like the oldest iOS device we support (min iOS 13.0) has 2Gb of RAM, processing // ~250k messages and ~1000 threads seems to take up enum _003_YDBToGRDBMigration: Migration { static let target: TargetMigrations.Identifier = .messagingKit static let identifier: String = "YDBToGRDBMigration" static let needsConfigSync: Bool = true static let minExpectedRunDuration: TimeInterval = 20 static let fetchedTables: [(TableRecord & FetchableRecord).Type] = [Identity.self] static let createdOrAlteredTables: [(TableRecord & FetchableRecord).Type] = [] static func migrate(_ db: Database) throws { guard let dbConnection: YapDatabaseConnection = SUKLegacy.newDatabaseConnection() else { // We want this setting to be on by default (even if there isn't a legacy database) db[.trimOpenGroupMessagesOlderThanSixMonths] = true SNLogNotTests("[Migration Warning] No legacy database, skipping \(target.key(with: self))") return } // MARK: - Read from Legacy Database let timestampNow: TimeInterval = Date().timeIntervalSince1970 var shouldFailMigration: Bool = false var legacyMigrations: Set = [] var contacts: Set = [] var legacyBlockedSessionIds: Set = [] var validProfileIds: Set = [] var contactThreadIds: Set = [] var legacyThreadIdToIdMap: [String: String] = [:] var legacyThreads: Set = [] var disappearingMessagesConfiguration: [String: SMKLegacy._DisappearingMessagesConfiguration] = [:] var closedGroupKeys: [String: [TimeInterval: SUKLegacy.KeyPair]] = [:] var closedGroupName: [String: String] = [:] var closedGroupFormation: [String: UInt64] = [:] var closedGroupModel: [String: SMKLegacy._GroupModel] = [:] var closedGroupZombieMemberIds: [String: Set] = [:] var openGroupServer: [String: String] = [:] var openGroupInfo: [String: SMKLegacy._OpenGroup] = [:] var openGroupUserCount: [String: Int64] = [:] var openGroupImage: [String: Data] = [:] var interactions: [String: [SMKLegacy._DBInteraction]] = [:] var attachments: [String: SMKLegacy._Attachment] = [:] var processedAttachmentIds: Set = [] var outgoingReadReceiptsTimestampsMs: [String: Set] = [:] var receivedMessageTimestamps: Set = [] var receivedCallUUIDs: [String: Set] = [:] var notifyPushServerJobs: Set = [] var messageReceiveJobs: Set = [] var messageSendJobs: Set = [] var attachmentUploadJobs: Set = [] var attachmentDownloadJobs: Set = [] var legacyPreferences: [String: Any] = [:] // Map the Legacy types for the NSKeyedUnarchivez self.mapLegacyTypesForNSKeyedUnarchiver() dbConnection.read { transaction in // MARK: --Migrations // Process the migrations (we don't want to bother running the old migrations as it would be // a waste of time, rather we include the logic from the old migrations in here and make the // same changes if the migration hasn't already run) transaction.enumerateKeys(inCollection: SMKLegacy.databaseMigrationCollection) { key, _ in guard let legacyMigration: SMKLegacy._DBMigration = SMKLegacy._DBMigration(rawValue: key) else { SNLogNotTests("[Migration Error] Found unknown migration") shouldFailMigration = true return } legacyMigrations.insert(legacyMigration) } Storage.update(progress: 0.01, for: self, in: target) // MARK: --Contacts SNLogNotTests("[Migration Info] \(target.key(with: self)) - Processing Contacts") transaction.enumerateRows(inCollection: SMKLegacy.contactCollection) { _, object, _, _ in guard let contact = object as? SMKLegacy._Contact else { return } contacts.insert(contact) /// Store a record of the all valid profiles (so we can create dummy entries if we need to for closed group members) validProfileIds.insert(contact.sessionID) } legacyBlockedSessionIds = Set(transaction.object( forKey: SMKLegacy.blockedPhoneNumbersKey, inCollection: SMKLegacy.blockListCollection ) as? [String] ?? []) Storage.update(progress: 0.02, for: self, in: target) // MARK: --Threads SNLogNotTests("[Migration Info] \(target.key(with: self)) - Processing Threads") transaction.enumerateKeysAndObjects(inCollection: SMKLegacy.threadCollection) { key, object, _ in guard let thread: SMKLegacy._Thread = object as? SMKLegacy._Thread else { return } legacyThreads.insert(thread) // Want to exclude threads which aren't visible (ie. threads which we started // but the user never ended up sending a message) if key.starts(with: SMKLegacy.contactThreadPrefix) && thread.shouldBeVisible { contactThreadIds.insert(key) } // Get the disappearing messages config disappearingMessagesConfiguration[thread.uniqueId] = transaction .object(forKey: thread.uniqueId, inCollection: SMKLegacy.disappearingMessagesCollection) .asType(SMKLegacy._DisappearingMessagesConfiguration.self) // Process group-specific info guard let groupThread: SMKLegacy._GroupThread = thread as? SMKLegacy._GroupThread else { legacyThreadIdToIdMap[thread.uniqueId] = thread.uniqueId.substring( from: SMKLegacy.contactThreadPrefix.count ) return } if groupThread.isClosedGroup { // The old threadId for closed groups was in the below format, we don't // really need the unnecessary complexity so process the key and extract // the publicKey from it // `g{base64String(Data(__textsecure_group__!{publicKey}))} let base64GroupId: String = String(thread.uniqueId.suffix(from: thread.uniqueId.index(after: thread.uniqueId.startIndex))) guard let groupIdData: Data = Data(base64Encoded: base64GroupId), let groupId: String = String(data: groupIdData, encoding: .utf8), let publicKey: String = groupId.split(separator: "!").last.map({ String($0) }) else { SNLogNotTests("[Migration Error] Unable to decode Closed Group") shouldFailMigration = true return } legacyThreadIdToIdMap[thread.uniqueId] = publicKey closedGroupName[thread.uniqueId] = groupThread.groupModel.groupName closedGroupModel[thread.uniqueId] = groupThread.groupModel closedGroupFormation[thread.uniqueId] = ((transaction.object(forKey: publicKey, inCollection: SMKLegacy.closedGroupFormationTimestampCollection) as? UInt64) ?? 0) closedGroupZombieMemberIds[thread.uniqueId] = transaction.object( forKey: publicKey, inCollection: SMKLegacy.closedGroupZombieMembersCollection ) as? Set // Note: If the user is no longer in a closed group then the group will still exist but the user // won't have the closed group public key anymore let keyCollection: String = "\(SMKLegacy.closedGroupKeyPairPrefix)\(publicKey)" transaction.enumerateKeysAndObjects(inCollection: keyCollection) { key, object, _ in guard let timestamp: TimeInterval = TimeInterval(key), let keyPair: SUKLegacy.KeyPair = object as? SUKLegacy.KeyPair else { return } closedGroupKeys[thread.uniqueId] = (closedGroupKeys[thread.uniqueId] ?? [:]) .setting(timestamp, keyPair) } } else if groupThread.isOpenGroup { guard let openGroup: SMKLegacy._OpenGroup = transaction.object(forKey: thread.uniqueId, inCollection: SMKLegacy.openGroupCollection) as? SMKLegacy._OpenGroup else { SNLogNotTests("[Migration Error] Unable to find open group info") shouldFailMigration = true return } // We want to migrate everyone over to using the domain name for open group // servers rather than the IP, also best to use HTTPS over HTTP where possible // so catch the case where we have the domain with HTTP (the 'defaultServer' // value contains a HTTPS scheme so we get IP HTTP -> HTTPS for free as well) let processedOpenGroupServer: String = { // Check if the server is a Session-run one based on it's guard OpenGroupManager.isSessionRunOpenGroup(server: openGroup.server) else { return openGroup.server } return OpenGroupAPI.defaultServer }() legacyThreadIdToIdMap[thread.uniqueId] = OpenGroup.idFor( roomToken: openGroup.room, server: processedOpenGroupServer ) openGroupServer[thread.uniqueId] = processedOpenGroupServer openGroupInfo[thread.uniqueId] = openGroup openGroupUserCount[thread.uniqueId] = ((transaction.object(forKey: openGroup.id, inCollection: SMKLegacy.openGroupUserCountCollection) as? Int64) ?? 0) openGroupImage[thread.uniqueId] = transaction.object(forKey: openGroup.id, inCollection: SMKLegacy.openGroupImageCollection) as? Data } } Storage.update(progress: 0.04, for: self, in: target) // MARK: --Interactions SNLogNotTests("[Migration Info] \(target.key(with: self)) - Processing Interactions") /// **Note:** There is no index on the collection column so unfortunately it takes the same amount of time to enumerate through all /// collections as it does to just get the count of collections, due to this, if the database is very large, importing thecollections can be /// very slow (~15s with 2,000,000 rows) - we want to show some kind of progress while enumerating so the below code creates a /// very rought guess of the number of collections based on the file size of the database (this shouldn't affect most users at all) let roughKbPerRow: CGFloat = 2.25 let oldDatabaseSizeBytes: CGFloat = (try? FileManager.default .attributesOfItem(atPath: SUKLegacy.legacyDatabaseFilepath)[.size] .asType(CGFloat.self)) .defaulting(to: 0) let roughNumRows: CGFloat = ((oldDatabaseSizeBytes / 1024) / roughKbPerRow) let startProgress: CGFloat = 0.04 let interactionsCompleteProgress: CGFloat = 0.19 var rowIndex: CGFloat = 0 transaction.enumerateKeysAndObjects(inCollection: SMKLegacy.interactionCollection) { _, object, _ in guard let interaction: SMKLegacy._DBInteraction = object as? SMKLegacy._DBInteraction else { SNLogNotTests("[Migration Error] Unable to process interaction") shouldFailMigration = true return } /// Prune interactions from OpenGroup thread interactions which are older than 6 months /// /// The old structure for the open group id was `g{base64String(Data(__loki_public_chat_group__!{server.room}))} /// so we process the uniqueThreadId to see if it matches that if interaction.uniqueThreadId.starts(with: SMKLegacy.groupThreadPrefix), let base64Data: Data = Data(base64Encoded: interaction.uniqueThreadId.substring(from: SMKLegacy.groupThreadPrefix.count)), let groupIdString: String = String(data: base64Data, encoding: .utf8), ( groupIdString.starts(with: SMKLegacy.openGroupIdPrefix) || groupIdString.starts(with: "http") ), interaction.timestamp < UInt64(floor((timestampNow - GarbageCollectionJob.approxSixMonthsInSeconds) * 1000)) { return } interactions[interaction.uniqueThreadId] = (interactions[interaction.uniqueThreadId] ?? []) .appending(interaction) rowIndex += 1 Storage.update( progress: min( interactionsCompleteProgress, ((rowIndex / roughNumRows) * (interactionsCompleteProgress - startProgress)) ), for: self, in: target ) } Storage.update(progress: interactionsCompleteProgress, for: self, in: target) // MARK: --Attachments SNLogNotTests("[Migration Info] \(target.key(with: self)) - Processing Attachments") transaction.enumerateKeysAndObjects(inCollection: SMKLegacy.attachmentsCollection) { key, object, _ in guard let attachment: SMKLegacy._Attachment = object as? SMKLegacy._Attachment else { SNLogNotTests("[Migration Error] Unable to process attachment") shouldFailMigration = true return } attachments[key] = attachment } Storage.update(progress: 0.21, for: self, in: target) // MARK: --Read Receipts transaction.enumerateKeysAndObjects(inCollection: SMKLegacy.outgoingReadReceiptManagerCollection) { key, object, _ in guard let timestampsMs: Set = object as? Set else { return } outgoingReadReceiptsTimestampsMs[key] = (outgoingReadReceiptsTimestampsMs[key] ?? Set()) .union(timestampsMs) } // MARK: --De-duping receivedMessageTimestamps = receivedMessageTimestamps.inserting( contentsOf: transaction .object( forKey: SMKLegacy.receivedMessageTimestampsKey, inCollection: SMKLegacy.receivedMessageTimestampsCollection ) .asType([UInt64].self) .defaulting(to: []) .asSet() ) transaction.enumerateKeysAndObjects(inCollection: SMKLegacy.receivedCallsCollection) { key, object, _ in guard let uuids: Set = object as? Set else { return } receivedCallUUIDs[key] = (receivedCallUUIDs[key] ?? Set()) .union(uuids) } // MARK: --Jobs SNLogNotTests("[Migration Info] \(target.key(with: self)) - Processing Jobs") transaction.enumerateRows(inCollection: SMKLegacy.notifyPushServerJobCollection) { _, object, _, _ in guard let job = object as? SMKLegacy._NotifyPNServerJob else { return } notifyPushServerJobs.insert(job) } transaction.enumerateRows(inCollection: SMKLegacy.messageReceiveJobCollection) { _, object, _, _ in guard let job = object as? SMKLegacy._MessageReceiveJob else { return } messageReceiveJobs.insert(job) } transaction.enumerateRows(inCollection: SMKLegacy.messageSendJobCollection) { _, object, _, _ in guard let job = object as? SMKLegacy._MessageSendJob else { return } messageSendJobs.insert(job) } transaction.enumerateRows(inCollection: SMKLegacy.attachmentUploadJobCollection) { _, object, _, _ in guard let job = object as? SMKLegacy._AttachmentUploadJob else { return } attachmentUploadJobs.insert(job) } transaction.enumerateRows(inCollection: SMKLegacy.attachmentDownloadJobCollection) { _, object, _, _ in guard let job = object as? SMKLegacy._AttachmentDownloadJob else { return } attachmentDownloadJobs.insert(job) } Storage.update(progress: 0.22, for: self, in: target) // MARK: --Preferences SNLogNotTests("[Migration Info] \(target.key(with: self)) - Processing Preferences") transaction.enumerateKeysAndObjects(inCollection: SMKLegacy.preferencesCollection) { key, object, _ in legacyPreferences[key] = object } transaction.enumerateKeysAndObjects(inCollection: SMKLegacy.additionalPreferencesCollection) { key, object, _ in legacyPreferences[key] = object } // Note: The 'int(forKey:inCollection:)' defaults to `0` which is an incorrect value // for the notification sound so catch it and default legacyPreferences[SMKLegacy.soundsGlobalNotificationKey] = (transaction .object( forKey: SMKLegacy.soundsGlobalNotificationKey, inCollection: SMKLegacy.soundsStorageNotificationCollection ) .asType(NSNumber.self)? .intValue) .defaulting(to: Preferences.Sound.defaultNotificationSound.rawValue) legacyPreferences[SMKLegacy.readReceiptManagerAreReadReceiptsEnabled] = (transaction .object( forKey: SMKLegacy.readReceiptManagerAreReadReceiptsEnabled, inCollection: SMKLegacy.readReceiptManagerCollection ) .asType(NSNumber.self)? .boolValue) .defaulting(to: false) legacyPreferences[SMKLegacy.typingIndicatorsEnabledKey] = (transaction .object( forKey: SMKLegacy.typingIndicatorsEnabledKey, inCollection: SMKLegacy.typingIndicatorsCollection ) .asType(NSNumber.self)? .boolValue) .defaulting(to: false) legacyPreferences[SMKLegacy.screenLockIsScreenLockEnabledKey] = (transaction .object( forKey: SMKLegacy.screenLockIsScreenLockEnabledKey, inCollection: SMKLegacy.screenLockCollection ) .asType(NSNumber.self)? .boolValue) .defaulting(to: false) legacyPreferences[SMKLegacy.screenLockScreenLockTimeoutSecondsKey] = (transaction .object( forKey: SMKLegacy.screenLockScreenLockTimeoutSecondsKey, inCollection: SMKLegacy.screenLockCollection) .asType(NSNumber.self)? .doubleValue) .defaulting(to: (15 * 60)) Storage.update(progress: 0.23, for: self, in: target) } // We can't properly throw within the 'enumerateKeysAndObjects' block so have to throw here guard !shouldFailMigration else { throw StorageError.migrationFailed } // Insert the data into GRDB let currentUserPublicKey: String = getUserHexEncodedPublicKey(db) // MARK: - Insert Contacts SNLogNotTests("[Migration Info] \(target.key(with: self)) - Inserting Contacts") try autoreleasepool { // Values for contact progress let contactStartProgress: CGFloat = 0.23 let progressPerContact: CGFloat = (0.05 / CGFloat(contacts.count)) try contacts.enumerated().forEach { index, legacyContact in let isCurrentUser: Bool = (legacyContact.sessionID == currentUserPublicKey) let contactThreadId: String = SMKLegacy._ContactThread.threadId(from: legacyContact.sessionID) // Create the "Profile" for the legacy contact try Profile( id: legacyContact.sessionID, name: (legacyContact.name ?? legacyContact.sessionID), nickname: legacyContact.nickname, profilePictureUrl: legacyContact.profilePictureURL, profilePictureFileName: legacyContact.profilePictureFileName, profileEncryptionKey: legacyContact.profileEncryptionKey?.keyData ).migrationSafeInsert(db) /// **Note:** The blow "shouldForce" flags are here to allow us to avoid having to run legacy migrations they /// replicate the behaviour of a number of the migrations and perform the changes if the migrations had never run /// `ContactsMigration` - Marked all existing contacts as trusted let shouldForceTrustContact: Bool = (!legacyMigrations.contains(.contactsMigration)) /// `MessageRequestsMigration` - Marked all existing contacts as isApproved and didApproveMe let shouldForceApproveContact: Bool = (!legacyMigrations.contains(.messageRequestsMigration)) /// `BlockingManagerRemovalMigration` - Removed the old blocking manager and updated contacts isBlocked flag accordingly let shouldForceBlockContact: Bool = ( !legacyMigrations.contains(.messageRequestsMigration) && legacyBlockedSessionIds.contains(legacyContact.sessionID) ) /// Looks like there are some cases where conversations would be visible in the old version but wouldn't in the new version /// it seems to be related to the `isApproved` and `didApproveMe` not being set correctly somehow, this logic is to /// ensure the flags are set correctly based on sent/received messages let interactionsForContact: [SMKLegacy._DBInteraction] = (interactions["\(SMKLegacy.contactThreadPrefix)\(legacyContact.sessionID)"] ?? []) let shouldForceIsApproved: Bool = interactionsForContact .contains(where: { $0 is SMKLegacy._DBOutgoingMessage }) let shouldForceDidApproveMe: Bool = interactionsForContact .contains(where: { $0 is SMKLegacy._DBIncomingMessage }) // Determine if this contact is a "real" contact (don't want to create contacts for // every user in the new structure but still want profiles for every user) if isCurrentUser || contactThreadIds.contains(contactThreadId) || legacyContact.isApproved || legacyContact.didApproveMe || legacyContact.isBlocked || legacyContact.hasBeenBlocked || shouldForceTrustContact || shouldForceApproveContact || shouldForceBlockContact || shouldForceIsApproved || shouldForceDidApproveMe { // Create the contact try Contact( id: legacyContact.sessionID, isTrusted: ( isCurrentUser || legacyContact.isTrusted || shouldForceTrustContact ), isApproved: ( isCurrentUser || legacyContact.isApproved || shouldForceApproveContact || shouldForceIsApproved ), isBlocked: ( !isCurrentUser && ( legacyContact.isBlocked || shouldForceBlockContact ) ), didApproveMe: ( isCurrentUser || legacyContact.didApproveMe || shouldForceApproveContact || shouldForceDidApproveMe ), hasBeenBlocked: (!isCurrentUser && (legacyContact.hasBeenBlocked || legacyContact.isBlocked)) ).migrationSafeInsert(db) } // Increment the progress for each contact Storage.update( progress: contactStartProgress + (progressPerContact * CGFloat(index + 1)), for: self, in: target ) } } // Clear out processed data (give the memory a change to be freed) contacts = [] legacyBlockedSessionIds = [] contactThreadIds = [] // MARK: - Insert Threads SNLogNotTests("[Migration Info] \(target.key(with: self)) - Inserting Threads & Interactions") var legacyInteractionToIdMap: [String: Int64] = [:] var legacyInteractionIdentifierToIdMap: [String: Int64] = [:] var legacyInteractionIdentifierToIdFallbackMap: [String: Int64] = [:] func identifier( for threadId: String, sentTimestamp: UInt64, recipients: [String], destination: Message.Destination?, variant: Interaction.Variant?, useFallback: Bool ) -> String { let recipientString: String = { if let destination: Message.Destination = destination { switch destination { case .contact(let publicKey): return publicKey default: break } } return (recipients.first ?? "0") }() return [ (useFallback ? // Fallback to seconds-based accuracy (instead of milliseconds) String("\(sentTimestamp)".prefix("\(Int(Date().timeIntervalSince1970))".count)) : "\(sentTimestamp)" ), (useFallback ? variant.map { "\($0)" } : nil), recipientString, threadId ] .compactMap { $0 } .joined(separator: "-") } // Values for thread progress var interactionCounter: CGFloat = 0 let allInteractionsCount: Int = interactions.map { $0.value.count }.reduce(0, +) let threadInteractionsStartProgress: CGFloat = 0.28 let progressPerInteraction: CGFloat = (0.70 / CGFloat(allInteractionsCount)) // Sort by id just so we can make the migration process more determinstic try legacyThreads.sorted(by: { lhs, rhs in lhs.uniqueId < rhs.uniqueId }).forEach { legacyThread in guard let threadId: String = legacyThreadIdToIdMap[legacyThread.uniqueId] else { SNLogNotTests("[Migration Error] Unable to migrate thread with no id mapping") throw StorageError.migrationFailed } let threadVariant: SessionThread.Variant let onlyNotifyForMentions: Bool switch legacyThread { case let groupThread as SMKLegacy._GroupThread: threadVariant = (groupThread.isOpenGroup ? .community : .legacyGroup) onlyNotifyForMentions = groupThread.isOnlyNotifyingForMentions default: threadVariant = .contact onlyNotifyForMentions = false } try autoreleasepool { try SessionThread( id: threadId, variant: threadVariant, creationDateTimestamp: legacyThread.creationDate.timeIntervalSince1970, shouldBeVisible: legacyThread.shouldBeVisible, isPinned: legacyThread.isPinned, messageDraft: ((legacyThread.messageDraft ?? "").isEmpty ? nil : legacyThread.messageDraft ), mutedUntilTimestamp: legacyThread.mutedUntilDate?.timeIntervalSince1970, onlyNotifyForMentions: onlyNotifyForMentions ).migrationSafeInsert(db) // Disappearing Messages Configuration if let config: SMKLegacy._DisappearingMessagesConfiguration = disappearingMessagesConfiguration[threadId] { try DisappearingMessagesConfiguration( threadId: threadId, isEnabled: config.isEnabled, durationSeconds: TimeInterval(config.durationSeconds) ).migrationSafeInsert(db) } else { try DisappearingMessagesConfiguration .defaultWith(threadId) .migrationSafeInsert(db) } // Closed Groups if legacyThread.isClosedGroup { guard let name: String = closedGroupName[legacyThread.uniqueId], let groupModel: SMKLegacy._GroupModel = closedGroupModel[legacyThread.uniqueId], let formationTimestamp: UInt64 = closedGroupFormation[legacyThread.uniqueId] else { SNLogNotTests("[Migration Error] Closed group missing required data") throw StorageError.migrationFailed } try ClosedGroup( threadId: threadId, name: name, formationTimestamp: TimeInterval(formationTimestamp) ).migrationSafeInsert(db) // Note: If a user has left a closed group then they won't actually have any keys // but they should still be able to browse the old messages so we do want to allow // this case and migrate the rest of the info try closedGroupKeys[legacyThread.uniqueId]?.forEach { timestamp, legacyKeys in try ClosedGroupKeyPair( threadId: threadId, publicKey: legacyKeys.publicKey, secretKey: legacyKeys.privateKey, receivedTimestamp: timestamp ).migrationSafeInsert(db) } // Create the 'GroupMember' models for the group (even if the current user is no longer // a member as these objects are used to generate the group avatar icon) func createDummyProfile(profileId: String) { SNLogNotTests("[Migration Warning] Closed group member with unknown user found - Creating empty profile") // Note: Need to upsert here because it's possible multiple quotes // will use the same invalid 'authorId' value resulting in a unique // constraint violation try? Profile( id: profileId, name: profileId ).migrationSafeSave(db) } try groupModel.groupMemberIds.forEach { memberId in try GroupMember( groupId: threadId, profileId: memberId, role: .standard, isHidden: false // Ignored: Didn't exist at time of migration ).migrationSafeInsert(db) if !validProfileIds.contains(memberId) { createDummyProfile(profileId: memberId) } } try groupModel.groupAdminIds.forEach { adminId in try GroupMember( groupId: threadId, profileId: adminId, role: .admin, isHidden: false // Ignored: Didn't exist at time of migration ).migrationSafeInsert(db) if !validProfileIds.contains(adminId) { createDummyProfile(profileId: adminId) } } try (closedGroupZombieMemberIds[legacyThread.uniqueId] ?? []).forEach { zombieId in try GroupMember( groupId: threadId, profileId: zombieId, role: .zombie, isHidden: false // Ignored: Didn't exist at time of migration ).migrationSafeInsert(db) if !validProfileIds.contains(zombieId) { createDummyProfile(profileId: zombieId) } } } // Open Groups if legacyThread.isOpenGroup { guard let openGroup: SMKLegacy._OpenGroup = openGroupInfo[legacyThread.uniqueId], let targetOpenGroupServer: String = openGroupServer[legacyThread.uniqueId] else { SNLogNotTests("[Migration Error] Open group missing required data") throw StorageError.migrationFailed } try OpenGroup( server: targetOpenGroupServer, roomToken: openGroup.room, publicKey: openGroup.publicKey, isActive: true, name: openGroup.name, roomDescription: nil, imageId: openGroup.imageID, imageData: openGroupImage[legacyThread.uniqueId], userCount: (openGroupUserCount[legacyThread.uniqueId] ?? 0), // Will be updated next poll infoUpdates: 0, sequenceNumber: 0, inboxLatestMessageId: 0, outboxLatestMessageId: 0 ) .migrationSafeInsert(db) } } try autoreleasepool { try interactions[legacyThread.uniqueId]? .sorted(by: { lhs, rhs in lhs.timestamp < rhs.timestamp }) // Maintain sort order .forEach { legacyInteraction in let serverHash: String? let variant: Interaction.Variant let authorId: String let body: String? let wasRead: Bool let expiresInSeconds: UInt32? let expiresStartedAtMs: UInt64? let openGroupServerMessageId: Int64? let recipientStateMap: [String: SMKLegacy._DBOutgoingMessageRecipientState]? let mostRecentFailureText: String? let quotedMessage: SMKLegacy._DBQuotedMessage? let linkPreview: SMKLegacy._DBLinkPreview? let linkPreviewVariant: LinkPreview.Variant var attachmentIds: [String] // Handle the common 'SMKLegacy._DBMessage' values first if let legacyMessage: SMKLegacy._DBMessage = legacyInteraction as? SMKLegacy._DBMessage { serverHash = legacyMessage.serverHash // The legacy code only considered '!= 0' ids as valid so set those // values to be null to avoid the unique constraint (it's also more // correct for the values to be null) // // Note: Looks like it was also possible for this to be set to the max // value which overflows when trying to convert to a signed version so // we essentially discard the information in those cases) openGroupServerMessageId = (Int64.zeroingOverflow(legacyMessage.openGroupServerMessageID) == 0 ? nil : Int64.zeroingOverflow(legacyMessage.openGroupServerMessageID) ) quotedMessage = legacyMessage.quotedMessage // Convert the 'OpenGroupInvitation' into a LinkPreview if let openGroupInvitationName: String = legacyMessage.openGroupInvitationName, let openGroupInvitationUrl: String = legacyMessage.openGroupInvitationURL { linkPreviewVariant = .openGroupInvitation linkPreview = SMKLegacy._DBLinkPreview( urlString: openGroupInvitationUrl, title: openGroupInvitationName, imageAttachmentId: nil ) } else { linkPreviewVariant = .standard linkPreview = legacyMessage.linkPreview } // Attachments for deleted messages won't exist attachmentIds = (legacyMessage.isDeleted ? [] : legacyMessage.attachmentIds ) } else { serverHash = nil openGroupServerMessageId = nil quotedMessage = nil linkPreviewVariant = .standard linkPreview = nil attachmentIds = [] } // Then handle the behaviours for each message type switch legacyInteraction { case let incomingMessage as SMKLegacy._DBIncomingMessage: // Note: We want to distinguish deleted messages from normal ones variant = (incomingMessage.isDeleted ? .standardIncomingDeleted : .standardIncoming ) authorId = incomingMessage.authorId body = incomingMessage.body wasRead = incomingMessage.wasRead expiresInSeconds = incomingMessage.expiresInSeconds expiresStartedAtMs = incomingMessage.expireStartedAt recipientStateMap = [:] mostRecentFailureText = nil case let outgoingMessage as SMKLegacy._DBOutgoingMessage: variant = .standardOutgoing authorId = currentUserPublicKey body = outgoingMessage.body wasRead = true // Outgoing messages are read by default expiresInSeconds = outgoingMessage.expiresInSeconds expiresStartedAtMs = outgoingMessage.expireStartedAt recipientStateMap = outgoingMessage.recipientStateMap mostRecentFailureText = outgoingMessage.mostRecentFailureText case let infoMessage as SMKLegacy._DBInfoMessage: // Note: The legacy 'TSInfoMessage' didn't store the author id so there is no // way to determine who actually triggered the info message authorId = currentUserPublicKey body = { // Note: Some message types stored additional info and constructed a string // at display time, instead we encode the data into the body of the message // as JSON so we want to continue that behaviour but not change the database // structure for some edge cases switch infoMessage.messageType { case .disappearingMessagesUpdate: guard let updateMessage: SMKLegacy._DisappearingConfigurationUpdateInfoMessage = infoMessage as? SMKLegacy._DisappearingConfigurationUpdateInfoMessage, let infoMessageData: Data = try? JSONEncoder().encode( DisappearingMessagesConfiguration.MessageInfo( senderName: updateMessage.createdByRemoteName, isEnabled: updateMessage.configurationIsEnabled, durationSeconds: TimeInterval(updateMessage.configurationDurationSeconds) ) ), let infoMessageString: String = String(data: infoMessageData, encoding: .utf8) else { break } return infoMessageString case .call: let messageInfo: CallMessage.MessageInfo = CallMessage.MessageInfo( state: { switch infoMessage.callState { case .incoming: return .incoming case .outgoing: return .outgoing case .missed: return .missed case .permissionDenied: return .permissionDenied case .unknown: return .unknown } }() ) guard let messageInfoData: Data = try? JSONEncoder().encode(messageInfo), let messageInfoDataString: String = String(data: messageInfoData, encoding: .utf8) else { break } return messageInfoDataString default: break } return ((infoMessage.body ?? "").isEmpty ? infoMessage.customMessage : infoMessage.body ) }() wasRead = infoMessage.wasRead expiresInSeconds = nil // Info messages don't expire expiresStartedAtMs = nil // Info messages don't expire recipientStateMap = [:] mostRecentFailureText = nil switch infoMessage.messageType { case .groupCreated: variant = .infoClosedGroupCreated case .groupUpdated: variant = .infoClosedGroupUpdated case .groupCurrentUserLeft: variant = .infoClosedGroupCurrentUserLeft case .disappearingMessagesUpdate: variant = .infoDisappearingMessagesUpdate case .screenshotNotification: variant = .infoScreenshotNotification case .mediaSavedNotification: variant = .infoMediaSavedNotification case .call: variant = .infoCall case .messageRequestAccepted: variant = .infoMessageRequestAccepted } default: SNLogNotTests("[Migration Error] Unsupported interaction type") throw StorageError.migrationFailed } // Insert the data let interaction: Interaction do { interaction = try Interaction( serverHash: { switch variant { // Don't store the 'serverHash' for these so sync messages // are seen as duplicates case .infoDisappearingMessagesUpdate: return nil default: return serverHash } }(), messageUuid: { guard variant == .infoCall else { return nil } /// **Note:** Unfortunately there is no good way to properly match this UUID up with the correct /// interaction (and it was previously stored as a Set so the values will be unsorted anyway); luckily /// we are only using this value for updating and de-duping purposes at this stage so it _shouldn't_ /// matter if the values end up being assigned to the wrong interactions, we do still want to try and /// store each value through so mutate the list as we process each UUID /// /// **Note:** It looks like these values were stored against the sessionId rather than the legacy /// thread unique id return receivedCallUUIDs[threadId]?.popFirst() }(), threadId: threadId, authorId: authorId, variant: variant, body: body, timestampMs: Int64.zeroingOverflow(legacyInteraction.timestamp), receivedAtTimestampMs: Int64.zeroingOverflow(legacyInteraction.receivedAtTimestamp), wasRead: wasRead, hasMention: Interaction.isUserMentioned( db, threadId: threadId, body: body, quoteAuthorId: quotedMessage?.authorId ), // For both of these '0' used to be equivalent to null expiresInSeconds: ((expiresInSeconds ?? 0) > 0 ? expiresInSeconds.map { TimeInterval($0) } : nil ), expiresStartedAtMs: ((expiresStartedAtMs ?? 0) > 0 ? expiresStartedAtMs.map { Double($0) } : nil ), linkPreviewUrl: linkPreview?.urlString, // Only a soft link so save to set openGroupServerMessageId: openGroupServerMessageId, openGroupWhisperMods: false, openGroupWhisperTo: nil ).migrationSafeInserted(db) } catch { switch error { // Ignore duplicate interactions case DatabaseError.SQLITE_CONSTRAINT_UNIQUE: SNLogNotTests("[Migration Warning] Found duplicate message of variant: \(variant); skipping") return default: SNLogNotTests("[Migration Error] Failed to insert interaction") throw StorageError.migrationFailed } } // Insert a 'ControlMessageProcessRecord' if needed (for duplication prevention) try ControlMessageProcessRecord( threadId: threadId, variant: variant, timestampMs: Int64.zeroingOverflow(legacyInteraction.timestamp) )?.migrationSafeInsert(db) // Remove timestamps we created records for (they will be protected by unique // constraints so don't need legacy process records) receivedMessageTimestamps.remove(legacyInteraction.timestamp) guard let interactionId: Int64 = interaction.id else { SNLogNotTests("[Migration Error] Failed to insert interaction") throw StorageError.migrationFailed } // Store the interactionId in the lookup map to simplify job creation later let legacyIdentifier: String = identifier( for: threadId, sentTimestamp: legacyInteraction.timestamp, recipients: ((legacyInteraction as? SMKLegacy._DBOutgoingMessage)? .recipientStateMap? .keys .map { $0 }) .defaulting(to: []), destination: (threadVariant == .contact ? .contact(publicKey: threadId) : nil ), variant: variant, useFallback: false ) let legacyIdentifierFallback: String = identifier( for: threadId, sentTimestamp: legacyInteraction.timestamp, recipients: ((legacyInteraction as? SMKLegacy._DBOutgoingMessage)? .recipientStateMap? .keys .map { $0 }) .defaulting(to: []), destination: (threadVariant == .contact ? .contact(publicKey: threadId) : nil ), variant: variant, useFallback: true ) legacyInteractionToIdMap[legacyInteraction.uniqueId] = interactionId legacyInteractionIdentifierToIdMap[legacyIdentifier] = interactionId legacyInteractionIdentifierToIdFallbackMap[legacyIdentifierFallback] = interactionId // Handle the recipient states // Note: Inserting an Interaction into the database will automatically create a 'RecipientState' // for outgoing messages try recipientStateMap?.forEach { recipientId, legacyState in try RecipientState( interactionId: interactionId, recipientId: recipientId, state: { switch legacyState.state { case .failed: return .failed case .sending: return .sending case .skipped: return .skipped case .sent: return .sent } }(), readTimestampMs: legacyState.readTimestamp, mostRecentFailureText: (legacyState.state == .failed ? mostRecentFailureText : nil ) ).migrationSafeSave(db) } // Handle any quote if let quotedMessage: SMKLegacy._DBQuotedMessage = quotedMessage { var quoteAttachmentId: String? = quotedMessage.quotedAttachments .flatMap { attachmentInfo in return [ // Prioritise the thumbnail as it means we won't // need to generate a new one attachmentInfo.thumbnailAttachmentStreamId, attachmentInfo.thumbnailAttachmentPointerId, attachmentInfo.attachmentId ] .compactMap { $0 } } .first { attachmentId -> Bool in attachments[attachmentId] != nil } // It looks like there can be cases where a quote can be quoting an // interaction that isn't associated with a profile we know about (eg. // if you join an open group and one of the first messages is a quote of // an older message not cached to the device) - this will cause a foreign // key constraint violation so in these cases just create an empty profile if !validProfileIds.contains(quotedMessage.authorId) { SNLogNotTests("[Migration Warning] Quote with unknown author found - Creating empty profile") // Note: Need to upsert here because it's possible multiple quotes // will use the same invalid 'authorId' value resulting in a unique // constraint violation try Profile( id: quotedMessage.authorId, name: quotedMessage.authorId ).migrationSafeSave(db) } // Note: It looks like there is a way for a quote to not have it's // associated attachmentId so let's try our best to track down the // original interaction and re-create the attachment link before // falling back to having no attachment in the quote if quoteAttachmentId == nil && !quotedMessage.quotedAttachments.isEmpty { quoteAttachmentId = interactions[legacyThread.uniqueId]? .first(where: { $0.timestamp == quotedMessage.timestamp && ( // Outgoing messages don't store the 'authorId' so we // need to compare against the 'currentUserPublicKey' // for those or cast to a TSIncomingMessage otherwise quotedMessage.authorId == currentUserPublicKey || quotedMessage.authorId == ($0 as? SMKLegacy._DBIncomingMessage)?.authorId ) }) .asType(SMKLegacy._DBMessage.self)? .attachmentIds .first SNLogNotTests([ "[Migration Warning] Quote with invalid attachmentId found", (quoteAttachmentId == nil ? "Unable to reconcile, leaving attachment blank" : "Original interaction found, using source attachment" ) ].joined(separator: " - ")) } // Setup the attachment and add it to the lookup (if it exists) let attachmentId: String? = try attachmentId( db, for: quoteAttachmentId, isQuotedMessage: true, attachments: attachments, processedAttachmentIds: &processedAttachmentIds ) // Create the quote try Quote( interactionId: interactionId, authorId: quotedMessage.authorId, timestampMs: Int64.zeroingOverflow(quotedMessage.timestamp), body: quotedMessage.body, attachmentId: attachmentId ).migrationSafeInsert(db) } // Handle any LinkPreview if let linkPreview: SMKLegacy._DBLinkPreview = linkPreview, let urlString: String = linkPreview.urlString { // Note: The `legacyInteraction.timestamp` value is in milliseconds let timestamp: TimeInterval = LinkPreview.timestampFor(sentTimestampMs: Double(legacyInteraction.timestamp)) // Setup the attachment and add it to the lookup (if it exists - we do actually // support link previews with no image attachments so no need to throw migration // errors in those cases) let attachmentId: String? = try attachmentId( db, for: linkPreview.imageAttachmentId, attachments: attachments, processedAttachmentIds: &processedAttachmentIds ) // Note: It's possible for there to be duplicate values here so we use 'save' // instead of insert (ie. upsert) try LinkPreview( url: urlString, timestamp: timestamp, variant: linkPreviewVariant, title: linkPreview.title, attachmentId: attachmentId ).migrationSafeSave(db) } // Handle any attachments try attachmentIds.enumerated().forEach { index, legacyAttachmentId in let maybeAttachmentId: String? = (try attachmentId( db, for: legacyAttachmentId, interactionVariant: variant, attachments: attachments, processedAttachmentIds: &processedAttachmentIds )) .defaulting( // It looks like somehow messages could exist in the old database which // referenced attachments but had no attachments in the database; doing // nothing here results in these messages appearing as empty message // bubbles so instead we want to insert invalid attachments instead to: try invalidAttachmentId( db, for: legacyAttachmentId, attachments: attachments, processedAttachmentIds: &processedAttachmentIds ) ) guard let attachmentId: String = maybeAttachmentId else { SNLogNotTests("[Migration Warning] Failed to create invalid attachment for missing attachment") return } // Link the attachment to the interaction and add to the id lookup try InteractionAttachment( albumIndex: index, interactionId: interactionId, attachmentId: attachmentId ).migrationSafeInsert(db) } // Increment the progress for each contact Storage.update( progress: ( threadInteractionsStartProgress + (progressPerInteraction * (interactionCounter + 1)) ), for: self, in: target ) interactionCounter += 1 } } } // Clear out processed data (give the memory a change to be freed) legacyThreads = [] disappearingMessagesConfiguration = [:] closedGroupKeys = [:] closedGroupName = [:] closedGroupFormation = [:] closedGroupModel = [:] closedGroupZombieMemberIds = [:] openGroupInfo = [:] openGroupUserCount = [:] openGroupImage = [:] interactions = [:] attachments = [:] // MARK: --Received Message Timestamps // Insert a 'ControlMessageProcessRecord' for any remaining 'receivedMessageTimestamp' // entries as "legacy" try ControlMessageProcessRecord.generateLegacyProcessRecords( db, receivedMessageTimestamps: receivedMessageTimestamps.map { Int64.zeroingOverflow($0) } ) // Clear out processed data (give the memory a change to be freed) receivedMessageTimestamps = [] // MARK: - Insert Jobs SNLogNotTests("[Migration Info] \(target.key(with: self)) - Inserting Jobs") // MARK: --notifyPushServer try autoreleasepool { try notifyPushServerJobs.forEach { legacyJob in _ = try Job( failureCount: legacyJob.failureCount, variant: .notifyPushServer, behaviour: .runOnce, nextRunTimestamp: 0, details: NotifyPushServerJob.Details( message: SnodeMessage( recipient: legacyJob.message.recipient, // Note: The legacy type had 'LosslessStringConvertible' so we need // to use '.description' to get it as a basic string data: legacyJob.message.data.description, ttl: legacyJob.message.ttl, timestampMs: legacyJob.message.timestamp ) ) )?.migrationSafeInserted(db) } } // MARK: --messageReceive try autoreleasepool { try messageReceiveJobs.forEach { legacyJob in // We haven't supported OpenGroup messageReceive jobs for a long time so if // we see any then just ignore them if legacyJob.openGroupID != nil && legacyJob.openGroupMessageServerID != nil { return } // We have changed how messageReceive jobs work - we now parse the message upon receipt and // the MessageReceiveJob only does the handling - as a result we need to do the same behaviour // here so we don't need to support the legacy behaviour guard let processedMessage: ProcessedMessage = try? Message.processRawReceivedMessage(db, serializedData: legacyJob.data, serverHash: legacyJob.serverHash) else { return } _ = try Job( failureCount: legacyJob.failureCount, variant: .messageReceive, behaviour: .runOnce, nextRunTimestamp: 0, threadId: processedMessage.threadId, details: MessageReceiveJob.Details( messages: [processedMessage.messageInfo], calledFromBackgroundPoller: legacyJob.isBackgroundPoll ) )?.migrationSafeInserted(db) } } // MARK: --messageSend var messageSendJobLegacyMap: [String: Job] = [:] try autoreleasepool { try messageSendJobs.forEach { legacyJob in // Fetch the threadId and interactionId this job should be associated with let threadId: String = { switch legacyJob.destination { case .contact(let publicKey): return publicKey case .closedGroup(let groupPublicKey): return groupPublicKey case .openGroup(let roomToken, let server, _, _, _): return OpenGroup.idFor(roomToken: roomToken, server: server) case .openGroupInbox(_, _, let blindedPublicKey): return blindedPublicKey } }() let interactionId: Int64? = { // The 'Legacy.Job' 'id' value was "(timestamp)(num jobs for this timestamp)" // so we can reverse-engineer an approximate timestamp by extracting it from // the id (this value is unlikely to match exactly though) let fallbackTimestamp: UInt64 = legacyJob.id .map { UInt64($0.prefix("\(SnodeAPI.currentOffsetTimestampMs())".count)) } .defaulting(to: 0) let legacyIdentifier: String = identifier( for: threadId, sentTimestamp: (legacyJob.message.sentTimestamp ?? fallbackTimestamp), recipients: (legacyJob.message.recipient.map { [$0] } ?? []), destination: legacyJob.destination, variant: nil, useFallback: false ) if let matchingId: Int64 = legacyInteractionIdentifierToIdMap[legacyIdentifier] { return matchingId } // If we didn't find the correct interaction then we need to try the "fallback" // identifier which is less accurate (during testing this only happened for // 'ExpirationTimerUpdate' send jobs) let fallbackIdentifier: String = identifier( for: threadId, sentTimestamp: (legacyJob.message.sentTimestamp ?? fallbackTimestamp), recipients: (legacyJob.message.recipient.map { [$0] } ?? []), destination: legacyJob.destination, variant: { switch legacyJob.message { case is SMKLegacy._ExpirationTimerUpdate: return .infoDisappearingMessagesUpdate default: return nil } }(), useFallback: true ) return legacyInteractionIdentifierToIdFallbackMap[fallbackIdentifier] }() // Don't botther adding any 'MessageSend' jobs VisibleMessages which don't have associated // interactions switch legacyJob.message { case is SMKLegacy._VisibleMessage: guard interactionId != nil else { SNLogNotTests("[Migration Warning] Unable to find associated interaction to messageSend job, ignoring.") return } break default: break } let job: Job? = try Job( failureCount: legacyJob.failureCount, variant: .messageSend, behaviour: .runOnce, nextRunTimestamp: 0, threadId: threadId, // Note: There are some cases where there isn't a link between a // 'MessageSendJob' and an interaction (eg. ConfigurationMessage), // in these cases the 'interactionId' value will be nil interactionId: interactionId, details: MessageSendJob.Details( destination: legacyJob.destination, message: legacyJob.message.toNonLegacy() ) )?.migrationSafeInserted(db) if let oldId: String = legacyJob.id { messageSendJobLegacyMap[oldId] = job } } } // MARK: --attachmentUpload try autoreleasepool { try attachmentUploadJobs.forEach { legacyJob in guard let sendJob: Job = messageSendJobLegacyMap[legacyJob.messageSendJobID], let sendJobId: Int64 = sendJob.id else { SNLogNotTests("[Migration Error] attachmentUpload job missing associated MessageSendJob") throw StorageError.migrationFailed } let uploadJob: Job? = try Job( failureCount: legacyJob.failureCount, variant: .attachmentUpload, behaviour: .runOnce, threadId: sendJob.threadId, interactionId: sendJob.interactionId, details: AttachmentUploadJob.Details( messageSendJobId: sendJobId, attachmentId: legacyJob.attachmentID ) )?.migrationSafeInserted(db) // Add the dependency to the relevant MessageSendJob guard let uploadJobId: Int64 = uploadJob?.id else { SNLogNotTests("[Migration Error] attachmentUpload job was not created") throw StorageError.migrationFailed } try JobDependencies( jobId: sendJobId, dependantId: uploadJobId ).migrationSafeInsert(db) } } // MARK: --attachmentDownload try autoreleasepool { try attachmentDownloadJobs.forEach { legacyJob in guard let interactionId: Int64 = legacyInteractionToIdMap[legacyJob.tsMessageID] else { // This can happen if an UnsendRequest came before an AttachmentDownloadJob completed SNLogNotTests("[Migration Warning] attachmentDownload job with no interaction found - ignoring") return } guard processedAttachmentIds.contains(legacyJob.attachmentID) else { // Unsure how this case can occur but it seemed to happen when testing internally SNLogNotTests("[Migration Warning] attachmentDownload job unable to find attachment - ignoring") return } _ = try Job( failureCount: legacyJob.failureCount, variant: .attachmentDownload, behaviour: .runOnce, nextRunTimestamp: 0, threadId: legacyThreadIdToIdMap[legacyJob.threadID], interactionId: interactionId, details: AttachmentDownloadJob.Details( attachmentId: legacyJob.attachmentID ) )?.migrationSafeInserted(db) } } // MARK: --sendReadReceipts try autoreleasepool { try outgoingReadReceiptsTimestampsMs.forEach { threadId, timestampsMs in _ = try Job( variant: .sendReadReceipts, behaviour: .recurring, threadId: threadId, details: SendReadReceiptsJob.Details( destination: .contact(publicKey: threadId), timestampMsValues: timestampsMs ) )?.migrationSafeInserted(db) } } Storage.update(progress: 0.99, for: self, in: target) // MARK: - Preferences SNLogNotTests("[Migration Info] \(target.key(with: self)) - Inserting Preferences") db[.defaultNotificationSound] = Preferences.Sound(rawValue: legacyPreferences[SMKLegacy.soundsGlobalNotificationKey] as? Int ?? -1) .defaulting(to: Preferences.Sound.defaultNotificationSound) db[.playNotificationSoundInForeground] = (legacyPreferences[SMKLegacy.preferencesKeyNotificationSoundInForeground] as? Bool == true) db[.preferencesNotificationPreviewType] = Preferences.NotificationPreviewType(rawValue: legacyPreferences[SMKLegacy.preferencesKeyNotificationPreviewType] as? Int ?? -1) .defaulting(to: .defaultPreviewType) if let lastPushToken: String = legacyPreferences[SMKLegacy.preferencesKeyLastRecordedPushToken] as? String { db[.lastRecordedPushToken] = lastPushToken } if let lastVoipToken: String = legacyPreferences[SMKLegacy.preferencesKeyLastRecordedVoipToken] as? String { db[.lastRecordedVoipToken] = lastVoipToken } db[.areReadReceiptsEnabled] = (legacyPreferences[SMKLegacy.readReceiptManagerAreReadReceiptsEnabled] as? Bool == true) db[.typingIndicatorsEnabled] = (legacyPreferences[SMKLegacy.typingIndicatorsEnabledKey] as? Bool == true) db[.isScreenLockEnabled] = (legacyPreferences[SMKLegacy.screenLockIsScreenLockEnabledKey] as? Bool == true) // Note: 'screenLockTimeoutSeconds' has been removed, but we want to avoid changing the behaviour // of old migrations when possible db.unsafeSet( key: "screenLockTimeoutSeconds", value: (legacyPreferences[SMKLegacy.screenLockScreenLockTimeoutSecondsKey] as? Double) .defaulting(to: (15 * 60)) ) db[.areLinkPreviewsEnabled] = (legacyPreferences[SMKLegacy.preferencesKeyAreLinkPreviewsEnabled] as? Bool == true) db[.areCallsEnabled] = (legacyPreferences[SMKLegacy.preferencesKeyAreCallsEnabled] as? Bool == true) db[.hasHiddenMessageRequests] = CurrentAppContext().appUserDefaults() .bool(forKey: SMKLegacy.userDefaultsHasHiddenMessageRequests) // Note: The 'hasViewedSeed' was originally stored on standard user defaults db[.hasViewedSeed] = UserDefaults.standard.bool(forKey: SMKLegacy.userDefaultsHasViewedSeedKey) db[.hasSavedThread] = (legacyPreferences[SMKLegacy.preferencesKeyHasSavedThreadKey] as? Bool == true) db[.hasSentAMessage] = (legacyPreferences[SMKLegacy.preferencesKeyHasSentAMessageKey] as? Bool == true) db[.isReadyForAppExtensions] = CurrentAppContext().appUserDefaults().bool(forKey: SMKLegacy.preferencesKeyIsReadyForAppExtensions) // We want this setting to be on by default db[.trimOpenGroupMessagesOlderThanSixMonths] = true Storage.update(progress: 1, for: self, in: target) // In case this is the last migration } // MARK: - Convenience private static func attachmentId( _ db: Database, for legacyAttachmentId: String?, interactionVariant: Interaction.Variant? = nil, isQuotedMessage: Bool = false, attachments: [String: SMKLegacy._Attachment], processedAttachmentIds: inout Set ) throws -> String? { guard let legacyAttachmentId: String = legacyAttachmentId else { return nil } guard !processedAttachmentIds.contains(legacyAttachmentId) else { guard isQuotedMessage else { SNLogNotTests("[Migration Error] Attempted to process duplicate attachment") throw StorageError.migrationFailed } return legacyAttachmentId } guard let legacyAttachment: SMKLegacy._Attachment = attachments[legacyAttachmentId] else { SNLogNotTests("[Migration Warning] Missing attachment - interaction will show a \"failed\" attachment") return nil } let processedLocalRelativeFilePath: String? = (legacyAttachment as? SMKLegacy._AttachmentStream)? .localRelativeFilePath .map { filePath -> String in // The old 'localRelativeFilePath' seemed to have a leading forward slash (want // to get rid of it so we can correctly use 'appendingPathComponent') guard filePath.starts(with: "/") else { return filePath } return String(filePath.suffix(from: filePath.index(after: filePath.startIndex))) } let state: Attachment.State = { switch legacyAttachment { case let stream as SMKLegacy._AttachmentStream: // Outgoing or already downloaded switch interactionVariant { case .standardOutgoing: return (stream.isUploaded ? .uploaded : .uploading) default: return .downloaded } // All other cases can just be set to 'pendingDownload' default: return .pendingDownload } }() let size: CGSize = { switch legacyAttachment { case let stream as SMKLegacy._AttachmentStream: // First try to get an image size using the 'localRelativeFilePath' value if let localRelativeFilePath: String = processedLocalRelativeFilePath, let specificImageSize: CGSize = Attachment.imageSize( contentType: stream.contentType, originalFilePath: URL(fileURLWithPath: Attachment.attachmentsFolder) .appendingPathComponent(localRelativeFilePath) .path ), specificImageSize != .zero { return specificImageSize } // Then fallback to trying to get the size from the 'originalFilePath' guard let originalFilePath: String = Attachment.originalFilePath(id: legacyAttachmentId, mimeType: stream.contentType, sourceFilename: stream.sourceFilename) else { return .zero } return Attachment .imageSize( contentType: stream.contentType, originalFilePath: originalFilePath ) .defaulting(to: .zero) case let pointer as SMKLegacy._AttachmentPointer: return pointer.mediaSize default: return CGSize.zero } }() let (isValid, duration): (Bool, TimeInterval?) = { guard let stream: SMKLegacy._AttachmentStream = legacyAttachment as? SMKLegacy._AttachmentStream, let originalFilePath: String = Attachment.originalFilePath( id: legacyAttachmentId, mimeType: stream.contentType, sourceFilename: stream.sourceFilename ) else { return (false, nil) } if stream.isAudio { if let cachedDuration: TimeInterval = stream.cachedAudioDurationSeconds?.doubleValue, cachedDuration > 0 { return (true, cachedDuration) } let attachmentVailidityInfo = Attachment.determineValidityAndDuration( contentType: stream.contentType, localRelativeFilePath: processedLocalRelativeFilePath, originalFilePath: originalFilePath ) return (attachmentVailidityInfo.isValid, attachmentVailidityInfo.duration) } if stream.isVisualMedia { let attachmentVailidityInfo = Attachment.determineValidityAndDuration( contentType: stream.contentType, localRelativeFilePath: processedLocalRelativeFilePath, originalFilePath: originalFilePath ) return (attachmentVailidityInfo.isValid, attachmentVailidityInfo.duration) } return (true, nil) }() _ = try Attachment( // Note: The legacy attachment object used a UUID string for it's id as well // and saved files using these id's so just used the existing id so we don't // need to bother renaming files as part of the migration id: legacyAttachmentId, serverId: "\(legacyAttachment.serverId)", variant: (legacyAttachment.attachmentType == .voiceMessage ? .voiceMessage : .standard), state: state, contentType: legacyAttachment.contentType, byteCount: UInt(legacyAttachment.byteCount), creationTimestamp: (legacyAttachment as? SMKLegacy._AttachmentStream)? .creationTimestamp.timeIntervalSince1970, sourceFilename: legacyAttachment.sourceFilename, downloadUrl: legacyAttachment.downloadURL, localRelativeFilePath: processedLocalRelativeFilePath, width: (size == .zero ? nil : UInt(size.width)), height: (size == .zero ? nil : UInt(size.height)), duration: duration, isValid: isValid, encryptionKey: legacyAttachment.encryptionKey, digest: { switch legacyAttachment { case let stream as SMKLegacy._AttachmentStream: return stream.digest case let pointer as SMKLegacy._AttachmentPointer: return pointer.digest default: return nil } }(), caption: legacyAttachment.caption ).migrationSafeInserted(db) processedAttachmentIds.insert(legacyAttachmentId) return legacyAttachmentId } private static func invalidAttachmentId( _ db: Database, for legacyAttachmentId: String, interactionVariant: Interaction.Variant? = nil, attachments: [String: SMKLegacy._Attachment], processedAttachmentIds: inout Set ) throws -> String { guard !processedAttachmentIds.contains(legacyAttachmentId) else { return legacyAttachmentId } _ = try Attachment( // Note: The legacy attachment object used a UUID string for it's id as well // and saved files using these id's so just used the existing id so we don't // need to bother renaming files as part of the migration id: legacyAttachmentId, serverId: nil, variant: .standard, state: .invalid, contentType: "", byteCount: 0, creationTimestamp: (TimeInterval(SnodeAPI.currentOffsetTimestampMs()) / 1000), sourceFilename: nil, downloadUrl: nil, localRelativeFilePath: nil, width: nil, height: nil, duration: nil, isValid: false, encryptionKey: nil, digest: nil, caption: nil ).migrationSafeInserted(db) processedAttachmentIds.insert(legacyAttachmentId) return legacyAttachmentId } private static func mapLegacyTypesForNSKeyedUnarchiver() { NSKeyedUnarchiver.setClass( SMKLegacy._Thread.self, forClassName: "TSThread" ) NSKeyedUnarchiver.setClass( SMKLegacy._ContactThread.self, forClassName: "TSContactThread" ) NSKeyedUnarchiver.setClass( SMKLegacy._GroupThread.self, forClassName: "TSGroupThread" ) NSKeyedUnarchiver.setClass( SMKLegacy._GroupModel.self, forClassName: "TSGroupModel" ) NSKeyedUnarchiver.setClass( SMKLegacy._OpenGroup.self, forClassName: "SNOpenGroupV2" ) NSKeyedUnarchiver.setClass( SMKLegacy._Contact.self, forClassName: "SNContact" ) NSKeyedUnarchiver.setClass( SMKLegacy._DBInteraction.self, forClassName: "TSInteraction" ) NSKeyedUnarchiver.setClass( SMKLegacy._DBMessage.self, forClassName: "TSMessage" ) NSKeyedUnarchiver.setClass( SMKLegacy._DBQuotedMessage.self, forClassName: "TSQuotedMessage" ) NSKeyedUnarchiver.setClass( SMKLegacy._DBQuotedMessage._DBAttachmentInfo.self, forClassName: "OWSAttachmentInfo" ) NSKeyedUnarchiver.setClass( SMKLegacy._DBLinkPreview.self, forClassName: "SessionServiceKit.OWSLinkPreview" // Very old legacy name ) NSKeyedUnarchiver.setClass( SMKLegacy._DBLinkPreview.self, forClassName: "SessionMessagingKit.OWSLinkPreview" ) NSKeyedUnarchiver.setClass( SMKLegacy._DBIncomingMessage.self, forClassName: "TSIncomingMessage" ) NSKeyedUnarchiver.setClass( SMKLegacy._DBOutgoingMessage.self, forClassName: "TSOutgoingMessage" ) NSKeyedUnarchiver.setClass( SMKLegacy._DBOutgoingMessageRecipientState.self, forClassName: "TSOutgoingMessageRecipientState" ) NSKeyedUnarchiver.setClass( SMKLegacy._DBInfoMessage.self, forClassName: "TSInfoMessage" ) NSKeyedUnarchiver.setClass( SMKLegacy._DisappearingConfigurationUpdateInfoMessage.self, forClassName: "OWSDisappearingConfigurationUpdateInfoMessage" ) NSKeyedUnarchiver.setClass( SMKLegacy._DataExtractionNotificationInfoMessage.self, forClassName: "SNDataExtractionNotificationInfoMessage" ) NSKeyedUnarchiver.setClass( SMKLegacy._Attachment.self, forClassName: "TSAttachment" ) NSKeyedUnarchiver.setClass( SMKLegacy._AttachmentStream.self, forClassName: "TSAttachmentStream" ) NSKeyedUnarchiver.setClass( SMKLegacy._AttachmentPointer.self, forClassName: "TSAttachmentPointer" ) NSKeyedUnarchiver.setClass( SMKLegacy._NotifyPNServerJob.self, forClassName: "SessionMessagingKit.NotifyPNServerJob" ) NSKeyedUnarchiver.setClass( SMKLegacy._NotifyPNServerJob._SnodeMessage.self, forClassName: "SessionSnodeKit.SnodeMessage" ) NSKeyedUnarchiver.setClass( SMKLegacy._MessageSendJob.self, forClassName: "SessionMessagingKit.SNMessageSendJob" ) NSKeyedUnarchiver.setClass( SMKLegacy._MessageReceiveJob.self, forClassName: "SessionMessagingKit.MessageReceiveJob" ) NSKeyedUnarchiver.setClass( SMKLegacy._AttachmentUploadJob.self, forClassName: "SessionMessagingKit.AttachmentUploadJob" ) NSKeyedUnarchiver.setClass( SMKLegacy._AttachmentDownloadJob.self, forClassName: "SessionMessagingKit.AttachmentDownloadJob" ) NSKeyedUnarchiver.setClass( SMKLegacy._Message.self, forClassName: "SNMessage" ) NSKeyedUnarchiver.setClass( SMKLegacy._VisibleMessage.self, forClassName: "SNVisibleMessage" ) NSKeyedUnarchiver.setClass( SMKLegacy._Quote.self, forClassName: "SNQuote" ) NSKeyedUnarchiver.setClass( SMKLegacy._LinkPreview.self, forClassName: "SNLinkPreview" ) NSKeyedUnarchiver.setClass( SMKLegacy._Profile.self, forClassName: "SNProfile" ) NSKeyedUnarchiver.setClass( SMKLegacy._OpenGroupInvitation.self, forClassName: "SNOpenGroupInvitation" ) NSKeyedUnarchiver.setClass( SMKLegacy._ControlMessage.self, forClassName: "SNControlMessage" ) NSKeyedUnarchiver.setClass( SMKLegacy._ReadReceipt.self, forClassName: "SNReadReceipt" ) NSKeyedUnarchiver.setClass( SMKLegacy._TypingIndicator.self, forClassName: "SNTypingIndicator" ) NSKeyedUnarchiver.setClass( SMKLegacy._ClosedGroupControlMessage.self, forClassName: "SessionMessagingKit.ClosedGroupControlMessage" ) NSKeyedUnarchiver.setClass( SMKLegacy._ClosedGroupControlMessage._KeyPairWrapper.self, forClassName: "ClosedGroupControlMessage.SNKeyPairWrapper" ) NSKeyedUnarchiver.setClass( SMKLegacy._DataExtractionNotification.self, forClassName: "SessionMessagingKit.DataExtractionNotification" ) NSKeyedUnarchiver.setClass( SMKLegacy._ExpirationTimerUpdate.self, forClassName: "SNExpirationTimerUpdate" ) NSKeyedUnarchiver.setClass( SMKLegacy._ConfigurationMessage.self, forClassName: "SNConfigurationMessage" ) NSKeyedUnarchiver.setClass( SMKLegacy._UnsendRequest.self, forClassName: "SNUnsendRequest" ) NSKeyedUnarchiver.setClass( SMKLegacy._MessageRequestResponse.self, forClassName: "SNMessageRequestResponse" ) NSKeyedUnarchiver.setClass( SMKLegacy._Contact._LegacyProfileKey.self, forClassName: "OWSAES256Key" ) } } fileprivate extension Int64 { static func zeroingOverflow(_ value: UInt64) -> Int64 { return (value > UInt64(Int64.max) ? 0 : Int64(value)) } }