Introduce conversation view mapping; rework conversation view scrolling.

Signal.xcodeproj/project.pbxproj

@@ -165,6 +165,7 @@
 		34A8B3512190A40E00218A25 /* MediaAlbumCellView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34A8B3502190A40E00218A25 /* MediaAlbumCellView.swift */; };
 		34ABB2C42090C59700C727A6 /* OWSResaveCollectionDBMigration.m in Sources */ = {isa = PBXBuildFile; fileRef = 34ABB2C22090C59600C727A6 /* OWSResaveCollectionDBMigration.m */; };
 		34ABB2C52090C59700C727A6 /* OWSResaveCollectionDBMigration.h in Headers */ = {isa = PBXBuildFile; fileRef = 34ABB2C32090C59700C727A6 /* OWSResaveCollectionDBMigration.h */; };
+		34ABC0E421DD20C500ED9469 /* ConversationMessageMapping.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34ABC0E321DD20C500ED9469 /* ConversationMessageMapping.swift */; };
 		34AC09DD211B39B100997B47 /* ViewControllerUtils.h in Headers */ = {isa = PBXBuildFile; fileRef = 34AC09BF211B39AE00997B47 /* ViewControllerUtils.h */; settings = {ATTRIBUTES = (Public, ); }; };
 		34AC09DE211B39B100997B47 /* OWSNavigationController.h in Headers */ = {isa = PBXBuildFile; fileRef = 34AC09C0211B39AE00997B47 /* OWSNavigationController.h */; settings = {ATTRIBUTES = (Public, ); }; };
 		34AC09DF211B39B100997B47 /* OWSNavigationController.m in Sources */ = {isa = PBXBuildFile; fileRef = 34AC09C1211B39AF00997B47 /* OWSNavigationController.m */; };
@@ -832,6 +833,7 @@
 		34A8B3502190A40E00218A25 /* MediaAlbumCellView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaAlbumCellView.swift; sourceTree = "<group>"; };
 		34ABB2C22090C59600C727A6 /* OWSResaveCollectionDBMigration.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = OWSResaveCollectionDBMigration.m; sourceTree = "<group>"; };
 		34ABB2C32090C59700C727A6 /* OWSResaveCollectionDBMigration.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OWSResaveCollectionDBMigration.h; sourceTree = "<group>"; };
+		34ABC0E321DD20C500ED9469 /* ConversationMessageMapping.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ConversationMessageMapping.swift; sourceTree = "<group>"; };
 		34AC09BF211B39AE00997B47 /* ViewControllerUtils.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ViewControllerUtils.h; sourceTree = "<group>"; };
 		34AC09C0211B39AE00997B47 /* OWSNavigationController.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OWSNavigationController.h; sourceTree = "<group>"; };
 		34AC09C1211B39AF00997B47 /* OWSNavigationController.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = OWSNavigationController.m; sourceTree = "<group>"; };
@@ -1613,6 +1615,7 @@
 				34D1F0681F8678AA0066283D /* ConversationInputTextView.m */,
 				34D1F0691F8678AA0066283D /* ConversationInputToolbar.h */,
 				34D1F06A1F8678AA0066283D /* ConversationInputToolbar.m */,
+				34ABC0E321DD20C500ED9469 /* ConversationMessageMapping.swift */,
 				343A65971FC4CFE7000477A1 /* ConversationScrollButton.h */,
 				343A65961FC4CFE6000477A1 /* ConversationScrollButton.m */,
 				34D1F06D1F8678AA0066283D /* ConversationViewController.h */,
@@ -3529,6 +3532,7 @@
 				34D8C0281ED3673300188D7C /* DebugUITableViewController.m in Sources */,
 				45F32C222057297A00A300D5 /* MediaDetailViewController.m in Sources */,
 				34B3F8851E8DF1700035BE1A /* NewGroupViewController.m in Sources */,
+				34ABC0E421DD20C500ED9469 /* ConversationMessageMapping.swift in Sources */,
 				34D8C0271ED3673300188D7C /* DebugUIMessages.m in Sources */,
 				34DBF003206BD5A500025978 /* OWSMessageTextView.m in Sources */,
 				34D1F0B41F86D31D0066283D /* ConversationCollectionView.m in Sources */,

Signal/src/ViewControllers/ConversationView/ConversationMessageMapping.swift

@@ -0,0 +1,324 @@
+//
+//  Copyright (c) 2019 Open Whisper Systems. All rights reserved.
+//
+
+import Foundation
+
+@objc
+public class ConversationMessageMapping: NSObject {
+    private let viewName: String
+    private let group: String?
+
+    // The desired number of the items to load BEFORE the pivot (see below).
+    @objc
+    public var desiredLength: UInt
+
+    typealias ItemId = String
+
+    // The list currently loaded items
+    private var itemIds = [ItemId]()
+
+    // We could use this to detect synchronization errors.
+    private var snapshotOfLastUpdate: UInt64?
+
+    // When we enter a conversation, we want to load up to N interactions. This
+    // in the "initial load window".
+    //
+    // We subsequently expand the load window in two directions using two very
+    // different behaviors.
+    //
+    // * We expand the load window "upwards" (backwards in time) only when
+    //   loadMore() is called, in "pages".
+    // * We auto-expand the load window "downwards" (forward in time) to include
+    //   any new interactions created after the initial load.
+    //
+    // We define the "pivot" as the last item in the initial load window.  This
+    // value is only set once.
+    //
+    // For example, if you enter a conversation with messages, 1..15:
+    //
+    // 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
+    //
+    // We initially load just the last 5 (if 5 is the initial desired length):
+    //
+    // 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
+    //                      |      pivot ^ | <-- load window
+    // pivot: 15, desired length=5.
+    //
+    // If a few more messages (16..18) are sent or received, we'll always load
+    // them immediately (they're after the pivot):
+    //
+    // 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
+    //                      |      pivot ^        | <-- load window
+    // pivot: 15, desired length=5.
+    //
+    // To load an additional page of items (perhaps due to user scrolling
+    // upward), we extend the desired length and thereby load more items
+    // before the pivot.
+    //
+    // 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
+    //           |                 pivot ^        | <-- load window
+    // pivot: 15, desired length=10.
+    //
+    // To reiterate:
+    //
+    // * The pivot doesn't move.
+    // * The desired length applies _before_ the pivot.
+    // * Everything after the pivot is auto-loaded.
+    //
+    // And note: we use the pivot's sort id, not its uniqueId, which works
+    // even if the pivot itself is deleted.
+    private var pivotSortId: UInt64?
+
+    @objc
+    public var canLoadMore = false
+
+    @objc
+    public required init(group: String?, desiredLength: UInt) {
+        self.viewName = TSMessageDatabaseViewExtensionName
+        self.group = group
+        self.desiredLength = desiredLength
+    }
+
+    @objc
+    public func loadedUniqueIds() -> [String] {
+        return itemIds
+    }
+
+    @objc
+    public func contains(uniqueId: String) -> Bool {
+        return loadedUniqueIds().contains(uniqueId)
+    }
+
+    // This method can be used to extend the desired length
+    // and update.
+    @objc
+    public func update(withDesiredLength desiredLength: UInt, transaction: YapDatabaseReadTransaction) {
+        assert(desiredLength >= self.desiredLength)
+
+        self.desiredLength = desiredLength
+
+        update(transaction: transaction)
+    }
+
+    // This is the core method of the class. It updates the state to
+    // reflect the latest database state & the current desired length.
+    @objc
+    public func update(transaction: YapDatabaseReadTransaction) {
+        AssertIsOnMainThread()
+
+        guard let view = transaction.ext(viewName) as? YapDatabaseAutoViewTransaction else {
+            owsFailDebug("Could not load view.")
+            return
+        }
+        guard let group = group else {
+            owsFailDebug("No group.")
+            return
+        }
+
+        // Deserializing interactions is expensive, so we only
+        // do that when necessary.
+        let sortIdForItemId: (String) -> UInt64? = { (itemId) in
+            guard let interaction = TSInteraction.fetch(uniqueId: itemId, transaction: transaction) else {
+                owsFailDebug("Could not load interaction.")
+                return nil
+            }
+            return interaction.sortId
+        }
+
+        // If we have a "pivot", load all items AFTER the pivot and up to minDesiredLength items BEFORE the pivot.
+        // If we do not have a "pivot", load up to minDesiredLength BEFORE the pivot.
+        var newItemIds = [ItemId]()
+        var canLoadMore = false
+        let desiredLength = self.desiredLength
+        // Not all items "count" towards the desired length. On an initial load, all items count.  Subsequently,
+        // only items above the pivot count.
+        var afterPivotCount: UInt = 0
+        var beforePivotCount: UInt = 0
+        // (void (^)(NSString *collection, NSString *key, id object, NSUInteger index, BOOL *stop))block;
+        view.enumerateKeys(inGroup: group, with: NSEnumerationOptions.reverse) { (_, key, _, stop) in
+            let itemId = key
+
+            // Load "uncounted" items after the pivot if possible.
+            //
+            // As an optimization, we can skip this check (which requires
+            // deserializing the interaction) if beforePivotCount is zero,
+            // e.g. we haven't "passed" the pivot yet.
+            if beforePivotCount == 0,
+                let pivotSortId = self.pivotSortId {
+                if let sortId = sortIdForItemId(itemId) {
+                    let isAfterPivot = sortId > pivotSortId
+                    if isAfterPivot {
+                        newItemIds.append(itemId)
+                        afterPivotCount += 1
+                        return
+                    }
+                } else {
+                    owsFailDebug("Could not determine sort id for interaction: \(itemId)")
+                }
+            }
+
+            // Load "counted" items unless the load window overflows.
+            if beforePivotCount >= desiredLength {
+                // Overflow
+                canLoadMore = true
+                stop.pointee = true
+            } else {
+                newItemIds.append(itemId)
+                beforePivotCount += 1
+            }
+        }
+
+        // The items need to be reversed, since we load them in reverse order.
+        self.itemIds = Array(newItemIds.reversed())
+        self.canLoadMore = canLoadMore
+        self.snapshotOfLastUpdate = transaction.connection.snapshot
+
+        // Establish the pivot, if necessary and possible.
+        //
+        // Deserializing interactions is expensive. We only need to deserialize
+        // interactions that are "after" the pivot.  So there would be performance
+        // benefits to moving the pivot after each update to the last loaded item.
+        //
+        // However, this would undesirable side effects. The desired length for
+        // conversations with very short disappearing message durations would
+        // continuously grow as messages appeared and disappeared.
+        //
+        // NOTE: if we do end up "moving" the pivot, we also need to increment
+        // `self.desiredLength` by `afterPivotCount`.
+        if self.pivotSortId == nil {
+            if let newLastItemId = newItemIds.first {
+                // newItemIds is in reverse order, so its "first" element is actually last.
+                if let sortId = sortIdForItemId(newLastItemId) {
+                    // Update the pivot.
+                    self.pivotSortId = sortId
+                } else {
+                    owsFailDebug("Could not determine sort id for interaction: \(newLastItemId)")
+                }
+            }
+        }
+    }
+
+    // Tries to ensure that the load window includes a given item.
+    // On success, returns the index path of that item.
+    // On failure, returns nil.
+    @objc
+    public func ensureLoadWindowContains(uniqueId: String,
+                                         transaction: YapDatabaseReadTransaction) -> IndexPath? {
+        if let oldIndex = loadedUniqueIds().firstIndex(of: uniqueId) {
+            return IndexPath(row: oldIndex, section: 0)
+        }
+        guard let view = transaction.ext(viewName) as? YapDatabaseAutoViewTransaction else {
+            owsFailDebug("Could not load view.")
+            return nil
+        }
+        guard let group = group else {
+            owsFailDebug("No group.")
+            return nil
+        }
+
+        let indexPtr: UnsafeMutablePointer<UInt> = UnsafeMutablePointer<UInt>.allocate(capacity: 1)
+        let wasFound = view.getGroup(nil, index: indexPtr, forKey: uniqueId, inCollection: TSInteraction.collection())
+        guard wasFound else {
+            owsFailDebug("Could not find interaction.")
+            return nil
+        }
+        let index = indexPtr.pointee
+        let threadInteractionCount = view.numberOfItems(inGroup: group)
+        guard index < threadInteractionCount else {
+            owsFailDebug("Invalid index.")
+            return nil
+        }
+        // This math doesn't take into account the number of items loaded _after_ the pivot.
+        // That's fine; it's okay to load too many interactions here.
+        let desiredWindowSize: UInt = threadInteractionCount - index
+        self.update(withDesiredLength: desiredWindowSize, transaction: transaction)
+
+        guard let newIndex = loadedUniqueIds().firstIndex(of: uniqueId) else {
+            owsFailDebug("Couldn't find interaction.")
+            return nil
+        }
+        return IndexPath(row: newIndex, section: 0)
+    }
+
+    @objc
+    public class ConversationMessageMappingDiff: NSObject {
+        @objc
+        public let addedItemIds: Set<String>
+        @objc
+        public let removedItemIds: Set<String>
+        @objc
+        public let updatedItemIds: Set<String>
+
+        init(addedItemIds: Set<String>, removedItemIds: Set<String>, updatedItemIds: Set<String>) {
+            self.addedItemIds = addedItemIds
+            self.removedItemIds = removedItemIds
+            self.updatedItemIds = updatedItemIds
+        }
+    }
+
+    // Updates and then calculates which items were inserted, removed or modified.
+    @objc
+    public func updateAndCalculateDiff(transaction: YapDatabaseReadTransaction,
+                                       notifications: [NSNotification]) -> ConversationMessageMappingDiff? {
+        let oldItemIds = Set(self.itemIds)
+        self.update(transaction: transaction)
+        let newItemIds = Set(self.itemIds)
+
+        let removedItemIds = oldItemIds.subtracting(newItemIds)
+        let addedItemIds = newItemIds.subtracting(oldItemIds)
+        // We only notify for updated items that a) were previously loaded b) weren't also inserted or removed.
+        let updatedItemIds = (self.updatedItemIds(for: notifications)
+            .subtracting(addedItemIds)
+            .subtracting(removedItemIds)
+            .intersection(oldItemIds))
+
+        return ConversationMessageMappingDiff(addedItemIds: addedItemIds,
+                                              removedItemIds: removedItemIds,
+                                              updatedItemIds: updatedItemIds)
+    }
+
+    // For performance reasons, the database modification notifications are used
+    // to determine which items were modified.  If YapDatabase ever changes the
+    // structure or semantics of these notifications, we'll need to update this
+    // code to reflect that.
+    private func updatedItemIds(for notifications: [NSNotification]) -> Set<String> {
+        var updatedItemIds = Set<String>()
+        for notification in notifications {
+            // Unpack the YDB notification, looking for row changes.
+            guard let userInfo =
+                notification.userInfo else {
+                    owsFailDebug("Missing userInfo.")
+                    continue
+            }
+            guard let viewChangesets =
+                userInfo[YapDatabaseExtensionsKey] as? NSDictionary else {
+                    // No changes for any views, skip.
+                    continue
+            }
+            guard let changeset =
+                viewChangesets[viewName] as? NSDictionary else {
+                    // No changes for this view, skip.
+                    continue
+            }
+            // This constant matches a private constant in YDB.
+            let changeset_key_changes: String = "changes"
+            guard let changesetChanges = changeset[changeset_key_changes] as? [Any] else {
+                owsFailDebug("Missing changeset changes.")
+                continue
+            }
+            for change in changesetChanges {
+                if change as? YapDatabaseViewSectionChange != nil {
+                    // Ignore.
+                } else if let rowChange = change as? YapDatabaseViewRowChange {
+                    updatedItemIds.insert(rowChange.collectionKey.key)
+                } else {
+                    owsFailDebug("Invalid change: \(type(of: change)).")
+                    continue
+                }
+            }
+        }
+
+        return updatedItemIds
+    }
+}

Signal/src/ViewControllers/ConversationView/ConversationViewController.m

@@ -196,6 +196,7 @@ typedef enum : NSUInteger {
 @property (nonatomic, nullable) NSDate *lastReloadDate;
 
 @property (nonatomic) CGFloat scrollDistanceToBottomSnapshot;
+@property (nonatomic, nullable) NSNumber *lastKnownDistanceFromBottom;
 
 @end
 
@@ -2954,36 +2955,6 @@ typedef enum : NSUInteger {
     return OWSPrimaryStorage.sharedManager.dbReadWriteConnection;
 }
 
-- (BOOL)isScrolledToBottom
-{
-    CGFloat contentHeight = self.safeContentHeight;
-
-    // This is a bit subtle.
-    //
-    // The _wrong_ way to determine if we're scrolled to the bottom is to
-    // measure whether the collection view's content is "near" the bottom edge
-    // of the collection view.  This is wrong because the collection view
-    // might not have enough content to fill the collection view's bounds
-    // _under certain conditions_ (e.g. with the keyboard dismissed).
-    //
-    // What we're really interested in is something a bit more subtle:
-    // "Is the scroll view scrolled down as far as it can, "at rest".
-    //
-    // To determine that, we find the appropriate "content offset y" if
-    // the scroll view were scrolled down as far as possible.  IFF the
-    // actual "content offset y" is "near" that value, we return YES.
-    const CGFloat kIsAtBottomTolerancePts = 5;
-    // Note the usage of MAX() to handle the case where there isn't enough
-    // content to fill the collection view at its current size.
-    CGFloat contentOffsetYBottom
-        = MAX(0.f, contentHeight + self.collectionView.contentInset.bottom - self.collectionView.bounds.size.height);
-
-    CGFloat distanceFromBottom = contentOffsetYBottom - self.collectionView.contentOffset.y;
-    BOOL isScrolledToBottom = distanceFromBottom <= kIsAtBottomTolerancePts;
-
-    return isScrolledToBottom;
-}
-
 #pragma mark - Audio
 
 - (void)requestRecordingVoiceMemo
@@ -3774,6 +3745,13 @@ typedef enum : NSUInteger {
 
 - (void)scrollViewDidScroll:(UIScrollView *)scrollView
 {
+    // Constantly update the lastKnownDistanceFromBottom,
+    // unless we're presenting the menu actions which
+    // temporarily gmeddles with the content insets.
+    if (!OWSWindowManager.sharedManager.isPresentingMenuActions) {
+        self.lastKnownDistanceFromBottom = @(self.safeDistanceFromBottom);
+    }
+
     [self updateLastVisibleSortId];
 
     __weak ConversationViewController *weakSelf = self;
@@ -4270,6 +4248,66 @@ typedef enum : NSUInteger {
     conversationViewCell.isCellVisible = NO;
 }
 
+// We use this hook to ensure scroll state continuity.  As the collection
+// view's content size changes, we want to keep the same cells in view.
+- (CGPoint)collectionView:(UICollectionView *)collectionView
+    targetContentOffsetForProposedContentOffset:(CGPoint)proposedContentOffset
+{
+    if (self.lastKnownDistanceFromBottom) {
+        // Adjust the content offset to reflect the "last known" distance
+        // from the bottom of the content.
+        CGFloat contentOffsetYBottom = self.maxContentOffsetY;
+        CGFloat contentOffsetY = contentOffsetYBottom - self.lastKnownDistanceFromBottom.floatValue;
+        proposedContentOffset.y = contentOffsetY;
+    }
+
+    return proposedContentOffset;
+}
+
+#pragma mark - Scroll State
+
+- (BOOL)isScrolledToBottom
+{
+    CGFloat distanceFromBottom = self.safeDistanceFromBottom;
+    const CGFloat kIsAtBottomTolerancePts = 5;
+    BOOL isScrolledToBottom = distanceFromBottom <= kIsAtBottomTolerancePts;
+    return isScrolledToBottom;
+}
+
+- (CGFloat)safeDistanceFromBottom
+{
+    // This is a bit subtle.
+    //
+    // The _wrong_ way to determine if we're scrolled to the bottom is to
+    // measure whether the collection view's content is "near" the bottom edge
+    // of the collection view.  This is wrong because the collection view
+    // might not have enough content to fill the collection view's bounds
+    // _under certain conditions_ (e.g. with the keyboard dismissed).
+    //
+    // What we're really interested in is something a bit more subtle:
+    // "Is the scroll view scrolled down as far as it can, "at rest".
+    //
+    // To determine that, we find the appropriate "content offset y" if
+    // the scroll view were scrolled down as far as possible.  IFF the
+    // actual "content offset y" is "near" that value, we return YES.
+    CGFloat contentOffsetYBottom = self.maxContentOffsetY;
+    CGFloat distanceFromBottom = contentOffsetYBottom - self.collectionView.contentOffset.y;
+    return distanceFromBottom;
+}
+
+- (CGFloat)maxContentOffsetY
+{
+    CGFloat contentHeight = self.safeContentHeight;
+
+    // Note the usage of MAX() to handle the case where there isn't enough
+    // content to fill the collection view at its current size.
+    CGFloat contentOffsetYBottom
+        = MAX(0.f, contentHeight + self.collectionView.contentInset.bottom - self.collectionView.bounds.size.height);
+    //    OWSLogVerbose(@"self.collectionView.contentInset: %@",
+    //    NSStringFromUIEdgeInsets(self.collectionView.contentInset));
+    return contentOffsetYBottom;
+}
+
 #pragma mark - ContactsPickerDelegate
 
 - (void)contactsPickerDidCancel:(ContactsPicker *)contactsPicker
@@ -4430,6 +4468,8 @@ typedef enum : NSUInteger {
 
     OWSLogVerbose(@"");
 
+    // TODO:
+    //
     // HACK to work around radar #28167779
     // "UICollectionView performBatchUpdates can trigger a crash if the collection view is flagged for layout"
     // more: https://github.com/PSPDFKit-labs/radar.apple.com/tree/master/28167779%20-%20CollectionViewBatchingIssue
@@ -4471,23 +4511,15 @@ typedef enum : NSUInteger {
     } else if (conversationUpdate.conversationUpdateType == ConversationUpdateType_Reload) {
         [self resetContentAndLayout];
         [self updateLastVisibleSortId];
-        [self scrollToBottomAnimated:NO];
         return;
     }
 
     OWSAssertDebug(conversationUpdate.conversationUpdateType == ConversationUpdateType_Diff);
     OWSAssertDebug(conversationUpdate.updateItems);
 
-    BOOL wasAtBottom = [self isScrolledToBottom];
+    // We want to auto-scroll to the bottom of the conversation
-    // We want sending messages to feel snappy.  So, if the only
+    // if the user is inserting new interactions.
-    // update is a new outgoing message AND we're already scrolled to
+    __block BOOL scrollToBottom = NO;
-    // the bottom of the conversation, skip the scroll animation.
-    __block BOOL shouldAnimateScrollToBottom = !wasAtBottom;
-    // We want to scroll to the bottom if the user:
-    //
-    // a) already was at the bottom of the conversation.
-    // b) is inserting new interactions.
-    __block BOOL scrollToBottom = wasAtBottom;
 
     void (^batchUpdates)(void) = ^{
         OWSAssertIsOnMainThread();
@@ -4518,7 +4550,6 @@ typedef enum : NSUInteger {
                         TSOutgoingMessage *outgoingMessage = (TSOutgoingMessage *)viewItem.interaction;
                         if (!outgoingMessage.isFromLinkedDevice) {
                             scrollToBottom = YES;
-                            shouldAnimateScrollToBottom = NO;
                         }
                     }
 
@@ -4545,8 +4576,8 @@ typedef enum : NSUInteger {
 
         [self updateLastVisibleSortId];
 
-        if (scrollToBottom && shouldAnimateUpdates) {
+        if (scrollToBottom) {
-            [self scrollToBottomAnimated:shouldAnimateScrollToBottom];
+            [self scrollToBottomAnimated:NO];
         }
     };
 
@@ -4659,6 +4690,14 @@ typedef enum : NSUInteger {
     [self updateShowLoadMoreHeader];
 }
 
+- (void)conversationViewModelDidReset
+{
+    OWSAssertIsOnMainThread();
+
+    // Scroll to bottom to get view back to a known good state.
+    [self scrollToBottomAnimated:NO];
+}
+
 @end
 
 NS_ASSUME_NONNULL_END

Signal/src/ViewControllers/ConversationView/ConversationViewModel.h

@@ -1,5 +1,5 @@
 //
-//  Copyright (c) 2018 Open Whisper Systems. All rights reserved.
+//  Copyright (c) 2019 Open Whisper Systems. All rights reserved.
 //
 
 NS_ASSUME_NONNULL_BEGIN
@@ -70,6 +70,8 @@ typedef NS_ENUM(NSUInteger, ConversationUpdateItemType) {
 - (void)conversationViewModelDidLoadPrevPage;
 - (void)conversationViewModelRangeDidChange;
 
+- (void)conversationViewModelDidReset;
+
 - (BOOL)isObservingVMUpdates;
 
 - (ConversationStyle *)conversationStyle;

Signal/src/ViewControllers/ConversationView/ConversationViewModel.m

@@ -1,5 +1,5 @@
 //
-//  Copyright (c) 2018 Open Whisper Systems. All rights reserved.
+//  Copyright (c) 2019 Open Whisper Systems. All rights reserved.
 //
 
 #import "ConversationViewModel.h"
@@ -128,8 +128,6 @@ static const int kConversationInitialMaxRangeSize = 300;
 // Never show more than n messages in conversation view at a time.
 static const int kYapDatabaseRangeMaxLength = 25000;
 
-static const int kYapDatabaseRangeMinLength = 0;
-
 #pragma mark -
 
 @interface ConversationViewModel ()
@@ -141,24 +139,20 @@ static const int kYapDatabaseRangeMinLength = 0;
 // The mapping must be updated in lockstep with the uiDatabaseConnection.
 //
 // * The first (required) step is to update uiDatabaseConnection using beginLongLivedReadTransaction.
-// * The second (required) step is to update messageMappings.
+// * The second (required) step is to update messageMapping. The desired length of the mapping
-// * The third (optional) step is to update the messageMappings range using
+//   can be modified at this time.
-//   updateMessageMappingRangeOptions.
+// * The third (optional) step is to update the view items using reloadViewItems.
-// * The fourth (optional) step is to update the view items using reloadViewItems.
 // * The steps must be done in strict order.
 // * If we do any of the steps, we must do all of the required steps.
-// * We can't use messageMappings or viewItems after the first step until we've
+// * We can't use messageMapping or viewItems after the first step until we've
 //   done the last step; i.e.. we can't do any layout, since that uses the view
 //   items which haven't been updated yet.
-// * If the first and/or second steps changes the set of messages
+// * Afterward, we must prod the view controller to update layout & view state.
-//   their ordering and/or their state, we must do the third and fourth steps.
+@property (nonatomic) ConversationMessageMapping *messageMapping;
-// * If we do the third step, we must call resetContentAndLayout afterward.
-@property (nonatomic) YapDatabaseViewMappings *messageMappings;
 
 @property (nonatomic) NSArray<id<ConversationViewItem>> *viewItems;
 @property (nonatomic) NSMutableDictionary<NSString *, id<ConversationViewItem>> *viewItemCache;
 
-@property (nonatomic) NSUInteger lastRangeLength;
 @property (nonatomic, nullable) ThreadDynamicInteractions *dynamicInteractions;
 @property (nonatomic) BOOL hasClearedUnreadMessagesIndicator;
 @property (nonatomic, nullable) NSDate *collapseCutoffDate;
@@ -311,13 +305,13 @@ static const int kYapDatabaseRangeMinLength = 0;
 
     // We need to update the "unread indicator" _before_ we determine the initial range
     // size, since it depends on where the unread indicator is placed.
-    self.lastRangeLength = 0;
     self.typingIndicatorsSender = [self.typingIndicators typingRecipientIdForThread:self.thread];
+    self.collapseCutoffDate = [NSDate new];
 
     [self ensureDynamicInteractions];
     [self.primaryStorage updateUIDatabaseConnectionToLatest];
 
-    [self createNewMessageMappings];
+    [self createNewMessageMapping];
     if (![self reloadViewItems]) {
         OWSFailDebug(@"failed to reload view items in configureForThread.");
     }
@@ -334,6 +328,10 @@ static const int kYapDatabaseRangeMinLength = 0;
                                              selector:@selector(uiDatabaseDidUpdate:)
                                                  name:OWSUIDatabaseConnectionDidUpdateNotification
                                                object:self.primaryStorage.dbNotificationObject];
+    [[NSNotificationCenter defaultCenter] addObserver:self
+                                             selector:@selector(applicationWillEnterForeground:)
+                                                 name:OWSApplicationWillEnterForegroundNotification
+                                               object:nil];
 }
 
 - (void)viewDidLoad
@@ -348,25 +346,11 @@ static const int kYapDatabaseRangeMinLength = 0;
 
 - (BOOL)canLoadMoreItems
 {
-    __block BOOL result;
+    if (self.messageMapping.desiredLength >= kYapDatabaseRangeMaxLength) {
-    [self.uiDatabaseConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) {
-        YapDatabaseViewTransaction *messageDatabaseView = [transaction ext:TSMessageDatabaseViewExtensionName];
-        result = [self canLoadMoreItems:messageDatabaseView];
-    }];
-    return result;
-}
-
-- (BOOL)canLoadMoreItems:(YapDatabaseViewTransaction *)messageDatabaseView
-{
-    OWSAssertDebug(messageDatabaseView);
-
-    if (self.lastRangeLength >= kYapDatabaseRangeMaxLength) {
         return NO;
     }
 
-    NSUInteger loadWindowSize = [self.messageMappings numberOfItemsInGroup:self.thread.uniqueId];
+    return self.messageMapping.canLoadMore;
-    NSUInteger totalMessageCount = [messageDatabaseView numberOfItemsInGroup:self.thread.uniqueId];
-    return loadWindowSize < totalMessageCount;
 }
 
 - (void)applicationDidEnterBackground:(NSNotification *)notification
@@ -400,7 +384,7 @@ static const int kYapDatabaseRangeMinLength = 0;
         // Ensure view items are updated before trying to scroll to the
         // unread indicator.
         //
-        // loadNMoreMessages calls resetMappings which calls ensureDynamicInteractions,
+        // loadNMoreMessages calls resetMapping which calls ensureDynamicInteractions,
         // which may move the unread indicator, and for scrollToUnreadIndicatorAnimated
         // to work properly, the view items need to be updated to reflect that change.
         [self.primaryStorage updateUIDatabaseConnectionToLatest];
@@ -413,65 +397,61 @@ static const int kYapDatabaseRangeMinLength = 0;
 {
     [self.delegate conversationViewModelWillLoadMoreItems];
 
-    self.lastRangeLength = MIN(self.lastRangeLength + numberOfMessagesToLoad, (NSUInteger)kYapDatabaseRangeMaxLength);
+    [self resetMappingWithAdditionalLength:numberOfMessagesToLoad];
-
-    [self resetMappings];
 
     [self.delegate conversationViewModelDidLoadMoreItems];
 }
 
-- (void)updateMessageMappingRangeOptions
+- (NSUInteger)initialMessageMappingLength
 {
-    NSUInteger rangeLength = 0;
+    NSUInteger rangeLength = kYapDatabasePageSize;
 
-    if (self.lastRangeLength == 0) {
+    // If this is the first time we're configuring the range length,
-        // If this is the first time we're configuring the range length,
+    // try to take into account the position of the unread indicator
-        // try to take into account the position of the unread indicator
+    // and the "focus message".
-        // and the "focus message".
+    OWSAssertDebug(self.dynamicInteractions);
-        OWSAssertDebug(self.dynamicInteractions);
 
-        if (self.focusMessageIdOnOpen) {
+    if (self.focusMessageIdOnOpen) {
-            OWSAssertDebug(self.dynamicInteractions.focusMessagePosition);
+        OWSAssertDebug(self.dynamicInteractions.focusMessagePosition);
-            if (self.dynamicInteractions.focusMessagePosition) {
+        if (self.dynamicInteractions.focusMessagePosition) {
-                OWSLogVerbose(@"ensuring load of focus message: %@", self.dynamicInteractions.focusMessagePosition);
+            OWSLogVerbose(@"ensuring load of focus message: %@", self.dynamicInteractions.focusMessagePosition);
-                rangeLength = MAX(rangeLength, 1 + self.dynamicInteractions.focusMessagePosition.unsignedIntegerValue);
+            rangeLength = MAX(rangeLength, 1 + self.dynamicInteractions.focusMessagePosition.unsignedIntegerValue);
-            }
-        }
-
-        if (self.dynamicInteractions.unreadIndicator) {
-            NSUInteger unreadIndicatorPosition
-                = (NSUInteger)self.dynamicInteractions.unreadIndicator.unreadIndicatorPosition;
-
-            // If there is an unread indicator, increase the initial load window
-            // to include it.
-            OWSAssertDebug(unreadIndicatorPosition > 0);
-            OWSAssertDebug(unreadIndicatorPosition <= kYapDatabaseRangeMaxLength);
-
-            // We'd like to include at least N seen messages,
-            // to give the user the context of where they left off the conversation.
-            const NSUInteger kPreferredSeenMessageCount = 1;
-            rangeLength = MAX(rangeLength, unreadIndicatorPosition + kPreferredSeenMessageCount);
         }
     }
 
+    if (self.dynamicInteractions.unreadIndicator) {
+        NSUInteger unreadIndicatorPosition
+            = (NSUInteger)self.dynamicInteractions.unreadIndicator.unreadIndicatorPosition;
+
+        // If there is an unread indicator, increase the initial load window
+        // to include it.
+        OWSAssertDebug(unreadIndicatorPosition > 0);
+        OWSAssertDebug(unreadIndicatorPosition <= kYapDatabaseRangeMaxLength);
+
+        // We'd like to include at least N seen messages,
+        // to give the user the context of where they left off the conversation.
+        const NSUInteger kPreferredSeenMessageCount = 1;
+        rangeLength = MAX(rangeLength, unreadIndicatorPosition + kPreferredSeenMessageCount);
+    }
+
+    return rangeLength;
+}
+
+- (void)updateMessageMappingWithAdditionalLength:(NSUInteger)additionalLength
+{
+    // Range size should monotonically increase.
+    NSUInteger rangeLength = self.messageMapping.desiredLength + additionalLength;
+
     // Always try to load at least a single page of messages.
     rangeLength = MAX(rangeLength, kYapDatabasePageSize);
 
-    // Range size should monotonically increase.
-    rangeLength = MAX(rangeLength, self.lastRangeLength);
-
     // Enforce max range size.
     rangeLength = MIN(rangeLength, kYapDatabaseRangeMaxLength);
 
-    self.lastRangeLength = rangeLength;
+    [self.uiDatabaseConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) {
+        [self.messageMapping updateWithDesiredLength:rangeLength transaction:transaction];
+    }];
 
-    YapDatabaseViewRangeOptions *rangeOptions =
-        [YapDatabaseViewRangeOptions flexibleRangeWithLength:rangeLength offset:0 from:YapDatabaseViewEnd];
-
-    rangeOptions.maxLength = MAX(rangeLength, kYapDatabaseRangeMaxLength);
-    rangeOptions.minLength = kYapDatabaseRangeMinLength;
-
-    [self.messageMappings setRangeOptions:rangeOptions forGroup:self.thread.uniqueId];
     [self.delegate conversationViewModelRangeDidChange];
     self.collapseCutoffDate = [NSDate new];
 }
@@ -480,7 +460,7 @@ static const int kYapDatabaseRangeMinLength = 0;
 {
     OWSAssertIsOnMainThread();
 
-    const int currentMaxRangeSize = (int)self.lastRangeLength;
+    const int currentMaxRangeSize = (int)self.messageMapping.desiredLength;
     const int maxRangeSize = MAX(kConversationInitialMaxRangeSize, currentMaxRangeSize);
 
     self.dynamicInteractions = [ThreadUtil ensureDynamicInteractionsForThread:self.thread
@@ -547,13 +527,8 @@ static const int kYapDatabaseRangeMinLength = 0;
         return;
     }
 
-    // External database modifications can't be converted into incremental updates,
+    // External database modifications (e.g. changes from another process such as the SAE)
-    // so rebuild everything.  This is expensive and usually isn't necessary, but
+    // are "flushed" using touchDbAsync when the app re-enters the foreground.
-    // there's no alternative.
-    //
-    // We don't need to do this if we're not observing db modifications since we'll
-    // do it when we resume.
-    [self resetMappings];
 }
 
 - (void)uiDatabaseWillUpdate:(NSNotification *)notification
@@ -570,7 +545,7 @@ static const int kYapDatabaseRangeMinLength = 0;
 
     OWSLogVerbose(@"");
 
-    NSArray *notifications = notification.userInfo[OWSUIDatabaseConnectionNotificationsKey];
+    NSArray<NSNotification *> *notifications = notification.userInfo[OWSUIDatabaseConnectionNotificationsKey];
     OWSAssertDebug([notifications isKindOfClass:[NSArray class]]);
 
     YapDatabaseAutoViewConnection *messageDatabaseView =
@@ -578,54 +553,46 @@ static const int kYapDatabaseRangeMinLength = 0;
     OWSAssertDebug([messageDatabaseView isKindOfClass:[YapDatabaseAutoViewConnection class]]);
     if (![messageDatabaseView hasChangesForGroup:self.thread.uniqueId inNotifications:notifications]) {
         [self.uiDatabaseConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) {
-            [self.messageMappings updateWithTransaction:transaction];
+            [self.messageMapping updateWithTransaction:transaction];
         }];
 
         [self.delegate conversationViewModelDidUpdate:ConversationUpdate.minorUpdate];
         return;
     }
 
-    NSArray<YapDatabaseViewSectionChange *> *sectionChanges = nil;
+    __block ConversationMessageMappingDiff *_Nullable diff = nil;
-    NSArray<YapDatabaseViewRowChange *> *rowChanges = nil;
+    [self.uiDatabaseConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) {
-    [messageDatabaseView getSectionChanges:&sectionChanges
+        diff = [self.messageMapping updateAndCalculateDiffWithTransaction:transaction notifications:notifications];
-                                rowChanges:&rowChanges
+    }];
-                          forNotifications:notifications
+    if (!diff) {
-                              withMappings:self.messageMappings];
+        OWSFailDebug(@"Could not determine diff");
-
+        // resetMapping will call delegate.conversationViewModelDidUpdate.
-    if ([sectionChanges count] == 0 && [rowChanges count] == 0) {
+        [self resetMapping];
-        // YapDatabase will ignore insertions within the message mapping's
+        [self.delegate conversationViewModelDidReset];
-        // range that are not within the current mapping's contents.  We
+        return;
-        // may need to extend the mapping's contents to reflect the current
+    }
-        // range.
+    if (diff.addedItemIds.count < 1 && diff.removedItemIds.count < 1 && diff.updatedItemIds.count < 1) {
-        [self updateMessageMappingRangeOptions];
+        OWSFailDebug(@"Unexpectedly empty diff.");
-        return [self.delegate conversationViewModelDidUpdate:ConversationUpdate.minorUpdate];
     }
 
+    NSMutableSet<NSString *> *diffAddedItemIds = [diff.addedItemIds mutableCopy];
+    NSMutableSet<NSString *> *diffRemovedItemIds = [diff.removedItemIds mutableCopy];
+    NSMutableSet<NSString *> *diffUpdatedItemIds = [diff.updatedItemIds mutableCopy];
     for (TSOutgoingMessage *unsavedOutgoingMessage in self.unsavedOutgoingMessages) {
         // unsavedOutgoingMessages should only exist for a short period (usually 30-50ms) before
         // they are saved and moved into the `persistedViewItems`
         OWSAssertDebug(unsavedOutgoingMessage.timestamp >= ([NSDate ows_millisecondTimeStamp] - 1 * kSecondInMs));
-        NSUInteger index = [rowChanges indexOfObjectPassingTest:^BOOL(
-            YapDatabaseViewRowChange *_Nonnull rowChange, NSUInteger idx, BOOL *_Nonnull stop) {
-            return [rowChange.collectionKey.key isEqualToString:unsavedOutgoingMessage.uniqueId];
-        }];
 
-        if (index != NSNotFound) {
+        BOOL isFound = ([diff.addedItemIds containsObject:unsavedOutgoingMessage.uniqueId] ||
-            // Replace the "Insert" RowChange to be an "Update" RowChange.
+            [diff.removedItemIds containsObject:unsavedOutgoingMessage.uniqueId] ||
-            YapDatabaseViewRowChange *rowChange = rowChanges[index];
+            [diff.updatedItemIds containsObject:unsavedOutgoingMessage.uniqueId]);
-            OWSAssertDebug(rowChange);
+        if (isFound) {
-
+            // Convert the "insert" to an "update".
-            OWSLogVerbose(@"unsaved item has since been saved. collection key: %@", rowChange.collectionKey.key);
+            if ([diffAddedItemIds containsObject:unsavedOutgoingMessage.uniqueId]) {
-
+                OWSLogVerbose(@"Converting insert to update: %@", unsavedOutgoingMessage.uniqueId);
-            YapDatabaseViewRowChange *update =
+                [diffAddedItemIds removeObject:unsavedOutgoingMessage.uniqueId];
-                [YapDatabaseViewRowChange updateCollectionKey:rowChange.collectionKey
+                [diffUpdatedItemIds removeObject:unsavedOutgoingMessage.uniqueId];
-                                                      inGroup:rowChange.originalGroup
+            }
-                                                      atIndex:rowChange.finalIndex
-                                                  withChanges:YapDatabaseViewChangedObject];
-
-            NSMutableArray<YapDatabaseViewRowChange *> *mutableRowChanges = [rowChanges mutableCopy];
-            mutableRowChanges[index] = update;
-            rowChanges = [mutableRowChanges copy];
 
             // Remove the unsavedOutgoingViewItem since it now exists as a persistedViewItem
             NSMutableArray<TSOutgoingMessage *> *unsavedOutgoingMessages = [self.unsavedOutgoingMessages mutableCopy];
@@ -643,58 +610,36 @@ static const int kYapDatabaseRangeMinLength = 0;
     // reloadViewItems.
     BOOL hasMalformedRowChange = NO;
     NSMutableSet<NSString *> *updatedItemSet = [NSMutableSet new];
-    for (YapDatabaseViewRowChange *rowChange in rowChanges) {
+    for (NSString *uniqueId in diffUpdatedItemIds) {
-        switch (rowChange.type) {
+        id<ConversationViewItem> _Nullable viewItem = self.viewItemCache[uniqueId];
-            case YapDatabaseViewChangeUpdate: {
+        if (viewItem) {
-                YapCollectionKey *collectionKey = rowChange.collectionKey;
+            [self reloadInteractionForViewItem:viewItem];
-                if (collectionKey.key) {
+            [updatedItemSet addObject:viewItem.itemId];
-                    id<ConversationViewItem> _Nullable viewItem = self.viewItemCache[collectionKey.key];
+        } else {
-                    if (viewItem) {
+            OWSFailDebug(@"Update is missing view item");
-                        [self reloadInteractionForViewItem:viewItem];
+            hasMalformedRowChange = YES;
-                        [updatedItemSet addObject:viewItem.itemId];
-                    } else {
-                        OWSFailDebug(@"Update is missing view item");
-                        hasMalformedRowChange = YES;
-                    }
-                } else {
-                    OWSFailDebug(@"Update is missing collection key");
-                    hasMalformedRowChange = YES;
-                }
-                break;
-            }
-            case YapDatabaseViewChangeDelete: {
-                // Discard cached view items after deletes.
-                YapCollectionKey *collectionKey = rowChange.collectionKey;
-                if (collectionKey.key) {
-                    [self.viewItemCache removeObjectForKey:collectionKey.key];
-                } else {
-                    OWSFailDebug(@"Delete is missing collection key");
-                    hasMalformedRowChange = YES;
-                }
-                break;
-            }
-            default:
-                break;
-        }
-        if (hasMalformedRowChange) {
-            break;
         }
     }
+    for (NSString *uniqueId in diffRemovedItemIds) {
+        [self.viewItemCache removeObjectForKey:uniqueId];
+    }
 
     if (hasMalformedRowChange) {
         // These errors seems to be very rare; they can only be reproduced
         // using the more extreme actions in the debug UI.
         OWSFailDebug(@"hasMalformedRowChange");
-        // resetMappings will call delegate.conversationViewModelDidUpdate.
+        // resetMapping will call delegate.conversationViewModelDidUpdate.
-        [self resetMappings];
+        [self resetMapping];
+        [self.delegate conversationViewModelDidReset];
         return;
     }
 
     if (![self reloadViewItems]) {
         // These errors are rare.
-        OWSFailDebug(@"could not reload view items; hard resetting message mappings.");
+        OWSFailDebug(@"could not reload view items; hard resetting message mapping.");
-        // resetMappings will call delegate.conversationViewModelDidUpdate.
+        // resetMapping will call delegate.conversationViewModelDidUpdate.
-        [self resetMappings];
+        [self resetMapping];
+        [self.delegate conversationViewModelDidReset];
         return;
     }
 
@@ -718,9 +663,9 @@ static const int kYapDatabaseRangeMinLength = 0;
 
     if (![self reloadViewItems]) {
         // These errors are rare.
-        OWSFailDebug(@"could not reload view items; hard resetting message mappings.");
+        OWSFailDebug(@"could not reload view items; hard resetting message mapping.");
-        // resetMappings will call delegate.conversationViewModelDidUpdate.
+        // resetMapping will call delegate.conversationViewModelDidUpdate.
-        [self resetMappings];
+        [self resetMapping];
         return;
     }
 
@@ -928,12 +873,18 @@ static const int kYapDatabaseRangeMinLength = 0;
             case ConversationUpdateItemType_Insert: {
                 id<ConversationViewItem> viewItem = updateItem.viewItem;
                 OWSAssertDebug(viewItem);
-                if (([viewItem.interaction isKindOfClass:[TSIncomingMessage class]] ||
+                switch (viewItem.interaction.interactionType) {
-                        [viewItem.interaction isKindOfClass:[TSOutgoingMessage class]])
+                    case OWSInteractionType_IncomingMessage:
-                    && updateItem.newIndex >= oldViewItemCount) {
+                    case OWSInteractionType_OutgoingMessage:
-                    continue;
+                    case OWSInteractionType_TypingIndicator:
+                        if (updateItem.newIndex < oldViewItemCount) {
+                            isOnlyModifyingLastMessage = NO;
+                        }
+                        break;
+                    default:
+                        isOnlyModifyingLastMessage = NO;
+                        break;
                 }
-                isOnlyModifyingLastMessage = NO;
                 break;
             }
             case ConversationUpdateItemType_Update: {
@@ -942,12 +893,18 @@ static const int kYapDatabaseRangeMinLength = 0;
                     continue;
                 }
                 OWSAssertDebug(viewItem);
-                if (([viewItem.interaction isKindOfClass:[TSIncomingMessage class]] ||
+                switch (viewItem.interaction.interactionType) {
-                        [viewItem.interaction isKindOfClass:[TSOutgoingMessage class]])
+                    case OWSInteractionType_IncomingMessage:
-                    && updateItem.newIndex >= oldViewItemCount) {
+                    case OWSInteractionType_OutgoingMessage:
-                    continue;
+                    case OWSInteractionType_TypingIndicator:
+                        if (updateItem.newIndex < updateItems.count - 1) {
+                            isOnlyModifyingLastMessage = NO;
+                        }
+                        break;
+                    default:
+                        isOnlyModifyingLastMessage = NO;
+                        break;
                 }
-                isOnlyModifyingLastMessage = NO;
                 break;
             }
         }
@@ -956,40 +913,51 @@ static const int kYapDatabaseRangeMinLength = 0;
     return shouldAnimateRowUpdates;
 }
 
-- (void)createNewMessageMappings
+- (void)createNewMessageMapping
 {
-    if (self.thread.uniqueId.length > 0) {
+    if (self.thread.uniqueId.length < 1) {
-        self.messageMappings = [[YapDatabaseViewMappings alloc] initWithGroups:@[ self.thread.uniqueId ]
-                                                                          view:TSMessageDatabaseViewExtensionName];
-    } else {
         OWSFailDebug(@"uniqueId unexpectedly empty for thread: %@", self.thread);
-        self.messageMappings =
-            [[YapDatabaseViewMappings alloc] initWithGroups:@[] view:TSMessageDatabaseViewExtensionName];
     }
 
+    self.messageMapping = [[ConversationMessageMapping alloc] initWithGroup:self.thread.uniqueId
+                                                              desiredLength:self.initialMessageMappingLength];
+
     [self.uiDatabaseConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) {
-        [self.messageMappings updateWithTransaction:transaction];
+        [self.messageMapping updateWithTransaction:transaction];
     }];
-    // We need to impose the range restrictions on the mappings immediately to avoid
-    // doing a great deal of unnecessary work and causing a perf hotspot.
-    [self updateMessageMappingRangeOptions];
 }
 
-- (void)resetMappings
+// This is more expensive than incremental updates.
+//
+// It can be used to get back to a known good state, or to respon
+// It can be done to get back to a known good state
+- (void)resetMapping
 {
-    OWSAssertDebug(self.messageMappings);
+    // Don't extend the mapping's desired length.
+    [self resetMappingWithAdditionalLength:0];
+}
+
+- (void)resetMappingWithAdditionalLength:(NSUInteger)additionalLength
+{
+    OWSAssertDebug(self.messageMapping);
+
+    [self updateMessageMappingWithAdditionalLength:additionalLength];
 
-    if (self.messageMappings != nil) {
-        // Make sure our mapping and range state is up-to-date.
-        [self.uiDatabaseConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) {
-            [self.messageMappings updateWithTransaction:transaction];
-        }];
-        [self updateMessageMappingRangeOptions];
-    }
     self.collapseCutoffDate = [NSDate new];
 
     [self ensureDynamicInteractions];
 
+    if (![self reloadViewItems]) {
+        OWSFailDebug(@"failed to reload view items in resetMapping.");
+    }
+
+    [self.delegate conversationViewModelDidUpdate:ConversationUpdate.reloadUpdate];
+}
+
+- (void)touchDbAsync
+{
+    OWSLogInfo(@"");
+
     // There appears to be a bug in YapDatabase that sometimes delays modifications
     // made in another process (e.g. the SAE) from showing up in other processes.
     // There's a simple workaround: a trivial write to the database flushes changes
@@ -997,8 +965,11 @@ static const int kYapDatabaseRangeMinLength = 0;
     [self.editingDatabaseConnection asyncReadWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) {
         [transaction setObject:[NSUUID UUID].UUIDString forKey:@"conversation_view_noop_mod" inCollection:@"temp"];
     }];
+}
 
-    [self.delegate conversationViewModelDidUpdate:ConversationUpdate.reloadUpdate];
+- (void)applicationWillEnterForeground:(NSNotification *)notification
+{
+    [self touchDbAsync];
 }
 
 #pragma mark - View Items
@@ -1198,7 +1169,7 @@ static const int kYapDatabaseRangeMinLength = 0;
     NSMutableArray<id<ConversationViewItem>> *viewItems = [NSMutableArray new];
     NSMutableDictionary<NSString *, id<ConversationViewItem>> *viewItemCache = [NSMutableDictionary new];
 
-    NSUInteger count = [self.messageMappings numberOfItemsInSection:0];
+    NSArray<NSString *> *loadedUniqueIds = [self.messageMapping loadedUniqueIds];
     BOOL isGroupThread = self.thread.isGroupThread;
     ConversationStyle *conversationStyle = self.delegate.conversationStyle;
 
@@ -1216,43 +1187,43 @@ static const int kYapDatabaseRangeMinLength = 0;
                                                                               transaction:transaction
                                                                         conversationStyle:conversationStyle];
               }
-              [viewItems addObject:viewItem];
               OWSAssertDebug(!viewItemCache[interaction.uniqueId]);
               viewItemCache[interaction.uniqueId] = viewItem;
+              [viewItems addObject:viewItem];
 
               return viewItem;
           };
 
+    NSMutableSet<NSString *> *interactionIds = [NSMutableSet new];
+    BOOL canLoadMoreItems = self.messageMapping.canLoadMore;
     [self.uiDatabaseConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) {
         NSMutableArray<TSInteraction *> *interactions = [NSMutableArray new];
-        NSMutableSet<NSString *> *interactionIds = [NSMutableSet new];
 
-        YapDatabaseViewTransaction *messageDatabaseView = [transaction ext:TSMessageDatabaseViewExtensionName];
+        YapDatabaseViewTransaction *viewTransaction = [transaction ext:TSMessageDatabaseViewExtensionName];
-        OWSAssertDebug(messageDatabaseView);
+        OWSAssertDebug(viewTransaction);
-        for (NSUInteger row = 0; row < count; row++) {
+        for (NSString *uniqueId in loadedUniqueIds) {
-            TSInteraction *interaction =
+            TSInteraction *_Nullable interaction =
-                [messageDatabaseView objectAtRow:row inSection:0 withMappings:self.messageMappings];
+                [TSInteraction fetchObjectWithUniqueID:uniqueId transaction:transaction];
             if (!interaction) {
-                OWSFailDebug(
+                OWSFailDebug(@"missing interaction in message mapping: %@.", uniqueId);
-                    @"missing interaction in message mappings: %lu / %lu.", (unsigned long)row, (unsigned long)count);
                 // TODO: Add analytics.
                 hasError = YES;
                 continue;
             }
             if (!interaction.uniqueId) {
-                OWSFailDebug(@"invalid interaction in message mappings: %lu / %lu: %@.",
+                OWSFailDebug(@"invalid interaction in message mapping: %@.", interaction);
-                    (unsigned long)row,
-                    (unsigned long)count,
-                    interaction);
                 // TODO: Add analytics.
                 hasError = YES;
                 continue;
             }
             [interactions addObject:interaction];
+            if ([interactionIds containsObject:interaction.uniqueId]) {
+                OWSFailDebug(@"Duplicate interaction: %@", interaction.uniqueId);
+                continue;
+            }
             [interactionIds addObject:interaction.uniqueId];
         }
 
-        BOOL canLoadMoreItems = [self canLoadMoreItems:messageDatabaseView];
         OWSContactOffersInteraction *_Nullable offers =
             [self tryToBuildContactOffersInteractionWithTransaction:transaction
                                                  loadedInteractions:interactions
@@ -1285,8 +1256,13 @@ static const int kYapDatabaseRangeMinLength = 0;
 
     if (self.unsavedOutgoingMessages.count > 0) {
         [self.uiDatabaseConnection readWithBlock:^(YapDatabaseReadTransaction *_Nonnull transaction) {
-            for (TSOutgoingMessage *outgoingMessage in self.unsavedOutgoingMessages) {
+            for (TSOutgoingMessage *outgoingMessage in [self.unsavedOutgoingMessages copy]) {
+                if ([interactionIds containsObject:outgoingMessage.uniqueId]) {
+                    OWSFailDebug(@"Duplicate interaction: %@", outgoingMessage.uniqueId);
+                    continue;
+                }
                 tryToAddViewItem(outgoingMessage, transaction);
+                [interactionIds addObject:outgoingMessage.uniqueId];
             }
         }];
     }
@@ -1583,125 +1559,26 @@ static const int kYapDatabaseRangeMinLength = 0;
     OWSAssertDebug(quotedReply.timestamp > 0);
     OWSAssertDebug(quotedReply.authorId.length > 0);
 
-    // TODO:
-    // We try to find the index of the item within the current thread's
-    // interactions that includes the "quoted interaction".
-    //
-    // NOTE: There are two indices:
-    //
-    // * The "group index" of the member of the database views group at
-    //   the db conneciton's current checkpoint.
-    // * The "index row/section" in the message mapping.
-    //
-    // NOTE: Since the range _IS NOT_ filtered by author,
-    // and timestamp collisions are possible, it's possible
-    // for:
-    //
-    // * The range to include more than the "quoted interaction".
-    // * The range to be non-empty but NOT include the "quoted interaction",
-    //   although this would be a bug.
-    __block TSInteraction *_Nullable quotedInteraction;
-    __block NSUInteger threadInteractionCount = 0;
-    __block NSNumber *_Nullable groupIndex = nil;
-
     if (quotedReply.isRemotelySourced) {
         return nil;
     }
 
+    __block NSIndexPath *_Nullable indexPath = nil;
     [self.uiDatabaseConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) {
-        quotedInteraction = [ThreadUtil findInteractionInThreadByTimestamp:quotedReply.timestamp
+        TSInteraction *_Nullable quotedInteraction =
-                                                                  authorId:quotedReply.authorId
+            [ThreadUtil findInteractionInThreadByTimestamp:quotedReply.timestamp
-                                                            threadUniqueId:self.thread.uniqueId
+                                                  authorId:quotedReply.authorId
-                                                               transaction:transaction];
+                                            threadUniqueId:self.thread.uniqueId
+                                               transaction:transaction];
         if (!quotedInteraction) {
             return;
         }
 
-        YapDatabaseAutoViewTransaction *_Nullable extension =
+        indexPath = [self.messageMapping ensureLoadWindowContainsWithUniqueId:quotedInteraction.uniqueId
-            [transaction extension:TSMessageDatabaseViewExtensionName];
+                                                                  transaction:transaction];
-        if (!extension) {
-            OWSFailDebug(@"Couldn't load view.");
-            return;
-        }
-
-        threadInteractionCount = [extension numberOfItemsInGroup:self.thread.uniqueId];
-
-        groupIndex = [self findGroupIndexOfThreadInteraction:quotedInteraction transaction:transaction];
     }];
 
-    if (!quotedInteraction || !groupIndex) {
+    return indexPath;
-        return nil;
-    }
-
-    NSUInteger indexRow = 0;
-    NSUInteger indexSection = 0;
-    BOOL isInMappings = [self.messageMappings getRow:&indexRow
-                                             section:&indexSection
-                                            forIndex:groupIndex.unsignedIntegerValue
-                                             inGroup:self.thread.uniqueId];
-
-    if (!isInMappings) {
-        NSInteger desiredWindowSize = MAX(0, 1 + (NSInteger)threadInteractionCount - groupIndex.integerValue);
-        NSUInteger oldLoadWindowSize = [self.messageMappings numberOfItemsInGroup:self.thread.uniqueId];
-        NSInteger additionalItemsToLoad = MAX(0, desiredWindowSize - (NSInteger)oldLoadWindowSize);
-        if (additionalItemsToLoad < 1) {
-            OWSLogError(@"Couldn't determine how to load quoted reply.");
-            return nil;
-        }
-
-        // Try to load more messages so that the quoted message
-        // is in the load window.
-        //
-        // This may fail if the quoted message is very old, in which
-        // case we'll load the max number of messages.
-        [self loadNMoreMessages:(NSUInteger)additionalItemsToLoad];
-
-        // `loadNMoreMessages` will reset the mapping and possibly
-        // integrate new changes, so we need to reload the "group index"
-        // of the quoted message.
-        [self.uiDatabaseConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) {
-            groupIndex = [self findGroupIndexOfThreadInteraction:quotedInteraction transaction:transaction];
-        }];
-
-        if (!quotedInteraction || !groupIndex) {
-            OWSLogError(@"Failed to find quoted reply in group.");
-            return nil;
-        }
-
-        isInMappings = [self.messageMappings getRow:&indexRow
-                                            section:&indexSection
-                                           forIndex:groupIndex.unsignedIntegerValue
-                                            inGroup:self.thread.uniqueId];
-
-        if (!isInMappings) {
-            OWSLogError(@"Could not load quoted reply into mapping.");
-            return nil;
-        }
-    }
-
-    // The mapping indices and view item indices don't always align for corrupt mappings.
-    __block TSInteraction *_Nullable interaction;
-    [self.uiDatabaseConnection readWithBlock:^(YapDatabaseReadTransaction *transaction) {
-        YapDatabaseViewTransaction *messageDatabaseView = [transaction ext:TSMessageDatabaseViewExtensionName];
-        OWSAssertDebug(messageDatabaseView);
-        interaction =
-            [messageDatabaseView objectAtRow:indexRow inSection:indexSection withMappings:self.messageMappings];
-    }];
-    if (!interaction) {
-        OWSFailDebug(@"Could not locate interaction for quoted reply.");
-        return nil;
-    }
-    id<ConversationViewItem> _Nullable viewItem = self.viewItemCache[interaction.uniqueId];
-    if (!viewItem) {
-        OWSFailDebug(@"Could not locate view item for quoted reply.");
-        return nil;
-    }
-    NSUInteger viewItemIndex = [self.viewItems indexOfObject:viewItem];
-    if (viewItemIndex == NSNotFound) {
-        OWSFailDebug(@"Could not locate view item index for quoted reply.");
-        return nil;
-    }
-    return [NSIndexPath indexPathForRow:(NSInteger)viewItemIndex inSection:0];
 }
 
 - (nullable NSNumber *)findGroupIndexOfThreadInteraction:(TSInteraction *)interaction
@@ -1749,7 +1626,23 @@ static const int kYapDatabaseRangeMinLength = 0;
     // Update the view items if necessary.
     // We don't have to do this if they haven't been configured yet.
     if (didChange && self.viewItems != nil) {
-        [self updateForTransientItems];
+        // When we receive an incoming message, we clear any typing indicators
+        // from that sender.  Ideally, we'd like both changes (disappearance of
+        // the typing indicators, appearance of the incoming message) to show up
+        // in the view at the same time, rather than as a "jerky" two-step
+        // visual change.
+        //
+        // Unfortunately, the view model learns of these changes by separate
+        // channels: the incoming message is a database modification and the
+        // typing indicator change arrives via this notification.
+        //
+        // Therefore we pause briefly before updating the view model to reflect
+        // typing indicators state changes so that the database modification
+        // can usually arrive first and update the view to reflect both changes.
+        __weak ConversationViewModel *weakSelf = self;
+        dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.1f * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{
+            [weakSelf updateForTransientItems];
+        });
     }
 }