Removed the wip pruning logic (no longer needed)
Fixed broken unit tests
This commit is contained in:
Morgan Pretty
parent
54fc75cd85
commit
5db254303a
|
|
@ -443,121 +443,6 @@ internal extension SessionUtil {
|
|||
|
||||
return updated
|
||||
}
|
||||
|
||||
// MARK: - Pruning
|
||||
|
||||
static func pruningIfNeeded(
|
||||
_ db: Database,
|
||||
conf: UnsafeMutablePointer<config_object>?
|
||||
) throws {
|
||||
// First make sure we are actually thowing the correct size constraint error (don't want to prune contacts
|
||||
// as a result of some other random error
|
||||
do {
|
||||
try CExceptionHelper.performSafely { config_push(conf).deallocate() }
|
||||
return // If we didn't error then no need to prune
|
||||
}
|
||||
catch {
|
||||
guard (error as NSError).userInfo["NSLocalizedDescription"] as? String == "Config data is too large" else {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
// Extract the contact data from the config
|
||||
var allContactData: [String: ContactData] = extractContacts(
|
||||
from: conf,
|
||||
latestConfigSentTimestampMs: 0
|
||||
)
|
||||
|
||||
// Remove the current user profile info (shouldn't be in there but just in case)
|
||||
let userPublicKey: String = getUserHexEncodedPublicKey(db)
|
||||
var cUserPublicKey: [CChar] = userPublicKey.cArray.nullTerminated()
|
||||
contacts_erase(conf, &cUserPublicKey)
|
||||
|
||||
/// Do the following in stages (we want to prune as few contacts as possible because we are essentially deleting data and removing these
|
||||
/// contacts will result in not just contact data but also associated conversation data for the contact being removed from linked devices
|
||||
///
|
||||
///
|
||||
/// **Step 1** First of all we want to try to detect spam-attacks (ie. if someone creates a bunch of accounts and messages you, and you
|
||||
/// systematically block every one of those accounts - this can quickly add up)
|
||||
///
|
||||
/// We will do this by filtering the contact data to only include blocked contacts, grouping those contacts into contacts created within the
|
||||
/// same hour and then only including groups that have more than 10 contacts (ie. if you blocked 20 users within an hour we expect those
|
||||
/// contacts were spammers)
|
||||
let blockSpamBatchingResolution: TimeInterval = (60 * 60)
|
||||
// TODO: Do we want to only do this case for contacts that were created over X time ago? (to avoid unintentionally unblocking accounts that were recently blocked
|
||||
let likelySpammerContacts: [ContactData] = allContactData
|
||||
.values
|
||||
.filter { $0.contact.isBlocked }
|
||||
.grouped(by: { $0.created / blockSpamBatchingResolution })
|
||||
.filter { _, values in values.count > 20 }
|
||||
.values
|
||||
.flatMap { $0 }
|
||||
|
||||
if !likelySpammerContacts.isEmpty {
|
||||
likelySpammerContacts.forEach { contact in
|
||||
var cSessionId: [CChar] = contact.contact.id.cArray.nullTerminated()
|
||||
contacts_erase(conf, &cSessionId)
|
||||
|
||||
allContactData.removeValue(forKey: contact.contact.id)
|
||||
}
|
||||
|
||||
// If we are no longer erroring then we can stop here
|
||||
do { return try CExceptionHelper.performSafely { config_push(conf).deallocate() } }
|
||||
catch {}
|
||||
}
|
||||
|
||||
/// We retrieve the `CONVO_INFO_VOLATILE` records and one-to-one conversation message counts as they will be relevant for subsequent checks
|
||||
let volatileThreadInfo: [String: VolatileThreadInfo] = SessionUtil
|
||||
.config(for: .convoInfoVolatile, publicKey: userPublicKey)
|
||||
.wrappedValue
|
||||
.map { SessionUtil.extractConvoVolatileInfo(from: $0) }
|
||||
.defaulting(to: [])
|
||||
.reduce(into: [:]) { result, next in result[next.threadId] = next }
|
||||
let conversationMessageCounts: [String: Int] = try SessionThread
|
||||
.filter(SessionThread.Columns.variant == SessionThread.Variant.contact)
|
||||
.select(.id)
|
||||
.annotated(with: SessionThread.interactions.count)
|
||||
.asRequest(of: ThreadCount.self)
|
||||
.fetchAll(db)
|
||||
.reduce(into: [:]) { result, next in result[next.id] = next.interactionCount }
|
||||
|
||||
/// **Step 2** Next up we want to remove contact records which are likely to be invalid, this means contacts which:
|
||||
/// - Aren't blocked
|
||||
/// - Have no `name` value
|
||||
/// - Have no `CONVO_INFO_VOLATILE` record
|
||||
/// - Have no messages in their one-to-one conversations
|
||||
///
|
||||
/// Any contacts that meet the above criteria are likely either invalid contacts or are contacts which the user hasn't seen or interacted
|
||||
/// with for 30+ days
|
||||
let likelyInvalidContacts: [ContactData] = allContactData
|
||||
.values
|
||||
.filter { !$0.contact.isBlocked }
|
||||
.filter { $0.profile.name.isEmpty }
|
||||
.filter { volatileThreadInfo[$0.contact.id] == nil }
|
||||
.filter { (conversationMessageCounts[$0.contact.id] ?? 0) == 0 }
|
||||
|
||||
if !likelyInvalidContacts.isEmpty {
|
||||
likelyInvalidContacts.forEach { contact in
|
||||
var cSessionId: [CChar] = contact.contact.id.cArray.nullTerminated()
|
||||
contacts_erase(conf, &cSessionId)
|
||||
|
||||
allContactData.removeValue(forKey: contact.contact.id)
|
||||
}
|
||||
|
||||
// If we are no longer erroring then we can stop here
|
||||
do { return try CExceptionHelper.performSafely { config_push(conf).deallocate() } }
|
||||
catch {}
|
||||
}
|
||||
|
||||
|
||||
// TODO: Exclude contacts that have no profile info(?)
|
||||
// TODO: Exclude contacts that have a CONVO_INFO_VOLATILE record
|
||||
// TODO: Exclude contacts that have a conversation with messages in the database (ie. only delete "empty" contacts)
|
||||
|
||||
// TODO: Start pruning valid contacts which have really old conversations...
|
||||
|
||||
print("RAWR")
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - External Outgoing Changes
|
||||
|
|
|
|||
|
|
@ -74,35 +74,6 @@ class ConfigContactsSpec {
|
|||
}
|
||||
.to(throwError(NSError(domain: "cpp_exception", code: -2, userInfo: ["NSLocalizedDescription": "Config data is too large"])))
|
||||
}
|
||||
|
||||
// MARK: -- can catch size limit errors thrown when dumping
|
||||
it("can catch size limit errors thrown when dumping") {
|
||||
var randomGenerator: ARC4RandomNumberGenerator = ARC4RandomNumberGenerator(seed: 1000)
|
||||
|
||||
try (0..<100000).forEach { index in
|
||||
var contact: contacts_contact = try createContact(
|
||||
for: index,
|
||||
in: conf,
|
||||
rand: &randomGenerator,
|
||||
maxing: .allProperties
|
||||
)
|
||||
contacts_set(conf, &contact)
|
||||
}
|
||||
|
||||
expect(contacts_size(conf)).to(equal(100000))
|
||||
expect(config_needs_push(conf)).to(beTrue())
|
||||
expect(config_needs_dump(conf)).to(beTrue())
|
||||
|
||||
expect {
|
||||
try CExceptionHelper.performSafely {
|
||||
var dump: UnsafeMutablePointer<UInt8>? = nil
|
||||
var dumpLen: Int = 0
|
||||
config_dump(conf, &dump, &dumpLen)
|
||||
dump?.deallocate()
|
||||
}
|
||||
}
|
||||
.to(throwError(NSError(domain: "cpp_exception", code: -2, userInfo: ["NSLocalizedDescription": "Config data is too large"])))
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - when checking size limits
|
||||
|
|
@ -225,70 +196,6 @@ class ConfigContactsSpec {
|
|||
}
|
||||
}
|
||||
|
||||
// MARK: - when pruning
|
||||
context("when pruning") {
|
||||
var mockStorage: Storage!
|
||||
var seed: Data!
|
||||
var identity: (ed25519KeyPair: KeyPair, x25519KeyPair: KeyPair)!
|
||||
var edSK: [UInt8]!
|
||||
var error: UnsafeMutablePointer<CChar>?
|
||||
var conf: UnsafeMutablePointer<config_object>?
|
||||
|
||||
beforeEach {
|
||||
mockStorage = Storage(
|
||||
customWriter: try! DatabaseQueue(),
|
||||
customMigrations: [
|
||||
SNUtilitiesKit.migrations(),
|
||||
SNMessagingKit.migrations()
|
||||
]
|
||||
)
|
||||
seed = Data(hex: "0123456789abcdef0123456789abcdef")
|
||||
|
||||
// FIXME: Would be good to move these into the libSession-util instead of using Sodium separately
|
||||
identity = try! Identity.generate(from: seed)
|
||||
edSK = identity.ed25519KeyPair.secretKey
|
||||
|
||||
// Initialize a brand new, empty config because we have no dump data to deal with.
|
||||
error = nil
|
||||
conf = nil
|
||||
_ = contacts_init(&conf, &edSK, nil, 0, error)
|
||||
error?.deallocate()
|
||||
}
|
||||
|
||||
it("does something") {
|
||||
mockStorage.write { db in
|
||||
try SessionThread.fetchOrCreate(db, id: "1", variant: .contact, shouldBeVisible: true)
|
||||
try SessionThread.fetchOrCreate(db, id: "2", variant: .contact, shouldBeVisible: true)
|
||||
try SessionThread.fetchOrCreate(db, id: "3", variant: .contact, shouldBeVisible: true)
|
||||
_ = try Interaction(
|
||||
threadId: "1",
|
||||
authorId: "1",
|
||||
variant: .standardIncoming,
|
||||
body: "Test1"
|
||||
).inserted(db)
|
||||
_ = try Interaction(
|
||||
threadId: "1",
|
||||
authorId: "2",
|
||||
variant: .standardIncoming,
|
||||
body: "Test2"
|
||||
).inserted(db)
|
||||
_ = try Interaction(
|
||||
threadId: "3",
|
||||
authorId: "3",
|
||||
variant: .standardIncoming,
|
||||
body: "Test3"
|
||||
).inserted(db)
|
||||
|
||||
try SessionUtil.pruningIfNeeded(
|
||||
db,
|
||||
conf: conf
|
||||
)
|
||||
|
||||
expect(contacts_size(conf)).to(equal(0))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - generates config correctly
|
||||
|
||||
it("generates config correctly") {
|
||||
|
|
|
|||
|
|
@ -55,9 +55,9 @@ open class Storage {
|
|||
// If a custom writer was provided then use that (for unit testing)
|
||||
guard customWriter == nil else {
|
||||
dbWriter = customWriter
|
||||
perform(migrations: (customMigrations ?? []), async: false, onProgressUpdate: nil, onComplete: { _, _ in })
|
||||
isValid = true
|
||||
Storage.internalHasCreatedValidInstance.mutate { $0 = true }
|
||||
perform(migrations: (customMigrations ?? []), async: false, onProgressUpdate: nil, onComplete: { _, _ in })
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue