Removed the wip pruning logic (no longer needed)
Fixed broken unit tests
This commit is contained in:
parent
54fc75cd85
commit
5db254303a
|
@ -443,121 +443,6 @@ internal extension SessionUtil {
|
||||||
|
|
||||||
return updated
|
return updated
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: - Pruning
|
|
||||||
|
|
||||||
static func pruningIfNeeded(
|
|
||||||
_ db: Database,
|
|
||||||
conf: UnsafeMutablePointer<config_object>?
|
|
||||||
) throws {
|
|
||||||
// First make sure we are actually thowing the correct size constraint error (don't want to prune contacts
|
|
||||||
// as a result of some other random error
|
|
||||||
do {
|
|
||||||
try CExceptionHelper.performSafely { config_push(conf).deallocate() }
|
|
||||||
return // If we didn't error then no need to prune
|
|
||||||
}
|
|
||||||
catch {
|
|
||||||
guard (error as NSError).userInfo["NSLocalizedDescription"] as? String == "Config data is too large" else {
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extract the contact data from the config
|
|
||||||
var allContactData: [String: ContactData] = extractContacts(
|
|
||||||
from: conf,
|
|
||||||
latestConfigSentTimestampMs: 0
|
|
||||||
)
|
|
||||||
|
|
||||||
// Remove the current user profile info (shouldn't be in there but just in case)
|
|
||||||
let userPublicKey: String = getUserHexEncodedPublicKey(db)
|
|
||||||
var cUserPublicKey: [CChar] = userPublicKey.cArray.nullTerminated()
|
|
||||||
contacts_erase(conf, &cUserPublicKey)
|
|
||||||
|
|
||||||
/// Do the following in stages (we want to prune as few contacts as possible because we are essentially deleting data and removing these
|
|
||||||
/// contacts will result in not just contact data but also associated conversation data for the contact being removed from linked devices
|
|
||||||
///
|
|
||||||
///
|
|
||||||
/// **Step 1** First of all we want to try to detect spam-attacks (ie. if someone creates a bunch of accounts and messages you, and you
|
|
||||||
/// systematically block every one of those accounts - this can quickly add up)
|
|
||||||
///
|
|
||||||
/// We will do this by filtering the contact data to only include blocked contacts, grouping those contacts into contacts created within the
|
|
||||||
/// same hour and then only including groups that have more than 10 contacts (ie. if you blocked 20 users within an hour we expect those
|
|
||||||
/// contacts were spammers)
|
|
||||||
let blockSpamBatchingResolution: TimeInterval = (60 * 60)
|
|
||||||
// TODO: Do we want to only do this case for contacts that were created over X time ago? (to avoid unintentionally unblocking accounts that were recently blocked
|
|
||||||
let likelySpammerContacts: [ContactData] = allContactData
|
|
||||||
.values
|
|
||||||
.filter { $0.contact.isBlocked }
|
|
||||||
.grouped(by: { $0.created / blockSpamBatchingResolution })
|
|
||||||
.filter { _, values in values.count > 20 }
|
|
||||||
.values
|
|
||||||
.flatMap { $0 }
|
|
||||||
|
|
||||||
if !likelySpammerContacts.isEmpty {
|
|
||||||
likelySpammerContacts.forEach { contact in
|
|
||||||
var cSessionId: [CChar] = contact.contact.id.cArray.nullTerminated()
|
|
||||||
contacts_erase(conf, &cSessionId)
|
|
||||||
|
|
||||||
allContactData.removeValue(forKey: contact.contact.id)
|
|
||||||
}
|
|
||||||
|
|
||||||
// If we are no longer erroring then we can stop here
|
|
||||||
do { return try CExceptionHelper.performSafely { config_push(conf).deallocate() } }
|
|
||||||
catch {}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// We retrieve the `CONVO_INFO_VOLATILE` records and one-to-one conversation message counts as they will be relevant for subsequent checks
|
|
||||||
let volatileThreadInfo: [String: VolatileThreadInfo] = SessionUtil
|
|
||||||
.config(for: .convoInfoVolatile, publicKey: userPublicKey)
|
|
||||||
.wrappedValue
|
|
||||||
.map { SessionUtil.extractConvoVolatileInfo(from: $0) }
|
|
||||||
.defaulting(to: [])
|
|
||||||
.reduce(into: [:]) { result, next in result[next.threadId] = next }
|
|
||||||
let conversationMessageCounts: [String: Int] = try SessionThread
|
|
||||||
.filter(SessionThread.Columns.variant == SessionThread.Variant.contact)
|
|
||||||
.select(.id)
|
|
||||||
.annotated(with: SessionThread.interactions.count)
|
|
||||||
.asRequest(of: ThreadCount.self)
|
|
||||||
.fetchAll(db)
|
|
||||||
.reduce(into: [:]) { result, next in result[next.id] = next.interactionCount }
|
|
||||||
|
|
||||||
/// **Step 2** Next up we want to remove contact records which are likely to be invalid, this means contacts which:
|
|
||||||
/// - Aren't blocked
|
|
||||||
/// - Have no `name` value
|
|
||||||
/// - Have no `CONVO_INFO_VOLATILE` record
|
|
||||||
/// - Have no messages in their one-to-one conversations
|
|
||||||
///
|
|
||||||
/// Any contacts that meet the above criteria are likely either invalid contacts or are contacts which the user hasn't seen or interacted
|
|
||||||
/// with for 30+ days
|
|
||||||
let likelyInvalidContacts: [ContactData] = allContactData
|
|
||||||
.values
|
|
||||||
.filter { !$0.contact.isBlocked }
|
|
||||||
.filter { $0.profile.name.isEmpty }
|
|
||||||
.filter { volatileThreadInfo[$0.contact.id] == nil }
|
|
||||||
.filter { (conversationMessageCounts[$0.contact.id] ?? 0) == 0 }
|
|
||||||
|
|
||||||
if !likelyInvalidContacts.isEmpty {
|
|
||||||
likelyInvalidContacts.forEach { contact in
|
|
||||||
var cSessionId: [CChar] = contact.contact.id.cArray.nullTerminated()
|
|
||||||
contacts_erase(conf, &cSessionId)
|
|
||||||
|
|
||||||
allContactData.removeValue(forKey: contact.contact.id)
|
|
||||||
}
|
|
||||||
|
|
||||||
// If we are no longer erroring then we can stop here
|
|
||||||
do { return try CExceptionHelper.performSafely { config_push(conf).deallocate() } }
|
|
||||||
catch {}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// TODO: Exclude contacts that have no profile info(?)
|
|
||||||
// TODO: Exclude contacts that have a CONVO_INFO_VOLATILE record
|
|
||||||
// TODO: Exclude contacts that have a conversation with messages in the database (ie. only delete "empty" contacts)
|
|
||||||
|
|
||||||
// TODO: Start pruning valid contacts which have really old conversations...
|
|
||||||
|
|
||||||
print("RAWR")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: - External Outgoing Changes
|
// MARK: - External Outgoing Changes
|
||||||
|
|
|
@ -74,35 +74,6 @@ class ConfigContactsSpec {
|
||||||
}
|
}
|
||||||
.to(throwError(NSError(domain: "cpp_exception", code: -2, userInfo: ["NSLocalizedDescription": "Config data is too large"])))
|
.to(throwError(NSError(domain: "cpp_exception", code: -2, userInfo: ["NSLocalizedDescription": "Config data is too large"])))
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: -- can catch size limit errors thrown when dumping
|
|
||||||
it("can catch size limit errors thrown when dumping") {
|
|
||||||
var randomGenerator: ARC4RandomNumberGenerator = ARC4RandomNumberGenerator(seed: 1000)
|
|
||||||
|
|
||||||
try (0..<100000).forEach { index in
|
|
||||||
var contact: contacts_contact = try createContact(
|
|
||||||
for: index,
|
|
||||||
in: conf,
|
|
||||||
rand: &randomGenerator,
|
|
||||||
maxing: .allProperties
|
|
||||||
)
|
|
||||||
contacts_set(conf, &contact)
|
|
||||||
}
|
|
||||||
|
|
||||||
expect(contacts_size(conf)).to(equal(100000))
|
|
||||||
expect(config_needs_push(conf)).to(beTrue())
|
|
||||||
expect(config_needs_dump(conf)).to(beTrue())
|
|
||||||
|
|
||||||
expect {
|
|
||||||
try CExceptionHelper.performSafely {
|
|
||||||
var dump: UnsafeMutablePointer<UInt8>? = nil
|
|
||||||
var dumpLen: Int = 0
|
|
||||||
config_dump(conf, &dump, &dumpLen)
|
|
||||||
dump?.deallocate()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
.to(throwError(NSError(domain: "cpp_exception", code: -2, userInfo: ["NSLocalizedDescription": "Config data is too large"])))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: - when checking size limits
|
// MARK: - when checking size limits
|
||||||
|
@ -225,70 +196,6 @@ class ConfigContactsSpec {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: - when pruning
|
|
||||||
context("when pruning") {
|
|
||||||
var mockStorage: Storage!
|
|
||||||
var seed: Data!
|
|
||||||
var identity: (ed25519KeyPair: KeyPair, x25519KeyPair: KeyPair)!
|
|
||||||
var edSK: [UInt8]!
|
|
||||||
var error: UnsafeMutablePointer<CChar>?
|
|
||||||
var conf: UnsafeMutablePointer<config_object>?
|
|
||||||
|
|
||||||
beforeEach {
|
|
||||||
mockStorage = Storage(
|
|
||||||
customWriter: try! DatabaseQueue(),
|
|
||||||
customMigrations: [
|
|
||||||
SNUtilitiesKit.migrations(),
|
|
||||||
SNMessagingKit.migrations()
|
|
||||||
]
|
|
||||||
)
|
|
||||||
seed = Data(hex: "0123456789abcdef0123456789abcdef")
|
|
||||||
|
|
||||||
// FIXME: Would be good to move these into the libSession-util instead of using Sodium separately
|
|
||||||
identity = try! Identity.generate(from: seed)
|
|
||||||
edSK = identity.ed25519KeyPair.secretKey
|
|
||||||
|
|
||||||
// Initialize a brand new, empty config because we have no dump data to deal with.
|
|
||||||
error = nil
|
|
||||||
conf = nil
|
|
||||||
_ = contacts_init(&conf, &edSK, nil, 0, error)
|
|
||||||
error?.deallocate()
|
|
||||||
}
|
|
||||||
|
|
||||||
it("does something") {
|
|
||||||
mockStorage.write { db in
|
|
||||||
try SessionThread.fetchOrCreate(db, id: "1", variant: .contact, shouldBeVisible: true)
|
|
||||||
try SessionThread.fetchOrCreate(db, id: "2", variant: .contact, shouldBeVisible: true)
|
|
||||||
try SessionThread.fetchOrCreate(db, id: "3", variant: .contact, shouldBeVisible: true)
|
|
||||||
_ = try Interaction(
|
|
||||||
threadId: "1",
|
|
||||||
authorId: "1",
|
|
||||||
variant: .standardIncoming,
|
|
||||||
body: "Test1"
|
|
||||||
).inserted(db)
|
|
||||||
_ = try Interaction(
|
|
||||||
threadId: "1",
|
|
||||||
authorId: "2",
|
|
||||||
variant: .standardIncoming,
|
|
||||||
body: "Test2"
|
|
||||||
).inserted(db)
|
|
||||||
_ = try Interaction(
|
|
||||||
threadId: "3",
|
|
||||||
authorId: "3",
|
|
||||||
variant: .standardIncoming,
|
|
||||||
body: "Test3"
|
|
||||||
).inserted(db)
|
|
||||||
|
|
||||||
try SessionUtil.pruningIfNeeded(
|
|
||||||
db,
|
|
||||||
conf: conf
|
|
||||||
)
|
|
||||||
|
|
||||||
expect(contacts_size(conf)).to(equal(0))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// MARK: - generates config correctly
|
// MARK: - generates config correctly
|
||||||
|
|
||||||
it("generates config correctly") {
|
it("generates config correctly") {
|
||||||
|
|
|
@ -55,9 +55,9 @@ open class Storage {
|
||||||
// If a custom writer was provided then use that (for unit testing)
|
// If a custom writer was provided then use that (for unit testing)
|
||||||
guard customWriter == nil else {
|
guard customWriter == nil else {
|
||||||
dbWriter = customWriter
|
dbWriter = customWriter
|
||||||
perform(migrations: (customMigrations ?? []), async: false, onProgressUpdate: nil, onComplete: { _, _ in })
|
|
||||||
isValid = true
|
isValid = true
|
||||||
Storage.internalHasCreatedValidInstance.mutate { $0 = true }
|
Storage.internalHasCreatedValidInstance.mutate { $0 = true }
|
||||||
|
perform(migrations: (customMigrations ?? []), async: false, onProgressUpdate: nil, onComplete: { _, _ in })
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue