2022-03-29 06:18:26 +02:00
import { app , clipboard , dialog , Notification } from 'electron' ;
2023-02-20 05:11:04 +01:00
import fs from 'fs' ;
import path from 'path' ;
2023-08-02 06:50:35 +02:00
import * as BetterSqlite3 from '@signalapp/better-sqlite3' ;
2023-02-20 05:11:04 +01:00
import rimraf from 'rimraf' ;
2022-03-29 06:18:26 +02:00
import {
2022-04-21 03:29:14 +02:00
chunk ,
2022-05-20 05:19:48 +02:00
compact ,
2022-04-21 06:04:11 +02:00
difference ,
2022-03-10 00:51:15 +01:00
forEach ,
2022-03-29 06:18:26 +02:00
fromPairs ,
2022-09-14 05:53:44 +02:00
isArray ,
2022-03-10 00:51:15 +01:00
isEmpty ,
isNumber ,
2022-03-29 06:18:26 +02:00
isObject ,
isString ,
last ,
map ,
2023-01-25 07:46:30 +01:00
omit ,
2023-05-11 09:29:01 +02:00
uniq ,
2022-03-29 06:18:26 +02:00
} from 'lodash' ;
2023-07-26 11:26:46 +02:00
import { base64_variants , from_base64 , to_hex } from 'libsodium-wrappers-sumo' ;
2023-02-08 00:50:23 +01:00
import { ConversationAttributes } from '../models/conversationAttributes' ;
2023-02-20 05:11:04 +01:00
import { PubKey } from '../session/types/PubKey' ; // checked - only node
import { redactAll } from '../util/privacy' ; // checked - only node
2022-08-08 01:50:48 +02:00
import {
arrayStrToJson ,
assertValidConversationAttributes ,
2022-08-12 06:48:20 +02:00
ATTACHMENT_DOWNLOADS_TABLE ,
CLOSED_GROUP_V2_KEY_PAIRS_TABLE ,
CONVERSATIONS_TABLE ,
2022-08-08 01:50:48 +02:00
formatRowOfConversation ,
2022-08-12 06:48:20 +02:00
GUARD_NODE_TABLE ,
HEX_KEY ,
IDENTITY_KEYS_TABLE ,
ITEMS_TABLE ,
2022-08-08 01:50:48 +02:00
jsonToObject ,
2022-08-12 06:48:20 +02:00
LAST_HASHES_TABLE ,
MESSAGES_FTS_TABLE ,
MESSAGES_TABLE ,
NODES_FOR_PUBKEY_TABLE ,
2022-08-08 01:50:48 +02:00
objectToJSON ,
2022-08-12 06:48:20 +02:00
OPEN_GROUP_ROOMS_V2_TABLE ,
2022-08-08 01:50:48 +02:00
toSqliteBoolean ,
} from './database_utility' ;
2023-02-20 05:11:04 +01:00
import { LocaleMessagesType } from './locale' ; // checked - only node
import { StorageItem } from './storage_item' ; // checked - only node
2022-01-07 00:32:58 +01:00
2023-02-20 05:11:04 +01:00
import { OpenGroupV2Room } from '../data/opengroups' ;
2023-01-25 07:46:30 +01:00
import {
2023-02-20 05:11:04 +01:00
CONFIG_DUMP_TABLE ,
2023-01-25 07:46:30 +01:00
MsgDuplicateSearchOpenGroup ,
2023-03-16 00:49:06 +01:00
roomHasBlindEnabled ,
SaveConversationReturn ,
2023-01-25 07:46:30 +01:00
UnprocessedDataNode ,
UnprocessedParameter ,
UpdateLastHashType ,
} from '../types/sqlSharedTypes' ;
2022-01-07 00:32:58 +01:00
2023-03-16 00:49:06 +01:00
import { KNOWN_BLINDED_KEYS_ITEM , SettingsKey } from '../data/settings-key' ;
2022-08-12 06:48:20 +02:00
import {
getSQLCipherIntegrityCheck ,
openAndMigrateDatabase ,
updateSchema ,
} from './migration/signalMigrations' ;
2023-01-19 01:23:51 +01:00
import {
assertGlobalInstance ,
assertGlobalInstanceOrInstance ,
closeDbInstance ,
initDbInstanceWith ,
isInstanceInitialized ,
} from './sqlInstance' ;
2023-02-01 05:03:58 +01:00
import { configDumpData } from './sql_calls/config_dump' ;
2023-05-11 09:29:01 +02:00
import { Quote } from '../receiver/types' ;
2022-01-07 00:32:58 +01:00
2023-07-26 11:26:46 +02:00
// eslint:disable: function-name non-literal-fs-path
2022-02-14 06:42:53 +01:00
2022-08-12 06:48:20 +02:00
const MAX_PUBKEYS_MEMBERS = 300 ;
2022-02-15 04:42:39 +01:00
2022-08-12 06:48:20 +02:00
function getSQLIntegrityCheck ( db : BetterSqlite3.Database ) {
const checkResult = db . pragma ( 'quick_check' , { simple : true } ) ;
if ( checkResult !== 'ok' ) {
return checkResult ;
2022-02-15 04:42:39 +01:00
}
2022-02-04 04:32:06 +01:00
2022-08-12 06:48:20 +02:00
return undefined ;
2022-02-04 04:32:06 +01:00
}
2022-08-12 06:48:20 +02:00
function openAndSetUpSQLCipher ( filePath : string , { key } : { key : string } ) {
return openAndMigrateDatabase ( filePath , key ) ;
2022-04-21 08:06:20 +02:00
}
2022-08-12 06:48:20 +02:00
function setSQLPassword ( password : string ) {
2023-01-19 01:23:51 +01:00
if ( ! assertGlobalInstance ( ) ) {
2022-08-12 06:48:20 +02:00
throw new Error ( 'setSQLPassword: db is not initialized' ) ;
2022-05-04 06:54:38 +02:00
}
2021-07-08 05:00:20 +02:00
2022-08-12 06:48:20 +02:00
// If the password isn't hex then we need to derive a key from it
const deriveKey = HEX_KEY . test ( password ) ;
const value = deriveKey ? ` ' ${ password } ' ` : ` "x' ${ password } '" ` ;
2023-01-19 01:23:51 +01:00
assertGlobalInstance ( ) . pragma ( ` rekey = ${ value } ` ) ;
2019-08-07 00:53:41 +02:00
}
2022-08-12 06:48:20 +02:00
function vacuumDatabase ( db : BetterSqlite3.Database ) {
if ( ! db ) {
throw new Error ( 'vacuum: db is not initialized' ) ;
2019-08-07 00:53:41 +02:00
}
2022-08-12 06:48:20 +02:00
const start = Date . now ( ) ;
console . info ( 'Vacuuming DB. This might take a while.' ) ;
db . exec ( 'VACUUM;' ) ;
console . info ( ` Vacuuming DB Finished in ${ Date . now ( ) - start } ms. ` ) ;
2019-08-07 00:53:41 +02:00
}
2022-03-29 06:18:26 +02:00
let databaseFilePath : string | undefined ;
2018-07-27 03:13:56 +02:00
2022-03-29 06:18:26 +02:00
function _initializePaths ( configDir : string ) {
2019-01-29 02:12:39 +01:00
const dbDir = path . join ( configDir , 'sql' ) ;
2021-12-08 04:15:54 +01:00
fs . mkdirSync ( dbDir , { recursive : true } ) ;
2022-08-08 01:50:48 +02:00
console . info ( 'Made sure db folder exists at:' , dbDir ) ;
2021-07-08 05:00:20 +02:00
databaseFilePath = path . join ( dbDir , 'db.sqlite' ) ;
2019-01-29 02:12:39 +01:00
}
2022-03-10 00:51:15 +01:00
function showFailedToStart() {
const notification = new Notification ( {
title : 'Session failed to start' ,
body : 'Please start from terminal and open a github issue' ,
} ) ;
notification . show ( ) ;
}
2022-03-29 09:03:02 +02:00
async function initializeSql ( {
2022-03-29 06:18:26 +02:00
configDir ,
key ,
messages ,
passwordAttempt ,
} : {
configDir : string ;
key : string ;
messages : LocaleMessagesType ;
passwordAttempt : boolean ;
} ) {
2022-04-13 06:55:22 +02:00
console . info ( 'initializeSql sqlnode' ) ;
2023-01-19 01:23:51 +01:00
if ( isInstanceInitialized ( ) ) {
2018-07-27 03:13:56 +02:00
throw new Error ( 'Cannot initialize more than once!' ) ;
}
if ( ! isString ( configDir ) ) {
throw new Error ( 'initialize: configDir is required!' ) ;
}
if ( ! isString ( key ) ) {
2019-02-20 02:32:44 +01:00
throw new Error ( 'initialize: key is required!' ) ;
}
if ( ! isObject ( messages ) ) {
throw new Error ( 'initialize: message is required!' ) ;
2018-07-27 03:13:56 +02:00
}
2019-01-29 02:12:39 +01:00
_initializePaths ( configDir ) ;
2018-11-07 05:16:49 +01:00
2021-07-08 05:00:20 +02:00
let db ;
2019-02-20 02:32:44 +01:00
try {
2022-03-29 06:18:26 +02:00
if ( ! databaseFilePath ) {
throw new Error ( 'databaseFilePath is not set' ) ;
}
2021-07-08 05:00:20 +02:00
db = openAndSetUpSQLCipher ( databaseFilePath , { key } ) ;
2022-03-29 06:18:26 +02:00
if ( ! db ) {
throw new Error ( 'db is not set' ) ;
}
2023-06-27 07:52:25 +02:00
await updateSchema ( db ) ;
2019-02-20 02:32:44 +01:00
2019-02-27 19:17:22 +01:00
// test database
2020-03-03 05:17:30 +01:00
2021-07-08 05:00:20 +02:00
const cipherIntegrityResult = getSQLCipherIntegrityCheck ( db ) ;
if ( cipherIntegrityResult ) {
console . log ( 'Database cipher integrity check failed:' , cipherIntegrityResult ) ;
throw new Error ( ` Cipher integrity check failed: ${ cipherIntegrityResult } ` ) ;
}
const integrityResult = getSQLIntegrityCheck ( db ) ;
if ( integrityResult ) {
console . log ( 'Database integrity check failed:' , integrityResult ) ;
throw new Error ( ` Integrity check failed: ${ integrityResult } ` ) ;
2020-03-03 05:17:30 +01:00
}
2021-07-08 05:00:20 +02:00
// At this point we can allow general access to the database
2023-01-19 01:23:51 +01:00
initDbInstanceWith ( db ) ;
2021-07-08 05:00:20 +02:00
2022-04-21 03:29:14 +02:00
console . info ( 'total message count before cleaning: ' , getMessageCount ( ) ) ;
console . info ( 'total conversation count before cleaning: ' , getConversationCount ( ) ) ;
2022-08-08 01:50:48 +02:00
cleanUpOldOpengroupsOnStart ( ) ;
cleanUpUnusedNodeForKeyEntriesOnStart ( ) ;
2022-04-21 03:29:14 +02:00
printDbStats ( ) ;
console . info ( 'total message count after cleaning: ' , getMessageCount ( ) ) ;
console . info ( 'total conversation count after cleaning: ' , getConversationCount ( ) ) ;
// Clear any already deleted db entries on each app start.
vacuumDatabase ( db ) ;
2019-02-20 02:32:44 +01:00
} catch ( error ) {
2022-04-21 03:29:14 +02:00
console . error ( 'error' , error ) ;
2019-07-05 05:21:32 +02:00
if ( passwordAttempt ) {
throw error ;
}
2019-02-20 02:32:44 +01:00
console . log ( 'Database startup error:' , error . stack ) ;
2022-03-29 06:18:26 +02:00
const button = await dialog . showMessageBox ( {
2021-06-16 03:20:31 +02:00
buttons : [ messages . copyErrorAndQuit , messages . clearAllData ] ,
2019-02-20 02:32:44 +01:00
defaultId : 0 ,
detail : redactAll ( error . stack ) ,
2021-06-16 03:20:31 +02:00
message : messages.databaseError ,
2019-02-20 02:32:44 +01:00
noLink : true ,
type : 'error' ,
} ) ;
2022-03-29 06:18:26 +02:00
if ( button . response === 0 ) {
2021-04-22 10:03:58 +02:00
clipboard . writeText ( ` Database startup error: \ n \ n ${ redactAll ( error . stack ) } ` ) ;
2019-02-20 02:32:44 +01:00
} else {
2023-01-19 01:23:51 +01:00
closeDbInstance ( ) ;
2022-03-10 00:51:15 +01:00
showFailedToStart ( ) ;
2019-02-20 02:32:44 +01:00
}
app . exit ( 1 ) ;
return false ;
}
return true ;
2018-07-27 03:13:56 +02:00
}
2021-07-08 05:00:20 +02:00
function removeDB ( configDir = null ) {
2023-01-19 01:23:51 +01:00
if ( isInstanceInitialized ( ) ) {
2021-07-08 05:00:20 +02:00
throw new Error ( 'removeDB: Cannot erase database when it is open!' ) ;
}
if ( ! databaseFilePath && configDir ) {
2019-01-29 02:12:39 +01:00
_initializePaths ( configDir ) ;
}
2022-03-29 06:18:26 +02:00
if ( databaseFilePath ) {
rimraf . sync ( databaseFilePath ) ;
rimraf . sync ( ` ${ databaseFilePath } -shm ` ) ;
rimraf . sync ( ` ${ databaseFilePath } -wal ` ) ;
}
2018-07-27 03:13:56 +02:00
}
2018-12-06 01:33:11 +01:00
// Password hash
2018-12-09 23:25:36 +01:00
const PASS_HASH_ID = 'passHash' ;
2021-07-08 05:00:20 +02:00
function getPasswordHash() {
const item = getItemById ( PASS_HASH_ID ) ;
2018-12-06 01:33:11 +01:00
return item && item . value ;
}
2022-03-29 06:18:26 +02:00
function savePasswordHash ( hash : string ) {
2018-12-06 01:33:11 +01:00
if ( isEmpty ( hash ) ) {
2022-03-29 06:18:26 +02:00
removePasswordHash ( ) ;
return ;
2018-12-06 01:33:11 +01:00
}
2018-12-09 23:25:36 +01:00
const data = { id : PASS_HASH_ID , value : hash } ;
2022-03-29 06:18:26 +02:00
createOrUpdateItem ( data ) ;
2018-12-06 01:33:11 +01:00
}
2021-07-08 05:00:20 +02:00
function removePasswordHash() {
2022-03-29 06:18:26 +02:00
removeItemById ( PASS_HASH_ID ) ;
2018-12-06 01:33:11 +01:00
}
2022-03-29 06:18:26 +02:00
function getIdentityKeyById ( id : string , instance : BetterSqlite3.Database ) {
2021-01-12 06:11:05 +01:00
return getById ( IDENTITY_KEYS_TABLE , id , instance ) ;
2018-11-07 05:16:49 +01:00
}
2021-07-08 05:00:20 +02:00
function getGuardNodes() {
2022-03-29 06:18:26 +02:00
const nodes = assertGlobalInstance ( )
. prepare ( ` SELECT ed25519PubKey FROM ${ GUARD_NODE_TABLE } ; ` )
. all ( ) ;
2020-03-23 05:00:51 +01:00
if ( ! nodes ) {
return null ;
}
return nodes ;
}
2022-03-29 06:18:26 +02:00
function updateGuardNodes ( nodes : Array < string > ) {
assertGlobalInstance ( ) . transaction ( ( ) = > {
assertGlobalInstance ( ) . exec ( ` DELETE FROM ${ GUARD_NODE_TABLE } ` ) ;
2020-03-24 08:22:51 +01:00
nodes . map ( edkey = >
2022-03-29 06:18:26 +02:00
assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
` INSERT INTO ${ GUARD_NODE_TABLE } (
2020-03-23 05:00:51 +01:00
ed25519PubKey
2021-07-08 05:00:20 +02:00
) values ( $ed25519PubKey ) `
)
. run ( {
ed25519PubKey : edkey ,
} )
) ;
} ) ( ) ;
2020-03-23 05:00:51 +01:00
}
2022-03-29 06:18:26 +02:00
function createOrUpdateItem ( data : StorageItem , instance? : BetterSqlite3.Database ) {
createOrUpdate ( ITEMS_TABLE , data , instance ) ;
2018-11-07 05:16:49 +01:00
}
2023-02-21 07:09:08 +01:00
function getItemById ( id : string , instance? : BetterSqlite3.Database ) {
return getById ( ITEMS_TABLE , id , instance ) ;
2018-11-07 05:16:49 +01:00
}
2021-07-08 05:00:20 +02:00
function getAllItems() {
2022-03-29 06:18:26 +02:00
const rows = assertGlobalInstance ( )
. prepare ( ` SELECT json FROM ${ ITEMS_TABLE } ORDER BY id ASC; ` )
. all ( ) ;
2018-11-07 05:16:49 +01:00
return map ( rows , row = > jsonToObject ( row . json ) ) ;
}
2022-03-29 06:18:26 +02:00
function removeItemById ( id : string ) {
removeById ( ITEMS_TABLE , id ) ;
2018-11-07 05:16:49 +01:00
}
2022-03-29 06:18:26 +02:00
function createOrUpdate ( table : string , data : StorageItem , instance? : BetterSqlite3.Database ) {
2018-11-07 05:16:49 +01:00
const { id } = data ;
if ( ! id ) {
throw new Error ( 'createOrUpdate: Provided data did not have a truthy id' ) ;
}
2022-03-29 06:18:26 +02:00
assertGlobalInstanceOrInstance ( instance )
2021-07-08 05:00:20 +02:00
. prepare (
` INSERT OR REPLACE INTO ${ table } (
2018-11-07 05:16:49 +01:00
id ,
json
) values (
$id ,
$json
2021-07-08 05:00:20 +02:00
) `
)
. run ( {
2021-07-12 08:00:04 +02:00
id ,
2021-07-08 05:00:20 +02:00
json : objectToJSON ( data ) ,
} ) ;
2018-11-07 05:16:49 +01:00
}
2022-03-29 06:18:26 +02:00
function getById ( table : string , id : string , instance? : BetterSqlite3.Database ) {
const row = assertGlobalInstanceOrInstance ( instance )
. prepare ( ` SELECT * FROM ${ table } WHERE id = $ id; ` )
. get ( {
id ,
} ) ;
2018-11-07 05:16:49 +01:00
if ( ! row ) {
return null ;
}
return jsonToObject ( row . json ) ;
}
2022-03-29 06:18:26 +02:00
function removeById ( table : string , id : string ) {
2018-11-07 05:16:49 +01:00
if ( ! Array . isArray ( id ) ) {
2022-03-29 06:18:26 +02:00
assertGlobalInstance ( )
. prepare ( ` DELETE FROM ${ table } WHERE id = $ id; ` )
. run ( { id } ) ;
2018-11-07 05:16:49 +01:00
return ;
}
if ( ! id . length ) {
throw new Error ( 'removeById: No ids to delete!' ) ;
}
// Our node interface doesn't seem to allow you to replace one single ? with an array
2022-03-29 06:18:26 +02:00
assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare ( ` DELETE FROM ${ table } WHERE id IN ( ${ id . map ( ( ) = > '?' ) . join ( ', ' ) } ); ` )
. run ( { id } ) ;
2018-11-07 05:16:49 +01:00
}
// Conversations
2022-03-29 06:18:26 +02:00
function getSwarmNodesForPubkey ( pubkey : string ) {
const row = assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare ( ` SELECT * FROM ${ NODES_FOR_PUBKEY_TABLE } WHERE pubkey = $ pubkey; ` )
. get ( {
pubkey ,
} ) ;
2019-01-09 02:33:21 +01:00
if ( ! row ) {
2019-01-30 04:23:36 +01:00
return [ ] ;
2019-01-09 02:33:21 +01:00
}
2020-07-06 07:28:22 +02:00
return jsonToObject ( row . json ) ;
}
2022-03-29 06:18:26 +02:00
function updateSwarmNodesForPubkey ( pubkey : string , snodeEdKeys : Array < string > ) {
assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
` INSERT OR REPLACE INTO ${ NODES_FOR_PUBKEY_TABLE } (
2020-07-06 07:28:22 +02:00
pubkey ,
json
) values (
$pubkey ,
$json
2021-07-08 05:00:20 +02:00
) ; `
)
. run ( {
pubkey ,
json : objectToJSON ( snodeEdKeys ) ,
} ) ;
2019-01-09 02:33:21 +01:00
}
2021-07-08 05:00:20 +02:00
function getConversationCount() {
2022-03-29 06:18:26 +02:00
const row = assertGlobalInstance ( )
. prepare ( ` SELECT count(*) from ${ CONVERSATIONS_TABLE } ; ` )
. get ( ) ;
2018-09-21 03:47:19 +02:00
if ( ! row ) {
2021-04-22 10:03:58 +02:00
throw new Error ( ` getConversationCount: Unable to get count of ${ CONVERSATIONS_TABLE } ` ) ;
2018-09-21 03:47:19 +02:00
}
return row [ 'count(*)' ] ;
}
2023-03-16 00:49:06 +01:00
/ * *
* Because the argument list can change when saving a conversation ( and actually doing a lot of other stuff ) ,
* it is not a good idea to try to use it to update a conversation while doing migrations .
* Because everytime you ' ll update the saveConversation with a new argument , the migration you wrote a month ago still relies on the old way .
* Because of that , there is no ` instance ` argument here , and you should not add one as this is only needed during migrations ( which will break if you do it )
* /
2023-07-26 11:26:46 +02:00
2023-03-16 00:49:06 +01:00
function saveConversation ( data : ConversationAttributes ) : SaveConversationReturn {
2022-08-08 01:50:48 +02:00
const formatted = assertValidConversationAttributes ( data ) ;
2018-09-21 03:47:19 +02:00
2019-01-16 05:44:13 +01:00
const {
id ,
active_at ,
type ,
members ,
2022-08-08 01:50:48 +02:00
nickname ,
profileKey ,
zombies ,
left ,
expireTimer ,
lastMessageStatus ,
lastMessage ,
lastJoinedTimestamp ,
groupAdmins ,
isKickedFromGroup ,
avatarPointer ,
avatarImageId ,
triggerNotificationsFor ,
isTrustedForAttachmentDownload ,
isApproved ,
didApproveMe ,
avatarInProfile ,
displayNameInProfile ,
conversationIdOrigin ,
2023-04-03 04:03:23 +02:00
priority ,
2023-03-10 06:39:48 +01:00
markedAsUnread ,
2022-08-08 01:50:48 +02:00
} = formatted ;
2023-03-10 06:39:48 +01:00
const omited = omit ( formatted ) ;
2023-01-25 07:46:30 +01:00
const keys = Object . keys ( omited ) ;
const columnsCommaSeparated = keys . join ( ', ' ) ;
const valuesArgs = keys . map ( k = > ` $ ${ k } ` ) . join ( ', ' ) ;
2022-08-24 08:09:29 +02:00
const maxLength = 300 ;
// shorten the last message as we never need more than `maxLength` chars (and it bloats the redux/ipc calls uselessly.
2022-08-08 01:50:48 +02:00
const shortenedLastMessage =
2022-08-24 08:09:29 +02:00
isString ( lastMessage ) && lastMessage . length > maxLength
? lastMessage . substring ( 0 , maxLength )
: lastMessage ;
2023-03-16 00:49:06 +01:00
assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
2022-08-08 01:50:48 +02:00
` INSERT OR REPLACE INTO ${ CONVERSATIONS_TABLE } (
2023-01-25 07:46:30 +01:00
$ { columnsCommaSeparated }
2022-08-08 01:50:48 +02:00
) values (
2023-01-25 07:46:30 +01:00
$ { valuesArgs }
2022-08-08 01:50:48 +02:00
) `
2021-07-08 05:00:20 +02:00
)
. run ( {
id ,
active_at ,
type ,
2022-08-08 01:50:48 +02:00
members : members && members . length ? arrayStrToJson ( members ) : '[]' ,
nickname ,
profileKey ,
zombies : zombies && zombies . length ? arrayStrToJson ( zombies ) : '[]' ,
left : toSqliteBoolean ( left ) ,
expireTimer ,
lastMessageStatus ,
lastMessage : shortenedLastMessage ,
lastJoinedTimestamp ,
groupAdmins : groupAdmins && groupAdmins . length ? arrayStrToJson ( groupAdmins ) : '[]' ,
isKickedFromGroup : toSqliteBoolean ( isKickedFromGroup ) ,
avatarPointer ,
avatarImageId ,
triggerNotificationsFor ,
isTrustedForAttachmentDownload : toSqliteBoolean ( isTrustedForAttachmentDownload ) ,
2023-04-03 04:03:23 +02:00
priority ,
2022-08-08 01:50:48 +02:00
isApproved : toSqliteBoolean ( isApproved ) ,
didApproveMe : toSqliteBoolean ( didApproveMe ) ,
avatarInProfile ,
displayNameInProfile ,
conversationIdOrigin ,
2023-03-10 06:39:48 +01:00
markedAsUnread : toSqliteBoolean ( markedAsUnread ) ,
2021-07-08 05:00:20 +02:00
} ) ;
2023-03-16 00:49:06 +01:00
return fetchConvoMemoryDetails ( id ) ;
}
function fetchConvoMemoryDetails ( convoId : string ) : SaveConversationReturn {
const hasMentionedUsUnread = ! ! getFirstUnreadMessageWithMention ( convoId ) ;
const unreadCount = getUnreadCountByConversation ( convoId ) ;
const lastReadTimestampMessageSentTimestamp = getLastMessageReadInConversation ( convoId ) ;
2023-03-30 05:15:59 +02:00
// TODOLATER it would be nice to be able to remove the lastMessage and lastMessageStatus from the conversation table, and just return it when saving the conversation
2023-03-29 05:30:35 +02:00
// and saving it in memory only.
// But we'd need to update a bunch of things as we do some logic before setting the lastUpdate text and status mostly in `getMessagePropStatus` and `getNotificationText()`
// const lastMessages = getLastMessagesByConversation(convoId, 1) as Array:Record<string, any>>;
2023-03-16 00:49:06 +01:00
return {
mentionedUs : hasMentionedUsUnread ,
unreadCount ,
lastReadTimestampMessage : lastReadTimestampMessageSentTimestamp ,
} ;
2018-09-21 03:47:19 +02:00
}
2022-03-29 06:18:26 +02:00
function removeConversation ( id : string | Array < string > ) {
2018-09-21 03:47:19 +02:00
if ( ! Array . isArray ( id ) ) {
2022-03-29 06:18:26 +02:00
assertGlobalInstance ( )
. prepare ( ` DELETE FROM ${ CONVERSATIONS_TABLE } WHERE id = $ id; ` )
. run ( {
id ,
} ) ;
2018-09-21 03:47:19 +02:00
return ;
}
if ( ! id . length ) {
throw new Error ( 'removeConversation: No ids to delete!' ) ;
}
// Our node interface doesn't seem to allow you to replace one single ? with an array
2022-03-29 06:18:26 +02:00
assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare ( ` DELETE FROM ${ CONVERSATIONS_TABLE } WHERE id IN ( ${ id . map ( ( ) = > '?' ) . join ( ', ' ) } ); ` )
. run ( id ) ;
2018-09-21 03:47:19 +02:00
}
2023-03-16 00:49:06 +01:00
export function getIdentityKeys ( db : BetterSqlite3.Database ) {
const row = db . prepare ( ` SELECT * FROM ${ ITEMS_TABLE } WHERE id = $ id; ` ) . get ( {
id : 'identityKey' ,
} ) ;
if ( ! row ) {
return null ;
}
try {
const parsedIdentityKey = jsonToObject ( row . json ) ;
if (
! parsedIdentityKey ? . value ? . pubKey ||
! parsedIdentityKey ? . value ? . ed25519KeyPair ? . privateKey
) {
return null ;
}
const publicKeyBase64 = parsedIdentityKey ? . value ? . pubKey ;
const publicKeyHex = to_hex ( from_base64 ( publicKeyBase64 , base64_variants . ORIGINAL ) ) ;
const ed25519PrivateKeyUintArray = parsedIdentityKey ? . value ? . ed25519KeyPair ? . privateKey ;
2023-03-30 05:15:59 +02:00
// TODOLATER migrate the ed25519KeyPair for all the users already logged in to a base64 representation
2023-03-16 00:49:06 +01:00
const privateEd25519 = new Uint8Array ( Object . values ( ed25519PrivateKeyUintArray ) ) ;
if ( ! privateEd25519 || isEmpty ( privateEd25519 ) ) {
return null ;
}
return {
publicKeyHex ,
privateEd25519 ,
} ;
} catch ( e ) {
return null ;
}
}
function getUsBlindedInThatServerIfNeeded (
convoId : string ,
instance? : BetterSqlite3.Database
) : string | undefined {
const usNaked = getIdentityKeys ( assertGlobalInstanceOrInstance ( instance ) ) ? . publicKeyHex ;
if ( ! usNaked ) {
return undefined ;
}
const room = getV2OpenGroupRoom ( convoId , instance ) as OpenGroupV2Room | null ;
if ( ! room || ! roomHasBlindEnabled ( room ) || ! room . serverPublicKey ) {
return usNaked ;
}
const blinded = getItemById ( KNOWN_BLINDED_KEYS_ITEM , instance ) ;
// this is essentially a duplicate of getUsBlindedInThatServer made at a database level (with db from the DB directly and not cached on the renderer side)
try {
const allBlinded = JSON . parse ( blinded ? . value ) ;
const found = allBlinded . find (
( m : any ) = > m . serverPublicKey === room . serverPublicKey && m . realSessionId === usNaked
) ;
const blindedId = found ? . blindedId ;
return isString ( blindedId ) ? blindedId : usNaked ;
} catch ( e ) {
2023-05-10 05:29:38 +02:00
console . error ( 'getUsBlindedInThatServerIfNeeded failed with ' , e . message ) ;
2023-03-16 00:49:06 +01:00
}
return usNaked ;
}
2023-02-21 07:09:08 +01:00
function getConversationById ( id : string , instance? : BetterSqlite3.Database ) {
const row = assertGlobalInstanceOrInstance ( instance )
2022-03-29 06:18:26 +02:00
. prepare ( ` SELECT * FROM ${ CONVERSATIONS_TABLE } WHERE id = $ id; ` )
. get ( {
id ,
} ) ;
2018-09-21 03:47:19 +02:00
2023-03-16 00:49:06 +01:00
const unreadCount = getUnreadCountByConversation ( id , instance ) || 0 ;
const mentionedUsStillUnread = ! ! getFirstUnreadMessageWithMention ( id , instance ) ;
return formatRowOfConversation ( row , 'getConversationById' , unreadCount , mentionedUsStillUnread ) ;
2018-09-21 03:47:19 +02:00
}
2021-07-08 05:00:20 +02:00
function getAllConversations() {
2022-03-29 06:18:26 +02:00
const rows = assertGlobalInstance ( )
2022-08-08 01:50:48 +02:00
. prepare ( ` SELECT * FROM ${ CONVERSATIONS_TABLE } ORDER BY id ASC; ` )
2021-07-08 05:00:20 +02:00
. all ( ) ;
2023-03-16 00:49:06 +01:00
return ( rows || [ ] ) . map ( m = > {
const unreadCount = getUnreadCountByConversation ( m . id ) || 0 ;
const mentionedUsStillUnread = ! ! getFirstUnreadMessageWithMention ( m . id ) ;
return formatRowOfConversation ( m , 'getAllConversations' , unreadCount , mentionedUsStillUnread ) ;
} ) ;
2018-09-21 03:47:19 +02:00
}
2022-03-29 06:18:26 +02:00
function getPubkeysInPublicConversation ( conversationId : string ) {
2022-09-14 05:53:44 +02:00
const conversation = getV2OpenGroupRoom ( conversationId ) ;
if ( ! conversation ) {
return [ ] ;
}
const hasBlindOn = Boolean (
conversation . capabilities &&
isArray ( conversation . capabilities ) &&
conversation . capabilities ? . includes ( 'blind' )
) ;
2023-07-26 14:06:11 +02:00
const whereClause = hasBlindOn ? "AND source LIKE '15%'" : '' ; // the LIKE content has to be ' and not "
2022-09-14 05:53:44 +02:00
2022-03-29 06:18:26 +02:00
const rows = assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
` SELECT DISTINCT source FROM ${ MESSAGES_TABLE } WHERE
2023-07-26 14:06:11 +02:00
conversationId = $conversationId $ { whereClause }
ORDER BY received_at DESC LIMIT $ { MAX_PUBKEYS_MEMBERS } ; `
2021-07-08 05:00:20 +02:00
)
. all ( {
conversationId ,
} ) ;
2019-11-13 05:52:17 +01:00
return map ( rows , row = > row . source ) ;
}
2022-03-29 06:18:26 +02:00
function searchConversations ( query : string ) {
const rows = assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
2022-08-08 01:50:48 +02:00
` SELECT * FROM ${ CONVERSATIONS_TABLE } WHERE
2019-01-14 22:47:19 +01:00
(
2022-08-08 01:50:48 +02:00
displayNameInProfile LIKE $displayNameInProfile OR
nickname LIKE $nickname
2023-02-21 07:09:08 +01:00
) AND active_at > 0
2022-02-02 07:50:59 +01:00
ORDER BY active_at DESC
2021-07-08 05:00:20 +02:00
LIMIT $limit `
)
. all ( {
2022-08-08 01:50:48 +02:00
displayNameInProfile : ` % ${ query } % ` ,
nickname : ` % ${ query } % ` ,
2022-03-29 06:18:26 +02:00
limit : 50 ,
2021-07-08 05:00:20 +02:00
} ) ;
2018-09-21 03:47:19 +02:00
2023-03-16 00:49:06 +01:00
return ( rows || [ ] ) . map ( m = > {
const unreadCount = getUnreadCountByConversation ( m . id ) ;
const mentionedUsStillUnread = ! ! getFirstUnreadMessageWithMention ( m . id ) ;
const formatted = formatRowOfConversation (
m ,
'searchConversations' ,
unreadCount ,
mentionedUsStillUnread
) ;
return formatted ;
} ) ;
2018-09-21 03:47:19 +02:00
}
2022-01-25 06:58:29 +01:00
// order by clause is the same as orderByClause but with a table prefix so we cannot reuse it
const orderByMessageCoalesceClause = ` ORDER BY COALESCE( ${ MESSAGES_TABLE } .serverTimestamp, ${ MESSAGES_TABLE } .sent_at, ${ MESSAGES_TABLE } .received_at) DESC ` ;
2022-01-20 06:50:42 +01:00
2022-03-29 06:18:26 +02:00
function searchMessages ( query : string , limit : number ) {
2022-02-02 05:54:00 +01:00
if ( ! limit ) {
throw new Error ( 'searchMessages limit must be set' ) ;
}
2022-03-29 06:18:26 +02:00
const rows = assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
` SELECT
2022-01-20 06:50:42 +01:00
$ { MESSAGES_TABLE } . json ,
2022-02-01 03:46:30 +01:00
snippet ( $ { MESSAGES_FTS_TABLE } , - 1 , '<<left>>' , '<<right>>' , '...' , 5 ) as snippet
2021-06-11 04:09:57 +02:00
FROM $ { MESSAGES_FTS_TABLE }
2023-06-27 07:52:25 +02:00
INNER JOIN $ { MESSAGES_TABLE } on $ { MESSAGES_FTS_TABLE } . rowid = $ { MESSAGES_TABLE } . rowid
2019-01-14 22:47:19 +01:00
WHERE
2023-06-27 07:52:25 +02:00
$ { MESSAGES_FTS_TABLE } . body match $query
2022-01-25 06:58:29 +01:00
$ { orderByMessageCoalesceClause }
2021-07-08 05:00:20 +02:00
LIMIT $limit ; `
)
. all ( {
2021-07-12 08:00:04 +02:00
query ,
2022-02-02 05:54:00 +01:00
limit ,
2021-07-08 05:00:20 +02:00
} ) ;
2019-01-14 22:47:19 +01:00
return map ( rows , row = > ( {
. . . jsonToObject ( row . json ) ,
snippet : row.snippet ,
} ) ) ;
}
2023-02-21 07:09:08 +01:00
/ * *
* Search for matching messages in a specific conversation .
* Currently unused but kept as we want to add it back at some point .
* /
2022-03-29 06:18:26 +02:00
function searchMessagesInConversation ( query : string , conversationId : string , limit : number ) {
const rows = assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
` SELECT
2022-01-24 07:03:11 +01:00
$ { MESSAGES_TABLE } . json ,
snippet ( $ { MESSAGES_FTS_TABLE } , - 1 , '<<left>>' , '<<right>>' , '...' , 15 ) as snippet
FROM $ { MESSAGES_FTS_TABLE }
INNER JOIN $ { MESSAGES_TABLE } on $ { MESSAGES_FTS_TABLE } . id = $ { MESSAGES_TABLE } . id
2019-01-14 22:47:19 +01:00
WHERE
2022-01-24 07:03:11 +01:00
$ { MESSAGES_FTS_TABLE } match $query AND
$ { MESSAGES_TABLE } . conversationId = $conversationId
2022-01-25 06:58:29 +01:00
$ { orderByMessageCoalesceClause }
2022-01-24 07:03:11 +01:00
LIMIT $limit ; `
2021-07-08 05:00:20 +02:00
)
. all ( {
query ,
conversationId ,
limit : limit || 100 ,
} ) ;
2019-01-14 22:47:19 +01:00
return map ( rows , row = > ( {
. . . jsonToObject ( row . json ) ,
snippet : row.snippet ,
} ) ) ;
}
2021-07-08 05:00:20 +02:00
function getMessageCount() {
2022-03-29 06:18:26 +02:00
const row = assertGlobalInstance ( )
. prepare ( ` SELECT count(*) from ${ MESSAGES_TABLE } ; ` )
. get ( ) ;
2018-08-07 01:18:58 +02:00
if ( ! row ) {
2021-04-22 10:03:58 +02:00
throw new Error ( ` getMessageCount: Unable to get count of ${ MESSAGES_TABLE } ` ) ;
2018-08-07 01:18:58 +02:00
}
return row [ 'count(*)' ] ;
}
2022-03-29 06:18:26 +02:00
function saveMessage ( data : any ) {
2018-07-27 03:13:56 +02:00
const {
2019-01-14 22:47:19 +01:00
body ,
2018-07-27 03:13:56 +02:00
conversationId ,
// eslint-disable-next-line camelcase
expires_at ,
hasAttachments ,
hasFileAttachments ,
hasVisualMediaAttachments ,
id ,
2019-08-09 05:34:48 +02:00
serverId ,
2020-09-04 07:11:24 +02:00
serverTimestamp ,
2018-07-27 03:13:56 +02:00
// eslint-disable-next-line camelcase
received_at ,
2018-11-16 00:03:43 +01:00
sent ,
2018-11-19 07:57:20 +01:00
// eslint-disable-next-line camelcase
2018-07-27 03:13:56 +02:00
sent_at ,
source ,
2018-08-07 21:33:56 +02:00
type ,
2018-07-27 03:13:56 +02:00
unread ,
2018-08-07 21:33:56 +02:00
expireTimer ,
expirationStartTimestamp ,
2018-07-27 03:13:56 +02:00
} = data ;
2021-02-19 03:01:24 +01:00
if ( ! id ) {
throw new Error ( 'id is required' ) ;
}
if ( ! conversationId ) {
throw new Error ( 'conversationId is required' ) ;
}
2018-08-07 21:33:56 +02:00
const payload = {
2021-07-08 05:00:20 +02:00
id ,
json : objectToJSON ( data ) ,
serverId ,
serverTimestamp ,
body ,
conversationId ,
expirationStartTimestamp ,
expires_at ,
expireTimer ,
hasAttachments ,
hasFileAttachments ,
hasVisualMediaAttachments ,
received_at ,
sent ,
sent_at ,
source ,
type : type || '' ,
unread ,
2018-08-07 21:33:56 +02:00
} ;
2022-03-29 06:18:26 +02:00
assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
` INSERT OR REPLACE INTO ${ MESSAGES_TABLE } (
2018-07-27 03:13:56 +02:00
id ,
json ,
2019-08-09 05:34:48 +02:00
serverId ,
2020-09-04 07:11:24 +02:00
serverTimestamp ,
2019-01-14 22:47:19 +01:00
body ,
2018-07-27 03:13:56 +02:00
conversationId ,
2018-08-07 21:33:56 +02:00
expirationStartTimestamp ,
2018-07-27 03:13:56 +02:00
expires_at ,
2018-08-07 21:33:56 +02:00
expireTimer ,
2018-07-27 03:13:56 +02:00
hasAttachments ,
hasFileAttachments ,
hasVisualMediaAttachments ,
received_at ,
2018-11-16 00:03:43 +01:00
sent ,
2018-07-27 03:13:56 +02:00
sent_at ,
source ,
2018-08-07 21:33:56 +02:00
type ,
2018-07-27 03:13:56 +02:00
unread
) values (
$id ,
$json ,
2019-08-09 05:34:48 +02:00
$serverId ,
2020-09-04 07:11:24 +02:00
$serverTimestamp ,
2019-01-14 22:47:19 +01:00
$body ,
2018-07-27 03:13:56 +02:00
$conversationId ,
2018-08-07 21:33:56 +02:00
$expirationStartTimestamp ,
2018-07-27 03:13:56 +02:00
$expires_at ,
2018-08-07 21:33:56 +02:00
$expireTimer ,
2018-07-27 03:13:56 +02:00
$hasAttachments ,
$hasFileAttachments ,
$hasVisualMediaAttachments ,
$received_at ,
2018-11-16 00:03:43 +01:00
$sent ,
2018-07-27 03:13:56 +02:00
$sent_at ,
$source ,
2018-08-07 21:33:56 +02:00
$type ,
2018-07-27 03:13:56 +02:00
$unread
2021-07-08 05:00:20 +02:00
) ; `
)
. run ( payload ) ;
2018-07-27 03:13:56 +02:00
2021-02-19 03:01:24 +01:00
return id ;
2018-07-27 03:13:56 +02:00
}
2022-03-29 06:18:26 +02:00
function saveSeenMessageHashes ( arrayOfHashes : Array < string > ) {
assertGlobalInstance ( ) . transaction ( ( ) = > {
map ( arrayOfHashes , saveSeenMessageHash ) ;
2021-07-08 05:00:20 +02:00
} ) ( ) ;
2018-11-19 07:57:20 +01:00
}
2022-05-04 06:54:38 +02:00
function updateLastHash ( data : UpdateLastHashType ) {
const { convoId , snode , hash , expiresAt , namespace } = data ;
if ( ! isNumber ( namespace ) ) {
throw new Error ( 'updateLastHash: namespace must be set to a number' ) ;
}
2022-03-29 06:18:26 +02:00
assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
2022-05-04 06:54:38 +02:00
` INSERT OR REPLACE INTO ${ LAST_HASHES_TABLE } (
2020-05-04 05:24:53 +02:00
id ,
2019-04-15 05:19:06 +02:00
snode ,
hash ,
2022-05-04 06:54:38 +02:00
expiresAt ,
namespace
2019-04-15 05:19:06 +02:00
) values (
2020-05-04 05:24:53 +02:00
$id ,
2019-04-15 05:19:06 +02:00
$snode ,
$hash ,
2022-05-04 06:54:38 +02:00
$expiresAt ,
$namespace
2021-07-08 05:00:20 +02:00
) `
)
. run ( {
2022-05-04 06:54:38 +02:00
id : convoId ,
2021-07-08 05:00:20 +02:00
snode ,
hash ,
expiresAt ,
2022-05-04 06:54:38 +02:00
namespace ,
2021-07-08 05:00:20 +02:00
} ) ;
2019-04-15 05:19:06 +02:00
}
2022-03-29 06:18:26 +02:00
function saveSeenMessageHash ( data : any ) {
2019-01-16 05:44:13 +01:00
const { expiresAt , hash } = data ;
2021-07-30 02:36:25 +02:00
try {
2022-03-29 06:18:26 +02:00
assertGlobalInstance ( )
2021-07-30 02:36:25 +02:00
. prepare (
` INSERT INTO seenMessages (
2018-11-19 07:57:20 +01:00
expiresAt ,
hash
2020-06-30 10:04:17 +02:00
) values (
$expiresAt ,
$hash
2021-07-08 05:00:20 +02:00
) ; `
2021-07-30 02:36:25 +02:00
)
. run ( {
expiresAt ,
hash ,
} ) ;
} catch ( e ) {
2022-01-20 06:48:59 +01:00
console . error ( 'saveSeenMessageHash failed:' , e . message ) ;
2021-07-30 02:36:25 +02:00
}
2018-11-19 07:57:20 +01:00
}
2021-07-08 05:00:20 +02:00
function cleanLastHashes() {
2022-03-29 06:18:26 +02:00
assertGlobalInstance ( )
2022-05-04 06:54:38 +02:00
. prepare ( ` DELETE FROM ${ LAST_HASHES_TABLE } WHERE expiresAt <= $ now; ` )
2022-03-29 06:18:26 +02:00
. run ( {
now : Date.now ( ) ,
} ) ;
2019-04-15 05:19:06 +02:00
}
2021-07-08 05:00:20 +02:00
function cleanSeenMessages() {
2022-03-29 06:18:26 +02:00
assertGlobalInstance ( )
. prepare ( 'DELETE FROM seenMessages WHERE expiresAt <= $now;' )
. run ( {
now : Date.now ( ) ,
} ) ;
2018-11-19 07:57:20 +01:00
}
2022-03-29 06:18:26 +02:00
function saveMessages ( arrayOfMessages : Array < any > ) {
console . info ( 'saveMessages of length: ' , arrayOfMessages . length ) ;
assertGlobalInstance ( ) . transaction ( ( ) = > {
map ( arrayOfMessages , saveMessage ) ;
2021-07-08 05:00:20 +02:00
} ) ( ) ;
2018-07-27 03:13:56 +02:00
}
2022-03-29 06:18:26 +02:00
function removeMessage ( id : string , instance? : BetterSqlite3.Database ) {
2022-10-06 06:15:14 +02:00
if ( ! isString ( id ) ) {
throw new Error ( 'removeMessage: only takes single message to delete!' ) ;
2018-07-27 03:13:56 +02:00
return ;
}
2022-10-06 06:15:14 +02:00
assertGlobalInstanceOrInstance ( instance )
. prepare ( ` DELETE FROM ${ MESSAGES_TABLE } WHERE id = $ id; ` )
. run ( { id } ) ;
}
function removeMessagesByIds ( ids : Array < string > , instance? : BetterSqlite3.Database ) {
if ( ! Array . isArray ( ids ) ) {
throw new Error ( 'removeMessagesByIds only allowed an array of strings' ) ;
}
if ( ! ids . length ) {
throw new Error ( 'removeMessagesByIds: No ids to delete!' ) ;
2018-07-27 03:13:56 +02:00
}
2023-06-27 07:52:25 +02:00
const start = Date . now ( ) ;
2018-07-27 03:13:56 +02:00
2022-03-29 06:18:26 +02:00
assertGlobalInstanceOrInstance ( instance )
2022-10-06 06:15:14 +02:00
. prepare ( ` DELETE FROM ${ MESSAGES_TABLE } WHERE id IN ( ${ ids . map ( ( ) = > '?' ) . join ( ', ' ) } ); ` )
. run ( ids ) ;
2023-06-27 07:52:25 +02:00
console . log ( ` removeMessagesByIds of length ${ ids . length } took ${ Date . now ( ) - start } ms ` ) ;
2022-10-06 06:15:14 +02:00
}
function removeAllMessagesInConversation (
conversationId : string ,
instance? : BetterSqlite3.Database
) {
if ( ! conversationId ) {
return ;
}
2023-06-27 07:52:25 +02:00
const inst = assertGlobalInstanceOrInstance ( instance ) ;
2022-10-06 06:15:14 +02:00
2023-06-27 11:08:00 +02:00
inst
. prepare ( ` DELETE FROM ${ MESSAGES_TABLE } WHERE conversationId = $ conversationId ` )
. run ( { conversationId } ) ;
2018-07-27 03:13:56 +02:00
}
2022-03-29 06:18:26 +02:00
function getMessageIdsFromServerIds ( serverIds : Array < string | number > , conversationId : string ) {
2020-04-20 01:52:47 +02:00
if ( ! Array . isArray ( serverIds ) ) {
return [ ] ;
}
// Sanitize the input as we're going to use it directly in the query
2022-08-08 01:50:48 +02:00
const validServerIds = serverIds . map ( Number ) . filter ( n = > ! Number . isNaN ( n ) ) ;
2020-04-20 01:52:47 +02:00
/ *
Sqlite3 doesn ' t have a good way to have ` IN ` query with another query .
See : https : //github.com/mapbox/node-sqlite3/issues/762.
So we have to use templating to insert the values .
* /
2022-03-29 06:18:26 +02:00
const rows = assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
` SELECT id FROM ${ MESSAGES_TABLE } WHERE
2022-08-08 01:50:48 +02:00
serverId IN ( $ { validServerIds . join ( ',' ) } ) AND
2021-07-08 05:00:20 +02:00
conversationId = $conversationId ; `
)
. all ( {
conversationId ,
} ) ;
2020-04-20 01:52:47 +02:00
return rows . map ( row = > row . id ) ;
2019-08-09 05:34:48 +02:00
}
2022-03-29 06:18:26 +02:00
function getMessageById ( id : string ) {
const row = assertGlobalInstance ( )
. prepare ( ` SELECT * FROM ${ MESSAGES_TABLE } WHERE id = $ id; ` )
. get ( {
id ,
} ) ;
2018-07-27 03:13:56 +02:00
if ( ! row ) {
return null ;
}
return jsonToObject ( row . json ) ;
}
2023-01-19 07:27:32 +01:00
// serverIds are not unique so we need the conversationId
function getMessageByServerId ( conversationId : string , serverId : number ) {
2022-08-22 08:40:14 +02:00
const row = assertGlobalInstance ( )
2023-01-19 07:27:32 +01:00
. prepare (
` SELECT * FROM ${ MESSAGES_TABLE } WHERE conversationId = $ conversationId AND serverId = $ serverId; `
)
2022-08-22 08:40:14 +02:00
. get ( {
2023-01-19 07:27:32 +01:00
conversationId ,
2022-08-22 08:40:14 +02:00
serverId ,
} ) ;
if ( ! row ) {
return null ;
}
return jsonToObject ( row . json ) ;
}
2022-04-14 02:56:34 +02:00
function getMessagesCountBySender ( { source } : { source : string } ) {
if ( ! source ) {
throw new Error ( 'source must be set' ) ;
}
const count = assertGlobalInstance ( )
. prepare (
` SELECT count(*) FROM ${ MESSAGES_TABLE } WHERE
source = $source ; `
)
. get ( {
source ,
} ) ;
if ( ! count ) {
return 0 ;
}
return count [ 'count(*)' ] || 0 ;
}
2023-05-11 09:29:01 +02:00
function getMessagesBySenderAndSentAt (
propsList : Array < {
source : string ;
timestamp : number ;
} >
) {
const db = assertGlobalInstance ( ) ;
const rows = [ ] ;
2023-07-26 11:26:46 +02:00
// eslint-disable-next-line no-restricted-syntax
2023-05-15 07:24:30 +02:00
for ( const msgProps of propsList ) {
const { source , timestamp } = msgProps ;
2023-05-11 09:29:01 +02:00
const _rows = db
. prepare (
` SELECT json FROM ${ MESSAGES_TABLE } WHERE
2021-09-20 05:47:59 +02:00
source = $source AND
sent_at = $timestamp ; `
2023-05-11 09:29:01 +02:00
)
. all ( {
source ,
timestamp ,
} ) ;
rows . push ( . . . _rows ) ;
}
2021-09-20 05:47:59 +02:00
2023-05-18 09:24:56 +02:00
return uniq ( map ( rows , row = > jsonToObject ( row . json ) ) ) ;
2021-09-20 05:47:59 +02:00
}
2022-04-13 05:52:05 +02:00
function filterAlreadyFetchedOpengroupMessage (
2023-01-25 07:46:30 +01:00
msgDetails : MsgDuplicateSearchOpenGroup
) : MsgDuplicateSearchOpenGroup {
2022-08-08 01:50:48 +02:00
const filteredNonBlinded = msgDetails . filter ( msg = > {
2022-04-13 05:52:05 +02:00
const rows = assertGlobalInstance ( )
. prepare (
` SELECT source, serverTimestamp FROM ${ MESSAGES_TABLE } WHERE
source = $sender AND
2021-07-08 05:00:20 +02:00
serverTimestamp = $serverTimestamp ; `
2022-04-13 05:52:05 +02:00
)
. all ( {
sender : msg.sender ,
serverTimestamp : msg.serverTimestamp ,
} ) ;
if ( rows . length ) {
console . info (
2022-04-13 08:51:53 +02:00
` filtering out already received sogs message from ${ msg . sender } at ${ msg . serverTimestamp } `
2022-04-13 05:52:05 +02:00
) ;
return false ;
}
return true ;
} ) ;
2022-08-08 01:50:48 +02:00
return filteredNonBlinded ;
2021-06-07 07:54:44 +02:00
}
2023-03-16 00:49:06 +01:00
function getUnreadByConversation ( conversationId : string , sentBeforeTimestamp : number ) {
2022-03-29 06:18:26 +02:00
const rows = assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
2022-08-08 01:50:48 +02:00
` SELECT * FROM ${ MESSAGES_TABLE } WHERE
2018-08-09 03:32:10 +02:00
unread = $unread AND
2023-03-16 00:49:06 +01:00
conversationId = $conversationId AND
COALESCE ( serverTimestamp , sent_at ) <= $sentBeforeTimestamp
$ { orderByClauseASC } ; `
2021-07-08 05:00:20 +02:00
)
. all ( {
2023-03-10 01:14:23 +01:00
unread : toSqliteBoolean ( true ) ,
2021-07-08 05:00:20 +02:00
conversationId ,
2023-03-16 00:49:06 +01:00
sentBeforeTimestamp ,
2021-07-08 05:00:20 +02:00
} ) ;
2018-07-27 03:13:56 +02:00
return map ( rows , row = > jsonToObject ( row . json ) ) ;
}
2022-05-20 05:19:48 +02:00
/ * *
* Warning : This does not start expiration timer
* /
function markAllAsReadByConversationNoExpiration (
2022-10-17 08:15:08 +02:00
conversationId : string ,
returnMessagesUpdated : boolean
) : Array < number > {
let toReturn : Array < number > = [ ] ;
if ( returnMessagesUpdated ) {
const messagesUnreadBefore = assertGlobalInstance ( )
. prepare (
` SELECT json FROM ${ MESSAGES_TABLE } WHERE
unread = $unread AND
conversationId = $conversationId ; `
)
. all ( {
2023-03-10 01:14:23 +01:00
unread : toSqliteBoolean ( true ) ,
2022-10-17 08:15:08 +02:00
conversationId ,
} ) ;
toReturn = compact ( messagesUnreadBefore . map ( row = > jsonToObject ( row . json ) . sent_at ) ) ;
}
2022-05-20 05:19:48 +02:00
assertGlobalInstance ( )
. prepare (
` UPDATE ${ MESSAGES_TABLE } SET
unread = 0 , json = json_set ( json , '$.unread' , 0 )
WHERE unread = $unread AND
conversationId = $conversationId ; `
)
. run ( {
2023-03-10 01:14:23 +01:00
unread : toSqliteBoolean ( true ) ,
2022-05-20 05:19:48 +02:00
conversationId ,
} ) ;
2022-10-17 08:15:08 +02:00
return toReturn ;
2022-05-20 05:19:48 +02:00
}
2023-03-16 00:49:06 +01:00
function getUnreadCountByConversation (
conversationId : string ,
instance? : BetterSqlite3.Database
) : number {
const row = assertGlobalInstanceOrInstance ( instance )
2021-07-08 05:00:20 +02:00
. prepare (
2022-08-08 01:50:48 +02:00
` SELECT count(*) FROM ${ MESSAGES_TABLE } WHERE
2020-10-30 06:02:18 +01:00
unread = $unread AND
2022-08-08 01:50:48 +02:00
conversationId = $conversationId ; `
2021-07-08 05:00:20 +02:00
)
. get ( {
2023-03-10 01:14:23 +01:00
unread : toSqliteBoolean ( true ) ,
2021-07-08 05:00:20 +02:00
conversationId ,
} ) ;
2020-10-30 06:02:18 +01:00
if ( ! row ) {
2023-03-10 06:39:48 +01:00
throw new Error ( ` Unable to get unread count of ${ conversationId } ` ) ;
2020-10-30 06:02:18 +01:00
}
return row [ 'count(*)' ] ;
}
2022-03-29 06:18:26 +02:00
function getMessageCountByType ( conversationId : string , type = '%' ) {
const row = assertGlobalInstance ( )
2022-02-14 13:01:50 +01:00
. prepare (
2022-03-10 00:51:15 +01:00
` SELECT count(*) from ${ MESSAGES_TABLE }
WHERE conversationId = $conversationId
2022-02-17 08:09:18 +01:00
AND type = $type ; `
2022-02-14 13:01:50 +01:00
)
. get ( {
conversationId ,
2022-02-17 08:09:18 +01:00
type ,
2022-02-14 13:01:50 +01:00
} ) ;
if ( ! row ) {
throw new Error (
` getIncomingMessagesCountByConversation: Unable to get incoming messages count of ${ conversationId } `
) ;
}
return row [ 'count(*)' ] ;
}
2020-01-13 05:37:55 +01:00
// Note: Sorting here is necessary for getting the last message (with limit 1)
2021-07-08 05:00:20 +02:00
// be sure to update the sorting order to sort messages on redux too (sortMessages)
2022-01-25 06:58:29 +01:00
const orderByClause = 'ORDER BY COALESCE(serverTimestamp, sent_at, received_at) DESC' ;
2022-01-27 06:22:53 +01:00
const orderByClauseASC = 'ORDER BY COALESCE(serverTimestamp, sent_at, received_at) ASC' ;
2022-01-19 07:24:20 +01:00
2023-05-11 09:29:01 +02:00
function getMessagesByConversation (
conversationId : string ,
{ messageId = null , returnQuotes = false } = { }
) : { messages : Array < Record < string , any > > ; quotes : Array < Quote > } {
2022-02-07 01:44:07 +01:00
const absLimit = 30 ;
2022-01-19 02:35:32 +01:00
// If messageId is given it means we are opening the conversation to that specific messageId,
// or that we just scrolled to it by a quote click and needs to load around it.
// If messageId is null, it means we are just opening the convo to the last unread message, or at the bottom
const firstUnread = getFirstUnreadMessageIdInConversation ( conversationId ) ;
2023-01-19 01:23:51 +01:00
const numberOfMessagesInConvo = getMessagesCountByConversation ( conversationId ) ;
2022-03-09 01:44:30 +01:00
const floorLoadAllMessagesInConvo = 70 ;
2023-05-11 09:29:01 +02:00
let messages : Array < Record < string , any > > = [ ] ;
let quotes = [ ] ;
2022-01-19 02:35:32 +01:00
if ( messageId || firstUnread ) {
const messageFound = getMessageById ( messageId || firstUnread ) ;
if ( messageFound && messageFound . conversationId === conversationId ) {
2023-06-22 14:12:30 +02:00
const start = Date . now ( ) ;
const msgTimestamp =
messageFound . serverTimestamp || messageFound . sent_at || messageFound . received_at ;
2022-01-19 07:24:20 +01:00
2023-06-22 14:12:30 +02:00
const commonArgs = {
conversationId ,
msgTimestamp ,
limit :
numberOfMessagesInConvo < floorLoadAllMessagesInConvo
? floorLoadAllMessagesInConvo
: absLimit ,
} ;
const messagesBefore = assertGlobalInstance ( )
. prepare (
` SELECT id, conversationId, json
FROM $ { MESSAGES_TABLE } WHERE conversationId = $conversationId AND COALESCE ( serverTimestamp , sent_at , received_at ) <= $msgTimestamp
$ { orderByClause }
LIMIT $limit `
2022-01-19 02:35:32 +01:00
)
2023-06-22 14:12:30 +02:00
. all ( commonArgs ) ;
const messagesAfter = assertGlobalInstance ( )
. prepare (
` SELECT id, conversationId, json
FROM $ { MESSAGES_TABLE } WHERE conversationId = $conversationId AND COALESCE ( serverTimestamp , sent_at , received_at ) > $msgTimestamp
$ { orderByClauseASC }
LIMIT $limit `
2022-01-19 02:35:32 +01:00
)
2023-06-22 14:12:30 +02:00
. all ( commonArgs ) ;
console . info ( ` getMessagesByConversation around took ${ Date . now ( ) - start } ms ` ) ;
// sorting is made in redux already when rendered, but some things are made outside of redux, so let's make sure the order is right
2023-07-04 11:46:10 +02:00
messages = map ( [ . . . messagesBefore , . . . messagesAfter ] , row = > jsonToObject ( row . json ) ) . sort (
2023-06-22 14:12:30 +02:00
( a , b ) = > {
return (
( b . serverTimestamp || b . sent_at || b . received_at ) -
( a . serverTimestamp || a . sent_at || a . received_at )
) ;
}
) ;
2022-01-19 02:35:32 +01:00
}
2022-01-20 06:48:59 +01:00
console . info (
2022-01-19 02:35:32 +01:00
` getMessagesByConversation: Could not find messageId ${ messageId } in db with conversationId: ${ conversationId } . Just fetching the convo as usual. `
) ;
2023-05-11 09:29:01 +02:00
} else {
const limit =
numberOfMessagesInConvo < floorLoadAllMessagesInConvo
? floorLoadAllMessagesInConvo
: absLimit * 2 ;
2022-01-19 07:24:20 +01:00
2023-05-11 09:29:01 +02:00
const rows = assertGlobalInstance ( )
. prepare (
`
2020-10-30 03:19:57 +01:00
SELECT json FROM $ { MESSAGES_TABLE } WHERE
2022-01-19 02:35:32 +01:00
conversationId = $conversationId
$ { orderByClause }
LIMIT $limit ;
`
2023-05-11 09:29:01 +02:00
)
. all ( {
conversationId ,
limit ,
} ) ;
2022-01-19 07:24:20 +01:00
2023-05-11 09:29:01 +02:00
messages = map ( rows , row = > jsonToObject ( row . json ) ) ;
}
if ( returnQuotes ) {
2023-05-15 05:54:18 +02:00
quotes = uniq ( messages . filter ( message = > message . quote ) . map ( message = > message . quote ) ) ;
2023-05-11 09:29:01 +02:00
}
return { messages , quotes } ;
2022-01-19 02:35:32 +01:00
}
2022-03-29 06:18:26 +02:00
function getLastMessagesByConversation ( conversationId : string , limit : number ) {
2022-01-19 02:35:32 +01:00
if ( ! isNumber ( limit ) ) {
throw new Error ( 'limit must be a number' ) ;
}
2022-03-29 06:18:26 +02:00
const rows = assertGlobalInstance ( )
2022-01-19 02:35:32 +01:00
. prepare (
`
SELECT json FROM $ { MESSAGES_TABLE } WHERE
conversationId = $conversationId
$ { orderByClause }
2018-11-13 05:35:45 +01:00
LIMIT $limit ;
2021-07-08 05:00:20 +02:00
`
)
. all ( {
conversationId ,
limit ,
} ) ;
2018-07-27 03:13:56 +02:00
return map ( rows , row = > jsonToObject ( row . json ) ) ;
}
2022-02-02 07:50:59 +01:00
/ * *
* This is the oldest message so we cannot reuse getLastMessagesByConversation
* /
2022-03-29 06:18:26 +02:00
function getOldestMessageInConversation ( conversationId : string ) {
const rows = assertGlobalInstance ( )
2022-01-27 06:22:53 +01:00
. prepare (
`
SELECT json FROM $ { MESSAGES_TABLE } WHERE
conversationId = $conversationId
$ { orderByClauseASC }
LIMIT $limit ;
`
)
. all ( {
conversationId ,
limit : 1 ,
} ) ;
return map ( rows , row = > jsonToObject ( row . json ) ) ;
}
2022-03-29 06:18:26 +02:00
function hasConversationOutgoingMessage ( conversationId : string ) {
const row = assertGlobalInstance ( )
2021-12-08 04:15:54 +01:00
. prepare (
`
SELECT count ( * ) FROM $ { MESSAGES_TABLE } WHERE
conversationId = $conversationId AND
type IS 'outgoing'
`
)
. get ( {
conversationId ,
} ) ;
if ( ! row ) {
throw new Error ( 'hasConversationOutgoingMessage: Unable to get coun' ) ;
}
return Boolean ( row [ 'count(*)' ] ) ;
}
2022-03-29 06:18:26 +02:00
function getFirstUnreadMessageIdInConversation ( conversationId : string ) {
const rows = assertGlobalInstance ( )
2021-07-22 02:20:09 +02:00
. prepare (
`
SELECT id FROM $ { MESSAGES_TABLE } WHERE
conversationId = $conversationId AND
unread = $unread
ORDER BY serverTimestamp ASC , serverId ASC , sent_at ASC , received_at ASC
LIMIT 1 ;
`
)
. all ( {
conversationId ,
2023-03-16 00:49:06 +01:00
unread : toSqliteBoolean ( true ) ,
2021-07-22 02:20:09 +02:00
} ) ;
if ( rows . length === 0 ) {
return undefined ;
}
return rows [ 0 ] . id ;
}
2023-03-16 00:49:06 +01:00
/ * *
* Returns the last read message timestamp in the specific conversation ( the columns ` serverTimestamp ` || ` sent_at ` )
* /
function getLastMessageReadInConversation ( conversationId : string ) : number | null {
const rows = assertGlobalInstance ( )
. prepare (
`
SELECT MAX ( MAX ( COALESCE ( serverTimestamp , 0 ) ) , MAX ( COALESCE ( sent_at , 0 ) ) ) AS max_sent_at
FROM $ { MESSAGES_TABLE } WHERE
conversationId = $conversationId AND
unread = $unread ;
`
)
. get ( {
conversationId ,
unread : toSqliteBoolean ( false ) , // we want to find the message read with the higher sent_at timestamp
} ) ;
return rows ? . max_sent_at || null ;
}
2022-08-08 01:50:48 +02:00
function getFirstUnreadMessageWithMention (
conversationId : string ,
2023-03-16 00:49:06 +01:00
instance? : BetterSqlite3.Database
2022-08-08 01:50:48 +02:00
) : string | undefined {
2023-03-16 00:49:06 +01:00
const ourPkInThatConversation = getUsBlindedInThatServerIfNeeded ( conversationId , instance ) ;
if ( ! ourPkInThatConversation || ! ourPkInThatConversation . length ) {
2022-02-07 01:44:07 +01:00
throw new Error ( 'getFirstUnreadMessageWithMention needs our pubkey but nothing was given' ) ;
}
2023-03-16 00:49:06 +01:00
const likeMatch = ` %@ ${ ourPkInThatConversation } % ` ;
2022-02-07 01:44:07 +01:00
2023-03-30 05:15:59 +02:00
// TODOLATER make this use the fts search table rather than this one?
2023-03-16 00:49:06 +01:00
const rows = assertGlobalInstanceOrInstance ( instance )
2022-02-07 01:44:07 +01:00
. prepare (
`
2023-03-30 05:15:59 +02:00
SELECT id FROM $ { MESSAGES_TABLE } WHERE
2022-02-07 01:44:07 +01:00
conversationId = $conversationId AND
unread = $unread AND
body LIKE $likeMatch
ORDER BY serverTimestamp ASC , serverId ASC , sent_at ASC , received_at ASC
LIMIT 1 ;
`
)
. all ( {
conversationId ,
2023-03-16 00:49:06 +01:00
unread : toSqliteBoolean ( true ) ,
2022-02-07 01:44:07 +01:00
likeMatch ,
} ) ;
if ( rows . length === 0 ) {
return undefined ;
}
return rows [ 0 ] . id ;
}
2022-03-29 06:18:26 +02:00
function getMessagesBySentAt ( sentAt : number ) {
const rows = assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
2022-05-20 05:19:48 +02:00
` SELECT json FROM ${ MESSAGES_TABLE }
2018-07-27 03:13:56 +02:00
WHERE sent_at = $sent_at
2021-07-08 05:00:20 +02:00
ORDER BY received_at DESC ; `
)
. all ( {
sent_at : sentAt ,
} ) ;
2018-07-27 03:13:56 +02:00
return map ( rows , row = > jsonToObject ( row . json ) ) ;
}
2022-05-04 06:54:38 +02:00
function getLastHashBySnode ( convoId : string , snode : string , namespace : number ) {
if ( ! isNumber ( namespace ) ) {
throw new Error ( 'getLastHashBySnode: namespace must be set to a number' ) ;
}
2022-03-29 06:18:26 +02:00
const row = assertGlobalInstance ( )
2022-05-04 06:54:38 +02:00
. prepare (
` SELECT * FROM ${ LAST_HASHES_TABLE } WHERE snode = $ snode AND id = $ id AND namespace = $ namespace; `
)
2021-07-08 05:00:20 +02:00
. get ( {
2021-07-12 08:00:04 +02:00
snode ,
2021-07-08 05:00:20 +02:00
id : convoId ,
2022-05-04 06:54:38 +02:00
namespace ,
2021-07-08 05:00:20 +02:00
} ) ;
2019-04-15 05:19:06 +02:00
if ( ! row ) {
return null ;
}
2020-05-04 05:24:53 +02:00
return row . hash ;
2019-04-15 05:19:06 +02:00
}
2022-03-29 06:18:26 +02:00
function getSeenMessagesByHashList ( hashes : Array < string > ) {
const rows = assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare ( ` SELECT * FROM seenMessages WHERE hash IN ( ${ hashes . map ( ( ) = > '?' ) . join ( ', ' ) } ); ` )
. all ( hashes ) ;
2018-11-19 07:57:20 +01:00
return map ( rows , row = > row . hash ) ;
}
2021-07-08 05:00:20 +02:00
function getExpiredMessages() {
2018-07-27 03:13:56 +02:00
const now = Date . now ( ) ;
2022-03-29 06:18:26 +02:00
const rows = assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
` SELECT json FROM ${ MESSAGES_TABLE } WHERE
2018-07-27 03:13:56 +02:00
expires_at IS NOT NULL AND
expires_at <= $expires_at
2021-07-08 05:00:20 +02:00
ORDER BY expires_at ASC ; `
)
. all ( {
expires_at : now ,
} ) ;
2018-07-27 03:13:56 +02:00
return map ( rows , row = > jsonToObject ( row . json ) ) ;
}
2021-07-08 05:00:20 +02:00
function getOutgoingWithoutExpiresAt() {
2022-03-29 06:18:26 +02:00
const rows = assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
`
2020-10-30 03:19:57 +01:00
SELECT json FROM $ { MESSAGES_TABLE }
2018-08-07 21:33:56 +02:00
WHERE
2018-08-09 03:32:10 +02:00
expireTimer > 0 AND
expires_at IS NULL AND
type IS 'outgoing'
2018-08-07 21:33:56 +02:00
ORDER BY expires_at ASC ;
2021-07-08 05:00:20 +02:00
`
)
. all ( ) ;
2018-08-07 21:33:56 +02:00
return map ( rows , row = > jsonToObject ( row . json ) ) ;
}
2021-07-08 05:00:20 +02:00
function getNextExpiringMessage() {
2022-03-29 06:18:26 +02:00
const rows = assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
`
2020-10-30 03:19:57 +01:00
SELECT json FROM $ { MESSAGES_TABLE }
2018-08-09 03:32:10 +02:00
WHERE expires_at > 0
2018-07-27 03:13:56 +02:00
ORDER BY expires_at ASC
LIMIT 1 ;
2021-07-08 05:00:20 +02:00
`
)
. all ( ) ;
2018-07-27 03:13:56 +02:00
return map ( rows , row = > jsonToObject ( row . json ) ) ;
}
2021-02-19 01:57:23 +01:00
/* Unproccessed a received messages not yet processed */
2023-01-25 07:46:30 +01:00
const unprocessed : UnprocessedDataNode = {
saveUnprocessed : ( data : UnprocessedParameter ) = > {
const { id , timestamp , version , attempts , envelope , senderIdentity , messageHash } = data ;
if ( ! id ) {
throw new Error ( ` saveUnprocessed: id was falsey: ${ id } ` ) ;
}
2018-07-27 03:13:56 +02:00
2023-01-25 07:46:30 +01:00
assertGlobalInstance ( )
. prepare (
` INSERT OR REPLACE INTO unprocessed (
id ,
timestamp ,
version ,
attempts ,
envelope ,
senderIdentity ,
serverHash
) values (
$id ,
$timestamp ,
$version ,
$attempts ,
$envelope ,
$senderIdentity ,
$messageHash
) ; `
)
. run ( {
id ,
timestamp ,
version ,
attempts ,
envelope ,
senderIdentity ,
messageHash ,
} ) ;
} ,
2018-07-27 03:13:56 +02:00
2023-01-25 07:46:30 +01:00
updateUnprocessedAttempts : ( id : string , attempts : number ) = > {
assertGlobalInstance ( )
. prepare ( 'UPDATE unprocessed SET attempts = $attempts WHERE id = $id;' )
. run ( {
id ,
attempts ,
} ) ;
} ,
2018-07-27 03:13:56 +02:00
2023-01-25 07:46:30 +01:00
updateUnprocessedWithData : ( id : string , data : UnprocessedParameter ) = > {
const { source , decrypted , senderIdentity } = data ;
2018-07-27 03:13:56 +02:00
2023-01-25 07:46:30 +01:00
assertGlobalInstance ( )
. prepare (
` UPDATE unprocessed SET
source = $source ,
decrypted = $decrypted ,
senderIdentity = $senderIdentity
WHERE id = $id ; `
)
. run ( {
id ,
source ,
decrypted ,
senderIdentity ,
} ) ;
} ,
2018-07-27 03:13:56 +02:00
2023-01-25 07:46:30 +01:00
getUnprocessedById : ( id : string ) = > {
const row = assertGlobalInstance ( )
. prepare ( 'SELECT * FROM unprocessed WHERE id = $id;' )
. get ( {
id ,
} ) ;
2019-02-05 02:23:50 +01:00
2023-01-25 07:46:30 +01:00
return row ;
} ,
2018-07-27 03:13:56 +02:00
2023-01-25 07:46:30 +01:00
getUnprocessedCount : ( ) = > {
const row = assertGlobalInstance ( )
. prepare ( 'SELECT count(*) from unprocessed;' )
. get ( ) ;
2018-09-29 00:51:26 +02:00
2023-01-25 07:46:30 +01:00
if ( ! row ) {
throw new Error ( 'getMessageCount: Unable to get count of unprocessed' ) ;
}
2018-09-29 00:51:26 +02:00
2023-01-25 07:46:30 +01:00
return row [ 'count(*)' ] ;
} ,
2018-09-29 00:51:26 +02:00
2023-01-25 07:46:30 +01:00
getAllUnprocessed : ( ) = > {
const rows = assertGlobalInstance ( )
. prepare ( 'SELECT * FROM unprocessed ORDER BY timestamp ASC;' )
. all ( ) ;
2018-07-27 03:13:56 +02:00
2023-01-25 07:46:30 +01:00
return rows ;
} ,
2018-07-27 03:13:56 +02:00
2023-01-25 07:46:30 +01:00
removeUnprocessed : ( id : string ) : void = > {
if ( Array . isArray ( id ) ) {
2023-05-10 05:29:38 +02:00
console . error ( 'removeUnprocessed only supports single ids at a time' ) ;
2023-01-25 07:46:30 +01:00
throw new Error ( 'removeUnprocessed only supports single ids at a time' ) ;
}
2022-03-29 06:18:26 +02:00
assertGlobalInstance ( )
. prepare ( 'DELETE FROM unprocessed WHERE id = $id;' )
. run ( { id } ) ;
2023-01-25 07:46:30 +01:00
} ,
2018-07-27 03:13:56 +02:00
2023-01-25 07:46:30 +01:00
removeAllUnprocessed : ( ) = > {
assertGlobalInstance ( )
. prepare ( 'DELETE FROM unprocessed;' )
. run ( ) ;
} ,
} ;
2018-07-27 03:13:56 +02:00
2022-03-29 06:18:26 +02:00
function getNextAttachmentDownloadJobs ( limit : number ) {
const timestamp = Date . now ( ) ;
2019-01-30 21:15:07 +01:00
2022-03-29 06:18:26 +02:00
const rows = assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
` SELECT json FROM ${ ATTACHMENT_DOWNLOADS_TABLE }
2019-01-30 21:15:07 +01:00
WHERE pending = 0 AND timestamp < $timestamp
ORDER BY timestamp DESC
2021-07-08 05:00:20 +02:00
LIMIT $limit ; `
)
. all ( {
limit ,
timestamp ,
} ) ;
2019-01-30 21:15:07 +01:00
return map ( rows , row = > jsonToObject ( row . json ) ) ;
}
2021-07-08 05:00:20 +02:00
2022-03-29 06:18:26 +02:00
function saveAttachmentDownloadJob ( job : any ) {
2019-01-30 21:15:07 +01:00
const { id , pending , timestamp } = job ;
if ( ! id ) {
2021-04-22 10:03:58 +02:00
throw new Error ( 'saveAttachmentDownloadJob: Provided job did not have a truthy id' ) ;
2019-01-30 21:15:07 +01:00
}
2022-03-29 06:18:26 +02:00
assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
` INSERT OR REPLACE INTO ${ ATTACHMENT_DOWNLOADS_TABLE } (
2019-01-30 21:15:07 +01:00
id ,
pending ,
timestamp ,
json
) values (
$id ,
$pending ,
$timestamp ,
$json
2021-07-08 05:00:20 +02:00
) `
)
. run ( {
id ,
pending ,
timestamp ,
json : objectToJSON ( job ) ,
} ) ;
2019-01-30 21:15:07 +01:00
}
2021-07-08 05:00:20 +02:00
2022-03-29 06:18:26 +02:00
function setAttachmentDownloadJobPending ( id : string , pending : 1 | 0 ) {
assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare ( ` UPDATE ${ ATTACHMENT_DOWNLOADS_TABLE } SET pending = $ pending WHERE id = $ id; ` )
. run ( {
id ,
pending ,
} ) ;
2019-01-30 21:15:07 +01:00
}
2021-07-08 05:00:20 +02:00
function resetAttachmentDownloadPending() {
2022-03-29 06:18:26 +02:00
assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare ( ` UPDATE ${ ATTACHMENT_DOWNLOADS_TABLE } SET pending = 0 WHERE pending != 0; ` )
. run ( ) ;
2019-01-30 21:15:07 +01:00
}
2022-03-29 06:18:26 +02:00
function removeAttachmentDownloadJob ( id : string ) {
removeById ( ATTACHMENT_DOWNLOADS_TABLE , id ) ;
2019-01-30 21:15:07 +01:00
}
2021-07-08 05:00:20 +02:00
function removeAllAttachmentDownloadJobs() {
2022-03-29 06:18:26 +02:00
assertGlobalInstance ( ) . exec ( ` DELETE FROM ${ ATTACHMENT_DOWNLOADS_TABLE } ; ` ) ;
2019-01-30 21:15:07 +01:00
}
2018-11-07 05:16:49 +01:00
// All data in database
2021-07-08 05:00:20 +02:00
function removeAll() {
2022-03-29 06:18:26 +02:00
assertGlobalInstance ( ) . exec ( `
2021-07-08 05:00:20 +02:00
DELETE FROM $ { IDENTITY_KEYS_TABLE } ;
DELETE FROM $ { ITEMS_TABLE } ;
DELETE FROM unprocessed ;
2022-05-04 06:54:38 +02:00
DELETE FROM $ { LAST_HASHES_TABLE } ;
2021-07-08 05:00:20 +02:00
DELETE FROM $ { NODES_FOR_PUBKEY_TABLE } ;
DELETE FROM $ { CLOSED_GROUP_V2_KEY_PAIRS_TABLE } ;
DELETE FROM seenMessages ;
DELETE FROM $ { CONVERSATIONS_TABLE } ;
DELETE FROM $ { MESSAGES_TABLE } ;
DELETE FROM $ { ATTACHMENT_DOWNLOADS_TABLE } ;
DELETE FROM $ { MESSAGES_FTS_TABLE } ;
2023-02-20 05:11:04 +01:00
DELETE FROM $ { CONFIG_DUMP_TABLE } ;
2021-07-08 05:00:20 +02:00
` );
}
function removeAllConversations() {
2022-03-29 06:18:26 +02:00
assertGlobalInstance ( )
. prepare ( ` DELETE FROM ${ CONVERSATIONS_TABLE } ; ` )
. run ( ) ;
2021-07-08 05:00:20 +02:00
}
2022-03-29 06:18:26 +02:00
function getMessagesWithVisualMediaAttachments ( conversationId : string , limit? : number ) {
const rows = assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
` SELECT json FROM ${ MESSAGES_TABLE } WHERE
2018-07-27 03:13:56 +02:00
conversationId = $conversationId AND
hasVisualMediaAttachments = 1
ORDER BY received_at DESC
2021-07-08 05:00:20 +02:00
LIMIT $limit ; `
)
. all ( {
conversationId ,
limit ,
} ) ;
2018-07-27 03:13:56 +02:00
return map ( rows , row = > jsonToObject ( row . json ) ) ;
}
2022-03-29 06:18:26 +02:00
function getMessagesWithFileAttachments ( conversationId : string , limit : number ) {
const rows = assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
` SELECT json FROM ${ MESSAGES_TABLE } WHERE
2018-07-27 03:13:56 +02:00
conversationId = $conversationId AND
hasFileAttachments = 1
ORDER BY received_at DESC
2021-07-08 05:00:20 +02:00
LIMIT $limit ; `
)
. all ( {
conversationId ,
limit ,
} ) ;
2018-07-27 03:13:56 +02:00
return map ( rows , row = > jsonToObject ( row . json ) ) ;
}
2018-08-07 01:18:58 +02:00
2022-03-29 06:18:26 +02:00
function getExternalFilesForMessage ( message : any ) {
2022-08-08 01:50:48 +02:00
const { attachments , quote , preview } = message ;
2022-03-29 06:18:26 +02:00
const files : Array < any > = [ ] ;
2018-08-07 01:18:58 +02:00
forEach ( attachments , attachment = > {
const { path : file , thumbnail , screenshot } = attachment ;
if ( file ) {
files . push ( file ) ;
}
if ( thumbnail && thumbnail . path ) {
files . push ( thumbnail . path ) ;
}
if ( screenshot && screenshot . path ) {
files . push ( screenshot . path ) ;
}
} ) ;
if ( quote && quote . attachments && quote . attachments . length ) {
forEach ( quote . attachments , attachment = > {
const { thumbnail } = attachment ;
if ( thumbnail && thumbnail . path ) {
files . push ( thumbnail . path ) ;
}
} ) ;
}
2019-01-16 04:03:56 +01:00
if ( preview && preview . length ) {
forEach ( preview , item = > {
const { image } = item ;
if ( image && image . path ) {
files . push ( image . path ) ;
}
} ) ;
}
2018-08-07 01:18:58 +02:00
return files ;
}
2022-08-08 01:50:48 +02:00
function getExternalFilesForConversation (
conversationAvatar :
| string
| {
path? : string | undefined ;
}
| undefined
) {
2018-09-21 03:47:19 +02:00
const files = [ ] ;
2022-08-08 01:50:48 +02:00
if ( isString ( conversationAvatar ) ) {
files . push ( conversationAvatar ) ;
2018-09-21 03:47:19 +02:00
}
2022-08-08 01:50:48 +02:00
if ( isObject ( conversationAvatar ) ) {
const avatarObj = conversationAvatar as Record < string , any > ;
if ( isString ( avatarObj . path ) ) {
files . push ( avatarObj . path ) ;
}
2018-09-21 03:47:19 +02:00
}
return files ;
}
2022-08-08 01:50:48 +02:00
function removeKnownAttachments ( allAttachments : Array < string > ) {
2018-08-07 01:18:58 +02:00
const lookup = fromPairs ( map ( allAttachments , file = > [ file , true ] ) ) ;
const chunkSize = 50 ;
2021-07-08 05:00:20 +02:00
const total = getMessageCount ( ) ;
2021-04-22 10:03:58 +02:00
console . log ( ` removeKnownAttachments: About to iterate through ${ total } messages ` ) ;
2018-08-07 01:18:58 +02:00
let count = 0 ;
let complete = false ;
let id = '' ;
while ( ! complete ) {
2022-03-29 06:18:26 +02:00
const rows = assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
` SELECT json FROM ${ MESSAGES_TABLE }
2018-08-07 01:18:58 +02:00
WHERE id > $id
ORDER BY id ASC
2021-07-08 05:00:20 +02:00
LIMIT $chunkSize ; `
)
. all ( {
id ,
chunkSize ,
} ) ;
2018-08-07 01:18:58 +02:00
const messages = map ( rows , row = > jsonToObject ( row . json ) ) ;
forEach ( messages , message = > {
const externalFiles = getExternalFilesForMessage ( message ) ;
forEach ( externalFiles , file = > {
delete lookup [ file ] ;
} ) ;
} ) ;
const lastMessage = last ( messages ) ;
if ( lastMessage ) {
( { id } = lastMessage ) ;
}
complete = messages . length < chunkSize ;
count += messages . length ;
}
2021-06-11 04:04:07 +02:00
console . log ( ` removeKnownAttachments: Done processing ${ count } ${ MESSAGES_TABLE } ` ) ;
2018-08-07 01:18:58 +02:00
2018-09-21 03:47:19 +02:00
complete = false ;
count = 0 ;
// Though conversations.id is a string, this ensures that, when coerced, this
// value is still a string but it's smaller than every other string.
2022-03-29 06:18:26 +02:00
( id as any ) = 0 ;
2018-09-21 03:47:19 +02:00
2021-07-08 05:00:20 +02:00
const conversationTotal = getConversationCount ( ) ;
2018-09-21 03:47:19 +02:00
console . log (
2021-06-11 04:09:57 +02:00
` removeKnownAttachments: About to iterate through ${ conversationTotal } ${ CONVERSATIONS_TABLE } `
2018-09-21 03:47:19 +02:00
) ;
while ( ! complete ) {
2022-08-08 01:50:48 +02:00
const conversations = assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
2022-08-08 01:50:48 +02:00
` SELECT * FROM ${ CONVERSATIONS_TABLE }
2018-09-21 03:47:19 +02:00
WHERE id > $id
ORDER BY id ASC
2021-07-08 05:00:20 +02:00
LIMIT $chunkSize ; `
)
. all ( {
id ,
chunkSize ,
} ) ;
2018-09-21 03:47:19 +02:00
forEach ( conversations , conversation = > {
2022-08-08 01:50:48 +02:00
const avatar = ( conversation as ConversationAttributes ) ? . avatarInProfile ;
const externalFiles = getExternalFilesForConversation ( avatar ) ;
2018-09-21 03:47:19 +02:00
forEach ( externalFiles , file = > {
delete lookup [ file ] ;
} ) ;
} ) ;
const lastMessage = last ( conversations ) ;
if ( lastMessage ) {
( { id } = lastMessage ) ;
}
complete = conversations . length < chunkSize ;
count += conversations . length ;
}
2021-06-11 04:09:57 +02:00
console . log ( ` removeKnownAttachments: Done processing ${ count } ${ CONVERSATIONS_TABLE } ` ) ;
2018-09-21 03:47:19 +02:00
2018-08-07 01:18:58 +02:00
return Object . keys ( lookup ) ;
}
2020-09-22 00:58:21 +02:00
2022-03-29 06:18:26 +02:00
function getMessagesCountByConversation (
conversationId : string ,
instance? : BetterSqlite3.Database | null
2022-04-14 02:56:34 +02:00
) : number {
2022-03-29 06:18:26 +02:00
const row = assertGlobalInstanceOrInstance ( instance )
2021-07-08 05:00:20 +02:00
. prepare ( ` SELECT count(*) from ${ MESSAGES_TABLE } WHERE conversationId = $ conversationId; ` )
. get ( { conversationId } ) ;
2020-09-22 07:36:39 +02:00
return row ? row [ 'count(*)' ] : 0 ;
}
2021-01-12 06:11:05 +01:00
/ * *
* The returned array is ordered based on the timestamp , the latest is at the end .
2022-03-29 06:18:26 +02:00
* @param groupPublicKey string | PubKey
2021-01-12 06:11:05 +01:00
* /
2023-03-08 07:39:29 +01:00
function getAllEncryptionKeyPairsForGroup (
groupPublicKey : string | PubKey ,
db? : BetterSqlite3.Database
) {
const rows = getAllEncryptionKeyPairsForGroupRaw ( groupPublicKey , db ) ;
2021-02-01 01:35:06 +01:00
return map ( rows , row = > jsonToObject ( row . json ) ) ;
}
2023-03-08 07:39:29 +01:00
function getAllEncryptionKeyPairsForGroupRaw (
groupPublicKey : string | PubKey ,
db? : BetterSqlite3.Database
) {
2022-03-29 06:18:26 +02:00
const pubkeyAsString = ( groupPublicKey as PubKey ) . key
? ( groupPublicKey as PubKey ) . key
: groupPublicKey ;
2023-03-08 07:39:29 +01:00
const rows = assertGlobalInstanceOrInstance ( db )
2021-07-08 05:00:20 +02:00
. prepare (
` SELECT * FROM ${ CLOSED_GROUP_V2_KEY_PAIRS_TABLE } WHERE groupPublicKey = $ groupPublicKey ORDER BY timestamp ASC; `
)
. all ( {
groupPublicKey : pubkeyAsString ,
} ) ;
2021-01-12 06:11:05 +01:00
2021-02-01 01:35:06 +01:00
return rows ;
2021-01-12 06:11:05 +01:00
}
2023-03-08 07:39:29 +01:00
function getLatestClosedGroupEncryptionKeyPair (
groupPublicKey : string ,
db? : BetterSqlite3.Database
) {
const rows = getAllEncryptionKeyPairsForGroup ( groupPublicKey , db ) ;
2021-01-12 06:11:05 +01:00
if ( ! rows || rows . length === 0 ) {
return undefined ;
}
return rows [ rows . length - 1 ] ;
}
2022-03-29 06:18:26 +02:00
function addClosedGroupEncryptionKeyPair (
groupPublicKey : string ,
keypair : object ,
instance? : BetterSqlite3.Database
) {
2021-01-12 06:11:05 +01:00
const timestamp = Date . now ( ) ;
2022-03-29 06:18:26 +02:00
assertGlobalInstanceOrInstance ( instance )
2021-07-08 05:00:20 +02:00
. prepare (
` INSERT OR REPLACE INTO ${ CLOSED_GROUP_V2_KEY_PAIRS_TABLE } (
2021-01-12 06:11:05 +01:00
groupPublicKey ,
timestamp ,
json
) values (
$groupPublicKey ,
$timestamp ,
$json
2021-07-08 05:00:20 +02:00
) ; `
)
. run ( {
groupPublicKey ,
timestamp ,
json : objectToJSON ( keypair ) ,
} ) ;
2021-01-12 06:11:05 +01:00
}
2022-03-29 06:18:26 +02:00
function removeAllClosedGroupEncryptionKeyPairs ( groupPublicKey : string ) {
assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare (
` DELETE FROM ${ CLOSED_GROUP_V2_KEY_PAIRS_TABLE } WHERE groupPublicKey = $ groupPublicKey `
)
. run ( {
groupPublicKey ,
} ) ;
2021-01-12 06:11:05 +01:00
}
2021-04-15 10:14:04 +02:00
/ * *
* Related to Opengroup V2
* /
2022-07-28 05:53:23 +02:00
function getAllV2OpenGroupRooms ( instance? : BetterSqlite3.Database ) : Array < OpenGroupV2Room > {
const rows = assertGlobalInstanceOrInstance ( instance )
2022-03-29 06:18:26 +02:00
. prepare ( ` SELECT json FROM ${ OPEN_GROUP_ROOMS_V2_TABLE } ; ` )
. all ( ) ;
2021-04-15 10:14:04 +02:00
2022-08-08 01:50:48 +02:00
if ( ! rows ) {
return [ ] ;
2021-04-15 10:14:04 +02:00
}
2022-07-28 05:53:23 +02:00
return rows . map ( r = > jsonToObject ( r . json ) ) as Array < OpenGroupV2Room > ;
2021-04-15 10:14:04 +02:00
}
2023-02-24 05:44:29 +01:00
function getV2OpenGroupRoom ( conversationId : string , db? : BetterSqlite3.Database ) {
const row = assertGlobalInstanceOrInstance ( db )
2021-07-08 05:00:20 +02:00
. prepare (
2022-08-08 01:50:48 +02:00
` SELECT json FROM ${ OPEN_GROUP_ROOMS_V2_TABLE } WHERE conversationId = $ conversationId; `
2021-07-08 05:00:20 +02:00
)
. get ( {
2022-08-08 01:50:48 +02:00
conversationId ,
2021-07-08 05:00:20 +02:00
} ) ;
2021-04-15 10:14:04 +02:00
if ( ! row ) {
return null ;
}
return jsonToObject ( row . json ) ;
}
2022-07-28 05:53:23 +02:00
function saveV2OpenGroupRoom ( opengroupsv2Room : OpenGroupV2Room , instance? : BetterSqlite3.Database ) {
2021-04-15 10:14:04 +02:00
const { serverUrl , roomId , conversationId } = opengroupsv2Room ;
2022-07-28 05:53:23 +02:00
assertGlobalInstanceOrInstance ( instance )
2021-07-08 05:00:20 +02:00
. prepare (
` INSERT OR REPLACE INTO ${ OPEN_GROUP_ROOMS_V2_TABLE } (
2021-04-15 10:14:04 +02:00
serverUrl ,
roomId ,
conversationId ,
json
) values (
$serverUrl ,
$roomId ,
$conversationId ,
$json
2021-07-08 05:00:20 +02:00
) `
)
. run ( {
serverUrl ,
roomId ,
conversationId ,
json : objectToJSON ( opengroupsv2Room ) ,
} ) ;
2021-04-15 10:14:04 +02:00
}
2022-03-29 06:18:26 +02:00
function removeV2OpenGroupRoom ( conversationId : string ) {
assertGlobalInstance ( )
2021-07-08 05:00:20 +02:00
. prepare ( ` DELETE FROM ${ OPEN_GROUP_ROOMS_V2_TABLE } WHERE conversationId = $ conversationId ` )
. run ( {
conversationId ,
} ) ;
2021-04-15 10:14:04 +02:00
}
2021-06-11 07:14:52 +02:00
2023-01-19 01:23:51 +01:00
/ * *
* Others
* /
2022-04-21 06:04:11 +02:00
function getEntriesCountInTable ( tbl : string ) {
try {
2022-04-21 03:29:14 +02:00
const row = assertGlobalInstance ( )
. prepare ( ` SELECT count(*) from ${ tbl } ; ` )
. get ( ) ;
return row [ 'count(*)' ] ;
2022-04-21 06:04:11 +02:00
} catch ( e ) {
2022-08-18 10:33:53 +02:00
console . error ( e ) ;
2022-04-21 06:04:11 +02:00
return 0 ;
2022-04-21 03:29:14 +02:00
}
2022-04-21 06:04:11 +02:00
}
2022-04-21 03:29:14 +02:00
2022-04-21 06:04:11 +02:00
function printDbStats() {
2022-04-21 03:29:14 +02:00
[
'attachment_downloads' ,
'conversations' ,
'encryptionKeyPairsForClosedGroupV2' ,
'guardNodes' ,
'identityKeys' ,
'items' ,
'lastHashes' ,
'loki_schema' ,
'messages' ,
'messages_fts' ,
'messages_fts_config' ,
'messages_fts_content' ,
'messages_fts_data' ,
'messages_fts_docsize' ,
'messages_fts_idx' ,
'nodesForPubkey' ,
'openGroupRoomsV2' ,
'seenMessages' ,
'sqlite_sequence' ,
'sqlite_stat1' ,
'sqlite_stat4' ,
'unprocessed' ,
] . forEach ( i = > {
2022-04-21 06:04:11 +02:00
console . log ( ` ${ i } count ` , getEntriesCountInTable ( i ) ) ;
2022-04-21 03:29:14 +02:00
} ) ;
}
2022-04-21 06:04:11 +02:00
/ * *
* Remove all the unused entries in the snodes for pubkey table .
* This table is used to know which snodes we should contact to send a message to a recipient
* /
2022-08-08 01:50:48 +02:00
function cleanUpUnusedNodeForKeyEntriesOnStart() {
// we have to keep private and closed group ids
2022-04-21 06:04:11 +02:00
const allIdsToKeep =
assertGlobalInstance ( )
. prepare (
2023-02-24 05:44:29 +01:00
` SELECT id FROM ${ CONVERSATIONS_TABLE } WHERE id NOT LIKE 'http%'
2022-04-21 06:04:11 +02:00
`
)
. all ( )
. map ( m = > m . id ) || [ ] ;
const allEntriesInSnodeForPubkey =
assertGlobalInstance ( )
. prepare ( ` SELECT pubkey FROM ${ NODES_FOR_PUBKEY_TABLE } ; ` )
. all ( )
. map ( m = > m . pubkey ) || [ ] ;
const swarmUnused = difference ( allEntriesInSnodeForPubkey , allIdsToKeep ) ;
if ( swarmUnused . length ) {
const start = Date . now ( ) ;
const chunks = chunk ( swarmUnused , 500 ) ;
chunks . forEach ( ch = > {
assertGlobalInstance ( )
. prepare (
` DELETE FROM ${ NODES_FOR_PUBKEY_TABLE } WHERE pubkey IN ( ${ ch . map ( ( ) = > '?' ) . join ( ',' ) } ); `
)
. run ( ch ) ;
} ) ;
console . log ( ` Removing of ${ swarmUnused . length } unused swarms took ${ Date . now ( ) - start } ms ` ) ;
}
}
2022-04-21 08:06:20 +02:00
function cleanUpMessagesJson() {
console . info ( 'cleanUpMessagesJson ' ) ;
const start = Date . now ( ) ;
assertGlobalInstance ( ) . transaction ( ( ) = > {
assertGlobalInstance ( ) . exec ( `
UPDATE $ { MESSAGES_TABLE } SET
json = json_remove ( json , '$.schemaVersion' , '$.recipients' , '$.decrypted_at' , '$.sourceDevice' )
` );
} ) ( ) ;
console . info ( ` cleanUpMessagesJson took ${ Date . now ( ) - start } ms ` ) ;
}
2022-08-08 01:50:48 +02:00
function cleanUpOldOpengroupsOnStart() {
2022-05-10 06:33:40 +02:00
const ourNumber = getItemById ( 'number_id' ) ;
if ( ! ourNumber || ! ourNumber . value ) {
2022-05-13 03:30:57 +02:00
console . info ( 'cleanUpOldOpengroups: ourNumber is not set' ) ;
2022-05-10 06:33:40 +02:00
return ;
}
2022-09-01 05:09:30 +02:00
let pruneSetting = getItemById ( SettingsKey . settingsOpengroupPruning ) ? . value ;
2022-05-18 03:47:42 +02:00
if ( pruneSetting === undefined ) {
2023-02-16 01:02:45 +01:00
console . info ( 'Prune settings is undefined (and not explicitly false), forcing it to true.' ) ;
2022-09-01 05:09:30 +02:00
createOrUpdateItem ( { id : SettingsKey.settingsOpengroupPruning , value : true } ) ;
pruneSetting = true ;
2022-05-18 03:47:42 +02:00
}
if ( ! pruneSetting ) {
console . info ( 'Prune setting not enabled, skipping cleanUpOldOpengroups' ) ;
return ;
}
2023-02-24 05:44:29 +01:00
const rows = assertGlobalInstance ( )
. prepare (
` SELECT id FROM ${ CONVERSATIONS_TABLE } WHERE
type = 'group' AND
id LIKE 'http%'
ORDER BY id ASC ; `
)
. all ( ) ;
const v2ConvosIds = map ( rows , row = > row . id ) ;
2022-08-08 01:50:48 +02:00
if ( ! v2ConvosIds || ! v2ConvosIds . length ) {
2022-05-13 03:30:57 +02:00
console . info ( 'cleanUpOldOpengroups: v2Convos is empty' ) ;
2022-05-10 06:33:40 +02:00
return ;
}
2022-08-12 05:21:57 +02:00
console . info ( ` Count of v2 opengroup convos to clean: ${ v2ConvosIds . length } ` ) ;
2022-05-13 11:48:45 +02:00
// For each open group, if it has more than 2000 messages, we remove all the messages
// older than 6 months. So this does not limit the size of open group history to 2000
// messages but to 6 months.
//
// This is the only way we can clean up conversations objects from users which just
// sent messages a while ago and with whom we never interacted. This is only for open
// groups, and is because ALL the conversations are cached in the redux store. Having
// a very large number of conversations (unused) is causing the performance of the app
// to deteriorate a lot.
//
// Another fix would be to not cache all the conversations in the redux store, but it
// ain't going to happen any time soon as it would a pretty big change of the way we
// do things and would break a lot of the app.
2022-04-20 08:33:59 +02:00
const maxMessagePerOpengroupConvo = 2000 ;
2022-04-14 02:56:34 +02:00
// first remove very old messages for each opengroups
2022-05-13 03:30:57 +02:00
const db = assertGlobalInstance ( ) ;
db . transaction ( ( ) = > {
2022-08-08 01:50:48 +02:00
v2ConvosIds . forEach ( convoId = > {
2022-05-18 03:47:42 +02:00
const messagesInConvoBefore = getMessagesCountByConversation ( convoId ) ;
if ( messagesInConvoBefore >= maxMessagePerOpengroupConvo ) {
const minute = 1000 * 60 ;
const sixMonths = minute * 60 * 24 * 30 * 6 ;
const limitTimestamp = Date . now ( ) - sixMonths ;
const countToRemove = assertGlobalInstance ( )
. prepare (
` SELECT count(*) from ${ MESSAGES_TABLE } WHERE serverTimestamp <= $ serverTimestamp AND conversationId = $ conversationId; `
)
. get ( { conversationId : convoId , serverTimestamp : limitTimestamp } ) [ 'count(*)' ] ;
const start = Date . now ( ) ;
assertGlobalInstance ( )
. prepare (
`
2022-05-14 18:23:51 +02:00
DELETE FROM $ { MESSAGES_TABLE } WHERE serverTimestamp <= $serverTimestamp AND conversationId = $conversationId `
2022-05-18 03:47:42 +02:00
)
. run ( { conversationId : convoId , serverTimestamp : limitTimestamp } ) ; // delete messages older than 6 months ago.
const messagesInConvoAfter = getMessagesCountByConversation ( convoId ) ;
console . info (
` Cleaning ${ countToRemove } messages older than 6 months in public convo: ${ convoId } took ${ Date . now ( ) -
start } ms . Old message count : $ { messagesInConvoBefore } , new message count : $ { messagesInConvoAfter } `
) ;
2023-03-10 06:39:48 +01:00
// no need to update the `unreadCount` during the migration anymore.
// `saveConversation` is broken when called with a argument without all the required fields.
// and this makes little sense as the unreadCount will be updated on opening
2023-03-16 00:49:06 +01:00
// const unreadCount = get UnreadCountByConversation(convoId);
// const convoProps = get ConversationById(convoId);
2023-03-10 06:39:48 +01:00
// if (convoProps) {
2023-03-16 00:49:06 +01:00
// convoProps.unread Count = unread Count;
2023-03-10 06:39:48 +01:00
// saveConversation(convoProps);
// }
2022-08-12 05:21:57 +02:00
} else {
console . info (
` Not cleaning messages older than 6 months in public convo: ${ convoId } . message count: ${ messagesInConvoBefore } `
) ;
2022-05-18 03:47:42 +02:00
}
} ) ;
2022-04-14 02:56:34 +02:00
2022-05-05 07:15:00 +02:00
// now, we might have a bunch of private conversation, without any interaction and no messages
// those are the conversation of the old members in the opengroups we just cleaned.
const allInactiveConvos = assertGlobalInstance ( )
. prepare (
`
2022-04-14 02:56:34 +02:00
SELECT id FROM $ { CONVERSATIONS_TABLE } WHERE type = 'private' AND ( active_at IS NULL OR active_at = 0 ) `
2022-05-05 07:15:00 +02:00
)
. all ( ) ;
2022-04-14 02:56:34 +02:00
2022-05-05 07:15:00 +02:00
const ourPubkey = ourNumber . value . split ( '.' ) [ 0 ] ;
const allInactiveAndWithoutMessagesConvo = allInactiveConvos
. map ( c = > c . id as string )
. filter ( convoId = > {
2023-07-26 11:26:46 +02:00
return ! ! ( convoId !== ourPubkey && getMessagesCountBySender ( { source : convoId } ) === 0 ) ;
2022-05-05 07:15:00 +02:00
} ) ;
if ( allInactiveAndWithoutMessagesConvo . length ) {
console . info (
` Removing ${ allInactiveAndWithoutMessagesConvo . length } completely inactive convos `
) ;
const start = Date . now ( ) ;
2022-04-21 03:29:14 +02:00
2022-05-05 07:15:00 +02:00
const chunks = chunk ( allInactiveAndWithoutMessagesConvo , 500 ) ;
chunks . forEach ( ch = > {
2022-05-13 03:30:57 +02:00
db . prepare (
` DELETE FROM ${ CONVERSATIONS_TABLE } WHERE id IN ( ${ ch . map ( ( ) = > '?' ) . join ( ',' ) } ); `
) . run ( ch ) ;
2022-05-05 07:15:00 +02:00
} ) ;
2022-04-21 03:29:14 +02:00
2022-05-05 07:15:00 +02:00
console . info (
` Removing of ${
allInactiveAndWithoutMessagesConvo . length
} completely inactive convos done in $ { Date . now ( ) - start } ms `
) ;
}
2022-04-21 06:04:11 +02:00
2022-05-05 07:15:00 +02:00
cleanUpMessagesJson ( ) ;
2022-05-13 03:30:57 +02:00
} ) ( ) ;
2022-04-14 02:56:34 +02:00
}
2022-03-29 23:36:04 +02:00
export type SqlNodeType = typeof sqlNode ;
2023-01-19 01:23:51 +01:00
export function close() {
closeDbInstance ( ) ;
}
2022-03-29 23:36:04 +02:00
export const sqlNode = {
2022-03-29 09:03:02 +02:00
initializeSql ,
2022-03-29 06:18:26 +02:00
close ,
removeDB ,
setSQLPassword ,
getPasswordHash ,
savePasswordHash ,
removePasswordHash ,
getIdentityKeyById ,
createOrUpdateItem ,
getItemById ,
getAllItems ,
removeItemById ,
getSwarmNodesForPubkey ,
updateSwarmNodesForPubkey ,
getGuardNodes ,
updateGuardNodes ,
getConversationCount ,
saveConversation ,
2023-03-16 00:49:06 +01:00
fetchConvoMemoryDetails ,
2022-03-29 06:18:26 +02:00
getConversationById ,
removeConversation ,
getAllConversations ,
getPubkeysInPublicConversation ,
removeAllConversations ,
searchConversations ,
searchMessages ,
searchMessagesInConversation ,
getMessageCount ,
saveMessage ,
cleanSeenMessages ,
cleanLastHashes ,
saveSeenMessageHashes ,
saveSeenMessageHash ,
updateLastHash ,
saveMessages ,
removeMessage ,
2022-10-06 06:15:14 +02:00
removeMessagesByIds ,
removeAllMessagesInConversation ,
2022-03-29 06:18:26 +02:00
getUnreadByConversation ,
2022-05-20 05:19:48 +02:00
markAllAsReadByConversationNoExpiration ,
2022-03-29 06:18:26 +02:00
getUnreadCountByConversation ,
getMessageCountByType ,
2022-04-14 02:56:34 +02:00
2022-04-13 05:52:05 +02:00
filterAlreadyFetchedOpengroupMessage ,
2023-05-11 09:29:01 +02:00
getMessagesBySenderAndSentAt ,
2022-03-29 06:18:26 +02:00
getMessageIdsFromServerIds ,
getMessageById ,
getMessagesBySentAt ,
2022-08-22 08:40:14 +02:00
getMessageByServerId ,
2022-03-29 06:18:26 +02:00
getSeenMessagesByHashList ,
getLastHashBySnode ,
getExpiredMessages ,
getOutgoingWithoutExpiresAt ,
getNextExpiringMessage ,
getMessagesByConversation ,
getLastMessagesByConversation ,
getOldestMessageInConversation ,
getFirstUnreadMessageIdInConversation ,
getFirstUnreadMessageWithMention ,
hasConversationOutgoingMessage ,
2023-01-25 07:46:30 +01:00
// add all the calls related to the unprocessed cache of incoming messages
. . . unprocessed ,
2022-03-29 06:18:26 +02:00
getNextAttachmentDownloadJobs ,
saveAttachmentDownloadJob ,
setAttachmentDownloadJobPending ,
resetAttachmentDownloadPending ,
removeAttachmentDownloadJob ,
removeAllAttachmentDownloadJobs ,
2022-03-29 23:36:04 +02:00
removeKnownAttachments ,
2022-03-29 06:18:26 +02:00
removeAll ,
getMessagesWithVisualMediaAttachments ,
getMessagesWithFileAttachments ,
2022-08-12 06:48:20 +02:00
getMessagesCountByConversation ,
2022-03-29 06:18:26 +02:00
getAllEncryptionKeyPairsForGroup ,
getLatestClosedGroupEncryptionKeyPair ,
addClosedGroupEncryptionKeyPair ,
removeAllClosedGroupEncryptionKeyPairs ,
// open group v2
getV2OpenGroupRoom ,
saveV2OpenGroupRoom ,
getAllV2OpenGroupRooms ,
removeV2OpenGroupRoom ,
2023-01-19 01:23:51 +01:00
// config dumps
. . . configDumpData ,
2022-03-29 06:18:26 +02:00
} ;