From 94a3aefe1973233a51443827634e0cfaab866e0b Mon Sep 17 00:00:00 2001 From: Xashyar <34171942+Xashyar@users.noreply.github.com> Date: Thu, 13 Dec 2018 22:35:46 +0330 Subject: [PATCH 01/35] Adding Appveyor SVG Status Badge (#2978) --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c0c7418d1..643d8fa90 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ + [![Build Status](https://travis-ci.org/signalapp/Signal-Desktop.svg?branch=master)](https://travis-ci.org/signalapp/Signal-Desktop) +[![Build Status](https://ci.appveyor.com/api/projects/status/github/signalapp/Signal-Desktop?branch=master&svg=true)](https://ci.appveyor.com/project/Signal-Desktop/signal-desktop) + Signal Desktop ========================== - Signal Desktop is an Electron application that links with Signal on [Android](https://github.com/signalapp/Signal-Android) or [iOS](https://github.com/signalapp/Signal-iOS). From e10fb47bb3b9f7093cd2fe8c4aa957621aeceb5e Mon Sep 17 00:00:00 2001 From: Xashyar <34171942+Xashyar@users.noreply.github.com> Date: Thu, 13 Dec 2018 22:36:11 +0330 Subject: [PATCH 02/35] Issue Template, Small Edits (#2975) --- .github/ISSUE_TEMPLATE.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index b0eed63cc..d7bfed42f 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,6 +1,6 @@ -### Steps to reproduce +### Steps to Reproduce @@ -33,11 +33,11 @@ Lastly, be sure to preview your issue before saving. Thanks! 2. step two 3. step three -Actual result: +Actual Result: -Expected result: +Expected Result: @@ -48,9 +48,9 @@ How to take screenshots on all OSes: https://www.take-a-screenshot.org/ You can drag and drop images into this text box. --> -### Platform info +### Platform Info -Signal version: +Signal Version: @@ -58,16 +58,16 @@ Operating System: -Linked device version: +Linked Device Version: -### Link to debug log +### Link to Debug Log From 3f78a3c466db2d8e3803b9ad45dba764b37e667f Mon Sep 17 00:00:00 2001 From: sha-265 <4103710+sha-265@users.noreply.github.com> Date: Fri, 14 Dec 2018 19:27:46 +0200 Subject: [PATCH 03/35] Fix text alignment for RTL quoted messages (#2980) --- stylesheets/_modules.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/stylesheets/_modules.scss b/stylesheets/_modules.scss index fc02f732c..fbac9783b 100644 --- a/stylesheets/_modules.scss +++ b/stylesheets/_modules.scss @@ -728,6 +728,7 @@ font-size: 14px; line-height: 18px; color: $color-gray-90; + text-align: start; a { color: $color-gray-90; From 0b60af1c848ba4895ea1bdd5829c6127dc8c2d61 Mon Sep 17 00:00:00 2001 From: Herohtar Date: Wed, 2 Jan 2019 14:22:47 -0600 Subject: [PATCH 04/35] Don't compare a numeric value with a string when using the identity operator (#2989) Fix comparison for emoji injection at cursor location --- js/views/conversation_view.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/views/conversation_view.js b/js/views/conversation_view.js index 1d86224c0..4fdec8581 100644 --- a/js/views/conversation_view.js +++ b/js/views/conversation_view.js @@ -1456,7 +1456,7 @@ const colons = `:${emojiData[e.index].short_name}:`; const textarea = this.$messageField[0]; - if (textarea.selectionStart || textarea.selectionStart === '0') { + if (textarea.selectionStart || textarea.selectionStart === 0) { const startPos = textarea.selectionStart; const endPos = textarea.selectionEnd; From 9ffe7c5836bf0cef9d373d2a51e99dfabf56c725 Mon Sep 17 00:00:00 2001 From: Hugo Torzuoli Date: Wed, 2 Jan 2019 23:25:15 +0100 Subject: [PATCH 05/35] Update Licence Year (#3009) Update License Year --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 643d8fa90..6cc4fd8e5 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,6 @@ The form and manner of this distribution makes it eligible for export under the ## License -Copyright 2014-2018 Open Whisper Systems +Copyright 2014-2019 Open Whisper Systems Licensed under the GPLv3: http://www.gnu.org/licenses/gpl-3.0.html From a21d63e450b667d18a0bafb53881e2c9c96c3dc1 Mon Sep 17 00:00:00 2001 From: Herohtar Date: Wed, 2 Jan 2019 16:34:18 -0600 Subject: [PATCH 06/35] Make notification initials consistent with everything else (#3006) Make notification initials consistent with Avatar component --- js/models/conversations.js | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/js/models/conversations.js b/js/models/conversations.js index 98b8377a7..c46901aad 100644 --- a/js/models/conversations.js +++ b/js/models/conversations.js @@ -1771,6 +1771,21 @@ } }, + getInitials(name) { + if (!name) { + return null; + } + + const cleaned = name.replace(/[^A-Za-z\s]+/g, '').replace(/\s+/g, ' '); + const parts = cleaned.split(' '); + const initials = parts.map(part => part.trim()[0]); + if (!initials.length) { + return null; + } + + return initials.slice(0, 2).join(''); + }, + isPrivate() { return this.get('type') === 'private'; }, @@ -1802,7 +1817,7 @@ } else if (this.isPrivate()) { return { color, - content: title ? title.trim()[0] : '#', + content: this.getInitials(title) || '#', }; } return { url: 'images/group_default.png', color }; From 22ca4f9cc7e9f9d36b0e61a20a4b8e85108b8055 Mon Sep 17 00:00:00 2001 From: Herohtar Date: Wed, 2 Jan 2019 16:42:31 -0600 Subject: [PATCH 07/35] Change tray icon click to always show/focus window (#2984) * Added function to always show the window on tray icon click and reassigned click event * Refactored the code to force the window on top into its own function --- app/tray_icon.js | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/app/tray_icon.js b/app/tray_icon.js index ab3f34ac7..ce30c082a 100644 --- a/app/tray_icon.js +++ b/app/tray_icon.js @@ -17,6 +17,17 @@ function createTrayIcon(getMainWindow, messages) { tray = new Tray(iconNoNewMessages); + tray.forceOnTop = mainWindow => { + if (mainWindow) { + // On some versions of GNOME the window may not be on top when restored. + // This trick should fix it. + // Thanks to: https://github.com/Enrico204/Whatsapp-Desktop/commit/6b0dc86b64e481b455f8fce9b4d797e86d000dc1 + mainWindow.setAlwaysOnTop(true); + mainWindow.focus(); + mainWindow.setAlwaysOnTop(false); + } + } + tray.toggleWindowVisibility = () => { const mainWindow = getMainWindow(); if (mainWindow) { @@ -25,17 +36,24 @@ function createTrayIcon(getMainWindow, messages) { } else { mainWindow.show(); - // On some versions of GNOME the window may not be on top when restored. - // This trick should fix it. - // Thanks to: https://github.com/Enrico204/Whatsapp-Desktop/commit/6b0dc86b64e481b455f8fce9b4d797e86d000dc1 - mainWindow.setAlwaysOnTop(true); - mainWindow.focus(); - mainWindow.setAlwaysOnTop(false); + tray.forceOnTop(mainWindow); } } tray.updateContextMenu(); }; + tray.showWindow = () => { + const mainWindow = getMainWindow(); + if (mainWindow) { + if (!mainWindow.isVisible()) { + mainWindow.show(); + } + + tray.forceOnTop(mainWindow); + } + tray.updateContextMenu(); + }; + tray.updateContextMenu = () => { const mainWindow = getMainWindow(); @@ -70,7 +88,7 @@ function createTrayIcon(getMainWindow, messages) { } }; - tray.on('click', tray.toggleWindowVisibility); + tray.on('click', tray.showWindow); tray.setToolTip(messages.trayTooltip.message); tray.updateContextMenu(); From 775e31c854f95c6c45eadfe8bf77c5bdb1b54ffb Mon Sep 17 00:00:00 2001 From: Scott Nonnenberg Date: Thu, 3 Jan 2019 13:44:42 -0800 Subject: [PATCH 08/35] Lint fixes after recent PRs --- README.md | 5 ++--- app/tray_icon.js | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6cc4fd8e5..91b5bbe49 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,8 @@ - [![Build Status](https://travis-ci.org/signalapp/Signal-Desktop.svg?branch=master)](https://travis-ci.org/signalapp/Signal-Desktop) [![Build Status](https://ci.appveyor.com/api/projects/status/github/signalapp/Signal-Desktop?branch=master&svg=true)](https://ci.appveyor.com/project/Signal-Desktop/signal-desktop) -Signal Desktop -========================== +# Signal Desktop + Signal Desktop is an Electron application that links with Signal on [Android](https://github.com/signalapp/Signal-Android) or [iOS](https://github.com/signalapp/Signal-iOS). diff --git a/app/tray_icon.js b/app/tray_icon.js index ce30c082a..407c2584e 100644 --- a/app/tray_icon.js +++ b/app/tray_icon.js @@ -26,7 +26,7 @@ function createTrayIcon(getMainWindow, messages) { mainWindow.focus(); mainWindow.setAlwaysOnTop(false); } - } + }; tray.toggleWindowVisibility = () => { const mainWindow = getMainWindow(); From 47f834cf5ca7a761566624c7659beeab55910936 Mon Sep 17 00:00:00 2001 From: Scott Nonnenberg Date: Thu, 13 Dec 2018 11:12:33 -0800 Subject: [PATCH 09/35] Encrypt device name on account create, on first launch if needed --- js/background.js | 1 + js/modules/crypto.js | 53 ++++- js/modules/web_api.js | 12 ++ libtextsecure/account_manager.js | 220 +++++++++++++-------- libtextsecure/protobufs.js | 3 + libtextsecure/storage/user.js | 8 + libtextsecure/test/account_manager_test.js | 20 +- protos/DeviceName.proto | 7 + test/crypto_test.js | 39 +++- ts/util/lint/exceptions.json | 14 +- 10 files changed, 282 insertions(+), 95 deletions(-) create mode 100644 protos/DeviceName.proto diff --git a/js/background.js b/js/background.js index 4b7cd9107..3482e6515 100644 --- a/js/background.js +++ b/js/background.js @@ -681,6 +681,7 @@ textsecure.storage.user.getDeviceId() != '1' ) { window.getSyncRequest(); + window.getAccountManager().maybeUpdateDeviceName(); } const udSupportKey = 'hasRegisterSupportForUnauthenticatedDelivery'; diff --git a/js/modules/crypto.js b/js/modules/crypto.js index 0071e8943..31f9fb97d 100644 --- a/js/modules/crypto.js +++ b/js/modules/crypto.js @@ -1,5 +1,5 @@ /* eslint-env browser */ -/* global dcodeIO */ +/* global dcodeIO, libsignal */ /* eslint-disable camelcase, no-bitwise */ @@ -10,9 +10,11 @@ module.exports = { concatenateBytes, constantTimeEqual, decryptAesCtr, + decryptDeviceName, decryptSymmetric, deriveAccessKey, encryptAesCtr, + encryptDeviceName, encryptSymmetric, fromEncodedBinaryToArrayBuffer, getAccessKeyVerifier, @@ -30,6 +32,55 @@ module.exports = { // High-level Operations +async function encryptDeviceName(deviceName, identityPublic) { + const plaintext = bytesFromString(deviceName); + const ephemeralKeyPair = await libsignal.KeyHelper.generateIdentityKeyPair(); + const masterSecret = await libsignal.Curve.async.calculateAgreement( + identityPublic, + ephemeralKeyPair.privKey + ); + + const key1 = await hmacSha256(masterSecret, bytesFromString('auth')); + const syntheticIv = _getFirstBytes(await hmacSha256(key1, plaintext), 16); + + const key2 = await hmacSha256(masterSecret, bytesFromString('cipher')); + const cipherKey = await hmacSha256(key2, syntheticIv); + + const counter = getZeroes(16); + const ciphertext = await encryptAesCtr(cipherKey, plaintext, counter); + + return { + ephemeralPublic: ephemeralKeyPair.pubKey, + syntheticIv, + ciphertext, + }; +} + +async function decryptDeviceName( + { ephemeralPublic, syntheticIv, ciphertext } = {}, + identityPrivate +) { + const masterSecret = await libsignal.Curve.async.calculateAgreement( + ephemeralPublic, + identityPrivate + ); + + const key2 = await hmacSha256(masterSecret, bytesFromString('cipher')); + const cipherKey = await hmacSha256(key2, syntheticIv); + + const counter = getZeroes(16); + const plaintext = await decryptAesCtr(cipherKey, ciphertext, counter); + + const key1 = await hmacSha256(masterSecret, bytesFromString('auth')); + const ourSyntheticIv = _getFirstBytes(await hmacSha256(key1, plaintext), 16); + + if (!constantTimeEqual(ourSyntheticIv, syntheticIv)) { + throw new Error('decryptDeviceName: synthetic IV did not match'); + } + + return stringFromBytes(plaintext); +} + async function deriveAccessKey(profileKey) { const iv = getZeroes(12); const plaintext = getZeroes(16); diff --git a/js/modules/web_api.js b/js/modules/web_api.js index ec2748392..fb5ca5308 100644 --- a/js/modules/web_api.js +++ b/js/modules/web_api.js @@ -325,6 +325,7 @@ function HTTPError(message, providedCode, response, stack) { const URL_CALLS = { accounts: 'v1/accounts', + updateDeviceName: 'v1/accounts/name', attachment: 'v1/attachments', deliveryCert: 'v1/certificate/delivery', supportUnauthenticatedDelivery: 'v1/devices/unauthenticated_delivery', @@ -386,6 +387,7 @@ function initialize({ url, cdnUrl, certificateAuthority, proxyUrl }) { sendMessages, sendMessagesUnauth, setSignedPreKey, + updateDeviceName, }; function _ajax(param) { @@ -568,6 +570,16 @@ function initialize({ url, cdnUrl, certificateAuthority, proxyUrl }) { return response; } + function updateDeviceName(deviceName) { + return _ajax({ + call: 'updateDeviceName', + httpType: 'PUT', + jsonData: { + deviceName, + }, + }); + } + function getDevices() { return _ajax({ call: 'devices', diff --git a/libtextsecure/account_manager.js b/libtextsecure/account_manager.js index 960c18188..443c82e66 100644 --- a/libtextsecure/account_manager.js +++ b/libtextsecure/account_manager.js @@ -4,6 +4,7 @@ libsignal, WebSocketResource, btoa, + Signal, getString, libphonenumber, Event, @@ -45,6 +46,59 @@ requestSMSVerification(number) { return this.server.requestVerificationSMS(number); }, + async encryptDeviceName(name, providedIdentityKey) { + const identityKey = + providedIdentityKey || + (await textsecure.storage.protocol.getIdentityKeyPair()); + if (!identityKey) { + throw new Error( + 'Identity key was not provided and is not in database!' + ); + } + const encrypted = await Signal.Crypto.encryptDeviceName( + name, + identityKey.pubKey + ); + + const proto = new textsecure.protobuf.DeviceName(); + proto.ephemeralPublic = encrypted.ephemeralPublic; + proto.syntheticIv = encrypted.syntheticIv; + proto.ciphertext = encrypted.ciphertext; + + const arrayBuffer = proto.encode().toArrayBuffer(); + return Signal.Crypto.arrayBufferToBase64(arrayBuffer); + }, + async decryptDeviceName(base64) { + const identityKey = await textsecure.storage.protocol.getIdentityKeyPair(); + + const arrayBuffer = Signal.Crypto.base64ToArrayBuffer(base64); + const proto = textsecure.protobuf.DeviceName.decode(arrayBuffer); + const encrypted = { + ephemeralPublic: proto.ephemeralPublic.toArrayBuffer(), + syntheticIv: proto.syntheticIv.toArrayBuffer(), + ciphertext: proto.ciphertext.toArrayBuffer(), + }; + + const name = await Signal.Crypto.decryptDeviceName( + encrypted, + identityKey.privKey + ); + + return name; + }, + async maybeUpdateDeviceName() { + const isNameEncrypted = textsecure.storage.user.getDeviceNameEncrypted(); + if (isNameEncrypted) { + return; + } + const deviceName = await textsecure.storage.user.getDeviceName(); + const base64 = await this.encryptDeviceName(deviceName); + + await this.server.updateDeviceName(base64); + }, + async deviceNameIsEncrypted() { + await textsecure.storage.user.setDeviceNameEncrypted(); + }, registerSingleDevice(number, verificationCode) { const registerKeys = this.server.registerKeys.bind(this.server); const createAccount = this.createAccount.bind(this); @@ -335,7 +389,7 @@ }); }); }, - createAccount( + async createAccount( number, verificationCode, identityKeyPair, @@ -353,110 +407,106 @@ const previousNumber = getNumber(textsecure.storage.get('number_id')); - return this.server - .confirmCode( - number, - verificationCode, - password, - signalingKey, - registrationId, - deviceName, - { accessKey } - ) - .then(response => { - if (previousNumber && previousNumber !== number) { - window.log.warn( - 'New number is different from old number; deleting all previous data' - ); + const encryptedDeviceName = await this.encryptDeviceName( + deviceName, + identityKeyPair + ); + await this.deviceNameIsEncrypted(); - return textsecure.storage.protocol.removeAllData().then( - () => { - window.log.info('Successfully deleted previous data'); - return response; - }, - error => { - window.log.error( - 'Something went wrong deleting data from previous number', - error && error.stack ? error.stack : error - ); + const response = await this.server.confirmCode( + number, + verificationCode, + password, + signalingKey, + registrationId, + encryptedDeviceName, + { accessKey } + ); - return response; - } - ); - } + if (previousNumber && previousNumber !== number) { + window.log.warn( + 'New number is different from old number; deleting all previous data' + ); - return response; - }) - .then(async response => { - await Promise.all([ - textsecure.storage.remove('identityKey'), - textsecure.storage.remove('signaling_key'), - textsecure.storage.remove('password'), - textsecure.storage.remove('registrationId'), - textsecure.storage.remove('number_id'), - textsecure.storage.remove('device_name'), - textsecure.storage.remove('regionCode'), - textsecure.storage.remove('userAgent'), - textsecure.storage.remove('profileKey'), - textsecure.storage.remove('read-receipts-setting'), - ]); - - // update our own identity key, which may have changed - // if we're relinking after a reinstall on the master device - await textsecure.storage.protocol.saveIdentityWithAttributes(number, { - id: number, - publicKey: identityKeyPair.pubKey, - firstUse: true, - timestamp: Date.now(), - verified: textsecure.storage.protocol.VerifiedStatus.VERIFIED, - nonblockingApproval: true, - }); - - await textsecure.storage.put('identityKey', identityKeyPair); - await textsecure.storage.put('signaling_key', signalingKey); - await textsecure.storage.put('password', password); - await textsecure.storage.put('registrationId', registrationId); - if (profileKey) { - await textsecure.storage.put('profileKey', profileKey); - } - if (userAgent) { - await textsecure.storage.put('userAgent', userAgent); - } - - await textsecure.storage.put( - 'read-receipt-setting', - Boolean(readReceipts) + try { + await textsecure.storage.protocol.removeAllData(); + window.log.info('Successfully deleted previous data'); + } catch (error) { + window.log.error( + 'Something went wrong deleting data from previous number', + error && error.stack ? error.stack : error ); + } + } - await textsecure.storage.user.setNumberAndDeviceId( - number, - response.deviceId || 1, - deviceName - ); - await textsecure.storage.put( - 'regionCode', - libphonenumber.util.getRegionCodeForNumber(number) - ); - }); + await Promise.all([ + textsecure.storage.remove('identityKey'), + textsecure.storage.remove('signaling_key'), + textsecure.storage.remove('password'), + textsecure.storage.remove('registrationId'), + textsecure.storage.remove('number_id'), + textsecure.storage.remove('device_name'), + textsecure.storage.remove('regionCode'), + textsecure.storage.remove('userAgent'), + textsecure.storage.remove('profileKey'), + textsecure.storage.remove('read-receipts-setting'), + ]); + + // update our own identity key, which may have changed + // if we're relinking after a reinstall on the master device + await textsecure.storage.protocol.saveIdentityWithAttributes(number, { + id: number, + publicKey: identityKeyPair.pubKey, + firstUse: true, + timestamp: Date.now(), + verified: textsecure.storage.protocol.VerifiedStatus.VERIFIED, + nonblockingApproval: true, + }); + + await textsecure.storage.put('identityKey', identityKeyPair); + await textsecure.storage.put('signaling_key', signalingKey); + await textsecure.storage.put('password', password); + await textsecure.storage.put('registrationId', registrationId); + if (profileKey) { + await textsecure.storage.put('profileKey', profileKey); + } + if (userAgent) { + await textsecure.storage.put('userAgent', userAgent); + } + + await textsecure.storage.put( + 'read-receipt-setting', + Boolean(readReceipts) + ); + + await textsecure.storage.user.setNumberAndDeviceId( + number, + response.deviceId || 1, + deviceName + ); + await textsecure.storage.put( + 'regionCode', + libphonenumber.util.getRegionCodeForNumber(number) + ); }, - clearSessionsAndPreKeys() { + async clearSessionsAndPreKeys() { const store = textsecure.storage.protocol; window.log.info('clearing all sessions, prekeys, and signed prekeys'); - return Promise.all([ + await Promise.all([ store.clearPreKeyStore(), store.clearSignedPreKeysStore(), store.clearSessionStore(), ]); }, // Takes the same object returned by generateKeys - confirmKeys(keys) { + async confirmKeys(keys) { const store = textsecure.storage.protocol; const key = keys.signedPreKey; const confirmed = true; window.log.info('confirmKeys: confirming key', key.keyId); - return store.storeSignedPreKey(key.keyId, key.keyPair, confirmed); + await store.storeSignedPreKey(key.keyId, key.keyPair, confirmed); }, generateKeys(count, providedProgressCallback) { const progressCallback = diff --git a/libtextsecure/protobufs.js b/libtextsecure/protobufs.js index d2da5e2cc..80c91e9e5 100644 --- a/libtextsecure/protobufs.js +++ b/libtextsecure/protobufs.js @@ -36,6 +36,9 @@ loadProtoBufs('SubProtocol.proto'); loadProtoBufs('DeviceMessages.proto'); + // Just for encrypting device names + loadProtoBufs('DeviceName.proto'); + // Metadata-specific protos loadProtoBufs('UnidentifiedDelivery.proto'); })(); diff --git a/libtextsecure/storage/user.js b/libtextsecure/storage/user.js index efbfd8e2d..e8b7e0fd5 100644 --- a/libtextsecure/storage/user.js +++ b/libtextsecure/storage/user.js @@ -31,5 +31,13 @@ getDeviceName() { return textsecure.storage.get('device_name'); }, + + setDeviceNameEncrypted() { + return textsecure.storage.put('deviceNameEncrypted', true); + }, + + getDeviceNameEncrypted() { + return textsecure.storage.get('deviceNameEncrypted'); + }, }; })(); diff --git a/libtextsecure/test/account_manager_test.js b/libtextsecure/test/account_manager_test.js index 12ffda788..f1e98827d 100644 --- a/libtextsecure/test/account_manager_test.js +++ b/libtextsecure/test/account_manager_test.js @@ -1,3 +1,5 @@ +/* global libsignal */ + describe('AccountManager', () => { let accountManager; @@ -10,9 +12,14 @@ describe('AccountManager', () => { let signedPreKeys; const DAY = 1000 * 60 * 60 * 24; - beforeEach(() => { + beforeEach(async () => { + const identityKey = await libsignal.KeyHelper.generateIdentityKeyPair(); + originalProtocolStorage = window.textsecure.storage.protocol; window.textsecure.storage.protocol = { + getIdentityKeyPair() { + return identityKey; + }, loadSignedPreKeys() { return Promise.resolve(signedPreKeys); }, @@ -22,6 +29,17 @@ describe('AccountManager', () => { window.textsecure.storage.protocol = originalProtocolStorage; }); + describe('encrypted device name', () => { + it('roundtrips', async () => { + const deviceName = 'v2.5.0 on Ubunto 20.04'; + const encrypted = await accountManager.encryptDeviceName(deviceName); + assert.strictEqual(typeof encrypted, 'string'); + const decrypted = await accountManager.decryptDeviceName(encrypted); + + assert.strictEqual(decrypted, deviceName); + }); + }); + it('keeps three confirmed keys even if over a week old', () => { const now = Date.now(); signedPreKeys = [ diff --git a/protos/DeviceName.proto b/protos/DeviceName.proto new file mode 100644 index 000000000..ec2859b18 --- /dev/null +++ b/protos/DeviceName.proto @@ -0,0 +1,7 @@ +package signalservice; + +message DeviceName { + optional bytes ephemeralPublic = 1; + optional bytes syntheticIv = 2; + optional bytes ciphertext = 3; +} diff --git a/test/crypto_test.js b/test/crypto_test.js index fe0800132..a047c2b31 100644 --- a/test/crypto_test.js +++ b/test/crypto_test.js @@ -1,4 +1,4 @@ -/* global Signal, textsecure */ +/* global Signal, textsecure, libsignal */ 'use strict'; @@ -109,4 +109,41 @@ describe('Crypto', () => { throw new Error('Expected error to be thrown'); }); }); + + describe('encrypted device name', () => { + it('roundtrips', async () => { + const deviceName = 'v1.19.0 on Windows 10'; + const identityKey = await libsignal.KeyHelper.generateIdentityKeyPair(); + + const encrypted = await Signal.Crypto.encryptDeviceName( + deviceName, + identityKey.pubKey + ); + const decrypted = await Signal.Crypto.decryptDeviceName( + encrypted, + identityKey.privKey + ); + + assert.strictEqual(decrypted, deviceName); + }); + + it('fails if iv is changed', async () => { + const deviceName = 'v1.19.0 on Windows 10'; + const identityKey = await libsignal.KeyHelper.generateIdentityKeyPair(); + + const encrypted = await Signal.Crypto.encryptDeviceName( + deviceName, + identityKey.pubKey + ); + encrypted.syntheticIv = Signal.Crypto.getRandomBytes(16); + try { + await Signal.Crypto.decryptDeviceName(encrypted, identityKey.privKey); + } catch (error) { + assert.strictEqual( + error.message, + 'decryptDeviceName: synthetic IV did not match' + ); + } + }); + }); }); diff --git a/ts/util/lint/exceptions.json b/ts/util/lint/exceptions.json index 37489874e..6b264d1db 100644 --- a/ts/util/lint/exceptions.json +++ b/ts/util/lint/exceptions.json @@ -244,7 +244,7 @@ "rule": "jQuery-wrap(", "path": "js/background.js", "line": " wrap(", - "lineNumber": 727, + "lineNumber": 728, "reasonCategory": "falseMatch", "updated": "2018-10-18T22:23:00.485Z" }, @@ -252,7 +252,7 @@ "rule": "jQuery-wrap(", "path": "js/background.js", "line": " await wrap(", - "lineNumber": 1257, + "lineNumber": 1258, "reasonCategory": "falseMatch", "updated": "2018-10-26T22:43:23.229Z" }, @@ -319,7 +319,7 @@ "rule": "jQuery-wrap(", "path": "js/modules/crypto.js", "line": " return dcodeIO.ByteBuffer.wrap(arrayBuffer).toString('base64');", - "lineNumber": 271, + "lineNumber": 322, "reasonCategory": "falseMatch", "updated": "2018-10-05T23:12:28.961Z" }, @@ -327,7 +327,7 @@ "rule": "jQuery-wrap(", "path": "js/modules/crypto.js", "line": " return dcodeIO.ByteBuffer.wrap(base64string, 'base64').toArrayBuffer();", - "lineNumber": 274, + "lineNumber": 325, "reasonCategory": "falseMatch", "updated": "2018-10-05T23:12:28.961Z" }, @@ -335,7 +335,7 @@ "rule": "jQuery-wrap(", "path": "js/modules/crypto.js", "line": " return dcodeIO.ByteBuffer.wrap(key, 'binary').toArrayBuffer();", - "lineNumber": 278, + "lineNumber": 329, "reasonCategory": "falseMatch", "updated": "2018-10-05T23:12:28.961Z" }, @@ -343,7 +343,7 @@ "rule": "jQuery-wrap(", "path": "js/modules/crypto.js", "line": " return dcodeIO.ByteBuffer.wrap(string, 'utf8').toArrayBuffer();", - "lineNumber": 282, + "lineNumber": 333, "reasonCategory": "falseMatch", "updated": "2018-10-05T23:12:28.961Z" }, @@ -351,7 +351,7 @@ "rule": "jQuery-wrap(", "path": "js/modules/crypto.js", "line": " return dcodeIO.ByteBuffer.wrap(buffer).toString('utf8');", - "lineNumber": 285, + "lineNumber": 336, "reasonCategory": "falseMatch", "updated": "2018-10-05T23:12:28.961Z" }, From e4babdaef00af9f2ad306a1277e6d34dbead2da6 Mon Sep 17 00:00:00 2001 From: Scott Nonnenberg Date: Thu, 13 Dec 2018 13:41:42 -0800 Subject: [PATCH 10/35] Updates to backup infrastructure --- Gruntfile.js | 6 +- app/sql.js | 5 + js/modules/backup.js | 686 ++++++++++------------ js/modules/crypto.js | 82 ++- js/modules/data.js | 5 + js/modules/signal.js | 1 + js/modules/types/message.js | 20 +- package.json | 2 +- preload.js | 13 + test/backup_test.js | 382 ++++-------- test/crypto_test.js | 32 +- test/metadata/SecretSessionCipher_test.js | 16 +- ts/util/lint/exceptions.json | 10 +- yarn.lock | 49 +- 14 files changed, 599 insertions(+), 710 deletions(-) diff --git a/Gruntfile.js b/Gruntfile.js index 241b7449e..686ff659a 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -223,8 +223,8 @@ module.exports = grunt => { grunt.registerTask('getExpireTime', () => { grunt.task.requires('gitinfo'); const gitinfo = grunt.config.get('gitinfo'); - const commited = gitinfo.local.branch.current.lastCommitTime; - const time = Date.parse(commited) + 1000 * 60 * 60 * 24 * 90; + const committed = gitinfo.local.branch.current.lastCommitTime; + const time = Date.parse(committed) + 1000 * 60 * 60 * 24 * 90; grunt.file.write( 'config/local-production.json', `${JSON.stringify({ buildExpiration: time })}\n` @@ -263,7 +263,7 @@ module.exports = grunt => { app.client .execute(getMochaResults) .then(data => Boolean(data.value)), - 10000, + 25000, 'Expected to find window.mochaResults set!' ) ) diff --git a/app/sql.js b/app/sql.js index b52625628..c544756e4 100644 --- a/app/sql.js +++ b/app/sql.js @@ -19,6 +19,7 @@ module.exports = { createOrUpdateGroup, getGroupById, getAllGroupIds, + getAllGroups, bulkAddGroups, removeGroupById, removeAllGroups, @@ -567,6 +568,10 @@ async function getAllGroupIds() { const rows = await db.all('SELECT id FROM groups ORDER BY id ASC;'); return map(rows, row => row.id); } +async function getAllGroups() { + const rows = await db.all('SELECT id FROM groups ORDER BY id ASC;'); + return map(rows, row => jsonToObject(row.json)); +} async function bulkAddGroups(array) { return bulkAdd(GROUPS_TABLE, array); } diff --git a/js/modules/backup.js b/js/modules/backup.js index 7bd0c95cc..31b0fd6d3 100644 --- a/js/modules/backup.js +++ b/js/modules/backup.js @@ -7,22 +7,20 @@ /* eslint-env browser */ /* eslint-env node */ -/* eslint-disable no-param-reassign, guard-for-in, no-unreachable */ +/* eslint-disable no-param-reassign, guard-for-in */ const fs = require('fs'); const path = require('path'); const { map, fromPairs } = require('lodash'); +const tar = require('tar'); const tmp = require('tmp'); const pify = require('pify'); -const archiver = require('archiver'); const rimraf = require('rimraf'); const electronRemote = require('electron').remote; -const Attachment = require('./types/attachment'); const crypto = require('./crypto'); -const decompress = () => null; const { dialog, BrowserWindow } = electronRemote; module.exports = { @@ -111,100 +109,55 @@ function createOutputStream(writer) { }; } -async function exportContactAndGroupsToFile(db, parent) { +async function exportContactAndGroupsToFile(parent) { const writer = await createFileAndWriter(parent, 'db.json'); - return exportContactsAndGroups(db, writer); + return exportContactsAndGroups(writer); } -function exportContactsAndGroups(db, fileWriter) { - return new Promise((resolve, reject) => { - let storeNames = db.objectStoreNames; - storeNames = _.without( - storeNames, - 'messages', - 'items', - 'signedPreKeys', - 'preKeys', - 'identityKeys', - 'sessions', - 'unprocessed' - ); +function writeArray(stream, array) { + stream.write('['); - const exportedStoreNames = []; - if (storeNames.length === 0) { - throw new Error('No stores to export'); + for (let i = 0, max = array.length; i < max; i += 1) { + if (i > 0) { + stream.write(','); } - window.log.info('Exporting from these stores:', storeNames.join(', ')); - const stream = createOutputStream(fileWriter); + const item = array[i]; - stream.write('{'); + // We don't back up avatars; we'll get them in a future contact sync or profile fetch + const cleaned = _.omit(item, ['avatar', 'profileAvatar']); - _.each(storeNames, storeName => { - // Both the readwrite permission and the multi-store transaction are required to - // keep this function working. They serve to serialize all of these transactions, - // one per store to be exported. - const transaction = db.transaction(storeNames, 'readwrite'); - transaction.onerror = () => { - Whisper.Database.handleDOMException( - `exportToJsonFile transaction error (store: ${storeName})`, - transaction.error, - reject - ); - }; - transaction.oncomplete = () => { - window.log.info('transaction complete'); - }; + stream.write(JSON.stringify(stringify(cleaned))); + } - const store = transaction.objectStore(storeName); - const request = store.openCursor(); - let count = 0; - request.onerror = () => { - Whisper.Database.handleDOMException( - `exportToJsonFile request error (store: ${storeNames})`, - request.error, - reject - ); - }; - request.onsuccess = async event => { - if (count === 0) { - window.log.info('cursor opened'); - stream.write(`"${storeName}": [`); - } + stream.write(']'); +} - const cursor = event.target.result; - if (cursor) { - if (count > 0) { - stream.write(','); - } +function getPlainJS(collection) { + return collection.map(model => model.attributes); +} - // Preventing base64'd images from reaching the disk, making db.json too big - const item = _.omit(cursor.value, ['avatar', 'profileAvatar']); +async function exportContactsAndGroups(fileWriter) { + const stream = createOutputStream(fileWriter); - const jsonString = JSON.stringify(stringify(item)); - stream.write(jsonString); - cursor.continue(); - count += 1; - } else { - // no more - stream.write(']'); - window.log.info('Exported', count, 'items from store', storeName); + stream.write('{'); - exportedStoreNames.push(storeName); - if (exportedStoreNames.length < storeNames.length) { - stream.write(','); - } else { - window.log.info('Exported all stores'); - stream.write('}'); - - await stream.close(); - window.log.info('Finished writing all stores to disk'); - resolve(); - } - } - }; - }); + stream.write('"conversations": '); + const conversations = await window.Signal.Data.getAllConversations({ + ConversationCollection: Whisper.ConversationCollection, }); + window.log.info(`Exporting ${conversations.length} conversations`); + writeArray(stream, getPlainJS(conversations)); + + stream.write(','); + + stream.write('"groups": '); + const groups = await window.Signal.Data.getAllGroups(); + window.log.info(`Exporting ${groups.length} groups`); + writeArray(stream, groups); + + stream.write('}'); + await stream.close(); } async function importNonMessages(parent, options) { @@ -414,6 +367,14 @@ function readFileAsText(parent, name) { }); } +// Buffer instances are also Uint8Array instances, but they might be a view +// https://nodejs.org/docs/latest/api/buffer.html#buffer_buffers_and_typedarray +const toArrayBuffer = nodeBuffer => + nodeBuffer.buffer.slice( + nodeBuffer.byteOffset, + nodeBuffer.byteOffset + nodeBuffer.byteLength + ); + function readFileAsArrayBuffer(targetPath) { return new Promise((resolve, reject) => { // omitting the encoding to get a buffer back @@ -422,9 +383,7 @@ function readFileAsArrayBuffer(targetPath) { return reject(error); } - // Buffer instances are also Uint8Array instances - // https://nodejs.org/docs/latest/api/buffer.html#buffer_buffers_and_typedarray - return resolve(buffer.buffer); + return resolve(toArrayBuffer(buffer)); }); }); } @@ -468,7 +427,7 @@ function _getAnonymousAttachmentFileName(message, index) { return `${message.id}-${index}`; } -async function readAttachment(dir, attachment, name, options) { +async function readEncryptedAttachment(dir, attachment, name, options) { options = options || {}; const { key } = options; @@ -485,26 +444,29 @@ async function readAttachment(dir, attachment, name, options) { const isEncrypted = !_.isUndefined(key); if (isEncrypted) { - attachment.data = await crypto.decryptSymmetric(key, data); + attachment.data = await crypto.decryptAttachment( + key, + attachment.path, + data + ); } else { attachment.data = data; } } -async function writeThumbnail(attachment, options) { +async function writeQuoteThumbnail(attachment, options) { + if (!attachment || !attachment.thumbnail || !attachment.thumbnail.path) { + return; + } + const { dir, message, index, key, newKey } = options; const filename = `${_getAnonymousAttachmentFileName( message, index - )}-thumbnail`; + )}-quote-thumbnail`; const target = path.join(dir, filename); - const { thumbnail } = attachment; - if (!thumbnail || !thumbnail.data) { - return; - } - - await writeEncryptedAttachment(target, thumbnail.data, { + await writeEncryptedAttachment(target, attachment.thumbnail.path, { key, newKey, filename, @@ -512,25 +474,13 @@ async function writeThumbnail(attachment, options) { }); } -async function writeThumbnails(rawQuotedAttachments, options) { +async function writeQuoteThumbnails(quotedAttachments, options) { const { name } = options; - const { loadAttachmentData } = Signal.Migrations; - const promises = rawQuotedAttachments.map(async attachment => { - if (!attachment || !attachment.thumbnail || !attachment.thumbnail.path) { - return attachment; - } - - return Object.assign({}, attachment, { - thumbnail: await loadAttachmentData(attachment.thumbnail), - }); - }); - - const attachments = await Promise.all(promises); try { await Promise.all( - _.map(attachments, (attachment, index) => - writeThumbnail( + _.map(quotedAttachments, (attachment, index) => + writeQuoteThumbnail( attachment, Object.assign({}, options, { index, @@ -550,26 +500,57 @@ async function writeThumbnails(rawQuotedAttachments, options) { } async function writeAttachment(attachment, options) { + if (!_.isString(attachment.path)) { + throw new Error('writeAttachment: attachment.path was not a string!'); + } + const { dir, message, index, key, newKey } = options; const filename = _getAnonymousAttachmentFileName(message, index); const target = path.join(dir, filename); - if (!Attachment.hasData(attachment)) { - throw new TypeError("'attachment.data' is required"); - } - await writeEncryptedAttachment(target, attachment.data, { + await writeEncryptedAttachment(target, attachment.path, { key, newKey, filename, dir, }); + + if (attachment.thumbnail && _.isString(attachment.thumbnail.path)) { + const thumbnailName = `${_getAnonymousAttachmentFileName( + message, + index + )}-thumbnail`; + const thumbnailTarget = path.join(dir, thumbnailName); + await writeEncryptedAttachment(thumbnailTarget, attachment.thumbnail.path, { + key, + newKey, + filename: thumbnailName, + dir, + }); + } + + if (attachment.screenshot && _.isString(attachment.screenshot.path)) { + const screenshotName = `${_getAnonymousAttachmentFileName( + message, + index + )}-screenshot`; + const screenshotTarget = path.join(dir, screenshotName); + await writeEncryptedAttachment( + screenshotTarget, + attachment.screenshot.path, + { + key, + newKey, + filename: screenshotName, + dir, + } + ); + } } -async function writeAttachments(rawAttachments, options) { +async function writeAttachments(attachments, options) { const { name } = options; - const { loadAttachmentData } = Signal.Migrations; - const attachments = await Promise.all(rawAttachments.map(loadAttachmentData)); const promises = _.map(attachments, (attachment, index) => writeAttachment( attachment, @@ -591,17 +572,18 @@ async function writeAttachments(rawAttachments, options) { } } -async function writeAvatar(avatar, options) { - const { dir, message, index, key, newKey } = options; - const name = _getAnonymousAttachmentFileName(message, index); - const filename = `${name}-contact-avatar`; - - const target = path.join(dir, filename); - if (!avatar || !avatar.path) { +async function writeAvatar(contact, options) { + const { avatar } = contact || {}; + if (!avatar || !avatar.avatar || !avatar.avatar.path) { return; } - await writeEncryptedAttachment(target, avatar.data, { + const { dir, message, index, key, newKey } = options; + const name = _getAnonymousAttachmentFileName(message, index); + const filename = `${name}-contact-avatar`; + const target = path.join(dir, filename); + + await writeEncryptedAttachment(target, avatar.avatar.path, { key, newKey, filename, @@ -612,23 +594,9 @@ async function writeAvatar(avatar, options) { async function writeContactAvatars(contact, options) { const { name } = options; - const { loadAttachmentData } = Signal.Migrations; - const promises = contact.map(async item => { - if ( - !item || - !item.avatar || - !item.avatar.avatar || - !item.avatar.avatar.path - ) { - return null; - } - - return loadAttachmentData(item.avatar.avatar); - }); - try { await Promise.all( - _.map(await Promise.all(promises), (item, index) => + _.map(contact, (item, index) => writeAvatar( item, Object.assign({}, options, { @@ -648,7 +616,7 @@ async function writeContactAvatars(contact, options) { } } -async function writeEncryptedAttachment(target, data, options = {}) { +async function writeEncryptedAttachment(target, source, options = {}) { const { key, newKey, filename, dir } = options; if (fs.existsSync(target)) { @@ -661,7 +629,9 @@ async function writeEncryptedAttachment(target, data, options = {}) { } } - const ciphertext = await crypto.encryptSymmetric(key, data); + const { readAttachmentData } = Signal.Migrations; + const data = await readAttachmentData(source); + const ciphertext = await crypto.encryptAttachment(key, source, data); const writer = await createFileAndWriter(dir, filename); const stream = createOutputStream(writer); @@ -673,9 +643,9 @@ function _sanitizeFileName(filename) { return filename.toString().replace(/[^a-z0-9.,+()'#\- ]/gi, '_'); } -async function exportConversation(db, conversation, options) { - options = options || {}; +async function exportConversation(conversation, options = {}) { const { name, dir, attachmentsDir, key, newKey } = options; + if (!name) { throw new Error('Need a name!'); } @@ -691,143 +661,111 @@ async function exportConversation(db, conversation, options) { window.log.info('exporting conversation', name); const writer = await createFileAndWriter(dir, 'messages.json'); + const stream = createOutputStream(writer); + stream.write('{"messages":['); - return new Promise(async (resolve, reject) => { - // TODO: need to iterate through message ids, export using window.Signal.Data - const transaction = db.transaction('messages', 'readwrite'); - transaction.onerror = () => { - Whisper.Database.handleDOMException( - `exportConversation transaction error (conversation: ${name})`, - transaction.error, - reject - ); - }; - transaction.oncomplete = () => { - // this doesn't really mean anything - we may have attachment processing to do - }; + const CHUNK_SIZE = 50; + let count = 0; + let complete = false; - const store = transaction.objectStore('messages'); - const index = store.index('conversation'); - const range = window.IDBKeyRange.bound( - [conversation.id, 0], - [conversation.id, Number.MAX_VALUE] - ); + // We're looping from the most recent to the oldest + let lastReceivedAt = Number.MAX_VALUE; - let promiseChain = Promise.resolve(); - let count = 0; - const request = index.openCursor(range); - - const stream = createOutputStream(writer); - stream.write('{"messages":['); - - request.onerror = () => { - Whisper.Database.handleDOMException( - `exportConversation request error (conversation: ${name})`, - request.error, - reject - ); - }; - request.onsuccess = async event => { - const cursor = event.target.result; - if (cursor) { - const message = cursor.value; - const { attachments } = message; - - // skip message if it is disappearing, no matter the amount of time left - if (message.expireTimer) { - cursor.continue(); - return; - } - - if (count !== 0) { - stream.write(','); - } - - // eliminate attachment data from the JSON, since it will go to disk - // Note: this is for legacy messages only, which stored attachment data in the db - message.attachments = _.map(attachments, attachment => - _.omit(attachment, ['data']) - ); - // completely drop any attachments in messages cached in error objects - // TODO: move to lodash. Sadly, a number of the method signatures have changed! - message.errors = _.map(message.errors, error => { - if (error && error.args) { - error.args = []; - } - if (error && error.stack) { - error.stack = ''; - } - return error; - }); - - const jsonString = JSON.stringify(stringify(message)); - stream.write(jsonString); - - if (attachments && attachments.length > 0) { - const exportAttachments = () => - writeAttachments(attachments, { - dir: attachmentsDir, - name, - message, - key, - newKey, - }); - - // eslint-disable-next-line more/no-then - promiseChain = promiseChain.then(exportAttachments); - } - - const quoteThumbnails = message.quote && message.quote.attachments; - if (quoteThumbnails && quoteThumbnails.length > 0) { - const exportQuoteThumbnails = () => - writeThumbnails(quoteThumbnails, { - dir: attachmentsDir, - name, - message, - key, - newKey, - }); - - // eslint-disable-next-line more/no-then - promiseChain = promiseChain.then(exportQuoteThumbnails); - } - - const { contact } = message; - if (contact && contact.length > 0) { - const exportContactAvatars = () => - writeContactAvatars(contact, { - dir: attachmentsDir, - name, - message, - key, - newKey, - }); - - // eslint-disable-next-line more/no-then - promiseChain = promiseChain.then(exportContactAvatars); - } - - count += 1; - cursor.continue(); - } else { - try { - await Promise.all([stream.write(']}'), promiseChain, stream.close()]); - } catch (error) { - window.log.error( - 'exportConversation: error exporting conversation', - name, - ':', - error && error.stack ? error.stack : error - ); - reject(error); - return; - } - - window.log.info('done exporting conversation', name); - resolve(); + while (!complete) { + // eslint-disable-next-line no-await-in-loop + const collection = await window.Signal.Data.getMessagesByConversation( + conversation.id, + { + limit: CHUNK_SIZE, + receivedAt: lastReceivedAt, + MessageCollection: Whisper.MessageCollection, } - }; - }); + ); + const messages = getPlainJS(collection); + + for (let i = 0, max = messages.length; i < max; i += 1) { + const message = messages[i]; + if (count > 0) { + stream.write(','); + } + + count += 1; + + // skip message if it is disappearing, no matter the amount of time left + if (message.expireTimer) { + // eslint-disable-next-line no-continue + continue; + } + + const { attachments } = message; + // eliminate attachment data from the JSON, since it will go to disk + // Note: this is for legacy messages only, which stored attachment data in the db + message.attachments = _.map(attachments, attachment => + _.omit(attachment, ['data']) + ); + // completely drop any attachments in messages cached in error objects + // TODO: move to lodash. Sadly, a number of the method signatures have changed! + message.errors = _.map(message.errors, error => { + if (error && error.args) { + error.args = []; + } + if (error && error.stack) { + error.stack = ''; + } + return error; + }); + + const jsonString = JSON.stringify(stringify(message)); + stream.write(jsonString); + + if (attachments && attachments.length > 0) { + // eslint-disable-next-line no-await-in-loop + await writeAttachments(attachments, { + dir: attachmentsDir, + name, + message, + key, + newKey, + }); + } + + const quoteThumbnails = message.quote && message.quote.attachments; + if (quoteThumbnails && quoteThumbnails.length > 0) { + // eslint-disable-next-line no-await-in-loop + await writeQuoteThumbnails(quoteThumbnails, { + dir: attachmentsDir, + name, + message, + key, + newKey, + }); + } + + const { contact } = message; + if (contact && contact.length > 0) { + // eslint-disable-next-line no-await-in-loop + await writeContactAvatars(contact, { + dir: attachmentsDir, + name, + message, + key, + newKey, + }); + } + } + + const last = messages.length > 0 ? messages[messages.length - 1] : null; + if (last) { + lastReceivedAt = last.received_at; + } + + if (messages.length < CHUNK_SIZE) { + complete = true; + } + } + + stream.write(']}'); + await stream.close(); } // Goals for directory names: @@ -857,74 +795,40 @@ function _getConversationLoggingName(conversation) { return name; } -function exportConversations(db, options) { +async function exportConversations(options) { options = options || {}; const { messagesDir, attachmentsDir, key, newKey } = options; if (!messagesDir) { - return Promise.reject(new Error('Need a messages directory!')); + throw new Error('Need a messages directory!'); } if (!attachmentsDir) { - return Promise.reject(new Error('Need an attachments directory!')); + throw new Error('Need an attachments directory!'); } - return new Promise((resolve, reject) => { - const transaction = db.transaction('conversations', 'readwrite'); - transaction.onerror = () => { - Whisper.Database.handleDOMException( - 'exportConversations transaction error', - transaction.error, - reject - ); - }; - transaction.oncomplete = () => { - // not really very useful - fires at unexpected times - }; - - let promiseChain = Promise.resolve(); - const store = transaction.objectStore('conversations'); - const request = store.openCursor(); - request.onerror = () => { - Whisper.Database.handleDOMException( - 'exportConversations request error', - request.error, - reject - ); - }; - request.onsuccess = async event => { - const cursor = event.target.result; - if (cursor && cursor.value) { - const conversation = cursor.value; - const dirName = _getConversationDirName(conversation); - const name = _getConversationLoggingName(conversation); - - const process = async () => { - const dir = await createDirectory(messagesDir, dirName); - return exportConversation(db, conversation, { - name, - dir, - attachmentsDir, - key, - newKey, - }); - }; - - window.log.info('scheduling export for conversation', name); - // eslint-disable-next-line more/no-then - promiseChain = promiseChain.then(process); - cursor.continue(); - } else { - window.log.info('Done scheduling conversation exports'); - try { - await promiseChain; - } catch (error) { - reject(error); - return; - } - resolve(); - } - }; + const collection = await window.Signal.Data.getAllConversations({ + ConversationCollection: Whisper.ConversationCollection, }); + const conversations = collection.models; + + for (let i = 0, max = conversations.length; i < max; i += 1) { + const conversation = conversations[i]; + const dirName = _getConversationDirName(conversation); + const name = _getConversationLoggingName(conversation); + + // eslint-disable-next-line no-await-in-loop + const dir = await createDirectory(messagesDir, dirName); + // eslint-disable-next-line no-await-in-loop + await exportConversation(conversation, { + name, + dir, + attachmentsDir, + key, + newKey, + }); + } + + window.log.info('Done exporting conversations!'); } function getDirectory(options = {}) { @@ -968,9 +872,30 @@ async function loadAttachments(dir, getName, options) { const { message } = options; await Promise.all( - _.map(message.attachments, (attachment, index) => { + _.map(message.attachments, async (attachment, index) => { const name = getName(message, index, attachment); - return readAttachment(dir, attachment, name, options); + + await readEncryptedAttachment(dir, attachment, name, options); + + if (attachment.thumbnail && _.isString(attachment.thumbnail.path)) { + const thumbnailName = `${name}-thumbnail`; + await readEncryptedAttachment( + dir, + attachment.thumbnail, + thumbnailName, + options + ); + } + + if (attachment.screenshot && _.isString(attachment.screenshot.path)) { + const screenshotName = `${name}-screenshot`; + await readEncryptedAttachment( + dir, + attachment.screenshot, + screenshotName, + options + ); + } }) ); @@ -982,8 +907,8 @@ async function loadAttachments(dir, getName, options) { return null; } - const name = `${getName(message, index)}-thumbnail`; - return readAttachment(dir, thumbnail, name, options); + const name = `${getName(message, index)}-quote-thumbnail`; + return readEncryptedAttachment(dir, thumbnail, name, options); }) ); @@ -996,7 +921,7 @@ async function loadAttachments(dir, getName, options) { } const name = `${getName(message, index)}-contact-avatar`; - return readAttachment(dir, avatar, name, options); + return readEncryptedAttachment(dir, avatar, name, options); }) ); @@ -1179,31 +1104,22 @@ function getDirectoryForExport() { return getDirectory(); } -function createZip(zipDir, targetDir) { - return new Promise((resolve, reject) => { - const target = path.join(zipDir, 'messages.zip'); - const output = fs.createWriteStream(target); - const archive = archiver('zip', { +async function compressArchive(file, targetDir) { + const items = fs.readdirSync(targetDir); + return tar.c( + { + gzip: true, + file, cwd: targetDir, - }); + }, + items + ); +} - output.on('close', () => { - resolve(target); - }); - - archive.on('warning', error => { - window.log.warn(`Archive generation warning: ${error.stack}`); - }); - archive.on('error', reject); - - archive.pipe(output); - - // The empty string ensures that the base location of the files added to the zip - // is nothing. If you provide null, you get the absolute path you pulled the files - // from in the first place. - archive.directory(targetDir, ''); - - archive.finalize(); +async function decompressArchive(file, targetDir) { + return tar.x({ + file, + cwd: targetDir, }); } @@ -1211,6 +1127,13 @@ function writeFile(targetPath, contents) { return pify(fs.writeFile)(targetPath, contents); } +// prettier-ignore +const UNIQUE_ID = new Uint8Array([ + 1, 3, 4, 5, 6, 7, 8, 11, + 23, 34, 1, 34, 3, 5, 45, 45, + 1, 3, 4, 5, 6, 7, 8, 11, + 23, 34, 1, 34, 3, 5, 45, 45, +]); async function encryptFile(sourcePath, targetPath, options) { options = options || {}; @@ -1220,8 +1143,8 @@ async function encryptFile(sourcePath, targetPath, options) { } const plaintext = await readFileAsArrayBuffer(sourcePath); - const ciphertext = await crypto.encryptSymmetric(key, plaintext); - return writeFile(targetPath, ciphertext); + const ciphertext = await crypto.encryptFile(key, UNIQUE_ID, plaintext); + return writeFile(targetPath, Buffer.from(ciphertext)); } async function decryptFile(sourcePath, targetPath, options) { @@ -1233,7 +1156,7 @@ async function decryptFile(sourcePath, targetPath, options) { } const ciphertext = await readFileAsArrayBuffer(sourcePath); - const plaintext = await crypto.decryptSymmetric(key, ciphertext); + const plaintext = await crypto.decryptFile(key, UNIQUE_ID, ciphertext); return writeFile(targetPath, Buffer.from(plaintext)); } @@ -1246,9 +1169,9 @@ function deleteAll(pattern) { return pify(rimraf)(pattern); } -async function exportToDirectory(directory, options) { - throw new Error('Encrypted export/import is disabled'); +const ARCHIVE_NAME = 'messages.tar.gz'; +async function exportToDirectory(directory, options) { options = options || {}; if (!options.key) { @@ -1261,20 +1184,19 @@ async function exportToDirectory(directory, options) { stagingDir = await createTempDir(); encryptionDir = await createTempDir(); - const db = await Whisper.Database.open(); const attachmentsDir = await createDirectory(directory, 'attachments'); - await exportContactAndGroupsToFile(db, stagingDir); + await exportContactAndGroupsToFile(stagingDir); await exportConversations( - db, Object.assign({}, options, { messagesDir: stagingDir, attachmentsDir, }) ); - const zip = await createZip(encryptionDir, stagingDir); - await encryptFile(zip, path.join(directory, 'messages.zip'), options); + const archivePath = path.join(directory, ARCHIVE_NAME); + await compressArchive(archivePath, stagingDir); + await encryptFile(archivePath, path.join(directory, ARCHIVE_NAME), options); window.log.info('done backing up!'); return directory; @@ -1317,10 +1239,8 @@ async function importFromDirectory(directory, options) { groupLookup, }); - const zipPath = path.join(directory, 'messages.zip'); - if (fs.existsSync(zipPath)) { - throw new Error('Encrypted export/import is disabled'); - + const archivePath = path.join(directory, ARCHIVE_NAME); + if (fs.existsSync(archivePath)) { // we're in the world of an encrypted, zipped backup if (!options.key) { throw new Error( @@ -1336,9 +1256,9 @@ async function importFromDirectory(directory, options) { const attachmentsDir = path.join(directory, 'attachments'); - const decryptedZip = path.join(decryptionDir, 'messages.zip'); - await decryptFile(zipPath, decryptedZip, options); - await decompress(decryptedZip, stagingDir); + const decryptedArchivePath = path.join(decryptionDir, ARCHIVE_NAME); + await decryptFile(archivePath, decryptedArchivePath, options); + await decompressArchive(decryptedArchivePath, stagingDir); options = Object.assign({}, options, { attachmentsDir, diff --git a/js/modules/crypto.js b/js/modules/crypto.js index 31f9fb97d..403f4c6ee 100644 --- a/js/modules/crypto.js +++ b/js/modules/crypto.js @@ -11,10 +11,14 @@ module.exports = { constantTimeEqual, decryptAesCtr, decryptDeviceName, + decryptAttachment, + decryptFile, decryptSymmetric, deriveAccessKey, encryptAesCtr, encryptDeviceName, + encryptAttachment, + encryptFile, encryptSymmetric, fromEncodedBinaryToArrayBuffer, getAccessKeyVerifier, @@ -30,6 +34,24 @@ module.exports = { verifyAccessKey, }; +function arrayBufferToBase64(arrayBuffer) { + return dcodeIO.ByteBuffer.wrap(arrayBuffer).toString('base64'); +} +function base64ToArrayBuffer(base64string) { + return dcodeIO.ByteBuffer.wrap(base64string, 'base64').toArrayBuffer(); +} + +function fromEncodedBinaryToArrayBuffer(key) { + return dcodeIO.ByteBuffer.wrap(key, 'binary').toArrayBuffer(); +} + +function bytesFromString(string) { + return dcodeIO.ByteBuffer.wrap(string, 'utf8').toArrayBuffer(); +} +function stringFromBytes(buffer) { + return dcodeIO.ByteBuffer.wrap(buffer).toString('utf8'); +} + // High-level Operations async function encryptDeviceName(deviceName, identityPublic) { @@ -81,6 +103,48 @@ async function decryptDeviceName( return stringFromBytes(plaintext); } +// Path structure: 'fa/facdf99c22945b1c9393345599a276f4b36ad7ccdc8c2467f5441b742c2d11fa' +function getAttachmentLabel(path) { + const filename = path.slice(3); + return base64ToArrayBuffer(filename); +} + +const PUB_KEY_LENGTH = 32; +async function encryptAttachment(staticPublicKey, path, plaintext) { + const uniqueId = getAttachmentLabel(path); + return encryptFile(staticPublicKey, uniqueId, plaintext); +} + +async function decryptAttachment(staticPrivateKey, path, data) { + const uniqueId = getAttachmentLabel(path); + return decryptFile(staticPrivateKey, uniqueId, data); +} + +async function encryptFile(staticPublicKey, uniqueId, plaintext) { + const ephemeralKeyPair = await libsignal.KeyHelper.generateIdentityKeyPair(); + const agreement = await libsignal.Curve.async.calculateAgreement( + staticPublicKey, + ephemeralKeyPair.privKey + ); + const key = await hmacSha256(agreement, uniqueId); + + const prefix = ephemeralKeyPair.pubKey.slice(1); + return concatenateBytes(prefix, await encryptSymmetric(key, plaintext)); +} + +async function decryptFile(staticPrivateKey, uniqueId, data) { + const ephemeralPublicKey = _getFirstBytes(data, PUB_KEY_LENGTH); + const ciphertext = _getBytes(data, PUB_KEY_LENGTH, data.byteLength); + const agreement = await libsignal.Curve.async.calculateAgreement( + ephemeralPublicKey, + staticPrivateKey + ); + + const key = await hmacSha256(agreement, uniqueId); + + return decryptSymmetric(key, ciphertext); +} + async function deriveAccessKey(profileKey) { const iv = getZeroes(12); const plaintext = getZeroes(16); @@ -318,24 +382,6 @@ function trimBytes(buffer, length) { return _getFirstBytes(buffer, length); } -function arrayBufferToBase64(arrayBuffer) { - return dcodeIO.ByteBuffer.wrap(arrayBuffer).toString('base64'); -} -function base64ToArrayBuffer(base64string) { - return dcodeIO.ByteBuffer.wrap(base64string, 'base64').toArrayBuffer(); -} - -function fromEncodedBinaryToArrayBuffer(key) { - return dcodeIO.ByteBuffer.wrap(key, 'binary').toArrayBuffer(); -} - -function bytesFromString(string) { - return dcodeIO.ByteBuffer.wrap(string, 'utf8').toArrayBuffer(); -} -function stringFromBytes(buffer) { - return dcodeIO.ByteBuffer.wrap(buffer).toString('utf8'); -} - function getViewOfArrayBuffer(buffer, start, finish) { const source = new Uint8Array(buffer); const result = source.slice(start, finish); diff --git a/js/modules/data.js b/js/modules/data.js index a0a81c920..83cfa1153 100644 --- a/js/modules/data.js +++ b/js/modules/data.js @@ -50,6 +50,7 @@ module.exports = { createOrUpdateGroup, getGroupById, getAllGroupIds, + getAllGroups, bulkAddGroups, removeGroupById, removeAllGroups, @@ -395,6 +396,10 @@ async function getAllGroupIds() { const ids = await channels.getAllGroupIds(); return ids; } +async function getAllGroups() { + const groups = await channels.getAllGroups(); + return groups; +} async function bulkAddGroups(array) { await channels.bulkAddGroups(array); } diff --git a/js/modules/signal.js b/js/modules/signal.js index 96a05b475..9cb1bffe1 100644 --- a/js/modules/signal.js +++ b/js/modules/signal.js @@ -133,6 +133,7 @@ function initializeMigrations({ loadAttachmentData, loadQuoteData, loadMessage: MessageType.createAttachmentLoader(loadAttachmentData), + readAttachmentData, run, upgradeMessageSchema: (message, options = {}) => { const { maxVersion } = options; diff --git a/js/modules/types/message.js b/js/modules/types/message.js index 33faf2163..cc08e4d03 100644 --- a/js/modules/types/message.js +++ b/js/modules/types/message.js @@ -545,8 +545,6 @@ exports.createAttachmentDataWriter = ({ }); }; - // TODO: need to handle attachment thumbnails and video screenshots - const messageWithoutAttachmentData = Object.assign( {}, await writeThumbnails(message, { logger }), @@ -555,7 +553,23 @@ exports.createAttachmentDataWriter = ({ attachments: await Promise.all( (attachments || []).map(async attachment => { await writeExistingAttachmentData(attachment); - return omit(attachment, ['data']); + + if (attachment.screenshot && attachment.screenshot.data) { + await writeExistingAttachmentData(attachment.screenshot); + } + if (attachment.thumbnail && attachment.thumbnail.data) { + await writeExistingAttachmentData(attachment.thumbnail); + } + + return { + ...omit(attachment, ['data']), + ...(attachment.thumbnail + ? { thumbnail: omit(attachment.thumbnail, ['data']) } + : null), + ...(attachment.screenshot + ? { screenshot: omit(attachment.screenshot, ['data']) } + : null), + }; }) ), } diff --git a/package.json b/package.json index 0cb942536..30964590c 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,6 @@ "dependencies": { "@journeyapps/sqlcipher": "https://github.com/scottnonnenberg-signal/node-sqlcipher.git#ed4f4d179ac010c6347b291cbd4c2ebe5c773741", "@sindresorhus/is": "0.8.0", - "archiver": "2.1.1", "backbone": "1.3.3", "blob-util": "1.3.0", "blueimp-canvas-to-blob": "3.14.0", @@ -86,6 +85,7 @@ "rimraf": "2.6.2", "semver": "5.4.1", "spellchecker": "3.4.4", + "tar": "4.4.8", "testcheck": "1.0.0-rc.2", "tmp": "0.0.33", "to-arraybuffer": "1.0.1", diff --git a/preload.js b/preload.js index c80303396..ece3f61e0 100644 --- a/preload.js +++ b/preload.js @@ -274,3 +274,16 @@ window.Signal.Logs = require('./js/modules/logs'); // We pull this in last, because the native module involved appears to be sensitive to // /tmp mounted as noexec on Linux. require('./js/spell_check'); + +if (config.environment === 'test') { + /* eslint-disable global-require, import/no-extraneous-dependencies */ + window.test = { + glob: require('glob'), + fse: require('fs-extra'), + tmp: require('tmp'), + path: require('path'), + basePath: __dirname, + attachmentsPath: window.Signal.Migrations.attachmentsPath, + }; + /* eslint-enable global-require, import/no-extraneous-dependencies */ +} diff --git a/test/backup_test.js b/test/backup_test.js index 2ac94cea9..d8dbc001d 100644 --- a/test/backup_test.js +++ b/test/backup_test.js @@ -1,10 +1,6 @@ -/* global Signal: false */ -/* global Whisper: false */ -/* global assert: false */ -/* global textsecure: false */ -/* global _: false */ +/* global Signal, Whisper, assert, textsecure, _, libsignal */ -/* eslint-disable no-unreachable, no-console */ +/* eslint-disable no-console */ 'use strict'; @@ -240,8 +236,8 @@ describe('Backup', () => { }); describe('end-to-end', () => { - it('exports then imports to produce the same data we started with', async () => { - return; + it('exports then imports to produce the same data we started with', async function thisNeeded() { + this.timeout(6000); const { attachmentsPath, fse, glob, path, tmp } = window.test; const { @@ -249,46 +245,32 @@ describe('Backup', () => { loadAttachmentData, } = window.Signal.Migrations; - const key = new Uint8Array([ - 1, - 3, - 4, - 5, - 6, - 7, - 8, - 11, - 23, - 34, - 1, - 34, - 3, - 5, - 45, - 45, - 1, - 3, - 4, - 5, - 6, - 7, - 8, - 11, - 23, - 34, - 1, - 34, - 3, - 5, - 45, - 45, - ]); + const staticKeyPair = await libsignal.KeyHelper.generateIdentityKeyPair(); const attachmentsPattern = path.join(attachmentsPath, '**'); const OUR_NUMBER = '+12025550000'; const CONTACT_ONE_NUMBER = '+12025550001'; const CONTACT_TWO_NUMBER = '+12025550002'; + const toArrayBuffer = nodeBuffer => + nodeBuffer.buffer.slice( + nodeBuffer.byteOffset, + nodeBuffer.byteOffset + nodeBuffer.byteLength + ); + + const getFixture = target => toArrayBuffer(fse.readFileSync(target)); + + const FIXTURES = { + gif: getFixture('fixtures/giphy-7GFfijngKbeNy.gif'), + mp4: getFixture('fixtures/pixabay-Soap-Bubble-7141.mp4'), + jpg: getFixture('fixtures/koushik-chowdavarapu-105425-unsplash.jpg'), + mp3: getFixture('fixtures/incompetech-com-Agnus-Dei-X.mp3'), + txt: getFixture('fixtures/lorem-ipsum.txt'), + png: getFixture( + 'fixtures/freepngs-2cd43b_bed7d1327e88454487397574d87b64dc_mv2.png' + ), + }; + async function wrappedLoadAttachment(attachment) { return _.omit(await loadAttachmentData(attachment), ['path']); } @@ -376,16 +358,30 @@ describe('Backup', () => { }) ), attachments: await Promise.all( - (message.attachments || []).map(attachment => - wrappedLoadAttachment(attachment) - ) + (message.attachments || []).map(async attachment => { + await wrappedLoadAttachment(attachment); + + if (attachment.thumbnail) { + await wrappedLoadAttachment(attachment.thumbnail); + } + + if (attachment.screenshot) { + await wrappedLoadAttachment(attachment.screenshot); + } + + return attachment; + }) ), }); } let backupDir; try { - const ATTACHMENT_COUNT = 3; + // Seven total: + // - Five from image/video attachments + // - One from embedded contact avatar + // - Another from embedded quoted attachment thumbnail + const ATTACHMENT_COUNT = 7; const MESSAGE_COUNT = 1; const CONVERSATION_COUNT = 1; @@ -397,47 +393,20 @@ describe('Backup', () => { timestamp: 1524185933350, errors: [], attachments: [ + // Note: generates two more files: screenshot and thumbnail { - contentType: 'image/gif', - fileName: 'sad_cat.gif', - data: new Uint8Array([ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - ]).buffer, + contentType: 'video/mp4', + fileName: 'video.mp4', + data: FIXTURES.mp4, + }, + // Note: generates one more file: thumbnail + { + contentType: 'image/png', + fileName: 'landscape.png', + data: FIXTURES.png, }, ], hasAttachments: 1, - hasFileAttachments: undefined, hasVisualMediaAttachments: 1, quote: { text: "Isn't it cute?", @@ -450,43 +419,10 @@ describe('Backup', () => { }, { contentType: 'image/gif', - fileName: 'happy_cat.gif', + fileName: 'avatar.gif', thumbnail: { contentType: 'image/png', - data: new Uint8Array([ - 2, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - ]).buffer, + data: FIXTURES.gif, }, }, ], @@ -506,40 +442,7 @@ describe('Backup', () => { isProfile: false, avatar: { contentType: 'image/png', - data: new Uint8Array([ - 3, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - ]).buffer, + data: FIXTURES.png, }, }, }, @@ -552,107 +455,30 @@ describe('Backup', () => { console.log('Backup test: Create models, save to db/disk'); const message = await upgradeMessageSchema(messageWithAttachments); console.log({ message }); - const messageModel = new Whisper.Message(message); - const id = await window.Signal.Data.saveMessage( - messageModel.attributes, - { - Message: Whisper.Message, - } - ); - messageModel.set({ id }); + await window.Signal.Data.saveMessage(message, { + Message: Whisper.Message, + }); const conversation = { active_at: 1524185933350, color: 'orange', expireTimer: 0, id: CONTACT_ONE_NUMBER, - lastMessage: 'Heyo!', name: 'Someone Somewhere', profileAvatar: { contentType: 'image/jpeg', - data: new Uint8Array([ - 4, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - ]).buffer, + data: FIXTURES.jpeg, size: 64, }, - profileKey: new Uint8Array([ - 5, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - ]).buffer, + profileKey: 'BASE64KEY', profileName: 'Someone! 🤔', profileSharing: true, timestamp: 1524185933350, - tokens: [ - 'someone somewhere', - 'someone', - 'somewhere', - '2025550001', - '12025550001', - ], type: 'private', unreadCount: 0, verified: 0, + sealedSender: 0, + version: 2, }; console.log({ conversation }); await window.Signal.Data.saveConversation(conversation, { @@ -669,11 +495,13 @@ describe('Backup', () => { console.log('Backup test: Export!'); backupDir = tmp.dirSync().name; console.log({ backupDir }); - await Signal.Backup.exportToDirectory(backupDir, { key }); + await Signal.Backup.exportToDirectory(backupDir, { + key: staticKeyPair.pubKey, + }); - console.log('Backup test: Ensure that messages.zip exists'); - const zipPath = path.join(backupDir, 'messages.zip'); - const messageZipExists = fse.existsSync(zipPath); + console.log('Backup test: Ensure that messages.tar.gz exists'); + const archivePath = path.join(backupDir, 'messages.tar.gz'); + const messageZipExists = fse.existsSync(archivePath); assert.strictEqual(true, messageZipExists); console.log( @@ -688,43 +516,9 @@ describe('Backup', () => { await clearAllData(); console.log('Backup test: Import!'); - await Signal.Backup.importFromDirectory(backupDir, { key }); - - console.log('Backup test: ensure that all attachments were imported'); - const recreatedAttachmentFiles = removeDirs( - glob.sync(attachmentsPattern) - ); - console.log({ recreatedAttachmentFiles }); - assert.strictEqual(ATTACHMENT_COUNT, recreatedAttachmentFiles.length); - assert.deepEqual(attachmentFiles, recreatedAttachmentFiles); - - console.log('Backup test: Check messages'); - const messageCollection = await window.Signal.Data.getAllMessages({ - MessageCollection: Whisper.MessageCollection, + await Signal.Backup.importFromDirectory(backupDir, { + key: staticKeyPair.privKey, }); - assert.strictEqual(messageCollection.length, MESSAGE_COUNT); - const messageFromDB = removeId(messageCollection.at(0).attributes); - const expectedMessage = omitUndefinedKeys(message); - console.log({ messageFromDB, expectedMessage }); - assert.deepEqual(messageFromDB, expectedMessage); - - console.log( - 'Backup test: Check that all attachments were successfully imported' - ); - const messageWithAttachmentsFromDB = await loadAllFilesFromDisk( - messageFromDB - ); - const expectedMessageWithAttachments = omitUndefinedKeys( - messageWithAttachments - ); - console.log({ - messageWithAttachmentsFromDB, - expectedMessageWithAttachments, - }); - assert.deepEqual( - _.omit(messageWithAttachmentsFromDB, ['schemaVersion']), - expectedMessageWithAttachments - ); console.log('Backup test: Check conversations'); const conversationCollection = await window.Signal.Data.getAllConversations( @@ -741,6 +535,42 @@ describe('Backup', () => { _.omit(conversation, ['profileAvatar']) ); + console.log('Backup test: Check messages'); + const messageCollection = await window.Signal.Data.getAllMessages({ + MessageCollection: Whisper.MessageCollection, + }); + assert.strictEqual(messageCollection.length, MESSAGE_COUNT); + const messageFromDB = removeId(messageCollection.at(0).attributes); + const expectedMessage = messageFromDB; + console.log({ messageFromDB, expectedMessage }); + assert.deepEqual(messageFromDB, expectedMessage); + + console.log('Backup test: ensure that all attachments were imported'); + const recreatedAttachmentFiles = removeDirs( + glob.sync(attachmentsPattern) + ); + console.log({ recreatedAttachmentFiles }); + assert.strictEqual(ATTACHMENT_COUNT, recreatedAttachmentFiles.length); + assert.deepEqual(attachmentFiles, recreatedAttachmentFiles); + + console.log( + 'Backup test: Check that all attachments were successfully imported' + ); + const messageWithAttachmentsFromDB = await loadAllFilesFromDisk( + messageFromDB + ); + const expectedMessageWithAttachments = await loadAllFilesFromDisk( + omitUndefinedKeys(message) + ); + console.log({ + messageWithAttachmentsFromDB, + expectedMessageWithAttachments, + }); + assert.deepEqual( + messageWithAttachmentsFromDB, + expectedMessageWithAttachments + ); + console.log('Backup test: Clear all data'); await clearAllData(); diff --git a/test/crypto_test.js b/test/crypto_test.js index a047c2b31..44eb42877 100644 --- a/test/crypto_test.js +++ b/test/crypto_test.js @@ -44,7 +44,7 @@ describe('Crypto', () => { const encrypted = await Signal.Crypto.encryptSymmetric(key, plaintext); const uintArray = new Uint8Array(encrypted); - uintArray[2] = 9; + uintArray[2] += 2; try { await Signal.Crypto.decryptSymmetric(key, uintArray.buffer); @@ -69,7 +69,7 @@ describe('Crypto', () => { const encrypted = await Signal.Crypto.encryptSymmetric(key, plaintext); const uintArray = new Uint8Array(encrypted); - uintArray[uintArray.length - 3] = 9; + uintArray[uintArray.length - 3] += 2; try { await Signal.Crypto.decryptSymmetric(key, uintArray.buffer); @@ -94,7 +94,7 @@ describe('Crypto', () => { const encrypted = await Signal.Crypto.encryptSymmetric(key, plaintext); const uintArray = new Uint8Array(encrypted); - uintArray[35] = 9; + uintArray[35] += 9; try { await Signal.Crypto.decryptSymmetric(key, uintArray.buffer); @@ -146,4 +146,30 @@ describe('Crypto', () => { } }); }); + + describe('attachment encryption', () => { + it('roundtrips', async () => { + const staticKeyPair = await libsignal.KeyHelper.generateIdentityKeyPair(); + const message = 'this is my message'; + const plaintext = Signal.Crypto.bytesFromString(message); + const path = + 'fa/facdf99c22945b1c9393345599a276f4b36ad7ccdc8c2467f5441b742c2d11fa'; + + const encrypted = await Signal.Crypto.encryptAttachment( + staticKeyPair.pubKey.slice(1), + path, + plaintext + ); + const decrypted = await Signal.Crypto.decryptAttachment( + staticKeyPair.privKey, + path, + encrypted + ); + + const equal = Signal.Crypto.constantTimeEqual(plaintext, decrypted); + if (!equal) { + throw new Error('The output and input did not match!'); + } + }); + }); }); diff --git a/test/metadata/SecretSessionCipher_test.js b/test/metadata/SecretSessionCipher_test.js index c60805abd..6fb7beea3 100644 --- a/test/metadata/SecretSessionCipher_test.js +++ b/test/metadata/SecretSessionCipher_test.js @@ -148,7 +148,9 @@ InMemorySignalProtocolStore.prototype = { }; describe('SecretSessionCipher', () => { - it('successfully roundtrips', async () => { + it('successfully roundtrips', async function thisNeeded() { + this.timeout(4000); + const aliceStore = new InMemorySignalProtocolStore(); const bobStore = new InMemorySignalProtocolStore(); @@ -187,7 +189,9 @@ describe('SecretSessionCipher', () => { assert.strictEqual(decryptResult.sender.toString(), '+14151111111.1'); }); - it('fails when untrusted', async () => { + it('fails when untrusted', async function thisNeeded() { + this.timeout(4000); + const aliceStore = new InMemorySignalProtocolStore(); const bobStore = new InMemorySignalProtocolStore(); @@ -226,7 +230,9 @@ describe('SecretSessionCipher', () => { } }); - it('fails when expired', async () => { + it('fails when expired', async function thisNeeded() { + this.timeout(4000); + const aliceStore = new InMemorySignalProtocolStore(); const bobStore = new InMemorySignalProtocolStore(); @@ -264,7 +270,9 @@ describe('SecretSessionCipher', () => { } }); - it('fails when wrong identity', async () => { + it('fails when wrong identity', async function thisNeeded() { + this.timeout(4000); + const aliceStore = new InMemorySignalProtocolStore(); const bobStore = new InMemorySignalProtocolStore(); diff --git a/ts/util/lint/exceptions.json b/ts/util/lint/exceptions.json index 6b264d1db..c09d6ac1f 100644 --- a/ts/util/lint/exceptions.json +++ b/ts/util/lint/exceptions.json @@ -319,7 +319,7 @@ "rule": "jQuery-wrap(", "path": "js/modules/crypto.js", "line": " return dcodeIO.ByteBuffer.wrap(arrayBuffer).toString('base64');", - "lineNumber": 322, + "lineNumber": 38, "reasonCategory": "falseMatch", "updated": "2018-10-05T23:12:28.961Z" }, @@ -327,7 +327,7 @@ "rule": "jQuery-wrap(", "path": "js/modules/crypto.js", "line": " return dcodeIO.ByteBuffer.wrap(base64string, 'base64').toArrayBuffer();", - "lineNumber": 325, + "lineNumber": 41, "reasonCategory": "falseMatch", "updated": "2018-10-05T23:12:28.961Z" }, @@ -335,7 +335,7 @@ "rule": "jQuery-wrap(", "path": "js/modules/crypto.js", "line": " return dcodeIO.ByteBuffer.wrap(key, 'binary').toArrayBuffer();", - "lineNumber": 329, + "lineNumber": 45, "reasonCategory": "falseMatch", "updated": "2018-10-05T23:12:28.961Z" }, @@ -343,7 +343,7 @@ "rule": "jQuery-wrap(", "path": "js/modules/crypto.js", "line": " return dcodeIO.ByteBuffer.wrap(string, 'utf8').toArrayBuffer();", - "lineNumber": 333, + "lineNumber": 49, "reasonCategory": "falseMatch", "updated": "2018-10-05T23:12:28.961Z" }, @@ -351,7 +351,7 @@ "rule": "jQuery-wrap(", "path": "js/modules/crypto.js", "line": " return dcodeIO.ByteBuffer.wrap(buffer).toString('utf8');", - "lineNumber": 336, + "lineNumber": 52, "reasonCategory": "falseMatch", "updated": "2018-10-05T23:12:28.961Z" }, diff --git a/yarn.lock b/yarn.lock index 30399fda8..b2aa13b96 100644 --- a/yarn.lock +++ b/yarn.lock @@ -391,20 +391,6 @@ archiver-utils@^1.3.0: normalize-path "^2.0.0" readable-stream "^2.0.0" -archiver@2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/archiver/-/archiver-2.1.1.tgz#ff662b4a78201494a3ee544d3a33fe7496509ebc" - integrity sha1-/2YrSnggFJSj7lRNOjP+dJZQnrw= - dependencies: - archiver-utils "^1.3.0" - async "^2.0.0" - buffer-crc32 "^0.2.1" - glob "^7.0.0" - lodash "^4.8.0" - readable-stream "^2.0.0" - tar-stream "^1.5.0" - zip-stream "^1.2.0" - archiver@~2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/archiver/-/archiver-2.1.0.tgz#d2df2e8d5773a82c1dcce925ccc41450ea999afd" @@ -1363,6 +1349,11 @@ chownr@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.0.1.tgz#e2a75042a9551908bebd25b8523d5f9769d79181" +chownr@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.1.tgz#54726b8b8fff4df053c42187e801fb4412df1494" + integrity sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g== + chrome-trace-event@^0.1.1: version "0.1.2" resolved "https://registry.yarnpkg.com/chrome-trace-event/-/chrome-trace-event-0.1.2.tgz#90f36885d5345a50621332f0717b595883d5d982" @@ -3104,6 +3095,7 @@ file-sync-cmp@^0.1.0: file-type@^3.1.0: version "3.9.0" resolved "https://registry.yarnpkg.com/file-type/-/file-type-3.9.0.tgz#257a078384d1db8087bc449d107d52a52672b9e9" + integrity sha1-JXoHg4TR24CHvESdEH1SpSZyuek= file-uri-to-path@1: version "1.0.0" @@ -5485,12 +5477,27 @@ minipass@^2.2.1, minipass@^2.3.3: safe-buffer "^5.1.2" yallist "^3.0.0" +minipass@^2.3.4: + version "2.3.5" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-2.3.5.tgz#cacebe492022497f656b0f0f51e2682a9ed2d848" + integrity sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA== + dependencies: + safe-buffer "^5.1.2" + yallist "^3.0.0" + minizlib@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-1.1.0.tgz#11e13658ce46bc3a70a267aac58359d1e0c29ceb" dependencies: minipass "^2.2.1" +minizlib@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-1.1.1.tgz#6734acc045a46e61d596a43bb9d9cd326e19cc42" + integrity sha512-TrfjCjk4jLhcJyGMYymBH6oTXcWjYbUAXTHDbtnWHjZC25h0cdajHuPE1zxb4DVmu8crfh+HwH/WMuyLG0nHBg== + dependencies: + minipass "^2.2.1" + mississippi@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/mississippi/-/mississippi-2.0.0.tgz#3442a508fafc28500486feea99409676e4ee5a6f" @@ -8319,6 +8326,7 @@ string-width@^2.1.0, string-width@^2.1.1: string_decoder@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" + integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== dependencies: safe-buffer "~5.1.0" @@ -8486,6 +8494,19 @@ tar-stream@^1.5.0: readable-stream "^2.0.0" xtend "^4.0.0" +tar@4.4.8: + version "4.4.8" + resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.8.tgz#b19eec3fde2a96e64666df9fdb40c5ca1bc3747d" + integrity sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ== + dependencies: + chownr "^1.1.1" + fs-minipass "^1.2.5" + minipass "^2.3.4" + minizlib "^1.1.1" + mkdirp "^0.5.0" + safe-buffer "^5.1.2" + yallist "^3.0.2" + tar@^2.0.0: version "2.2.1" resolved "https://registry.yarnpkg.com/tar/-/tar-2.2.1.tgz#8e4d2a256c0e2185c6b18ad694aec968b83cb1d1" From 985b1d6aa647c3944b0ad28f995c495729dc741d Mon Sep 17 00:00:00 2001 From: Scott Nonnenberg Date: Sat, 1 Dec 2018 17:48:53 -0800 Subject: [PATCH 11/35] New staged attachments UI, multiple image attachments per message --- _locales/en/messages.json | 35 +- background.html | 14 +- images/add-caption-24.svg | 1 + images/x-16.svg | 1 + images/x-shadow-16.svg | 1 + js/modules/signal.js | 6 + js/views/attachment_preview_view.js | 16 - js/views/conversation_view.js | 67 +- js/views/file_input_view.js | 710 ++++++++++-------- libtextsecure/outgoing_message.js | 2 +- stylesheets/_modules.scss | 234 ++++++ ts/components/CaptionEditor.md | 72 ++ ts/components/CaptionEditor.tsx | 78 ++ ts/components/conversation/AttachmentList.md | 114 +++ ts/components/conversation/AttachmentList.tsx | 106 +++ ts/components/conversation/Image.md | 81 ++ ts/components/conversation/Image.tsx | 26 +- ts/components/conversation/ImageGrid.tsx | 2 +- ts/components/conversation/Message.tsx | 92 +-- .../conversation/StagedGenericAttachment.md | 44 ++ .../conversation/StagedGenericAttachment.tsx | 44 ++ ts/util/lint/exceptions.json | 452 ++++++----- 22 files changed, 1550 insertions(+), 648 deletions(-) create mode 100644 images/add-caption-24.svg create mode 100644 images/x-16.svg create mode 100644 images/x-shadow-16.svg delete mode 100644 js/views/attachment_preview_view.js create mode 100644 ts/components/CaptionEditor.md create mode 100644 ts/components/CaptionEditor.tsx create mode 100644 ts/components/conversation/AttachmentList.md create mode 100644 ts/components/conversation/AttachmentList.tsx create mode 100644 ts/components/conversation/StagedGenericAttachment.md create mode 100644 ts/components/conversation/StagedGenericAttachment.tsx diff --git a/_locales/en/messages.json b/_locales/en/messages.json index bbc988fc1..edd6b109a 100644 --- a/_locales/en/messages.json +++ b/_locales/en/messages.json @@ -172,6 +172,10 @@ "message": "Choose folder", "description": "Button to allow the user to find a folder on disk" }, + "chooseFile": { + "message": "Choose file", + "description": "Button to allow the user to find a file on disk" + }, "loadDataHeader": { "message": "Load your data", "description": "Header shown on the first screen in the data import process" @@ -542,15 +546,27 @@ "message": "Voice Message", "description": "Name for a voice message attachment" }, - "unsupportedFileType": { - "message": "Unsupported file type", - "description": "Displayed for outgoing unsupported attachment" - }, "dangerousFileType": { "message": "Attachment type not allowed for security reasons", "description": "Shown in toast when user attempts to send .exe file, for example" }, + "oneNonImageAtATimeToast": { + "message": + "When including a non-image attachment, the limit is one attachment per message.", + "description": + "An error popup when the user has attempted to add an attachment" + }, + "cannotMixImageAdnNonImageAttachments": { + "message": "You cannot mix non-image and image attachments in one message.", + "description": + "An error popup when the user has attempted to add an attachment" + }, + "maximumAttachments": { + "message": "You cannot add any more attachments to this message.", + "description": + "An error popup when the user has attempted to add an attachment" + }, "fileSizeWarning": { "message": "Sorry, the selected file exceeds message size restrictions." }, @@ -732,6 +748,12 @@ "description": "Shown in toast if user clicks on quote references messages not loaded in view, but in database" }, + "voiceNoteMustBeOnlyAttachment": { + "message": + "A voice note must be the only attachment included in a message.", + "description": + "Shown in toast if tries to record a voice note with any staged attachments" + }, "you": { "message": "You", "description": @@ -910,6 +932,11 @@ "description": "Used for the icon layered on top of an image in message bubbles" }, + "addACaption": { + "message": "Add a caption...", + "descripton": + "Used as the placeholder text in the caption editor text field" + }, "fileIconAlt": { "message": "File icon", "description": diff --git a/background.html b/background.html index dad6103eb..64ef85855 100644 --- a/background.html +++ b/background.html @@ -118,9 +118,9 @@