Contact sharing: protos and data pipeline

As of this commit: 82b76ccf37
This commit is contained in:
Scott Nonnenberg 2018-04-27 09:32:31 -07:00
parent b6a585a646
commit 3ea3e4e256
7 changed files with 868 additions and 27 deletions

View File

@ -588,6 +588,7 @@
message.set({
attachments: dataMessage.attachments,
body: dataMessage.body,
contact: dataMessage.contact,
conversationId: conversation.id,
decrypted_at: now,
errors: [],

View File

@ -570,6 +570,64 @@ async function writeAttachments(rawAttachments, options) {
}
}
async function writeAvatar(avatar, options) {
console.log('writeAvatar', { avatar, options });
const { dir, message, index, key, newKey } = options;
const name = _getAnonymousAttachmentFileName(message, index);
const filename = `${name}-contact-avatar`;
const target = path.join(dir, filename);
if (!avatar || !avatar.path) {
return;
}
await writeEncryptedAttachment(target, avatar.data, {
key,
newKey,
filename,
dir,
});
}
async function writeContactAvatars(contact, options) {
const { name } = options;
const { loadAttachmentData } = Signal.Migrations;
const promises = contact.map(async item => {
if (
!item ||
!item.avatar ||
!item.avatar.avatar ||
!item.avatar.avatar.path
) {
return null;
}
return loadAttachmentData(item.avatar.avatar);
});
try {
await Promise.all(
_.map(await Promise.all(promises), (item, index) =>
writeAvatar(
item,
Object.assign({}, options, {
index,
})
)
)
);
} catch (error) {
console.log(
'writeContactAvatars: error exporting conversation',
name,
':',
error && error.stack ? error.stack : error
);
throw error;
}
}
async function writeEncryptedAttachment(target, data, options = {}) {
const { key, newKey, filename, dir } = options;
@ -714,6 +772,21 @@ async function exportConversation(db, conversation, options) {
promiseChain = promiseChain.then(exportQuoteThumbnails);
}
const { contact } = message;
if (contact && contact.length > 0) {
const exportContactAvatars = () =>
writeContactAvatars(contact, {
dir: attachmentsDir,
name,
message,
key,
newKey,
});
// eslint-disable-next-line more/no-then
promiseChain = promiseChain.then(exportContactAvatars);
}
count += 1;
cursor.continue();
} else {
@ -870,27 +943,44 @@ function getDirContents(dir) {
});
}
function loadAttachments(dir, getName, options) {
async function loadAttachments(dir, getName, options) {
options = options || {};
const { message } = options;
const attachmentPromises = _.map(message.attachments, (attachment, index) => {
const name = getName(message, index, attachment);
return readAttachment(dir, attachment, name, options);
});
await Promise.all(
_.map(message.attachments, (attachment, index) => {
const name = getName(message, index, attachment);
return readAttachment(dir, attachment, name, options);
})
);
const quoteAttachments = message.quote && message.quote.attachments;
const thumbnailPromises = _.map(quoteAttachments, (attachment, index) => {
const thumbnail = attachment && attachment.thumbnail;
if (!thumbnail) {
return null;
}
await Promise.all(
_.map(quoteAttachments, (attachment, index) => {
const thumbnail = attachment && attachment.thumbnail;
if (!thumbnail) {
return null;
}
const name = `${getName(message, index, thumbnail)}-thumbnail`;
return readAttachment(dir, thumbnail, name, options);
});
const name = `${getName(message, index)}-thumbnail`;
return readAttachment(dir, thumbnail, name, options);
})
);
return Promise.all(attachmentPromises.concat(thumbnailPromises));
const { contact } = message;
await Promise.all(
_.map(contact, (item, index) => {
const avatar = item && item.avatar && item.avatar.avatar;
if (!avatar) {
return null;
}
const name = `${getName(message, index)}-contact-avatar`;
return readAttachment(dir, avatar, name, options);
})
);
console.log('loadAttachments', { message });
}
function saveMessage(db, message) {

View File

@ -1,4 +1,4 @@
const { isFunction, isString, omit } = require('lodash');
const { isFunction, isString, omit, compact, map } = require('lodash');
const Attachment = require('./attachment');
const Errors = require('./errors');
@ -29,6 +29,8 @@ const PRIVATE = 'private';
// - `hasAttachments?: 1 | 0`
// - `hasVisualMediaAttachments?: 1 | undefined` (for media gallery Media view)
// - `hasFileAttachments?: 1 | undefined` (for media gallery Documents view)
// Version 6
// - Contact: Write contact avatar to disk, ensure contact data is well-formed
const INITIAL_SCHEMA_VERSION = 0;
@ -37,7 +39,7 @@ const INITIAL_SCHEMA_VERSION = 0;
// add more upgrade steps, we could design a pipeline that does this
// incrementally, e.g. from version 0 / unknown -> 1, 1 --> 2, etc., similar to
// how we do database migrations:
exports.CURRENT_SCHEMA_VERSION = 5;
exports.CURRENT_SCHEMA_VERSION = 6;
// Public API
exports.GROUP = GROUP;
@ -154,6 +156,20 @@ exports._mapAttachments = upgradeAttachment => async (message, context) => {
return Object.assign({}, message, { attachments });
};
// Public API
// _mapContact :: (Contact -> Promise Contact) ->
// (Message, Context) ->
// Promise Message
exports._mapContact = upgradeContact => async (message, context) => {
const contextWithMessage = Object.assign({}, context, { message });
const upgradeWithContext = contact =>
upgradeContact(contact, contextWithMessage);
const contact = await Promise.all(
(message.contact || []).map(upgradeWithContext)
);
return Object.assign({}, message, { contact });
};
// _mapQuotedAttachments :: (QuotedAttachment -> Promise QuotedAttachment) ->
// (Message, Context) ->
// Promise Message
@ -194,6 +210,126 @@ exports._mapQuotedAttachments = upgradeAttachment => async (
});
};
function validateContact(contact, options = {}) {
const { messageId } = options;
const { name, number, email, address, organization } = contact;
if ((!name || !name.displayName) && !organization) {
console.log(
`Message ${messageId}: Contact had neither 'displayName' nor 'organization'`
);
return false;
}
if (
(!number || !number.length) &&
(!email || !email.length) &&
(!address || !address.length)
) {
console.log(
`Message ${messageId}: Contact had no included numbers, email or addresses`
);
return false;
}
return true;
}
function cleanContact(contact) {
function cleanBasicItem(item) {
if (!item.value) {
return null;
}
return Object.assign({}, item, {
type: item.type || 1,
});
}
function cleanAddress(address) {
if (!address) {
return null;
}
if (
!address.street &&
!address.pobox &&
!address.neighborhood &&
!address.city &&
!address.region &&
!address.postcode &&
!address.country
) {
return null;
}
return Object.assign({}, address, {
type: address.type || 1,
});
}
function cleanAvatar(avatar) {
if (!avatar) {
return null;
}
return {
avatar: Object.assign({}, avatar, {
isProfile: avatar.isProfile || false,
}),
};
}
function addArrayKey(key, array) {
if (!array || !array.length) {
return null;
}
return {
[key]: array,
};
}
return Object.assign(
{},
omit(contact, ['avatar', 'number', 'email', 'address']),
cleanAvatar(contact.avatar),
addArrayKey('number', compact(map(contact.number, cleanBasicItem))),
addArrayKey('email', compact(map(contact.email, cleanBasicItem))),
addArrayKey('address', compact(map(contact.address, cleanAddress)))
);
}
function idForLogging(message) {
return `${message.source}.${message.sourceDevice} ${message.sent_at}`;
}
exports._cleanAndWriteContactAvatar = upgradeAttachment => async (
contact,
context = {}
) => {
const { message } = context;
const { avatar } = contact;
const contactWithUpdatedAvatar =
avatar && avatar.avatar
? Object.assign({}, contact, {
avatar: Object.assign({}, avatar, {
avatar: await upgradeAttachment(avatar.avatar, context),
}),
})
: omit(contact, ['avatar']);
// eliminates empty numbers, emails, and addresses; adds type if not provided
const contactWithCleanedElements = cleanContact(contactWithUpdatedAvatar);
// We'll log if the contact is invalid, leave everything as-is
validateContact(contactWithCleanedElements, {
messageId: idForLogging(message),
});
return contactWithCleanedElements;
};
const toVersion0 = async message => exports.initializeSchemaVersion(message);
const toVersion1 = exports._withSchemaVersion(
@ -214,6 +350,13 @@ const toVersion4 = exports._withSchemaVersion(
);
const toVersion5 = exports._withSchemaVersion(5, initializeAttachmentMetadata);
const toVersion6 = exports._withSchemaVersion(
6,
exports._mapContact(
exports._cleanAndWriteContactAvatar(Attachment.migrateDataToFileSystem)
)
);
// UpgradeStep
exports.upgradeSchema = async (rawMessage, { writeNewAttachmentData } = {}) => {
if (!isFunction(writeNewAttachmentData)) {
@ -228,6 +371,7 @@ exports.upgradeSchema = async (rawMessage, { writeNewAttachmentData } = {}) => {
toVersion3,
toVersion4,
toVersion5,
toVersion6,
];
for (let i = 0, max = versions.length; i < max; i += 1) {
@ -269,10 +413,11 @@ exports.createAttachmentDataWriter = writeExistingAttachmentData => {
const message = exports.initializeSchemaVersion(rawMessage);
const { attachments, quote } = message;
const { attachments, quote, contact } = message;
const hasFilesToWrite =
(quote && quote.attachments && quote.attachments.length > 0) ||
(attachments && attachments.length > 0);
(attachments && attachments.length > 0) ||
(contact && contact.length > 0);
if (!hasFilesToWrite) {
return message;
@ -318,10 +463,26 @@ exports.createAttachmentDataWriter = writeExistingAttachmentData => {
return omit(thumbnail, ['data']);
});
const writeContactAvatar = async messageContact => {
const { avatar } = messageContact;
if (avatar && !avatar.avatar) {
return omit(messageContact, ['avatar']);
}
await writeExistingAttachmentData(avatar.avatar);
return Object.assign({}, messageContact, {
avatar: Object.assign({}, avatar, {
avatar: omit(avatar.avatar, ['data']),
}),
});
};
const messageWithoutAttachmentData = Object.assign(
{},
await writeThumbnails(message),
{
contact: await Promise.all((contact || []).map(writeContactAvatar)),
attachments: await Promise.all(
(attachments || []).map(async attachment => {
await writeExistingAttachmentData(attachment);

View File

@ -1065,6 +1065,14 @@ MessageReceiver.prototype.extend({
promises.push(this.handleAttachment(attachment));
}
if (
decrypted.contact &&
decrypted.contact.avatar &&
decrypted.contact.avatar.avatar
) {
promises.push(this.handleAttachment(decrypted.contact.avatar.avatar));
}
if (decrypted.quote && decrypted.quote.id) {
decrypted.quote.id = decrypted.quote.id.toNumber();
}

View File

@ -84,6 +84,73 @@ message DataMessage {
repeated QuotedAttachment attachments = 4;
}
message Contact {
message Name {
optional string givenName = 1;
optional string familyName = 2;
optional string prefix = 3;
optional string suffix = 4;
optional string middleName = 5;
optional string displayName = 6;
}
message Phone {
enum Type {
HOME = 1;
MOBILE = 2;
WORK = 3;
CUSTOM = 4;
}
optional string value = 1;
optional Type type = 2;
optional string label = 3;
}
message Email {
enum Type {
HOME = 1;
MOBILE = 2;
WORK = 3;
CUSTOM = 4;
}
optional string value = 1;
optional Type type = 2;
optional string label = 3;
}
message PostalAddress {
enum Type {
HOME = 1;
WORK = 2;
CUSTOM = 3;
}
optional Type type = 1;
optional string label = 2;
optional string street = 3;
optional string pobox = 4;
optional string neighborhood = 5;
optional string city = 6;
optional string region = 7;
optional string postcode = 8;
optional string country = 9;
}
message Avatar {
optional AttachmentPointer avatar = 1;
optional bool isProfile = 2;
}
optional Name name = 1;
repeated Phone number = 3;
repeated Email email = 4;
repeated PostalAddress address = 5;
optional Avatar avatar = 6;
optional string organization = 7;
}
optional string body = 1;
repeated AttachmentPointer attachments = 2;
optional GroupContext group = 3;
@ -92,6 +159,7 @@ message DataMessage {
optional bytes profileKey = 6;
optional uint64 timestamp = 7;
optional Quote quote = 8;
repeated Contact contact = 9;
}
message NullMessage {

View File

@ -283,6 +283,7 @@ describe('Backup', () => {
const OUR_NUMBER = '+12025550000';
const CONTACT_ONE_NUMBER = '+12025550001';
const CONTACT_TWO_NUMBER = '+12025550002';
async function wrappedLoadAttachment(attachment) {
return _.omit(await loadAttachmentData(attachment), ['path']);
@ -356,18 +357,31 @@ describe('Backup', () => {
return wrappedLoadAttachment(thumbnail);
});
const promises = (message.attachments || []).map(attachment =>
wrappedLoadAttachment(attachment)
);
return Object.assign({}, await loadThumbnails(message), {
attachments: await Promise.all(promises),
contact: await Promise.all(
(message.contact || []).map(async contact => {
return contact && contact.avatar && contact.avatar.avatar
? Object.assign({}, contact, {
avatar: Object.assign({}, contact.avatar, {
avatar: await wrappedLoadAttachment(
contact.avatar.avatar
),
}),
})
: contact;
})
),
attachments: await Promise.all(
(message.attachments || []).map(attachment =>
wrappedLoadAttachment(attachment)
)
),
});
}
let backupDir;
try {
const ATTACHMENT_COUNT = 2;
const ATTACHMENT_COUNT = 3;
const MESSAGE_COUNT = 1;
const CONVERSATION_COUNT = 1;
@ -473,6 +487,59 @@ describe('Backup', () => {
},
],
},
contact: [
{
name: {
displayName: 'Someone Somewhere',
},
number: [
{
value: CONTACT_TWO_NUMBER,
type: 1,
},
],
avatar: {
isProfile: false,
avatar: {
contentType: 'image/png',
data: new Uint8Array([
3,
2,
3,
4,
5,
6,
7,
8,
1,
2,
3,
4,
5,
6,
7,
8,
1,
2,
3,
4,
5,
6,
7,
8,
1,
2,
3,
4,
5,
6,
7,
8,
]).buffer,
},
},
},
],
};
console.log('Backup test: Clear all data');
@ -494,7 +561,7 @@ describe('Backup', () => {
profileAvatar: {
contentType: 'image/jpeg',
data: new Uint8Array([
3,
4,
2,
3,
4,
@ -530,7 +597,7 @@ describe('Backup', () => {
size: 64,
},
profileKey: new Uint8Array([
4,
5,
2,
3,
4,

View File

@ -63,6 +63,7 @@ describe('Message', () => {
path: 'ab/abcdefghi',
},
],
contact: [],
};
const writeExistingAttachmentData = attachment => {
@ -108,6 +109,56 @@ describe('Message', () => {
},
],
},
contact: [],
};
const writeExistingAttachmentData = attachment => {
assert.equal(attachment.path, 'ab/abcdefghi');
assert.deepEqual(
attachment.data,
stringToArrayBuffer('Its easy if you try')
);
};
const actual = await Message.createAttachmentDataWriter(
writeExistingAttachmentData
)(input);
assert.deepEqual(actual, expected);
});
it('should process contact avatars', async () => {
const input = {
body: 'Imagine there is no heaven…',
schemaVersion: 4,
attachments: [],
contact: [
{
name: 'john',
avatar: {
isProfile: false,
avatar: {
path: 'ab/abcdefghi',
data: stringToArrayBuffer('Its easy if you try'),
},
},
},
],
};
const expected = {
body: 'Imagine there is no heaven…',
schemaVersion: 4,
attachments: [],
contact: [
{
name: 'john',
avatar: {
isProfile: false,
avatar: {
path: 'ab/abcdefghi',
},
},
},
],
};
const writeExistingAttachmentData = attachment => {
@ -212,6 +263,7 @@ describe('Message', () => {
hasVisualMediaAttachments: undefined,
hasFileAttachments: 1,
schemaVersion: Message.CURRENT_SCHEMA_VERSION,
contact: [],
};
const expectedAttachmentData = stringToArrayBuffer(
@ -458,7 +510,7 @@ describe('Message', () => {
assert.deepEqual(result, message);
});
it('eliminates thumbnails with no data fielkd', async () => {
it('eliminates thumbnails with no data field', async () => {
const upgradeAttachment = sinon
.stub()
.throws(new Error("Shouldn't be called"));
@ -531,4 +583,398 @@ describe('Message', () => {
assert.deepEqual(result, expected);
});
});
describe('_mapContact', () => {
it('handles message with no contact field', async () => {
const upgradeContact = sinon
.stub()
.throws(new Error("Shouldn't be called"));
const upgradeVersion = Message._mapContact(upgradeContact);
const message = {
body: 'hey there!',
};
const expected = {
body: 'hey there!',
contact: [],
};
const result = await upgradeVersion(message);
assert.deepEqual(result, expected);
});
it('handles one contact', async () => {
const upgradeContact = contact => Promise.resolve(contact);
const upgradeVersion = Message._mapContact(upgradeContact);
const message = {
body: 'hey there!',
contact: [
{
name: {
displayName: 'Someone somewhere',
},
},
],
};
const expected = {
body: 'hey there!',
contact: [
{
name: {
displayName: 'Someone somewhere',
},
},
],
};
const result = await upgradeVersion(message);
assert.deepEqual(result, expected);
});
});
describe('_cleanAndWriteContactAvatar', () => {
const NUMBER = '+12025550099';
it('handles message with no avatar in contact', async () => {
const upgradeAttachment = sinon
.stub()
.throws(new Error("Shouldn't be called"));
const upgradeVersion = Message._cleanAndWriteContactAvatar(
upgradeAttachment
);
const message = {
body: 'hey there!',
contact: [
{
name: {
displayName: 'Someone Somewhere',
},
number: [
{
type: 1,
value: NUMBER,
},
],
},
],
};
const result = await upgradeVersion(message.contact[0], { message });
assert.deepEqual(result, message.contact[0]);
});
it('removes contact avatar if it has no sub-avatar', async () => {
const upgradeAttachment = sinon
.stub()
.throws(new Error("Shouldn't be called"));
const upgradeVersion = Message._cleanAndWriteContactAvatar(
upgradeAttachment
);
const message = {
body: 'hey there!',
contact: [
{
name: {
displayName: 'Someone Somewhere',
},
number: [
{
type: 1,
value: NUMBER,
},
],
avatar: {
isProfile: true,
},
},
],
};
const expected = {
name: {
displayName: 'Someone Somewhere',
},
number: [
{
type: 1,
value: NUMBER,
},
],
};
const result = await upgradeVersion(message.contact[0], { message });
assert.deepEqual(result, expected);
});
it('writes avatar to disk', async () => {
const upgradeAttachment = async () => {
return {
path: 'abc/abcdefg',
};
};
const upgradeVersion = Message._cleanAndWriteContactAvatar(
upgradeAttachment
);
const message = {
body: 'hey there!',
contact: [
{
name: {
displayName: 'Someone Somewhere',
},
number: [
{
type: 1,
value: NUMBER,
},
],
email: [
{
type: 2,
value: 'someone@somewhere.com',
},
],
address: [
{
type: 1,
street: '5 Somewhere Ave.',
},
],
avatar: {
otherKey: 'otherValue',
avatar: {
contentType: 'image/png',
data: stringToArrayBuffer('Its easy if you try'),
},
},
},
],
};
const expected = {
name: {
displayName: 'Someone Somewhere',
},
number: [
{
type: 1,
value: NUMBER,
},
],
email: [
{
type: 2,
value: 'someone@somewhere.com',
},
],
address: [
{
type: 1,
street: '5 Somewhere Ave.',
},
],
avatar: {
otherKey: 'otherValue',
isProfile: false,
avatar: {
path: 'abc/abcdefg',
},
},
};
const result = await upgradeVersion(message.contact[0], { message });
assert.deepEqual(result, expected);
});
it('removes number element if it ends up with no value', async () => {
const upgradeAttachment = sinon
.stub()
.throws(new Error("Shouldn't be called"));
const upgradeVersion = Message._cleanAndWriteContactAvatar(
upgradeAttachment
);
const message = {
body: 'hey there!',
contact: [
{
name: {
displayName: 'Someone Somewhere',
},
number: [
{
type: 1,
},
],
email: [
{
value: 'someone@somewhere.com',
},
],
},
],
};
const expected = {
name: {
displayName: 'Someone Somewhere',
},
email: [
{
type: 1,
value: 'someone@somewhere.com',
},
],
};
const result = await upgradeVersion(message.contact[0], { message });
assert.deepEqual(result, expected);
});
it('drops address if it has no real values', async () => {
const upgradeAttachment = sinon
.stub()
.throws(new Error("Shouldn't be called"));
const upgradeVersion = Message._cleanAndWriteContactAvatar(
upgradeAttachment
);
const message = {
body: 'hey there!',
contact: [
{
name: {
displayName: 'Someone Somewhere',
},
number: [
{
value: NUMBER,
},
],
address: [
{
type: 1,
},
],
},
],
};
const expected = {
name: {
displayName: 'Someone Somewhere',
},
number: [
{
value: NUMBER,
type: 1,
},
],
};
const result = await upgradeVersion(message.contact[0], { message });
assert.deepEqual(result, expected);
});
it('logs if contact has no name.displayName or organization', async () => {
const upgradeAttachment = sinon
.stub()
.throws(new Error("Shouldn't be called"));
const upgradeVersion = Message._cleanAndWriteContactAvatar(
upgradeAttachment
);
const message = {
body: 'hey there!',
source: NUMBER,
sourceDevice: '1',
sent_at: 1232132,
contact: [
{
name: {
name: 'Someone',
},
number: [
{
type: 1,
value: NUMBER,
},
],
},
],
};
const expected = {
name: {
name: 'Someone',
},
number: [
{
type: 1,
value: NUMBER,
},
],
};
const result = await upgradeVersion(message.contact[0], { message });
assert.deepEqual(result, expected);
});
it('removes invalid elements then logs if no values remain in contact', async () => {
const upgradeAttachment = sinon
.stub()
.throws(new Error("Shouldn't be called"));
const upgradeVersion = Message._cleanAndWriteContactAvatar(
upgradeAttachment
);
const message = {
body: 'hey there!',
source: NUMBER,
sourceDevice: '1',
sent_at: 1232132,
contact: [
{
name: {
displayName: 'Someone Somewhere',
},
number: [
{
type: 1,
},
],
email: [
{
type: 1,
},
],
},
],
};
const expected = {
name: {
displayName: 'Someone Somewhere',
},
};
const result = await upgradeVersion(message.contact[0], { message });
assert.deepEqual(result, expected);
});
it('handles a contact with just organization', async () => {
const upgradeAttachment = sinon
.stub()
.throws(new Error("Shouldn't be called"));
const upgradeVersion = Message._cleanAndWriteContactAvatar(
upgradeAttachment
);
const message = {
contact: [
{
organization: 'Somewhere Consulting',
number: [
{
type: 1,
value: NUMBER,
},
],
},
],
};
const result = await upgradeVersion(message.contact[0], { message });
assert.deepEqual(result, message.contact[0]);
});
});
});