write attachment path with absolute attachment to disk for opengroupv2

This commit is contained in:
Audric Ackermann 2021-04-30 14:50:01 +10:00
parent 9bf3cb1880
commit bdcdca206b
No known key found for this signature in database
GPG key ID: 999F434D76324AD4
6 changed files with 250 additions and 37 deletions

View file

@ -81,7 +81,6 @@ export const Avatar = (props: Props) => {
const { avatarPath, base64Data, size, memberAvatars, name } = props;
const [imageBroken, setImageBroken] = useState(false);
// contentType is not important
const { urlToLoad } = useEncryptedFileFetch(avatarPath || '', '');
const handleImageError = () => {
window.log.warn('Avatar: Image failed to load; failing over to placeholder', urlToLoad);

View file

@ -73,6 +73,8 @@ export interface ConversationAttributes {
type: string;
avatarPointer?: any;
avatar?: any;
/* Avatar hash is currently used for opengroupv2. it's sha256 hash of the base64 avatar data. */
avatarHash?: string;
server?: any;
channelId?: any;
nickname?: string;
@ -108,6 +110,7 @@ export interface ConversationAttributesOptionals {
type: string;
avatarPointer?: any;
avatar?: any;
avatarHash?: string;
server?: any;
channelId?: any;
nickname?: string;

View file

@ -1,10 +1,14 @@
import { getV2OpenGroupRoomByRoomId, saveV2OpenGroupRoom } from '../../data/opengroups';
import { OpenGroupV2CompactPollRequest, parseMessages } from './ApiUtil';
import {
getV2OpenGroupRoomByRoomId,
OpenGroupV2Room,
saveV2OpenGroupRoom,
} from '../../data/opengroups';
import { OpenGroupV2CompactPollRequest, OpenGroupV2Info, parseMessages } from './ApiUtil';
import { parseStatusCodeFromOnionRequest } from './OpenGroupAPIV2Parser';
import _ from 'lodash';
import { sendViaOnion } from '../../session/onions/onionSend';
import { OpenGroupMessageV2 } from './OpenGroupMessageV2';
import { getAuthToken } from './OpenGroupAPIV2';
import { downloadPreviewOpenGroupV2, getAuthToken } from './OpenGroupAPIV2';
const COMPACT_POLL_ENDPOINT = 'compact_poll';
@ -24,16 +28,63 @@ export const compactFetchEverything = async (
return result ? result : null;
};
export const getAllBase64AvatarForRooms = async (
serverUrl: string,
rooms: Set<string>,
abortSignal: AbortSignal
): Promise<Array<ParsedBase64Avatar> | null> => {
// fetch all we need
const allValidRoomInfos = await getAllValidRoomInfos(serverUrl, rooms);
if (!allValidRoomInfos?.length) {
window.log.info('getAllBase64AvatarForRooms: no valid roominfos got.');
return null;
}
if (abortSignal.aborted) {
window.log.info('preview download aborted, returning null');
return null;
}
// Currently this call will not abort if AbortSignal is aborted,
// but the call will return null.
const validPreviewBase64 = _.compact(
await Promise.all(
allValidRoomInfos.map(async room => {
try {
const base64 = await downloadPreviewOpenGroupV2(room);
if (base64) {
return {
roomId: room.roomId,
base64,
};
}
} catch (e) {
window.log.warn('getPreview failed for room', room);
}
return null;
})
)
);
if (abortSignal.aborted) {
window.log.info('preview download aborted, returning null');
return null;
}
return validPreviewBase64 ? validPreviewBase64 : null;
};
/**
* This return body to be used to do the compactPoll
* This function fetches the valid roomInfos from the database.
* It also makes sure that the pubkey for all those rooms are the same, or returns null.
*/
const getCompactPollRequest = async (
const getAllValidRoomInfos = async (
serverUrl: string,
rooms: Set<string>
): Promise<null | OpenGroupV2CompactPollRequest> => {
): Promise<Array<OpenGroupV2Room> | null> => {
const allServerPubKeys: Array<string> = [];
const roomsRequestInfos = _.compact(
// fetch all the roomInfos for the specified rooms.
// those invalid (like, not found in db) are excluded (with lodash compact)
const validRoomInfos = _.compact(
await Promise.all(
[...rooms].map(async roomId => {
try {
@ -45,22 +96,9 @@ const getCompactPollRequest = async (
window.log.warn('Could not find this room getMessages');
return null;
}
allServerPubKeys.push(fetchedInfo.serverPublicKey);
const {
lastMessageFetchedServerID,
lastMessageDeletedServerID,
token,
serverPublicKey,
} = fetchedInfo;
allServerPubKeys.push(serverPublicKey);
const roomRequestContent: Record<string, any> = {
room_id: roomId,
auth_token: token || '',
};
roomRequestContent.from_deletion_server_id = lastMessageDeletedServerID;
roomRequestContent.from_message_server_id = lastMessageFetchedServerID;
return roomRequestContent;
return fetchedInfo;
} catch (e) {
window.log.warn('failed to fetch roominfos for room', roomId);
return null;
@ -68,7 +106,7 @@ const getCompactPollRequest = async (
})
)
);
if (!roomsRequestInfos?.length) {
if (!validRoomInfos?.length) {
return null;
}
// double check that all those server pubkeys are the same
@ -84,13 +122,59 @@ const getCompactPollRequest = async (
window.log.warn('No pubkeys found:', allServerPubKeys);
return null;
}
return validRoomInfos;
};
/**
* This return body to be used to do the compactPoll
*/
const getCompactPollRequest = async (
serverUrl: string,
rooms: Set<string>
): Promise<null | OpenGroupV2CompactPollRequest> => {
const allValidRoomInfos = await getAllValidRoomInfos(serverUrl, rooms);
if (!allValidRoomInfos?.length) {
window.log.info('compactPoll: no valid roominfos got.');
return null;
}
const roomsRequestInfos = _.compact(
allValidRoomInfos.map(validRoomInfos => {
try {
const {
lastMessageFetchedServerID,
lastMessageDeletedServerID,
token,
roomId,
} = validRoomInfos;
const roomRequestContent: Record<string, any> = {
room_id: roomId,
auth_token: token || '',
};
roomRequestContent.from_deletion_server_id = lastMessageDeletedServerID;
roomRequestContent.from_message_server_id = lastMessageFetchedServerID;
return roomRequestContent;
} catch (e) {
window.log.warn('failed to fetch roominfos for room', validRoomInfos.roomId);
return null;
}
})
);
if (!roomsRequestInfos?.length) {
return null;
}
const body = JSON.stringify({
requests: roomsRequestInfos,
});
// getAllValidRoomInfos return null if the room have not all the same serverPublicKey.
// so being here, we know this is the case
return {
body,
server: serverUrl,
serverPubKey: firstPubkey,
serverPubKey: allValidRoomInfos[0].serverPublicKey,
endpoint: COMPACT_POLL_ENDPOINT,
};
};
@ -158,12 +242,20 @@ async function sendOpenGroupV2RequestCompactPoll(
export type ParsedDeletions = Array<{ id: number; deleted_message_id: number }>;
export type ParsedRoomCompactPollResults = {
type StatusCodeType = {
statusCode: number;
};
export type ParsedRoomCompactPollResults = StatusCodeType & {
roomId: string;
deletions: ParsedDeletions;
messages: Array<OpenGroupMessageV2>;
moderators: Array<string>;
statusCode: number;
};
export type ParsedBase64Avatar = {
roomId: string;
base64: string;
};
const parseCompactPollResult = async (

View file

@ -4,6 +4,8 @@ import { getOpenGroupV2ConversationId } from '../utils/OpenGroupUtils';
import { OpenGroupRequestCommonType } from './ApiUtil';
import {
compactFetchEverything,
getAllBase64AvatarForRooms,
ParsedBase64Avatar,
ParsedDeletions,
ParsedRoomCompactPollResults,
} from './OpenGroupAPIV2CompactPoll';
@ -13,8 +15,13 @@ import { getMessageIdsFromServerIds, removeMessage } from '../../data/data';
import { getV2OpenGroupRoom, saveV2OpenGroupRoom } from '../../data/opengroups';
import { OpenGroupMessageV2 } from './OpenGroupMessageV2';
import { handleOpenGroupV2Message } from '../../receiver/receiver';
import { DAYS, SECONDS } from '../../session/utils/Number';
import autoBind from 'auto-bind';
import { sha256 } from '../../session/crypto';
import { fromBase64ToArrayBuffer } from '../../session/utils/String';
const pollForEverythingInterval = 8 * 1000;
const pollForEverythingInterval = SECONDS * 6;
const pollForRoomAvatarInterval = DAYS * 1;
/**
* An OpenGroupServerPollerV2 polls for everything for a particular server. We should
@ -27,6 +34,7 @@ export class OpenGroupServerPoller {
private readonly serverUrl: string;
private readonly roomIdsToPoll: Set<string> = new Set();
private pollForEverythingTimer?: NodeJS.Timeout;
private pollForRoomAvatarTimer?: NodeJS.Timeout;
private readonly abortController: AbortController;
/**
@ -36,9 +44,11 @@ export class OpenGroupServerPoller {
* This is to ensure that we don't trigger too many request at the same time
*/
private isPolling = false;
private isPreviewPolling = false;
private wasStopped = false;
constructor(roomInfos: Array<OpenGroupRequestCommonType>) {
autoBind(this);
if (!roomInfos?.length) {
throw new Error('Empty roomInfos list');
}
@ -56,8 +66,14 @@ export class OpenGroupServerPoller {
});
this.abortController = new AbortController();
this.compactPoll = this.compactPoll.bind(this);
this.pollForEverythingTimer = global.setInterval(this.compactPoll, pollForEverythingInterval);
this.pollForRoomAvatarTimer = global.setInterval(
this.previewPerRoomPoll,
pollForRoomAvatarInterval
);
// first refresh of avatar rooms is in a day, force it now just in case
global.setTimeout(this.previewPerRoomPoll, SECONDS * 30);
}
/**
@ -94,7 +110,6 @@ export class OpenGroupServerPoller {
public getPolledRoomsCount() {
return this.roomIdsToPoll.size;
}
/**
* Stop polling.
* Requests currently being made will we canceled.
@ -102,10 +117,17 @@ export class OpenGroupServerPoller {
* This has to be used only for quiting the app.
*/
public stop() {
if (this.pollForRoomAvatarTimer) {
global.clearInterval(this.pollForRoomAvatarTimer);
}
if (this.pollForEverythingTimer) {
// cancel next ticks for each timer
global.clearInterval(this.pollForEverythingTimer);
// abort current requests
this.abortController?.abort();
this.pollForEverythingTimer = undefined;
this.pollForRoomAvatarTimer = undefined;
this.wasStopped = true;
}
}
@ -125,6 +147,60 @@ export class OpenGroupServerPoller {
return true;
}
private shouldPollPreview() {
if (this.wasStopped) {
window.log.error('Serverpoller was stopped. PollPreview should not happen');
return false;
}
if (!this.roomIdsToPoll.size) {
return false;
}
// return early if a poll is already in progress
if (this.isPreviewPolling) {
return false;
}
return true;
}
private async previewPerRoomPoll() {
if (!this.shouldPollPreview()) {
return;
}
// do everything with throwing so we can check only at one place
// what we have to clean
try {
this.isPreviewPolling = true;
// don't try to make the request if we are aborted
if (this.abortController.signal.aborted) {
throw new Error('Poller aborted');
}
let previewGotResults = await getAllBase64AvatarForRooms(
this.serverUrl,
this.roomIdsToPoll,
this.abortController.signal
);
// check that we are still not aborted
if (this.abortController.signal.aborted) {
throw new Error('Abort controller was canceled. Dropping preview request');
}
if (!previewGotResults) {
throw new Error('getPreview: no results');
}
// we were not aborted, make sure to filter out roomIds we are not polling for anymore
previewGotResults = previewGotResults.filter(result => this.roomIdsToPoll.has(result.roomId));
// ==> At this point all those results need to trigger conversation updates, so update what we have to update
await handleBase64AvatarUpdate(this.serverUrl, previewGotResults);
} catch (e) {
window.log.warn('Got error while preview fetch:', e);
} finally {
this.isPreviewPolling = false;
}
}
private async compactPoll() {
if (!this.shouldPoll()) {
return;
@ -276,3 +352,44 @@ const handleCompactPollResults = async (
})
);
};
const handleBase64AvatarUpdate = async (
serverUrl: string,
avatarResults: Array<ParsedBase64Avatar>
) => {
await Promise.all(
avatarResults.map(async res => {
const convoId = getOpenGroupV2ConversationId(serverUrl, res.roomId);
const convo = ConversationController.getInstance().get(convoId);
if (!convo) {
window.log.warn('Could not find convo for compactPoll', convoId);
return;
}
if (!res.base64) {
window.log.info('getPreview: no base64 data. skipping');
return;
}
const existingHash = convo.get('avatarHash');
const newHash = sha256(res.base64);
if (newHash !== existingHash) {
// write the file to the disk (automatically encrypted),
// ArrayBuffer
const { getAbsoluteAttachmentPath, processNewAttachment } = window.Signal.Migrations;
const upgradedAttachment = await processNewAttachment({
isRaw: true,
data: fromBase64ToArrayBuffer(res.base64),
url: `${serverUrl}/${res.roomId}`,
});
// update the hash on the conversationModel
convo.set({
avatar: await getAbsoluteAttachmentPath(upgradedAttachment.path),
avatarHash: newHash,
});
// trigger the write to db and refresh the UI
await convo.commit();
}
})
);
};

View file

@ -2,6 +2,7 @@ import * as MessageEncrypter from './MessageEncrypter';
import * as DecryptedAttachmentsManager from './DecryptedAttachmentsManager';
export { MessageEncrypter, DecryptedAttachmentsManager };
import crypto from 'crypto';
// libsodium-wrappers requires the `require` call to work
// tslint:disable-next-line: no-require-imports
@ -14,6 +15,13 @@ export async function getSodium(): Promise<typeof libsodiumwrappers> {
return libsodiumwrappers;
}
export const sha256 = (s: string) => {
return crypto
.createHash('sha256')
.update(s)
.digest('base64');
};
export const concatUInt8Array = (...args: Array<Uint8Array>): Uint8Array => {
const totalLength = args.reduce((acc, current) => acc + current.length, 0);

View file

@ -18,6 +18,7 @@ export { sendOnionRequestLsrpcDest };
import { getRandomSnodeAddress, markNodeUnreachable, Snode, updateSnodesFor } from './snodePool';
import { Constants } from '..';
import { sleepFor } from '../utils/Promise';
import { sha256 } from '../crypto';
/**
* Currently unused. If we need it again, be sure to update it to onion routing rather
@ -68,13 +69,6 @@ export async function getVersion(node: Snode, retries: number = 0): Promise<stri
}
}
const sha256 = (s: string) => {
return crypto
.createHash('sha256')
.update(s)
.digest('base64');
};
const getSslAgentForSeedNode = (seedNodeHost: string, isSsl = false) => {
let filePrefix = '';
let pubkey256 = '';