oxen-io/session-desktop
1
0
Fork 0
mirror of https://github.com/oxen-io/session-desktop.git synced 2023-12-14 02:12:57 +01:00
Code Issues Projects Releases Wiki Activity

Move onion requests to typescript

Browse source
This commit is contained in:
Maxim Shishmarev 2020-07-07 09:57:52 +10:00
parent d429390e62
commit 12f73e23f2
8 changed files with 329 additions and 328 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
js/background.js
View file

@ -292,7 +292,7 @@
window.lokiFeatureFlags.useFileOnionRequests
) {
// Initialize paths for onion requests
window.lokiSnodeAPI.buildNewOnionPaths();
window.OnionAPI.buildNewOnionPaths();
}
const currentPoWDifficulty = storage.get('PoWDifficulty', null);

2
js/modules/data.d.ts vendored
View file

@ -159,7 +159,7 @@ export function getPairingAuthorisationsFor(
export function removePairingAuthorisationsFor(pubKey: string): Promise<void>;
// Guard Nodes
export function getGuardNodes(): Promise<GuardNode>;
export function getGuardNodes(): Promise<Array<GuardNode>>;
export function updateGuardNodes(nodes: Array<string>): Promise<void>;
// Storage Items

6
js/modules/loki_app_dot_net_api.js
View file

@ -1,6 +1,6 @@
/* global log, textsecure, libloki, Signal, Whisper, ConversationController,
clearTimeout, MessageController, libsignal, StringView, window, _,
dcodeIO, Buffer, lokiSnodeAPI, TextDecoder, process */
dcodeIO, Buffer, TextDecoder, process */
const nodeFetch = require('node-fetch');
const { URL, URLSearchParams } = require('url');
const FormData = require('form-data');
@ -59,7 +59,7 @@ const sendViaOnion = async (srvPubKey, url, fetchOptions, options = {}) => {
// eslint-disable-next-line no-param-reassign
options.retry = 0;
// eslint-disable-next-line no-param-reassign
options.requestNumber = window.lokiSnodeAPI.assignOnionRequestNumber();
options.requestNumber = window.OnionAPI.assignOnionRequestNumber();
}
const payloadObj = {
@ -92,7 +92,7 @@ const sendViaOnion = async (srvPubKey, url, fetchOptions, options = {}) => {
let pathNodes = [];
try {
pathNodes = await lokiSnodeAPI.getOnionPath();
pathNodes = await window.OnionAPI.getOnionPath();
} catch (e) {
log.error(
`loki_app_dot_net:::sendViaOnion #${options.requestNumber} - getOnionPath Error ${e.code} ${e.message}`

316
js/modules/loki_snode_api.js
View file

@ -1,317 +1,11 @@
/* eslint-disable class-methods-use-this */
/* global window, textsecure, log, process, Buffer, StringView, dcodeIO */
// not sure I like this name but it's been than util
const primitives = require('./loki_primitives');
const is = require('@sindresorhus/is');
const nodeFetch = require('node-fetch');
const MIN_GUARD_COUNT = 2;
/* global window, Buffer, StringView, dcodeIO */
class LokiSnodeAPI {
constructor({ serverUrl, localUrl }) {
if (!is.string(serverUrl)) {
throw new Error('LokiSnodeAPI.initialize: Invalid server url');
}
this.serverUrl = serverUrl; // random.snode
this.localUrl = localUrl; // localhost.loki
this.swarmsPendingReplenish = {};
this.stopGetAllVersionPromiseControl = false;
this.onionPaths = [];
this.guardNodes = [];
this.onionRequestCounter = 0; // Request index for debugging
}
assignOnionRequestNumber() {
this.onionRequestCounter += 1;
return this.onionRequestCounter;
}
async testGuardNode(snode) {
log.info('Testing a candidate guard node ', snode);
// Send a post request and make sure it is OK
const endpoint = '/storage_rpc/v1';
const url = `https://${snode.ip}:${snode.port}${endpoint}`;
const ourPK = textsecure.storage.user.getNumber();
const pubKey = window.getStoragePubKey(ourPK); // truncate if testnet
const method = 'get_snodes_for_pubkey';
const params = { pubKey };
const body = {
jsonrpc: '2.0',
id: '0',
method,
params,
};
const fetchOptions = {
method: 'POST',
body: JSON.stringify(body),
headers: { 'Content-Type': 'application/json' },
timeout: 10000, // 10s, we want a smaller timeout for testing
};
process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
let response;
try {
// Log this line for testing
// curl -k -X POST -H 'Content-Type: application/json' -d '"+fetchOptions.body.replace(/"/g, "\\'")+"'", url
response = await nodeFetch(url, fetchOptions);
} catch (e) {
if (e.type === 'request-timeout') {
log.warn(`test timeout for node,`, snode);
}
return false;
} finally {
process.env.NODE_TLS_REJECT_UNAUTHORIZED = '1';
}
if (!response.ok) {
log.info(`Node failed the guard test:`, snode);
}
return response.ok;
}
async selectGuardNodes() {
const _ = window.Lodash;
// FIXME: handle rejections
let nodePool = await window.SnodePool.getRandomSnodePool();
if (nodePool.length === 0) {
log.error(`Could not select guard nodes: node pool is empty`);
return [];
}
let shuffled = _.shuffle(nodePool);
let guardNodes = [];
const DESIRED_GUARD_COUNT = 3;
if (shuffled.length < DESIRED_GUARD_COUNT) {
log.error(
`Could not select guard nodes: node pool is not big enough, pool size ${shuffled.length}, need ${DESIRED_GUARD_COUNT}, attempting to refresh randomPool`
);
await window.SnodePool.refreshRandomPool();
nodePool = await window.SnodePool.getRandomSnodePool();
shuffled = _.shuffle(nodePool);
if (shuffled.length < DESIRED_GUARD_COUNT) {
log.error(
`Could not select guard nodes: node pool is not big enough, pool size ${shuffled.length}, need ${DESIRED_GUARD_COUNT}, failing...`
);
return [];
}
}
// The use of await inside while is intentional:
// we only want to repeat if the await fails
// eslint-disable-next-line-no-await-in-loop
while (guardNodes.length < 3) {
if (shuffled.length < DESIRED_GUARD_COUNT) {
log.error(`Not enought nodes in the pool`);
break;
}
const candidateNodes = shuffled.splice(0, DESIRED_GUARD_COUNT);
// Test all three nodes at once
// eslint-disable-next-line no-await-in-loop
const idxOk = await Promise.all(
candidateNodes.map(n => this.testGuardNode(n))
);
const goodNodes = _.zip(idxOk, candidateNodes)
.filter(x => x[0])
.map(x => x[1]);
guardNodes = _.concat(guardNodes, goodNodes);
}
if (guardNodes.length < DESIRED_GUARD_COUNT) {
log.error(
`COULD NOT get enough guard nodes, only have: ${guardNodes.length}`
);
}
log.info('new guard nodes: ', guardNodes);
const edKeys = guardNodes.map(n => n.pubkey_ed25519);
await window.libloki.storage.updateGuardNodes(edKeys);
return guardNodes;
}
async getOnionPath(toExclude = null) {
const _ = window.Lodash;
let goodPaths = this.onionPaths.filter(x => !x.bad);
let attemptNumber = 0;
while (goodPaths.length < MIN_GUARD_COUNT) {
log.error(
`Must have at least 2 good onion paths, actual: ${goodPaths.length}, attempt #${attemptNumber} fetching more...`
);
// eslint-disable-next-line no-await-in-loop
await this.buildNewOnionPaths();
// should we add a delay? buildNewOnionPaths should act as one
// reload goodPaths now
attemptNumber += 1;
goodPaths = this.onionPaths.filter(x => !x.bad);
}
const paths = _.shuffle(goodPaths);
if (!toExclude) {
if (!paths[0]) {
log.error('LokiSnodeAPI::getOnionPath - no path in', paths);
return [];
}
if (!paths[0].path) {
log.error('LokiSnodeAPI::getOnionPath - no path in', paths[0]);
}
return paths[0].path;
}
// Select a path that doesn't contain `toExclude`
const otherPaths = paths.filter(
path =>
!_.some(path, node => node.pubkey_ed25519 === toExclude.pubkey_ed25519)
);
if (otherPaths.length === 0) {
// This should never happen!
// well it did happen, should we
// await this.buildNewOnionPaths();
// and restart call?
log.error(
`LokiSnodeAPI::getOnionPath - no paths without`,
toExclude.pubkey_ed25519,
'path count',
paths.length,
'goodPath count',
goodPaths.length,
'paths',
paths
);
throw new Error('No onion paths available after filtering');
}
if (!otherPaths[0].path) {
log.error(
'LokiSnodeAPI::getOnionPath - otherPaths no path in',
otherPaths[0]
);
}
return otherPaths[0].path;
}
markPathAsBad(path) {
this.onionPaths.forEach(p => {
if (!p.path) {
log.error('LokiSnodeAPI::markPathAsBad - no path in', p);
}
if (p.path === path) {
// eslint-disable-next-line no-param-reassign
p.bad = true;
}
});
}
async buildNewOnionPathsWorker() {
const _ = window.Lodash;
log.info('LokiSnodeAPI::buildNewOnionPaths - building new onion paths');
const allNodes = await window.SnodePool.getRandomSnodePool();
if (this.guardNodes.length === 0) {
// Not cached, load from DB
const nodes = await window.libloki.storage.getGuardNodes();
if (nodes.length === 0) {
log.warn(
'LokiSnodeAPI::buildNewOnionPaths - no guard nodes in DB. Will be selecting new guards nodes...'
);
} else {
// We only store the nodes' keys, need to find full entries:
const edKeys = nodes.map(x => x.ed25519PubKey);
this.guardNodes = allNodes.filter(
x => edKeys.indexOf(x.pubkey_ed25519) !== -1
);
if (this.guardNodes.length < edKeys.length) {
log.warn(
`LokiSnodeAPI::buildNewOnionPaths - could not find some guard nodes: ${this.guardNodes.length}/${edKeys.length} left`
);
}
}
// If guard nodes is still empty (the old nodes are now invalid), select new ones:
if (this.guardNodes.length < MIN_GUARD_COUNT) {
// TODO: don't throw away potentially good guard nodes
this.guardNodes = await this.selectGuardNodes();
}
}
// TODO: select one guard node and 2 other nodes randomly
let otherNodes = _.difference(allNodes, this.guardNodes);
if (otherNodes.length < 2) {
log.warn(
'LokiSnodeAPI::buildNewOnionPaths - Too few nodes to build an onion path! Refreshing pool and retrying'
);
await window.SnodePool.refreshRandomPool();
await this.buildNewOnionPaths();
return;
}
otherNodes = _.shuffle(otherNodes);
const guards = _.shuffle(this.guardNodes);
// Create path for every guard node:
const nodesNeededPerPaths = window.lokiFeatureFlags.onionRequestHops - 1;
// Each path needs X (nodesNeededPerPaths) nodes in addition to the guard node:
const maxPath = Math.floor(
Math.min(
guards.length,
nodesNeededPerPaths
? otherNodes.length / nodesNeededPerPaths
: otherNodes.length
)
);
// TODO: might want to keep some of the existing paths
this.onionPaths = [];
for (let i = 0; i < maxPath; i += 1) {
const path = [guards[i]];
for (let j = 0; j < nodesNeededPerPaths; j += 1) {
path.push(otherNodes[i * nodesNeededPerPaths + j]);
}
this.onionPaths.push({ path, bad: false });
}
log.info(`Built ${this.onionPaths.length} onion paths`, this.onionPaths);
}
async buildNewOnionPaths() {
// this function may be called concurrently make sure we only have one inflight
return primitives.allowOnlyOneAtATime('buildNewOnionPaths', async () => {
await this.buildNewOnionPathsWorker();
});
}
// ************** NOTE ***************
// This is not used by anything yet,
// but should be. Do not remove!!!
// ***********************************
async getLnsMapping(lnsName, timeout) {
// Returns { pubkey, error }
// pubkey is

8
preload.js
View file

@ -332,14 +332,12 @@ const { initialize: initializeWebAPI } = require('./js/modules/web_api');
window.WebAPI = initializeWebAPI();
window.seedNodeList = JSON.parse(config.seedNodeList);
const LokiSnodeAPI = require('./js/modules/loki_snode_api');
window.SenderKeyAPI = require('./js/modules/loki_sender_key_api');
window.lokiSnodeAPI = new LokiSnodeAPI({
serverUrl: config.serverUrl,
localUrl: config.localUrl,
});
const { OnionAPI } = require('./ts/session/onions');
window.OnionAPI = OnionAPI;
if (process.env.USE_STUBBED_NETWORK) {
const StubMessageAPI = require('./integration_test/stubs/stub_message_api');

310
ts/session/onions/index.ts Normal file
View file

@ -0,0 +1,310 @@
import { allowOnlyOneAtATime } from '../../../js/modules/loki_primitives';
import * as Data from '../../../js/modules/data';
import * as SnodePool from '../snode_api/snodePool';
import _ from 'lodash';
import fetch from 'node-fetch';
type Snode = SnodePool.Snode;
const MIN_GUARD_COUNT = 2;
interface SnodePath {
path: Array<Snode>;
bad: boolean;
}
class OnionPaths {
private onionPaths: Array<SnodePath> = [];
// This array is meant to store nodes will full info,
// so using GuardNode would not be correct (there is
// some naming issue here it seems)
private guardNodes: Array<Snode> = [];
private onionRequestCounter = 0; // Request index for debugging
public async buildNewOnionPaths() {
// this function may be called concurrently make sure we only have one inflight
return allowOnlyOneAtATime('buildNewOnionPaths', async () => {
await this.buildNewOnionPathsWorker();
});
}
public async getOnionPath(toExclude?: {
pubkey_ed25519: string;
}): Promise<Array<Snode>> {
const { log } = window;
let goodPaths = this.onionPaths.filter(x => !x.bad);
let attemptNumber = 0;
while (goodPaths.length < MIN_GUARD_COUNT) {
log.error(
`Must have at least 2 good onion paths, actual: ${goodPaths.length}, attempt #${attemptNumber} fetching more...`
);
// eslint-disable-next-line no-await-in-loop
await this.buildNewOnionPaths();
// should we add a delay? buildNewOnionPaths should act as one
// reload goodPaths now
attemptNumber += 1;
goodPaths = this.onionPaths.filter(x => !x.bad);
}
const paths = _.shuffle(goodPaths);
if (!toExclude) {
if (!paths[0]) {
log.error('LokiSnodeAPI::getOnionPath - no path in', paths);
return [];
}
if (!paths[0].path) {
log.error('LokiSnodeAPI::getOnionPath - no path in', paths[0]);
}
return paths[0].path;
}
// Select a path that doesn't contain `toExclude`
const otherPaths = paths.filter(
path =>
!_.some(
path.path,
node => node.pubkey_ed25519 === toExclude.pubkey_ed25519
)
);
if (otherPaths.length === 0) {
// This should never happen!
// well it did happen, should we
// await this.buildNewOnionPaths();
// and restart call?
log.error(
'LokiSnodeAPI::getOnionPath - no paths without',
toExclude.pubkey_ed25519,
'path count',
paths.length,
'goodPath count',
goodPaths.length,
'paths',
paths
);
throw new Error('No onion paths available after filtering');
}
if (!otherPaths[0].path) {
log.error(
'LokiSnodeAPI::getOnionPath - otherPaths no path in',
otherPaths[0]
);
}
return otherPaths[0].path;
}
public markPathAsBad(path: Array<Snode>) {
// TODO: we might want to remove the nodes from the
// node pool (but we don't know which node on the path
// is causing issues)
this.onionPaths.forEach(p => {
if (_.isEqual(p.path, path)) {
// eslint-disable-next-line no-param-reassign
p.bad = true;
}
});
}
public assignOnionRequestNumber() {
this.onionRequestCounter += 1;
return this.onionRequestCounter;
}
private async testGuardNode(snode: Snode) {
const { log } = window;
log.info('Testing a candidate guard node ', snode);
// Send a post request and make sure it is OK
const endpoint = '/storage_rpc/v1';
const url = `https://${snode.ip}:${snode.port}${endpoint}`;
const ourPK = window.textsecure.storage.user.getNumber();
const pubKey = window.getStoragePubKey(ourPK); // truncate if testnet
const method = 'get_snodes_for_pubkey';
const params = { pubKey };
const body = {
jsonrpc: '2.0',
id: '0',
method,
params,
};
const fetchOptions = {
method: 'POST',
body: JSON.stringify(body),
headers: { 'Content-Type': 'application/json' },
timeout: 10000, // 10s, we want a smaller timeout for testing
};
process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
let response;
try {
// Log this line for testing
// curl -k -X POST -H 'Content-Type: application/json' -d '"+fetchOptions.body.replace(/"/g, "\\'")+"'", url
response = await fetch(url, fetchOptions);
} catch (e) {
if (e.type === 'request-timeout') {
log.warn('test timeout for node,', snode);
}
return false;
} finally {
process.env.NODE_TLS_REJECT_UNAUTHORIZED = '1';
}
if (!response.ok) {
log.info('Node failed the guard test:', snode);
}
return response.ok;
}
private async selectGuardNodes(): Promise<Array<Snode>> {
const { log } = window;
const DESIRED_GUARD_COUNT = 3;
// `getRandomSnodePool` is expected to refresh itself on low nodes
const nodePool = await SnodePool.getRandomSnodePool();
if (nodePool.length < DESIRED_GUARD_COUNT) {
log.error(
'Could not select guard nodes. Not enough nodes in the pool: ',
nodePool.length
);
return [];
}
const shuffled = _.shuffle(nodePool);
let guardNodes: Array<Snode> = [];
// The use of await inside while is intentional:
// we only want to repeat if the await fails
// eslint-disable-next-line-no-await-in-loop
while (guardNodes.length < 3) {
if (shuffled.length < DESIRED_GUARD_COUNT) {
log.error('Not enought nodes in the pool');
break;
}
const candidateNodes = shuffled.splice(0, DESIRED_GUARD_COUNT);
// Test all three nodes at once
// eslint-disable-next-line no-await-in-loop
const idxOk = await Promise.all(
candidateNodes.map(n => this.testGuardNode(n))
);
const goodNodes = _.zip(idxOk, candidateNodes)
.filter(x => x[0])
.map(x => x[1]) as Array<Snode>;
guardNodes = _.concat(guardNodes, goodNodes);
}
if (guardNodes.length < DESIRED_GUARD_COUNT) {
log.error(
`COULD NOT get enough guard nodes, only have: ${guardNodes.length}`
);
}
log.info('new guard nodes: ', guardNodes);
const edKeys = guardNodes.map(n => _.pick(n, 'pubkey_ed25519'));
await window.libloki.storage.updateGuardNodes(edKeys);
return guardNodes;
}
private async buildNewOnionPathsWorker() {
const { log } = window;
log.info('LokiSnodeAPI::buildNewOnionPaths - building new onion paths');
const allNodes = await SnodePool.getRandomSnodePool();
if (this.guardNodes.length === 0) {
// Not cached, load from DB
const nodes = await Data.getGuardNodes();
if (nodes.length === 0) {
log.warn(
'LokiSnodeAPI::buildNewOnionPaths - no guard nodes in DB. Will be selecting new guards nodes...'
);
} else {
// We only store the nodes' keys, need to find full entries:
const edKeys = nodes.map(x => x.ed25519PubKey);
this.guardNodes = allNodes.filter(
x => edKeys.indexOf(x.pubkey_ed25519) !== -1
);
if (this.guardNodes.length < edKeys.length) {
log.warn(
`LokiSnodeAPI::buildNewOnionPaths - could not find some guard nodes: ${this.guardNodes.length}/${edKeys.length} left`
);
}
}
// If guard nodes is still empty (the old nodes are now invalid), select new ones:
if (this.guardNodes.length < MIN_GUARD_COUNT) {
// TODO: don't throw away potentially good guard nodes
this.guardNodes = await this.selectGuardNodes();
}
}
// TODO: select one guard node and 2 other nodes randomly
let otherNodes = _.difference(allNodes, this.guardNodes);
if (otherNodes.length < 2) {
log.warn(
'LokiSnodeAPI::buildNewOnionPaths - Too few nodes to build an onion path! Refreshing pool and retrying'
);
await SnodePool.refreshRandomPool();
await this.buildNewOnionPaths();
return;
}
otherNodes = _.shuffle(otherNodes);
const guards = _.shuffle(this.guardNodes);
// Create path for every guard node:
const nodesNeededPerPaths = window.lokiFeatureFlags.onionRequestHops - 1;
// Each path needs X (nodesNeededPerPaths) nodes in addition to the guard node:
const maxPath = Math.floor(
Math.min(
guards.length,
nodesNeededPerPaths
? otherNodes.length / nodesNeededPerPaths
: otherNodes.length
)
);
// TODO: might want to keep some of the existing paths
this.onionPaths = [];
for (let i = 0; i < maxPath; i += 1) {
const path = [guards[i]];
for (let j = 0; j < nodesNeededPerPaths; j += 1) {
path.push(otherNodes[i * nodesNeededPerPaths + j]);
}
this.onionPaths.push({ path, bad: false });
}
log.info(`Built ${this.onionPaths.length} onion paths`, this.onionPaths);
}
}
export const OnionAPI = new OnionPaths();

11
ts/session/snode_api/onions.ts
View file

@ -4,8 +4,7 @@ import https from 'https';
import { Snode } from './snodePool';
import ByteBuffer from 'bytebuffer';
import { StringUtils } from '../utils';
const BAD_PATH = 'bad_path';
import { OnionAPI } from '../onions';
enum RequestError {
BAD_PATH,
@ -401,15 +400,15 @@ export async function lokiOnionFetch(
body: any,
targetNode: Snode
): Promise<SnodeResponse | boolean> {
const { lokiSnodeAPI, log } = window;
const { log } = window;
// Loop until the result is not BAD_PATH
// tslint:disable-next-line no-constant-condition
while (true) {
// Get a path excluding `targetNode`:
// eslint-disable-next-line no-await-in-loop
const path = await lokiSnodeAPI.getOnionPath(targetNode);
const thisIdx = lokiSnodeAPI.assignOnionRequestNumber();
const path = await OnionAPI.getOnionPath(targetNode);
const thisIdx = OnionAPI.assignOnionRequestNumber();
// At this point I only care about BAD_PATH
@ -427,7 +426,7 @@ export async function lokiOnionFetch(
targetNode.port
}`
);
lokiSnodeAPI.markPathAsBad(path);
OnionAPI.markPathAsBad(path);
return false;
} else if (result === RequestError.OTHER) {
// could mean, fail to parse results

2
ts/session/snode_api/serviceNodeAPI.ts
View file

@ -261,7 +261,7 @@ export async function storeOnNode(
targetNode: Snode,
params: SendParams
): Promise<boolean> {
const { log, textsecure, lokiSnodeAPI } = window;
const { log, textsecure } = window;
let successiveFailures = 0;
while (successiveFailures < MAX_ACCEPTABLE_FAILURES) {