Merge remote-tracking branch 'origin/master'

cmake/win32.cmake

@@ -1,21 +1,6 @@
 enable_language(RC)
 set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lshlwapi")
 
-get_filename_component(LIBTUNTAP_IMPL ${TT_ROOT}/tuntap-windows.c ABSOLUTE)
-get_filename_component(EV_SRC "llarp/ev/ev_win32.cpp" ABSOLUTE)
-add_definitions(-DWIN32_LEAN_AND_MEAN -DWIN32 -DWINVER=0x0500 -D_WIN32_WINNT=0x0500)
-set(EXE_LIBS ${STATIC_LIB} ws2_32 iphlpapi)
-
-if(RELEASE_MOTTO)
-  add_definitions(-DLLARP_RELEASE_MOTTO="${RELEASE_MOTTO}")
-  add_definitions(-DRELEASE_MOTTO=${RELEASE_MOTTO})
-endif()
-
-if (NOT STATIC_LINK_RUNTIME AND NOT MSVC)
-  message("must ship compiler runtime libraries with this build: libwinpthread-1.dll, libgcc_s_dw2-1.dll, and libstdc++-6.dll")
-  message("for release builds, turn on STATIC_LINK_RUNTIME in cmake options")
-endif()
-
 if(NOT MSVC_VERSION)
   add_compile_options($<$<COMPILE_LANGUAGE:C>:-Wno-bad-function-cast>)
   add_compile_options($<$<COMPILE_LANGUAGE:C>:-Wno-cast-function-type>)
@@ -25,3 +10,18 @@ if(NOT MSVC_VERSION)
   add_compile_options(-fno-ident)
   set(FS_LIB stdc++fs)
 endif()
+
+get_filename_component(LIBTUNTAP_IMPL ${TT_ROOT}/tuntap-windows.c ABSOLUTE)
+get_filename_component(EV_SRC "llarp/ev/ev_win32.cpp" ABSOLUTE)
+add_definitions(-DWIN32_LEAN_AND_MEAN -DWIN32 -DWINVER=0x0500 -D_WIN32_WINNT=0x0500)
+set(EXE_LIBS ${STATIC_LIB} ${FS_LIB} ws2_32 iphlpapi)
+
+if(RELEASE_MOTTO)
+  add_definitions(-DLLARP_RELEASE_MOTTO="${RELEASE_MOTTO}")
+  add_definitions(-DRELEASE_MOTTO=${RELEASE_MOTTO})
+endif()
+
+if (NOT STATIC_LINK_RUNTIME AND NOT MSVC)
+  message("must ship compiler runtime libraries with this build: libwinpthread-1.dll, libgcc_s_dw2-1.dll, and libstdc++-6.dll")
+  message("for release builds, turn on STATIC_LINK_RUNTIME in cmake options")
+endif()

contrib/apparmor/usr.bin.lokinet

@@ -0,0 +1,24 @@
+# Last Modified: Sat May  4 18:48:24 2019
+#include <tunables/global>
+
+/usr/bin/lokinet {
+  #include <abstractions/base>
+
+  capability net_admin,
+  capability net_bind_service,
+
+  network inet dgram,
+  network inet6 dgram,
+  network netlink raw,
+
+  /dev/net/tun rw,
+  /lib/@{multiarch}/ld-*.so mr,
+  /usr/bin/lokinet mr,
+
+  owner /var/lib/lokinet/ rw,
+  owner /var/lib/lokinet/** rwk,
+
+  owner @{HOME}/.lokinet/ rw,
+  owner @{HOME}/.lokinet/** rwk,
+
+}

contrib/node-monitor/README.md

@@ -0,0 +1,26 @@
+# Building / Running
+
+after building lokinet
+
+```
+cd contrib/node-monitor
+npm install
+sudo node monitor_client.js
+```
+
+and wait 30 minutes for final score
+
+# Scoring
+
+Lokinet gets a point for everything it does correctly.
+
+Currently:
+- succesful ping to snode (1 per second after a snode is discovered)
+- session with snode established
+- got a router from exploration
+- Hanlding DHT S or R message
+- a path is built
+- obtained an exit via
+- granted exit
+- IntroSet publish confirmed
+- utp.session.sendq. has a min zero count (no blocked buffers on routers)

contrib/node-monitor/ini.js

@@ -0,0 +1,62 @@
+function iniToJSON(data) {
+  const lines = data.split(/\n/)
+  var section = 'unknown'
+  var config = {}
+  for(var i in lines) {
+    var line = lines[i].trim()
+    if (line.match(/#/)) {
+      var parts = line.split(/#/)
+      line = parts[0].trim()
+    }
+    // done reducing
+    if (!line) continue
+
+    // check for section
+    if (line[0] == '[' && line[line.length - 1] == ']') {
+      section = line.substring(1, line.length -1)
+      //console.log('found section', section)
+      continue
+    }
+    // key value pair
+    if (line.match(/=/)) {
+      var parts = line.split(/=/)
+      var key = parts.shift().trim()
+      var value = parts.join('=').trim()
+      if (value === 'true')  value = true
+      if (value === 'false') value = false
+      //console.log('key/pair ['+section+']', key, '=', value)
+      if (config[section] === undefined) config[section] = {}
+      config[section][key]=value
+      continue
+    }
+    console.error('config ['+section+'] not section or key/value pair', line)
+  }
+  return config
+}
+
+function jsonToINI(json) {
+  var lastSection = 'unknown'
+  var config = ''
+  for(var section in json) {
+    config += "\n" + '[' + section + ']' + "\n"
+    var keys = json[section]
+    for(var key in keys) {
+      //console.log('key', key, 'value', keys[key])
+      // if keys[key] is an array, then we need to send the same key each time
+      if (keys[key] !== undefined && keys[key].constructor.name == 'Array') {
+        for(var i in keys[key]) {
+          var v = keys[key][i]
+          config += key + '=' + v + "\n"
+        }
+      } else {
+        config += key + '=' + keys[key] + "\n"
+      }
+    }
+  }
+  return config
+}
+
+module.exports = {
+  iniToJSON: iniToJSON,
+  jsonToINI: jsonToINI,
+}

contrib/node-monitor/monitor_client.js

@@ -0,0 +1,322 @@
+const lokinet = require('./lokinet') // currently using the 0.5 api
+const fs      = require('fs')
+const ping    = require ("net-ping")
+// horrible library that shells out to use ping...
+//const ping    = require('ping')
+
+// score successful actions
+// while we monitor snode and loki addresses
+
+var session = ping.createSession ()
+
+// probably should nuke profiles.dat each round to level the playing field
+if (fs.existsSync('profiles.dat')) {
+  console.log('cleaning router profiles')
+  fs.unlinkSync('profiles.dat')
+}
+
+//
+// start config
+//
+
+// FIXME: take in a binary_path as an option...
+var lokinet_config = {
+  binary_path : '../../lokinet',
+// clients will default to i2p.rocks
+//  bootstrap_url   : 'http://206.81.100.174/n-st-1.signed',
+//  rpc_ip          : '127.0.0.1',
+//  rpc_port        : 28082,
+//  auto_config_test_host: 'www.google.com',
+//  auto_config_test_port: 80,
+//  testnet         : true,
+//  verbose         : true,
+}
+
+score_time  = null
+score_total = 0
+
+var lokinet_version = 'unknown'
+
+var known_snodes = []
+var snodes_stats = {}
+function addSnode(snode) {
+  var idx = known_snodes.indexOf(snode)
+  if (idx == -1) {
+    known_snodes.push(snode)
+    function lookupSnode(snode) {
+      lokinet.lookup(snode, function(records) {
+        console.log(snode, 'mapped to', records)
+        // if no response or not found, retry
+        if (records === undefined || records === null) {
+          // failure retry
+          setTimeout(function() {
+            console.log('retry lookup for', snode)
+            lookupSnode(snode)
+          }, 5000)
+          return
+        }
+        if (!records) {
+          console.log('records is false', records)
+          return
+        }
+        if (!records.length) {
+          console.log('record is empty array')
+          return
+        }
+        var ip = records[0]
+        // reset stats
+        snodes_stats[snode] = {
+          onlines: 0,
+          offlines: 0,
+        }
+        // trigger monitor update?
+        var snodeMonitor = setInterval(function() {
+          /*
+          ping.sys.probe(ip, function(isAlive){
+            if (isAlive) {
+              //console.log(snode, ip, 'is online')
+              snodes_stats[snode].onlines++
+              score_total++
+            } else {
+              console.log(snode, ip, 'is offline')
+              snodes_stats[snode].offlines++
+            }
+          })
+          */
+          session.pingHost(ip, function(err, target) {
+            if (err) {
+              console.warn(target, 'ping err', err);
+              console.log(snode, ip, 'is offline')
+              snodes_stats[snode].offlines++
+            } else {
+              //console.log (target + ": Alive")
+              //console.log(snode, ip, 'is online')
+              snodes_stats[snode].onlines++
+              score_total++
+            }
+          })
+        }, 1000)
+      })
+    }
+    lookupSnode(snode)
+  }
+}
+
+setInterval(function() {
+  // can be known but not mapped yet...
+  console.log('snode score card,', known_snodes.length, 'known routers')
+  for(var snode in snodes_stats) {
+    var stats = snodes_stats[snode]
+    var total = stats.onlines + stats.offlines
+    var onlinePer = (stats.onlines / total) * 100
+    console.log(snode, stats, onlinePer + '%')
+  }
+}, 30 * 1000)
+
+lokinet.onMessage = function(data) {
+  console.log(`monitor: ${data}`)
+  var lines = data.split(/\n/)
+  for(var i in lines) {
+    var tline = lines[i].trim()
+    // 	lokinet-0.4.0-59e6a4bc (dev build)
+    if (tline.match('lokinet-0.')) {
+      var parts = tline.split('lokinet-0.')
+      lokinet_version = parts[1]
+      console.log('VERSION', parts[1])
+    }
+    if (tline.match('Using config file:')) {
+      // get bootstrap info
+      var parts = tline.split('Using config file: ')
+      console.log('CONFIGFILE', parts[1])
+    }
+    if (tline.match('Added bootstrap node')) {
+      // get bootstrap info
+      var parts = tline.split('Added bootstrap node ')
+      addSnode(parts[1])
+      console.log('BOOTSTRAP', parts[1])
+    }
+    if (tline.match('Set Up networking for')) {
+      // get interface info
+      var parts = tline.split('Set Up networking for ')
+      // default:9j4uido1ai7ucirbncbqii1395e8ccd6cjomo9cccp3ztx1ukwio.loki
+      console.log('INTERFACE', parts[1])
+    }
+    //
+    if (tline.match('session with ')) {
+      // potential node
+      var parts = tline.split('session with ')
+      if (parts.length) {
+        var one = parts[1].replace(' established', '')
+        var two = one.split('.snode') // snode\u001b[0;0m
+        var snode = two[0] + '.snode'
+        addSnode(snode)
+        console.log('SESSION', snode)
+        score_total++
+      } else {
+        console.log('unknown session line', tline)
+      }
+    }
+    if (tline.match(' routers from exploration')) {
+      var parts = tline.split('\tgot ')
+      if (parts.length > 1) {
+        var routers = parts[1].replace(' routers from exploration', '')
+        console.log('ROUTERS', routers)
+        if (routers && routers != '0') {
+          score_total++
+        }
+      } else {
+        console.log('unknown exploration line', tline)
+      }
+    }
+    // ?
+    if (tline.match('Handle DHT message S relayed=0')) {
+      score_total++
+    }
+    // relay DHT packet
+    if (tline.match('Handle DHT message R relayed=0')) {
+      score_total++
+    }
+    //
+    if (tline.match(' is built, took ')) {
+      // path TX=8826efd28bede66c59782af9894eed08 RX=a994770a8c6d61700b8ca65e4a3adea3 on OBContext:default:3y3ch3m6c8xgmutxwe8fger6n963hn3mkn6tk14j67w1zdxj1n1y.loki-icxqqcpd3sfkjbqifn53h7rmusqa1fyxwqyfrrcgkd37xcikwa7y.loki is built, took 2321 ms
+      var parts = tline.split('path TX=')
+      if (parts.length > 1) {
+        var two     = parts[1].split(' RX=')
+        if (two.length > 1) {
+          var tx      = two[0]
+          var three   = two[1].split(' on OBContext:')
+          if (three.length > 1) {
+            var rx      = three[0]
+            var four    = three[1].split(' is built, took ')
+            if (four.length > 1) {
+              var context = four[0]
+              var ms      = four[1]
+              console.log('PATH BUILT', tx, rx, context, ms)
+            }
+          }
+        }
+      }
+      score_total++
+    }
+    // happen right after the built, took
+    //TX=53e833fcbad1395647f198201d4931e5 RX=b4768bc4b91b86b51b513319f443f538 on default:dnhi78pwrn6cd5yz83i83816nqudwym57s5x4bp3j4g5kbeigw5y.loki built latency=291
+    if (tline.match(' is built latency ')) {
+      // path latency info
+    }
+    //[WRN] (166) Tue Apr 30 15:26:25 2019 PDT path/pathbuilder.cpp:319	SNode::8ti485iig9q1wd6k9qr6dqmswrqp9hceohcrtwwa7pn6wcxx9wdy.snode failed to select hop 3
+    // not a sure but snode extract
+    if (tline.match('.snode failed to select hop')) {
+      var parts = tline.split('SNode::')
+      //console.log('failed to select hop parts', parts)
+      if (parts.length > 1) {
+        var two = parts[1].split(' failed to select hop ')
+        if (two.length) {
+          var snode = two[0]
+          var hops = two[1]
+          //console.log('hopSelectionFailure', snode, hops)
+        }
+      }
+    }
+    //obtained an exit via nc9y88xsr7cqmpytc6tbo5dbf5c9fa3is5dezmdwxwgfkuhemg3o.snode
+    if (tline.match('obtained an exit via ')) {
+      var parts = tline.split('obtained an exit via ')
+      if (parts.length > 1) {
+        var two = parts[1].split('.snode')
+        var snode = two[0]+'.snode'
+        addSnode(snode)
+      }
+      score_total++
+    }
+    //TX=219222f0f390e40ce47745af292001f2 RX=671c259dd9590173019694fc10433b63 on SNode::nc9y88xsr7cqmpytc6tbo5dbf5c9fa3is5dezmdwxwgfkuhemg3o.snode nc9y88xsr7cqmpytc6tbo5dbf5c9fa3is5dezmdwxwgfkuhemg3o.snode Granted exit
+    if (tline.match('Granted exit')) {
+      var one = tline.split('TX=')
+      if (one.length > 1) {
+        var two = one[1].split(' RX=')
+        if (two.length > 1) {
+          var three = two[1].split(' on SNode::')
+          if (three.length > 1) {
+            var tx    = two[0]
+            var rx    = three[0]
+            console.log('EXIT', tx, rx, 'rest', three[1])
+          }
+        }
+      }
+      score_total++
+    }
+    //service/endpoint.cpp:652	default:kzow69uftho8ukm8g4kf88y5zka84azfrmfx5okkmrg7ucdz5d1o.loki IntroSet publish confirmed
+    if (tline.match('IntroSet publish confirmed')) {
+      score_total++
+    }
+  }
+}
+lokinet.onError = function(data) {
+  console.log(`monitorerr: ${data}`)
+  // start on 2019-04-30T15:40:55.540314745-07:00 X Records
+  // buffer until
+  // end on \n\n
+  var lines = data.toString().split(/\n/)
+  for(var i in lines) {
+    var tline = lines[i].trim()
+    if (tline.match('utp.session.sendq.')) {
+      //console.log('sendq')
+      var one = tline.split('utp.session.sendq.')
+      // parts[1] = 8ti485iig9q1wd6k9qr6dqmswrqp9hceohcrtwwa7pn6wcxx9wdy.snode [
+      // count = 282, total = 3, min = 0, max = 1 ]
+      if (one.length > 1) {
+        var two = one[1].split(' [ ')
+        if (two.length > 1) {
+          var snode = two[0]
+          var three = two[1].split(', ')
+          if (three.length > 3) {
+            var count = three[0]
+            var total = three[1]
+            var min = three[2]
+            if (min == 'min = 0') {
+              //console.log('no bloat')
+              // not sure if this is related to our version or not...
+              score_total++
+            }
+            var max = three[3]
+            console.log('UTPSENDQ', snode, count, total, min, max)
+          }
+        }
+      }
+    }
+  }
+}
+
+//
+// end config
+//
+
+function checkIP(cb) {
+  lokinet.getLokiNetIP(function(ip) {
+    if (ip === undefined) {
+      checkIP(cb)
+      return
+    }
+    //console.log('lokinet interface ip', ip)
+    cb(ip)
+  })
+}
+
+lokinet.startClient(lokinet_config, function() {
+  // lokinet isn't necessarily running at this point
+  console.log('Starting monitor')
+  score_time = Date.now()
+  checkIP(function(ip) {
+    console.log('lokinet interface ip', ip)
+    lokinet.findLokiNetDNS(function(servers) {
+      console.log('monitor detected DNS Servers', servers)
+    })
+    setInterval(function() {
+      console.log('score', score_total, 'time', ((Date.now() -  score_time)/1000))
+    }, 30000)
+    // maybe run until a specific time and quit for comparison
+    // 1200s (two 10min sessions)
+    setTimeout(function() {
+      console.log(lokinet_config.binary_path, 'version', lokinet_version, 'final score', score_total)
+      process.exit()
+    }, 3 * 600 * 1000) // 3x 10m sessions worth
+  })
+})

contrib/node-monitor/package-lock.json

@@ -0,0 +1,29 @@
+{
+  "name": "lokinet-monitor",
+  "version": "0.0.1",
+  "lockfileVersion": 1,
+  "requires": true,
+  "dependencies": {
+    "nan": {
+      "version": "2.10.0",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz",
+      "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA=="
+    },
+    "net-ping": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/net-ping/-/net-ping-1.2.3.tgz",
+      "integrity": "sha512-ZKxj/kVPKL2RIsV9nR6I8nMT8Pi3k6ciTBKxD/6gd5lga9qcNmlyqNv+dbXqYGBvHsmG9yIpsfajr8X054x2fQ==",
+      "requires": {
+        "raw-socket": "*"
+      }
+    },
+    "raw-socket": {
+      "version": "1.6.4",
+      "resolved": "https://registry.npmjs.org/raw-socket/-/raw-socket-1.6.4.tgz",
+      "integrity": "sha512-ab/By3tp0gTDzwEJxgKUv+uIma0JFVnJElNMKNAqNiET+GQ2VAfR6eUVfnm0yRG/vxGu8gO2BYWhR30sQOTebg==",
+      "requires": {
+        "nan": "2.10.*"
+      }
+    }
+  }
+}

contrib/node-monitor/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "lokinet-monitor",
+  "version": "0.0.1",
+  "description": "Lokinet monitor",
+  "main": "monitor_client.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/loki-project/loki-network.git"
+  },
+  "keywords": [
+    "lokinet"
+  ],
+  "author": "Ryan Tharp",
+  "license": "ISC",
+  "bugs": {
+    "url": "https://github.com/loki-project/loki-network/issues"
+  },
+  "homepage": "https://github.com/loki-project/loki-network#readme",
+  "dependencies": {
+    "net-ping": "^1.2.3"
+  }
+}

llarp/CMakeLists.txt

@@ -251,7 +251,7 @@ if(WITH_SHARED)
   set(LIBS ${LIBS} Threads::Threads)
   target_link_libraries(${SHARED_LIB} PUBLIC ${ABYSS_LIB} ${CRYPTOGRAPHY_LIB} ${UTIL_LIB} ${PLATFORM_LIB} ${LIBS})
   if (WIN32)
-    target_link_libraries(${SHARED_LIB} PUBLIC ws2_32 iphlpapi)
+    target_link_libraries(${SHARED_LIB} PUBLIC ${FS_LIB} ws2_32 iphlpapi)
   else()
     install(TARGETS ${SHARED_LIB} LIBRARY DESTINATION lib)
   endif()

llarp/hook/shell.cpp

@@ -6,6 +6,10 @@
 #include <util/logger.hpp>
 #include <sys/wait.h>
 #include <unistd.h>
+#if !defined(__linux__) || !defined(_GNU_SOURCE)
+// Not all systems declare this variable
+extern char **environ;
+#endif
 #endif
 #if defined(Darwin)
 #include <crt_externs.h>

osx-setup/make-release.sh

@@ -11,6 +11,6 @@ echo "Building package $VERSION"
 mkdir -p pkg1
 rm pkg1/lokinet.pkg
 pkgbuild --root osx-pkg --scripts scripts --identifier network.loki.lokinet --version $VERSION pkg1/lokinet.pkg
-rm lokinet-v0.4.pkg
+rm lokinet-v$VERSION.pkg
 productbuild --distribution distribution.xml --resources resources --package-path pkg1 --version $VERSION lokinet-v$VERSION.pkg
 

readme.md

@@ -66,7 +66,6 @@ Build requirements:
 * CMake
 * C++ 17 capable C++ compiler
 * gcovr (if generating test coverage with gcc)
-* IMPORTANT NOTE: To use the optimiser, make sure the default -DNDEBUG macro is removed before generating (see #400)
 
 ### Linux
 

test/CMakeLists.txt

@@ -67,5 +67,5 @@ if(NOT WIN32)
     target_link_libraries(${TEST_EXE} PUBLIC absl::variant)
 else()
     target_sources(${TEST_EXE} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/win32/test.rc")
-    target_link_libraries(${TEST_EXE} PUBLIC ws2_32 iphlpapi shlwapi)
+    target_link_libraries(${TEST_EXE} PUBLIC ${FS_LIB} ws2_32 iphlpapi shlwapi)
 endif(NOT WIN32)

ui-win32/Properties/AssemblyInfo.cs

@@ -32,10 +32,10 @@ using System.Runtime.InteropServices;
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("0.4.0.0")]
-[assembly: AssemblyFileVersion("0.4.0.0")]
+[assembly: AssemblyVersion("0.4.1.0")]
+[assembly: AssemblyFileVersion("0.4.1.0")]
 #if DEBUG
-[assembly: AssemblyInformationalVersion("0.4.0-dev-{chash:8}")]
+[assembly: AssemblyInformationalVersion("0.4.1-dev-{chash:8}")]
 #else
-[assembly: AssemblyInformationalVersion("0.4.0 (RELEASE_CODENAME)")]
+[assembly: AssemblyInformationalVersion("0.4.1 (RELEASE_CODENAME)")]
 #endif

win32-setup/lokinet-win32.iss

@@ -39,18 +39,18 @@ OutputDir={#DevPath}win32-setup
 OutputBaseFilename=lokinet-win32
 Compression=lzma
 SolidCompression=yes
-VersionInfoVersion=0.4.0
+VersionInfoVersion=0.4.1
 VersionInfoCompany=Loki Project
 VersionInfoDescription=LokiNET for Microsoft® Windows® NT™
 #ifndef RELEASE
-VersionInfoTextVersion=0.4.0-dev-{#VCSRev}
-VersionInfoProductTextVersion=0.4.0-dev-{#VCSRev}
+VersionInfoTextVersion=0.4.1-dev-{#VCSRev}
+VersionInfoProductTextVersion=0.4.1-dev-{#VCSRev}
 #else
-VersionInfoTextVersion=0.4.0
-VersionInfoProductTextVersion=0.4.0 ({#Codename})
+VersionInfoTextVersion=0.4.1
+VersionInfoProductTextVersion=0.4.1 ({#Codename})
 #endif
 VersionInfoProductName=LokiNET
-VersionInfoProductVersion=0.4.0
+VersionInfoProductVersion=0.4.1
 InternalCompressLevel=ultra64
 MinVersion=0,5.0
 ArchitecturesInstallIn64BitMode=x64