2015-06-25 20:09:41 +02:00
|
|
|
import logging
|
|
|
|
import time
|
|
|
|
import sys
|
2019-01-20 16:20:41 +01:00
|
|
|
import socket
|
2018-01-27 12:19:37 +01:00
|
|
|
from collections import defaultdict
|
2015-06-25 20:09:41 +02:00
|
|
|
|
|
|
|
import gevent
|
|
|
|
import msgpack
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
from gevent.server import StreamServer
|
|
|
|
from gevent.pool import Pool
|
2015-06-25 20:09:41 +02:00
|
|
|
|
2018-01-27 12:20:40 +01:00
|
|
|
import util
|
2019-01-20 16:20:41 +01:00
|
|
|
from util import helper
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
from Debug import Debug
|
2019-03-15 21:06:59 +01:00
|
|
|
from .Connection import Connection
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
from Config import config
|
Version 0.3.1, rev238, Connection encryption using TLS, One click site clone feature, Encryption stats, Disable encryption startup parameter, Disable ssl compression startup parameter, Exchange supported encryption methods at handshake, Alternative open port checker, Option to store site privatekey in users.json, Torrent tracker swap, Test for bip32 based site creation, cloning and sslcert creation, Fix for Chrome plugin on OSX, Separate siteSign websocket command, Update pybitcointools to major speedup, Re-add sslwrap for python 0.2.9+, Disable SSL compression to save memory and better performance
2015-06-10 00:29:30 +02:00
|
|
|
from Crypt import CryptConnection
|
2015-09-13 23:17:13 +02:00
|
|
|
from Crypt import CryptHash
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
from Tor import TorManager
|
2018-01-27 12:19:37 +01:00
|
|
|
from Site import SiteManager
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
|
|
|
|
|
2018-02-08 17:58:44 +01:00
|
|
|
class ConnectionServer(object):
|
2015-06-25 20:09:41 +02:00
|
|
|
def __init__(self, ip=None, port=None, request_handler=None):
|
2019-01-20 16:16:55 +01:00
|
|
|
if not ip:
|
|
|
|
if config.fileserver_ip_type == "ipv6":
|
|
|
|
ip = "::1"
|
|
|
|
else:
|
|
|
|
ip = "127.0.0.1"
|
|
|
|
port = 15441
|
2015-06-25 20:09:41 +02:00
|
|
|
self.ip = ip
|
|
|
|
self.port = port
|
|
|
|
self.last_connection_id = 1 # Connection id incrementer
|
|
|
|
self.log = logging.getLogger("ConnServer")
|
2019-01-23 02:13:13 +01:00
|
|
|
self.port_opened = {}
|
2018-01-27 12:19:37 +01:00
|
|
|
self.peer_blacklist = SiteManager.peer_blacklist
|
2015-06-25 20:09:41 +02:00
|
|
|
|
2018-04-28 21:51:14 +02:00
|
|
|
self.tor_manager = TorManager(self.ip, self.port)
|
2015-06-25 20:09:41 +02:00
|
|
|
self.connections = [] # Connections
|
2017-02-16 20:56:07 +01:00
|
|
|
self.whitelist = config.ip_local # No flood protection on this ips
|
2015-06-25 20:09:41 +02:00
|
|
|
self.ip_incoming = {} # Incoming connections from ip in the last minute to avoid connection flood
|
2018-03-14 22:26:59 +01:00
|
|
|
self.broken_ssl_ips = {} # Peerids of broken ssl connections
|
2015-06-25 20:09:41 +02:00
|
|
|
self.ips = {} # Connection by ip
|
2016-03-19 18:14:09 +01:00
|
|
|
self.has_internet = True # Internet outage detection
|
2015-06-25 20:09:41 +02:00
|
|
|
|
2017-06-20 20:28:09 +02:00
|
|
|
self.stream_server = None
|
2019-01-23 02:13:13 +01:00
|
|
|
self.stream_server_proxy = None
|
2018-04-04 15:33:02 +02:00
|
|
|
self.running = False
|
2019-09-19 16:33:45 +02:00
|
|
|
self.stopping = False
|
2020-02-07 16:38:42 +01:00
|
|
|
self.thread_checker = None
|
2015-06-25 20:09:41 +02:00
|
|
|
|
2017-10-13 01:21:05 +02:00
|
|
|
self.stat_recv = defaultdict(lambda: defaultdict(int))
|
|
|
|
self.stat_sent = defaultdict(lambda: defaultdict(int))
|
2015-06-25 20:09:41 +02:00
|
|
|
self.bytes_recv = 0
|
|
|
|
self.bytes_sent = 0
|
2018-01-19 02:16:33 +01:00
|
|
|
self.num_recv = 0
|
|
|
|
self.num_sent = 0
|
2015-06-25 20:09:41 +02:00
|
|
|
|
2018-09-02 02:19:51 +02:00
|
|
|
self.num_incoming = 0
|
|
|
|
self.num_outgoing = 0
|
2019-01-20 16:20:41 +01:00
|
|
|
self.had_external_incoming = False
|
2018-09-02 02:19:51 +02:00
|
|
|
|
2018-09-05 14:32:23 +02:00
|
|
|
self.timecorrection = 0.0
|
2019-01-23 02:13:13 +01:00
|
|
|
self.pool = Pool(500) # do not accept more than 500 connections
|
2018-09-05 14:32:23 +02:00
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
# Bittorrent style peerid
|
2018-04-29 00:59:45 +02:00
|
|
|
self.peer_id = "-UT3530-%s" % CryptHash.random(12, "base64")
|
2015-06-25 20:09:41 +02:00
|
|
|
|
|
|
|
# Check msgpack version
|
|
|
|
if msgpack.version[0] == 0 and msgpack.version[1] < 4:
|
|
|
|
self.log.error(
|
2018-02-27 02:48:15 +01:00
|
|
|
"Error: Unsupported msgpack version: %s (<0.4.0), please run `sudo apt-get install python-pip; sudo pip install msgpack --upgrade`" %
|
2015-06-25 20:09:41 +02:00
|
|
|
str(msgpack.version)
|
|
|
|
)
|
|
|
|
sys.exit(0)
|
|
|
|
|
2018-04-03 14:45:04 +02:00
|
|
|
if request_handler:
|
|
|
|
self.handleRequest = request_handler
|
2015-06-25 20:09:41 +02:00
|
|
|
|
2018-05-31 19:43:39 +02:00
|
|
|
def start(self, check_connections=True):
|
2019-09-19 16:33:45 +02:00
|
|
|
if self.stopping:
|
|
|
|
return False
|
2015-06-25 20:09:41 +02:00
|
|
|
self.running = True
|
2018-05-31 19:43:39 +02:00
|
|
|
if check_connections:
|
|
|
|
self.thread_checker = gevent.spawn(self.checkConnections)
|
2015-09-24 22:08:08 +02:00
|
|
|
CryptConnection.manager.loadCerts()
|
2018-07-10 03:31:39 +02:00
|
|
|
if config.tor != "disable":
|
2018-04-29 01:00:37 +02:00
|
|
|
self.tor_manager.start()
|
2018-04-03 14:45:04 +02:00
|
|
|
if not self.port:
|
|
|
|
self.log.info("No port found, not binding")
|
|
|
|
return False
|
|
|
|
|
2019-01-26 03:13:47 +01:00
|
|
|
self.log.debug("Binding to: %s:%s, (msgpack: %s), supported crypt: %s" % (
|
2019-01-20 16:18:48 +01:00
|
|
|
self.ip, self.port, ".".join(map(str, msgpack.version)),
|
2019-01-26 03:13:47 +01:00
|
|
|
CryptConnection.manager.crypt_supported
|
2019-01-20 16:18:48 +01:00
|
|
|
))
|
2015-06-25 20:09:41 +02:00
|
|
|
try:
|
2018-04-03 14:45:04 +02:00
|
|
|
self.stream_server = StreamServer(
|
2019-01-23 02:13:13 +01:00
|
|
|
(self.ip, self.port), self.handleIncomingConnection, spawn=self.pool, backlog=100
|
2018-04-03 14:45:04 +02:00
|
|
|
)
|
2019-03-15 21:06:59 +01:00
|
|
|
except Exception as err:
|
2019-01-23 02:13:13 +01:00
|
|
|
self.log.info("StreamServer create error: %s" % Debug.formatException(err))
|
2019-01-20 16:18:24 +01:00
|
|
|
|
2018-04-29 02:44:46 +02:00
|
|
|
def listen(self):
|
2019-04-15 15:46:37 +02:00
|
|
|
if not self.running:
|
2019-09-19 16:33:45 +02:00
|
|
|
return None
|
|
|
|
|
2019-01-23 02:13:13 +01:00
|
|
|
if self.stream_server_proxy:
|
|
|
|
gevent.spawn(self.listenProxy)
|
2018-04-29 02:44:46 +02:00
|
|
|
try:
|
|
|
|
self.stream_server.serve_forever()
|
2019-03-15 21:06:59 +01:00
|
|
|
except Exception as err:
|
2018-04-29 02:44:46 +02:00
|
|
|
self.log.info("StreamServer listen error: %s" % err)
|
2019-09-19 16:33:45 +02:00
|
|
|
return False
|
2020-02-07 16:38:42 +01:00
|
|
|
self.log.debug("Stopped.")
|
2015-06-25 20:09:41 +02:00
|
|
|
|
|
|
|
def stop(self):
|
2020-02-07 16:38:42 +01:00
|
|
|
self.log.debug("Stopping %s" % self.stream_server)
|
2019-09-19 16:33:45 +02:00
|
|
|
self.stopping = True
|
2015-06-25 20:09:41 +02:00
|
|
|
self.running = False
|
2020-02-07 16:38:42 +01:00
|
|
|
if self.thread_checker:
|
|
|
|
gevent.kill(self.thread_checker)
|
2017-06-20 20:28:09 +02:00
|
|
|
if self.stream_server:
|
|
|
|
self.stream_server.stop()
|
2015-06-25 20:09:41 +02:00
|
|
|
|
2019-04-15 15:46:53 +02:00
|
|
|
def closeConnections(self):
|
|
|
|
self.log.debug("Closing all connection: %s" % len(self.connections))
|
|
|
|
for connection in self.connections[:]:
|
|
|
|
connection.close("Close all connections")
|
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
def handleIncomingConnection(self, sock, addr):
|
2019-04-15 15:47:05 +02:00
|
|
|
if config.offline:
|
|
|
|
sock.close()
|
|
|
|
return False
|
|
|
|
|
2019-01-20 16:20:01 +01:00
|
|
|
ip, port = addr[0:2]
|
|
|
|
ip = ip.lower()
|
|
|
|
if ip.startswith("::ffff:"): # IPv6 to IPv4 mapping
|
|
|
|
ip = ip.replace("::ffff:", "", 1)
|
2018-09-02 02:19:51 +02:00
|
|
|
self.num_incoming += 1
|
2015-06-25 20:09:41 +02:00
|
|
|
|
2019-01-20 16:20:41 +01:00
|
|
|
if not self.had_external_incoming and not helper.isPrivateIp(ip):
|
|
|
|
self.had_external_incoming = True
|
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
# Connection flood protection
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
if ip in self.ip_incoming and ip not in self.whitelist:
|
2015-06-25 20:09:41 +02:00
|
|
|
self.ip_incoming[ip] += 1
|
2016-01-05 20:18:17 +01:00
|
|
|
if self.ip_incoming[ip] > 6: # Allow 6 in 1 minute from same ip
|
2015-06-25 20:09:41 +02:00
|
|
|
self.log.debug("Connection flood detected from %s" % ip)
|
|
|
|
time.sleep(30)
|
|
|
|
sock.close()
|
|
|
|
return False
|
|
|
|
else:
|
2015-09-24 22:08:08 +02:00
|
|
|
self.ip_incoming[ip] = 1
|
2015-06-25 20:09:41 +02:00
|
|
|
|
|
|
|
connection = Connection(self, ip, port, sock)
|
|
|
|
self.connections.append(connection)
|
2018-02-08 17:59:27 +01:00
|
|
|
if ip not in config.ip_local:
|
|
|
|
self.ips[ip] = connection
|
2015-06-25 20:09:41 +02:00
|
|
|
connection.handleIncomingConnection(sock)
|
|
|
|
|
2018-02-13 13:18:40 +01:00
|
|
|
def handleMessage(self, *args, **kwargs):
|
|
|
|
pass
|
|
|
|
|
2018-08-26 02:52:23 +02:00
|
|
|
def getConnection(self, ip=None, port=None, peer_id=None, create=True, site=None, is_tracker_connection=False):
|
2019-01-23 02:11:31 +01:00
|
|
|
ip_type = helper.getIpType(ip)
|
|
|
|
has_per_site_onion = (ip.endswith(".onion") or self.port_opened.get(ip_type, None) == False) and self.tor_manager.start_onions and site
|
|
|
|
if has_per_site_onion: # Site-unique connection for Tor
|
2018-03-14 22:29:00 +01:00
|
|
|
if ip.endswith(".onion"):
|
|
|
|
site_onion = self.tor_manager.getOnion(site.address)
|
|
|
|
else:
|
|
|
|
site_onion = self.tor_manager.getOnion("global")
|
2017-04-09 12:02:24 +02:00
|
|
|
key = ip + site_onion
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
else:
|
|
|
|
key = ip
|
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
# Find connection by ip
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
if key in self.ips:
|
|
|
|
connection = self.ips[key]
|
2015-09-24 22:08:08 +02:00
|
|
|
if not peer_id or connection.handshake.get("peer_id") == peer_id: # Filter by peer_id
|
|
|
|
if not connection.connected and create:
|
|
|
|
succ = connection.event_connected.get() # Wait for connection
|
|
|
|
if not succ:
|
|
|
|
raise Exception("Connection event return error")
|
|
|
|
return connection
|
|
|
|
|
2016-08-10 12:51:38 +02:00
|
|
|
# Recover from connection pool
|
|
|
|
for connection in self.connections:
|
|
|
|
if connection.ip == ip:
|
|
|
|
if peer_id and connection.handshake.get("peer_id") != peer_id: # Does not match
|
|
|
|
continue
|
2017-04-09 12:02:24 +02:00
|
|
|
if ip.endswith(".onion") and self.tor_manager.start_onions and ip.replace(".onion", "") != connection.target_onion:
|
2016-08-10 12:51:38 +02:00
|
|
|
# For different site
|
|
|
|
continue
|
|
|
|
if not connection.connected and create:
|
|
|
|
succ = connection.event_connected.get() # Wait for connection
|
|
|
|
if not succ:
|
|
|
|
raise Exception("Connection event return error")
|
|
|
|
return connection
|
2015-06-25 20:09:41 +02:00
|
|
|
|
|
|
|
# No connection found
|
2019-04-15 15:47:17 +02:00
|
|
|
if create and not config.offline: # Allow to create new connection if not found
|
2015-06-25 20:09:41 +02:00
|
|
|
if port == 0:
|
|
|
|
raise Exception("This peer is not connectable")
|
2018-01-27 12:19:37 +01:00
|
|
|
|
2018-09-21 08:05:27 +02:00
|
|
|
if (ip, port) in self.peer_blacklist and not is_tracker_connection:
|
2018-01-27 12:19:37 +01:00
|
|
|
raise Exception("This peer is blacklisted")
|
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
try:
|
2019-01-23 02:11:31 +01:00
|
|
|
if has_per_site_onion: # Lock connection to site
|
2018-08-26 02:52:23 +02:00
|
|
|
connection = Connection(self, ip, port, target_onion=site_onion, is_tracker_connection=is_tracker_connection)
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
else:
|
2018-08-26 02:52:23 +02:00
|
|
|
connection = Connection(self, ip, port, is_tracker_connection=is_tracker_connection)
|
2018-09-02 02:19:51 +02:00
|
|
|
self.num_outgoing += 1
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
self.ips[key] = connection
|
2015-06-25 20:09:41 +02:00
|
|
|
self.connections.append(connection)
|
2018-10-30 04:40:34 +01:00
|
|
|
connection.log("Connecting... (site: %s)" % site)
|
2015-06-25 20:09:41 +02:00
|
|
|
succ = connection.connect()
|
|
|
|
if not succ:
|
2017-02-27 00:02:24 +01:00
|
|
|
connection.close("Connection event return error")
|
2015-06-25 20:09:41 +02:00
|
|
|
raise Exception("Connection event return error")
|
|
|
|
|
2019-03-15 21:06:59 +01:00
|
|
|
except Exception as err:
|
2017-02-27 00:02:24 +01:00
|
|
|
connection.close("%s Connect error: %s" % (ip, Debug.formatException(err)))
|
2015-06-25 20:09:41 +02:00
|
|
|
raise err
|
2018-01-27 12:20:40 +01:00
|
|
|
|
|
|
|
if len(self.connections) > config.global_connected_limit:
|
|
|
|
gevent.spawn(self.checkMaxConnections)
|
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
return connection
|
|
|
|
else:
|
|
|
|
return None
|
|
|
|
|
|
|
|
def removeConnection(self, connection):
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
# Delete if same as in registry
|
|
|
|
if self.ips.get(connection.ip) == connection:
|
2015-06-25 20:09:41 +02:00
|
|
|
del self.ips[connection.ip]
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
# Site locked connection
|
2017-04-09 12:02:24 +02:00
|
|
|
if connection.target_onion:
|
|
|
|
if self.ips.get(connection.ip + connection.target_onion) == connection:
|
|
|
|
del self.ips[connection.ip + connection.target_onion]
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
# Cert pinned connection
|
|
|
|
if connection.cert_pin and self.ips.get(connection.ip + "#" + connection.cert_pin) == connection:
|
|
|
|
del self.ips[connection.ip + "#" + connection.cert_pin]
|
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
if connection in self.connections:
|
|
|
|
self.connections.remove(connection)
|
|
|
|
|
|
|
|
def checkConnections(self):
|
2016-03-12 23:09:26 +01:00
|
|
|
run_i = 0
|
2020-02-07 16:38:42 +01:00
|
|
|
time.sleep(15)
|
2015-06-25 20:09:41 +02:00
|
|
|
while self.running:
|
2016-03-12 23:09:26 +01:00
|
|
|
run_i += 1
|
2015-09-02 19:15:55 +02:00
|
|
|
self.ip_incoming = {} # Reset connected ips counter
|
2016-03-19 18:14:09 +01:00
|
|
|
last_message_time = 0
|
2017-10-03 14:32:31 +02:00
|
|
|
s = time.time()
|
2015-06-25 20:09:41 +02:00
|
|
|
for connection in self.connections[:]: # Make a copy
|
2018-05-31 19:43:09 +02:00
|
|
|
if connection.ip.endswith(".onion") or config.tor == "always":
|
2018-01-27 12:19:57 +01:00
|
|
|
timeout_multipler = 2
|
|
|
|
else:
|
|
|
|
timeout_multipler = 1
|
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
idle = time.time() - max(connection.last_recv_time, connection.start_time, connection.last_message_time)
|
2018-02-08 17:58:44 +01:00
|
|
|
if connection.last_message_time > last_message_time and not connection.is_private_ip:
|
|
|
|
# Message from local IPs does not means internet connection
|
|
|
|
last_message_time = connection.last_message_time
|
2015-06-25 20:09:41 +02:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
if connection.unpacker and idle > 30:
|
|
|
|
# Delete the unpacker if not needed
|
2015-06-25 20:09:41 +02:00
|
|
|
del connection.unpacker
|
|
|
|
connection.unpacker = None
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
|
2017-10-13 01:19:34 +02:00
|
|
|
elif connection.last_cmd_sent == "announce" and idle > 20: # Bootstrapper connection close after 20 sec
|
2018-10-30 04:41:02 +01:00
|
|
|
connection.close("[Cleanup] Tracker connection, idle: %.3fs" % idle)
|
2015-06-25 20:09:41 +02:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
if idle > 60 * 60:
|
|
|
|
# Wake up after 1h
|
2018-10-30 04:41:02 +01:00
|
|
|
connection.close("[Cleanup] After wakeup, idle: %.3fs" % idle)
|
2015-06-25 20:09:41 +02:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
elif idle > 20 * 60 and connection.last_send_time < time.time() - 10:
|
2016-03-18 20:01:23 +01:00
|
|
|
# Idle more than 20 min and we have not sent request in last 10 sec
|
2016-04-25 02:24:05 +02:00
|
|
|
if not connection.ping():
|
2017-02-27 00:02:24 +01:00
|
|
|
connection.close("[Cleanup] Ping timeout")
|
2015-06-25 20:09:41 +02:00
|
|
|
|
2018-01-27 12:19:57 +01:00
|
|
|
elif idle > 10 * timeout_multipler and connection.incomplete_buff_recv > 0:
|
2016-04-25 02:24:05 +02:00
|
|
|
# Incomplete data with more than 10 sec idle
|
2017-02-27 00:02:24 +01:00
|
|
|
connection.close("[Cleanup] Connection buff stalled")
|
2015-06-25 20:09:41 +02:00
|
|
|
|
2018-01-27 12:19:57 +01:00
|
|
|
elif idle > 10 * timeout_multipler and connection.protocol == "?": # No connection after 10 sec
|
2017-02-27 00:02:24 +01:00
|
|
|
connection.close(
|
2017-10-03 14:34:21 +02:00
|
|
|
"[Cleanup] Connect timeout: %.3fs" % idle
|
2015-07-12 20:36:46 +02:00
|
|
|
)
|
2015-06-25 20:09:41 +02:00
|
|
|
|
2018-01-27 12:19:57 +01:00
|
|
|
elif idle > 10 * timeout_multipler and connection.waiting_requests and time.time() - connection.last_send_time > 10 * timeout_multipler:
|
2017-10-03 14:34:21 +02:00
|
|
|
# Sent command and no response in 10 sec
|
2017-02-27 00:05:27 +01:00
|
|
|
connection.close(
|
2017-10-13 01:19:34 +02:00
|
|
|
"[Cleanup] Command %s timeout: %.3fs" % (connection.last_cmd_sent, time.time() - connection.last_send_time)
|
2017-02-27 00:05:27 +01:00
|
|
|
)
|
2016-03-12 23:09:26 +01:00
|
|
|
|
|
|
|
elif idle < 60 and connection.bad_actions > 40:
|
2017-02-27 00:02:24 +01:00
|
|
|
connection.close(
|
|
|
|
"[Cleanup] Too many bad actions: %s" % connection.bad_actions
|
|
|
|
)
|
2016-03-12 23:09:26 +01:00
|
|
|
|
2018-01-19 02:16:50 +01:00
|
|
|
elif idle > 5 * 60 and connection.sites == 0:
|
2017-02-27 00:02:24 +01:00
|
|
|
connection.close(
|
|
|
|
"[Cleanup] No site for connection"
|
|
|
|
)
|
2016-04-25 02:24:05 +02:00
|
|
|
|
2017-10-03 14:32:31 +02:00
|
|
|
elif run_i % 90 == 0:
|
2016-03-12 23:09:26 +01:00
|
|
|
# Reset bad action counter every 30 min
|
|
|
|
connection.bad_actions = 0
|
2016-03-19 18:14:09 +01:00
|
|
|
|
|
|
|
# Internet outage detection
|
2018-01-19 02:16:50 +01:00
|
|
|
if time.time() - last_message_time > max(60, 60 * 10 / max(1, float(len(self.connections)) / 50)):
|
2016-04-09 19:31:26 +02:00
|
|
|
# Offline: Last message more than 60-600sec depending on connection number
|
2018-08-29 19:53:24 +02:00
|
|
|
if self.has_internet and last_message_time:
|
2016-03-19 18:14:09 +01:00
|
|
|
self.has_internet = False
|
|
|
|
self.onInternetOffline()
|
|
|
|
else:
|
|
|
|
# Online
|
|
|
|
if not self.has_internet:
|
|
|
|
self.has_internet = True
|
|
|
|
self.onInternetOnline()
|
|
|
|
|
2018-09-05 14:32:23 +02:00
|
|
|
self.timecorrection = self.getTimecorrection()
|
|
|
|
|
2017-10-03 14:32:31 +02:00
|
|
|
if time.time() - s > 0.01:
|
|
|
|
self.log.debug("Connection cleanup in %.3fs" % (time.time() - s))
|
2020-02-07 16:38:42 +01:00
|
|
|
|
|
|
|
time.sleep(15)
|
2018-04-12 19:25:50 +02:00
|
|
|
self.log.debug("Checkconnections ended")
|
2017-10-03 14:32:31 +02:00
|
|
|
|
2018-01-27 12:20:40 +01:00
|
|
|
@util.Noparallel(blocking=False)
|
|
|
|
def checkMaxConnections(self):
|
|
|
|
if len(self.connections) < config.global_connected_limit:
|
|
|
|
return 0
|
|
|
|
|
|
|
|
s = time.time()
|
|
|
|
num_connected_before = len(self.connections)
|
|
|
|
self.connections.sort(key=lambda connection: connection.sites)
|
|
|
|
num_closed = 0
|
|
|
|
for connection in self.connections:
|
|
|
|
idle = time.time() - max(connection.last_recv_time, connection.start_time, connection.last_message_time)
|
|
|
|
if idle > 60:
|
|
|
|
connection.close("Connection limit reached")
|
|
|
|
num_closed += 1
|
|
|
|
if num_closed > config.global_connected_limit * 0.1:
|
|
|
|
break
|
|
|
|
|
|
|
|
self.log.debug("Closed %s connections of %s after reached limit %s in %.3fs" % (
|
|
|
|
num_closed, num_connected_before, config.global_connected_limit, time.time() - s
|
|
|
|
))
|
|
|
|
return num_closed
|
|
|
|
|
2016-03-19 18:14:09 +01:00
|
|
|
def onInternetOnline(self):
|
|
|
|
self.log.info("Internet online")
|
|
|
|
|
|
|
|
def onInternetOffline(self):
|
2019-01-20 16:20:41 +01:00
|
|
|
self.had_external_incoming = False
|
2016-03-19 18:14:09 +01:00
|
|
|
self.log.info("Internet offline")
|
2018-08-29 19:53:44 +02:00
|
|
|
|
2018-09-05 14:32:01 +02:00
|
|
|
def getTimecorrection(self):
|
2018-08-29 19:53:44 +02:00
|
|
|
corrections = sorted([
|
|
|
|
connection.handshake.get("time") - connection.handshake_time + connection.last_ping_delay
|
|
|
|
for connection in self.connections
|
|
|
|
if connection.handshake.get("time") and connection.last_ping_delay
|
|
|
|
])
|
2020-01-22 16:37:07 +01:00
|
|
|
if len(corrections) < 9:
|
2018-08-29 19:53:44 +02:00
|
|
|
return 0.0
|
2019-03-15 21:06:59 +01:00
|
|
|
mid = int(len(corrections) / 2 - 1)
|
2018-08-29 19:53:44 +02:00
|
|
|
median = (corrections[mid - 1] + corrections[mid] + corrections[mid + 1]) / 3
|
|
|
|
return median
|