2015-06-25 20:09:41 +02:00
|
|
|
import logging
|
|
|
|
import random
|
|
|
|
import string
|
|
|
|
import time
|
|
|
|
import sys
|
|
|
|
|
|
|
|
import gevent
|
|
|
|
import msgpack
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
from gevent.server import StreamServer
|
|
|
|
from gevent.pool import Pool
|
2015-06-25 20:09:41 +02:00
|
|
|
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
from Debug import Debug
|
|
|
|
from Connection import Connection
|
|
|
|
from Config import config
|
Version 0.3.1, rev238, Connection encryption using TLS, One click site clone feature, Encryption stats, Disable encryption startup parameter, Disable ssl compression startup parameter, Exchange supported encryption methods at handshake, Alternative open port checker, Option to store site privatekey in users.json, Torrent tracker swap, Test for bip32 based site creation, cloning and sslcert creation, Fix for Chrome plugin on OSX, Separate siteSign websocket command, Update pybitcointools to major speedup, Re-add sslwrap for python 0.2.9+, Disable SSL compression to save memory and better performance
2015-06-10 00:29:30 +02:00
|
|
|
from Crypt import CryptConnection
|
2015-09-13 23:17:13 +02:00
|
|
|
from Crypt import CryptHash
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
|
|
|
|
|
|
|
|
class ConnectionServer:
|
2015-06-25 20:09:41 +02:00
|
|
|
def __init__(self, ip=None, port=None, request_handler=None):
|
|
|
|
self.ip = ip
|
|
|
|
self.port = port
|
|
|
|
self.last_connection_id = 1 # Connection id incrementer
|
|
|
|
self.log = logging.getLogger("ConnServer")
|
|
|
|
self.port_opened = None
|
|
|
|
|
|
|
|
self.connections = [] # Connections
|
|
|
|
self.ip_incoming = {} # Incoming connections from ip in the last minute to avoid connection flood
|
2015-09-02 19:15:55 +02:00
|
|
|
self.broken_ssl_peer_ids = {} # Peerids of broken ssl connections
|
2015-06-25 20:09:41 +02:00
|
|
|
self.ips = {} # Connection by ip
|
|
|
|
|
|
|
|
self.running = True
|
|
|
|
self.thread_checker = gevent.spawn(self.checkConnections)
|
|
|
|
|
|
|
|
self.bytes_recv = 0
|
|
|
|
self.bytes_sent = 0
|
|
|
|
|
|
|
|
# Bittorrent style peerid
|
2015-09-13 23:17:13 +02:00
|
|
|
self.peer_id = "-ZN0%s-%s" % (config.version.replace(".", ""), CryptHash.random(12, "base64"))
|
2015-06-25 20:09:41 +02:00
|
|
|
|
|
|
|
# Check msgpack version
|
|
|
|
if msgpack.version[0] == 0 and msgpack.version[1] < 4:
|
|
|
|
self.log.error(
|
2015-07-12 20:36:46 +02:00
|
|
|
"Error: Unsupported msgpack version: %s (<0.4.0), please run `sudo pip install msgpack-python --upgrade`" %
|
2015-06-25 20:09:41 +02:00
|
|
|
str(msgpack.version)
|
|
|
|
)
|
|
|
|
sys.exit(0)
|
|
|
|
|
|
|
|
if port: # Listen server on a port
|
|
|
|
self.pool = Pool(1000) # do not accept more than 1000 connections
|
2015-07-12 20:36:46 +02:00
|
|
|
self.stream_server = StreamServer(
|
|
|
|
(ip.replace("*", ""), port), self.handleIncomingConnection, spawn=self.pool, backlog=100
|
|
|
|
)
|
2015-06-25 20:09:41 +02:00
|
|
|
if request_handler:
|
|
|
|
self.handleRequest = request_handler
|
|
|
|
|
|
|
|
def start(self):
|
|
|
|
self.running = True
|
2015-09-24 22:08:08 +02:00
|
|
|
CryptConnection.manager.loadCerts()
|
2015-06-25 20:09:41 +02:00
|
|
|
self.log.debug("Binding to: %s:%s, (msgpack: %s), supported crypt: %s" % (
|
|
|
|
self.ip, self.port,
|
|
|
|
".".join(map(str, msgpack.version)), CryptConnection.manager.crypt_supported)
|
|
|
|
)
|
|
|
|
try:
|
|
|
|
self.stream_server.serve_forever() # Start normal connection server
|
|
|
|
except Exception, err:
|
|
|
|
self.log.info("StreamServer bind error, must be running already: %s" % err)
|
|
|
|
|
|
|
|
def stop(self):
|
|
|
|
self.running = False
|
|
|
|
self.stream_server.stop()
|
|
|
|
|
|
|
|
def handleIncomingConnection(self, sock, addr):
|
|
|
|
ip, port = addr
|
|
|
|
|
|
|
|
# Connection flood protection
|
|
|
|
if ip in self.ip_incoming:
|
|
|
|
self.ip_incoming[ip] += 1
|
|
|
|
if self.ip_incoming[ip] > 3: # Allow 3 in 1 minute from same ip
|
|
|
|
self.log.debug("Connection flood detected from %s" % ip)
|
|
|
|
time.sleep(30)
|
|
|
|
sock.close()
|
|
|
|
return False
|
|
|
|
else:
|
2015-09-24 22:08:08 +02:00
|
|
|
self.ip_incoming[ip] = 1
|
2015-06-25 20:09:41 +02:00
|
|
|
|
|
|
|
connection = Connection(self, ip, port, sock)
|
|
|
|
self.connections.append(connection)
|
|
|
|
self.ips[ip] = connection
|
|
|
|
connection.handleIncomingConnection(sock)
|
|
|
|
|
|
|
|
def getConnection(self, ip=None, port=None, peer_id=None, create=True):
|
|
|
|
# Find connection by ip
|
|
|
|
if ip in self.ips:
|
|
|
|
connection = self.ips[ip]
|
2015-09-24 22:08:08 +02:00
|
|
|
if not peer_id or connection.handshake.get("peer_id") == peer_id: # Filter by peer_id
|
|
|
|
if not connection.connected and create:
|
|
|
|
succ = connection.event_connected.get() # Wait for connection
|
|
|
|
if not succ:
|
|
|
|
raise Exception("Connection event return error")
|
|
|
|
return connection
|
|
|
|
|
2015-06-25 20:09:41 +02:00
|
|
|
# Recover from connection pool
|
|
|
|
for connection in self.connections:
|
|
|
|
if connection.ip == ip:
|
2015-09-24 22:08:08 +02:00
|
|
|
if peer_id and connection.handshake.get("peer_id") != peer_id: # Does not match
|
|
|
|
continue
|
2015-06-25 20:09:41 +02:00
|
|
|
if not connection.connected and create:
|
|
|
|
succ = connection.event_connected.get() # Wait for connection
|
|
|
|
if not succ:
|
|
|
|
raise Exception("Connection event return error")
|
|
|
|
return connection
|
|
|
|
|
|
|
|
# No connection found
|
|
|
|
if create: # Allow to create new connection if not found
|
|
|
|
if port == 0:
|
|
|
|
raise Exception("This peer is not connectable")
|
|
|
|
try:
|
|
|
|
connection = Connection(self, ip, port)
|
|
|
|
self.ips[ip] = connection
|
|
|
|
self.connections.append(connection)
|
|
|
|
succ = connection.connect()
|
|
|
|
if not succ:
|
|
|
|
connection.close()
|
|
|
|
raise Exception("Connection event return error")
|
|
|
|
|
|
|
|
except Exception, err:
|
|
|
|
self.log.debug("%s Connect error: %s" % (ip, Debug.formatException(err)))
|
|
|
|
connection.close()
|
|
|
|
raise err
|
|
|
|
return connection
|
|
|
|
else:
|
|
|
|
return None
|
|
|
|
|
|
|
|
def removeConnection(self, connection):
|
|
|
|
self.log.debug("Removing %s..." % connection)
|
|
|
|
if self.ips.get(connection.ip) == connection: # Delete if same as in registry
|
|
|
|
del self.ips[connection.ip]
|
|
|
|
if connection in self.connections:
|
|
|
|
self.connections.remove(connection)
|
|
|
|
|
|
|
|
def checkConnections(self):
|
|
|
|
while self.running:
|
|
|
|
time.sleep(60) # Sleep 1 min
|
2015-09-02 19:15:55 +02:00
|
|
|
self.ip_incoming = {} # Reset connected ips counter
|
|
|
|
self.broken_ssl_peer_ids = {} # Reset broken ssl peerids count
|
2015-06-25 20:09:41 +02:00
|
|
|
for connection in self.connections[:]: # Make a copy
|
|
|
|
idle = time.time() - max(connection.last_recv_time, connection.start_time, connection.last_message_time)
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
if connection.unpacker and idle > 30:
|
|
|
|
# Delete the unpacker if not needed
|
2015-06-25 20:09:41 +02:00
|
|
|
del connection.unpacker
|
|
|
|
connection.unpacker = None
|
|
|
|
connection.log("Unpacker deleted")
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
if idle > 60 * 60:
|
|
|
|
# Wake up after 1h
|
2015-06-25 20:09:41 +02:00
|
|
|
connection.log("[Cleanup] After wakeup, idle: %s" % idle)
|
|
|
|
connection.close()
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
elif idle > 20 * 60 and connection.last_send_time < time.time() - 10:
|
|
|
|
# Idle more than 20 min and we not send request in last 10 sec
|
2015-06-25 20:09:41 +02:00
|
|
|
if not connection.ping(): # send ping request
|
|
|
|
connection.close()
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
elif idle > 10 and connection.incomplete_buff_recv > 0:
|
|
|
|
# Incompelte data with more than 10 sec idle
|
2015-06-25 20:09:41 +02:00
|
|
|
connection.log("[Cleanup] Connection buff stalled")
|
|
|
|
connection.close()
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
elif idle > 10 and connection.waiting_requests and time.time() - connection.last_send_time > 10:
|
|
|
|
# Sent command and no response in 10 sec
|
|
|
|
connection.log(
|
|
|
|
"[Cleanup] Command %s timeout: %s" % (connection.last_cmd, time.time() - connection.last_send_time)
|
|
|
|
)
|
2015-06-25 20:09:41 +02:00
|
|
|
connection.close()
|
|
|
|
|
|
|
|
elif idle > 60 and connection.protocol == "?": # No connection after 1 min
|
|
|
|
connection.log("[Cleanup] Connect timeout: %s" % idle)
|
|
|
|
connection.close()
|